Fixed-horizon actor-critic trainer that collects trajectories from MOOSE reporters and runs a PPO update on top of reusable RL-core components (observation history, trajectory buffer, mini-batch sampler, value estimator, and PPO loss). More...

#include <LibtorchDRLControlTrainer.h>

Inheritance diagram for LibtorchDRLControlTrainer:

[legend]

Public Types
typedef DataFileName	DataFileParameterType

Public Member Functions
	LibtorchDRLControlTrainer (const InputParameters &parameters)
	Build the PPO-based DRL trainer.

virtual void	execute () override
	Pull fresh rollout data from the reporters and trigger training when ready.

Real	averageEpisodeReward ()
	Return the current average episodic reward.

Real	stdEpisodeReward ()
	Return the current episodic reward standard deviation.

std::vector< Real >	sampleAverageEpsiodeRewards ()
	Return per-sample mean episodic rewards from the latest update window.

std::vector< Real >	sampleStdEpsiodeRewards ()
	Return per-sample episodic reward standard deviations from the latest update window.

void	trainController (const LibtorchRLTrajectoryBuffer::TensorBatch &batch)
	Run the PPO update on a flattened on-policy batch.

const Moose::LibtorchActorNeuralNet &	controlNeuralNet () const
	Return the current actor network.

unsigned int	seed () const
	Return the trainer seed used for sampling and shuffling.

unsigned int	timestepWindow () const
	Return the number of raw reporter entries per training transition.

unsigned int	inputTimesteps () const
	Return the number of observation time levels stacked into the network inputs.

unsigned int	numberOfObservations () const
	Return the number of observation reporters read at each time level.

unsigned int	numberOfControlSignals () const
	Return the number of control signals produced by the actor.

virtual void	initialize ()

virtual void	finalize ()

virtual void	threadJoin (const UserObject &) final

virtual Real	spatialValue (const Point &) const

virtual const std::vector< Point >	spatialPoints () const

void	setPrimaryThreadCopy (UserObject *primary)

UserObject *	primaryThreadCopy ()

SubProblem &	getSubProblem () const

bool	shouldDuplicateInitialExecution () const

void	gatherSum (T &value)

void	gatherMax (T &value)

void	gatherMin (T &value)

void	gatherProxyValueMax (T1 &proxy, T2 &value)

void	gatherProxyValueMin (T1 &proxy, T2 &value)

std::set< UserObjectName >	getDependObjects () const

const std::set< std::string > &	getRequestedItems () override

const std::set< std::string > &	getSuppliedItems () override

unsigned int	systemNumber () const

virtual bool	needThreadedCopy () const

virtual bool	enabled () const

std::shared_ptr< MooseObject >	getSharedPtr ()

std::shared_ptr< const MooseObject >	getSharedPtr () const

bool	isKokkosObject () const

MooseApp &	getMooseApp () const

const std::string &	type () const

const std::string &	name () const

std::string	typeAndName () const

MooseObjectParameterName	uniqueParameterName (const std::string &parameter_name) const

MooseObjectName	uniqueName () const

const InputParameters &	parameters () const

const hit::Node *	getHitNode () const

bool	hasBase () const

const std::string &	getBase () const

const T &	getParam (const std::string &name) const

std::vector< std::pair< T1, T2 > >	getParam (const std::string &param1, const std::string &param2) const

const T *	queryParam (const std::string &name) const

const T &	getRenamedParam (const std::string &old_name, const std::string &new_name) const

T	getCheckedPointerParam (const std::string &name, const std::string &error_string="") const

bool	isParamValid (const std::string &name) const

bool	isParamSetByUser (const std::string &name) const

void	connectControllableParams (const std::string &parameter, const std::string &object_type, const std::string &object_name, const std::string &object_parameter) const

void	paramError (const std::string &param, Args... args) const

void	paramWarning (const std::string &param, Args... args) const

void	paramWarning (const std::string &param, Args... args) const

void	paramInfo (const std::string &param, Args... args) const

std::string	messagePrefix (const bool hit_prefix=true) const

std::string	errorPrefix (const std::string &) const

void	mooseError (Args &&... args) const

void	mooseDocumentedError (const std::string &repo_name, const unsigned int issue_num, Args &&... args) const

void	mooseErrorNonPrefixed (Args &&... args) const

void	mooseWarning (Args &&... args) const

void	mooseWarning (Args &&... args) const

void	mooseWarningNonPrefixed (Args &&... args) const

void	mooseWarningNonPrefixed (Args &&... args) const

void	mooseDeprecated (Args &&... args) const

void	mooseDeprecated (Args &&... args) const

void	mooseDeprecatedNoTrace (Args &&... args) const

void	mooseInfo (Args &&... args) const

void	callMooseError (std::string msg, const bool with_prefix, const hit::Node *node=nullptr, const bool show_trace=true) const

std::string	getDataFileName (const std::string &param) const

std::string	getDataFileNameByName (const std::string &relative_path) const

std::string	getDataFilePath (const std::string &relative_path) const

virtual void	initialSetup ()

virtual void	timestepSetup ()

virtual void	jacobianSetup ()

virtual void	residualSetup ()

virtual void	customSetup (const ExecFlagType &)

const ExecFlagEnum &	getExecuteOnEnum () const

UserObjectName	getUserObjectName (const std::string &param_name) const

const T &	getUserObject (const std::string &param_name, bool is_dependency=true) const

const T &	getUserObjectByName (const UserObjectName &object_name, bool is_dependency=true) const

const UserObjectBase &	getUserObjectBase (const std::string &param_name, bool is_dependency=true) const

const UserObjectBase &	getUserObjectBaseByName (const UserObjectName &object_name, bool is_dependency=true) const

const std::vector< MooseVariableScalar * > &	getCoupledMooseScalarVars ()

const std::set< TagID > &	getScalarVariableCoupleableVectorTags () const

const std::set< TagID > &	getScalarVariableCoupleableMatrixTags () const

const GenericMaterialProperty< T, is_ad > &	getGenericMaterialProperty (const std::string &name, MaterialData &material_data, const unsigned int state=0)

const GenericMaterialProperty< T, is_ad > &	getGenericMaterialProperty (const std::string &name, const unsigned int state=0)

const GenericMaterialProperty< T, is_ad > &	getGenericMaterialProperty (const std::string &name, const unsigned int state=0)

const MaterialProperty< T > &	getMaterialProperty (const std::string &name, MaterialData &material_data, const unsigned int state=0)

const MaterialProperty< T > &	getMaterialProperty (const std::string &name, const unsigned int state=0)

const MaterialProperty< T > &	getMaterialProperty (const std::string &name, const unsigned int state=0)

const ADMaterialProperty< T > &	getADMaterialProperty (const std::string &name, MaterialData &material_data)

const ADMaterialProperty< T > &	getADMaterialProperty (const std::string &name)

const ADMaterialProperty< T > &	getADMaterialProperty (const std::string &name)

const MaterialProperty< T > &	getMaterialPropertyOld (const std::string &name, MaterialData &material_data)

const MaterialProperty< T > &	getMaterialPropertyOld (const std::string &name)

const MaterialProperty< T > &	getMaterialPropertyOld (const std::string &name)

const MaterialProperty< T > &	getMaterialPropertyOlder (const std::string &name, MaterialData &material_data)

const MaterialProperty< T > &	getMaterialPropertyOlder (const std::string &name)

const MaterialProperty< T > &	getMaterialPropertyOlder (const std::string &name)

const GenericMaterialProperty< T, is_ad > &	getGenericMaterialPropertyByName (const MaterialPropertyName &name, MaterialData &material_data, const unsigned int state)

const GenericMaterialProperty< T, is_ad > &	getGenericMaterialPropertyByName (const MaterialPropertyName &name, const unsigned int state=0)

const GenericMaterialProperty< T, is_ad > &	getGenericMaterialPropertyByName (const MaterialPropertyName &name, const unsigned int state=0)

const MaterialProperty< T > &	getMaterialPropertyByName (const MaterialPropertyName &name, MaterialData &material_data, const unsigned int state=0)

const MaterialProperty< T > &	getMaterialPropertyByName (const MaterialPropertyName &name, const unsigned int state=0)

const MaterialProperty< T > &	getMaterialPropertyByName (const MaterialPropertyName &name, const unsigned int state=0)

const ADMaterialProperty< T > &	getADMaterialPropertyByName (const MaterialPropertyName &name, MaterialData &material_data)

const ADMaterialProperty< T > &	getADMaterialPropertyByName (const MaterialPropertyName &name)

const ADMaterialProperty< T > &	getADMaterialPropertyByName (const MaterialPropertyName &name)

const MaterialProperty< T > &	getMaterialPropertyOldByName (const MaterialPropertyName &name, MaterialData &material_data)

const MaterialProperty< T > &	getMaterialPropertyOldByName (const MaterialPropertyName &name)

const MaterialProperty< T > &	getMaterialPropertyOldByName (const MaterialPropertyName &name)

const MaterialProperty< T > &	getMaterialPropertyOlderByName (const MaterialPropertyName &name, MaterialData &material_data)

const MaterialProperty< T > &	getMaterialPropertyOlderByName (const MaterialPropertyName &name)

const MaterialProperty< T > &	getMaterialPropertyOlderByName (const MaterialPropertyName &name)

Moose::Kokkos::MaterialProperty< T, dimension >	getKokkosMaterialPropertyByName (const std::string &prop_name_in)

Moose::Kokkos::MaterialProperty< T, dimension >	getKokkosMaterialPropertyOldByName (const std::string &prop_name)

Moose::Kokkos::MaterialProperty< T, dimension >	getKokkosMaterialPropertyOlderByName (const std::string &prop_name)

Moose::Kokkos::MaterialProperty< T, dimension >	getKokkosMaterialProperty (const std::string &name)

Moose::Kokkos::MaterialProperty< T, dimension >	getKokkosMaterialPropertyOld (const std::string &name)

Moose::Kokkos::MaterialProperty< T, dimension >	getKokkosMaterialPropertyOlder (const std::string &name)

std::pair< const MaterialProperty< T > *, std::set< SubdomainID > >	getBlockMaterialProperty (const MaterialPropertyName &name)

std::pair< Moose::Kokkos::MaterialProperty< T, dimension >, std::set< SubdomainID > >	getKokkosBlockMaterialProperty (const MaterialPropertyName &name)

const GenericMaterialProperty< T, is_ad > &	getGenericZeroMaterialProperty (const std::string &name)

const GenericMaterialProperty< T, is_ad > &	getGenericZeroMaterialProperty ()

const GenericMaterialProperty< T, is_ad > &	getGenericZeroMaterialPropertyByName (const std::string &prop_name)

const MaterialProperty< T > &	getZeroMaterialProperty (Ts... args)

std::set< SubdomainID >	getMaterialPropertyBlocks (const std::string &name)

std::vector< SubdomainName >	getMaterialPropertyBlockNames (const std::string &name)

std::set< BoundaryID >	getMaterialPropertyBoundaryIDs (const std::string &name)

std::vector< BoundaryName >	getMaterialPropertyBoundaryNames (const std::string &name)

void	checkBlockAndBoundaryCompatibility (std::shared_ptr< MaterialBase > discrete)

std::unordered_map< SubdomainID, std::vector< MaterialBase * > >	buildRequiredMaterials (bool allow_stateful=true)

void	statefulPropertiesAllowed (bool)

virtual bool	getMaterialPropertyCalled () const

virtual const std::unordered_set< unsigned int > &	getMatPropDependencies () const

virtual void	resolveOptionalProperties ()

const GenericMaterialProperty< T, is_ad > &	getPossiblyConstantGenericMaterialPropertyByName (const MaterialPropertyName &prop_name, MaterialData &material_data, const unsigned int state)

bool	isImplicit ()

Moose::StateArg	determineState () const

virtual void	subdomainSetup () override

virtual void	subdomainSetup () override

bool	hasUserObject (const std::string &param_name) const

bool	hasUserObject (const std::string &param_name) const

bool	hasUserObject (const std::string &param_name) const

bool	hasUserObject (const std::string &param_name) const

bool	hasUserObjectByName (const UserObjectName &object_name) const

bool	hasUserObjectByName (const UserObjectName &object_name) const

bool	hasUserObjectByName (const UserObjectName &object_name) const

bool	hasUserObjectByName (const UserObjectName &object_name) const

const GenericOptionalMaterialProperty< T, is_ad > &	getGenericOptionalMaterialProperty (const std::string &name, const unsigned int state=0)

const GenericOptionalMaterialProperty< T, is_ad > &	getGenericOptionalMaterialProperty (const std::string &name, const unsigned int state=0)

const OptionalMaterialProperty< T > &	getOptionalMaterialProperty (const std::string &name, const unsigned int state=0)

const OptionalMaterialProperty< T > &	getOptionalMaterialProperty (const std::string &name, const unsigned int state=0)

const OptionalADMaterialProperty< T > &	getOptionalADMaterialProperty (const std::string &name)

const OptionalADMaterialProperty< T > &	getOptionalADMaterialProperty (const std::string &name)

const OptionalMaterialProperty< T > &	getOptionalMaterialPropertyOld (const std::string &name)

const OptionalMaterialProperty< T > &	getOptionalMaterialPropertyOld (const std::string &name)

const OptionalMaterialProperty< T > &	getOptionalMaterialPropertyOlder (const std::string &name)

const OptionalMaterialProperty< T > &	getOptionalMaterialPropertyOlder (const std::string &name)

MaterialBase &	getMaterial (const std::string &name)

MaterialBase &	getMaterial (const std::string &name)

MaterialBase &	getMaterialByName (const std::string &name, bool no_warn=false)

MaterialBase &	getMaterialByName (const std::string &name, bool no_warn=false)

bool	hasMaterialProperty (const std::string &name)

bool	hasMaterialProperty (const std::string &name)

bool	hasMaterialPropertyByName (const std::string &name)

bool	hasMaterialPropertyByName (const std::string &name)

bool	hasADMaterialProperty (const std::string &name)

bool	hasADMaterialProperty (const std::string &name)

bool	hasADMaterialPropertyByName (const std::string &name)

bool	hasADMaterialPropertyByName (const std::string &name)

bool	hasKokkosMaterialProperty (const std::string &name)

bool	hasKokkosMaterialProperty (const std::string &name)

bool	hasKokkosMaterialPropertyByName (const std::string &name)

bool	hasKokkosMaterialPropertyByName (const std::string &name)

bool	hasGenericMaterialProperty (const std::string &name)

bool	hasGenericMaterialProperty (const std::string &name)

bool	hasGenericMaterialPropertyByName (const std::string &name)

bool	hasGenericMaterialPropertyByName (const std::string &name)

const Function &	getFunction (const std::string &name) const

const Function &	getFunctionByName (const FunctionName &name) const

bool	hasFunction (const std::string &param_name) const

bool	hasFunctionByName (const FunctionName &name) const

Moose::Kokkos::Function	getKokkosFunction (const std::string &name) const

const T &	getKokkosFunction (const std::string &name) const

Moose::Kokkos::Function	getKokkosFunctionByName (const FunctionName &name) const

const T &	getKokkosFunctionByName (const FunctionName &name) const

bool	hasKokkosFunction (const std::string &param_name) const

bool	hasKokkosFunctionByName (const FunctionName &name) const

bool	isDefaultPostprocessorValue (const std::string &param_name, const unsigned int index=0) const

bool	hasPostprocessor (const std::string &param_name, const unsigned int index=0) const

bool	hasPostprocessorByName (const PostprocessorName &name) const

std::size_t	coupledPostprocessors (const std::string &param_name) const

const PostprocessorName &	getPostprocessorName (const std::string &param_name, const unsigned int index=0) const

const VectorPostprocessorValue &	getVectorPostprocessorValue (const std::string &param_name, const std::string &vector_name) const

const VectorPostprocessorValue &	getVectorPostprocessorValue (const std::string &param_name, const std::string &vector_name, bool needs_broadcast) const

const VectorPostprocessorValue &	getVectorPostprocessorValueByName (const VectorPostprocessorName &name, const std::string &vector_name) const

const VectorPostprocessorValue &	getVectorPostprocessorValueByName (const VectorPostprocessorName &name, const std::string &vector_name, bool needs_broadcast) const

const VectorPostprocessorValue &	getVectorPostprocessorValueOld (const std::string &param_name, const std::string &vector_name) const

const VectorPostprocessorValue &	getVectorPostprocessorValueOld (const std::string &param_name, const std::string &vector_name, bool needs_broadcast) const

const VectorPostprocessorValue &	getVectorPostprocessorValueOldByName (const VectorPostprocessorName &name, const std::string &vector_name) const

const VectorPostprocessorValue &	getVectorPostprocessorValueOldByName (const VectorPostprocessorName &name, const std::string &vector_name, bool needs_broadcast) const

const ScatterVectorPostprocessorValue &	getScatterVectorPostprocessorValue (const std::string &param_name, const std::string &vector_name) const

const ScatterVectorPostprocessorValue &	getScatterVectorPostprocessorValueByName (const VectorPostprocessorName &name, const std::string &vector_name) const

const ScatterVectorPostprocessorValue &	getScatterVectorPostprocessorValueOld (const std::string &param_name, const std::string &vector_name) const

const ScatterVectorPostprocessorValue &	getScatterVectorPostprocessorValueOldByName (const VectorPostprocessorName &name, const std::string &vector_name) const

bool	hasVectorPostprocessor (const std::string &param_name, const std::string &vector_name) const

bool	hasVectorPostprocessor (const std::string &param_name) const

bool	hasVectorPostprocessorByName (const VectorPostprocessorName &name, const std::string &vector_name) const

bool	hasVectorPostprocessorByName (const VectorPostprocessorName &name) const

const VectorPostprocessorName &	getVectorPostprocessorName (const std::string &param_name) const

T &	getSampler (const std::string &name)

Sampler &	getSampler (const std::string &name)

T &	getSamplerByName (const SamplerName &name)

Sampler &	getSamplerByName (const SamplerName &name)

virtual void	meshChanged ()

virtual void	meshDisplaced ()

PerfGraph &	perfGraph ()

const PostprocessorValue &	getPostprocessorValue (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValue (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValueOld (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValueOld (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValueOlder (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValueOlder (const std::string &param_name, const unsigned int index=0) const

virtual const PostprocessorValue &	getPostprocessorValueByName (const PostprocessorName &name) const

virtual const PostprocessorValue &	getPostprocessorValueByName (const PostprocessorName &name) const

const PostprocessorValue &	getPostprocessorValueOldByName (const PostprocessorName &name) const

const PostprocessorValue &	getPostprocessorValueOldByName (const PostprocessorName &name) const

const PostprocessorValue &	getPostprocessorValueOlderByName (const PostprocessorName &name) const

const PostprocessorValue &	getPostprocessorValueOlderByName (const PostprocessorName &name) const

bool	isVectorPostprocessorDistributed (const std::string &param_name) const

bool	isVectorPostprocessorDistributed (const std::string &param_name) const

bool	isVectorPostprocessorDistributedByName (const VectorPostprocessorName &name) const

bool	isVectorPostprocessorDistributedByName (const VectorPostprocessorName &name) const

const Distribution &	getDistribution (const std::string &name) const

const T &	getDistribution (const std::string &name) const

const Distribution &	getDistribution (const std::string &name) const

const T &	getDistribution (const std::string &name) const

const Distribution &	getDistributionByName (const DistributionName &name) const

const T &	getDistributionByName (const std::string &name) const

const Distribution &	getDistributionByName (const DistributionName &name) const

const T &	getDistributionByName (const std::string &name) const

const Parallel::Communicator &	comm () const

processor_id_type	n_processors () const

processor_id_type	processor_id () const

const std::string &	modelMetaDataName () const
	Accessor for the name of the model meta data.

const FileName &	getModelDataFileName () const
	Get the associated filename.

bool	hasModelData () const
	Check if we need to load model data (if the filename parameter is used)

bool	hasExplicitModelData () const
	Check if the filename parameter is explicitly set (ignoring recover state)


template<typename T , typename... Args>
T &	declareModelData (const std::string &data_name, Args &&... args)
	Declare model data for loading from file as well as restart.


template<typename T , typename... Args>
const T &	getModelData (const std::string &data_name, Args &&... args) const
	Retrieve model data from the interface.

Static Public Member Functions
static InputParameters	validParams ()
	Return the input parameters supported by this trainer.

static void	callMooseError (MooseApp const app, const InputParameters &params, std::string msg, const bool with_prefix, const hit::Node node, const bool show_trace=true)

static void	sort (typename std::vector< T > &vector)

static void	sortDFS (typename std::vector< T > &vector)

static void	cyclicDependencyError (CyclicDependencyException< T2 > &e, const std::string &header, NameFunc &&name_func)

static void	cyclicDependencyError (CyclicDependencyException< T2 > &e, const std::string &header)

Public Attributes
	usingCombinedWarningSolutionWarnings

const ConsoleStream	_console

Static Public Attributes
static const std::string	type_param

static const std::string	name_param

static const std::string	unique_name_param

static const std::string	app_param

static const std::string	moose_base_param

static const std::string	kokkos_object_param

static constexpr PropertyValue::id_type	default_property_id

static constexpr PropertyValue::id_type	zero_property_id

static constexpr auto	SYSTEM

static constexpr auto	NAME

Protected Member Functions
void	computeEpisodeRewardStatistics ()
	Compute the average episodic reward statistics for the latest samples.

void	resetData ()
	Reset the stored rollout data after an update.

virtual void	addPostprocessorDependencyHelper (const PostprocessorName &name) const override

virtual void	addVectorPostprocessorDependencyHelper (const VectorPostprocessorName &name) const override

virtual void	addUserObjectDependencyHelper (const UserObjectBase &uo) const override

void	addReporterDependencyHelper (const ReporterName &reporter_name) override

void	flagInvalidSolutionInternal (const InvalidSolutionID invalid_solution_id) const

InvalidSolutionID	registerInvalidSolutionInternal (const std::string &message, const bool warning) const

const ReporterContextBase &	getReporterContextBaseByName (const ReporterName &reporter_name) const

const ReporterName &	getReporterName (const std::string &param_name) const

T &	declareRestartableData (const std::string &data_name, Args &&... args)

ManagedValue< T >	declareManagedRestartableDataWithContext (const std::string &data_name, void *context, Args &&... args)

const T &	getRestartableData (const std::string &data_name) const

T &	declareRestartableDataWithContext (const std::string &data_name, void *context, Args &&... args)

T &	declareRecoverableData (const std::string &data_name, Args &&... args)

T &	declareRestartableDataWithObjectName (const std::string &data_name, const std::string &object_name, Args &&... args)

T &	declareRestartableDataWithObjectNameWithContext (const std::string &data_name, const std::string &object_name, void *context, Args &&... args)

std::string	restartableName (const std::string &data_name) const

const T &	getMeshProperty (const std::string &data_name, const std::string &prefix)

const T &	getMeshProperty (const std::string &data_name)

bool	hasMeshProperty (const std::string &data_name, const std::string &prefix) const

bool	hasMeshProperty (const std::string &data_name, const std::string &prefix) const

bool	hasMeshProperty (const std::string &data_name) const

bool	hasMeshProperty (const std::string &data_name) const

std::string	meshPropertyName (const std::string &data_name) const

PerfID	registerTimedSection (const std::string &section_name, const unsigned int level) const

PerfID	registerTimedSection (const std::string &section_name, const unsigned int level, const std::string &live_message, const bool print_dots=true) const

std::string	timedSectionName (const std::string &section_name) const

bool	isCoupledScalar (const std::string &var_name, unsigned int i=0) const

unsigned int	coupledScalarComponents (const std::string &var_name) const

unsigned int	coupledScalar (const std::string &var_name, unsigned int comp=0) const

libMesh::Order	coupledScalarOrder (const std::string &var_name, unsigned int comp=0) const

const VariableValue &	coupledScalarValue (const std::string &var_name, unsigned int comp=0) const

const ADVariableValue &	adCoupledScalarValue (const std::string &var_name, unsigned int comp=0) const

const GenericVariableValue< is_ad > &	coupledGenericScalarValue (const std::string &var_name, unsigned int comp=0) const

const GenericVariableValue< false > &	coupledGenericScalarValue (const std::string &var_name, const unsigned int comp) const

const GenericVariableValue< true > &	coupledGenericScalarValue (const std::string &var_name, const unsigned int comp) const

const VariableValue &	coupledVectorTagScalarValue (const std::string &var_name, const std::string &tag_param_name, unsigned int comp=0) const

const VariableValue &	coupledMatrixTagScalarValue (const std::string &var_name, const std::string &tag_param_name, unsigned int comp=0) const

const VariableValue &	coupledScalarValueOld (const std::string &var_name, unsigned int comp=0) const

const VariableValue &	coupledScalarValueOlder (const std::string &var_name, unsigned int comp=0) const

const VariableValue &	coupledScalarDot (const std::string &var_name, unsigned int comp=0) const

const ADVariableValue &	adCoupledScalarDot (const std::string &var_name, unsigned int comp=0) const

const VariableValue &	coupledScalarDotDot (const std::string &var_name, unsigned int comp=0) const

const VariableValue &	coupledScalarDotOld (const std::string &var_name, unsigned int comp=0) const

const VariableValue &	coupledScalarDotDotOld (const std::string &var_name, unsigned int comp=0) const

const VariableValue &	coupledScalarDotDu (const std::string &var_name, unsigned int comp=0) const

const VariableValue &	coupledScalarDotDotDu (const std::string &var_name, unsigned int comp=0) const

Moose::Kokkos::VariableValue	kokkosCoupledVectorTagScalarValueByName (const std::string &var_name, const std::string &tag_name, unsigned int comp=0) const

Moose::Kokkos::VariableValue	kokkosCoupledVectorTagScalarValue (const std::string &var_name, const std::string &tag_param_name, unsigned int comp=0) const

Moose::Kokkos::VariableValue	kokkosCoupledScalarValue (const std::string &var_name, unsigned int comp=0) const

Moose::Kokkos::VariableValue	kokkosCoupledScalarValueOld (const std::string &var_name, unsigned int comp=0) const

Moose::Kokkos::VariableValue	kokkosCoupledScalarValueOlder (const std::string &var_name, unsigned int comp=0) const

Moose::Kokkos::VariableValue	kokkosCoupledScalarDot (const std::string &var_name, unsigned int comp=0) const

Moose::Kokkos::Scalar< const Real >	kokkosCoupledScalarDotDu (const std::string &var_name, unsigned int comp=0) const

const MooseVariableScalar *	getScalarVar (const std::string &var_name, unsigned int comp) const

virtual void	checkMaterialProperty (const std::string &name, const unsigned int state)

virtual void	getKokkosMaterialPropertyHook (const std::string &, const unsigned int)

void	markMatPropRequested (const std::string &)

MaterialPropertyName	getMaterialPropertyName (const std::string &name) const

void	checkExecutionStage ()

const T &	getReporterValue (const std::string &param_name, const std::size_t time_index=0)

const T &	getReporterValue (const std::string &param_name, ReporterMode mode, const std::size_t time_index=0)

const T &	getReporterValue (const std::string &param_name, const std::size_t time_index=0)

const T &	getReporterValue (const std::string &param_name, ReporterMode mode, const std::size_t time_index=0)

const T &	getReporterValueByName (const ReporterName &reporter_name, const std::size_t time_index=0)

const T &	getReporterValueByName (const ReporterName &reporter_name, ReporterMode mode, const std::size_t time_index=0)

const T &	getReporterValueByName (const ReporterName &reporter_name, const std::size_t time_index=0)

const T &	getReporterValueByName (const ReporterName &reporter_name, ReporterMode mode, const std::size_t time_index=0)

bool	hasReporterValue (const std::string &param_name) const

bool	hasReporterValue (const std::string &param_name) const

bool	hasReporterValue (const std::string &param_name) const

bool	hasReporterValue (const std::string &param_name) const

bool	hasReporterValueByName (const ReporterName &reporter_name) const

bool	hasReporterValueByName (const ReporterName &reporter_name) const

bool	hasReporterValueByName (const ReporterName &reporter_name) const

bool	hasReporterValueByName (const ReporterName &reporter_name) const

const GenericMaterialProperty< T, is_ad > *	defaultGenericMaterialProperty (const std::string &name)

const GenericMaterialProperty< T, is_ad > *	defaultGenericMaterialProperty (const std::string &name)

const MaterialProperty< T > *	defaultMaterialProperty (const std::string &name)

const MaterialProperty< T > *	defaultMaterialProperty (const std::string &name)

const ADMaterialProperty< T > *	defaultADMaterialProperty (const std::string &name)

const ADMaterialProperty< T > *	defaultADMaterialProperty (const std::string &name)

Static Protected Member Functions
static std::string	meshPropertyName (const std::string &data_name, const std::string &prefix)

Protected Attributes
const std::vector< ReporterName >	_state_names
	Observation reporter names.

std::vector< const std::vector< std::vector< Real > > * >	_state_value_pointers
	Pointers to the current values of the observations We can have multiple observations, multiple samples, multiple timesteps.

const std::vector< Real >	_state_shift_factors
	Shifting constants for the observations.

const std::vector< Real >	_state_scaling_factors
	Scaling constants for the observations.

const std::vector< ReporterName >	_action_names
	Control reporter names.

const std::vector< Real >	_action_scaling_factors
	Multiplicative action scaling embedded in the actor outputs.

std::vector< const std::vector< std::vector< Real > > * >	_action_value_pointers
	Pointers to the current values of the control signals We can have multiple control signals, multiple samples, multiple timesteps.

const std::vector< ReporterName >	_log_probability_names
	Log probability reporter names.

std::vector< const std::vector< std::vector< Real > > * >	_log_probability_value_pointers
	Pointers to the current values of the control log probabilities We can have multiple control signals, multiple samples, multiple timesteps.

const ReporterName	_reward_name
	Reward reporter name.

const std::vector< std::vector< Real > > *	_reward_value_pointer
	Pointer to the current values of the reward We can have multiple samples, multiple timesteps.

const unsigned int	_input_timesteps
	Number of timesteps to fetch from the reporters to be the input of then eural nets.

unsigned int	_num_inputs
	Number of inputs for the control and critic neural nets.

unsigned int	_num_outputs
	Number of outputs for the control neural network.

const unsigned int	_num_epochs
	Number of epochs for the training of the emulator.

const std::vector< unsigned int >	_num_critic_neurons_per_layer
	Number of neurons within the hidden layers in the critic neural net.

const Real	_critic_learning_rate
	The learning rate for the optimization algorithm for the critic.

const std::vector< unsigned int >	_num_control_neurons_per_layer
	Number of neurons within the hidden layers in the control neural net.

const Real	_control_learning_rate
	The learning rate for the optimization algorithm for the control.

const unsigned int	_update_frequency
	Number of transients to run and collect data from before updating the controller neural net.

const Real	_clip_param
	The clip parameter used while clamping the advantage value.

const Real	_decay_factor
	Decaying factor that is used when calculating the return from the reward.

const Real	_lambda_factor
	GAE lambda factor used while estimating advantages and returns.

const std::string	_filename_base
	Name of the pytorch output file.

const bool	_read_from_file
	Switch indicating if an already existing neural net should be read from a file or not.

const bool	_shift_outputs
	Currently, the controls are executed after the user objects at initial in moose.

const bool	_average_reward_over_timestep_window
	Average rewards over each reporter timestep window instead of sampling one boundary value.

Real	_average_episode_reward
	Storage for the current average episode reward.

Real	_std_episode_reward
	Storage for the current episode reward standard deviation.

std::vector< Real >	_sample_average_episode_reward
	Per-sample mean episodic rewards over the latest update window.

std::vector< Real >	_sample_std_episode_reward
	Per-sample episodic reward standard deviations over the latest update window.

const bool	_standardize_advantage
	Switch to enable the standardization of the advantages.

const unsigned int	_loss_print_frequency
	The frequency the loss should be printed.

const unsigned int	_seed
	Base seed for stochastic optimizers and policy sampling.

std::vector< Real >	_min_values
	Optional lower bounds for each control signal.

std::vector< Real >	_max_values
	Optional upper bounds for each control signal.

std::shared_ptr< Moose::LibtorchActorNeuralNet >	_control_nn
	Pointer to the control (or actor) neural net object.

std::shared_ptr< Moose::LibtorchArtificialNeuralNet >	_critic_nn
	Pointer to the critic neural net object.

Real	_highest_reward
	Best average episode reward seen so far while training.

Real	_entropy_coeff
	Entropy bonus coefficient used in the PPO actor loss.

std::unique_ptr< torch::optim::Adam >	_actor_optimizer
	Adam optimizer used to update the actor network.

std::unique_ptr< torch::optim::Adam >	_critic_optimizer
	Adam optimizer used to update the critic network.

const Moose::CoordinateSystemType &	_coord_sys

const THREAD_ID	_tid

SubProblem &	_subproblem

FEProblemBase &	_fe_problem

SystemBase &	_sys

Assembly &	_assembly

const bool	_duplicate_initial_execution

std::set< std::string >	_depend_uo

const bool &	_enabled

MooseApp &	_app

Factory &	_factory

ActionFactory &	_action_factory

const std::string &	_type

const std::string &	_name

const InputParameters &	_pars

const ExecFlagEnum &	_execute_enum

const ExecFlagType &	_current_execute_flag

MooseApp &	_restartable_app

const std::string	_restartable_system_name

const THREAD_ID	_restartable_tid

const bool	_restartable_read_only

FEProblemBase &	_mci_feproblem

FEProblemBase &	_mdi_feproblem

MooseApp &	_pg_moose_app

const std::string	_prefix

FEProblemBase &	_sc_fe_problem

const THREAD_ID	_sc_tid

const Real &	_real_zero

const VariableValue &	_scalar_zero

const Point &	_point_zero

const InputParameters &	_mi_params

const std::string	_mi_name

const MooseObjectName	_mi_moose_object_name

FEProblemBase &	_mi_feproblem

SubProblem &	_mi_subproblem

const THREAD_ID	_mi_tid

const bool	_is_kokkos_object

const Moose::MaterialDataType	_material_data_type

MaterialData &	_material_data

bool	_stateful_allowed

bool	_get_material_property_called

std::vector< std::unique_ptr< PropertyValue > >	_default_properties

std::unordered_set< unsigned int >	_material_property_dependencies

const MaterialPropertyName	_get_suffix

const bool	_use_interpolated_state

const InputParameters &	_ti_params

FEProblemBase &	_ti_feproblem

bool	_is_implicit

Real &	_t

const Real &	_t_old

int &	_t_step

Real &	_dt

Real &	_dt_old

bool	_is_transient

const Parallel::Communicator &	_communicator

Static Protected Attributes
static const std::string	_interpolated_old

static const std::string	_interpolated_older

Private Member Functions
void	getReporterPointers (const std::vector< ReporterName > &reporter_names, std::vector< const std::vector< std::vector< Real > > * > &pointer_storage)
	Resolve reporter names into cached pointer storage.

bool	collectTrajectoriesFromReporters ()
	Pull trajectories out of the reporters and append them to the trajectory buffer.

const Moose::FunctionBase *	getKokkosFunctionByNameHelper (const FunctionName &name) const

const UserObjectBase &	getUserObjectFromFEProblem (const UserObjectName &object_name, const THREAD_ID tid=0) const

const T &	castUserObject (const UserObjectBase &uo_base, const std::string &param_name="") const

void	mooseObjectError (const std::string &param_name, std::stringstream &oss) const

const std::string &	userObjectType (const UserObjectBase &uo) const

const std::string &	userObjectName (const UserObjectBase &uo) const

const PostprocessorName &	getPostprocessorNameInternal (const std::string &param_name, const unsigned int index, const bool allow_default_value=true) const

bool	isDefaultPostprocessorValueByName (const PostprocessorName &name) const

PostprocessorValue	getDefaultPostprocessorValueByName (const PostprocessorName &name) const

void	checkParam (const std::string &param_name, const unsigned int index=std::numeric_limits< unsigned int >::max()) const

bool	postprocessorsAdded () const

const VectorPostprocessorValue &	getVectorPostprocessorByNameHelper (const VectorPostprocessorName &name, const std::string &vector_name, bool broadcast, std::size_t t_index) const

const VectorPostprocessorContext< VectorPostprocessorValue > &	getVectorPostprocessorContextByNameHelper (const VectorPostprocessorName &name, const std::string &vector_name) const

bool	vectorPostprocessorsAdded () const

bool	reportersAdded () const

void	possiblyCheckHasReporter (const ReporterName &reporter_name, const std::string &param_name="") const

RestartableDataValue &	registerRestartableDataOnApp (std::unique_ptr< RestartableDataValue > data, THREAD_ID tid) const

void	registerRestartableNameWithFilterOnApp (const std::string &name, Moose::RESTARTABLE_FILTER filter)

RestartableData< T > &	declareRestartableDataHelper (const std::string &data_name, void *context, Args &&... args) const

virtual std::string	meshPropertyPrefix (const std::string &data_name) const

const RestartableDataValue &	getMeshPropertyInternal (const std::string &data_name, const std::string &prefix) const

void	mooseErrorInternal (Args &&... args) const

const VariableValue *	getDefaultValue (const std::string &var_name) const

const ADVariableValue *	getADDefaultValue (const std::string &var_name) const

void	checkVar (const std::string &var_name) const

void	validateExecutionerType (const std::string &name, const std::string &fn_name) const

Moose::Kokkos::Variable	kokkosCoupledVectorTagScalarVariable (const std::string &var_name, const std::string &tag_name, unsigned int comp) const

Moose::MaterialDataType	getMaterialDataType (const std::set< BoundaryID > &boundary_ids) const

unsigned int	getMaxQps () const

void	addConsumedPropertyName (const MooseObjectName &obj_name, const std::string &prop_name)

const PostprocessorValue &	getPostprocessorValueInternal (const std::string &param_name, unsigned int index, std::size_t t_index) const

const PostprocessorValue &	getPostprocessorValueInternal (const std::string &param_name, unsigned int index, std::size_t t_index) const

const PostprocessorValue &	getPostprocessorValueByNameInternal (const PostprocessorName &name, std::size_t t_index) const

const PostprocessorValue &	getPostprocessorValueByNameInternal (const PostprocessorName &name, std::size_t t_index) const

void	possiblyCheckHasVectorPostprocessor (const std::string &param_name, const std::string &vector_name) const

void	possiblyCheckHasVectorPostprocessor (const std::string &param_name, const std::string &vector_name) const

void	possiblyCheckHasVectorPostprocessorByName (const VectorPostprocessorName &name, const std::string &vector_name) const

void	possiblyCheckHasVectorPostprocessorByName (const VectorPostprocessorName &name, const std::string &vector_name) const

Static Private Member Functions
static const hit::Node *	getHitNode (const InputParameters &params)

static std::string	messagePrefix (const InputParameters &params, const bool hit_prefix)

Private Attributes
unsigned int	_update_counter
	Counter for number of transient simulations that have been run before updating the controller.

const LibtorchRLTrajectoryAssembler	_trajectory_assembler
	Builds rollout trajectories from the raw reporter sequences.

LibtorchRLTrajectoryBuffer	_trajectory_buffer
	Accumulated on-policy rollout data waiting to be flattened and trained on.

const LibtorchRLMiniBatchSampler	_sampler
	Mini-batch sampler used to split flattened rollout data for PPO updates.

const LibtorchRLValueEstimator	_value_estimator
	Helper that builds value targets and advantages from collected trajectories.

const LibtorchRLPPOLoss	_ppo_loss
	PPO loss helper for the actor and critic updates.

UserObject *	_primary_thread_copy

std::set< std::string >	_supplied_uo

const ParallelParamObject &	_parent

const MooseBase &	_si_moose_base

const FEProblemBase *	_si_problem

ExecFlagEnum	_empty_execute_enum

const MooseObject &	_fni_object

const InputParameters &	_fni_params

FEProblemBase &	_fni_feproblem

const THREAD_ID	_fni_tid

const MooseObject &	_uoi_moose_object

const FEProblemBase &	_uoi_feproblem

const THREAD_ID	_uoi_tid

const MooseObject &	_ppi_moose_object

const InputParameters &	_ppi_params

const FEProblemBase &	_ppi_feproblem

std::map< PostprocessorName, std::unique_ptr< PostprocessorValue > >	_default_values

const bool	_broadcast_by_default

const MooseObject &	_vpi_moose_object

const FEProblemBase &	_vpi_feproblem

const THREAD_ID	_vpi_tid

const InputParameters &	_ri_params

FEProblemBase &	_ri_fe_problem_base

const ReporterData &	_ri_reporter_data

const MooseObject &	_ri_moose_object

const InputParameters &	_dni_params

FEProblemBase &	_dni_feproblem

const MooseObject *const	_dni_moose_object_ptr

const InputParameters &	_si_params

FEProblemBase &	_si_feproblem

THREAD_ID	_si_tid

const RestartableDataMapName	_metaname

std::string	_restartable_name

MooseApp &	_meta_data_app

const MooseObject *const	_meta_data_object

const InputParameters &	_sc_parameters

const std::string &	_sc_name

const bool	_sc_is_implicit

std::unordered_map< std::string, std::vector< MooseVariableScalar * > >	_coupled_scalar_vars

std::unordered_map< std::string, std::unique_ptr< VariableValue > >	_default_value

std::unordered_map< std::string, std::unique_ptr< ADVariableValue > >	_dual_default_value

std::vector< MooseVariableScalar * >	_coupled_moose_scalar_vars

std::unordered_map< std::string, std::vector< MooseVariableFieldBase * > >	_sc_coupled_vars

std::set< TagID >	_sc_coupleable_vector_tags

std::set< TagID >	_sc_coupleable_matrix_tags

const MooseObject &	_mi_moose_object

const bool	_mi_boundary_restricted

const std::set< SubdomainID > &	_mi_block_ids

const std::set< BoundaryID > &	_mi_boundary_ids

std::vector< std::unique_ptr< OptionalMaterialPropertyProxyBase< MaterialPropertyInterface > > >	_optional_property_proxies

const std::string	_ti_name

const MooseObject &	_model_object
	Reference to the MooseObject that uses this interface.

const std::string	_model_meta_data_name
	The model meta data name.

PublicRestartable	_model_restartable
	Member for interfacing with the framework's restartable system.

Detailed Description

Fixed-horizon actor-critic trainer that collects trajectories from MOOSE reporters and runs a PPO update on top of reusable RL-core components (observation history, trajectory buffer, mini-batch sampler, value estimator, and PPO loss).

Definition at line 31 of file LibtorchDRLControlTrainer.h.

Constructor & Destructor Documentation

◆ LibtorchDRLControlTrainer()

LibtorchDRLControlTrainer::LibtorchDRLControlTrainer ( const InputParameters & parameters )

Build the PPO-based DRL trainer.

Parameters

parameters Input parameters for the trainer.

Definition at line 142 of file LibtorchDRLControlTrainer.C.

  : SurrogateTrainerBase(parameters),
    _state_names(getParam<std::vector<ReporterName>>("observation")),
    _state_shift_factors(isParamSetByUser("observation_shift_factors")
                             ? getParam<std::vector<Real>>("observation_shift_factors")
                             : std::vector<Real>(_state_names.size(), 0.0)),
    _state_scaling_factors(isParamSetByUser("observation_scaling_factors")
                               ? getParam<std::vector<Real>>("observation_scaling_factors")
                               : std::vector<Real>(_state_names.size(), 1.0)),
    _action_names(getParam<std::vector<ReporterName>>("control")),
    _action_scaling_factors(isParamSetByUser("action_scaling_factors")
                                ? getParam<std::vector<Real>>("action_scaling_factors")
                                : std::vector<Real>(_action_names.size(), 1.0)),
    _log_probability_names(getParam<std::vector<ReporterName>>("log_probability")),
    _reward_name(getParam<ReporterName>("reward")),
    _reward_value_pointer(&getReporterValueByName<std::vector<std::vector<Real>>>(_reward_name)),
    _input_timesteps(getParam<unsigned int>("input_timesteps")),
    _num_inputs(_input_timesteps * _state_names.size()),
    _num_outputs(_action_names.size()),
    _num_epochs(getParam<unsigned int>("num_epochs")),
    _num_critic_neurons_per_layer(
        getParam<std::vector<unsigned int>>("num_critic_neurons_per_layer")),
    _critic_learning_rate(getParam<Real>("critic_learning_rate")),
    _num_control_neurons_per_layer(
        getParam<std::vector<unsigned int>>("num_control_neurons_per_layer")),
    _control_learning_rate(getParam<Real>("control_learning_rate")),
    _update_frequency(getParam<unsigned int>("update_frequency")),
    _clip_param(getParam<Real>("clip_parameter")),
    _decay_factor(getParam<Real>("decay_factor")),
    _lambda_factor(getParam<Real>("lambda_factor")),
    _filename_base(isParamValid("filename_base") ? getParam<std::string>("filename_base") : ""),
    _read_from_file(getParam<bool>("read_from_file")),
    _shift_outputs(getParam<bool>("shift_outputs")),
    _average_reward_over_timestep_window(getParam<bool>("average_reward_over_timestep_window")),
    _average_episode_reward(0.0),
    _std_episode_reward(0.0),
    _standardize_advantage(getParam<bool>("standardize_advantage")),
    _loss_print_frequency(getParam<unsigned int>("loss_print_frequency")),
    _seed(getParam<unsigned int>("seed")),
    _min_values(getParam<std::vector<Real>>("min_control_value")),
    _max_values(getParam<std::vector<Real>>("max_control_value")),
    _highest_reward(-1e8),
    _entropy_coeff(getParam<Real>("entropy_coeff")),
    _update_counter(_update_frequency),
    _trajectory_assembler(_input_timesteps,
                          getParam<unsigned int>("timestep_window"),
                          _shift_outputs,
                          _average_reward_over_timestep_window),
    _sampler(LibtorchRLMiniBatchSampler()),
    _value_estimator(_decay_factor, _lambda_factor),
    _ppo_loss(_clip_param, _entropy_coeff)
{
  if (_state_names.size() != _state_shift_factors.size())
    paramError("observation_shift_factors",
               "The number of shift factors is not the same as the number of observations!");
 
  if (_state_names.size() != _state_scaling_factors.size())
    paramError("observation_scaling_factors",
               "The number of normalization coefficients is not the same as the number of "
               "observations!");
 
  if (_action_names.size() != _log_probability_names.size())
    paramError("log_probability",
               "The number of log-probability reporters must match the number of control "
               "reporters.");
 
  if (_action_names.size() != _action_scaling_factors.size())
    paramError("action_scaling_factors",
               "The number of action scaling factors must match the number of control "
               "reporters.");
 
  // We establish the links with the chosen reporters
  getReporterPointers(_state_names, _state_value_pointers);
  getReporterPointers(_action_names, _action_value_pointers);
  getReporterPointers(_log_probability_names, _log_probability_value_pointers);
 
  bool filename_valid = isParamValid("filename_base");
  const LibtorchObservationHistoryHelper observation_history(_input_timesteps);
  const auto input_shift_factors =
      observation_history.expandObservationFactors(_state_shift_factors);
  const auto input_scaling_factors =
      observation_history.expandObservationFactors(_state_scaling_factors);
 
  // Initializing the control neural net so that the control can grab it right away
  _control_nn = std::make_shared<Moose::LibtorchActorNeuralNet>(
      filename_valid ? _filename_base + "_control.net" : "control.net",
      _num_inputs,
      _num_outputs,
      _num_control_neurons_per_layer,
      getParam<std::vector<std::string>>("control_activation_functions"),
      _min_values,
      _max_values,
      torch::kCPU,
      torch::kDouble,
      true,
      input_shift_factors,
      input_scaling_factors,
      _action_scaling_factors,
      getParam<bool>("state_independent_std"));
 
  // We read parameters for the control neural net if it is requested
  if (_read_from_file)
  {
    Moose::loadLibtorchActorNeuralNetState(*_control_nn, _control_nn->name());
    _console << "Loaded requested .pt file." << std::endl;
  }
  else if (filename_valid)
    torch::save(_control_nn, _control_nn->name());
 
  // Initialize the critic neural net
  _critic_nn = std::make_shared<Moose::LibtorchArtificialNeuralNet>(
      filename_valid ? _filename_base + "_ctiric.net" : "ctiric.net",
      _num_inputs,
      1,
      _num_critic_neurons_per_layer,
      getParam<std::vector<std::string>>("critic_activation_functions"),
      torch::kCPU,
      torch::kDouble,
      true,
      input_shift_factors,
      input_scaling_factors);
 
  _actor_optimizer = std::make_unique<torch::optim::Adam>(
      _control_nn->parameters(), torch::optim::AdamOptions(_control_learning_rate));
  _critic_optimizer = std::make_unique<torch::optim::Adam>(
      _critic_nn->parameters(), torch::optim::AdamOptions(_critic_learning_rate));
 
  // We read parameters for the critic neural net if it is requested
  if (_read_from_file)
  {
    try
    {
      Moose::loadLibtorchArtificialNeuralNetState(*_critic_nn, _critic_nn->name());
      _console << "Loaded requested .pt file." << std::endl;
    }
    catch (const c10::Error & e)
    {
      mooseError("The requested pytorch file could not be loaded for the critic neural net.\n",
                 e.msg());
    }
  }
  else if (filename_valid)
    torch::save(_critic_nn, _critic_nn->name());
 
  // A loaded checkpoint already contains initialized parameters. Reinitializing here would
  // silently discard the actor and critic states requested with read_from_file.
  if (!_read_from_file)
  {
    _control_nn->initializeNeuralNetwork(
        Moose::makeLibtorchCPUGenerator(static_cast<uint64_t>(_seed)));
    _critic_nn->initializeNeuralNetwork(
        Moose::makeLibtorchCPUGenerator(static_cast<uint64_t>(_seed) + 1));
  }
}

Member Function Documentation

◆ averageEpisodeReward()

Real LibtorchDRLControlTrainer::averageEpisodeReward ( )

inline

Return the current average episodic reward.

Returns: Average episodic reward over the latest training window.

Definition at line 54 of file LibtorchDRLControlTrainer.h.

54{ return _average_episode_reward; }

Referenced by DRLRewardReporter::execute().

◆ collectTrajectoriesFromReporters()

bool LibtorchDRLControlTrainer::collectTrajectoriesFromReporters ( )

private

Pull trajectories out of the reporters and append them to the trajectory buffer.

Returns: whether any trajectories were collected on this rank.

Definition at line 416 of file LibtorchDRLControlTrainer.C.

{
  auto trajectories = _trajectory_assembler.assemble(_state_value_pointers,
                                                     _action_value_pointers,
                                                     _log_probability_value_pointers,
                                                     *_reward_value_pointer);
  for (auto & trajectory : trajectories)
    _trajectory_buffer.addTrajectory(std::move(trajectory));
  return !trajectories.empty();
}

Referenced by execute().

◆ computeEpisodeRewardStatistics()

void LibtorchDRLControlTrainer::computeEpisodeRewardStatistics ( )

protected

Compute the average episodic reward statistics for the latest samples.

Definition at line 331 of file LibtorchDRLControlTrainer.C.

{
  const auto statistics = _trajectory_buffer.rewardStatistics();
  _average_episode_reward = statistics.mean;
  _std_episode_reward = statistics.std;
  _sample_average_episode_reward = statistics.trajectory_means;
  _sample_std_episode_reward = statistics.trajectory_stds;
}

Referenced by execute().

◆ controlNeuralNet()

const Moose::LibtorchActorNeuralNet & LibtorchDRLControlTrainer::controlNeuralNet ( ) const

inline

Return the current actor network.

Definition at line 79 of file LibtorchDRLControlTrainer.h.

79{ return *_control_nn; }

Referenced by LibtorchNeuralNetControlTransfer::execute(), SamplerDRLControlTransfer::execute(), and SamplerDRLControlTransfer::executeToMultiapp().

◆ declareModelData()

template<typename T , typename... Args>

T & RestartableModelInterface::declareModelData	(	const std::string &	data_name,
		Args &&...	args
	)

inherited

Declare model data for loading from file as well as restart.

Definition at line 76 of file RestartableModelInterface.h.

{
  return _model_restartable.declareRestartableData<T>(data_name, std::forward<Args>(args)...);
}

◆ execute()

void LibtorchDRLControlTrainer::execute ( )

overridevirtual

Pull fresh rollout data from the reporters and trigger training when ready.

Implements GeneralUserObject.

Definition at line 298 of file LibtorchDRLControlTrainer.C.

{
  // Only executions that collect trajectories count toward update_frequency, so an execution
  // without complete trajectories, such as on INITIAL, neither triggers nor delays training.
  bool collected_trajectories = collectTrajectoriesFromReporters();
  _communicator.max(collected_trajectories);
  if (!collected_trajectories)
    return;
 
  mooseAssert(_update_counter > 0, "The update counter is reset after every training.");
  if (--_update_counter != 0)
    return;
 
  computeEpisodeRewardStatistics();
  _communicator.broadcast(_average_episode_reward, 0);
  _communicator.broadcast(_std_episode_reward, 0);
 
  if (processor_id() == 0 && _average_episode_reward > _highest_reward)
  {
    torch::save(_control_nn, _control_nn->name() + "_best");
    _highest_reward = _average_episode_reward;
  }
  _communicator.broadcast(_highest_reward, 0);
 
  if (processor_id() == 0)
    _value_estimator.computeValueTargets(_trajectory_buffer, *_critic_nn);
  const auto batch = _trajectory_buffer.flatten();
 
  trainController(batch);
  resetData();
}

◆ finalize()

virtual void SurrogateTrainerBase::finalize ( )

inlinevirtualinherited

Implements GeneralUserObject.

Reimplemented in SurrogateTrainer, and PODReducedBasisTrainer.

Definition at line 39 of file SurrogateTrainer.h.

39{} // not required, but available

◆ getModelData()

template<typename T , typename... Args>

const T & RestartableModelInterface::getModelData	(	const std::string &	data_name,
		Args &&...	args
	)		const

inherited

Retrieve model data from the interface.

Definition at line 83 of file RestartableModelInterface.h.

{
  return _model_restartable.getRestartableData<T>(data_name, std::forward<Args>(args)...);
}

◆ getModelDataFileName()

const FileName & RestartableModelInterface::getModelDataFileName ( ) const

inherited

Get the associated filename.

Definition at line 33 of file RestartableModelInterface.C.

{
  return _model_object.getParam<FileName>("filename");
}

◆ getReporterPointers()

void LibtorchDRLControlTrainer::getReporterPointers	(	const std::vector< ReporterName > &	reporter_names,
		std::vector< const std::vector< std::vector< Real > > * > &	pointer_storage
	)

private

Resolve reporter names into cached pointer storage.

Parameters

reporter_names	Reporter names to look up.
pointer_storage	Output vector that receives the reporter pointers.

Definition at line 428 of file LibtorchDRLControlTrainer.C.

{
  pointer_storage.clear();
  for (const auto & name : reporter_names)
    pointer_storage.push_back(&getReporterValueByName<std::vector<std::vector<Real>>>(name));
}

Referenced by LibtorchDRLControlTrainer().

◆ hasExplicitModelData()

bool RestartableModelInterface::hasExplicitModelData ( ) const

inherited

Check if the filename parameter is explicitly set (ignoring recover state)

Definition at line 39 of file RestartableModelInterface.C.

{
  return _model_object.isParamValid("filename");
}

Referenced by RestartableModelInterface::hasModelData().

◆ hasModelData()

bool RestartableModelInterface::hasModelData ( ) const

inherited

Check if we need to load model data (if the filename parameter is used)

Definition at line 45 of file RestartableModelInterface.C.

{
  return hasExplicitModelData() || _model_object.getMooseApp().isRecovering();
}

◆ initialize()

virtual void SurrogateTrainerBase::initialize ( )

inlinevirtualinherited

Implements GeneralUserObject.

Reimplemented in ActiveLearningGaussianProcess, SurrogateTrainer, and PODReducedBasisTrainer.

Definition at line 38 of file SurrogateTrainer.h.

38{} // not required, but available

◆ inputTimesteps()

unsigned int LibtorchDRLControlTrainer::inputTimesteps ( ) const

inline

Return the number of observation time levels stacked into the network inputs.

Definition at line 94 of file LibtorchDRLControlTrainer.h.

94{ return _input_timesteps; }

Referenced by SamplerDRLControlTransfer::getDRLControl().

◆ modelMetaDataName()

const std::string & RestartableModelInterface::modelMetaDataName ( ) const

inlineinherited

Accessor for the name of the model meta data.

Definition at line 47 of file RestartableModelInterface.h.

47{ return _model_meta_data_name; }

RestartableModelInterface::_model_meta_data_name

const std::string _model_meta_data_name

The model meta data name.

Definition RestartableModelInterface.h:64

Referenced by MappingOutput::output(), and SurrogateTrainerOutput::output().

◆ numberOfControlSignals()

unsigned int LibtorchDRLControlTrainer::numberOfControlSignals ( ) const

inline

Return the number of control signals produced by the actor.

Definition at line 104 of file LibtorchDRLControlTrainer.h.

104{ return _action_names.size(); }

Referenced by SamplerDRLControlTransfer::getDRLControl().

◆ numberOfObservations()

unsigned int LibtorchDRLControlTrainer::numberOfObservations ( ) const

inline

Return the number of observation reporters read at each time level.

Definition at line 99 of file LibtorchDRLControlTrainer.h.

99{ return _state_names.size(); }

Referenced by SamplerDRLControlTransfer::getDRLControl().

◆ resetData()

void LibtorchDRLControlTrainer::resetData ( )

protected

Reset the stored rollout data after an update.

Definition at line 409 of file LibtorchDRLControlTrainer.C.

{
  _trajectory_buffer.clear();
  _update_counter = _update_frequency;
}

Referenced by execute().

◆ sampleAverageEpsiodeRewards()

std::vector< Real > LibtorchDRLControlTrainer::sampleAverageEpsiodeRewards ( )

inline

Return per-sample mean episodic rewards from the latest update window.

Definition at line 63 of file LibtorchDRLControlTrainer.h.

63{ return _sample_average_episode_reward; }

Referenced by DRLRewardReporter::execute().

◆ sampleStdEpsiodeRewards()

std::vector< Real > LibtorchDRLControlTrainer::sampleStdEpsiodeRewards ( )

inline

Return per-sample episodic reward standard deviations from the latest update window.

Definition at line 68 of file LibtorchDRLControlTrainer.h.

68{ return _sample_std_episode_reward; }

Referenced by DRLRewardReporter::execute().

◆ seed()

unsigned int LibtorchDRLControlTrainer::seed ( ) const

inline

Return the trainer seed used for sampling and shuffling.

Definition at line 84 of file LibtorchDRLControlTrainer.h.

84{ return _seed; }

Referenced by SamplerDRLControlTransfer::executeToMultiapp().

◆ stdEpisodeReward()

Real LibtorchDRLControlTrainer::stdEpisodeReward ( )

inline

Return the current episodic reward standard deviation.

Definition at line 58 of file LibtorchDRLControlTrainer.h.

58{ return _std_episode_reward; }

Referenced by DRLRewardReporter::execute().

◆ threadJoin()

virtual void SurrogateTrainerBase::threadJoin ( const UserObject & )

inlinefinalvirtualinherited

Reimplemented from GeneralUserObject.

Definition at line 40 of file SurrogateTrainer.h.

40{} // GeneralUserObjects are not threaded

◆ timestepWindow()

unsigned int LibtorchDRLControlTrainer::timestepWindow ( ) const

inline

Return the number of raw reporter entries per training transition.

Definition at line 89 of file LibtorchDRLControlTrainer.h.

89{ return getParam<unsigned int>("timestep_window"); }

Referenced by SamplerDRLControlTransfer::getDRLControl().

◆ trainController()

void LibtorchDRLControlTrainer::trainController ( const LibtorchRLTrajectoryBuffer::TensorBatch & batch )

Run the PPO update on a flattened on-policy batch.

Parameters

batch Flattened trajectory batch to train on.

Definition at line 341 of file LibtorchDRLControlTrainer.C.

{
  bool has_batch_data = batch.size();
  _communicator.max(has_batch_data);
 
  if (!has_batch_data)
    return;
 
  // We only train on the rank 0 partition. Libtorch should still be able to
  // fetch the local threads which are available.
  if (processor_id() == 0 && batch.size())
  {
    auto shuffle_generator = Moose::makeLibtorchCPUGenerator(
        static_cast<uint64_t>(_seed) + static_cast<uint64_t>(_fe_problem.timeStep()));
 
    for (unsigned int epoch = 0; epoch < _num_epochs; ++epoch)
    {
      const auto mini_batches = _sampler.sample(
          batch, getParam<unsigned int>("batch_size"), _standardize_advantage, shuffle_generator);
      bool printed_losses = false;
      for (const auto & mini_batch : mini_batches)
      {
        const auto losses = _ppo_loss.compute(*_control_nn, *_critic_nn, mini_batch);
 
        _actor_optimizer->zero_grad();
        losses.actor_loss.backward();
        _actor_optimizer->step();
 
        _critic_optimizer->zero_grad();
        losses.critic_loss.backward();
        _critic_optimizer->step();
 
        if (_loss_print_frequency && epoch % _loss_print_frequency == 0 && !printed_losses)
        {
          _console << "Epoch: " << epoch << " | Actor Loss: " << COLOR_GREEN
                   << losses.actor_loss.item<double>() << COLOR_DEFAULT
                   << " | Critic Loss: " << COLOR_GREEN << losses.critic_loss.item<double>()
                   << COLOR_DEFAULT << std::endl;
          printed_losses = true;
        }
      }
    }
 
    _console << "Best model so far: " << _highest_reward << std::endl;
  }
 
  // It is time to send the trained data to every other processor so that the neural networks
  // are the same on all ranks. TODO: Make sure this can be done on a GPU as well.
  for (auto & param : _control_nn->named_parameters())
  {
    MPI_Bcast(param.value().data_ptr(), param.value().numel(), MPI_DOUBLE, 0, _communicator.get());
  }
 
  for (auto & param : _critic_nn->named_parameters())
  {
    MPI_Bcast(param.value().data_ptr(), param.value().numel(), MPI_DOUBLE, 0, _communicator.get());
  }
 
  // Save the controller neural net so our controller can read it, we also save the critic if we
  // want to continue training
  if (processor_id() == 0)
  {
    torch::save(_control_nn, _control_nn->name());
    torch::save(_critic_nn, _critic_nn->name());
  }
}

Referenced by execute().

◆ validParams()

InputParameters LibtorchDRLControlTrainer::validParams ( )

static

Return the input parameters supported by this trainer.

Definition at line 18 of file LibtorchDRLControlTrainer.C.

{
  InputParameters params = SurrogateTrainerBase::validParams();
 
  params.addClassDescription(
      "Trains a neural network controller using fixed-horizon PPO on top of the libtorch RL core.");
 
  params.addRequiredParam<std::vector<ReporterName>>(
      "observation", "Reporter values containing the observation values from the model.");
  params.addParam<std::vector<Real>>(
      "observation_shift_factors",
      {},
      "Optional offsets applied to the observed state values before scaling.");
  params.addParam<std::vector<Real>>(
      "observation_scaling_factors",
      {},
      "Optional multipliers applied after shifting the observed state values.");
  params.addRequiredParam<std::vector<ReporterName>>(
      "control",
      "Reporters containing the values of the controlled quantities (control signals) from the "
      "model simulations.");
  params.addParam<std::vector<Real>>(
      "action_scaling_factors",
      {},
      "Scale factors embedded into the trained policy outputs so transferred and checkpointed "
      "controllers operate in physical units.");
  params.addRequiredParam<std::vector<ReporterName>>(
      "log_probability",
      "Reporters containing the log probabilities of the actions taken during the simulations.");
  params.addRequiredParam<ReporterName>(
      "reward", "Reporter containing the earned time-dependent rewards from the simulation.");
  params.addRangeCheckedParam<unsigned int>(
      "input_timesteps",
      1,
      "1<=input_timesteps",
      "Number of time steps to use in the input data, if larger than 1, "
      "data from the previous timesteps will be used as inputs in the training.");
 
  params.addRequiredParam<unsigned int>("num_epochs", "Number of epochs for the training.");
 
  params.addRequiredRangeCheckedParam<Real>("critic_learning_rate",
                                            "0<critic_learning_rate",
                                            "Learning rate used by the critic optimizer.");
  params.addRequiredParam<std::vector<unsigned int>>("num_critic_neurons_per_layer",
                                                     "Hidden-layer widths for the critic network.");
  params.addParam<std::vector<std::string>>(
      "critic_activation_functions",
      std::vector<std::string>({"relu"}),
      "Activation name for each critic hidden layer, or one shared value for all layers.");
 
  params.addRequiredRangeCheckedParam<Real>("control_learning_rate",
                                            "0<control_learning_rate",
                                            "Learning rate used by the actor optimizer.");
  params.addRequiredParam<std::vector<unsigned int>>(
      "num_control_neurons_per_layer",
      "Number of neurons per layer for the control neural network.");
  params.addParam<std::vector<std::string>>(
      "control_activation_functions",
      std::vector<std::string>({"relu"}),
      "Activation name for each actor hidden layer, or one shared value for all layers.");
 
  params.addParam<std::string>("filename_base",
                               "Base filename used when writing actor and critic checkpoints.");
 
  params.addParam<unsigned int>(
      "seed", 11, "Random number generator seed for stochastic optimizers.");
 
  params.addParam<Real>(
      "clip_parameter", 0.2, "Clip parameter used while clamping the advantage value.");
  params.addRangeCheckedParam<unsigned int>(
      "update_frequency",
      1,
      "1<=update_frequency",
      "Number of transient simulation data to collect for updating the controller neural network.");
 
  params.addRangeCheckedParam<Real>(
      "decay_factor",
      1.0,
      "0.0<=decay_factor<=1.0",
      "Discount factor used when building PPO return and GAE targets.");
 
  params.addRangeCheckedParam<Real>("lambda_factor", 1.0, "0.0<=lambda_factor<=1.0", "GAE lambda.");
 
  params.addParam<bool>(
      "read_from_file", false, "Switch to read the neural network parameters from a file.");
  params.addParam<bool>(
      "shift_outputs",
      true,
      "Whether to shift rollout outputs so observations and actions line up in time.");
  params.addParam<bool>(
      "standardize_advantage",
      true,
      "Switch to enable the shifting and normalization of the advantages in the PPO algorithm.");
  params.addParam<unsigned int>(
      "loss_print_frequency", 0, "Print PPO loss values every N updates. Use 0 to stay quiet.");
  params.addParam<unsigned int>("batch_size", 100, "Number of flattened samples per mini-batch.");
  params.addParam<std::vector<Real>>(
      "min_control_value", {}, "Optional lower bounds for each control signal.");
  params.addParam<std::vector<Real>>(
      "max_control_value", {}, "Optional upper bounds for each control signal.");
  params.addParam<bool>(
      "state_independent_std",
      true,
      "If true, learn one Gaussian log-std parameter per action dimension. If false, learn a "
      "state-dependent std head as in the older MOOSE actor implementation.");
 
  params.addParam<Real>(
      "entropy_coeff", 0.01, "Entropy bonus coefficient used in the PPO actor loss.");
 
  params.addRangeCheckedParam<unsigned int>(
      "timestep_window",
      1,
      "1<=timestep_window",
      "Use every nth reporter timestep when assembling trajectories. Observation history lags "
      "also span this many timesteps, so it should match the controller's "
      "'num_steps_in_period'.");
  params.addParam<bool>("average_reward_over_timestep_window",
                        false,
                        "Average reward reporter values over each timestep_window interval instead "
                        "of sampling one value at the end of the interval.");
 
  return params;
}

Member Data Documentation

◆ _action_names

const std::vector<ReporterName> LibtorchDRLControlTrainer::_action_names

protected

Control reporter names.

Definition at line 131 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer(), and numberOfControlSignals().

◆ _action_scaling_factors

const std::vector<Real> LibtorchDRLControlTrainer::_action_scaling_factors

protected

Multiplicative action scaling embedded in the actor outputs.

Definition at line 134 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _action_value_pointers

std::vector<const std::vector<std::vector<Real> > *> LibtorchDRLControlTrainer::_action_value_pointers

protected

Pointers to the current values of the control signals We can have multiple control signals, multiple samples, multiple timesteps.

Definition at line 138 of file LibtorchDRLControlTrainer.h.

Referenced by collectTrajectoriesFromReporters(), and LibtorchDRLControlTrainer().

◆ _actor_optimizer

std::unique_ptr<torch::optim::Adam> LibtorchDRLControlTrainer::_actor_optimizer

protected

Adam optimizer used to update the actor network.

Definition at line 239 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer(), and trainController().

◆ _average_episode_reward

Real LibtorchDRLControlTrainer::_average_episode_reward

protected

Storage for the current average episode reward.

Definition at line 205 of file LibtorchDRLControlTrainer.h.

Referenced by averageEpisodeReward(), computeEpisodeRewardStatistics(), and execute().

◆ _average_reward_over_timestep_window

const bool LibtorchDRLControlTrainer::_average_reward_over_timestep_window

protected

Average rewards over each reporter timestep window instead of sampling one boundary value.

Definition at line 202 of file LibtorchDRLControlTrainer.h.

◆ _clip_param

const Real LibtorchDRLControlTrainer::_clip_param

protected

The clip parameter used while clamping the advantage value.

Definition at line 181 of file LibtorchDRLControlTrainer.h.

◆ _control_learning_rate

const Real LibtorchDRLControlTrainer::_control_learning_rate

protected

The learning rate for the optimization algorithm for the control.

Definition at line 175 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _control_nn

std::shared_ptr<Moose::LibtorchActorNeuralNet> LibtorchDRLControlTrainer::_control_nn

protected

Pointer to the control (or actor) neural net object.

Definition at line 229 of file LibtorchDRLControlTrainer.h.

Referenced by controlNeuralNet(), execute(), LibtorchDRLControlTrainer(), and trainController().

◆ _critic_learning_rate

const Real LibtorchDRLControlTrainer::_critic_learning_rate

protected

The learning rate for the optimization algorithm for the critic.

Definition at line 169 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _critic_nn

std::shared_ptr<Moose::LibtorchArtificialNeuralNet> LibtorchDRLControlTrainer::_critic_nn

protected

Pointer to the critic neural net object.

Definition at line 231 of file LibtorchDRLControlTrainer.h.

Referenced by execute(), LibtorchDRLControlTrainer(), and trainController().

◆ _critic_optimizer

std::unique_ptr<torch::optim::Adam> LibtorchDRLControlTrainer::_critic_optimizer

protected

Adam optimizer used to update the critic network.

Definition at line 241 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer(), and trainController().

◆ _decay_factor

const Real LibtorchDRLControlTrainer::_decay_factor

protected

Decaying factor that is used when calculating the return from the reward.

Definition at line 184 of file LibtorchDRLControlTrainer.h.

◆ _entropy_coeff

Real LibtorchDRLControlTrainer::_entropy_coeff

protected

Entropy bonus coefficient used in the PPO actor loss.

Definition at line 236 of file LibtorchDRLControlTrainer.h.

◆ _filename_base

const std::string LibtorchDRLControlTrainer::_filename_base

protected

Name of the pytorch output file.

This is used for loading and storing already existing data

Definition at line 190 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _highest_reward

Real LibtorchDRLControlTrainer::_highest_reward

protected

Best average episode reward seen so far while training.

Definition at line 234 of file LibtorchDRLControlTrainer.h.

Referenced by execute(), and trainController().

◆ _input_timesteps

const unsigned int LibtorchDRLControlTrainer::_input_timesteps

protected

Number of timesteps to fetch from the reporters to be the input of then eural nets.

Definition at line 155 of file LibtorchDRLControlTrainer.h.

Referenced by inputTimesteps(), and LibtorchDRLControlTrainer().

◆ _lambda_factor

const Real LibtorchDRLControlTrainer::_lambda_factor

protected

GAE lambda factor used while estimating advantages and returns.

Definition at line 186 of file LibtorchDRLControlTrainer.h.

◆ _log_probability_names

const std::vector<ReporterName> LibtorchDRLControlTrainer::_log_probability_names

protected

Log probability reporter names.

Definition at line 141 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _log_probability_value_pointers

std::vector<const std::vector<std::vector<Real> > *> LibtorchDRLControlTrainer::_log_probability_value_pointers

protected

Pointers to the current values of the control log probabilities We can have multiple control signals, multiple samples, multiple timesteps.

Definition at line 145 of file LibtorchDRLControlTrainer.h.

Referenced by collectTrajectoriesFromReporters(), and LibtorchDRLControlTrainer().

◆ _loss_print_frequency

const unsigned int LibtorchDRLControlTrainer::_loss_print_frequency

protected

The frequency the loss should be printed.

Definition at line 218 of file LibtorchDRLControlTrainer.h.

Referenced by trainController().

◆ _max_values

std::vector<Real> LibtorchDRLControlTrainer::_max_values

protected

Optional upper bounds for each control signal.

Definition at line 226 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _min_values

std::vector<Real> LibtorchDRLControlTrainer::_min_values

protected

Optional lower bounds for each control signal.

Definition at line 224 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _model_meta_data_name

const std::string RestartableModelInterface::_model_meta_data_name

privateinherited

The model meta data name.

This is used to store the restartable data within the RestartableDataMap.

Definition at line 64 of file RestartableModelInterface.h.

Referenced by RestartableModelInterface::modelMetaDataName(), and RestartableModelInterface::RestartableModelInterface().

◆ _model_object

const MooseObject& RestartableModelInterface::_model_object

privateinherited

Reference to the MooseObject that uses this interface.

Definition at line 60 of file RestartableModelInterface.h.

Referenced by RestartableModelInterface::getModelDataFileName(), RestartableModelInterface::hasExplicitModelData(), RestartableModelInterface::hasModelData(), and RestartableModelInterface::RestartableModelInterface().

◆ _model_restartable

PublicRestartable RestartableModelInterface::_model_restartable

privateinherited

Member for interfacing with the framework's restartable system.

We need this because we would like to have the capability to handle the model data separately from the other data members used for checkpointing.

Definition at line 71 of file RestartableModelInterface.h.

Referenced by RestartableModelInterface::declareModelData(), and RestartableModelInterface::getModelData().

◆ _num_control_neurons_per_layer

const std::vector<unsigned int> LibtorchDRLControlTrainer::_num_control_neurons_per_layer

protected

Number of neurons within the hidden layers in the control neural net.

Definition at line 172 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _num_critic_neurons_per_layer

const std::vector<unsigned int> LibtorchDRLControlTrainer::_num_critic_neurons_per_layer

protected

Number of neurons within the hidden layers in the critic neural net.

Definition at line 166 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _num_epochs

const unsigned int LibtorchDRLControlTrainer::_num_epochs

protected

Number of epochs for the training of the emulator.

Definition at line 163 of file LibtorchDRLControlTrainer.h.

Referenced by trainController().

◆ _num_inputs

unsigned int LibtorchDRLControlTrainer::_num_inputs

protected

Number of inputs for the control and critic neural nets.

Definition at line 158 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _num_outputs

unsigned int LibtorchDRLControlTrainer::_num_outputs

protected

Number of outputs for the control neural network.

Definition at line 160 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _ppo_loss

const LibtorchRLPPOLoss LibtorchDRLControlTrainer::_ppo_loss

private

PPO loss helper for the actor and critic updates.

Definition at line 270 of file LibtorchDRLControlTrainer.h.

Referenced by trainController().

◆ _read_from_file

const bool LibtorchDRLControlTrainer::_read_from_file

protected

Switch indicating if an already existing neural net should be read from a file or not.

This can be used to load existing torch files (from previous MOOSE runs for retraining and further manipulation)

Definition at line 195 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _reward_name

const ReporterName LibtorchDRLControlTrainer::_reward_name

protected

Reward reporter name.

Definition at line 148 of file LibtorchDRLControlTrainer.h.

◆ _reward_value_pointer

const std::vector<std::vector<Real> >* LibtorchDRLControlTrainer::_reward_value_pointer

protected

Pointer to the current values of the reward We can have multiple samples, multiple timesteps.

Definition at line 152 of file LibtorchDRLControlTrainer.h.

Referenced by collectTrajectoriesFromReporters().

◆ _sample_average_episode_reward

std::vector<Real> LibtorchDRLControlTrainer::_sample_average_episode_reward

protected

Per-sample mean episodic rewards over the latest update window.

Definition at line 210 of file LibtorchDRLControlTrainer.h.

Referenced by computeEpisodeRewardStatistics(), and sampleAverageEpsiodeRewards().

◆ _sample_std_episode_reward

std::vector<Real> LibtorchDRLControlTrainer::_sample_std_episode_reward

protected

Per-sample episodic reward standard deviations over the latest update window.

Definition at line 212 of file LibtorchDRLControlTrainer.h.

Referenced by computeEpisodeRewardStatistics(), and sampleStdEpsiodeRewards().

◆ _sampler

const LibtorchRLMiniBatchSampler LibtorchDRLControlTrainer::_sampler

private

Mini-batch sampler used to split flattened rollout data for PPO updates.

Definition at line 266 of file LibtorchDRLControlTrainer.h.

Referenced by trainController().

◆ _seed

const unsigned int LibtorchDRLControlTrainer::_seed

protected

Base seed for stochastic optimizers and policy sampling.

Definition at line 221 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer(), seed(), and trainController().

◆ _shift_outputs

const bool LibtorchDRLControlTrainer::_shift_outputs

protected

Currently, the controls are executed after the user objects at initial in moose.

So using a shift can realign the corresponding input-output values while reading the reporters

Definition at line 200 of file LibtorchDRLControlTrainer.h.

◆ _standardize_advantage

const bool LibtorchDRLControlTrainer::_standardize_advantage

protected

Switch to enable the standardization of the advantages.

Definition at line 215 of file LibtorchDRLControlTrainer.h.

Referenced by trainController().

◆ _state_names

const std::vector<ReporterName> LibtorchDRLControlTrainer::_state_names

protected

Observation reporter names.

Definition at line 118 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer(), and numberOfObservations().

◆ _state_scaling_factors

const std::vector<Real> LibtorchDRLControlTrainer::_state_scaling_factors

protected

Scaling constants for the observations.

Definition at line 128 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _state_shift_factors

const std::vector<Real> LibtorchDRLControlTrainer::_state_shift_factors

protected

Shifting constants for the observations.

Definition at line 125 of file LibtorchDRLControlTrainer.h.

Referenced by LibtorchDRLControlTrainer().

◆ _state_value_pointers

std::vector<const std::vector<std::vector<Real> > *> LibtorchDRLControlTrainer::_state_value_pointers

protected

Pointers to the current values of the observations We can have multiple observations, multiple samples, multiple timesteps.

Definition at line 122 of file LibtorchDRLControlTrainer.h.

Referenced by collectTrajectoriesFromReporters(), and LibtorchDRLControlTrainer().

◆ _std_episode_reward

Real LibtorchDRLControlTrainer::_std_episode_reward

protected

Storage for the current episode reward standard deviation.

Definition at line 207 of file LibtorchDRLControlTrainer.h.

Referenced by computeEpisodeRewardStatistics(), execute(), and stdEpisodeReward().

◆ _trajectory_assembler

const LibtorchRLTrajectoryAssembler LibtorchDRLControlTrainer::_trajectory_assembler

private

Builds rollout trajectories from the raw reporter sequences.

Definition at line 262 of file LibtorchDRLControlTrainer.h.

Referenced by collectTrajectoriesFromReporters().

◆ _trajectory_buffer

LibtorchRLTrajectoryBuffer LibtorchDRLControlTrainer::_trajectory_buffer

private

Accumulated on-policy rollout data waiting to be flattened and trained on.

Definition at line 264 of file LibtorchDRLControlTrainer.h.

Referenced by collectTrajectoriesFromReporters(), computeEpisodeRewardStatistics(), execute(), and resetData().

◆ _update_counter

unsigned int LibtorchDRLControlTrainer::_update_counter

private

Counter for number of transient simulations that have been run before updating the controller.

Definition at line 259 of file LibtorchDRLControlTrainer.h.

Referenced by execute(), and resetData().

◆ _update_frequency

const unsigned int LibtorchDRLControlTrainer::_update_frequency

protected

Number of transients to run and collect data from before updating the controller neural net.

Definition at line 178 of file LibtorchDRLControlTrainer.h.

Referenced by resetData().

◆ _value_estimator

const LibtorchRLValueEstimator LibtorchDRLControlTrainer::_value_estimator

private

Helper that builds value targets and advantages from collected trajectories.

Definition at line 268 of file LibtorchDRLControlTrainer.h.

Referenced by execute().

The documentation for this class was generated from the following files:

stochastic_tools/include/libtorch/surrogates/LibtorchDRLControlTrainer.h
stochastic_tools/src/libtorch/trainers/LibtorchDRLControlTrainer.C

Public Types

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Static Protected Attributes

Private Member Functions

Static Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ LibtorchDRLControlTrainer()

Member Function Documentation

◆ averageEpisodeReward()

◆ collectTrajectoriesFromReporters()

◆ computeEpisodeRewardStatistics()

◆ controlNeuralNet()

◆ declareModelData()

◆ execute()

◆ finalize()

◆ getModelData()

◆ getModelDataFileName()

◆ getReporterPointers()

◆ hasExplicitModelData()

◆ hasModelData()

◆ initialize()

◆ inputTimesteps()

◆ modelMetaDataName()

◆ numberOfControlSignals()

◆ numberOfObservations()

◆ resetData()

◆ sampleAverageEpsiodeRewards()

◆ sampleStdEpsiodeRewards()

◆ seed()

◆ stdEpisodeReward()

◆ threadJoin()

◆ timestepWindow()

◆ trainController()

◆ validParams()

Member Data Documentation

◆ _action_names

◆ _action_scaling_factors

◆ _action_value_pointers

◆ _actor_optimizer

◆ _average_episode_reward

◆ _average_reward_over_timestep_window

◆ _clip_param

◆ _control_learning_rate

◆ _control_nn

◆ _critic_learning_rate

◆ _critic_nn

◆ _critic_optimizer

◆ _decay_factor

◆ _entropy_coeff

◆ _filename_base

◆ _highest_reward

◆ _input_timesteps

◆ _lambda_factor

◆ _log_probability_names

◆ _log_probability_value_pointers

◆ _loss_print_frequency

◆ _max_values

◆ _min_values

◆ _model_meta_data_name

◆ _model_object

◆ _model_restartable

◆ _num_control_neurons_per_layer

◆ _num_critic_neurons_per_layer

◆ _num_epochs

◆ _num_inputs

◆ _num_outputs

◆ _ppo_loss

◆ _read_from_file

◆ _reward_name

◆ _reward_value_pointer

◆ _sample_average_episode_reward

◆ _sample_std_episode_reward