A time-dependent, neural-network-based controller which is associated with a Proximal Policy Optimization. More...

#include <LibtorchDRLControl.h>

Inheritance diagram for LibtorchDRLControl:

Public Types
typedef DataFileName	DataFileParameterType

Public Member Functions
	LibtorchDRLControl (const InputParameters &parameters)
	Construct using input parameters.

virtual void	initialSetup () override
	Restore any restartable controller state after base setup completes.

virtual void	execute () override
	Compute the actions and their corresponding logarithmic probabilities.

Real	getSignalLogProbability (const unsigned int signal_index) const
	Get the logarithmic probability of (signal_index)-th signal of the control neural net.

void	loadControlNeuralNet (const Moose::LibtorchActorNeuralNet &input_nn)
	Copy an actor network into this DRL controller.

virtual void	loadControlNeuralNet (const Moose::LibtorchArtificialNeuralNet &input_nn) override
	Reject loading a plain artificial neural network into this actor-based controller.

virtual void	loadControlNeuralNetFromFile () override
	Load the actor neural network from the configured checkpoint file.

void	setPolicySampleSeed (uint64_t seed)
	Reset the owned policy-sampling generator to a known seed.

unsigned int	numStepsInPeriod () const
	Return the number of time steps between policy evaluations.

unsigned int	inputTimesteps () const
	Return the number of observation time levels stacked into the actor input.

unsigned int	numberOfObservations () const
	Return the number of observations read at each time level.

Real	getSignal (const unsigned int signal_index) const

unsigned int	numberOfControlSignals () const

const Moose::LibtorchNeuralNetBase &	controlNeuralNet () const

bool	hasControlNeuralNet () const

std::vector< std::string > &	getDependencies ()

virtual bool	enabled () const

std::shared_ptr< MooseObject >	getSharedPtr ()

std::shared_ptr< const MooseObject >	getSharedPtr () const

bool	isKokkosObject () const

MooseApp &	getMooseApp () const

const std::string &	type () const

const std::string &	name () const

std::string	typeAndName () const

MooseObjectParameterName	uniqueParameterName (const std::string &parameter_name) const

MooseObjectName	uniqueName () const

const InputParameters &	parameters () const

const hit::Node *	getHitNode () const

bool	hasBase () const

const std::string &	getBase () const

const T &	getParam (const std::string &name) const

std::vector< std::pair< T1, T2 > >	getParam (const std::string &param1, const std::string &param2) const

const T *	queryParam (const std::string &name) const

const T &	getRenamedParam (const std::string &old_name, const std::string &new_name) const

T	getCheckedPointerParam (const std::string &name, const std::string &error_string="") const

bool	isParamValid (const std::string &name) const

bool	isParamSetByUser (const std::string &name) const

void	connectControllableParams (const std::string &parameter, const std::string &object_type, const std::string &object_name, const std::string &object_parameter) const

void	paramError (const std::string &param, Args... args) const

void	paramWarning (const std::string &param, Args... args) const

void	paramWarning (const std::string &param, Args... args) const

void	paramInfo (const std::string &param, Args... args) const

std::string	messagePrefix (const bool hit_prefix=true) const

std::string	errorPrefix (const std::string &) const

void	mooseError (Args &&... args) const

void	mooseDocumentedError (const std::string &repo_name, const unsigned int issue_num, Args &&... args) const

void	mooseErrorNonPrefixed (Args &&... args) const

void	mooseWarning (Args &&... args) const

void	mooseWarning (Args &&... args) const

void	mooseWarningNonPrefixed (Args &&... args) const

void	mooseWarningNonPrefixed (Args &&... args) const

void	mooseDeprecated (Args &&... args) const

void	mooseDeprecated (Args &&... args) const

void	mooseDeprecatedNoTrace (Args &&... args) const

void	mooseInfo (Args &&... args) const

void	callMooseError (std::string msg, const bool with_prefix, const hit::Node *node=nullptr, const bool show_trace=true) const

std::string	getDataFileName (const std::string &param) const

std::string	getDataFileNameByName (const std::string &relative_path) const

std::string	getDataFilePath (const std::string &relative_path) const

bool	isImplicit ()

Moose::StateArg	determineState () const

virtual void	timestepSetup ()

virtual void	jacobianSetup ()

virtual void	residualSetup ()

virtual void	subdomainSetup ()

virtual void	customSetup (const ExecFlagType &)

const ExecFlagEnum &	getExecuteOnEnum () const

const Function &	getFunction (const std::string &name) const

const Function &	getFunctionByName (const FunctionName &name) const

bool	hasFunction (const std::string &param_name) const

bool	hasFunctionByName (const FunctionName &name) const

Moose::Kokkos::Function	getKokkosFunction (const std::string &name) const

const T &	getKokkosFunction (const std::string &name) const

Moose::Kokkos::Function	getKokkosFunctionByName (const FunctionName &name) const

const T &	getKokkosFunctionByName (const FunctionName &name) const

bool	hasKokkosFunction (const std::string &param_name) const

bool	hasKokkosFunctionByName (const FunctionName &name) const

UserObjectName	getUserObjectName (const std::string &param_name) const

const T &	getUserObject (const std::string &param_name, bool is_dependency=true) const

const T &	getUserObjectByName (const UserObjectName &object_name, bool is_dependency=true) const

const UserObjectBase &	getUserObjectBase (const std::string &param_name, bool is_dependency=true) const

const UserObjectBase &	getUserObjectBaseByName (const UserObjectName &object_name, bool is_dependency=true) const

bool	hasUserObject (const std::string &param_name) const

bool	hasUserObject (const std::string &param_name) const

bool	hasUserObject (const std::string &param_name) const

bool	hasUserObject (const std::string &param_name) const

bool	hasUserObjectByName (const UserObjectName &object_name) const

bool	hasUserObjectByName (const UserObjectName &object_name) const

bool	hasUserObjectByName (const UserObjectName &object_name) const

bool	hasUserObjectByName (const UserObjectName &object_name) const

PerfGraph &	perfGraph ()

bool	isDefaultPostprocessorValue (const std::string &param_name, const unsigned int index=0) const

bool	hasPostprocessor (const std::string &param_name, const unsigned int index=0) const

bool	hasPostprocessorByName (const PostprocessorName &name) const

std::size_t	coupledPostprocessors (const std::string &param_name) const

const PostprocessorName &	getPostprocessorName (const std::string &param_name, const unsigned int index=0) const

const VectorPostprocessorValue &	getVectorPostprocessorValue (const std::string &param_name, const std::string &vector_name) const

const VectorPostprocessorValue &	getVectorPostprocessorValue (const std::string &param_name, const std::string &vector_name, bool needs_broadcast) const

const VectorPostprocessorValue &	getVectorPostprocessorValueByName (const VectorPostprocessorName &name, const std::string &vector_name) const

const VectorPostprocessorValue &	getVectorPostprocessorValueByName (const VectorPostprocessorName &name, const std::string &vector_name, bool needs_broadcast) const

const VectorPostprocessorValue &	getVectorPostprocessorValueOld (const std::string &param_name, const std::string &vector_name) const

const VectorPostprocessorValue &	getVectorPostprocessorValueOld (const std::string &param_name, const std::string &vector_name, bool needs_broadcast) const

const VectorPostprocessorValue &	getVectorPostprocessorValueOldByName (const VectorPostprocessorName &name, const std::string &vector_name) const

const VectorPostprocessorValue &	getVectorPostprocessorValueOldByName (const VectorPostprocessorName &name, const std::string &vector_name, bool needs_broadcast) const

const ScatterVectorPostprocessorValue &	getScatterVectorPostprocessorValue (const std::string &param_name, const std::string &vector_name) const

const ScatterVectorPostprocessorValue &	getScatterVectorPostprocessorValueByName (const VectorPostprocessorName &name, const std::string &vector_name) const

const ScatterVectorPostprocessorValue &	getScatterVectorPostprocessorValueOld (const std::string &param_name, const std::string &vector_name) const

const ScatterVectorPostprocessorValue &	getScatterVectorPostprocessorValueOldByName (const VectorPostprocessorName &name, const std::string &vector_name) const

bool	hasVectorPostprocessor (const std::string &param_name, const std::string &vector_name) const

bool	hasVectorPostprocessor (const std::string &param_name) const

bool	hasVectorPostprocessorByName (const VectorPostprocessorName &name, const std::string &vector_name) const

bool	hasVectorPostprocessorByName (const VectorPostprocessorName &name) const

const VectorPostprocessorName &	getVectorPostprocessorName (const std::string &param_name) const

const PostprocessorValue &	getPostprocessorValue (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValue (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValueOld (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValueOld (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValueOlder (const std::string &param_name, const unsigned int index=0) const

const PostprocessorValue &	getPostprocessorValueOlder (const std::string &param_name, const unsigned int index=0) const

virtual const PostprocessorValue &	getPostprocessorValueByName (const PostprocessorName &name) const

virtual const PostprocessorValue &	getPostprocessorValueByName (const PostprocessorName &name) const

const PostprocessorValue &	getPostprocessorValueOldByName (const PostprocessorName &name) const

const PostprocessorValue &	getPostprocessorValueOldByName (const PostprocessorName &name) const

const PostprocessorValue &	getPostprocessorValueOlderByName (const PostprocessorName &name) const

const PostprocessorValue &	getPostprocessorValueOlderByName (const PostprocessorName &name) const

bool	isVectorPostprocessorDistributed (const std::string &param_name) const

bool	isVectorPostprocessorDistributed (const std::string &param_name) const

bool	isVectorPostprocessorDistributedByName (const VectorPostprocessorName &name) const

bool	isVectorPostprocessorDistributedByName (const VectorPostprocessorName &name) const

const Parallel::Communicator &	comm () const

processor_id_type	n_processors () const

processor_id_type	processor_id () const

Static Public Member Functions
static InputParameters	validParams ()
	Return the input parameters supported by this control.

static void	callMooseError (MooseApp const app, const InputParameters &params, std::string msg, const bool with_prefix, const hit::Node node, const bool show_trace=true)

static MultiMooseEnum	getExecuteOptions ()

Public Attributes
	usingCombinedWarningSolutionWarnings

const ConsoleStream	_console

Static Public Attributes
static const std::string	type_param

static const std::string	name_param

static const std::string	unique_name_param

static const std::string	app_param

static const std::string	moose_base_param

static const std::string	kokkos_object_param

Protected Member Functions
void	applyControlSignals ()
	Apply the current smoothed control signals, including optional offsets.

Real	computeControlOffset (unsigned int control_i) const
	Return the offset to add to a control signal at the current time.

void	conditionalParameterError (const std::string &param_name, const std::vector< std::string > &conditional_param, bool should_be_defined=true)

void	updateCurrentObservation ()

torch::Tensor	prepareInputTensor ()

bool	hasControllableParameterByName (const std::string &name) const

void	flagInvalidSolutionInternal (const InvalidSolutionID invalid_solution_id) const

InvalidSolutionID	registerInvalidSolutionInternal (const std::string &message, const bool warning) const

PerfID	registerTimedSection (const std::string &section_name, const unsigned int level) const

PerfID	registerTimedSection (const std::string &section_name, const unsigned int level, const std::string &live_message, const bool print_dots=true) const

std::string	timedSectionName (const std::string &section_name) const

virtual void	addUserObjectDependencyHelper (const UserObjectBase &) const

T &	declareRestartableData (const std::string &data_name, Args &&... args)

ManagedValue< T >	declareManagedRestartableDataWithContext (const std::string &data_name, void *context, Args &&... args)

const T &	getRestartableData (const std::string &data_name) const

T &	declareRestartableDataWithContext (const std::string &data_name, void *context, Args &&... args)

T &	declareRecoverableData (const std::string &data_name, Args &&... args)

T &	declareRestartableDataWithObjectName (const std::string &data_name, const std::string &object_name, Args &&... args)

T &	declareRestartableDataWithObjectNameWithContext (const std::string &data_name, const std::string &object_name, void *context, Args &&... args)

std::string	restartableName (const std::string &data_name) const

virtual void	addPostprocessorDependencyHelper (const PostprocessorName &) const

virtual void	addVectorPostprocessorDependencyHelper (const VectorPostprocessorName &) const

const ReporterContextBase &	getReporterContextBaseByName (const ReporterName &reporter_name) const

const ReporterName &	getReporterName (const std::string &param_name) const

virtual void	addReporterDependencyHelper (const ReporterName &)

ControllableParameter	getControllableParameter (const std::string &param_name)

ControllableParameter	getControllableParameter (const std::string &param_name)

ControllableParameter	getControllableParameterByName (const std::string &param_name)

ControllableParameter	getControllableParameterByName (const std::string &tag, const std::string &object_name, const std::string &param_name)

ControllableParameter	getControllableParameterByName (const MooseObjectName &object_name, const std::string &param_name)

ControllableParameter	getControllableParameterByName (const MooseObjectParameterName &param_name)

ControllableParameter	getControllableParameterByName (const std::string &param_name)

ControllableParameter	getControllableParameterByName (const std::string &tag, const std::string &object_name, const std::string &param_name)

ControllableParameter	getControllableParameterByName (const MooseObjectName &object_name, const std::string &param_name)

ControllableParameter	getControllableParameterByName (const MooseObjectParameterName &param_name)

T	getControllableValue (const std::string &name, bool warn_when_values_differ=true)

T	getControllableValue (const std::string &name, bool warn_when_values_differ=true)

T	getControllableValueByName (const std::string &name, bool warn_when_values_differ=true)

T	getControllableValueByName (const std::string &object_name, const std::string &param_name, bool warn_when_values_differ=true)

T	getControllableValueByName (const MooseObjectName &object_name, const std::string &param_name, bool warn_when_values_differ=true)

T	getControllableValueByName (const std::string &tag, const std::string &object_name, const std::string &param_name, bool warn_when_values_differ=true)

T	getControllableValueByName (const MooseObjectParameterName &desired, bool warn_when_values_differ=true)

T	getControllableValueByName (const std::string &name, bool warn_when_values_differ=true)

T	getControllableValueByName (const std::string &object_name, const std::string &param_name, bool warn_when_values_differ=true)

T	getControllableValueByName (const MooseObjectName &object_name, const std::string &param_name, bool warn_when_values_differ=true)

T	getControllableValueByName (const std::string &tag, const std::string &object_name, const std::string &param_name, bool warn_when_values_differ=true)

T	getControllableValueByName (const MooseObjectParameterName &desired, bool warn_when_values_differ=true)

void	setControllableValue (const std::string &name, const T &value)

void	setControllableValue (const std::string &name, const T &value)

void	setControllableValueByName (const std::string &name, const T &value)

void	setControllableValueByName (const std::string &object_name, const std::string &param_name, const T &value)

void	setControllableValueByName (const MooseObjectName &object_name, const std::string &param_name, const T &value)

void	setControllableValueByName (const std::string &tag, const std::string &object_name, const std::string &param_name, const T &value)

void	setControllableValueByName (const MooseObjectParameterName &name, const T &value)

void	setControllableValueByName (const std::string &name, const T &value)

void	setControllableValueByName (const std::string &object_name, const std::string &param_name, const T &value)

void	setControllableValueByName (const MooseObjectName &object_name, const std::string &param_name, const T &value)

void	setControllableValueByName (const std::string &tag, const std::string &object_name, const std::string &param_name, const T &value)

void	setControllableValueByName (const MooseObjectParameterName &name, const T &value)

const T &	getReporterValue (const std::string &param_name, const std::size_t time_index=0)

const T &	getReporterValue (const std::string &param_name, ReporterMode mode, const std::size_t time_index=0)

const T &	getReporterValue (const std::string &param_name, const std::size_t time_index=0)

const T &	getReporterValue (const std::string &param_name, ReporterMode mode, const std::size_t time_index=0)

const T &	getReporterValueByName (const ReporterName &reporter_name, const std::size_t time_index=0)

const T &	getReporterValueByName (const ReporterName &reporter_name, ReporterMode mode, const std::size_t time_index=0)

const T &	getReporterValueByName (const ReporterName &reporter_name, const std::size_t time_index=0)

const T &	getReporterValueByName (const ReporterName &reporter_name, ReporterMode mode, const std::size_t time_index=0)

bool	hasReporterValue (const std::string &param_name) const

bool	hasReporterValue (const std::string &param_name) const

bool	hasReporterValue (const std::string &param_name) const

bool	hasReporterValue (const std::string &param_name) const

bool	hasReporterValueByName (const ReporterName &reporter_name) const

bool	hasReporterValueByName (const ReporterName &reporter_name) const

bool	hasReporterValueByName (const ReporterName &reporter_name) const

bool	hasReporterValueByName (const ReporterName &reporter_name) const

Protected Attributes
std::vector< Real > &	_current_control_signal_log_probabilities
	The log probability of control signals from the last evaluation of the controller.

std::vector< Real > &	_previous_control_signal
	The smoothed control signal from the previous execution, saved for restart/recover.

std::vector< Real > &	_current_smoothed_signal
	The current smoothed control signal applied to the controllable parameters.

std::vector< Real >	_control_offsets
	Constant offsets added to each control signal before it is applied.

bool	_has_control_offsets
	Whether any offset data was provided by the user.

std::shared_ptr< Moose::LibtorchActorNeuralNet >	_actor_nn
	Actor network used when the controller operates as a stochastic policy.

at::Generator	_policy_generator
	Owned libtorch CPU generator used for policy sampling.

std::vector< std::uint8_t > &	_policy_generator_state
	Restartable serialized state for the owned policy-sampling generator.

const unsigned int	_num_steps_in_period
	Number of time steps between policy evaluations.

const Real	_control_smoothing_factor
	Exponential smoothing factor in (0, 1] applied to the control signal at each time step.

const bool	_stochastic
	Whether to sample actions stochastically instead of using the deterministic actor output.

std::vector< Real >	_current_observation

std::vector< std::vector< Real > > &	_old_observations

const std::vector< std::string > &	_control_names

std::vector< Real > &	_current_control_signals

const std::vector< PostprocessorName > &	_observation_names

std::vector< const Real * >	_observation_values

const unsigned int	_input_timesteps

const std::vector< Real >	_observation_shift_factors

const std::vector< Real >	_observation_scaling_factors

const std::vector< Real >	_action_scaling_factors

const LibtorchObservationHistoryHelper	_observation_history

std::shared_ptr< Moose::LibtorchNeuralNetBase >	_nn

FEProblemBase &	_fe_problem

std::vector< std::string >	_depends_on

const bool &	_enabled

MooseApp &	_app

Factory &	_factory

ActionFactory &	_action_factory

const std::string &	_type

const std::string &	_name

const InputParameters &	_pars

MooseApp &	_pg_moose_app

const std::string	_prefix

const InputParameters &	_ti_params

FEProblemBase &	_ti_feproblem

bool	_is_implicit

Real &	_t

const Real &	_t_old

int &	_t_step

Real &	_dt

Real &	_dt_old

bool	_is_transient

const ExecFlagEnum &	_execute_enum

const ExecFlagType &	_current_execute_flag

MooseApp &	_restartable_app

const std::string	_restartable_system_name

const THREAD_ID	_restartable_tid

const bool	_restartable_read_only

const Parallel::Communicator &	_communicator

Private Member Functions
bool	shouldEvaluatePolicy () const
	Report whether the policy is evaluated at the current time step.

void	restorePolicyGeneratorState ()
	Restore the owned libtorch generator state from restartable storage.

void	savePolicyGeneratorState ()
	Mirror the owned libtorch generator state into restartable storage.

const Moose::FunctionBase *	getKokkosFunctionByNameHelper (const FunctionName &name) const

const UserObjectBase &	getUserObjectFromFEProblem (const UserObjectName &object_name, const THREAD_ID tid=0) const

const T &	castUserObject (const UserObjectBase &uo_base, const std::string &param_name="") const

void	mooseObjectError (const std::string &param_name, std::stringstream &oss) const

const std::string &	userObjectType (const UserObjectBase &uo) const

const std::string &	userObjectName (const UserObjectBase &uo) const

RestartableDataValue &	registerRestartableDataOnApp (std::unique_ptr< RestartableDataValue > data, THREAD_ID tid) const

void	registerRestartableNameWithFilterOnApp (const std::string &name, Moose::RESTARTABLE_FILTER filter)

RestartableData< T > &	declareRestartableDataHelper (const std::string &data_name, void *context, Args &&... args) const

const PostprocessorName &	getPostprocessorNameInternal (const std::string &param_name, const unsigned int index, const bool allow_default_value=true) const

bool	isDefaultPostprocessorValueByName (const PostprocessorName &name) const

PostprocessorValue	getDefaultPostprocessorValueByName (const PostprocessorName &name) const

void	checkParam (const std::string &param_name, const unsigned int index=std::numeric_limits< unsigned int >::max()) const

bool	postprocessorsAdded () const

const VectorPostprocessorValue &	getVectorPostprocessorByNameHelper (const VectorPostprocessorName &name, const std::string &vector_name, bool broadcast, std::size_t t_index) const

const VectorPostprocessorContext< VectorPostprocessorValue > &	getVectorPostprocessorContextByNameHelper (const VectorPostprocessorName &name, const std::string &vector_name) const

bool	vectorPostprocessorsAdded () const

bool	reportersAdded () const

void	possiblyCheckHasReporter (const ReporterName &reporter_name, const std::string &param_name="") const

const PostprocessorValue &	getPostprocessorValueInternal (const std::string &param_name, unsigned int index, std::size_t t_index) const

const PostprocessorValue &	getPostprocessorValueInternal (const std::string &param_name, unsigned int index, std::size_t t_index) const

const PostprocessorValue &	getPostprocessorValueByNameInternal (const PostprocessorName &name, std::size_t t_index) const

const PostprocessorValue &	getPostprocessorValueByNameInternal (const PostprocessorName &name, std::size_t t_index) const

void	possiblyCheckHasVectorPostprocessor (const std::string &param_name, const std::string &vector_name) const

void	possiblyCheckHasVectorPostprocessor (const std::string &param_name, const std::string &vector_name) const

void	possiblyCheckHasVectorPostprocessorByName (const VectorPostprocessorName &name, const std::string &vector_name) const

void	possiblyCheckHasVectorPostprocessorByName (const VectorPostprocessorName &name, const std::string &vector_name) const

Static Private Member Functions
static const hit::Node *	getHitNode (const InputParameters &params)

static std::string	messagePrefix (const InputParameters &params, const bool hit_prefix)

Private Attributes
InputParameterWarehouse &	_input_parameter_warehouse

const ParallelParamObject &	_parent

const MooseBase &	_si_moose_base

const FEProblemBase *	_si_problem

const std::string	_ti_name

ExecFlagEnum	_empty_execute_enum

const MooseObject &	_fni_object

const InputParameters &	_fni_params

FEProblemBase &	_fni_feproblem

const THREAD_ID	_fni_tid

const MooseObject &	_uoi_moose_object

const FEProblemBase &	_uoi_feproblem

const THREAD_ID	_uoi_tid

const RestartableDataMapName	_metaname

std::string	_restartable_name

const MooseObject &	_ppi_moose_object

const InputParameters &	_ppi_params

const FEProblemBase &	_ppi_feproblem

std::map< PostprocessorName, std::unique_ptr< PostprocessorValue > >	_default_values

const bool	_broadcast_by_default

const MooseObject &	_vpi_moose_object

const FEProblemBase &	_vpi_feproblem

const THREAD_ID	_vpi_tid

const InputParameters &	_ri_params

FEProblemBase &	_ri_fe_problem_base

const ReporterData &	_ri_reporter_data

const MooseObject &	_ri_moose_object

Detailed Description

A time-dependent, neural-network-based controller which is associated with a Proximal Policy Optimization.

We use this neural net for the training of a controller. The additional functionality in this controller is the addition of the variability (using an assumed Gaussian distribution) to avoid overfitting. This control is supposed to be used in conjunction with LibtorchDRLControlTrainer.

Definition at line 26 of file LibtorchDRLControl.h.

Constructor & Destructor Documentation

◆ LibtorchDRLControl()

LibtorchDRLControl::LibtorchDRLControl ( const InputParameters & parameters )

Construct using input parameters.

Definition at line 71 of file LibtorchDRLControl.C.

  : LibtorchNeuralNetControl(parameters),
    _current_control_signal_log_probabilities(declareRestartableData<std::vector<Real>>(
        "current_control_signal_log_probabilities", std::vector<Real>(_control_names.size(), 0.0))),
    _previous_control_signal(declareRestartableData<std::vector<Real>>(
        "previous_control_signal", std::vector<Real>(_control_names.size(), 0.0))),
    _current_smoothed_signal(declareRestartableData<std::vector<Real>>(
        "current_smoothed_signal", std::vector<Real>(_control_names.size(), 0.0))),
    _control_offsets(isParamSetByUser("control_offsets")
                         ? getParam<std::vector<Real>>("control_offsets")
                         : std::vector<Real>(_control_names.size(), 0.0)),
    _has_control_offsets(isParamSetByUser("control_offsets")),
    _policy_generator(Moose::makeLibtorchCPUGenerator()),
    _policy_generator_state(declareRestartableData<std::vector<std::uint8_t>>(
        "policy_generator_state", std::vector<std::uint8_t>())),
    _num_steps_in_period(getParam<unsigned int>("num_steps_in_period")),
    _control_smoothing_factor(getParam<Real>("control_smoothing_factor")),
    _stochastic(getParam<bool>("stochastic"))
{
  const auto & execute_on = getParam<ExecFlagEnum>("execute_on");
  if (execute_on.size() != 1 || !execute_on.contains(EXEC_TIMESTEP_BEGIN))
    paramError("execute_on", "LibtorchDRLControl only supports 'TIMESTEP_BEGIN' for 'execute_on'.");
 
  if (isParamValid("seed"))
    setPolicySampleSeed(getParam<unsigned int>("seed"));
 
  if (_control_offsets.size() != _control_names.size())
    paramError("control_offsets",
               "The number of control offsets must match the number of controlled parameters.");
 
  savePolicyGeneratorState();
}

Member Function Documentation

◆ applyControlSignals()

void LibtorchDRLControl::applyControlSignals ( )

protected

Apply the current smoothed control signals, including optional offsets.

Definition at line 218 of file LibtorchDRLControl.C.

{
  for (const auto control_i : index_range(_control_names))
  {
    const Real applied_signal =
        computeControlOffset(control_i) + _current_smoothed_signal[control_i];
    setControllableValueByName<Real>(_control_names[control_i], applied_signal);
  }
}

Referenced by execute().

◆ computeControlOffset()

Real LibtorchDRLControl::computeControlOffset ( unsigned int control_i ) const

protected

Return the offset to add to a control signal at the current time.

Parameters

control_i Index of the control signal.

Definition at line 212 of file LibtorchDRLControl.C.

{
  return _control_offsets[control_i];
}

Referenced by applyControlSignals().

◆ execute()

void LibtorchDRLControl::execute ( )

overridevirtual

Compute the actions and their corresponding logarithmic probabilities.

Reimplemented from LibtorchNeuralNetControl.

Definition at line 153 of file LibtorchDRLControl.C.

{
  if (!_actor_nn)
  {
    mooseAssert(!_nn, "LibtorchDRLControl should not store a non-actor controller network.");
    if (_has_control_offsets)
      applyControlSignals();
    return;
  }
 
  // Transfers also execute the control once at step 0, before the transient starts.
  mooseAssert(_current_execute_flag == EXEC_TIMESTEP_BEGIN || _t_step == 0,
              "LibtorchDRLControl should only execute on TIMESTEP_BEGIN or from a transfer "
              "before the first time step.");
 
  const unsigned int n_controls = _control_names.size();
  const bool first_control_execution = _old_observations.empty();
 
  // Fill a vector with the current observation values.
  updateCurrentObservation();
 
  // Seed the observation history with the initial observation when the control first runs.
  if (first_control_execution)
    _observation_history.initializeHistory(_current_observation, _old_observations);
 
  if (shouldEvaluatePolicy())
  {
    torch::Tensor input_tensor = prepareInputTensor();
    torch::Tensor action = _actor_nn->evaluate(input_tensor, _stochastic, _policy_generator);
    savePolicyGeneratorState();
 
    if (_stochastic)
    {
      torch::Tensor log_probability = _actor_nn->logProbability(action);
      _current_control_signal_log_probabilities = {log_probability.data_ptr<Real>(),
                                                   log_probability.data_ptr<Real>() +
                                                       log_probability.size(1)};
    }
    else
      _current_control_signal_log_probabilities.assign(n_controls, 0.0);
 
    _current_control_signals = {action.data_ptr<Real>(), action.data_ptr<Real>() + action.size(1)};
 
    // The history only advances on evaluations, so each lag spans one period. This matches the
    // trainer, which stacks observations sampled every timestep_window steps.
    _observation_history.advanceHistory(_current_observation, _old_observations);
  }
 
  _previous_control_signal = _current_smoothed_signal;
 
  for (const auto i : index_range(_current_smoothed_signal))
    _current_smoothed_signal[i] =
        _previous_control_signal[i] +
        _control_smoothing_factor * (_current_control_signals[i] - _previous_control_signal[i]);
 
  applyControlSignals();
}

Referenced by SamplerDRLControlTransfer::execute(), and SamplerDRLControlTransfer::executeToMultiapp().

◆ getSignalLogProbability()

Real LibtorchDRLControl::getSignalLogProbability ( const unsigned int signal_index ) const

Get the logarithmic probability of (signal_index)-th signal of the control neural net.

Parameters

signal_index The index of the signal

Returns: The logarithmic probability of the (signal_index)-th signal

Definition at line 295 of file LibtorchDRLControl.C.

{
  mooseAssert(signal_index < _control_names.size(),
              "The index of the requested control signal is not in the [0," +
                  std::to_string(_control_names.size()) + ") range!");
  return _current_control_signal_log_probabilities[signal_index];
}

Referenced by LibtorchDRLLogProbabilityPostprocessor::getValue().

◆ initialSetup()

void LibtorchDRLControl::initialSetup ( )

overridevirtual

Restore any restartable controller state after base setup completes.

Reimplemented from LibtorchNeuralNetControl.

Definition at line 105 of file LibtorchDRLControl.C.

{
  LibtorchNeuralNetControl::initialSetup();
  restorePolicyGeneratorState();
  savePolicyGeneratorState();
}

◆ inputTimesteps()

unsigned int LibtorchDRLControl::inputTimesteps ( ) const

inline

Return the number of observation time levels stacked into the actor input.

Definition at line 87 of file LibtorchDRLControl.h.

87{ return _input_timesteps; }

LibtorchNeuralNetControl::_input_timesteps

const unsigned int _input_timesteps

Referenced by SamplerDRLControlTransfer::getDRLControl().

◆ loadControlNeuralNet() [1/2]

void LibtorchDRLControl::loadControlNeuralNet ( const Moose::LibtorchActorNeuralNet & input_nn )

Copy an actor network into this DRL controller.

Parameters

input_nn Actor network to copy into the controller.

Definition at line 229 of file LibtorchDRLControl.C.

{
  _actor_nn = std::make_shared<Moose::LibtorchActorNeuralNet>(input_nn);
  _nn = _actor_nn;
}

Referenced by SamplerDRLControlTransfer::execute(), SamplerDRLControlTransfer::executeToMultiapp(), and loadControlNeuralNet().

◆ loadControlNeuralNet() [2/2]

void LibtorchDRLControl::loadControlNeuralNet ( const Moose::LibtorchArtificialNeuralNet & input_nn )

overridevirtual

Reject loading a plain artificial neural network into this actor-based controller.

Parameters

input_nn Artificial neural network passed through the base-class interface.

Reimplemented from LibtorchNeuralNetControl.

Definition at line 236 of file LibtorchDRLControl.C.

{
  const auto * check = dynamic_cast<const Moose::LibtorchActorNeuralNet *>(&input_nn);
  if (!check)
    mooseError("This needs to be a LibtorchActorNeuralNet!");
  loadControlNeuralNet(*check);
}

◆ loadControlNeuralNetFromFile()

void LibtorchDRLControl::loadControlNeuralNetFromFile ( )

overridevirtual

Load the actor neural network from the configured checkpoint file.

Reimplemented from LibtorchNeuralNetControl.

Definition at line 113 of file LibtorchDRLControl.C.

{
  const auto & filename = getParam<std::string>("filename");
  unsigned int num_inputs = _observation_names.size() * _input_timesteps;
  unsigned int num_outputs = _control_names.size();
  std::vector<unsigned int> num_neurons_per_layer =
      getParam<std::vector<unsigned int>>("num_neurons_per_layer");
  std::vector<std::string> activation_functions =
      isParamSetByUser("activation_function")
          ? getParam<std::vector<std::string>>("activation_function")
          : std::vector<std::string>({"relu"});
 
  const std::vector<Real> & minimum_values = getParam<std::vector<Real>>("min_control_value");
  const std::vector<Real> & maximum_values = getParam<std::vector<Real>>("max_control_value");
  const auto input_shift_factors =
      _observation_history.expandObservationFactors(_observation_shift_factors);
  const auto input_scaling_factors =
      _observation_history.expandObservationFactors(_observation_scaling_factors);
 
  _actor_nn =
      std::make_shared<Moose::LibtorchActorNeuralNet>(filename,
                                                      num_inputs,
                                                      num_outputs,
                                                      num_neurons_per_layer,
                                                      activation_functions,
                                                      minimum_values,
                                                      maximum_values,
                                                      torch::kCPU,
                                                      torch::kDouble,
                                                      true,
                                                      input_shift_factors,
                                                      input_scaling_factors,
                                                      _action_scaling_factors,
                                                      getParam<bool>("state_independent_std"));
 
  Moose::loadLibtorchActorNeuralNetState(*_actor_nn, filename);
  _nn = _actor_nn;
}

◆ numberOfObservations()

unsigned int LibtorchDRLControl::numberOfObservations ( ) const

inline

Return the number of observations read at each time level.

Definition at line 92 of file LibtorchDRLControl.h.

92{ return _observation_names.size(); }

Referenced by SamplerDRLControlTransfer::getDRLControl().

◆ numStepsInPeriod()

unsigned int LibtorchDRLControl::numStepsInPeriod ( ) const

inline

Return the number of time steps between policy evaluations.

Definition at line 82 of file LibtorchDRLControl.h.

82{ return _num_steps_in_period; }

Referenced by SamplerDRLControlTransfer::getDRLControl().

◆ restorePolicyGeneratorState()

void LibtorchDRLControl::restorePolicyGeneratorState ( )

private

Restore the owned libtorch generator state from restartable storage.

Definition at line 265 of file LibtorchDRLControl.C.

{
  if (!_stochastic || _policy_generator_state.empty())
    return;
 
  auto state_tensor =
      torch::from_blob(_policy_generator_state.data(),
                       {static_cast<long>(_policy_generator_state.size())},
                       torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU))
          .clone();
  std::lock_guard<std::mutex> lock(_policy_generator.mutex());
  _policy_generator.set_state(state_tensor);
}

Referenced by initialSetup().

◆ savePolicyGeneratorState()

void LibtorchDRLControl::savePolicyGeneratorState ( )

private

Mirror the owned libtorch generator state into restartable storage.

Definition at line 280 of file LibtorchDRLControl.C.

{
  if (!_stochastic)
  {
    _policy_generator_state.clear();
    return;
  }
 
  std::lock_guard<std::mutex> lock(_policy_generator.mutex());
  const auto state_tensor = _policy_generator.get_state().contiguous();
  const auto * data = state_tensor.data_ptr<std::uint8_t>();
  _policy_generator_state.assign(data, data + static_cast<std::size_t>(state_tensor.numel()));
}

Referenced by execute(), initialSetup(), LibtorchDRLControl(), and setPolicySampleSeed().

◆ setPolicySampleSeed()

void LibtorchDRLControl::setPolicySampleSeed ( uint64_t seed )

Reset the owned policy-sampling generator to a known seed.

Parameters

seed	Seed assigned to the policy-sampling generator.

Definition at line 245 of file LibtorchDRLControl.C.

{
  {
    std::lock_guard<std::mutex> lock(_policy_generator.mutex());
    _policy_generator.set_current_seed(seed);
  }
  savePolicyGeneratorState();
}

Referenced by SamplerDRLControlTransfer::executeToMultiapp(), and LibtorchDRLControl().

◆ shouldEvaluatePolicy()

bool LibtorchDRLControl::shouldEvaluatePolicy ( ) const

private

Report whether the policy is evaluated at the current time step.

Definition at line 255 of file LibtorchDRLControl.C.

{
  // Evaluate at step 0 and on the first step of every period: 1, P + 1, 2P + 1, ... Tying the
  // schedule to the time step keeps it aligned with the trainer's reporter downsampling.
  if (_t_step <= 0)
    return true;
  return (static_cast<unsigned int>(_t_step) - 1) % _num_steps_in_period == 0;
}

Referenced by execute().

◆ validParams()

InputParameters LibtorchDRLControl::validParams ( )

static

Return the input parameters supported by this control.

Definition at line 22 of file LibtorchDRLControl.C.

{
  InputParameters params = LibtorchNeuralNetControl::validParams();
  params.addClassDescription(
      "Sets the value of multiple 'Real' input parameters and postprocessors based on a Deep "
      "Reinforcement Learning (DRL) neural network trained using a PPO algorithm.");
  params.set<ExecFlagEnum>("execute_on") = EXEC_TIMESTEP_BEGIN;
  params.suppressParameter<bool>("torch_script_format");
 
  params.addParam<unsigned int>("seed", "Seed for the random number generator.");
 
  params.addRangeCheckedParam<unsigned int>(
      "num_steps_in_period",
      1,
      "1<=num_steps_in_period",
      "Number of time steps between policy evaluations. The observation history advances once "
      "per period, so the trainer's 'timestep_window' should match this value.");
  params.addRangeCheckedParam<Real>(
      "control_smoothing_factor",
      1.0,
      "control_smoothing_factor>0 & control_smoothing_factor<=1",
      "Exponential smoothing factor for the applied control signal: the fraction of the change "
      "from the previously applied signal toward the new policy action that is applied at each "
      "time step. A value of 1 applies the policy action directly.");
 
  params.addParam<std::vector<Real>>(
      "control_offsets",
      {},
      "Optional constant offsets added to each policy signal before setting the controlled "
      "parameter. This can be used to train a policy on deviations from a nominal signal.");
 
  params.addParam<bool>(
      "stochastic",
      true,
      "If true, sample from the policy distribution; otherwise use the deterministic action.");
 
  params.addParam<std::vector<Real>>(
      "min_control_value", {}, "Optional lower bounds for each control signal.");
  params.addParam<std::vector<Real>>(
      "max_control_value", {}, "Optional upper bounds for each control signal.");
  params.addParam<bool>(
      "state_independent_std",
      true,
      "If true, interpret the unbounded Gaussian actor as learning one log-std per action "
      "dimension. If false, use a state-dependent std head.");
 
  return params;
}

Member Data Documentation

◆ _actor_nn

std::shared_ptr<Moose::LibtorchActorNeuralNet> LibtorchDRLControl::_actor_nn

protected

Actor network used when the controller operates as a stochastic policy.

Definition at line 120 of file LibtorchDRLControl.h.

Referenced by execute(), loadControlNeuralNet(), and loadControlNeuralNetFromFile().

◆ _control_offsets

std::vector<Real> LibtorchDRLControl::_control_offsets

protected

Constant offsets added to each control signal before it is applied.

Definition at line 115 of file LibtorchDRLControl.h.

Referenced by computeControlOffset(), and LibtorchDRLControl().

◆ _control_smoothing_factor

const Real LibtorchDRLControl::_control_smoothing_factor

protected

Exponential smoothing factor in (0, 1] applied to the control signal at each time step.

Definition at line 129 of file LibtorchDRLControl.h.

Referenced by execute().

◆ _current_control_signal_log_probabilities

std::vector<Real>& LibtorchDRLControl::_current_control_signal_log_probabilities

protected

The log probability of control signals from the last evaluation of the controller.

Definition at line 107 of file LibtorchDRLControl.h.

Referenced by execute(), and getSignalLogProbability().

◆ _current_smoothed_signal

std::vector<Real>& LibtorchDRLControl::_current_smoothed_signal

protected

The current smoothed control signal applied to the controllable parameters.

Definition at line 112 of file LibtorchDRLControl.h.

Referenced by applyControlSignals(), and execute().

◆ _has_control_offsets

bool LibtorchDRLControl::_has_control_offsets

protected

Whether any offset data was provided by the user.

Definition at line 117 of file LibtorchDRLControl.h.

Referenced by execute().

◆ _num_steps_in_period

const unsigned int LibtorchDRLControl::_num_steps_in_period

protected

Number of time steps between policy evaluations.

Definition at line 127 of file LibtorchDRLControl.h.

Referenced by numStepsInPeriod(), and shouldEvaluatePolicy().

◆ _policy_generator

at::Generator LibtorchDRLControl::_policy_generator

protected

Owned libtorch CPU generator used for policy sampling.

Definition at line 122 of file LibtorchDRLControl.h.

Referenced by execute(), restorePolicyGeneratorState(), savePolicyGeneratorState(), and setPolicySampleSeed().

◆ _policy_generator_state

std::vector<std::uint8_t>& LibtorchDRLControl::_policy_generator_state

protected

Restartable serialized state for the owned policy-sampling generator.

Definition at line 124 of file LibtorchDRLControl.h.

Referenced by restorePolicyGeneratorState(), and savePolicyGeneratorState().

◆ _previous_control_signal

std::vector<Real>& LibtorchDRLControl::_previous_control_signal

protected

The smoothed control signal from the previous execution, saved for restart/recover.

Definition at line 110 of file LibtorchDRLControl.h.

Referenced by execute().

◆ _stochastic

const bool LibtorchDRLControl::_stochastic

protected

Whether to sample actions stochastically instead of using the deterministic actor output.

Definition at line 131 of file LibtorchDRLControl.h.

Referenced by execute(), restorePolicyGeneratorState(), and savePolicyGeneratorState().

The documentation for this class was generated from the following files:

stochastic_tools/include/libtorch/controls/LibtorchDRLControl.h
stochastic_tools/src/libtorch/controls/LibtorchDRLControl.C

Public Types

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Protected Member Functions

Protected Attributes

Private Member Functions

Static Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ LibtorchDRLControl()

Member Function Documentation

◆ applyControlSignals()

◆ computeControlOffset()

◆ execute()

◆ getSignalLogProbability()

◆ initialSetup()

◆ inputTimesteps()

◆ loadControlNeuralNet() [1/2]

◆ loadControlNeuralNet() [2/2]

◆ loadControlNeuralNetFromFile()

◆ numberOfObservations()

◆ numStepsInPeriod()

◆ restorePolicyGeneratorState()

◆ savePolicyGeneratorState()

◆ setPolicySampleSeed()

◆ shouldEvaluatePolicy()

◆ validParams()

Member Data Documentation

◆ _actor_nn

◆ _control_offsets

◆ _control_smoothing_factor

◆ _current_control_signal_log_probabilities

◆ _current_smoothed_signal

◆ _has_control_offsets

◆ _num_steps_in_period

◆ _policy_generator

◆ _policy_generator_state

◆ _previous_control_signal

◆ _stochastic