Reusable continuous-action distribution interface for actor policies. More...

#include <LibtorchActionDistribution.h>

Inheritance diagram for Moose::LibtorchActionDistribution:

Public Member Functions
	LibtorchActionDistribution (const std::string &name, unsigned int num_inputs, unsigned int num_outputs, torch::DeviceType device_type=torch::kCPU, torch::ScalarType scalar_type=torch::kDouble, const std::vector< Real > &output_scaling_factors={})
	Build an action-distribution module for an actor network.

virtual void	initialize (c10::optional< at::Generator > generator=c10::nullopt)=0
	Initialize the trainable distribution parameters.

virtual void	reset (const torch::Tensor &input)=0
	Refresh cached distribution parameters from the latest actor features.

virtual torch::Tensor	sample (c10::optional< at::Generator > generator=c10::nullopt) const =0
	Draw a stochastic action sample in physical units.

virtual torch::Tensor	deterministicAction () const =0
	Return the deterministic action used for evaluation.

virtual torch::Tensor	logProbability (const torch::Tensor &action) const =0
	Evaluate the log-probability of an action under the current distribution.

virtual torch::Tensor	entropy () const =0
	Compute the entropy of the current distribution.

virtual bool	isBounded () const =0
	Tell callers whether the distribution enforces explicit action bounds.

void	synchronizeScalingFactorsFromBuffer ()
	Sync cached scaling metadata from the registered buffers after loading state.

Protected Member Functions
torch::Tensor	prepareFeatures (const torch::Tensor &input) const
	Convert actor features to the configured device and scalar type.

torch::Tensor	prepareAction (const torch::Tensor &action) const
	Convert actions to the configured device and scalar type.

const torch::Tensor &	actionScaleTensor () const
	Return the registered tensor that stores per-action scaling factors.

Protected Attributes
const std::string	_name
	Module name used for registration and serialization.

const unsigned int	_num_inputs
	Number of actor features feeding this distribution.

const unsigned int	_num_outputs
	Number of action dimensions produced by this distribution.

const torch::DeviceType	_device_type
	Torch device used by this distribution.

const torch::ScalarType	_data_type
	Torch scalar type used by this distribution.

std::vector< Real >	_output_scaling_factors
	Cached per-action scaling factors applied in physical units.

torch::Tensor	_action_scale_tensor
	Registered libtorch buffer holding the per-action scaling factors.

Detailed Description

Reusable continuous-action distribution interface for actor policies.

Definition at line 27 of file LibtorchActionDistribution.h.

Constructor & Destructor Documentation

◆ LibtorchActionDistribution()

Moose::LibtorchActionDistribution::LibtorchActionDistribution	(	const std::string &	name,
		unsigned int	num_inputs,
		unsigned int	num_outputs,
		torch::DeviceType	device_type = `torch::kCPU`,
		torch::ScalarType	scalar_type = `torch::kDouble`,
		const std::vector< Real > &	output_scaling_factors = `{}`
	)

Build an action-distribution module for an actor network.

Parameters

name	Module name used for registration and serialization.
num_inputs	Number of actor features feeding the distribution.
num_outputs	Number of action dimensions produced by the distribution.
device_type	Torch device used by the module.
scalar_type	Torch scalar type used by the module.
output_scaling_factors	Optional per-action scaling applied in physical units.

Definition at line 52 of file LibtorchActionDistribution.C.

  : _name(name),
    _num_inputs(num_inputs),
    _num_outputs(num_outputs),
    _device_type(device_type),
    _data_type(data_type),
    _output_scaling_factors(setActionScalingFactors(output_scaling_factors, num_outputs))
{
  auto action_scale = _output_scaling_factors;
  LibtorchUtils::vectorToTensor(action_scale, _action_scale_tensor);
  _action_scale_tensor = register_buffer(
      "action_scale", _action_scale_tensor.transpose(0, 1).to(_data_type).to(_device_type));
}

Member Function Documentation

◆ actionScaleTensor()

const torch::Tensor & Moose::LibtorchActionDistribution::actionScaleTensor ( ) const

inlineprotected

Return the registered tensor that stores per-action scaling factors.

Definition at line 104 of file LibtorchActionDistribution.h.

104{ return _action_scale_tensor; }

Referenced by Moose::LibtorchGaussianActionDistribution::deterministicAction(), Moose::LibtorchBetaActionDistribution::deterministicAction(), Moose::LibtorchGaussianActionDistribution::entropy(), Moose::LibtorchBetaActionDistribution::entropy(), Moose::LibtorchGaussianActionDistribution::logProbability(), Moose::LibtorchBetaActionDistribution::logProbability(), Moose::LibtorchGaussianActionDistribution::sample(), and Moose::LibtorchBetaActionDistribution::sample().

◆ deterministicAction()

virtual torch::Tensor Moose::LibtorchActionDistribution::deterministicAction ( ) const

pure virtual

Return the deterministic action used for evaluation.

Implemented in Moose::LibtorchGaussianActionDistribution, and Moose::LibtorchBetaActionDistribution.

◆ entropy()

virtual torch::Tensor Moose::LibtorchActionDistribution::entropy ( ) const

pure virtual

Compute the entropy of the current distribution.

Implemented in Moose::LibtorchGaussianActionDistribution, and Moose::LibtorchBetaActionDistribution.

◆ initialize()

virtual void Moose::LibtorchActionDistribution::initialize ( c10::optional< at::Generator > generator = c10::nullopt )

pure virtual

Initialize the trainable distribution parameters.

Parameters

generator Optional random-number generator used for reproducible initialization.

Implemented in Moose::LibtorchGaussianActionDistribution, and Moose::LibtorchBetaActionDistribution.

◆ isBounded()

virtual bool Moose::LibtorchActionDistribution::isBounded ( ) const

pure virtual

Tell callers whether the distribution enforces explicit action bounds.

Implemented in Moose::LibtorchGaussianActionDistribution, and Moose::LibtorchBetaActionDistribution.

◆ logProbability()

virtual torch::Tensor Moose::LibtorchActionDistribution::logProbability ( const torch::Tensor & action ) const

pure virtual

Evaluate the log-probability of an action under the current distribution.

Parameters

action Action tensor in physical units.

Implemented in Moose::LibtorchGaussianActionDistribution, and Moose::LibtorchBetaActionDistribution.

◆ prepareAction()

torch::Tensor Moose::LibtorchActionDistribution::prepareAction ( const torch::Tensor & action ) const

protected

Convert actions to the configured device and scalar type.

Parameters

action Raw action tensor.

Definition at line 92 of file LibtorchActionDistribution.C.

{
  auto scaled_action = action;
  if (_data_type != scaled_action.scalar_type())
    scaled_action = scaled_action.to(_data_type);
  if (_device_type != scaled_action.device().type())
    scaled_action = scaled_action.to(_device_type);
  return scaled_action;
}

Referenced by Moose::LibtorchGaussianActionDistribution::logProbability(), and Moose::LibtorchBetaActionDistribution::logProbability().

◆ prepareFeatures()

torch::Tensor Moose::LibtorchActionDistribution::prepareFeatures ( const torch::Tensor & input ) const

protected

Convert actor features to the configured device and scalar type.

Parameters

input Raw actor feature tensor.

Definition at line 81 of file LibtorchActionDistribution.C.

{
  auto features = input;
  if (_data_type != features.scalar_type())
    features = features.to(_data_type);
  if (_device_type != features.device().type())
    features = features.to(_device_type);
  return features;
}

Referenced by Moose::LibtorchGaussianActionDistribution::reset(), and Moose::LibtorchBetaActionDistribution::reset().

◆ reset()

virtual void Moose::LibtorchActionDistribution::reset ( const torch::Tensor & input )

pure virtual

Refresh cached distribution parameters from the latest actor features.

Parameters

input Feature tensor coming from the actor body.

Implemented in Moose::LibtorchGaussianActionDistribution, and Moose::LibtorchBetaActionDistribution.

◆ sample()

virtual torch::Tensor Moose::LibtorchActionDistribution::sample ( c10::optional< at::Generator > generator = c10::nullopt ) const

pure virtual

Draw a stochastic action sample in physical units.

Parameters

generator Optional random-number generator used for sampling.

Implemented in Moose::LibtorchGaussianActionDistribution, and Moose::LibtorchBetaActionDistribution.

◆ synchronizeScalingFactorsFromBuffer()

void Moose::LibtorchActionDistribution::synchronizeScalingFactorsFromBuffer ( )

Sync cached scaling metadata from the registered buffers after loading state.

Definition at line 73 of file LibtorchActionDistribution.C.

{
  auto action_scale =
      _action_scale_tensor.detach().reshape({-1}).to(torch::kCPU).to(torch::kDouble);
  LibtorchUtils::tensorToVector(action_scale, _output_scaling_factors);
}