LibtorchRLTrajectoryBuffer Class Reference

On-policy trajectory storage for fixed-horizon RL training. More...

#include <LibtorchRLTrajectoryBuffer.h>

Classes
struct	RewardStatistics
	Reward statistics over the stored trajectories. More...
struct	TensorBatch
	Flattened tensor data assembled from the stored trajectories. More...
struct	Trajectory
	Vector data for one on-policy trajectory. More...

Public Member Functions
void	addTrajectory (Trajectory trajectory)
	Append one trajectory to the on-policy buffer.
void	clear ()
	Clear every stored trajectory.
bool	empty () const
	Return true if the buffer contains no trajectories.
std::size_t	numTrajectories () const
	Return the number of stored trajectories.
std::size_t	numTransitions () const
	Count the total number of transitions stored across every trajectory.
std::vector< Trajectory > &	trajectories ()
	Return mutable access to the stored trajectories.
const std::vector< Trajectory > &	trajectories () const
	Return read-only access to the stored trajectories.
TensorBatch	flatten () const
	Flatten every stored trajectory into one tensor batch.
RewardStatistics	rewardStatistics () const
	Compute reward statistics over the stored trajectories.

Static Private Member Functions
static void	validateTrajectory (const Trajectory &trajectory)
	Validate a trajectory before it is stored.

Private Attributes
std::vector< Trajectory >	_trajectories
	Stored on-policy trajectories.

Detailed Description

On-policy trajectory storage for fixed-horizon RL training.

Definition at line 24 of file LibtorchRLTrajectoryBuffer.h.

Member Function Documentation

addTrajectory()

void LibtorchRLTrajectoryBuffer::addTrajectory

(

Trajectory

trajectory

)

Append one trajectory to the on-policy buffer.

Parameters

trajectory

Trajectory to store.

Definition at line 67 of file LibtorchRLTrajectoryBuffer.C.

{
  validateTrajectory(trajectory);
  _trajectories.push_back(std::move(trajectory));
}

Referenced by LibtorchDRLControlTrainer::collectTrajectoriesFromReporters().

clear()

void LibtorchRLTrajectoryBuffer::clear

(

)

Clear every stored trajectory.

Definition at line 74 of file LibtorchRLTrajectoryBuffer.C.

{
  _trajectories.clear();
}

Referenced by LibtorchDRLControlTrainer::resetData().

empty()

bool LibtorchRLTrajectoryBuffer::empty ( ) const

inline

Return true if the buffer contains no trajectories.

Definition at line 105 of file LibtorchRLTrajectoryBuffer.h.

{ return _trajectories.empty(); }

flatten()

LibtorchRLTrajectoryBuffer::TensorBatch LibtorchRLTrajectoryBuffer::flatten

(

)

const

Flatten every stored trajectory into one tensor batch.

Returns

Tensor batch ready for mini-batch sampling.

Definition at line 89 of file LibtorchRLTrajectoryBuffer.C.

{
  TensorBatch batch;
 
  if (_trajectories.empty())
    return batch;
 
  std::vector<std::vector<Real>> observations;
  std::vector<std::vector<Real>> next_observations;
  std::vector<std::vector<Real>> actions;
  std::vector<std::vector<Real>> log_probabilities;
  std::vector<Real> rewards;
  std::vector<Real> value_targets;
  std::vector<Real> advantages;
 
  observations.reserve(numTransitions());
  next_observations.reserve(numTransitions());
  actions.reserve(numTransitions());
  log_probabilities.reserve(numTransitions());
  rewards.reserve(numTransitions());
  value_targets.reserve(numTransitions());
  advantages.reserve(numTransitions());
 
  for (const auto & trajectory : _trajectories)
  {
    if (!trajectory.value_targets.empty() &&
        trajectory.value_targets.size() != trajectory.rewards.size())
      mooseError("Value targets must match the reward length of the trajectory.");
    if (!trajectory.advantages.empty() && trajectory.advantages.size() != trajectory.rewards.size())
      mooseError("Advantages must match the reward length of the trajectory.");
 
    observations.insert(
        observations.end(), trajectory.observations.begin(), trajectory.observations.end());
    next_observations.insert(next_observations.end(),
                             trajectory.next_observations.begin(),
                             trajectory.next_observations.end());
    actions.insert(actions.end(), trajectory.actions.begin(), trajectory.actions.end());
    log_probabilities.insert(log_probabilities.end(),
                             trajectory.log_probabilities.begin(),
                             trajectory.log_probabilities.end());
    rewards.insert(rewards.end(), trajectory.rewards.begin(), trajectory.rewards.end());
    value_targets.insert(
        value_targets.end(), trajectory.value_targets.begin(), trajectory.value_targets.end());
    advantages.insert(advantages.end(), trajectory.advantages.begin(), trajectory.advantages.end());
  }
 
  batch.observations = bufferMatrixToTensor(observations);
  batch.next_observations = bufferMatrixToTensor(next_observations);
  batch.actions = bufferMatrixToTensor(actions);
  batch.log_probabilities = bufferMatrixToTensor(log_probabilities);
  batch.rewards = bufferVectorToColumnTensor(rewards);
  batch.value_targets = bufferVectorToColumnTensor(value_targets);
  batch.advantages = bufferVectorToColumnTensor(advantages);
 
  return batch;
}

Referenced by LibtorchDRLControlTrainer::execute().

numTrajectories()

std::size_t LibtorchRLTrajectoryBuffer::numTrajectories ( ) const

inline

Return the number of stored trajectories.

Definition at line 110 of file LibtorchRLTrajectoryBuffer.h.

{ return _trajectories.size(); }

numTransitions()

std::size_t LibtorchRLTrajectoryBuffer::numTransitions

(

)

const

Count the total number of transitions stored across every trajectory.

Returns

Total transition count.

Definition at line 80 of file LibtorchRLTrajectoryBuffer.C.

{
  std::size_t transitions = 0;
  for (const auto & trajectory : _trajectories)
    transitions += trajectory.rewards.size();
  return transitions;
}

Referenced by flatten(), and rewardStatistics().

rewardStatistics()

LibtorchRLTrajectoryBuffer::RewardStatistics LibtorchRLTrajectoryBuffer::rewardStatistics

(

)

const

Compute reward statistics over the stored trajectories.

Returns

Overall and per-trajectory reward means and standard deviations.

Definition at line 147 of file LibtorchRLTrajectoryBuffer.C.

{
  RewardStatistics statistics;
 
  const auto num_transitions = numTransitions();
  if (!num_transitions)
    return statistics;
 
  const auto squared_deviation_sum = [](const std::vector<Real> & values, const Real mean)
  {
    return std::transform_reduce(values.begin(),
                                 values.end(),
                                 0.0,
                                 std::plus<>(),
                                 [mean](const Real value)
                                 { return (value - mean) * (value - mean); });
  };
 
  Real reward_sum = 0.0;
  for (const auto & trajectory : _trajectories)
  {
    const auto & rewards = trajectory.rewards;
    if (rewards.empty())
      continue;
 
    const Real trajectory_sum = std::accumulate(rewards.begin(), rewards.end(), 0.0);
    const Real trajectory_mean = trajectory_sum / rewards.size();
    statistics.trajectory_means.push_back(trajectory_mean);
    statistics.trajectory_stds.push_back(
        std::sqrt(squared_deviation_sum(rewards, trajectory_mean) / rewards.size()));
    reward_sum += trajectory_sum;
  }
  statistics.mean = reward_sum / num_transitions;
 
  // The deviations are taken about the overall mean so the spread between trajectories counts.
  Real overall_squared_deviation_sum = 0.0;
  for (const auto & trajectory : _trajectories)
    overall_squared_deviation_sum += squared_deviation_sum(trajectory.rewards, statistics.mean);
  statistics.std = std::sqrt(overall_squared_deviation_sum / num_transitions);
 
  return statistics;
}

Referenced by LibtorchDRLControlTrainer::computeEpisodeRewardStatistics().

trajectories() [1/2]

std::vector< Trajectory > & LibtorchRLTrajectoryBuffer::trajectories ( )

inline

Return mutable access to the stored trajectories.

Definition at line 121 of file LibtorchRLTrajectoryBuffer.h.

{ return _trajectories; }

Referenced by LibtorchRLValueEstimator::computeValueTargets().

trajectories() [2/2]

const std::vector< Trajectory > & LibtorchRLTrajectoryBuffer::trajectories ( ) const

inline

Return read-only access to the stored trajectories.

Definition at line 126 of file LibtorchRLTrajectoryBuffer.h.

{ return _trajectories; }

validateTrajectory()

void LibtorchRLTrajectoryBuffer::validateTrajectory ( const Trajectory &  trajectory )

staticprivate

Validate a trajectory before it is stored.

Parameters

trajectory

Trajectory to validate.

Definition at line 191 of file LibtorchRLTrajectoryBuffer.C.

{
  const auto num_steps = trajectory.rewards.size();
 
  if (trajectory.observations.size() != num_steps)
    mooseError("RL trajectory observations must match the reward sequence length.");
 
  if (trajectory.next_observations.size() != num_steps)
    mooseError("RL trajectory next observations must match the reward sequence length.");
 
  if (trajectory.actions.size() != num_steps)
    mooseError("RL trajectory actions must match the reward sequence length.");
 
  if (trajectory.log_probabilities.size() != num_steps)
    mooseError("RL trajectory log probabilities must match the reward sequence length.");
 
  if (!trajectory.terminals.empty() && trajectory.terminals.size() != num_steps)
    mooseError("RL trajectory terminal flags must match the reward sequence length.");
 
  if (!trajectory.value_targets.empty() && trajectory.value_targets.size() != num_steps)
    mooseError("RL trajectory value targets must match the reward sequence length.");
 
  if (!trajectory.advantages.empty() && trajectory.advantages.size() != num_steps)
    mooseError("RL trajectory advantages must match the reward sequence length.");
}

Referenced by addTrajectory().

Member Data Documentation

_trajectories

std::vector<Trajectory> LibtorchRLTrajectoryBuffer::_trajectories

private

Stored on-policy trajectories.

Definition at line 148 of file LibtorchRLTrajectoryBuffer.h.

Referenced by addTrajectory(), clear(), empty(), flatten(), numTrajectories(), numTransitions(), rewardStatistics(), trajectories(), and trajectories().

---

The documentation for this class was generated from the following files:

• stochastic_tools/include/libtorch/utils/LibtorchRLTrajectoryBuffer.h
• stochastic_tools/src/libtorch/utils/LibtorchRLTrajectoryBuffer.C