Stochastic Tools Requirements Traceability Matrix

[monte_carlo]
  type = CSVDiff
  input = monte_carlo.i
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'Monte Carlo;'
[]

[lhs]
  type = CSVDiff
  input = lhs.i
  csvdiff = 'lhs_csv_study_results_0001.csv'
  detail = 'latin hypercube;'
[]

[cartesian_product]
  type = CSVDiff
  input = cartesian_product.i
  csvdiff = 'cartesian_product_csv_study_results_0001.csv'
  detail = 'cartesian product;'
[]

[csv]
  type = CSVDiff
  input = csv.i
  cli_args = 'Outputs/file_base=monte_carlo_csv'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'CSV;'
[]

[csv_indices]
  type = CSVDiff
  input = csv.i
  cli_args = 'ParameterStudy/csv_column_indices=\'0 1\' Outputs/file_base=monte_carlo_csv'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'CSV when specifiying column indices;'
[]

[csv_names]
  type = CSVDiff
  input = csv.i
  cli_args = 'ParameterStudy/csv_column_names=\'study_sampler_0 study_sampler_1\' Outputs/file_base=monte_carlo_csv'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'CSV when specifiying column names;'
[]

[input_matrix]
  type = CSVDiff
  input = input_matrix.i
  cli_args = 'Outputs/file_base=monte_carlo_csv'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  detail = 'input matrix;'
[]

[monte_carlo_num_samples]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=monte-carlo ParameterStudy/distributions=uniform'
  expect_err = 'The num_samples parameter is required to build the requested sampling type\.'
  detail = '\'num_samples\' parameter for Monte Carlo sampling;'
[]

[monte_carlo_distributions]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=monte-carlo ParameterStudy/num_samples=1'
  expect_err = 'The distributions parameter is required to build the requested sampling type\.'
  detail = '\'distributions\' parameter for Monte Carlo sampling;'
[]

[monte_carlo_unused]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=monte-carlo
                  ParameterStudy/num_samples=1
                  ParameterStudy/distributions=uniform
                  ParameterStudy/linear_space_items=\'0 1 1\''
  expect_err = 'The following parameters are unused for the selected sampling type:'
  detail = '\'linear_space_items\' parameter with Monte Carlo sampling;'
[]

[lhs_num_samples]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=lhs ParameterStudy/distributions=uniform'
  expect_err = 'The num_samples parameter is required to build the requested sampling type\.'
  detail = '\'num_samples\' parameter for latin hypercube sampling;'
[]

[lhs_distributions]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=lhs ParameterStudy/num_samples=1'
  expect_err = 'The distributions parameter is required to build the requested sampling type\.'
  detail = '\'distributions\' parameter for latin hypercube sampling;'
[]

[cartesian_product]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=cartesian-product'
  expect_err = 'The linear_space_items parameter is required to build the requested sampling type\.'
  detail = '\'linear_space_items\' parameter for cartesian product sampling;'
[]

[csv]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=csv'
  expect_err = 'The csv_samples_file parameter is required to build the requested sampling type\.'
  detail = '\'samples_file\' parameter for CSV sampling;'
[]

[csv_both]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=csv
                  ParameterStudy/csv_samples_file=gold/monte_carlo_csv_study_samples_0001.csv
                  ParameterStudy/csv_column_indices=1
                  ParameterStudy/csv_column_names=foo'
  expect_err = '\'csv_column_indices\' and \'csv_column_names\' cannot both be set\.'
  detail = 'both \'csv_column_indices\' and \'csv_column_names\' parameters are specified for CSV sampling;'
[]

[input_matrix]
  type = RunException
  input = sampling_error.i
  cli_args = 'ParameterStudy/sampling_type=input-matrix'
  expect_err = 'The input_matrix parameter is required to build the requested sampling type\.'
  detail = '\'matrix\' parameter for input matrix sampling;'
[]

[distributions]
  type = CSVDiff
  input = distributions.i
  csvdiff = distributions_csv_study_results_0001.csv
  requirement = 'The system shall be able to setup a Monte Carlo parameter study with uniform, normal, Weibull, lognormal, and truncated normal distributions.'
[]

[missing]
  type = RunException
  input = distributions_error.i
  cli_args = 'ParameterStudy/distributions=\'uniform uniform\'
                  ParameterStudy/uniform_upper_bound=\'1 1\''
  expect_err = 'The uniform_lower_bound parameter is required to build the listed distributions\.'
  detail = 'missing;'
[]

[inconsistent]
  type = RunException
  input = distributions_error.i
  cli_args = 'ParameterStudy/distributions=\'uniform uniform\'
                  ParameterStudy/uniform_lower_bound=\'0\'
                  ParameterStudy/uniform_upper_bound=\'1 1\''
  expect_err = 'The number of entries in uniform_lower_bound does not match the number of required entries \(\d+\) to build the listed distributions\.'
  detail = 'inconsitent size;'
[]

[unused]
  type = RunException
  input = distributions_error.i
  cli_args = 'ParameterStudy/distributions=\'uniform uniform\'
                  ParameterStudy/uniform_lower_bound=\'0 0\'
                  ParameterStudy/uniform_upper_bound=\'1 1\'
                  ParameterStudy/normal_mean=1'
  expect_err = 'The following parameters are unused for the listed distributions:'
  detail = 'unused;'
[]

[true]
  type = JSONDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/output_type=json
                  ParameterStudy/show_study_objects=true
                  --color off'
  jsondiff = monte_carlo_json.json
  rel_err = 1e-4
  expect_out = 'Type:\s+StatisticsReporter'
  detail = 'compute statistics;'
[]

[false]
  type = RunApp
  input = monte_carlo.i
  cli_args = 'ParameterStudy/compute_statistics=false
                  ParameterStudy/show_study_objects=true
                  --color off'
  absent_out = 'Type:\s+StatisticsReporter'
  detail = 'not compute statistics;'
[]

[unused]
  type = RunException
  input = monte_carlo.i
  cli_args = 'ParameterStudy/compute_statistics=false
                  ParameterStudy/statistics=mean
                  ParameterStudy/ci_levels=0.1
                  ParameterStudy/ci_replicates=10'
  expect_err = 'The following parameters are unused since statistics are not being computed: ci_levels, ci_replicates, statistics'
  detail = 'error if there are unused statistics parameters;'
[]

[none]
  type = RunApp
  input = monte_carlo.i
  cli_args = 'ParameterStudy/output_type=none
                  ParameterStudy/compute_statistics=false
                  ParameterStudy/show_study_objects=true
                  --color off'
  expect_out = 'outputs : none'
  absent_out = 'Base Type:\s+Output'
  detail = 'no output;'
[]

[csv]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/output_type=csv
                  ParameterStudy/compute_statistics=false
                  ParameterStudy/show_study_objects=true
                  --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'Type:\s+CSV'
  absent_out = 'Type:\s+JSON'
  detail = 'CSV output;'
[]

[json]
  type = JSONDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/output_type=json
                  ParameterStudy/show_study_objects=true
                  --color off'
  jsondiff = monte_carlo_json.json
  rel_err = 1e-4
  expect_out = 'Type:\s+JSON'
  absent_out = 'Type:\s+CSV'
  detail = 'JSON output;'
[]

[normal]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=normal --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : normal'
  detail = 'one sub-app per sample;'
[]

[batch_reset]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=batch-reset --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-reset'
  detail = 'batches with one sub-app per processor while re-initializing;'
[]

[batch_restore]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=batch-restore --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore'
  detail = 'batches one sub-app per processor while restoring;'
[]

[batch_keep_solution]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=batch-keep-solution --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'keep_solution_during_restore : 1\n.*\n\s*mode : batch-restore'
  detail = 'batches one sub-app per processor while restoring and keeping solution;'
[]

[batch_no_restore]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true ParameterStudy/multiapp_mode=batch-no-restore --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore\n\s*no_restore : 1'
  detail = 'batches one sub-app per processor without restoring;'
[]

[parsed]
  type = CSVDiff
  input = distributions.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = distributions_csv_study_results_0001.csv
  expect_out = 'mode : batch-reset'
  detail = 'parameters are parsed;'
[]

[not_controllable]
  type = CSVDiff
  input = monte_carlo_not_controllable.i
  cli_args = 'Outputs/file_base=monte_carlo_csv ParameterStudy/sampler_column_names=\'BCs_left_value BCs_right_value\' ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-reset'
  detail = 'parameters are not controllable;'
[]

[transient]
  type = CSVDiff
  input = monte_carlo_transient.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_transient_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore'
  detail = 'parameters are controllable in a transient problem;'
[]

[pseudo_transient]
  type = CSVDiff
  input = monte_carlo_pseudo_transient.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_pseudo_transient_csv_study_results_0001.csv'
  expect_out = 'keep_solution_during_restore : 1\n.*\n\s*mode : batch-restore'
  detail = 'parameters are controllable in a pseudo-transient problem;'
[]

[steady]
  type = CSVDiff
  input = monte_carlo.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore\n\s*no_restore : 1'
  detail = 'parameters are controllable in a steady-state problem;'
[]

[eigen]
  type = CSVDiff
  input = monte_carlo_eigen.i
  cli_args = 'ParameterStudy/show_study_objects=true --color off'
  csvdiff = 'monte_carlo_eigen_csv_study_results_0001.csv'
  expect_out = 'mode : batch-restore\n\s*no_restore : 1'
  detail = 'parameters are controllable in a eigenvalue problem;'
[]

[create]
  type = CheckFiles
  input = surrogate.i
  check_files = 'surrogate_trainer_poly_regression.rd/data'
  requirement = 'The system shall be able to create a surrogate that can be read in by file and use in aux kernels.'
[]

[var]
  type = RunApp
  input = surrogate_aux.i
  prereq = create
  detail = 'standard field variables and'
[]

[array_var]
  type = RunApp
  input = surrogate_array_aux.i
  prereq = create
  detail = 'array field variables.'
[]

[parameter_unspecified]
  type = RunException
  input = surrogate_aux.i
  cli_args = 'AuxKernels/u_aux/parameters=\'funx 3.14 var pp foo\''
  prereq = create
  expect_err = '\'foo\' is not listed in \'scalar_parameters\'\, \'coupled_variables\'\, or \'coupled_array_variables\'.'
  detail = 'a parameter is undefined,'
[]

[scalar_missing]
  type = RunException
  input = surrogate_aux.i
  cli_args = 'AuxKernels/u_aux/parameters=\'funx 3.14 var pp foo\' AuxKernels/u_aux/scalar_parameters=\'funx funy pp foo\''
  prereq = create
  expect_err = '\'foo\' is not a postprocessor or a function.'
  detail = 'a scalar parameter is not found, or'
[]

[wrong_array_size]
  type = RunException
  input = surrogate_array_aux.i
  cli_args = 'AuxVariables/u/components=3'
  prereq = create
  expect_err = 'The number of components in \'\w+\' \(\d+\) does not match the number of components in \'\w+\' \(\d+\).'
  detail = 'a coupled array variable does not match the number of components.'
[]

[scalar_surrogate_aux]
  type = 'CSVDiff'
  input = 'surrogate_scalar_aux.i'
  csvdiff = 'surrogate_scalar_aux_out.csv'
  abs_zero = 1e-06
  group = 'simple'
  recover = false
  requirement = 'The system shall be able to evaluate a surrogate at given scalar variable values and insert the result into a scalar variable'
[]

[scalar_surrogate_aux_pp_arg]
  type = 'CSVDiff'
  input = 'surrogate_scalar_aux.i'
  csvdiff = 'scalar_surrogate_aux_pp_arg.csv'
  abs_zero = 1e-06
  group = 'simple'
  recover = false
  cli_args = 'Outputs/file_base=scalar_surrogate_aux_pp_arg AuxScalarKernels/scalar_eval/parameters="Tpp 5"'
  requirement = 'The system shall be able to evaluate a surrogate at given postprocessor values and insert the result into a scalar variable'
[]

[read-parameters]
  type = CSVDiff
  input = libtorch_drl_control.i
  csvdiff = parameter_read.csv
  cli_args = "Outputs/file_base='parameter_read'"
  # A retried time step executes the control again, drawing another policy sample.
  restep = false
  requirement = "The system shall be able to read a neural network from a parameter file and use it to control a transient process."
  capabilities = 'libtorch'
  recover = false # Addressed in #32802
[]

[read-parameters-period]
  type = CSVDiff
  input = libtorch_drl_control.i
  csvdiff = parameter_read_period.csv
  cli_args = "Controls/src_control/num_steps_in_period=2 Outputs/file_base='parameter_read_period'"
  # A retried time step executes the control again, drawing another policy sample.
  restep = false
  requirement = "The system shall be able to hold a DRL control signal over a multi-step control period and advance the observation history once per period."
  capabilities = 'libtorch'
[]

[read-parameters-bounded]
  type = CSVDiff
  input = libtorch_drl_control.i
  csvdiff = parameter_read_bounded.csv
  # mynet_control_bounded.net is an untrained Beta actor with the architecture of src_control,
  # saved by the LibtorchDRLControlTrainer in transfers/sampler_drl_control_transfer/trainer.i.
  # With action_scaling_factors = 100, the bounds limit left_flux to [-100, 100], narrower than
  # the range the unbounded controller reaches in parameter_read.csv.
  cli_args = "Controls/src_control/filename='mynet_control_bounded.net' Controls/src_control/min_control_value=-1 Controls/src_control/max_control_value=1 Outputs/file_base='parameter_read_bounded'"
  # A retried time step executes the control again, drawing another policy sample.
  restep = false
  requirement = "The system shall be able to control a transient process with a bounded neural network read from a parameter file, sampling each action within the bounds."
  capabilities = 'libtorch'
[]

[read-parameters-smoothed]
  type = CSVDiff
  input = libtorch_drl_control.i
  csvdiff = parameter_read_smoothed.csv
  # The control drives a ConstantPostprocessor instead of the BC so that the smoothed value
  # applied_flux is output next to the policy action left_flux.
  cli_args = "Controls/src_control/control_smoothing_factor=0.5 Controls/src_control/parameters='Postprocessors/applied_flux/value' Postprocessors/applied_flux/type=ConstantPostprocessor Postprocessors/applied_flux/execute_on='INITIAL TIMESTEP_END' Outputs/file_base='parameter_read_smoothed'"
  # A retried time step executes the control again, drawing another policy sample.
  restep = false
  requirement = "The system shall be able to exponentially smooth the control signal applied by a DRL controller."
  capabilities = 'libtorch'
[]

[without-nn]
  type = CSVDiff
  input = libtorch_drl_control.i
  csvdiff = run_wo_nn.csv
  cli_args = "Outputs/file_base='run_wo_nn' Controls/inactive='src_control' Postprocessors/left_flux/control_name='src_control_empty' Postprocessors/log_prob_left_flux/control_name='src_control_empty'"
  requirement = "The system shall be able to run with a DRL controller without having a neural net initialized in it."
  capabilities = 'libtorch'
[]

[invalid-execute-on]
  type = RunException
  input = libtorch_drl_control.i
  cli_args = "Controls/src_control/execute_on='TIMESTEP_END'"
  expect_err = "LibtorchDRLControl only supports 'TIMESTEP_BEGIN' for 'execute_on'."
  requirement = "The system shall reject DRL controller execute flags other than timestep begin."
  capabilities = 'libtorch'
[]

[uniform]
  type = CSVDiff
  input = uniform.i
  csvdiff = 'uniform_out.csv'
  allow_test_objects = true

  detail = 'uniform,'
[]

[weibull]
  type = CSVDiff
  input = weibull.i
  csvdiff = 'weibull_out.csv'
  allow_test_objects = true

  detail = 'Weibull (3 parameter),'
[]

[kernel1d_gaussian]
  type = CSVDiff
  input = kernel_density_1d_gaussian.i
  csvdiff = 'kernel_density_1d_gaussian_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and data file as input,'
[]

[kernel1d_uniform]
  type = CSVDiff
  input = kernel_density_1d_uniform.i
  csvdiff = 'kernel_density_1d_uniform_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Uniform kernel and data file as input,'
[]

[kernel1d_datvec]
  type = CSVDiff
  input = kernel_density_1d_datvec.i
  csvdiff = 'kernel_density_1d_datvec_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and data vector as input,'
[]

[kernel1d_userbw]
  type = CSVDiff
  input = kernel_density_1d_userbw.i
  csvdiff = 'kernel_density_1d_userbw_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and user defined bandwidth,'
[]

[kernel1d_sdbw]
  type = CSVDiff
  input = kernel_density_1d_sdbw.i
  csvdiff = 'kernel_density_1d_sdbw_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and standard deviation as bandwidth,'
[]

[normal]
  type = CSVDiff
  input = normal.i
  csvdiff = 'normal_out.csv'
  allow_test_objects = true

  detail = 'normal,'
[]

[maxwellian]
  type = CSVDiff
  input = maxwellian.i
  csvdiff = 'maxwellian_out.csv'
  allow_test_objects = true
  detail = 'maxwellian,'
[]

[truncated_normal]
  type = CSVDiff
  input = truncated_normal.i
  csvdiff = 'truncated_normal_out.csv'
  allow_test_objects = true

  detail = 'truncated normal,'
[]

[lognormal]
  type = CSVDiff
  input = lognormal.i
  csvdiff = 'lognormal_out.csv'
  allow_test_objects = true

  detail = 'lognormal,'
[]

[gamma]
  type = CSVDiff
  input = gamma.i
  csvdiff = 'gamma_out.csv'
  allow_test_objects = true

  detail = 'gamma,'
[]

[beta]
  type = CSVDiff
  input = beta.i
  csvdiff = 'beta_out.csv'
  allow_test_objects = true

  detail = 'beta,'
[]

[fdist]
  type = CSVDiff
  input = fdist.i
  csvdiff = 'fdist_out.csv'
  allow_test_objects = true

  detail = 'F,'
[]

[studentt]
  type = CSVDiff
  input = studentt.i
  csvdiff = 'studentt_out.csv'
  allow_test_objects = true

  detail = 'student t,'
[]

[johnsonsb]
  type = CSVDiff
  input = johnsonsb.i
  csvdiff = 'johnsonsb_out.csv'
  allow_test_objects = true

  detail = 'Johnson Special Bounded (SB), and'
[]

[logistic]
  type = CSVDiff
  input = logistic.i
  csvdiff = 'logistic_out.csv'
  allow_test_objects = true

  detail = 'logistic distributions.'
[]

[normal_direct]
  type = CSVDiff
  input = normal_direct.i
  csvdiff = 'normal_direct_out.csv'
  allow_test_objects = true
  requirement = 'The system shall provide a normal distribution with the ability to directly call methods with distribution inputs.'
  design = 'Normal.md'
  issues = '#12720'
[]

[normal_direct_type_error]
  type = RunException
  input = normal_direct_type_error.i
  allow_test_objects = true
  expect_err = "The 'cdf' object failed to retrieve 'this_is_the_wrong_type' distribution with the desired type."
  requirement = 'The system shall produce an error if a distribution is retrieved with the differing type than supplied.'
  design = 'Normal.md'
  issues = '#12720'
[]

[libtorch_drl_reward_linear]
  type = CSVDiff
  input = drl_reward.i
  csvdiff = drl_reward_linear.csv
  requirement = "The system shall be able to evaluate a linear DRL reward function with a given, time-dependent target."
  issues = '#19571'
  design = 'ScaledAbsDifferenceDRLRewardFunction.md'
  cli_args = "Outputs/file_base='drl_reward_linear'"
  capabilities = 'libtorch'
[]

[generate]
  type = 'CSVDiff'
  input = 'random_ic_distribution_test.i'
  csvdiff = 'random_ic_distribution_test_initial_histo_0000.csv'
  mesh_mode = 'replicated' # sensitive to difference in partitioning

  requirement = 'The system shall generate parallel agnostic random initial conditions using a distribution function.'
[]

[test_err_distribution_and_min_max]
  type = 'RunException'
  input = 'random_ic_distribution_test.i'
  cli_args = 'ICs/u_aux/min=0 ICs/u_aux/max=1'
  prereq = 'generate'
  expect_err = "Cannot use together with 'min' or 'max' parameter"

  requirement = 'The system shall generate an error the random initial condition is used with both a distribution and min or max value defined.'
[]

[loglikelihood_scalar]
  type = 'JSONDiff'
  input = main.i
  allow_test_objects = true
  jsondiff = 'loglikelihood_scalar.json'
  min_parallel = 2
  max_parallel = 2
  detail = 'using a Gaussian type with log-likelihoods requested and model/experiment values are scalars; '
[]

[likelihood_scalar]
  type = 'JSONDiff'
  input = main.i
  allow_test_objects = true
  jsondiff = 'likelihood_scalar.json'
  cli_args = "Likelihood/gaussian/log_likelihood='false' Outputs/file_base='likelihood_scalar'"
  min_parallel = 2
  max_parallel = 2
  detail = 'using a Gaussian type with likelihoods requested and model/experiment values are scalars; '
[]

[truncatedgaussian]
  type = 'JSONDiff'
  input = main.i
  allow_test_objects = true
  jsondiff = 'truncatedgaussian.json'
  cli_args = "Likelihood/gaussian/type='TruncatedGaussian' Likelihood/gaussian/lower_bound='-1.0' Likelihood/gaussian/upper_bound='1.0' Outputs/file_base='truncatedgaussian'"
  min_parallel = 2
  max_parallel = 2
  detail = 'using a truncated Gaussian type with likelihoods requested and model/experiment values are scalars. '
[]

[extremevalue]
  type = 'JSONDiff'
  input = main.i
  allow_test_objects = true
  jsondiff = 'extremevalue.json'
  cli_args = "Likelihood/gaussian/type='ExtremeValue' Outputs/file_base='extremevalue'"
  min_parallel = 2
  max_parallel = 2
  detail = 'using a Extreme Value type with likelihoods requested and model/experiment values are scalars. '
[]

[truncatedgaussian_bounds]
  type = RunException
  input = main.i
  allow_test_objects = True
  cli_args = "Likelihood/gaussian/type=TruncatedGaussian Likelihood/gaussian/lower_bound=1.0 Likelihood/gaussian/upper_bound=-1.0"
  detail = 'the specified lower bound is greater than the upper bound; '
  expect_err = "The specified lower bound should be less than the upper bound."
[]

[single_normal]
  type = CSVDiff
  input = parent_single.i
  cli_args = MultiApps/sub/mode=normal
  csvdiff = 'parent_single_out_storage_0002.csv'

  detail = 'for a single parameter, '
[]

[single_batch]
  type = CSVDiff
  input = parent_single.i
  csvdiff = 'parent_single_out_storage_0002.csv'
  prereq = parent/single_normal
  max_parallel = 3

  detail = 'for a single parameter for a batch of sub-applications,'
[]

[multiple_normal]
  type = CSVDiff
  input = parent_multiple.i
  cli_args = MultiApps/sub/mode=normal
  csvdiff = 'parent_multiple_out_storage_0002.csv'
  prereq = parent/single_batch

  detail = 'for multiple parameters,'
[]

[vector]
  type = CSVDiff
  input = parent_vector.i
  csvdiff = 'parent_vector_out_storage_0002.csv parent_vector_out_prop_A_0002.csv parent_vector_out_prop_B_0002.csv parent_vector_out_prop_C_0002.csv parent_vector_out_prop_D_0002.csv'
  max_parallel = 3

  detail = 'for vector parameters for a batch of sub-applications,'
[]

[multiple_batch]
  type = CSVDiff
  input = parent_multiple.i
  csvdiff = 'parent_multiple_out_storage_0002.csv'
  prereq = parent/multiple_normal
  max_parallel = 3

  detail = 'for multiple parameters for a batch of sub-applications,'
[]

[multiple_batch_with_index]
  type = CSVDiff
  input = parent_multiple.i
  csvdiff = 'parent_multiple_out_storage_0002.csv'
  cli_args = "Controls/cmdline/param_names='Mesh/ymax[1] Mesh/xmax[0]'"
  prereq = parent/multiple_normal
  max_parallel = 3

  detail = 'for multiple parameters using their global column indexes for a batch of sub-applications,'
[]

[input_variable]
  type = RunApp
  input = parent_input_variable.i
  detail = 'for input variables, and'
[]

[input_variable_with_index]
  type = RunApp
  input = parent_input_variable.i
  cli_args = "Controls/cmdline/param_names='var1[0]'"
  detail = 'for input variables with column index specified.'
[]

[wrong_batch_mode]
  type = RunException
  input = parent_single.i
  cli_args = MultiApps/sub/mode=batch-restore
  expect_err = "The MultiApp object, 'sub', supplied to the 'cmdline' object is setup to run in 'batch-restore' mode, when using this mode command line arguments cannot be modified; batch-reset mode should be used instead."
  requirement = "The system shall report an error when the supplied sampler object operates in a mode that does not allow for command line arguments to be modified."
  prereq = parent/multiple_batch
  max_parallel = 3
[]

[wrong_execute_on]
  type = RunException
  input = parent_single.i
  cli_args = Samplers/sample/execute_on=INITIAL
  expect_err = "The sampler object, 'sample', is being used by the 'cmdline' object, thus the 'execute_on' of the sampler must include 'PRE_MULTIAPP_SETUP'."
  requirement = "The system shall report an error when the supplied sampler does not use the correct execution flags."
  prereq = parent/multiple_batch
  max_parallel = 3
[]

[vector_no_bracket]
  type = RunException
  input = parent_vector.i
  cli_args = "Controls/cmdline/param_names='Mesh/xmax[0] Materials/const/prop_values[1,(1.5),2,2] Mesh/ymax'"
  expect_err = "If the bracket is used, it must be provided to every parameter."
  requirement = "The system shall report an error when '[]' syntax is not used for all parameters."
  max_parallel = 3
[]

[vector_out_of_bound]
  type = RunException
  input = parent_vector.i
  cli_args = "Controls/cmdline/param_names='Mesh/xmax[0] Materials/const/prop_values[1,(1.5),2,2] Mesh/ymax[4]'"
  expect_err = "The provided global column index \(4\) for Mesh/ymax is out of bound."
  requirement = "The system shall report an error when provided global column index is out of bound."
  max_parallel = 3
[]

[normal]
  type = CSVDiff
  input = parent_full_solve.i
  csvdiff = parent_full_solve_out_storage_0002.csv
  cli_args = MultiApps/runner/mode=normal
  requirement = "The system shall support pulling postprocessor data from a sub-application for each row of sampled data."
[]

[batch]
  type = CheckFiles
  input = parent_full_solve.i
  check_files = 'parent_full_solve_out_runner0.e parent_full_solve_out_runner1.e parent_full_solve_out_runner2.e parent_full_solve_out_runner3.e parent_full_solve_out_runner4.e'
  prereq = normal
  requirement = "The system shall support running sub-applications in batches."
[]

[n1]
  type = CSVDiff
  input = parent_full_solve.i
  csvdiff = parent_full_solve_out_storage_0002.csv
  max_parallel = 1
  prereq = batch

  detail = 'on a single processor,'
[]

[n2]
  type = CSVDiff
  input = parent_full_solve.i
  csvdiff = parent_full_solve_out_storage_0002.csv
  max_parallel = 2
  min_parallel = 2
  prereq = batch_csv/n1

  detail = 'on multiple processors, and'
[]

[n2_restore]
  type = CSVDiff
  input = parent_full_solve.i
  csvdiff = parent_full_solve_out_storage_0002.csv
  max_parallel = 2
  min_parallel = 2
  cli_args = "MultiApps/runner/mode=batch-restore"
  prereq = batch_csv/n2

  detail = 'on multiple processors using in-memory backup.'
[]

[normal]
  type = CSVDiff
  input = parent_transient.i
  cli_args = MultiApps/runner/mode=normal
  csvdiff = 'parent_transient_out_storage_0000.csv parent_transient_out_storage_0001.csv parent_transient_out_storage_0002.csv parent_transient_out_storage_0003.csv'

  detail = 'with individual sub-applications,'
[]

[batch]
  type = CSVDiff
  input = parent_transient.i
  csvdiff = 'parent_transient_out_storage_0000.csv parent_transient_out_storage_0001.csv parent_transient_out_storage_0002.csv parent_transient_out_storage_0003.csv'
  prereq = transient/normal
  max_parallel = 5

  detail = 'with sub-applications batches using in-memory restore functionality.'
[]

[mode_error]
  type = RunException
  input = parent_transient.i
  cli_args = MultiApps/runner/mode=batch-reset
  expect_err = "The supplied mode, 'batch-reset', currently"
  requirement = "The SamplerTransientMultiApp object shall error if the 'batch-reset' mode is supplied."
[]

[normal]
  type = CSVDiff
  input = parent_transient.i
  cli_args = 'MultiApps/runner/mode=normal runner:Outputs/csv=true'
  csvdiff = 'parent_transient_out_runner0.csv
                 parent_transient_out_runner1.csv
                 parent_transient_out_runner2.csv
                 parent_transient_out_runner3.csv
                 parent_transient_out_runner4.csv'
  prereq = transient/batch
  detail = 'individual sub-applications;'
[]

[batch]
  type = CSVDiff
  input = parent_transient.i
  cli_args = 'MultiApps/runner/mode=batch-restore runner:Outputs/csv=true'
  csvdiff = 'parent_transient_out_runner0.csv
                 parent_transient_out_runner1.csv
                 parent_transient_out_runner2.csv
                 parent_transient_out_runner3.csv
                 parent_transient_out_runner4.csv'
  prereq = file_output/normal
  detail = 'sub-application batches;'
[]

[single]
  type = CSVDiff
  input = parent_single.i
  csvdiff = 'parent_single_out_sub0.csv parent_single_out_sub1.csv parent_single_out_sub2.csv'

  detail = 'for a single parameter and'
[]

[multiple]
  type = CSVDiff
  input = parent_multiple.i
  csvdiff = 'parent_multiple_out_sub0.csv parent_multiple_out_sub1.csv parent_multiple_out_sub2.csv'

  detail = 'for a multiple parameters.'
[]

[wrong_multiapp_type]
  type = RunException
  input = parent_wrong_multiapp_type.i
  expect_err = "The MultiApp must construct its sub-apps in initial setup but not during its creation"

  requirement = "The system shall report an error when sub-apps are constructed too early thus unable to be changed by samplers."
[]

[wrong_num_params]
  type = RunException
  input = parent_wrong_num_params.i
  expect_err = "The number of columns \(2\) must match the number of parameters \(3\)"

  requirement = "The system shall report an error when the number of samples differs from the number of command line parameters."
[]

[function]
  type = JSONDiff
  input = conditional_main.i
  jsondiff = 'conditional_main_out.json'
  detail = 'with a parsed function dependent on time and sampler values;'
[]

[error]
  type = RunException
  input = conditional_main.i
  cli_args = MultiApps/runner/mode=normal
  expect_err = "Conditionally run sampler multiapp only works in batch modes."
  detail = 'error if running in normal mode.'
[]

[normal]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  jsondiff = main_out.json

  detail = "normal execution,"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  cli_args = 'MultiApps/runner/mode=batch-restore'
  jsondiff = main_out.json
  prereq = dynamic/normal

  detail = "batch execution with memory-based restoring, and"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  cli_args = 'MultiApps/runner/mode=batch-reset'
  jsondiff = main_out.json
  prereq = dynamic/batch-restore

  detail = "batch execution with reset-based restoring."
[]

[error]
  type = RunException
  input = main.i
  allow_test_objects = true
  expect_err = "The size of the sampler has changed; SamplerTransientMultiApp object do not support dynamic Sampler output."

  requirement = "The system shall report an error when the size of a sampler is altered an sub-applications are progressing with time with the main application."
[]

[normal]
  type = CSVDiff
  input = stm.i
  csvdiff = stm_out_storage_0001.csv
  min_slots = 3 # heavy memory usage (200+ multiapps)
  detail = 'normal multiapp mode;'
[]

[batch]
  type = CSVDiff
  input = stm.i
  cli_args = 'MultiApps/main/mode=batch-reset'
  csvdiff = stm_out_storage_0001.csv
  detail = 'batch multiapp mode;'
[]

[normal]
  type = CSVDiff
  input = stm_batch.i
  cli_args = 'Outputs/file_base=stm_out'
  csvdiff = stm_out_storage_0001.csv
  min_slots = 3 # heavy memory usage (200+ multiapps)
  detail = 'normal multiapp mode;'
[]

[batch_reset]
  type = CSVDiff
  input = stm_batch.i
  cli_args = 'Outputs/file_base=stm_out MultiApps/main/mode=batch-reset'
  csvdiff = stm_out_storage_0001.csv
  detail = 'batch-reset multiapp mode;'
[]

[batch_restore]
  type = CSVDiff
  input = stm_batch.i
  cli_args = 'Outputs/file_base=stm_out MultiApps/main/mode=batch-restore'
  csvdiff = stm_out_storage_0001.csv
  detail = 'batch-restore multiapp mode;'
[]

[normal]
  type = CSVDiff
  input = 'stm_small_batch.i'
  cli_args = 'MultiApps/main/mode=normal main:Outputs/csv=true main:sub:Outputs/csv=true main:sub:subsub:Outputs/csv=true'
  csvdiff = 'stm_small_batch_out_main0.csv stm_small_batch_out_main0_sub0.csv stm_small_batch_out_main0_sub0_subsub0.csv stm_small_batch_out_main0_sub0_subsub1.csv
                 stm_small_batch_out_main1.csv stm_small_batch_out_main1_sub0.csv stm_small_batch_out_main1_sub0_subsub0.csv stm_small_batch_out_main1_sub0_subsub1.csv
                 stm_small_batch_out_main2.csv stm_small_batch_out_main2_sub0.csv stm_small_batch_out_main2_sub0_subsub0.csv stm_small_batch_out_main2_sub0_subsub1.csv
                 stm_small_batch_out_main3.csv stm_small_batch_out_main3_sub0.csv stm_small_batch_out_main3_sub0_subsub0.csv stm_small_batch_out_main3_sub0_subsub1.csv'
  detail = 'normal multiapp mode;'
[]

[batch_reset]
  type = CSVDiff
  input = 'stm_small_batch.i'
  cli_args = 'MultiApps/main/mode=batch-reset main:Outputs/csv=true main:sub:Outputs/csv=true main:sub:subsub:Outputs/csv=true'
  csvdiff = 'stm_small_batch_out_main0.csv stm_small_batch_out_main0_sub0.csv stm_small_batch_out_main0_sub0_subsub0.csv stm_small_batch_out_main0_sub0_subsub1.csv
                 stm_small_batch_out_main1.csv stm_small_batch_out_main1_sub0.csv stm_small_batch_out_main1_sub0_subsub0.csv stm_small_batch_out_main1_sub0_subsub1.csv
                 stm_small_batch_out_main2.csv stm_small_batch_out_main2_sub0.csv stm_small_batch_out_main2_sub0_subsub0.csv stm_small_batch_out_main2_sub0_subsub1.csv
                 stm_small_batch_out_main3.csv stm_small_batch_out_main3_sub0.csv stm_small_batch_out_main3_sub0_subsub0.csv stm_small_batch_out_main3_sub0_subsub1.csv'
  detail = 'batch-reset multiapp mode;'
[]

[batch_restore]
  type = CSVDiff
  input = 'stm_small_batch.i'
  cli_args = 'MultiApps/main/mode=batch-restore main:Outputs/csv=true main:sub:Outputs/csv=true main:sub:subsub:Outputs/csv=true'
  csvdiff = 'stm_small_batch_out_main0.csv stm_small_batch_out_main0_sub0.csv stm_small_batch_out_main0_sub0_subsub0.csv stm_small_batch_out_main0_sub0_subsub1.csv
                 stm_small_batch_out_main1.csv stm_small_batch_out_main1_sub0.csv stm_small_batch_out_main1_sub0_subsub0.csv stm_small_batch_out_main1_sub0_subsub1.csv
                 stm_small_batch_out_main2.csv stm_small_batch_out_main2_sub0.csv stm_small_batch_out_main2_sub0_subsub0.csv stm_small_batch_out_main2_sub0_subsub1.csv
                 stm_small_batch_out_main3.csv stm_small_batch_out_main3_sub0.csv stm_small_batch_out_main3_sub0_subsub0.csv stm_small_batch_out_main3_sub0_subsub1.csv'
  detail = 'batch-restore multiapp mode;'
[]

[normal_less_procs]
  type = CSVDiff
  input = main.i
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  detail = 'with less processors than rows in normal mode;'
[]

[normal_more_procs]
  type = CSVDiff
  input = main.i
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  prereq = full_solve/normal_less_procs
  detail = 'with more processors than rows in normal mode;'
[]

[normal_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  prereq = full_solve/normal_more_procs
  detail = 'with specified minimum processors per app in normal mode;'
[]

[reset_less_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  prereq = full_solve/normal_min_procs
  detail = 'with less processors than rows in batch-reset mode;'
[]

[reset_more_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  prereq = full_solve/reset_less_procs
  detail = 'with more processors than rows in batch-reset mode;'
[]

[reset_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  prereq = full_solve/reset_more_procs
  detail = 'with specified minimum processors per app in batch-reset mode;'
[]

[restore_less_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  prereq = full_solve/reset_min_procs
  detail = 'with less processors than rows in batch-restore mode;'
[]

[restore_more_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  prereq = full_solve/restore_less_procs
  detail = 'with more processors than rows in batch-restore mode;'
[]

[restore_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2 Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  prereq = full_solve/restore_more_procs
  detail = 'with specified minimum processors per app in batch-restore mode;'
[]

[error_min_procs]
  type = RunException
  input = main.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=1'
  allow_test_objects = true
  expect_err = "Sampler and multiapp communicator configuration inconsistent. Please ensure that 'MultiApps/sub/min\(max\)_procs_per_app' and 'Samplers/sample/min\(max\)_procs_per_row' are the same."
  min_parallel = 2
  detail = 'error when partitionings do not match;'
[]

[normal_less_procs]
  type = CSVDiff
  input = main_transient.i
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  max_parallel = 5
  detail = 'with less processors than rows in normal mode;'
[]

[normal_more_procs]
  type = CSVDiff
  input = main_transient.i
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 6
  prereq = transient/normal_less_procs
  detail = 'with more processors than rows in normal mode;'
[]

[normal_min_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 4
  prereq = transient/normal_more_procs
  detail = 'with specified minimum processors per app in normal mode;'
[]

[restore_less_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  max_parallel = 5
  prereq = transient/normal_min_procs
  detail = 'with less processors than rows in batch-restore mode;'
[]

[restore_more_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 6
  prereq = transient/restore_less_procs
  detail = 'with more processors than rows in batch-restore mode;'
[]

[restore_min_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2 Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 4
  prereq = transient/restore_more_procs
  detail = 'with specified minimum processors per app in batch-restore mode;'
[]

[error_min_procs]
  type = RunException
  input = main_transient.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=1'
  allow_test_objects = true
  expect_err = "Sampler and multiapp communicator configuration inconsistent. Please ensure that 'MultiApps/sub/min\(max\)_procs_per_app' and 'Samplers/sample/min\(max\)_procs_per_row' are the same."
  min_parallel = 2
  detail = 'error when partitionings do not match;'
[]

[full_solve]
  type = CheckFiles
  input = parent_full_solve.i
  check_files = 'parent_full_solve_out_runner0.e parent_full_solve_out_runner1.e parent_full_solve_out_runner2.e parent_full_solve_out_runner3.e parent_full_solve_out_runner4.e'

  requirement = 'The system shall provide the ability to create a full-solve type sub-application from sampled data from distributions.'
[]

[transient]
  type = CheckFiles
  input = parent_transient.i
  check_files = 'parent_transient_out_runner0.e parent_transient_out_runner1.e parent_transient_out_runner2.e parent_transient_out_runner3.e parent_transient_out_runner4.e'

  requirement = 'The system shall provide the ability to create a transient sub-application from the sample data generated from distributions.'
[]

[transient_cmdline_control]
  type = Exodiff
  input = parent_transient_cmd_control.i
  exodiff = 'parent_transient_cmd_control_out_runner0.e parent_transient_cmd_control_out_runner1.e parent_transient_cmd_control_out_runner2.e parent_transient_cmd_control_out_runner3.e parent_transient_cmd_control_out_runner4.e'

  requirement = 'The system shall provide the ability to set a transient sub-application command line parameters from a sample distribution.'

  issues = '#17486 #17548'
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json

  detail = "using normal operation;"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  cli_args = MultiApps/runner/mode=batch-restore
  prereq = repeat/normal

  detail = "using in memory backup operation;"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  cli_args = MultiApps/runner/mode=batch-reset
  prereq = repeat/batch-restore

  detail = "using reset operation."
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"

  detail = "using normal operation;"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "MultiApps/runner/mode=batch-restore Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"
  prereq = new/normal

  detail = "using in memory backup operation;"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "MultiApps/runner/mode=batch-reset Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"
  prereq = new/batch-restore

  detail = "using reset operation."
[]

[cmdline_normal]
  type = CSVDiff
  input = main_full_solve.i
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'command line control in normal mode;'
[]

[cmdline_batch]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-reset"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'command line control in batch mode;'
[]

[transfer_normal]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in normal mode;'
[]

[transfer_batch_reset]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-reset Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in batch-reset mode;'
[]

[transfer_batch_restore]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-restore Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in batch-restore mode;'
[]

[cmdline]
  type = CSVDiff
  input = main_transient.i
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  detail = 'command line control;'
[]

[transfer_normal]
  type = CSVDiff
  input = main_transient.i
  cli_args = "Transfers/inactive='' Controls/active=''"
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  prereq = transient/cmdline
  detail = 'parameter transfer in normal mode;'
[]

[transfer_batch]
  type = CSVDiff
  input = main_transient.i
  cli_args = "MultiApps/sub/mode=batch-restore Transfers/inactive='' Controls/active=''"
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  prereq = transient/transfer_normal
  detail = 'parameter transfer in batch mode;'
[]

[ais_al_uval]
  type = 'JSONDiff'
  input = ais_al.i
  jsondiff = 'ais_al.json'
  detail = 'using the U learning function.'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[SingleProc_SingleRow_Ufunction]
  type = 'JSONDiff'
  input = main_adam.i
  jsondiff = 'single_proc_single_row_ufunction.json'
  cli_args = 'Executioner/num_steps=30' # Ensure we recover during the timesteps
  detail = 'a single row in the sampler using the U learning function;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[MultipleProc_MultipleRow_Ufunction]
  type = 'JSONDiff'
  input = main_adam.i
  min_parallel = 2
  jsondiff = 'multiple_proc_multiple_row_ufunction.json'
  cli_args = "Samplers/mc/num_batch=2 Outputs/file_base='multiple_proc_multiple_row_ufunction'"
  detail = 'multiple rows in the sampler using the U learning function and automatically increase the number of steps when re-training is needed;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 4 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[SingleProc_SingleRow_Ufunction_retraining]
  type = 'JSONDiff'
  input = main_adam.i
  jsondiff = 'single_proc_single_row_ufunction_retraining.json'
  cli_args = "Reporters/conditional/learning_function_threshold=5.0 Outputs/file_base='single_proc_single_row_ufunction_retraining'"
  detail = 'a single row in the sampler using the U learning function and automatically increase the number of steps when re-training is needed;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[COV_squared_exp]
  type = 'JSONDiff'
  input = main_adam.i
  jsondiff = 'cov_squared_exp.json'
  cli_args = "Reporters/conditional/learning_function='COV' Reporters/conditional/learning_function_threshold=0.025 Outputs/file_base='cov_squared_exp'"
  detail = 'a single row in the sampler using the COV learning function and automatically increase the number of steps when re-training is needed;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[SingleProc_SingleRow_Ufunction]
  type = 'JSONDiff'
  input = main_adam.i
  max_parallel = 1
  jsondiff = 'single_proc_single_row_ufunction.json'
  detail = 'a single row in the sampler using the U learning function;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[MultipleProc_MultipleRow_Ufunction]
  type = 'JSONDiff'
  input = main_adam.i
  min_parallel = 2
  max_parallel = 2
  jsondiff = 'multiple_proc_multiple_row_ufunction.json'
  cli_args = "Samplers/mc/num_batch=2 Outputs/file_base='multiple_proc_multiple_row_ufunction'"
  detail = 'multiple rows in the sampler using the U learning function;'
  rel_err = 1.0e-4
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 4 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[statistics]
  type = JSONDiff
  input = statistics_vpp.i
  jsondiff = statistics_vpp_out.json

  detail = "vectors of data from the postprocessing system and include"
[]

[ci]
  type = JSONDiff
  input = statistics_vpp_ci.i
  jsondiff = statistics_vpp_ci_out.json

  detail = "confidence level intervals for statistics calculations."
[]

[statistics]
  type = JSONDiff
  input = statistics_reporter.i
  jsondiff = statistics_reporter_out.json

  detail = "vector of data from the reporting system; including"
[]

[ci]
  type = JSONDiff
  input = statistics_reporter_ci.i
  jsondiff = statistics_reporter_ci_out.json

  detail = "confidence level intervals for statistics calculations"
[]

[error]
  type = RunException
  input = statistics_reporter_error.i
  expect_err = "Reporters/stats/reporters: The following reporter value\(s\) do not have a type supported by the StatisticsReporter:\nconst/const"

  detail = "and error if the supplied type is not supported."
[]

[basic]
  type = JSONDiff
  input = bca.i
  jsondiff = bca_out.json

  detail = "of a vector of data"
[]

[vector]
  type = JSONDiff
  input = bca_main.i
  jsondiff = bca_main_out.json

  detail = "of a vector of vector data"
  valgrind = HEAVY
[]

[replicated]
  type = JSONDiff
  input = bca_distributed.i
  jsondiff = bca_distributed_out.json
  cli_args = "VectorPostprocessors/data/parallel_type=replicated"
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is replicated and"
[]

[distributed]
  type = JSONDiff
  input = bca_distributed.i
  jsondiff = bca_distributed_out.json
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is distributed."
[]

[no_ci_levels]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels=''"
  expect_err = "If the 'ci_method' parameter is supplied then the 'ci_levels' must also be supplied with values in \(0, 1\)"

  detail = "the confidence level intervals are omitted;"
[]

[ci_levels_zero]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels='0'"
  expect_err = "The supplied levels must be greater than zero"

  detail = "the confidence level intervals are less than or equal to zero;"
[]

[ci_levels_one]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels='1'"
  expect_err = "The supplied levels must be less than 1\.0"

  detail = "the confidence level intervals are greater than or equal to one;"
[]

[no_input]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/vectorpostprocessors=''"
  expect_err = "The 'vectorpostprocessors' and/or 'reporters' parameters must be defined and non-empty"

  detail = "input is not provided."
[]

[basic]
  type = JSONDiff
  input = percentile.i
  jsondiff = percentile_out.json

  detail = "of a vector of data"
[]

[vector]
  type = JSONDiff
  input = percentile_main.i
  jsondiff = percentile_main_out.json

  detail = "of a vector of vector data"
  valgrind = HEAVY
[]

[replicated]
  type = JSONDiff
  input = percentile_distributed.i
  jsondiff = percentile_distributed_out.json
  cli_args = "VectorPostprocessors/data/parallel_type=replicated"
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is replicated or"
[]

[distributed]
  type = JSONDiff
  input = percentile_distributed.i
  jsondiff = percentile_distributed_out.json
  prereq = percentile/replicated
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "distributed across processors."
[]

[distributed]
  type = JSONDiff
  input = dp_main.i
  jsondiff = dp_main_out.json
  allow_test_objects = true
  detail = 'on distributed data sets;'
[]

[root]
  type = JSONDiff
  input = dp_main.i
  cli_args = 'Reporters/storage/parallel_type=ROOT'
  jsondiff = dp_main_out.json
  allow_test_objects = true
  detail = 'on root processor;'
[]

[distributed]
  type = JSONDiff
  input = dp_main.i
  jsondiff = dp_main_out.json
  allow_test_objects = true
  detail = 'on distributed data sets;'
  cli_args = 'Samplers/directperturbation/perturbation_method=forward_difference'
[]

[root]
  type = JSONDiff
  input = dp_main.i
  jsondiff = dp_main_out.json
  allow_test_objects = true
  detail = 'on root processor;'
  cli_args = 'Reporters/storage/parallel_type=ROOT Samplers/directperturbation/perturbation_method=forward_difference'
[]

[relative_sensitivity]
  type = JSONDiff
  input = dp_main.i
  cli_args = 'Reporters/sensitivities/relative_sensitivity=true Outputs/file_base=relative_sens'
  jsondiff = relative_sens.json
  allow_test_objects = true
  requirement = 'The system shall support the ability to compute direct perturbation-based relative sensitivity values.'
[]

[from_parallelstorage]
  type = JSONDiff
  requirement = 'The system should be able to map the solutions in a snapshot container into a latent space.'
  input = map_main.i
  jsondiff = 'map_training_data.json'
  min_parallel = 2
  max_parallel = 2
  capabilities = 'petsc>=3.14'
[]

[from_solutions]
  # Note: this test should result in the same reporter values as the one above. Due to reporter naming difference, we
  # needed two different gold files. If something causes mismatch between the values in the two gold files, look for bugs.
  type = JSONDiff
  requirement = 'The system should be able to extract and map variables in nonlinear system into a latent (low-dimensional) space.'
  input = load_main.i
  cli_args = "worker:VariableMappings/inactive='' worker:Reporters/inactive=''"
  jsondiff = 'map_variable.json'
  min_parallel = 2
  max_parallel = 2
  prereq = from_parallelstorage
  capabilities = 'petsc>=3.14'
[]

[distributed]
  type = JSONDiff
  input = morris.i
  jsondiff = morris_out.json
  allow_test_objects = true
  detail = 'distributed data;'
[]

[replicated]
  type = JSONDiff
  input = morris.i
  cli_args = 'VectorPostprocessors/gfun/parallel_type=REPLICATED'
  jsondiff = morris_out.json
  allow_test_objects = true
  detail = 'replicated data;'
[]

[distributed]
  type = JSONDiff
  input = morris_main.i
  jsondiff = morris_main_out.json
  allow_test_objects = true
  capabilities = 'method=opt'
  detail = 'distributed;'
[]

[root]
  type = JSONDiff
  input = morris_main.i
  cli_args = 'Reporters/storage/parallel_type=ROOT'
  jsondiff = morris_main_out.json
  allow_test_objects = true
  capabilities = 'method=opt'
  detail = 'on root processor;'
[]

[print-p2]
  type = JSONDiff
  requirement = 'The system should be able to print serialized snapshots in a distributed fashion in a json format.'
  input = parallel_storage_main.i
  jsondiff = 'parallel_storage_main_out.json parallel_storage_main_out.json.1'

  min_parallel = 2
  max_parallel = 2
[]

[sobol]
  type = JSONDiff
  input = sobol.i
  jsondiff = sobol_out.json
  allow_test_objects = true

  requirement = "The system shall support the ability to compute first, second, and total-effect Sobol sensitivity indices with a reporter."
[]

[sobol_vec]
  type = JSONDiff
  input = sobol_main.i
  jsondiff = sobol_main_out.json
  allow_test_objects = true

  requirement = "The system shall support the ability to compute Sobol sensitivity indices for vector-type data."
  capabilities = 'method=opt'
[]

[sobol_no_resample]
  type = JSONDiff
  input = sobol_no_resample.i
  jsondiff = sobol_no_resample_out.json
  allow_test_objects = true

  requirement = "The system shall support the ability to compute only first and total-effect Sobol sensitivity indices when resampling is not requested."
[]

[replicated]
  type = 'JSONDiff'
  input = 'statistics.i'
  jsondiff = 'statistics_out.json'
  allow_test_objects = true

  detail = 'is replicated and'
[]

[distributed]
  type = 'JSONDiff'
  input = 'statistics_distributed.i'
  jsondiff = 'statistics_distributed_out.json'
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = 'distributed.'
[]

[reporter]
  type = JSONDiff
  input = statistics_main.i
  jsondiff = statistics_main_out.json
  requirement = 'The system shall be capable of computing the statistics from vector reporter values.'
[]

[serial]
  type = JSONDiff
  input = sample_only.i
  jsondiff = sample_only_out.json
  detail = 'serialized output;'
[]

[parallel]
  type = JSONDiff
  input = sample_only.i
  cli_args = 'Reporters/matrix/parallel_type=distributed Outputs/file_base=sample_only_dist'
  min_parallel = 3
  max_parallel = 3
  jsondiff = 'sample_only_dist.json sample_only_dist.json.1 sample_only_dist.json.2'
  detail = 'distributed output;'
[]

[names]
  type = JSONDiff
  input = sample_only.i
  cli_args = 'Reporters/matrix/sampler_column_names=\'foo bar fah lala\' Outputs/file_base=sample_only_names'
  jsondiff = sample_only_names.json
  detail = 'specified column names;'
[]

[simulation]
  type = CSVDiff
  input = stochastic_matrix.i
  csvdiff = stochastic_matrix_out_matrix_0001.csv
  requirement = 'The system shall be able to output samples from a sampler along side stochastic simulation results.'
[]

[wrong_num_names]
  type = RunException
  input = sample_only.i
  cli_args = 'Reporters/matrix/sampler_column_names=foobar'
  expect_err = 'The number of column names specified \(1\) does not match the number of sampler columns \(4\)\.'
  detail = 'inputting the wrong number of column names;'
[]

[wrong_sampler]
  type = RunException
  input = stochastic_matrix.i
  cli_args = 'Samplers/wrong/type=InputMatrix Samplers/wrong/matrix=\'1 2 3 4\' Reporters/matrix/sampler=wrong'
  expect_err = 'Attempting to create a stochastic vector with a different sampler \(sample\) than the one specified at input \(wrong\)\.'
  detail = 'using a different sampler than the transfer\'s;'
[]

[stats]
  type = JSONDiff
  input = stats.i
  jsondiff = stats_out.json

  detail = "compute statistics."
[]

[ais]
  requirement = 'The system shall include an Adaptive Importance Sampling method for sampling distribution data.'
  type = 'JSONDiff'
  skip_keys = 'number_of_parts part'
  input = 'ais.i'
  cli_args = 'Executioner/num_steps=100' # This makes it so "recover" is performed during the simulation
  jsondiff = 'ais_out.json'
  valgrind = none
[]

[ais_error1]
  type = RunException
  input = 'ais_error1.i'
  detail = 'the selected sampler type is not of an adaptive type when performing adaptive decision;'
  expect_err = "The selected sampler is not an adaptive sampler."
[]

[al_1proc]
  type = 'JSONDiff'
  input = parallelAL.i
  allow_test_objects = true
  jsondiff = 'al1.json'
  max_parallel = 1
  cli_args = "Executioner/num_steps=2"
  detail = 'using a single processor and the Expected Improvment acquisition; '
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[al_4proc]
  type = 'JSONDiff'
  input = parallelAL.i
  allow_test_objects = true
  jsondiff = 'al4.json'
  cli_args = "Outputs/file_base='al4' Executioner/num_steps=2"
  min_parallel = 4
  max_parallel = 4
  detail = 'using four processors and the Expected Improvment acquisition; '
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 9 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[al_PoI]
  type = 'JSONDiff'
  input = parallelAL.i
  allow_test_objects = true
  jsondiff = 'al_poi.json'
  cli_args = "ParallelAcquisition/expectedimprovement/type='ProbabilityofImprovement' Outputs/file_base='al_poi' Executioner/num_steps=2"
  max_parallel = 1
  detail = 'using the Probability of Improvment acquisition; '
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[al_CoV]
  type = 'JSONDiff'
  input = parallelAL.i
  allow_test_objects = true
  jsondiff = 'al_cov.json'
  cli_args = "ParallelAcquisition/expectedimprovement/type='CoefficientOfVariation' Outputs/file_base='al_cov'"
  max_parallel = 1
  detail = 'using the Coefficient of Variation acquisition; '
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[al_EIGF]
  type = 'JSONDiff'
  input = parallelAL.i
  allow_test_objects = true
  jsondiff = 'al_eigf.json'
  cli_args = "ParallelAcquisition/expectedimprovement/type='ExpectedImprovementGlobalFit' Outputs/file_base='al_eigf'"
  max_parallel = 1
  detail = 'using the Expected Improvement for Global Fit acquisition; '
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[al_UFunc]
  type = 'JSONDiff'
  input = parallelAL.i
  allow_test_objects = true
  jsondiff = 'al_ufunc.json'
  cli_args = "ParallelAcquisition/expectedimprovement/type='UFunction' Outputs/file_base='al_ufunc'"
  max_parallel = 1
  detail = 'using the U-Function acquisition; '
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[al_UCB]
  type = 'JSONDiff'
  input = parallelAL.i
  allow_test_objects = true
  jsondiff = 'al_ucb.json'
  cli_args = "ParallelAcquisition/expectedimprovement/type='UpperConfidenceBound' Outputs/file_base='al_ucb'"
  max_parallel = 1
  detail = 'using the Upper Confidence Bound acquisition.'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 3 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[b_al]
  type = 'JSONDiff'
  input = BayesianALTrain.i
  allow_test_objects = true
  jsondiff = 'b_al.json'
  max_parallel = 1
  min_slots = 3 # Reserve extra CI memory for libtorch runtime
  detail = 'using the Bayesian posterior targeted acquisition; '
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
[]

[controllable]
  type = 'JSONDiff'
  skip_keys = 'number_of_parts part'
  input = 'pss.i'
  cli_args = 'Samplers/sample/num_samplessub=4 Samplers/sample/num_subsets=3 Samplers/sample/subset_probability=0.5 Outputs/file_base=pss_reduced
                  Executioner/num_steps=10' # This makes it so "recover" is performed during the simulation
  jsondiff = 'pss_reduced.json'
  detail = 'controllable and '
[]

[not_controllable]
  type = 'JSONDiff'
  skip_keys = 'number_of_parts part'
  input = 'pss_cli.i'
  cli_args = 'Outputs/file_base=pss_reduced
                  Executioner/num_steps=10' # This makes it so "recover" is performed during the simulation
  jsondiff = 'pss_reduced.json'
  prereq = pss_reduced/controllable
  detail = 'not controllable.'
[]

[pss1]
  requirement = 'The system shall include an Parallel Subset Simulation method for sampling distribution data.'
  type = 'JSONDiff'
  skip_keys = 'number_of_parts part'
  input = 'pss.i'
  jsondiff = 'pss_out.json'
  valgrind = none
  heavy = true
[]

[pss_error1]
  type = RunException
  input = 'pss_error1.i'
  detail = 'the selected sampler type is not of an adaptive type. '
  expect_err = "The selected sampler is not an adaptive sampler."
[]

[test]
  type = CSVDiff
  input = grid.i
  csvdiff = grid_out_data_0000.csv

  requirement = "The system shall include the ability to create a 1D Cartesian sampling scheme."
[]

[no_triplet]
  type = RunException
  input = grid.i
  cli_args = "Samplers/sample/linear_space_items='1 1'"
  expect_err = "The number of numeric items must be divisible by 3; min, max, divisions for each item are required."
  match_literal = True
  detail = 'the specified grid does not form a triplet,'
[]

[wrong_num_values]
  type = RunException
  input = grid.i
  cli_args = "Samplers/sample/nominal_values='1 1'"
  expect_err = "The number of values specified must match the number of triplets in 'linear_space_items'."
  match_literal = True
  detail = 'the number of nominal values specified does not match the number of sample columns,'
[]

[non_int]
  type = RunException
  input = grid.i
  cli_args = "Samplers/sample/linear_space_items='1 1 0.1 1 1 1 1 1 1'"
  expect_err = "The third entry for each item must be an integer; it provides the number of entries in the resulting item vector."
  match_literal = True
  detail = 'the number of grid points is not an integer, or'
[]

[negative]
  type = RunException
  input = grid.i
  cli_args = "Samplers/sample/linear_space_items='1 1 -1 1 1 1 1 1 1'"
  expect_err = "The third entry for each item must be positive; it provides the number of entries in the resulting item vector."
  match_literal = True
  detail = 'the number of grid points is negative.'
[]

[test]
  type = CSVDiff
  input = grid.i
  csvdiff = grid_out_data_0000.csv

  requirement = "The system shall include the ability to create a Cartesian product sampling scheme."
[]

[sampling_test]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_out_data_0000.csv csv_sampler_out_data_0001.csv'

  detail = 'distributed across processors, '
[]

[sampling_test_global]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_out_data_0000.csv csv_sampler_out_data_0001.csv'
  prereq = csv_sampler/sampling_test
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'replicated across processors, and '
[]

[sampling_test_distributed]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_distributed_out_data_0000.csv.0 csv_sampler_distributed_out_data_0000.csv.1 csv_sampler_distributed_out_data_0001.csv.0 csv_sampler_distributed_out_data_0001.csv.1'
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=csv_sampler_distributed_out'
  min_parallel = 2
  max_parallel = 2
  detail = 'distributed across processors with the output also distributed.'
[]

[csv_sampler_indices]
  requirement = "The CSV Sampler shall sample from a CSV file when column indices are provided."

  type = 'CSVDiff'
  input = 'csv_sampler_indices.i'
  csvdiff = 'csv_sampler_indices_out_data_0000.csv csv_sampler_indices_out_data_0001.csv'
[]

[csv_sampler_names]
  requirement = "The CSV Sampler shall sample from a CSV file when column names are provided."

  type = 'CSVDiff'
  input = 'csv_sampler_names.i'
  csvdiff = 'csv_sampler_indices_out_data_0000.csv csv_sampler_indices_out_data_0001.csv'
  cli_args = Outputs/file_base='csv_sampler_indices_out'
[]

[serial]
  type = CSVDiff
  input = directperturbation.i
  csvdiff = 'directperturbation_out_data_0001.csv'
  detail = "with replicated output;"
[]

[parallel]
  type = CSVDiff
  input = directperturbation.i
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=dp_cd_3'
  min_parallel = 3
  max_parallel = 3
  csvdiff = 'dp_cd_3_data_0001.csv.0 dp_cd_3_data_0001.csv.1 dp_cd_3_data_0001.csv.2'
  detail = "with parallel output;"
[]

[serial]
  type = CSVDiff
  input = directperturbation.i
  csvdiff = 'dp_fd_data_0001.csv'
  detail = "with replicated output;"
  cli_args = 'Outputs/file_base=dp_fd Samplers/directperturbation/perturbation_method=forward_difference'
[]

[parallel]
  type = CSVDiff
  input = directperturbation.i
  cli_args = 'Samplers/directperturbation/perturbation_method=forward_difference VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=dp_fd_3'
  min_parallel = 3
  max_parallel = 3
  csvdiff = 'dp_fd_3_data_0001.csv.0 dp_fd_3_data_0001.csv.1 dp_fd_3_data_0001.csv.2'
  detail = "with parallel output;"
[]

[constant]
  type = JSONDiff
  input = main.i
  jsondiff = main_constant_size_out.json
  allow_test_objects = true
  cli_args = "Samplers/dynamic/increment_rows=0 Outputs/file_base=main_constant_size_out"

  detail = "that remain constant in size and"
[]

[dynamic]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  allow_test_objects = true

  detail = "that are dynamic in size."
[]

[initial]
  type = CSVDiff
  input = initial.i
  csvdiff = 'initial_out_data_0000.csv initial_out_data_0001.csv initial_out_data_0002.csv initial_out_data_0003.csv'

  requirement = 'The system shall support the creation of data sampled from distribution during the initial setup of a simulation.'
[]

[sampling_test]
  type = CSVDiff
  input = input_matrix.i
  csvdiff = input_matrix_out_data_0001.csv

  issues = '#22512'
  requirement = 'The system shall be able to sample an input-defined matrix.'
[]

[dynamic_matrix]
  type = JSONDiff
  input = input_matrix_dynamic.i
  jsondiff = input_matrix_dynamic_out.json
  allow_test_objects = true

  issues = '#31171'
  requirement = 'The system shall be able to modify a input-defined sampling matrix during the simulation.'
[]

[dynamic_matrix_error]
  type = RunException
  input = input_matrix_dynamic.i
  cli_args = 'Controls/sample_control/num_cols_function=dynamic_num_rows'
  expect_err = 'Input matrix changed the number of columns from \d+ to \d+'
  allow_test_objects = true

  issues = '#31171'
  requirement = 'The system shall throw an error if the input-defined sampling matrix changed the number of columns during the simulation.'
[]

[global]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "using global matrix,"
[]

[local]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "a local matrix,"
[]

[row]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "or row-by-row."
[]

[global]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "using global matrix,"
[]

[local]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "a local matrix,"
[]

[row]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "or row-by-row."
[]

[visualize]
  type = CheckFiles
  command = "./latin_hypercube.py"
  check_files = latin_hypercube.pdf
  required_python_packages = 'pandas numpy matplotlib'
  valgrind = NONE

  requirement = "The system shall include a utility that visually displays results of plotting Latin Hypercube test."
[]

[pmcmc_base]
  type = 'JSONDiff'
  input = main_base.i
  allow_test_objects = true
  jsondiff = 'mcmc_base.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the base MCMC class; '
[]

[pmcmc_base_bounds]
  type = 'JSONDiff'
  input = main_base.i
  allow_test_objects = true
  jsondiff = 'mcmc_base_bounds.json'
  cli_args = "Samplers/sample/lower_bound='-2.0 -2.0' Samplers/sample/upper_bound='2.0 2.0' Outputs/file_base='mcmc_base_bounds'"
  min_parallel = 4
  max_parallel = 4
  detail = 'using the base MCMC class with bounds specified; '
[]

[imh_5prop]
  type = 'JSONDiff'
  input = main_imh.i
  allow_test_objects = true
  jsondiff = 'imh_5prop.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the independent Metropolis-Hastings class; '
[]

[des_5prop]
  type = 'JSONDiff'
  input = main_des.i
  allow_test_objects = true
  jsondiff = 'des_5prop.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the differential evolution sampler class; '
[]

[ss_5prop]
  type = 'JSONDiff'
  input = main_ss.i
  allow_test_objects = true
  jsondiff = 'ss_5prop.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the stretch sampler class; '
[]

[des_5prop_var]
  type = 'JSONDiff'
  input = main_des_var.i
  allow_test_objects = true
  jsondiff = 'des_5prop_var.json'
  min_parallel = 4
  max_parallel = 4
  detail = 'using the differential evolution sampler class with variance inferred; '
[]

[pmcmc_base_bounds1]
  type = RunException
  input = main_base.i
  cli_args = "Samplers/sample/lower_bound='-2.0 -2.0'"
  detail = 'only the lower bounds specified; '
  expect_err = "Both lower and upper bounds should be specified."
[]

[pmcmc_base_bounds2]
  type = RunException
  input = main_base.i
  cli_args = "Samplers/sample/lower_bound='-2.0 -2.0' Samplers/sample/upper_bound='2.0'"
  detail = 'the specified bounds are not of the same size; '
  expect_err = "Lower and upper bounds should be of the same size."
[]

[pmcmc_base_sizes]
  type = RunException
  input = main_base.i
  cli_args = "Samplers/sample/initial_values='0.1'"
  detail = 'the priors, bounds, and initial values are not of the same size; '
  expect_err = "The priors and initial values should be of the same size."
[]

[imh_std_sizes]
  type = RunException
  input = main_imh.i
  cli_args = "Samplers/sample/std_prop='0.05'"
  detail = 'the proposal stds, initial values, and priors are not of the same size; '
  expect_err = "The number of proposal stds, initial values, and priors should be the same."
[]

[des_scale_sizes]
  type = RunException
  input = main_des.i
  cli_args = "Samplers/sample/scales='1.0'"
  detail = 'the number of scales provided do not match the number of tunable params; '
  expect_err = "The number of scales provided should match the number of tunable params."
[]

[uniform]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_out_data_0000.csv monte_carlo_uniform_out_data_0001.csv'

  detail = 'a uniform distribution distributed across processors,'
[]

[uniform_global]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_out_data_0000.csv monte_carlo_uniform_out_data_0001.csv'
  prereq = monte_carlo/uniform
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'a uniform distribution replicated across processors,'
[]

[uniform_distributed]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_distributed_out_data_0000.csv.0 monte_carlo_uniform_distributed_out_data_0000.csv.1 monte_carlo_uniform_distributed_out_data_0001.csv.0 monte_carlo_uniform_distributed_out_data_0001.csv.1'
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=monte_carlo_uniform_distributed_out'
  min_parallel = 2
  max_parallel = 2
  detail = 'a uniform distribution distributed across processors (output is also distributed),'
[]

[weibull]
  type = 'CSVDiff'
  input = 'monte_carlo_weibull.i'
  csvdiff = 'monte_carlo_weibull_out_data_0000.csv monte_carlo_weibull_out_data_0001.csv'

  detail = 'a Wiebull distribution distributed across processors, and'
[]

[weibull_global]
  type = 'CSVDiff'
  input = 'monte_carlo_weibull.i'
  csvdiff = 'monte_carlo_weibull_out_data_0000.csv monte_carlo_weibull_out_data_0001.csv'
  prereq = monte_carlo/weibull
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'a Weibull distribution replicated across processors.'
[]

[serial]
  type = CSVDiff
  input = morris.i
  csvdiff = 'morris_out_data_0000.csv morris_out_data_0001.csv'
  detail = "with replicated output;"
[]

[parallel]
  type = CSVDiff
  input = morris.i
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=morris3'
  min_parallel = 3
  max_parallel = 3
  csvdiff = 'morris3_data_0000.csv.0 morris3_data_0000.csv.1 morris3_data_0000.csv.2 morris3_data_0001.csv.0 morris3_data_0001.csv.1 morris3_data_0001.csv.2'
  detail = "with parallel output;"
[]

[parallel_more]
  type = CSVDiff
  input = morris.i
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=morris5'
  min_parallel = 5
  max_parallel = 5
  csvdiff = 'morris5_data_0000.csv.0 morris5_data_0000.csv.1 morris5_data_0000.csv.2 morris5_data_0000.csv.3 morris5_data_0000.csv.4
                 morris5_data_0001.csv.0 morris5_data_0001.csv.1 morris5_data_0001.csv.2 morris5_data_0001.csv.3 morris5_data_0001.csv.4'
  detail = "with more processors than trajectories;"
[]

[serial]
  type = CSVDiff
  input = 'nested_monte_carlo.i'
  csvdiff = 'nested_monte_carlo_out_data_0001.csv'
  max_parallel = 1
  detail = 'in serial;'
[]

[parallel]
  type = CSVDiff
  input = 'nested_monte_carlo.i'
  csvdiff = 'nested_monte_carlo_out_data_0001.csv'
  min_parallel = 3
  prereq = nested_monte_carlo/serial
  detail = 'in parallel;'
[]

[error]
  type = RunException
  input = nested_monte_carlo.i
  cli_args = "Samplers/nested_mc/num_rows='10 3'"
  expect_err = "There must be a set of distributions for each loop defined by 'num_rows'"
  requirement = 'The system shall report an error when the number of nested Monte Carlo loops does not match the number of sets of distributions.'
[]

[resample]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_data_0000.csv'

  detail = "with the re-sampling matrix and"
[]

[no_resample]
  type = CSVDiff
  input = sobol.i
  cli_args = 'Samplers/sobol/resample=false Outputs/file_base=sobol_no_resample_out'
  csvdiff = 'sobol_no_resample_out_data_0000.csv'

  detail = "without the re-sampling matrix."
[]

[row_mismatch]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sample/num_rows=42"
  expect_err = "The supplied Sampler objects must have the same number of rows"

  detail = "with differing number of rows;"
[]

[col_mismatch]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sample/distributions='d0 d1'"
  expect_err = "The supplied Sampler objects must have the same number of columns"

  detail = "with differing number of columns; and"
[]

[same_matrix]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sobol/sampler_a=resample"
  expect_err = "The supplied sampler matrices must not be the same"

  detail = "if the matrices are the same."
[]

[VectorPostprocessors_sampler]
  requirement = 'The system shall be able to perform stochastic sampling based on global vector data.'
  type = 'CSVDiff'
  input = 'vectorpostprocessor.i'
  csvdiff = 'vectorpostprocessor_out_data_0002.csv'
[]

[cv_real]
  type = JSONDiff
  input = train_and_cv.i
  allow_test_objects = true
  cli_args = "Trainers/train/cv_surrogate='surrogate'"
  jsondiff = 'train_and_cv_out.json'
  detail = 'for a 1-dimensional polynomial regression surrogate with Real'
[]

[cv_vector_real]
  type = JSONDiff
  input = poly_reg_vec.i
  jsondiff = 'poly_reg_vec_out.json'
  detail = ' and std::vector<Real> response types.'
[]

[surr-error]
  type = RunException
  input = train_and_cv.i
  allow_test_objects = true
  expect_err = "To perform cross-validation, the option cv_surrogate needs to be specified to provide a Surrogate object for training and evaluation."
  detail = ' and throw an error if a SurrogateModel is not provided.'
[]

[GP_training]
  type = CheckFiles
  input = GP_squared_exponential_training.i
  check_files = 'gauss_process_training_GP_avg_trainer.rd/data'
  allow_test_objects = true
  detail = 'training a Gaussian process model and '
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_testing]
  type = CSVDiff
  input = GP_squared_exponential_testing.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_testing_out_samp_avg_0002.csv'
  detail = 'evaluating the trained Gaussian process model'
  prereq = store_load/GP_training
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_squared_exponential]
  type = CSVDiff
  input = GP_squared_exponential.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_out_samp_avg_0002.csv GP_squared_exponential_out_results_0002.csv'
  detail = 'a squared exponential kernel;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_exponential]
  type = CSVDiff
  input = GP_exponential.i
  allow_test_objects = true
  csvdiff = 'GP_exponential_out_samp_avg_0002.csv GP_exponential_out_results_0002.csv'
  detail = 'an exponential kernel;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_Matern_half_int]
  type = CSVDiff
  input = GP_Matern_half_int.i
  allow_test_objects = true
  csvdiff = 'GP_Matern_half_int_out_samp_avg_0002.csv GP_Matern_half_int_out_results_0002.csv'
  detail = 'Matern half integer kernel;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_squared_exponential_tuned_adam]
  type = CSVDiff
  input = GP_squared_exponential_tuned_adam.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_tuned_adam_out_samp_avg_0002.csv GP_squared_exponential_tuned_adam_out_results_0002.csv GP_squared_exponential_tuned_adam_out_hyperparams_0002.csv'
  detail = 'a squared exponential kernel using Adam;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_exponential_tuned_adam]
  type = CSVDiff
  input = GP_exponential_tuned_adam.i
  allow_test_objects = true
  csvdiff = 'GP_exponential_tuned_adam_out_samp_avg_0002.csv GP_exponential_tuned_adam_out_results_0002.csv GP_exponential_tuned_adam_out_hyperparams_0002.csv'
  detail = 'an exponential kernel using Adam;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_Matern_half_int_tuned_adam]
  type = CSVDiff
  input = GP_Matern_half_int_tuned_adam.i
  allow_test_objects = true
  csvdiff = 'GP_Matern_half_int_tuned_adam_samp_avg_0002.csv GP_Matern_half_int_tuned_adam_results_0002.csv GP_Matern_half_int_tuned_adam_hyperparams_0002.csv'
  detail = 'a Matern half integer kernel using Adam;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_Matern_half_int_tuned_adam_minibatch]
  type = CSVDiff
  input = GP_Matern_half_int_tuned_adam.i
  cli_args = "Trainers/GP_avg_trainer/batch_size=10 Outputs/out/file_base='GP_Matern_half_int_tuned_adam_minibatch'"
  allow_test_objects = true
  csvdiff = 'GP_Matern_half_int_tuned_adam_minibatch_samp_avg_0002.csv GP_Matern_half_int_tuned_adam_minibatch_results_0002.csv GP_Matern_half_int_tuned_adam_minibatch_hyperparams_0002.csv'
  detail = 'a Matern half integer kernel using Adam with mini-batch sampling;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[GP_squared_exponential_tuned_legacy_adam]
  type = CSVDiff
  input = GP_squared_exponential_tuned_adam.i
  cli_args = "Trainers/GP_avg_trainer/optimizer=legacy_adam Outputs/out/file_base='GP_squared_exponential_tuned_legacy_adam'"
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_tuned_legacy_adam_samp_avg_0002.csv GP_squared_exponential_tuned_legacy_adam_results_0002.csv GP_squared_exponential_tuned_legacy_adam_hyperparams_0002.csv'
  detail = 'a squared exponential kernel using legacy-compatible Adam with manually supplied gradients;'
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[optimization_adam_batch_size]
  type = RunException
  input = GP_squared_exponential_tuned_adam.i
  cli_args = "Trainers/GP_avg_trainer/batch_size='100'"
  detail = 'the batch size is greater than the training data set size for Adam optimization. '
  expect_err = "Batch size cannot be greater than the training data set size."
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[zero-timestep-window]
  type = RunException
  input = trainer_errors.i
  cli_args = "Trainers/nn_trainer/timestep_window=0"
  expect_err = "Range check failed for parameter Trainers/nn_trainer/timestep_window"
  detail = "the timestep window used to downsample the rollout data is zero, or"
  capabilities = 'libtorch'
[]

[mismatched-sample-count]
  type = RunException
  input = trainer_errors.i
  cli_args = "Reporters/data/real_vector_vector_values='300 301 302 | 0 1 2; 0 2 4 | 0 -1 -1; 0 -2 -2 | 0 1 2; 0 2 4'"
  expect_err = "Entry 0 of the observation reporters holds 1 samples, but the reward reporter holds 2 samples."
  detail = "the observation reporters hold a different number of rollout samples than the reward reporter."
  capabilities = 'libtorch'
[]

[train_and_evaluate]
  requirement = 'The system shall be able to train and evaluate a libtorch-based neural network in the same input file.'
  type = CSVDiff
  input = train_and_evaluate.i
  allow_test_objects = true
  cli_args = "Trainers/train/read_from_file=false"
  csvdiff = 'train_and_evaluate_out_results_0002.csv'
  capabilities = 'libtorch'
[]

[train]
  requirement = 'The system shall be able to train a libtorch-based neural network.'
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd/data mynet.pt'
  cli_args = "Trainers/train/read_from_file=false"
  allow_test_objects = true
  capabilities = 'libtorch'
[]

[evaluate]
  requirement = 'The system shall be able to evaluate a previously trained, libtorch-based neural network.'
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = train
  capabilities = 'libtorch'
[]

[retrain]
  requirement = 'The system shall be able to retrain a pretrained and saved libtorch-based neural network.'
  type = CSVDiff
  input = retrain.i
  allow_test_objects = true
  cli_args = "Trainers/train/read_from_file=true"
  csvdiff = 'retrain_out_results_0002.csv'
  prereq = evaluate
  capabilities = 'libtorch'
[]

[train-with-rel-tol]
  requirement = 'The system shall be able to train a libtorch-based neural network using a relative tolerance instead of fixed epoch number.'
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd/data mynet.pt'
  cli_args = "Trainers/train/read_from_file=false Trainers/train/rel_loss_tol=0.1"
  allow_test_objects = true
  capabilities = 'libtorch'
  prereq = retrain
[]

[train-standardized]
  requirement = 'The system shall be able to train a libtorch-based neural network with standardized input and output parameters.'
  type = CheckFiles
  input = train.i
  check_files = 'standardized_train.rd/data mynet.pt'
  cli_args = "Trainers/train/read_from_file=false Trainers/train/standardize_input=true Trainers/train/standardize_output=true Outputs/file_base=standardized"
  allow_test_objects = true
  capabilities = 'libtorch'
  prereq = train-with-rel-tol
[]

[evaluate-standardized]
  requirement = 'The system shall be able to evaluate a previously trained, libtorch-based neural network with standardized input and output parameters.'
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_standardized_results_0002.csv'
  cli_args = "Surrogates/surr/filename='standardized_train.rd' Outputs/file_base=evaluate_standardized"
  prereq = train-standardized
  capabilities = 'libtorch'
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_poly_chaos.rd/data'
  allow_test_objects = true

  detail = "trained with replicated stochastic data and"
[]

[evaluate]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train # Creates data files that this test uses
  allow_warnings = true
  detail = "evaluated separately (with replicated data);"
[]

[train_distributed]
  type = CheckFiles
  input = train.i
  cli_args = Reporters/storage/parallel_type=DISTRIBUTED
  check_files = 'train_out_poly_chaos.rd/data'
  prereq = surrogate/evaluate
  allow_test_objects = true

  detail = "trained with distributed stochastic data and"
[]

[evaluate_distributed]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train_distributed # Creates data files that this test uses
  allow_warnings = true
  detail = "evaluated separately (with distributed data);"
[]

[train_three_ranks]
  type = CheckFiles
  input = train.i
  cli_args = Reporters/storage/parallel_type=DISTRIBUTED
  check_files = 'train_out_poly_chaos.rd/data'
  prereq = surrogate/evaluate_distributed
  allow_test_objects = true
  min_parallel = 3

  detail = "trained with distributed stochastic data;"
[]

[evaluate_two_ranks]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train_three_ranks # Creates data files that this test uses
  max_parallel = 2

  detail = "evaluated separately with a different number of processors;"
[]

[evaluate_and_train]
  type = JSONDiff
  input = train_and_evaluate.i
  cli_args = 'Outputs/file_base=evaluate_out'
  jsondiff = evaluate_out.json
  prereq = surrogate/evaluate_two_ranks # Shares evaluate_out checkpoint files under recover
  allow_test_objects = true

  detail = "and be trained and evaluated in memory with a single input file."
[]

[untuned]
  type = 'JSONDiff'
  input = mogp_lmc.i
  jsondiff = "mogp_lmc_out.json"
  detail = "without tuning the hyperparameters."
  abs_zero = 1e-6
  rel_err = 1e-4
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[tuned]
  type = 'JSONDiff'
  input = mogp_lmc_tuned.i
  jsondiff = "mogp_lmc_tuned_out.json"
  detail = "with tuning the hyperparameters."
  abs_zero = 1e-6
  rel_err = 1e-4
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[load]
  type = 'JSONDiff'
  input = mogp_lmc_load.i
  jsondiff = "mogp_lmc_load_out.json"
  detail = "and load the trained surrogate from a file."
  prereq = lmc/tuned
  skip_keys = "reporters train_params"
  abs_zero = 1e-6
  rel_err = 1e-4
  capabilities = 'libtorch'
  compute_devices = 'cpu cuda'
  min_slots = 2 # Reserve extra CI memory for libtorch runtime in GP tests
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd/data'
  allow_test_objects = true
  detail = 'training then '
[]

[load]
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = nearest_point/train # Creates data files that this test uses
  detail = 'evaluating, '
[]

[train_and_load]
  type = CSVDiff
  input = cartesian.i
  allow_test_objects = true
  csvdiff = 'cartesian_out_results_0002.csv'
  detail = 'training and loading, and '
[]

[predictors]
  type = CSVDiff
  input = predictor_response.i
  allow_test_objects = true
  cli_args = 'Outputs/file_base=evaluate_out'
  csvdiff = 'evaluate_out_results_0002.csv'
  detail = 'using explictly specified predictors.'
[]

[vector]
  type = JSONDiff
  input = np_vec.i
  jsondiff = 'np_vec_out.json'
  requirement = 'The system shall be able to create nearest point surrogates with vector-type response.'
[]

[train]
  type = CheckFiles
  input = trainer.i
  check_files = 'trainer_out_pod_rb.rd/data'
  allow_test_objects = true
  detail = 'training using known 4D data '
  max_parallel = 5 # We need fewer processors than samples
[]

[load]
  type = CSVDiff
  input = surr.i
  allow_test_objects = true
  csvdiff = 'surr_out_res_0001.csv'
  prereq = pod_boundary/train # Creates data files that this test uses
  detail = 'and then evaluating new samples separately for new data.'
[]

[trainer_name]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'MultiApps/sub/trainer_name="doesnt_exist"'
  detail = 'the trainer object cannot be found.'
  expect_err = "Unable to find user object with name 'doesnt_exist'"
  max_parallel = 3
[]

[max_procs_per_app]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'MultiApps/sub/max_procs_per_app=2 Samplers/sample/max_procs_per_row=2'
  detail = 'the trainer is run with more than one processors per subapp.'
  expect_err = "does not support more than one processors per subapp"
[]

[more_procs_than_samples]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  detail = 'the number of samples is lower than the number of processors.'
  expect_err = "PODFullSolveMultiApp needs to be run with fewer processors \(detected (\d+)\) than samples \(detected 3\)"
  min_parallel = 4
[]

[trainer_name]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Transfers/snapshot/trainer_name="doesnt_exist"'
  detail = 'the trainer object cannot be found.'
  expect_err = "Unable to find user object with name 'doesnt_exist'"
  max_parallel = 3
[]

[var_names]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = "Trainers/pod_rb/var_names='doesnt_exist'"
  detail = 'the variable names cannot be found on sub-applications, '
  expect_err = "Variable 'doesnt_exist' not found on sub-application [0-9]+!"
  max_parallel = 3
[]

[en_limits]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = "Trainers/pod_rb/error_res='1e-2 1e-3'"
  detail = 'the number of energy limits and variable names do not match, '
  expect_err = "The number of elements is not equal to the number of elements in 'var_names'!"
  max_parallel = 3
[]

[tag_types]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/tag_types="op"'
  detail = 'the number of tag names and tag types do not match, '
  expect_err = "The number of elements is not equal to the number of elements in 'tag_names'!"
  max_parallel = 3
[]

[dir_tag_names]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/tag_types="doesnt_exist doesnt_exist doesnt_exist"'
  detail = "the Dirichlet tag types do not exist, "
  expect_err = "Tag type 'doesnt_exist' is not valid, available names are: op, op_dir, src, src_dir"
  max_parallel = 3
[]

[wrong_res_transfer]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/execute_on="final"'
  detail = 'and the residual generation is called before having the basis vectors.'
  expect_err = "There are no basis vectors available for residual generation. This indicates that the bases have not been created yet. The most common cause of this error is the wrong setting of the 'execute_on' flags in the PODFullSolveMultiApp and/or PODReducedBasisTrainer."
  match_literal = true
  max_parallel = 3
[]

[var_name]
  type = RunException
  input = trainer_and_surr.i
  allow_test_objects = true
  cli_args = "Surrogates/rbpod/change_rank='doesnt_exist'"
  detail = "the number of inputs in 'change_rank' and 'new_ranks' is not the same."
  expect_err = "The size of 'new_ranks' is not equal to the size of 'change_rank' 0 != 1"
[]

[train]
  type = CheckFiles
  input = trainer.i
  check_files = 'trainer_out_pod_rb.rd/data'
  allow_test_objects = true
  detail = 'training using known 3D data, '
  max_parallel = 3 # We need fewer processors than samples
[]

[save_ev]
  type = CSVDiff
  input = trainer.i
  csvdiff = 'eigenvalues_u.csv'
  cli_args = "Trainers/pod_rb/filenames='eigenvalues_u.csv'"
  allow_test_objects = true
  detail = 'saving the eigenvalues, '
  max_parallel = 3 # We need fewer processors than samples
[]

[load]
  type = CSVDiff
  input = surr.i
  allow_test_objects = true
  csvdiff = 'surr_out_res_0001.csv'
  prereq = pod_internal/train # Creates data files that this test uses
  detail = 'then evaluating new samples separately for new data, '
[]

[train_and_load]
  type = CSVDiff
  input = trainer_and_surr.i
  allow_test_objects = true
  csvdiff = 'trainer_and_surr_out_res_0002.csv'
  detail = 'and doing both together in one input file.'
  max_parallel = 3 # We need fewer processors than samples
[]

[monte_carlo_integration]
  type = CSVDiff
  input = main_2d_mc.i
  allow_test_objects = true
  csvdiff = 'main_2d_mc_out_pc_samp_0002.csv'
  detail = 'MonteCarlo sampler with Uniform distribution, '
[]

[monte_carlo_ols]
  type = JSONDiff
  input = ols_test.i
  jsondiff = ols_test_out.json
  detail = 'MonteCarlo sampler with Uniform distribution using least-squares regression, '
[]

[monte_carlo_ridge]
  type = JSONDiff
  input = ols_test.i
  cli_args = 'Trainers/train_ols/penalty=1.0 Trainers/train_pet_ols/penalty=1.0 Outputs/file_base=ridge_test_out'
  jsondiff = ridge_test_out.json
  detail = 'MonteCarlo sampler with Uniform distribution using Ridge regression, '
[]

[gauss_legendre_integration]
  type = CSVDiff
  input = main_2d_quad.i
  allow_test_objects = true
  csvdiff = 'main_2d_quad_out_pc_samp_0002.csv'
  detail = 'Quadrature sampler with Uniform distribution, and '
[]

[gauss_hermite]
  type = CSVDiff
  input = main_2dnorm_quad.i
  allow_test_objects = true
  csvdiff = 'main_2dnorm_quad_out_pc_samp_0002.csv'
  detail = 'Quadrature sampler with Normal distribution.'
[]

[legendre_moment_calculation]
  type = JSONDiff
  input = main_2d_quad_moment.i
  allow_test_objects = true
  jsondiff = 'main_2d_quad_moment_out.json'
  detail = 'statistical moments with Legendre polynomials, '
[]

[hermite_moment_calculation]
  type = JSONDiff
  input = main_2dnorm_quad_moment.i
  allow_test_objects = true
  jsondiff = 'main_2dnorm_quad_moment_out.json'
  detail = 'statistical moments with Hermite polynomials, '
[]

[legendre_local_sensitivity]
  type = JSONDiff
  input = main_2d_quad_locs.i
  allow_test_objects = true
  jsondiff = 'main_2d_quad_locs_out.json'
  detail = 'sampler and user defined local sensitivities with Legendre polynomials, '
[]

[hermite_local_sensitivity]
  type = JSONDiff
  input = main_2dnorm_quad_locs.i
  allow_test_objects = true
  jsondiff = 'main_2dnorm_quad_locs_out.json'
  detail = 'sampler and user defined local sensitivities with Hermite polynomials, and '
[]

[sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  jsondiff = 'sobol_out.json'
  detail = 'Sobol sensitivity indices.'
[]

[smolyak_sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  cli_args = 'Samplers/sample/sparse_grid=smolyak Outputs/file_base=smolyak'
  jsondiff = 'smolyak.json'
  detail = 'Smolyak and '
[]

[clenshaw_curtis_sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  cli_args = 'Samplers/sample/sparse_grid=clenshaw-curtis Outputs/file_base=clenshaw_curtis'
  jsondiff = 'clenshaw_curtis.json'
  detail = 'Clenshaw-Curtis methods.'
[]

[wrong_sampler]
  type = RunException
  input = error.i
  allow_test_objects = true
  expect_err = "Reporter value values/g_values of size [0-9]{3,4} does not match sampler size \([0-9]{2,3}\)."
  detail = "the number of samples does not match the number of results."
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd/data'
  allow_test_objects = true
  detail = 'training using known 3D data '
[]

[load]
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = poly_reg/train # Creates data files that this test uses
  detail = 'and then evaluating new samples separately for the same data'
[]

[train_and_load]
  type = CSVDiff
  input = train_and_evaluate.i
  allow_test_objects = true
  csvdiff = 'train_and_evaluate_out_results_0002.csv'
  detail = 'and then doing both on another 1D case.'
[]

[vector]
  type = JSONDiff
  input = poly_reg_vec.i
  jsondiff = 'poly_reg_vec_out.json'
  requirement = 'The system shall be able to create polynomial regression surrogate with vector-type response.'
[]

[n1]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal'
  max_parallel = 1

  detail = "on a single processor, "
[]

[n2]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal'
  prereq = normal/n1
  min_parallel = 2
  max_parallel = 2

  detail = "on multiple processors, "
[]

[n3]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_small_storage_0001.csv parent_small_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal
                  Samplers/mc/num_rows=3
                  Outputs/file_base=parent_small'
  min_parallel = 4
  max_parallel = 4

  detail = "and on more processors than samples."
[]

[n1]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  max_parallel = 1
  prereq = normal/n2

  detail = "on a single processor,"
[]

[n2]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  min_parallel = 2
  max_parallel = 2
  prereq = batch/n1

  detail = "on multiple processors, and"
[]

[n2_restore]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_data_0001.csv'
  min_parallel = 2
  max_parallel = 2
  prereq = batch/n2
  cli_args = MultiApps/runner/mode=batch-restore

  detail = "on multiple processors using in-memory sub-application restore."
[]

[batch_two_subapps]
  requirement = "The system shall include the ability to transfer stochastic results for two sub apps."
  issues = '#17079'
  type = CSVDiff
  input = parent_2sub.i
  csvdiff = 'parent_2sub_out_data0_0001.csv parent_2sub_out_data1_0001.csv parent_2sub_out_storage0_0001.csv parent_2sub_out_storage1_0001.csv'
[]

[StochasticToolsTransfer_execute_on_error]
  type = RunException
  input = parent.i
  cli_args = Transfers/data/execute_on=INITIAL
  expect_err = "The 'execute_on' parameter for the 'data' transfer was set"
  requirement = "The 'StochasticToolsTransfer object shall error if the 'execute_on' parameter is defined when the corresponding MultiApp object is running in batch mode."
[]

[StochasticToolsTransfer_execute_on_check]
  type = RunException
  input = parent.i
  cli_args = "Transfers/data/execute_on=INITIAL MultiApps/runner/mode=normal"
  expect_err = "MultiAppTransfer execute_on flags do not match associated from_multi_app execute_on flags"
  requirement = "The 'StochasticToolsTransfer' object shall error if the 'execute_on' parameter does not match the corresponding MultiApp object is running in normal mode."
[]

[direction_error]
  type = RunException
  input = parent.i
  cli_args = 'Transfers/data/to_multi_app=runner'
  expect_err = 'Transfers between multiapp are not currently supported for this transfer type'
  requirement = 'The system shall report a reasonable error if parameters for a trasnfer between multiapps are provided to stochastics transfer, which do not support this currently'
[]

[torch-parameters]
  type = JSONDiff
  input = libtorch_drl_control_trainer.i
  # The runner gold is a symlink to the trainer gold, so the network the control receives must
  # match the trainer's. Only the reporter type, time, and time step differ between the apps; the
  # time pattern is one segment so that it spans the list index in root['time_steps'][0]['time'].
  jsondiff = 'train_out.json train_out_runner0_json_out.json'
  ignored_regex_items = "reporters/nn_parameters/type time_steps.*\['time(_step)?"
  requirement = "The system shall be able to transfer an artificial neural network between a trainer object and a control object residing on different apps."
  capabilities = 'libtorch'
[]

[monte_carlo]
  type = CSVDiff
  input = monte_carlo.i
  csvdiff = 'monte_carlo_out_sub0.csv monte_carlo_out_sub1.csv monte_carlo_out_sub2.csv monte_carlo_out_sub3.csv monte_carlo_out_sub4.csv'
  requirement = "The system shall support the creation of a sub-application for each row of the stochastic data."
[]

[bounded-multi-sample-training]
  type = JSONDiff
  input = trainer.i
  jsondiff = 'trainer_json_out.json'
  min_parallel = 2
  requirement = "The system shall be able to train a bounded DRL controller in parallel from multiple seeded rollouts that evaluate the policy once per control period."
  capabilities = 'libtorch'
[]

[training-ignores-executions-without-trajectories]
  type = JSONDiff
  input = trainer.i
  # No trajectories have been transferred on INITIAL, so training matches the TIMESTEP_END run.
  cli_args = "Trainers/nn_trainer/execute_on='INITIAL TIMESTEP_END'"
  jsondiff = 'trainer_json_out.json'
  min_parallel = 2
  requirement = "The system shall not count DRL trainer executions that collect no trajectories toward the number of rollouts collected between policy updates."
  capabilities = 'libtorch'
[]

[period-multiple]
  type = RunException
  input = trainer.i
  cli_args = "Trainers/nn_trainer/timestep_window=4"
  expect_err = "The trainer 'nn_trainer' has timestep_window = 4, which is 2 times this control's num_steps_in_period = 2."
  detail = "the timestep window and control period, stating the window as a multiple of the period,"
  capabilities = 'libtorch'
[]

[period-fraction]
  type = RunException
  input = trainer.i
  cli_args = "Trainers/nn_trainer/timestep_window=1"
  expect_err = "The trainer 'nn_trainer' has timestep_window = 1, which is 1/2 of this control's num_steps_in_period = 2."
  detail = "the timestep window and control period, stating the window as a fraction of the period,"
  capabilities = 'libtorch'
[]

[input-timesteps]
  type = RunException
  input = trainer.i
  cli_args = "Trainers/nn_trainer/input_timesteps=3"
  expect_err = "The trainer 'nn_trainer' has input_timesteps = 3, but this control has input_timesteps = 2."
  detail = "the number of stacked observation time steps,"
  capabilities = 'libtorch'
[]

[observation-count]
  type = RunException
  input = trainer.i
  cli_args = "Trainers/nn_trainer/observation='storage/r_transfer:T_reporter:center_temp_tend:value storage/r_transfer:T_reporter:reward:value' Trainers/nn_trainer/observation_scaling_factors='0.03 1' Trainers/nn_trainer/observation_shift_factors='270 0'"
  expect_err = "The trainer 'nn_trainer' reads 2 observation\(s\), but this control reads 1."
  detail = "the number of observations, or"
  capabilities = 'libtorch'
[]

[control-count]
  type = RunException
  input = trainer.i
  cli_args = "Trainers/nn_trainer/control='storage/r_transfer:T_reporter:left_flux:value storage/r_transfer:T_reporter:left_flux:value' Trainers/nn_trainer/log_probability='storage/r_transfer:T_reporter:log_prob_left_flux:value storage/r_transfer:T_reporter:log_prob_left_flux:value' Trainers/nn_trainer/action_scaling_factors='100 100' Trainers/nn_trainer/min_control_value='0 0' Trainers/nn_trainer/max_control_value='2 2'"
  expect_err = "The trainer 'nn_trainer' produces 2 control signal\(s\), but this control sets 1 parameter\(s\)."
  detail = "the number of control signals."
  capabilities = 'libtorch'
[]

[wrong_multi_app]
  type = RunException
  input = wrong_multi_app.i
  expect_err = "The 'multi_app' parameter must provide either a 'SamplerTransientMultiApp' or 'SamplerFullSolveMultiApp' object."
  requirement = "The system shall produce an error if neither a 'SamplerTransientMultiApp' nor `SamplerFullSolveMultiApp` is provided in SamplerPostprocessorTransfer."
[]

[require_stochastic_results]
  type = RunException
  input = require_stochastic_results.i
  expect_err = "The object prescribed by the 'to_vector_postprocessor' parameter in data must be a 'StochasticResults' object"
  requirement = "The system shall produce an error if the 'result' object in 'SamplerPostprocessorTransfer' is not a 'StochasticResults object'."
[]

[sobol_from_multiapp]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_storage_0001.csv parent_out_storage_0002.csv parent_out_storage_0003.csv parent_out_storage_0004.csv parent_out_storage_0005.csv'

  requirement = "The system shall support the ability to transfer a single value from each sub-application for a set of stochastic data."
[]

[invalid_sub_pp_name]
  type = RunException
  input = parent.i
  cli_args = Transfers/data/from_postprocessor=wrong
  expect_err = "Unknown postprocesssor name 'wrong' on sub-application 'sub'"

  requirement = "The system shall report an error if the supplied name is invalid when attempting to
                  transfer a single value from a sub-application."
[]

[abort]
  type = RunException
  input = cartesian_diverge.i
  cli_args = 'MultiApps/sub/ignore_solve_not_converge=false
                  sub0:Executioner/error_on_dtmin=true'
  expect_err = "Solve failed and timestep already at or below dtmin, cannot continue!"

  detail = "abort run, "
[]

[last_computed]
  type = CSVDiff
  input = cartesian_diverge.i
  csvdiff = 'cartesian_diverge_out_storage_0001.csv cartesian_diverge_out_storage_0002.csv'

  detail = "transfer last computed postprocessor value, "
  valgrind = 'none' # too slow
[]

[nan]
  type = CheckFiles
  input = cartesian_diverge.i
  cli_args = 'Transfers/data/keep_solve_fail_value=false
                  Outputs/file_base=cartesian_diverge_nan'
  check_files = 'cartesian_diverge_nan_storage_0001.csv'
  file_expect_out = 'data:avg\nnan\n0\.269307.*\n0\.269307.*\n0\.269307.*\n0\.269307.*\n'
  detail = "or transfer NaN."
  valgrind = 'none' # too slow
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json

  detail = "in normal mode, "
[]

[batch]
  type = JSONDiff
  input = main_batch.i
  jsondiff = 'main_batch_out.json'

  detail = 'in batch mode, '
[]

[distributed]
  type = JSONDiff
  input = main.i
  cli_args = 'Reporters/storage/parallel_type=DISTRIBUTED Outputs/file_base=main_parallel'
  jsondiff = 'main_parallel.json main_parallel.json.1 main_parallel.json.2 main_parallel.json.3'
  min_parallel = 4
  max_parallel = 4

  detail = 'with distributed output, '
[]

[more_cpus]
  type = JSONDiff
  input = main_small.i
  jsondiff = 'main_small_out.json'
  min_parallel = 4

  detail = 'with more processors than samples, '
[]

[invalid_type]
  type = RunException
  input = main.i
  cli_args = "Transfers/data/from_reporter='mesh/sidesets'"
  expect_err = 'Reporter value mesh/sidesets is of unsupported type'

  detail = 'and error if transferring unsupported type.'
[]

[multiapp_type]
  type = RunException
  input = parent_multiapp_type_error.i
  expect_err = "The 'multi_app' parameter must provide either a 'SamplerTransientMultiApp' or 'SamplerFullSolveMultiApp' object."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if neither a 'SamplerTransientMultiApp' nor `SamplerFullSolveMultiApp` is provided in SamplerParameterTransfer."
[]

[control_missing]
  type = RunException
  input = parent_missing_control.i
  design = 'SamplerTransientMultiApp.md'
  expect_err = "The sub-application \(sub\) does not contain a SamplerReceiver control object."
  requirement = "The system shall produce an error if the sampler sub-application does not contain a Control object with the name 'stochastic'."
[]

[num_parameters_wrong]
  type = RunException
  input = parent_num_parameters_wrong.i
  expect_err = "Number of command line arguments does not match number of sampler columns."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if supplied vector of real values is not sized correctly within the SamplerParameterTransfer object."
[]

[not_vector]
  type = RunException
  input = parent_not_vector.i
  expect_err = "The Real parameter '[a-z,A-Z,/]+' expects a single value, but the vector supplying its values has a size of \d+\."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if a vector of values is supplied to a scalar parameter."
[]

[sampler_mismatch]
  type = RunException
  input = parent_transfer_wrong_sampler.i
  expect_err = "The supplied 'multi_app' must have the same Sampler object as this Transfer."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if sampling method differs between the sub-application and the associated sub-application data transfer."
[]

[monte_carlo]
  type = CSVDiff
  input = monte_carlo.i
  csvdiff = 'monte_carlo_out_sub0.csv monte_carlo_out_sub1.csv monte_carlo_out_sub2.csv monte_carlo_out_sub3.csv monte_carlo_out_sub4.csv'

  detail = 'using a Monte Carlo and'
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_sub00.csv sobol_out_sub01.csv sobol_out_sub02.csv sobol_out_sub03.csv sobol_out_sub04.csv sobol_out_sub05.csv sobol_out_sub06.csv sobol_out_sub07.csv sobol_out_sub08.csv sobol_out_sub09.csv sobol_out_sub10.csv sobol_out_sub11.csv sobol_out_sub12.csv sobol_out_sub13.csv sobol_out_sub14.csv sobol_out_sub15.csv sobol_out_sub16.csv sobol_out_sub17.csv'

  detail = 'Sobol sampling scheme.'
[]

[monte_carlo]
  type = CSVDiff
  input = parent.i
  csvdiff = 'parent_out_sub0.csv parent_out_sub1.csv parent_out_sub2.csv parent_out_sub3.csv parent_out_sub4.csv'
  requirement = "The system shall be capable of transferring vector data to sub-applications for each row of the stochastic data."
[]

[invalid_name]
  type = RunException
  input = parent.i
  cli_args = "Transfers/sub/parameters='foo bar fah lala' :Outputs/csv=false"
  expect_err = "The desired parameter '\w+' was not located for the 'stochastic' object, it either does not exist or has not been declared as controllable."
  detail = "if the vector parameter does not exist;"
[]

[extra_data]
  type = RunException
  input = parent.i
  cli_args = "Transfers/sub/parameters='BCs/left/value' :Outputs/csv=false"
  expect_err = "Number of command line arguments does not match number of sampler columns."
  detail = "if the sub-application does not consume all of the supplied data;"
[]

[batch_restore]
  detail = "batch-restore mode,"
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'serialization_batch_restore.json serialization_batch_restore.json.1 serialization_batch_restore.json.2 serialization_batch_restore.json.3'
  cli_args = "Outputs/file_base=serialization_batch_restore Transfers/solution_transfer/serialize_on_root=true MultiApps/worker/mode=batch-restore"
  min_parallel = 4
  max_parallel = 4
[]

[batch_reset]
  detail = "batch-reset mode, and"
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'serialization_batch_reset.json serialization_batch_reset.json.1 serialization_batch_reset.json.2 serialization_batch_reset.json.3'
  cli_args = "Outputs/file_base=serialization_batch_reset Transfers/solution_transfer/serialize_on_root=true MultiApps/worker/mode=batch-reset"
  min_parallel = 4
  max_parallel = 4
[]

[normal]
  detail = "normal mode."
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'serialization_normal.json serialization_normal.json.1 serialization_normal.json.2 serialization_normal.json.3'
  cli_args = "Outputs/file_base=serialization_normal Transfers/solution_transfer/serialize_on_root=true MultiApps/worker/mode=normal"
  min_parallel = 4
  max_parallel = 4
[]

[batch_restore]
  detail = "batch-restore mode,"
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'no_serialization_batch_restore.json no_serialization_batch_restore.json.1 no_serialization_batch_restore.json.2 no_serialization_batch_restore.json.3'
  cli_args = "Outputs/file_base=no_serialization_batch_restore Transfers/solution_transfer/serialize_on_root=false MultiApps/worker/mode=batch-restore Transfers/solution_transfer/variables='u' Reporters/parallel_storage/variables='u'"
  min_parallel = 4
  max_parallel = 4
[]

[batch_reset]
  detail = "batch-reset mode, and"
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'no_serialization_batch_reset.json no_serialization_batch_reset.json.1 no_serialization_batch_reset.json.2 no_serialization_batch_reset.json.3'
  cli_args = "Outputs/file_base=no_serialization_batch_reset Transfers/solution_transfer/serialize_on_root=false MultiApps/worker/mode=batch-reset Transfers/solution_transfer/variables='u' Reporters/parallel_storage/variables='u'"
  min_parallel = 4
  max_parallel = 4
[]

[normal]
  detail = "normal mode."
  type = JSONDiff
  input = sst_main.i
  jsondiff = 'no_serialization_normal.json no_serialization_normal.json.1 no_serialization_normal.json.2 no_serialization_normal.json.3'
  cli_args = "Outputs/file_base=no_serialization_normal Transfers/solution_transfer/serialize_on_root=false MultiApps/worker/mode=normal Transfers/solution_transfer/variables='u' Reporters/parallel_storage/variables='u'"
  min_parallel = 4
  max_parallel = 4
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_sub00.csv sobol_out_sub01.csv sobol_out_sub02.csv sobol_out_sub03.csv sobol_out_sub04.csv sobol_out_sub05.csv sobol_out_sub06.csv sobol_out_sub07.csv sobol_out_sub08.csv sobol_out_sub09.csv sobol_out_sub10.csv sobol_out_sub11.csv sobol_out_sub12.csv sobol_out_sub13.csv sobol_out_sub14.csv sobol_out_sub15.csv sobol_out_sub16.csv sobol_out_sub17.csv'

  requirement = "The system shall support the creation of a sub-application for each row sampled data generated from a Sobol scheme."
[]

[create_surrogate]
  requirement = 'The system shall be able to create a surrogate model for the solution coordinates in the latent space.'
  type = CheckFiles
  input = create_mapping_main.i
  check_files = 'create_mapping_main_rom_polyreg_v.rd/data create_mapping_main_rom_polyreg_v_aux.rd/data create_mapping_main_mapping_pod_mapping.rd/data'
  min_parallel = 2
  max_parallel = 2
  capabilities = 'petsc>=3.14'
[]

[with_surrogate]
  detail = 'coordinates determined by a surrogate model.'
  type = Exodiff
  input = inverse_map.i
  cli_args = "Outputs/file_base='with_surrogate'"
  exodiff = 'with_surrogate.e'
  min_parallel = 2
  max_parallel = 2
  prereq = create_surrogate
  capabilities = 'petsc>=3.14'
[]

[with_manual_input]
  detail = 'manually specified coordinates.'
  type = Exodiff
  input = inverse_map.i
  cli_args = "Outputs/file_base='with_manual' UserObjects/im/surrogate=''"
  exodiff = 'with_manual.e'
  min_parallel = 2
  max_parallel = 2
  prereq = create_surrogate
  capabilities = 'petsc>=3.14'
[]

[with_polyreg_csv]
  # Note: the reconstructed solution with 2 POD modes and a linear regression should be exact.
  detail = 'polynomial regression surrogates for the expansion coefficients.'
  type = CSVDiff
  input = inverse_map.i
  cli_args = "Outputs/file_base='with_surrogate'"
  csvdiff = 'with_surrogate.csv'
  min_parallel = 2
  max_parallel = 2
  prereq = create_surrogate
  capabilities = 'petsc>=3.14'
[]

[svd_1_proc_per_app]
  type = JSONDiff
  requirement = 'The system should be able to build a variable mapping by computing the SVD of a snapshot matrix in serial.'
  input = pod_mapping_main.i
  jsondiff = 'serial.json'
  cli_args = "Outputs/file_base=serial"
  skip_keys = "left_singular_vectors right_singular_vectors" # We do this because the negative of the eigenvectors are still eigenvectors and depending on the system it finds alternating signs
  max_parallel = 1
  capabilities = 'petsc>=3.14'
[]

[svd_2_proc_per_app]
  type = JSONDiff
  requirement = 'The system should be able to build a variable mapping by computing the SVD of a snapshot matrix in parallel.'
  input = pod_mapping_main.i
  jsondiff = 'two_procs_per_app.json'
  skip_keys = "left_singular_vectors right_singular_vectors" # We do this because the negative of the eigenvectors are still eigenvectors and depending on the system it finds alternating signs
  cli_args = "Outputs/file_base=two_procs_per_app Transfers/solution_transfer/serialize_on_root=false"
  min_parallel = 2
  max_parallel = 2
  capabilities = 'petsc>=3.14'
[]

[svd_2_proc_per_app_root]
  type = JSONDiff
  requirement = 'The system should be able to build a variable mapping by computing the SVD of a snapshot matrix in parallel with having data on only root processors.'
  input = pod_mapping_main.i
  jsondiff = 'two_procs_per_app_root.json'
  cli_args = "Outputs/file_base=two_procs_per_app_root Transfers/solution_transfer/serialize_on_root=true"
  skip_keys = "left_singular_vectors right_singular_vectors" # We do this because the negative of the eigenvectors are still eigenvectors and depending on the system it finds alternating signs
  min_parallel = 2
  max_parallel = 2
  capabilities = 'petsc>=3.14'
[]

[multiple]
  type = CSVDiff
  input = parent.i
  csvdiff = parent_out_storage_0002.csv

  requirement = "The system shall support the collection of stochastic data from multiple sub-applications."
[]

[get_global_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples'
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'get global matrix;'
[]

[get_local_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples'
  csvdiff = sampler_data_out_data_0001.csv
  prereq = get_sample_types/get_global_samples
  detail = 'get local matrix;'
[]

[get_next_local_row]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row'
  csvdiff = sampler_data_out_data_0001.csv
  prereq = get_sample_types/get_local_samples
  detail = 'get next local row;'
[]

[get_local_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples  VectorPostprocessors/data/parallel_type=distributed'
  csvdiff = 'sampler_data_out_data_0001.csv.0 sampler_data_out_data_0001.csv.1 sampler_data_out_data_0001.csv.2'
  min_parallel = 3
  max_parallel = 3
  prereq = get_sample_types/get_next_local_row
  detail = 'get local matrix;'
[]

[get_next_local_row]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row  VectorPostprocessors/data/parallel_type=distributed'
  csvdiff = 'sampler_data_out_data_0001.csv.0 sampler_data_out_data_0001.csv.1 sampler_data_out_data_0001.csv.2'
  min_parallel = 3
  max_parallel = 3
  prereq = parallel/get_local_samples
  detail = 'get next local row;'
[]

[one]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist'"
  csvdiff = sampler_data_out_data_0001.csv
  prereq = parallel/get_next_local_row
  detail = 'one column;'
[]

[multi]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist dist dist' Outputs/file_base=multi_col"
  csvdiff = multi_col_data_0001.csv
  prereq = cols/one
  detail = 'multiple columns;'
[]

[lots]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist' Outputs/file_base=lots_col"
  csvdiff = lots_col_data_0001.csv
  prereq = cols/multi
  detail = 'large number of columns;'
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = sobol_out_sobol_0002.csv
  allow_test_objects = true

  requirement = "The system shall support the ability to compute first, second, and total-effect Sobol sensitivity indices."
[]

[sobol_bootstrap]
  type = CSVDiff
  input = sobol_bootstrap.i
  csvdiff = sobol_bootstrap_out_sobol_0002.csv
  allow_test_objects = true

  requirement = "The system shall support the ability to compute confidence intervals on Sobol sensitivity indices."
[]

[replicated]
  type = CSVDiff
  input = parent.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=replicated_out'
  csvdiff = replicated_out_storage_0002.csv

  detail = "replicated on all processors and"
[]

[distributed]
  type = CSVDiff
  input = parent.i
  min_parallel = 3
  max_parallel = 3
  cli_args = 'VectorPostprocessors/storage/parallel_type=DISTRIBUTED Outputs/file_base=distributed_out'
  csvdiff = 'distributed_out_storage_0002.csv.0 distributed_out_storage_0002.csv.1 distributed_out_storage_0002.csv.2'

  detail = "distributed across many."
[]

[custom]
  type = CSVDiff
  input = parent.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=prefix_out Transfers/data/prefix=prefix'
  csvdiff = prefix_out_storage_0002.csv

  detail = "with custom prefix and"
[]

[none]
  type = CSVDiff
  input = parent.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=no_prefix_out Transfers/data/prefix='
  csvdiff = no_prefix_out_storage_0002.csv

  detail = "without a prefix."
[]

[complete]
  type = CSVDiff
  input = parent.i
  csvdiff = parent_out_storage.csv

  detail = "can be appended into a single data set or"
[]

[time]
  type = CSVDiff
  input = parent.i
  cli_args = VectorPostprocessors/storage/contains_complete_history=false
  csvdiff = 'parent_out_storage_0000.csv parent_out_storage_0001.csv parent_out_storage_0002.csv'

  detail = "or contain a single file per timestep."
[]

[normal]
  type = RunApp
  input = sub.i
  allow_test_objects = true
  command_proxy = 'stochastic_control.py 0'
  required_python_packages = requests
  recover = false
  restep = false
  valgrind = none # Cannot handle extra command-line from valgrind

  detail = 'in "normal" mode, '
[]

[batch_reset]
  type = RunApp
  input = sub.i
  allow_test_objects = true
  command_proxy = 'stochastic_control.py 1'
  required_python_packages = requests
  recover = false
  restep = false
  valgrind = none # Cannot handle extra command-line from valgrind

  detail = 'in "batch-reset" mode, '
[]

[batch_restore]
  type = RunApp
  input = sub.i
  allow_test_objects = true
  command_proxy = 'stochastic_control.py 2'
  required_python_packages = requests
  recover = false
  restep = false
  valgrind = none # Cannot handle extra command-line from valgrind

  detail = 'in "batch-restore" mode, '
[]

[batch_reset_multi]
  type = RunApp
  input = sub.i
  allow_test_objects = true
  command_proxy = 'stochastic_control.py 1 --multi-output'
  required_python_packages = requests
  recover = false
  restep = false
  valgrind = none # Cannot handle extra command-line from valgrind

  detail = 'in "batch-reset" mode with multiple outputs, and'
[]

[batch_restore_multi]
  type = RunApp
  input = sub.i
  allow_test_objects = true
  command_proxy = 'stochastic_control.py 2 --multi-output'
  required_python_packages = requests
  recover = false
  restep = false
  valgrind = none # Cannot handle extra command-line from valgrind

  detail = 'in "batch-restore" mode with multiple outputs.'
[]

[vector_config]
  type = RunApp
  input = vpp_test_runner.i
  issues = '#32196'
  command_proxy = 'test_stochastic_controls_vpp.py 0'
  required_python_packages = 'numpy requests'
  recover = false
  restep = false
  valgrind = none # Cannot handle extra command-line from valgrind
  requirement = "Python utilities for driving stochastic simulations shall support cache tolerances supplied on a per-parameter basis"
[]

[list_format]
  type = RunApp
  input = vpp_test_runner.i
  issues = '#32196'
  command_proxy = 'test_stochastic_controls_vpp.py 1'
  required_python_packages = 'numpy requests'
  recover = false
  restep = false
  valgrind = none # Cannot handle extra command-line from valgrind
  requirement = "Python utilities for driving stochastic simulations shall support output of arbitrary dimensions (after the first two) with caching"
[]

[nested_list_format]
  type = RunApp
  input = vpp_test_runner.i
  issues = '#32196'
  command_proxy = 'test_stochastic_controls_vpp.py 3'
  required_python_packages = 'numpy requests'
  recover = false
  restep = false
  valgrind = none # Cannot handle extra command-line from valgrind
  requirement = "Python utilities for driving stochastic simulations shall support output of arbitrary dimensions (after the first two) with caching, including nested lists"
[]

[artificial_net_affine_scaling]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.artificialNetAppliesAffineInputAndOutputScaling
  design = 'LibtorchArtificialNeuralNet.md'
  requirement = 'The system shall shift and scale the inputs of a feedforward neural network and scale its outputs.'
[]

[bounded_beta_log_probability]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.boundedBetaLogProbability
  requirement = 'The system shall compute the log-probability of an action drawn from a bounded Beta policy, including the Jacobian of the map from the unit interval to the action bounds.'
[]

[bounded_beta_large_logits]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.boundedBetaStaysFiniteForLargeLogits
  requirement = 'The system shall keep the actions, log-probabilities, and gradients of a bounded Beta policy finite for large distribution-parameter logits.'
[]

[gaussian_state_independent_std]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.gaussianActorUsesPhysicalActionScalingAndStateIndependentStd
  requirement = 'The system shall evaluate a Gaussian policy in physical action units, account for the action scaling in its log-probability, and by default use a standard deviation that is independent of the observation.'
[]

[gaussian_state_dependent_std]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.gaussianActorCanUseStateDependentStdWhenRequested
  requirement = 'The system shall optionally compute the standard deviation of a Gaussian policy from the observation.'
[]

[copied_policy_sampling]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.explicitGeneratorKeepsGaussianSamplingStableAcrossCopies
  requirement = 'The system shall sample identical actions from a policy and its copy when both use identically seeded random number generators.'
[]

[seeded_initialization]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.explicitGeneratorMakesInitializationIndependentOfConstructionOrder
  requirement = 'The system shall initialize policy network parameters identically from identically seeded random number generators, independent of the order in which the networks are constructed.'
[]

[load_archive]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.loadActorStateAcceptsTorchSaveArchive
  requirement = 'The system shall restore the parameters, buffers, and outputs of a policy network from a libtorch archive.'
[]

[hidden_layer_mismatch]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.loadActorStateRejectsHiddenLayerMismatch
  detail = 'hidden-layer structure differs from that of the receiving network;'
[]

[boundedness_mismatch]
  type = GoogleTest
  unit_test = LibtorchActorNeuralNetTest.loadActorStateRejectsBoundednessMismatch
  detail = 'action distribution is unbounded while that of the receiving network is bounded.'
[]

[observation_history]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.ObservationHistoryStacksCurrentAndTrajectoryData
  requirement = 'The system shall stack an observation with its previous values into a policy input, both during a simulation and from a recorded trajectory.'
[]

[malformed_stored_history]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.ObservationHistoryRejectsMalformedStoredHistory
  detail = 'whose stored entries do not match the number of input timesteps;'
[]

[zero_timesteps]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.ObservationHistoryRejectsZeroTimesteps
  detail = 'with zero input timesteps.'
[]

[gae_targets]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.ValueEstimatorComputesGAETargets
  detail = 'without terminal transitions;'
[]

[terminal_transitions]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.ValueEstimatorMasksTerminalTransitions
  detail = 'without bootstrapping from the value past a terminal transition.'
[]

[ppo_loss]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.PPOLossUsesStoredLogProbabilityAndValueTarget
  requirement = 'The system shall compute the clipped Proximal Policy Optimization actor loss with an entropy bonus, and the critic loss, from stored action log-probabilities and value targets.'
[]

[standardized_advantages]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.MiniBatchSamplerStandardizesAdvantagesPerBatch
  detail = 'with the advantages standardized within each mini-batch;'
[]

[seeded_shuffling]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.MiniBatchSamplerUsesExplicitGeneratorForDeterministicShuffling
  detail = 'in the same order when given identically seeded random number generators.'
[]

[window_stride]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.TrajectoryAssemblerDownsamplesWithWindowStride
  detail = 'by downsampling with the timestep window, stacking the observation history, and marking the final transition as terminal;'
[]

[averaged_rewards]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.TrajectoryAssemblerAveragesRewardsWithoutShift
  detail = 'with rewards averaged over each timestep window;'
[]

[complete_windows]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.TrajectoryAssemblerCountsCompleteWindows
  detail = 'from complete timestep windows only, skipping samples without one.'
[]

[mismatched_sample_counts]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.TrajectoryAssemblerRejectsMismatchedSampleCounts
  detail = 'from observation reporters that hold a different number of samples than the reward reporter;'
[]

[zero_timestep_window]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.TrajectoryAssemblerRejectsZeroTimestepWindow
  detail = 'with a timestep window of zero.'
[]

[reward_statistics]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.TrajectoryBufferRewardStatisticsIncludeSpreadBetweenTrajectories
  detail = 'computes reward statistics over all transitions and for each trajectory;'
[]

[flatten]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.TrajectoryBufferFlattensTrajectoriesInOrder
  detail = 'flattens trajectories of different lengths into a single batch in order;'
[]

[mismatched_lengths]
  type = GoogleTest
  unit_test = LibtorchRLCoreTest.TrajectoryBufferRejectsMismatchedTrajectoryLengths
  detail = 'rejects a trajectory whose fields do not match the length of its reward sequence.'
[]

[get_mean]
  type = GoogleTest
  unit_test = StochasticTools.getMean
  requirement = 'The system shall compute and provide access to the per-column mean and standard deviation of a data set via computeSet, getMean, and getStdDev.'
  capabilities = 'libtorch'
[]

[get_standardized]
  type = GoogleTest
  unit_test = StochasticTools.getStandardized
  requirement = 'The system shall center and scale a tensor using the mean and standard deviation computed by a Standardizer via getStandardized.'
  capabilities = 'libtorch'
[]

[get_destandardized]
  type = GoogleTest
  unit_test = StochasticTools.getDestandardized
  requirement = 'The system shall recover the original tensor values when reversing a Standardizer getStandardized transform via getDestandardized.'
  capabilities = 'libtorch'
[]

[get_descaled]
  type = GoogleTest
  unit_test = StochasticTools.getDescaled
  requirement = 'The system shall scale a tensor by a Standardizer computed standard deviation via getDescaled, without applying any mean shift.'
  capabilities = 'libtorch'
[]

[get_scaled]
  type = GoogleTest
  unit_test = StochasticTools.getScaled
  requirement = 'The system shall scale a tensor by dividing it by a Standardizer computed standard deviation via getScaled, without applying any mean shift.'
  capabilities = 'libtorch'
[]

[tensor_data_store_load]
  type = GoogleTest
  unit_test = StochasticTools.tensorDataStoreLoad
  requirement = 'The system shall store and load a 2-dimensional torch::Tensor as recoverable data, preserving its shape and values.'
  design = 'DataIO.md'
  capabilities = 'libtorch'
[]

[tensor_scalar_data_store_load]
  type = GoogleTest
  unit_test = StochasticTools.tensorScalarDataStoreLoad
  requirement = 'The system shall store and load a scalar (0-dimensional) torch::Tensor as recoverable data, preserving its rank and value.'
  design = 'DataIO.md'
  capabilities = 'libtorch'
[]

[tensor_vector_data_store_load]
  type = GoogleTest
  unit_test = StochasticTools.tensorVectorDataStoreLoad
  requirement = 'The system shall store and load a 1-dimensional torch::Tensor as recoverable data, preserving its shape and values.'
  design = 'DataIO.md'
  capabilities = 'libtorch'
[]

[tensor_data_store_load_non_contiguous]
  type = GoogleTest
  unit_test = StochasticTools.tensorDataStoreLoadNonContiguous
  requirement = 'The system shall store and load a non-contiguous torch::Tensor as recoverable data, preserving its shape and values.'
  design = 'DataIO.md'
  capabilities = 'libtorch'
[]

[standardizer_data_store_load]
  type = GoogleTest
  unit_test = StochasticTools.standardizerDataStoreLoad
  requirement = 'The system shall store and load a Standardizer with a computed mean and standard deviation as recoverable data, preserving both moments.'
  capabilities = 'libtorch'
[]

[empty_data_store_load]
  type = GoogleTest
  unit_test = StochasticTools.standardizerEmptyDataStoreLoad
  requirement = 'The system shall support storing and loading recoverable data for a Standardizer that has not yet computed a mean/standard deviation set.'
  issues = '#32193'
  capabilities = 'libtorch'
[]