Stochastic Tools Requirements Traceability Matrix

[uniform]
  type = CSVDiff
  input = uniform.i
  csvdiff = 'uniform_out.csv'
  allow_test_objects = true

  detail = 'uniform,'
[]

[weibull]
  type = CSVDiff
  input = weibull.i
  csvdiff = 'weibull_out.csv'
  allow_test_objects = true

  detail = 'Weibull (3 parameter),'
[]

[kernel1d_gaussian]
  type = CSVDiff
  input = kernel_density_1d_gaussian.i
  csvdiff = 'kernel_density_1d_gaussian_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and data file as input,'
[]

[kernel1d_uniform]
  type = CSVDiff
  input = kernel_density_1d_uniform.i
  csvdiff = 'kernel_density_1d_uniform_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Uniform kernel and data file as input,'
[]

[kernel1d_datvec]
  type = CSVDiff
  input = kernel_density_1d_datvec.i
  csvdiff = 'kernel_density_1d_datvec_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and data vector as input,'
[]

[kernel1d_userbw]
  type = CSVDiff
  input = kernel_density_1d_userbw.i
  csvdiff = 'kernel_density_1d_userbw_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and user defined bandwidth,'
[]

[kernel1d_sdbw]
  type = CSVDiff
  input = kernel_density_1d_sdbw.i
  csvdiff = 'kernel_density_1d_sdbw_out.csv'
  allow_test_objects = true
  abs_zero = 5e-7
  rel_err = 1e-3

  detail = 'Kernel Density 1D with a Gaussian kernel and standard deviation as bandwidth,'
[]

[normal]
  type = CSVDiff
  input = normal.i
  csvdiff = 'normal_out.csv'
  allow_test_objects = true

  detail = 'normal,'
[]

[truncated_normal]
  type = CSVDiff
  input = truncated_normal.i
  csvdiff = 'truncated_normal_out.csv'
  allow_test_objects = true

  detail = 'truncated normal,'
[]

[lognormal]
  type = CSVDiff
  input = lognormal.i
  csvdiff = 'lognormal_out.csv'
  allow_test_objects = true

  detail = 'lognormal,'
[]

[johnsonsb]
  type = CSVDiff
  input = johnsonsb.i
  csvdiff = 'johnsonsb_out.csv'
  allow_test_objects = true

  detail = 'Johnson Special Bounded (SB), and'
[]

[logistic]
  type = CSVDiff
  input = logistic.i
  csvdiff = 'logistic_out.csv'
  allow_test_objects = true

  detail = 'logistic distributions.'
[]

[normal_direct]
  type = CSVDiff
  input = normal_direct.i
  csvdiff = 'normal_direct_out.csv'
  allow_test_objects = true
  requirement = 'The system shall provide a normal distribution with the ability to directly call methods with distribution inputs.'
  design = 'Normal.md'
  issues = '#12720'
[]

[normal_direct_type_error]
  type = RunException
  input = normal_direct_type_error.i
  allow_test_objects = true
  expect_err = "The 'cdf' object failed to retrieve 'this_is_the_wrong_type' distribution with the desired type."
  requirement = 'The system shall produce an error if a distribution is retrieved with the differing type than supplied.'
  design = 'Normal.md'
  issues = '#12720'
[]

[generate]
  type = 'CSVDiff'
  input = 'random_ic_distribution_test.i'
  csvdiff = 'random_ic_distribution_test_initial_histo_0000.csv'
  mesh_mode = 'replicated' # sensitive to difference in partitioning

  requirement = 'The system shall generate parallel agnostic random initial conditions using a distribution function.'
[]

[test_err_distribution_and_min_max]
  type = 'RunException'
  input = 'random_ic_distribution_test.i'
  cli_args = 'ICs/u_aux/min=0 ICs/u_aux/max=1'
  prereq = 'generate'
  expect_err = "Cannot use together with 'min' or 'max' parameter"

  requirement = 'The system shall generate an error the random initial condition is used with both a distribution and min or max value defined.'
[]

[single_normal]
  type = CSVDiff
  input = master_single.i
  cli_args = MultiApps/sub/mode=normal
  csvdiff = 'master_single_out_storage_0002.csv'

  detail = 'for a single parameter, '
[]

[single_batch]
  type = CSVDiff
  input = master_single.i
  csvdiff = 'master_single_out_storage_0002.csv'
  prereq = master/single_normal
  max_parallel = 3

  detail = 'for a single parameter for a batch of sub-applications,'
[]

[multiple_normal]
  type = CSVDiff
  input = master_multiple.i
  cli_args = MultiApps/sub/mode=normal
  csvdiff = 'master_multiple_out_storage_0002.csv'
  prereq = master/single_batch

  detail = 'for multiple parameters, and'
[]

[vector]
  type = CSVDiff
  input = master_vector.i
  csvdiff = 'master_vector_out_storage_0002.csv master_vector_out_prop_A_0002.csv master_vector_out_prop_B_0002.csv master_vector_out_prop_C_0002.csv master_vector_out_prop_D_0002.csv'
  max_parallel = 3

  detail = 'for vector parameters for a batch of sub-applications,'
[]

[multiple_batch]
  type = CSVDiff
  input = master_multiple.i
  csvdiff = 'master_multiple_out_storage_0002.csv'
  prereq = master/multiple_normal
  max_parallel = 3

  detail = 'for multiple parameters for a batch of sub-applications.'
[]

[multiple_batch_with_index]
  type = CSVDiff
  input = master_multiple.i
  csvdiff = 'master_multiple_out_storage_0002.csv'
  cli_args = "Controls/cmdline/param_names='Mesh/ymax[1] Mesh/xmax[0]'"
  prereq = master/multiple_normal
  max_parallel = 3

  detail = 'for multiple parameters using their global column indexes for a batch of sub-applications.'
[]

[wrong_batch_mode]
  type = RunException
  input = master_single.i
  cli_args = MultiApps/sub/mode=batch-restore
  expect_err = "The MultiApp object, 'sub', supplied to the 'cmdline' object is setup to run in 'batch-restore' mode, when using this mode command line arguments cannot be modified; batch-reset mode should be used instead."
  requirement = "The system shall error when the supplied sampler object operates in a mode that does not allow for command line arguments to be modified."
  prereq = master/multiple_batch
  max_parallel = 3
[]

[wrong_execute_on]
  type = RunException
  input = master_single.i
  cli_args = Samplers/sample/execute_on=INITIAL
  expect_err = "The sampler object, 'sample', is being used by the 'cmdline' object, thus the 'execute_on' of the sampler must include 'PRE_MULTIAPP_SETUP'."
  requirement = "The system shall error when the supplied sampler does not use the correct execution flags."
  prereq = master/multiple_batch
  max_parallel = 3
[]

[vector_no_bracket]
  type = RunException
  input = master_vector.i
  cli_args = "Controls/cmdline/param_names='Mesh/xmax[0] Materials/const/prop_values[1,(1.5),2,2] Mesh/ymax'"
  expect_err = "If the bracket is used, it must be provided to every parameter."
  requirement = "The system shall error when '[]' syntax is not used for all parameters."
  max_parallel = 3
[]

[vector_out_of_bound]
  type = RunException
  input = master_vector.i
  cli_args = "Controls/cmdline/param_names='Mesh/xmax[0] Materials/const/prop_values[1,(1.5),2,2] Mesh/ymax[4]'"
  expect_err = "The provided global column index \(4\) for Mesh/ymax is out of bound."
  requirement = "The system shall error when provided global column index is out of bound."
  max_parallel = 3
[]

[normal]
  type = CSVDiff
  input = master_full_solve.i
  csvdiff = master_full_solve_out_storage_0002.csv
  cli_args = MultiApps/runner/mode=normal
  requirement = "The system shall support pulling postprocessor data from a sub-application for each row of sampled data."
[]

[exodus_1]
  type = CheckFiles
  input = master_full_solve.i
  check_files = master_full_solve_out_runner0.e
  check_not_exists = master_full_solve_out_runner1.e
  max_parallel = 1
  prereq = normal

  detail = 'on a single processor and'
[]

[exodus_2]
  type = CheckFiles
  input = master_full_solve.i
  check_files = 'master_full_solve_out_runner0.e master_full_solve_out_runner1.e'
  check_not_exists = master_full_solve_out_runner2.e
  max_parallel = 2
  min_parallel = 2
  prereq = batch/exodus_1

  detail = 'on a multiple processors.'
[]

[n1]
  type = CSVDiff
  input = master_full_solve.i
  csvdiff = master_full_solve_out_storage_0002.csv
  prereq = batch/exodus_2
  max_parallel = 1

  detail = 'on a single processor,'
[]

[n2]
  type = CSVDiff
  input = master_full_solve.i
  csvdiff = master_full_solve_out_storage_0002.csv
  max_parallel = 2
  min_parallel = 2
  prereq = batch_csv/n1

  detail = 'on multiple processors, and'
[]

[n2_restore]
  type = CSVDiff
  input = master_full_solve.i
  csvdiff = master_full_solve_out_storage_0002.csv
  max_parallel = 2
  min_parallel = 2
  prereq = batch_csv/n2
  cli_args = "MultiApps/runner/mode=batch-restore"

  detail = 'on multiple processors using in-memory backup.'
[]

[normal]
  type = CSVDiff
  input = master_transient.i
  cli_args = MultiApps/runner/mode=normal
  csvdiff = 'master_transient_out_storage_0000.csv master_transient_out_storage_0001.csv master_transient_out_storage_0002.csv master_transient_out_storage_0003.csv'

  detail = 'with individual sub-applications,'
[]

[batch]
  type = CSVDiff
  input = master_transient.i
  csvdiff = 'master_transient_out_storage_0000.csv master_transient_out_storage_0001.csv master_transient_out_storage_0002.csv master_transient_out_storage_0003.csv'
  prereq = transient/normal
  max_parallel = 5

  detail = 'with sub-applications batches using in-memory restore functionality.'
[]

[mode_error]
  type = RunException
  input = master_transient.i
  cli_args = MultiApps/runner/mode=batch-reset
  expect_err = "The supplied mode, 'batch-reset', currently"
  requirement = "The SamplerTransientMultiApp object shall error if the 'batch-reset' mode is supplied."
[]

[single]
  type = CSVDiff
  input = master_single.i
  csvdiff = 'master_single_out_sub0.csv master_single_out_sub1.csv master_single_out_sub2.csv'

  detail = 'for a single parameter and'
[]

[multiple]
  type = CSVDiff
  input = master_multiple.i
  csvdiff = 'master_multiple_out_sub0.csv master_multiple_out_sub1.csv master_multiple_out_sub2.csv'

  detail = 'for a multiple parameters.'
[]

[wrong_multiapp_type]
  type = RunException
  input = master_wrong_multiapp_type.i
  expect_err = "The MultiApp must construct its sub-apps in initial setup but not during its creation"

  requirement = "The system shall error when sub-apps are constructed too early thus unable to be changed by samplers."
[]

[wrong_num_params]
  type = RunException
  input = master_wrong_num_params.i
  expect_err = "The number of columns \(2\) must match the number of parameters \(3\)"

  requirement = "The system shall error when the number of samples differs from the number of command line parameters."
[]

[normal]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  jsondiff = main_out.json

  detail = "normal execution,"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  cli_args = 'MultiApps/runner/mode=batch-restore'
  jsondiff = main_out.json

  detail = "batch execution with memory-based restoring, and"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  allow_test_objects = true
  cli_args = 'MultiApps/runner/mode=batch-reset'
  jsondiff = main_out.json

  detail = "batch execution with reset-based restoring."
[]

[error]
  type = RunException
  input = main.i
  allow_test_objects = true
  expect_err = "The size of the sampler has changed; SamplerTransientMultiApp object do not support dynamic Sampler output."

  requirement = "The system shall error when the size of a sampler is altered an sub-applications are progressing with time with the main application."
[]

[normal_less_procs]
  type = CSVDiff
  input = main.i
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  detail = 'with less processors than rows in normal mode;'
[]

[normal_more_procs]
  type = CSVDiff
  input = main.i
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  detail = 'with more processors than rows in normal mode;'
[]

[normal_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  detail = 'with specified minimum processors per app in normal mode;'
[]

[reset_less_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  detail = 'with less processors than rows in batch-reset mode;'
[]

[reset_more_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  detail = 'with more processors than rows in batch-reset mode;'
[]

[reset_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-reset MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  detail = 'with specified minimum processors per app in batch-reset mode;'
[]

[restore_less_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  max_parallel = 5
  detail = 'with less processors than rows in batch-restore mode;'
[]

[restore_more_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 6
  detail = 'with more processors than rows in batch-restore mode;'
[]

[restore_min_procs]
  type = CSVDiff
  input = main.i
  cli_args = 'MultiApps/sub/mode=batch-restore MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2 Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = main_out_vpp_0001.csv
  min_parallel = 4
  detail = 'with specified minimum processors per app in batch-restore mode;'
[]

[error_min_procs]
  type = RunException
  input = main.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=1'
  allow_test_objects = true
  expect_err = "Sampler and multiapp communicator configuration inconsistent. Please ensure that 'MultiApps/sub/min\(max\)_procs_per_app' and 'Samplers/sample/min\(max\)_procs_per_row' are the same."
  min_parallel = 2
  detail = 'error when partitionings do not match;'
[]

[normal_less_procs]
  type = CSVDiff
  input = main_transient.i
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  max_parallel = 5
  detail = 'with less processors than rows in normal mode;'
[]

[normal_more_procs]
  type = CSVDiff
  input = main_transient.i
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 6
  detail = 'with more processors than rows in normal mode;'
[]

[normal_min_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 4
  detail = 'with specified minimum processors per app in normal mode;'
[]

[restore_less_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  max_parallel = 5
  detail = 'with less processors than rows in batch-restore mode;'
[]

[restore_more_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 6
  detail = 'with more processors than rows in batch-restore mode;'
[]

[restore_min_procs]
  type = CSVDiff
  input = main_transient.i
  cli_args = 'MultiApps/sub/mode=batch-restore MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=2 Controls/active="" Transfers/inactive=rep Reporters/check/reporters=vpp/pp:pp2'
  allow_test_objects = true
  csvdiff = 'main_transient_out_vpp_0001.csv main_transient_out_vpp_0002.csv main_transient_out_vpp_0003.csv'
  min_parallel = 4
  detail = 'with specified minimum processors per app in batch-restore mode;'
[]

[error_min_procs]
  type = RunException
  input = main_transient.i
  cli_args = 'MultiApps/sub/min_procs_per_app=2 Samplers/sample/min_procs_per_row=1'
  allow_test_objects = true
  expect_err = "Sampler and multiapp communicator configuration inconsistent. Please ensure that 'MultiApps/sub/min\(max\)_procs_per_app' and 'Samplers/sample/min\(max\)_procs_per_row' are the same."
  min_parallel = 2
  detail = 'error when partitionings do not match;'
[]

[full_solve]
  type = CheckFiles
  input = master_full_solve.i
  check_files = 'master_full_solve_out_runner0.e master_full_solve_out_runner1.e master_full_solve_out_runner2.e master_full_solve_out_runner3.e master_full_solve_out_runner4.e'

  requirement = 'The system shall provide the ability to create a full-solve type sub-application from sampled data from distributions.'
[]

[transient]
  type = CheckFiles
  input = master_transient.i
  check_files = 'master_transient_out_runner0.e master_transient_out_runner1.e master_transient_out_runner2.e master_transient_out_runner3.e master_transient_out_runner4.e'

  requirement = 'The system shall provide the ability to create a transient sub-application from the sample data generated from distributions.'
[]

[transient_cmdline_control]
  type = Exodiff
  input = master_transient_cmd_control.i
  exodiff = 'master_transient_cmd_control_out_runner0.e master_transient_cmd_control_out_runner1.e master_transient_cmd_control_out_runner2.e master_transient_cmd_control_out_runner3.e master_transient_cmd_control_out_runner4.e'

  requirement = 'The system shall provide the ability to set a transient sub-application command line parameters from a sample distribution.'

  issues = '#17486 #17548'
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json

  detail = "using normal operation;"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  cli_args = MultiApps/runner/mode=batch-restore

  detail = "using in memory backup operation;"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  cli_args = MultiApps/runner/mode=batch-reset

  detail = "using reset operation."
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"

  detail = "using normal operation;"
[]

[batch-restore]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "MultiApps/runner/mode=batch-restore Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"

  detail = "using in memory backup operation;"
[]

[batch-reset]
  type = JSONDiff
  input = main.i
  jsondiff = main_new_out.json
  cli_args = "MultiApps/runner/mode=batch-reset Outputs/file_base=main_new_out Samplers/dynamic/execute_on='INITIAL TIMESTEP_END'"

  detail = "using reset operation."
[]

[cmdline_normal]
  type = CSVDiff
  input = main_full_solve.i
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'command line control in normal mode;'
[]

[cmdline_batch]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-reset"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'command line control in batch mode;'
[]

[transfer_normal]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in normal mode;'
[]

[transfer_batch_reset]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-reset Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in batch-reset mode;'
[]

[transfer_batch_restore]
  type = CSVDiff
  input = main_full_solve.i
  cli_args = "MultiApps/sub/mode=batch-restore Transfers/inactive='' Controls/active=''"
  csvdiff = main_full_solve_out_storage_0001.csv
  detail = 'parameter transfer in batch-restore mode;'
[]

[cmdline]
  type = CSVDiff
  input = main_transient.i
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  detail = 'command line control;'
[]

[transfer_normal]
  type = CSVDiff
  input = main_transient.i
  cli_args = "Transfers/inactive='' Controls/active=''"
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  detail = 'parameter transfer in normal mode;'
[]

[transfer_batch]
  type = CSVDiff
  input = main_transient.i
  cli_args = "MultiApps/sub/mode=batch-restore Transfers/inactive='' Controls/active=''"
  csvdiff = 'main_transient_out_storage_0001.csv main_transient_out_storage_0002.csv main_transient_out_storage_0003.csv'
  detail = 'parameter transfer in batch mode;'
[]

[statistics]
  type = JSONDiff
  input = statistics_vpp.i
  jsondiff = statistics_vpp_out.json

  detail = "vectors of data from the postprocessing system and include"
[]

[ci]
  type = JSONDiff
  input = statistics_vpp_ci.i
  jsondiff = statistics_vpp_ci_out.json

  detail = "confidence level intervals for statistics calculations."
[]

[statistics]
  type = JSONDiff
  input = statistics_reporter.i
  jsondiff = statistics_reporter_out.json

  detail = "vector of data from the reporting system; including"
[]

[ci]
  type = JSONDiff
  input = statistics_reporter_ci.i
  jsondiff = statistics_reporter_ci_out.json

  detail = "confidence level intervals for statistics calculations"
[]

[error]
  type = RunException
  input = statistics_reporter_error.i
  expect_err = "The following reporter value\(s\) do not have a type supported by the StatisticsReporter:\n\s+const/const"

  detail = "and error if the supplied type is not supported."
[]

[basic]
  type = JSONDiff
  input = bca.i
  jsondiff = bca_out.json

  detail = "of a vector of data"
[]

[vector]
  type = JSONDiff
  input = bca_main.i
  jsondiff = bca_main_out.json

  detail = "of a vector of vector data"
[]

[replicated]
  type = JSONDiff
  input = bca_distributed.i
  jsondiff = bca_distributed_out.json
  cli_args = "VectorPostprocessors/data/parallel_type=replicated"
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is replicated and"
[]

[distributed]
  type = JSONDiff
  input = bca_distributed.i
  jsondiff = bca_distributed_out.json
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is distributed."
[]

[no_ci_levels]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels=''"
  expect_err = "If the 'ci_method' parameter is supplied then the 'ci_levels' must also be supplied with values in \(0, 1\)"

  detail = "the confidence level intervals are omitted;"
[]

[ci_levels_zero]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels='0'"
  expect_err = "The supplied levels must be greater than zero"

  detail = "the confidence level intervals are less than or equal to zero;"
[]

[ci_levels_one]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/ci_levels='1'"
  expect_err = "The supplied levels must be less than 1\.0"

  detail = "the confidence level intervals are greater than or equal to one;"
[]

[no_input]
  type = RunException
  input = error.i
  cli_args = "Reporters/stats/vectorpostprocessors=''"
  expect_err = "The 'vectorpostprocessors' and/or 'reporters' parameters must be defined and non-empty"

  detail = "input is not provided."
[]

[basic]
  type = JSONDiff
  input = percentile.i
  jsondiff = percentile_out.json

  detail = "of a vector of data"
[]

[vector]
  type = JSONDiff
  input = percentile_main.i
  jsondiff = percentile_main_out.json

  detail = "of a vector of vector data"
[]

[replicated]
  type = JSONDiff
  input = percentile_distributed.i
  jsondiff = percentile_distributed_out.json
  cli_args = "VectorPostprocessors/data/parallel_type=replicated"
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "using data that is replicated or"
[]

[distributed]
  type = JSONDiff
  input = percentile_distributed.i
  jsondiff = percentile_distributed_out.json
  prereq = percentile/replicated
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = "distributed across processors."
[]

[sobol]
  type = JSONDiff
  input = sobol.i
  jsondiff = sobol_out.json
  allow_test_objects = true

  requirement = "The system shall support the ability to compute first, second, and total-effect Sobol sensitivity indices with a reporter."
[]

[sobol_vec]
  type = JSONDiff
  input = sobol_main.i
  jsondiff = sobol_main_out.json
  allow_test_objects = true

  requirement = "The system shall support the ability to compute Sobol sensitivity indices for vector-type data."
  method = 'opt'
[]

[replicated]
  type = 'JSONDiff'
  input = 'statistics.i'
  jsondiff = 'statistics_out.json'
  allow_test_objects = true

  detail = 'is replicated and'
[]

[distributed]
  type = 'JSONDiff'
  input = 'statistics_distributed.i'
  jsondiff = 'statistics_distributed_out.json'
  min_parallel = 3
  max_parallel = 3
  allow_test_objects = true

  detail = 'distributed.'
[]

[reporter]
  type = JSONDiff
  input = statistics_main.i
  jsondiff = statistics_main_out.json
  requirement = 'The system shall be capable of computing the statistics from vector reporter values.'
[]

[stats]
  type = JSONDiff
  input = stats.i
  jsondiff = stats_out.json

  detail = "compute statistics."
[]

[ais]
  requirement = 'The system shall include an Adaptive Importance Sampling method for sampling distribution data.'
  type = 'JSONDiff'
  skip_keys = 'number_of_parts part'
  input = 'ais.i'
  jsondiff = 'ais_out.json'
[]

[pss1]
  requirement = 'The system shall include an Parallel Subset Simulation method for sampling distribution data.'
  type = 'JSONDiff'
  skip_keys = 'number_of_parts part'
  input = 'pss.i'
  jsondiff = 'pss_out.json'
[]

[pss_error1]
  type = RunException
  input = 'pss_error1.i'
  detail = 'the selected sampler type is not of an adaptive type. '
  expect_err = "The selected sampler is not an adaptive sampler."
[]

[test]
  type = CSVDiff
  input = grid.i
  csvdiff = grid_out_data_0000.csv

  requirement = "The system shall include the ability to create a Cartesian product sampling scheme."
[]

[sampling_test]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_out_data_0000.csv csv_sampler_out_data_0001.csv'

  detail = 'distributed across processors, '
[]

[sampling_test_global]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_out_data_0000.csv csv_sampler_out_data_0001.csv'
  prereq = csv_sampler/sampling_test
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'replicated across processors, and '
[]

[sampling_test_distributed]
  type = 'CSVDiff'
  input = 'csv_sampler.i'
  csvdiff = 'csv_sampler_distributed_out_data_0000.csv.0 csv_sampler_distributed_out_data_0000.csv.1 csv_sampler_distributed_out_data_0001.csv.0 csv_sampler_distributed_out_data_0001.csv.1'
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=csv_sampler_distributed_out'
  min_parallel = 2
  max_parallel = 2
  detail = 'distributed across processors with the output also distributed.'
[]

[csv_sampler_indices]
  requirement = "The CSV Sampler shall sample from a CSV file when column indices are provided."

  type = 'CSVDiff'
  input = 'csv_sampler_indices.i'
  csvdiff = 'csv_sampler_indices_out_data_0000.csv csv_sampler_indices_out_data_0001.csv'
[]

[csv_sampler_names]
  requirement = "The CSV Sampler shall sample from a CSV file when column names are provided."

  type = 'CSVDiff'
  input = 'csv_sampler_names.i'
  csvdiff = 'csv_sampler_indices_out_data_0000.csv csv_sampler_indices_out_data_0001.csv'
  cli_args = Outputs/file_base='csv_sampler_indices_out'
[]

[constant]
  type = JSONDiff
  input = main.i
  jsondiff = main_constant_size_out.json
  allow_test_objects = true
  cli_args = "Samplers/dynamic/increment_rows=0 Outputs/file_base=main_constant_size_out"

  detail = "that remain constant in size and"
[]

[dynamic]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json
  allow_test_objects = true

  detail = "that are dynamic in size."
[]

[initial]
  type = CSVDiff
  input = initial.i
  csvdiff = 'initial_out_data_0000.csv initial_out_data_0001.csv initial_out_data_0002.csv initial_out_data_0003.csv'

  requirement = 'The system shall support the creation of data sampled from distribution during the initial setup of a simulation.'
[]

[global]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "using global matrix,"
[]

[local]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "a local matrix,"
[]

[row]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row'
  csvdiff = 'latin_hypercube_out_data_0000.csv latin_hypercube_out_data_0001.csv'

  detail = "or row-by-row."
[]

[global]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "using global matrix,"
[]

[local]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "a local matrix,"
[]

[row]
  type = CSVDiff
  input = 'latin_hypercube.i'
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row Samplers/sample/num_rows=2
                  Outputs/file_base=latin_hypercube_small'
  csvdiff = 'latin_hypercube_small_data_0000.csv latin_hypercube_small_data_0001.csv'
  min_parallel = 4

  detail = "or row-by-row."
[]

[visualize]
  type = CheckFiles
  command = "./latin_hypercube.py"
  check_files = latin_hypercube.pdf
  required_python_packages = 'pandas numpy matplotlib'

  requirement = "The system shall include a utility that visually displays results of plotting Latin Hypercube test."
[]

[uniform]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_out_data_0000.csv monte_carlo_uniform_out_data_0001.csv'

  detail = 'a uniform distribution distributed across processors,'
[]

[uniform_global]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_out_data_0000.csv monte_carlo_uniform_out_data_0001.csv'
  prereq = monte_carlo/uniform
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'a uniform distribution replicated across processors,'
[]

[uniform_distributed]
  type = 'CSVDiff'
  input = 'monte_carlo_uniform.i'
  csvdiff = 'monte_carlo_uniform_distributed_out_data_0000.csv.0 monte_carlo_uniform_distributed_out_data_0000.csv.1 monte_carlo_uniform_distributed_out_data_0001.csv.0 monte_carlo_uniform_distributed_out_data_0001.csv.1'
  cli_args = 'VectorPostprocessors/data/parallel_type=DISTRIBUTED Outputs/file_base=monte_carlo_uniform_distributed_out'
  min_parallel = 2
  max_parallel = 2
  detail = 'a uniform distribution distributed across processors (output is also distributed),'
[]

[weibull]
  type = 'CSVDiff'
  input = 'monte_carlo_weibull.i'
  csvdiff = 'monte_carlo_weibull_out_data_0000.csv monte_carlo_weibull_out_data_0001.csv'

  detail = 'a Wiebull distribution distributed across processors, and'
[]

[weibull_global]
  type = 'CSVDiff'
  input = 'monte_carlo_weibull.i'
  csvdiff = 'monte_carlo_weibull_out_data_0000.csv monte_carlo_weibull_out_data_0001.csv'
  prereq = monte_carlo/weibull
  cli_args = VectorPostprocessors/data/sampler_method=get_global_samples

  detail = 'a Weibull distribution replicated across processors.'
[]

[serial]
  type = CSVDiff
  input = 'nested_monte_carlo.i'
  csvdiff = 'nested_monte_carlo_out_data_0001.csv'
  max_parallel = 1
  detail = 'in serial;'
[]

[parallel]
  type = CSVDiff
  input = 'nested_monte_carlo.i'
  csvdiff = 'nested_monte_carlo_out_data_0001.csv'
  min_parallel = 3
  detail = 'in parallel;'
[]

[error]
  type = RunException
  input = nested_monte_carlo.i
  cli_args = "Samplers/nested_mc/num_rows='10 3'"
  expect_err = "There must be a set of distributions for each loop defined by 'num_rows'"
  requirement = 'The system shall error out when the number of nested Monte Carlo loops does not match the number of sets of distributions.'
[]

[resample]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_data_0000.csv'

  detail = "with the re-sampling matrix and"
[]

[no_resample]
  type = CSVDiff
  input = sobol.i
  cli_args = 'Samplers/sobol/resample=false Outputs/file_base=sobol_no_resample_out'
  csvdiff = 'sobol_no_resample_out_data_0000.csv'

  detail = "without the re-sampling matrix."
[]

[row_mismatch]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sample/num_rows=42"
  expect_err = "The supplied Sampler objects must have the same number of rows"

  detail = "with differing number of rows;"
[]

[col_mismatch]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sample/distributions='d0 d1'"
  expect_err = "The supplied Sampler objects must have the same number of columns"

  detail = "with differing number of columns; and"
[]

[same_matrix]
  type = RunException
  input = sobol.i
  cli_args = "Samplers/sobol/sampler_a=resample"
  expect_err = "The supplied sampler matrices must not be the same"

  detail = "if the matrices are the same."
[]

[GP_training]
  type = CheckFiles
  input = GP_squared_exponential_training.i
  check_files = 'gauss_process_training_GP_avg_trainer.rd'
  allow_test_objects = true
  detail = 'training a Gaussian process model and '
[]

[GP_testing]
  type = CSVDiff
  input = GP_squared_exponential_testing.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_testing_out_samp_avg_0002.csv'
  detail = 'evaluating the trained Gaussian process model'
  prereq = store_load/GP_training
[]

[GP_squared_exponential]
  type = CSVDiff
  input = GP_squared_exponential.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_out_samp_avg_0002.csv GP_squared_exponential_out_results_0002.csv'
  detail = 'a squared exponential kernel;'
[]

[GP_exponential]
  type = CSVDiff
  input = GP_exponential.i
  allow_test_objects = true
  csvdiff = 'GP_exponential_out_samp_avg_0002.csv GP_exponential_out_results_0002.csv'
  detail = 'an exponential kernel;'
[]

[GP_Matern_half_int]
  type = CSVDiff
  input = GP_Matern_half_int.i
  allow_test_objects = true
  csvdiff = 'GP_Matern_half_int_out_samp_avg_0002.csv GP_Matern_half_int_out_results_0002.csv'
  detail = 'Matern half integer kernel;'
[]

[GP_squared_exponential_tuned]
  type = CSVDiff
  input = GP_squared_exponential_tuned.i
  allow_test_objects = true
  csvdiff = 'GP_squared_exponential_tuned_out_samp_avg_0002.csv GP_squared_exponential_tuned_out_results_0002.csv GP_squared_exponential_tuned_out_hyperparams_0002.csv'
  petsc_version = '>=3.10.0'
  detail = 'a squared exponential kernel;'
[]

[GP_exponential_tuned]
  type = CSVDiff
  input = GP_exponential_tuned.i
  allow_test_objects = true
  csvdiff = 'GP_exponential_tuned_out_samp_avg_0002.csv GP_exponential_tuned_out_results_0002.csv GP_exponential_tuned_out_hyperparams_0002.csv'
  petsc_version = '>=3.10.0'
  detail = 'an exponential kernel;'
[]

[GP_Matern_half_int_tuned]
  type = CSVDiff
  input = GP_Matern_half_int_tuned.i
  allow_test_objects = true
  csvdiff = 'GP_Matern_half_int_tuned_out_samp_avg_0002.csv GP_Matern_half_int_tuned_out_results_0002.csv GP_Matern_half_int_tuned_out_hyperparams_0002.csv'
  petsc_version = '>=3.10.0'
  detail = 'a Matern half integer kernel;'
[]

[optimization]
  type = RunException
  input = GP_squared_exponential_tuned.i
  cli_args = "Trainers/GP_avg_trainer/tuning_algorithm='none'"
  detail = 'the no optimization is selected while parameter tuning is required. '
  expect_err = "No tuning algorithm is selected for the hyper parameter optimization!"
[]

[train_and_evaluate]
  requirement = 'The system shall be able to train and evaluate a libtorch-based neural network in the same input file.'
  type = CSVDiff
  input = train_and_evaluate.i
  allow_test_objects = true
  cli_args = "Trainers/train/read_from_file=false"
  csvdiff = 'train_and_evaluate_out_results_0002.csv'
  libtorch = true
[]

[train]
  requirement = 'The system shall be able to train a libtorch-based neural network.'
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd mynet.pt'
  cli_args = "Trainers/train/read_from_file=false"
  allow_test_objects = true
  libtorch = true
[]

[evaluate]
  requirement = 'The system shall be able to evaluate a previously trained, libtorch-based neural network.'
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = train
  libtorch = true
[]

[retrain]
  requirement = 'The system shall be able to retrain a pretrained and saved libtorch-based neural network.'
  type = CSVDiff
  input = retrain.i
  allow_test_objects = true
  cli_args = "Trainers/train/read_from_file=true"
  csvdiff = 'retrain_out_results_0002.csv'
  prereq = evaluate
  libtorch = true
[]

[trainer-no-torch-error]
  requirement = 'The system shall be able to throw an error if the user requires libtorch-based objects without installing libtorch.'
  type = RunException
  input = train.i
  allow_test_objects = true
  expect_err = "PyTorch C\+\+ API \(libtorch\) must be installed to use this object, see https://mooseframework.inl.gov/modules/stochastic_tools/install_pytorch.html for instruction."
  libtorch = false
[]

[train-with-rel-tol]
  requirement = 'The system shall be able to train a libtorch-based neural network using a relative tolerance instead of fixed epoch number.'
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd mynet.pt'
  cli_args = "Trainers/train/read_from_file=false Trainers/train/rel_loss_tol=0.1"
  allow_test_objects = true
  libtorch = true
  prereq = retrain
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_poly_chaos.rd'
  allow_test_objects = true

  detail = "trained with replicated stochastic data and"
[]

[evaluate]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train # Creates data files that this test uses
  allow_warnings = true
  detail = "evaluated separately (with replicated data);"
[]

[train_distributed]
  type = CheckFiles
  input = train.i
  cli_args = Reporters/storage/parallel_type=DISTRIBUTED
  check_files = 'train_out_poly_chaos.rd'
  prereq = surrogate/evaluate
  allow_test_objects = true

  detail = "trained with distributed stochastic data and"
[]

[evaluate_distributed]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train_distributed # Creates data files that this test uses
  allow_warnings = true
  detail = "evaluated separately (with distributed data);"
[]

[train_three_ranks]
  type = CheckFiles
  input = train.i
  cli_args = Reporters/storage/parallel_type=DISTRIBUTED
  check_files = 'train_out_poly_chaos.rd'
  prereq = surrogate/evaluate_distributed
  allow_test_objects = true
  min_parallel = 3

  detail = "trained with distributed stochastic data;"
[]

[evaluate_two_ranks]
  type = JSONDiff
  input = evaluate.i
  jsondiff = evaluate_out.json
  prereq = surrogate/train_three_ranks # Creates data files that this test uses
  max_parallel = 2

  detail = "evaluated separately with a different number of processors;"
[]

[evaluate_and_train]
  type = JSONDiff
  input = train_and_evaluate.i
  cli_args = 'Outputs/file_base=evaluate_out'
  jsondiff = evaluate_out.json
  allow_test_objects = true

  detail = "and be trained and evaluated in memory with a single input file."
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd'
  allow_test_objects = true
  detail = 'training then '
[]

[load]
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = nearest_point/train # Creates data files that this test uses
  detail = 'evaluating, '
[]

[train_and_load]
  type = CSVDiff
  input = cartesian.i
  allow_test_objects = true
  csvdiff = 'cartesian_out_results_0002.csv'
  detail = 'training and loading, and '
[]

[predictors]
  type = CSVDiff
  input = predictor_response.i
  allow_test_objects = true
  cli_args = 'Outputs/file_base=evaluate_out'
  csvdiff = 'evaluate_out_results_0002.csv'
  detail = 'using explictly specified predictors.'
[]

[train]
  type = CheckFiles
  input = trainer.i
  check_files = 'trainer_out_pod_rb.rd'
  allow_test_objects = true
  detail = 'training using known 4D data '
[]

[load]
  type = CSVDiff
  input = surr.i
  allow_test_objects = true
  csvdiff = 'surr_out_res_0001.csv'
  prereq = pod_boundary/train # Creates data files that this test uses
  detail = 'and then evaluating new samples separately for new data.'
[]

[trainer_name]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'MultiApps/sub/trainer_name="doesnt_exist"'
  detail = 'the trainer object cannot be found.'
  expect_err = "Unable to find user object with name 'doesnt_exist'"
[]

[trainer_name]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Transfers/snapshot/trainer_name="doesnt_exist"'
  detail = 'the trainer object cannot be found.'
  expect_err = "Unable to find user object with name 'doesnt_exist'"
[]

[var_names]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = "Trainers/pod_rb/var_names='doesnt_exist'"
  detail = 'the variable names cannot be found on sub-applications, '
  expect_err = "Variable 'doesnt_exist' not found on sub-application [0-9]+!"
[]

[en_limits]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = "Trainers/pod_rb/error_res='1e-2 1e-3'"
  detail = 'the number of energy limits and variable names do not match, '
  expect_err = "The number of elements is not equal to the number of elements in 'var_names'!"
[]

[tag_types]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/tag_types="op"'
  detail = 'the number of tag names and tag types do not match, '
  expect_err = "The number of elements is not equal to the number of elements in 'tag_names'!"
[]

[dir_tag_names]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/tag_types="doesnt_exist doesnt_exist doesnt_exist"'
  detail = "the Dirichlet tag types do not exist, "
  expect_err = "Tag type 'doesnt_exist' is not valid, available names are: op, op_dir, src, src_dir"
[]

[wrong_res_transfer]
  type = RunException
  input = trainer.i
  allow_test_objects = true
  cli_args = 'Trainers/pod_rb/execute_on="final"'
  detail = 'and the residual generation is called before having the basis vectors.'
  expect_err = "There are no basis vectors available for residual generation. This indicates that the bases have not been created yet. The most common cause of this error is the wrong setting of the 'execute_on' flags in the PODFullSolveMultiApp and/or PODReducedBasisTrainer."
  match_literal = true
[]

[var_name]
  type = RunException
  input = trainer_and_surr.i
  allow_test_objects = true
  cli_args = "Surrogates/rbpod/change_rank='doesnt_exist'"
  detail = "the number of inputs in 'change_rank' and 'new_ranks' is not the same."
  expect_err = "The size of 'new_ranks' is not equal to the size of 'change_rank' 0 != 1"
[]

[train]
  type = CheckFiles
  input = trainer.i
  check_files = 'trainer_out_pod_rb.rd'
  allow_test_objects = true
  detail = 'training using known 3D data, '
[]

[save_ev]
  type = CSVDiff
  input = trainer.i
  csvdiff = 'eigenvalues_u.csv'
  cli_args = "Trainers/pod_rb/filenames='eigenvalues_u.csv'"
  allow_test_objects = true
  detail = 'saving the eigenvalues, '
[]

[load]
  type = CSVDiff
  input = surr.i
  allow_test_objects = true
  csvdiff = 'surr_out_res_0001.csv'
  prereq = pod_internal/train # Creates data files that this test uses
  detail = 'then evaluating new samples separately for new data, '
[]

[train_and_load]
  type = CSVDiff
  input = trainer_and_surr.i
  allow_test_objects = true
  csvdiff = 'trainer_and_surr_out_res_0002.csv'
  detail = 'and doing both together in one input file.'
[]

[monte_carlo_integration]
  type = CSVDiff
  input = main_2d_mc.i
  allow_test_objects = true
  csvdiff = 'main_2d_mc_out_pc_samp_0002.csv'
  detail = 'MonteCarlo sampler with Uniform distribution, '
[]

[gauss_legendre_integration]
  type = CSVDiff
  input = main_2d_quad.i
  allow_test_objects = true
  csvdiff = 'main_2d_quad_out_pc_samp_0002.csv'
  detail = 'Quadrature sampler with Uniform distribution, and '
[]

[gauss_hermite]
  type = CSVDiff
  input = main_2dnorm_quad.i
  allow_test_objects = true
  csvdiff = 'main_2dnorm_quad_out_pc_samp_0002.csv'
  detail = 'Quadrature sampler with Normal distribution.'
[]

[legendre_moment_calculation]
  type = JSONDiff
  input = main_2d_quad_moment.i
  allow_test_objects = true
  jsondiff = 'main_2d_quad_moment_out.json'
  detail = 'statistical moments with Legendre polynomials, '
[]

[hermite_moment_calculation]
  type = JSONDiff
  input = main_2dnorm_quad_moment.i
  allow_test_objects = true
  jsondiff = 'main_2dnorm_quad_moment_out.json'
  detail = 'statistical moments with Hermite polynomials, '
[]

[legendre_local_sensitivity]
  type = JSONDiff
  input = main_2d_quad_locs.i
  allow_test_objects = true
  jsondiff = 'main_2d_quad_locs_out.json'
  detail = 'sampler and user defined local sensitivities with Legendre polynomials, '
[]

[hermite_local_sensitivity]
  type = JSONDiff
  input = main_2dnorm_quad_locs.i
  allow_test_objects = true
  jsondiff = 'main_2dnorm_quad_locs_out.json'
  detail = 'sampler and user defined local sensitivities with Hermite polynomials, and '
[]

[sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  jsondiff = 'sobol_out.json'
  detail = 'Sobol sensitivity indices.'
[]

[smolyak_sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  cli_args = 'Samplers/sample/sparse_grid=smolyak Outputs/file_base=smolyak'
  jsondiff = 'smolyak.json'
  detail = 'Smolyak and '
[]

[clenshaw_curtis_sobol]
  type = JSONDiff
  input = sobol.i
  allow_test_objects = true
  cli_args = 'Samplers/sample/sparse_grid=clenshaw-curtis Outputs/file_base=clenshaw_curtis'
  jsondiff = 'clenshaw_curtis.json'
  detail = 'Clenshaw-Curtis methods.'
[]

[wrong_sampler]
  type = RunException
  input = error.i
  allow_test_objects = true
  expect_err = "Reporter value values/g_values of size [0-9]{3,4} does not match sampler size \([0-9]{2,3}\)."
  detail = "the number of samples does not match the number of results."
[]

[train]
  type = CheckFiles
  input = train.i
  check_files = 'train_out_train.rd'
  allow_test_objects = true
  detail = 'training using known 3D data '
[]

[load]
  type = CSVDiff
  input = evaluate.i
  allow_test_objects = true
  csvdiff = 'evaluate_out_results_0002.csv'
  prereq = poly_reg/train # Creates data files that this test uses
  detail = 'and then evaluating new samples separately for the same data'
[]

[train_and_load]
  type = CSVDiff
  input = train_and_evaluate.i
  allow_test_objects = true
  csvdiff = 'train_and_evaluate_out_results_0002.csv'
  detail = 'and then doing both on another 1D case.'
[]

[vector]
  type = CheckFiles
  input = poly_reg_vec.i
  check_files = 'poly_reg_vec_out.json'
  requirement = 'The sytem shall be able to create polynomial regression surrogate with vector-type response.'
[]

[n1]
  type = CSVDiff
  input = master.i
  csvdiff = 'master_out_storage_0001.csv master_out_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal'
  max_parallel = 1

  detail = "on a single processor, "
[]

[n2]
  type = CSVDiff
  input = master.i
  csvdiff = 'master_out_storage_0001.csv master_out_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal'
  prereq = normal/n1
  min_parallel = 2
  max_parallel = 2

  detail = "on multiple processors, "
[]

[n3]
  type = CSVDiff
  input = master.i
  csvdiff = 'master_small_storage_0001.csv master_small_data_0001.csv'
  cli_args = 'MultiApps/runner/mode=normal
                  Samplers/mc/num_rows=3
                  Outputs/file_base=master_small'
  min_parallel = 4
  max_parallel = 4

  detail = "and on more processors than samples."
[]

[n1]
  type = CSVDiff
  input = master.i
  csvdiff = 'master_out_storage_0001.csv master_out_data_0001.csv'
  max_parallel = 1
  prereq = normal/n2

  detail = "on a single processor,"
[]

[n2]
  type = CSVDiff
  input = master.i
  csvdiff = 'master_out_storage_0001.csv master_out_data_0001.csv'
  min_parallel = 2
  max_parallel = 2
  prereq = batch/n1

  detail = "on multiple processors, and"
[]

[n2_restore]
  type = CSVDiff
  input = master.i
  csvdiff = 'master_out_storage_0001.csv master_out_data_0001.csv'
  min_parallel = 2
  max_parallel = 2
  prereq = batch/n2
  cli_args = MultiApps/runner/mode=batch-restore

  detail = "on multiple processors using in-memory sub-application restore."
[]

[batch_two_subapps]
  requirement = "The system shall include the ability to transfer stochastic results for two sub apps."
  issues = '#17079'
  type = CSVDiff
  input = master_2sub.i
  csvdiff = 'master_2sub_out_data0_0001.csv master_2sub_out_data1_0001.csv master_2sub_out_storage0_0001.csv master_2sub_out_storage1_0001.csv'
[]

[StochasticToolsTransfer_execute_on_error]
  type = RunException
  input = master.i
  cli_args = Transfers/data/execute_on=INITIAL
  expect_err = "The 'execute_on' parameter for the 'data' transfer was set"
  requirement = "The 'StochasticToolsTransfer object shall error if the 'execute_on' parameter is defined when the corresponding MultiApp object is running in batch mode."
[]

[StochasticToolsTransfer_execute_on_check]
  type = RunException
  input = master.i
  cli_args = "Transfers/data/execute_on=INITIAL MultiApps/runner/mode=normal"
  expect_err = "MultiAppTransfer execute_on flags do not match associated from_multi_app execute_on flags"
  requirement = "The 'StochasticToolsTransfer' object shall error if the 'execute_on' parameter does not match the corresponding MultiApp object is running in normal mode."
[]

[direction_error]
  type = RunException
  input = master.i
  cli_args = 'Transfers/data/to_multi_app=runner'
  expect_err = 'Transfers between multiapp are not currently supported for this transfer type'
  requirement = 'The system shall report a reasonable error if parameters for a trasnfer between multiapps are provided to stochastics transfer, which do not support this currently'
[]

[monte_carlo]
  type = CSVDiff
  input = monte_carlo.i
  csvdiff = 'monte_carlo_out_sub0.csv monte_carlo_out_sub1.csv monte_carlo_out_sub2.csv monte_carlo_out_sub3.csv monte_carlo_out_sub4.csv'
  requirement = "The system shall support the creation of a sub-application for each row of the stochastic data."
[]

[wrong_multi_app]
  type = RunException
  input = wrong_multi_app.i
  expect_err = "The 'multi_app' parameter must provide either a 'SamplerTransientMultiApp' or 'SamplerFullSolveMultiApp' object."
  requirement = "The system shall produce an error if neither a 'SamplerTransientMultiApp' nor `SamplerFullSolveMultiApp` is provided in SamplerPostprocessorTransfer."
[]

[require_stochastic_results]
  type = RunException
  input = require_stochastic_results.i
  expect_err = "The object prescribed by the 'to_vector_postprocessor' parameter in data must be a 'StochasticResults' object"
  requirement = "The system shall produce an error if the 'result' object in 'SamplerPostprocessorTransfer' is not a 'StochasticResults object'."
[]

[sobol_from_multiapp]
  type = CSVDiff
  input = master.i
  csvdiff = 'master_out_storage_0001.csv master_out_storage_0002.csv master_out_storage_0003.csv master_out_storage_0004.csv master_out_storage_0005.csv'

  requirement = "The system shall support the ability to transfer a single value from each sub-application for a set of stochastic data."
[]

[invalid_sub_pp_name]
  type = RunException
  input = master.i
  cli_args = Transfers/data/from_postprocessor=wrong
  expect_err = "Unknown postprocesssor name 'wrong' on sub-application 'sub'"

  requirement = "The system shall error if the supplied name is invalid when attempting to
                  transfer a single value from a sub-application."

[]

[abort]
  type = RunException
  input = cartesian_diverge.i
  cli_args = 'MultiApps/sub/ignore_solve_not_converge=false
                  sub0:Executioner/error_on_dtmin=true'
  expect_err = "Solve failed and timestep already at or below dtmin, cannot continue!"

  detail = "abort run, "
[]

[last_computed]
  type = CSVDiff
  input = cartesian_diverge.i
  csvdiff = 'cartesian_diverge_out_storage_0001.csv cartesian_diverge_out_storage_0002.csv'

  detail = "transfer last computed postprocessor value, "
[]

[nan]
  type = CheckFiles
  input = cartesian_diverge.i
  cli_args = 'Transfers/data/keep_solve_fail_value=false
                  Outputs/file_base=cartesian_diverge_nan'
  check_files = 'cartesian_diverge_nan_storage_0001.csv'
  file_expect_out = 'data:avg\nnan\n0\.269307.*\n0\.269307.*\n0\.269307.*\n0\.269307.*\n'
  detail = "or transfer NaN."
[]

[normal]
  type = JSONDiff
  input = main.i
  jsondiff = main_out.json

  detail = "in normal mode, "
[]

[batch]
  type = JSONDiff
  input = main_batch.i
  jsondiff = 'main_batch_out.json'

  detail = 'in batch mode, '
[]

[distributed]
  type = JSONDiff
  input = main.i
  cli_args = 'Reporters/storage/parallel_type=DISTRIBUTED Outputs/file_base=main_parallel'
  jsondiff = 'main_parallel.json main_parallel.json.1 main_parallel.json.2 main_parallel.json.3'
  min_parallel = 4
  max_parallel = 4

  detail = 'with distributed output, '
[]

[more_cpus]
  type = JSONDiff
  input = main_small.i
  jsondiff = 'main_small_out.json'
  min_parallel = 4

  detail = 'with more processors than samples, '
[]

[invalid_type]
  type = RunException
  input = main.i
  cli_args = "Transfers/data/from_reporter='mesh/sidesets'"
  expect_err = 'Reporter value mesh/sidesets is of unsupported type'

  detail = 'and error if transferring unsupported type.'
[]

[multiapp_type]
  type = RunException
  input = master_multiapp_type_error.i
  expect_err = "The 'multi_app' parameter must provide either a 'SamplerTransientMultiApp' or 'SamplerFullSolveMultiApp' object."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if neither a 'SamplerTransientMultiApp' nor `SamplerFullSolveMultiApp` is provided in SamplerParameterTransfer."
[]

[control_missing]
  type = RunException
  input = master_missing_control.i
  design = 'SamplerTransientMultiApp.md'
  expect_err = "The sub-application \(sub\) does not contain a Control object with the name 'stochastic'."
  requirement = "The system shall produce an error if the sampler sub-application does not contain a Control object with the name 'stochastic'."
[]

[wrong_control]
  type = RunException
  input = master_wrong_control.i
  expect_err = "The sub-application \(sub\) Control object for the 'to_control' parameter must be of type 'SamplerReceiver'."
  design = 'SamplerTransientMultiApp.md'
  requirement = "The system shall produce an error if the sampler sub-application does not have a correct Control object with 'to_control' parameter being 'SamplerReceiver' type."
[]

[num_parameters_wrong]
  type = RunException
  input = master_num_parameters_wrong.i
  expect_err = "The supplied vector of Real values is not sized correctly, the Real parameter 'BCs/right/value requires a value but no more values are available in the supplied values which have a size of 2."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if supplied vector of real values is not sized correctly within the SamplerParameterTransfer object."
[]

[sampler_mismatch]
  type = RunException
  input = master_transfer_wrong_sampler.i
  expect_err = "The supplied 'multi_app' must have the same Sampler object as this Transfer."
  design = 'SamplerParameterTransfer.md'
  requirement = "The system shall produce an error if sampling method differs between the sub-application and the associated sub-application data transfer."
[]

[monte_carlo]
  type = CSVDiff
  input = monte_carlo.i
  csvdiff = 'monte_carlo_out_sub0.csv monte_carlo_out_sub1.csv monte_carlo_out_sub2.csv monte_carlo_out_sub3.csv monte_carlo_out_sub4.csv'

  detail = 'using a Monte Carlo and'
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_sub00.csv sobol_out_sub01.csv sobol_out_sub02.csv sobol_out_sub03.csv sobol_out_sub04.csv sobol_out_sub05.csv sobol_out_sub06.csv sobol_out_sub07.csv sobol_out_sub08.csv sobol_out_sub09.csv sobol_out_sub10.csv sobol_out_sub11.csv sobol_out_sub12.csv sobol_out_sub13.csv sobol_out_sub14.csv sobol_out_sub15.csv sobol_out_sub16.csv sobol_out_sub17.csv'

  detail = 'Sobol sampling scheme.'
[]

[monte_carlo]
  type = CSVDiff
  input = master.i
  csvdiff = 'master_out_sub0.csv master_out_sub1.csv master_out_sub2.csv master_out_sub3.csv master_out_sub4.csv'

  requirement = "The system shall be capable of transferring vector data to sub-applications for each row of the stochastic data."

[]

[not_enough_data]
  type = RunException
  input = master_not_enough_data.i
  expect_err = "The supplied vector of Real values is not sized correctly, the std::vector<Real> parameter 'Materials/mat/prop_values requires 2 values but only 1 are available in the supplied vector."

  detail = "is not sized correctly for stochastic data,"
[]

[size_mismatch]
  type = RunException
  input = master_size_mismatch.i
  expect_err = "The std::vector<Real> parameters being controlled must all be the same size:"

  detail = "is not sized uniformily across sub-applications,"
[]

[invalid_name]
  type = RunException
  input = master.i
  cli_args = "Transfers/sub/parameters=foo :Outputs/csv=false"
  expect_err = "The desired parameter 'foo' was not located for the 'stochastic' object, it either does not exist or has not been declared as controllable."

  detail = "if the vector parameter does not exist, and"
[]

[extra_data]
  type = RunException
  input = master.i
  cli_args = "Transfers/sub/parameters='BCs/left/value' :Outputs/csv=false"
  expect_err = "The number of values supplied \(4\) does not match the number of values consumed by setting parameter values \(1\)."

  detail = "if the sub-application does not consume all of the supplied data."
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = 'sobol_out_sub00.csv sobol_out_sub01.csv sobol_out_sub02.csv sobol_out_sub03.csv sobol_out_sub04.csv sobol_out_sub05.csv sobol_out_sub06.csv sobol_out_sub07.csv sobol_out_sub08.csv sobol_out_sub09.csv sobol_out_sub10.csv sobol_out_sub11.csv sobol_out_sub12.csv sobol_out_sub13.csv sobol_out_sub14.csv sobol_out_sub15.csv sobol_out_sub16.csv sobol_out_sub17.csv'

  requirement = "The system shall support the creation of a sub-application for each row sampled data generated from a Sobol scheme."
[]

[multiple]
  type = CSVDiff
  input = master.i
  csvdiff = master_out_storage_0002.csv

  requirement = "The system shall support the collection of stochastic data from multiple sub-applications."
[]

[get_global_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_global_samples'
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'get global matrix;'
[]

[get_local_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples'
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'get local matrix;'
[]

[get_next_local_row]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row'
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'get next local row;'
[]

[get_local_samples]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_local_samples  VectorPostprocessors/data/parallel_type=distributed'
  csvdiff = 'sampler_data_out_data_0001.csv.0 sampler_data_out_data_0001.csv.1 sampler_data_out_data_0001.csv.2'
  min_parallel = 3
  max_parallel = 3
  detail = 'get local matrix;'
[]

[get_next_local_row]
  type = CSVDiff
  input = sampler_data.i
  cli_args = 'VectorPostprocessors/data/sampler_method=get_next_local_row  VectorPostprocessors/data/parallel_type=distributed'
  csvdiff = 'sampler_data_out_data_0001.csv.0 sampler_data_out_data_0001.csv.1 sampler_data_out_data_0001.csv.2'
  min_parallel = 3
  max_parallel = 3
  detail = 'get next local row;'
[]

[one]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist'"
  csvdiff = sampler_data_out_data_0001.csv
  detail = 'one column;'
[]

[multi]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist dist dist' Outputs/file_base=multi_col"
  csvdiff = multi_col_data_0001.csv
  detail = 'multiple columns;'
[]

[lots]
  type = CSVDiff
  input = sampler_data.i
  cli_args = "Samplers/sample/distributions='dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist dist' Outputs/file_base=lots_col"
  csvdiff = lots_col_data_0001.csv
  detail = 'large number of columns;'
[]

[sobol]
  type = CSVDiff
  input = sobol.i
  csvdiff = sobol_out_sobol_0002.csv
  allow_test_objects = true

  requirement = "The system shall support the ability to compute first, second, and total-effect Sobol sensitivity indices."
[]

[sobol_bootstrap]
  type = CSVDiff
  input = sobol_bootstrap.i
  csvdiff = sobol_bootstrap_out_sobol_0002.csv
  allow_test_objects = true

  requirement = "The system shall support the ability to compute confidence intervals on Sobol sensitivity indices."
[]

[replicated]
  type = CSVDiff
  input = master.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=replicated_out'
  csvdiff = replicated_out_storage_0002.csv

  detail = "replicated on all processors and"
[]

[distributed]
  type = CSVDiff
  input = master.i
  min_parallel = 3
  max_parallel = 3
  cli_args = 'VectorPostprocessors/storage/parallel_type=DISTRIBUTED Outputs/file_base=distributed_out'
  csvdiff = 'distributed_out_storage_0002.csv.0 distributed_out_storage_0002.csv.1 distributed_out_storage_0002.csv.2'

  detail = "distributed across many."
[]

[custom]
  type = CSVDiff
  input = master.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=prefix_out Transfers/data/prefix=prefix'
  csvdiff = prefix_out_storage_0002.csv

  detail = "with custom prefix and"
[]

[none]
  type = CSVDiff
  input = master.i
  cli_args = 'VectorPostprocessors/storage/parallel_type=REPLICATED Outputs/file_base=no_prefix_out Transfers/data/prefix='
  csvdiff = no_prefix_out_storage_0002.csv

  detail = "without a prefix."
[]

[complete]
  type = CSVDiff
  input = master.i
  csvdiff = master_out_storage.csv

  detail = "can be appended into a single data set or"
[]

[time]
  type = CSVDiff
  input = master.i
  cli_args = VectorPostprocessors/storage/contains_complete_history=false
  csvdiff = 'master_out_storage_0000.csv master_out_storage_0001.csv master_out_storage_0002.csv'

  detail = "or contain a single file per timestep."
[]