Repository navigation
Expand file tree
/
Copy pathtesting.html
More file actions
262 lines (260 loc) · 20.6 KB
/
Copy pathtesting.html
File metadata and controls
262 lines (260 loc) · 20.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
<!-- HTML header for doxygen 1.9.1-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-SY496B9L99"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-SY496B9L99');
</script>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.16.1"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>MFC: Testing</title>
<meta name="description" content="Testing — MFC documentation. Open-source exascale multiphase flow solver." />
<meta name="keywords" content="exascale, fluid dynamics, cfd, computational fluid dynamics, compressible, hpc, bryngelson, colonius, subgrid, multiphase, frontier, summit, el capitan, aurora, amd gpu, gpu, nvidia"/>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="dynsections.js"></script>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="navtreedata.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript" src="cookie.js"></script>
<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="search/searchdata.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
extensions: ["tex2jax.js", "TeX/AMSmath.js", "TeX/AMSsymbols.js"],
jax: ["input/TeX","output/HTML-CSS"],
});
// This file is set as MATHJAX_CODEFILE in the Doxyfile. It configures how
// MathJax renders expressions in Markdown so that it is consistent with GitHub.
MathJax.Hub.Config({
extensions: ["tex2jax.js"],
jax: ["input/TeX", "output/HTML-CSS"],
tex2jax: {
inlineMath: [ ['$', '$'], ["\\(","\\)"] ],
displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
processEscapes: true,
ignoreClass: "line" // Ignore code blocks: https://web.archive.org/web/20120430100225/http://www.mathjax.org/docs/1.1/options/tex2jax.html
},
"HTML-CSS": {
fonts: ["TeX"]
}
});
</script>
<script type="text/javascript" async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js"></script>
<link href="doxygen.css" rel="stylesheet" type="text/css" />
<link rel="shortcut icon" href="icon.ico" type="image/x-icon" />
<link href="doxygen-awesome.css" rel="stylesheet" type="text/css"/>
<link href="doxygen-awesome-sidebar-only.css" rel="stylesheet" type="text/css"/>
<link href="custom.css" rel="stylesheet" type="text/css"/>
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr style="height: 56px;">
<td id="projectlogo"><img alt="Logo" src="icon.ico"/></td>
<td id="projectalign" style="padding-left: 0.5em;">
<div id="projectname">MFC
</div>
<div id="projectbrief">Exascale flow solver</div>
</td>
</tr>
</tbody>
</table>
</div>
<!-- Cross-navigation injected into sidebar via script below -->
<script>
document.addEventListener('DOMContentLoaded', function() {
var nav = document.createElement('div');
nav.id = 'mfc-nav';
var items = [
['../documentation/index.html', 'documentation', 'User Guide'],
['../api/index.html', 'api', 'API Documentation']
];
var path = window.location.pathname;
var apiPaths = ['/api/', '/pre_process/', '/simulation/', '/post_process/'];
for (var i = 0; i < items.length; i++) {
var a = document.createElement('a');
a.href = items[i][0];
a.textContent = items[i][2];
if (items[i][1] === 'api') {
for (var j = 0; j < apiPaths.length; j++) {
if (path.indexOf(apiPaths[j]) !== -1) { a.className = 'active'; break; }
}
} else {
if (path.indexOf('/' + items[i][1] + '/') !== -1) a.className = 'active';
}
nav.appendChild(a);
}
var sideNav = document.getElementById('side-nav');
if (sideNav) sideNav.insertBefore(nav, sideNav.firstChild);
});
</script>
<!-- end header part -->
<!-- Generated by Doxygen 1.16.1 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search/",'.html');
</script>
<script type="text/javascript">
$(function() { codefold.init(); });
</script>
<script type="text/javascript" src="menudata.js"></script>
<script type="text/javascript" src="menu.js"></script>
<script type="text/javascript">
$(function() {
initMenu('',true,false,'search.php','Search',true);
$(function() { init_search(); });
});
</script>
<div id="main-nav"></div>
</div><!-- top -->
<div id="side-nav" class="ui-resizable side-nav-resizable">
<div id="nav-tree">
<div id="nav-tree-contents">
<div id="nav-sync" class="sync"></div>
</div>
</div>
<div id="splitbar" style="-moz-user-select:none;"
class="ui-resizable-handle">
</div>
</div>
<script type="text/javascript">
$(function(){initNavTree('testing.html','',''); });
</script>
<div id="container">
<div id="doc-content">
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
onmouseover="return searchBox.OnSearchSelectShow()"
onmouseout="return searchBox.OnSearchSelectHide()"
onkeydown="return searchBox.OnSearchSelectKey(event)">
</div>
<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<div id="MSearchResults">
<div class="SRPage">
<div id="SRIndex">
<div id="SRResults"></div>
<div class="SRStatus" id="Loading">Loading...</div>
<div class="SRStatus" id="Searching">Searching...</div>
<div class="SRStatus" id="NoMatches">No Matches</div>
</div>
</div>
</div>
</div>
<div><div class="header">
<div class="headertitle"><div class="title">Testing </div></div>
</div><!--header-->
<div class="contents">
<div class="textblock"><h2 class="doxsection"><a class="anchor" id="autotoc_md585"></a>
Testing</h2>
<p>To run MFC's test suite, run </p><div class="fragment"><div class="line">./mfc.sh test -j <thread count></div>
</div><!-- fragment --><p>It will generate and run test cases, comparing their output to previous runs from versions of MFC considered accurate. <em>golden files</em>, stored in the <span class="tt">tests/</span> directory contain this data, aggregating <span class="tt">.dat</span> files generated when running MFC. A test is considered passing when our error tolerances are met in order to maintain a high level of stability and accuracy. <span class="tt">./mfc.sh test</span> has the following unique options:</p><ul>
<li><span class="tt">-l</span> outputs the full list of tests</li>
<li><span class="tt">--from</span> (<span class="tt">-f)</span> and <span class="tt">--to</span> (<span class="tt">t</span>) restrict testing to a range of contiguous slugs</li>
<li><span class="tt">--only</span> (<span class="tt">-o</span>) restricts testing to a non-contiguous range of tests whose trace contains a given whole trace element (see <a class="el" href="#selection-and-execution-pitfalls" title="Selection and Execution Pitfalls">Selection and Execution Pitfalls</a> for the exact matching rules)</li>
<li><span class="tt">--test-all</span> (<span class="tt">a</span>) test post process and ensure the Silo database files are correct</li>
<li><span class="tt">--percent</span> (<span class="tt">%</span>) to specify a percentage of the test suite to select at random and test</li>
<li><span class="tt">--max-attempts</span> (<span class="tt">-m</span>) the maximum number of attempts to make on a test before considering it failed</li>
<li><span class="tt">--no-examples</span> skips the testing of cases in the examples folder</li>
<li><span class="tt">--no-chemistry</span> skips every case that uses chemistry (<span class="tt">chemistry = 'T'</span>), including reacting example cases</li>
<li><span class="tt">--no-build</span> runs against existing binaries without rebuilding. Some cases (chemistry, analytic initial conditions) need their own build, which <span class="tt">./mfc.sh build</span> does not produce; build everything the suite needs with <span class="tt">./mfc.sh test --dry-run <options></span>. If any required binary is missing, <span class="tt">--no-build</span> stops before running any case and lists what is missing.</li>
<li><span class="tt">--rdma-mpi</span> runs additional tests where RDMA MPI is enabled.</li>
</ul>
<p>To specify a computer, pass the <span class="tt">-c</span> flag to <span class="tt">./mfc.sh run</span> like so: </p><div class="fragment"><div class="line">./mfc.sh test -j <thread count> -- -c <computer name></div>
</div><!-- fragment --><p> where <span class="tt"><computer name></span> could be <span class="tt">phoenix</span> or any of the others in the <a href="https://github.com/MFlowCode/MFC/tree/master/toolchain/templates">templates</a>). You can create new templates with the appropriate run commands or omit this option. The use of <span class="tt">--</span> in the above command passes options to the <span class="tt">./mfc.sh run</span> command underlying the <span class="tt">./mfc.sh test</span>.</p>
<h3 class="doxsection"><a class="anchor" id="autotoc_md586"></a>
Creating Tests</h3>
<p>Creating and updating test cases can be done with the following command line arguments:</p><ul>
<li><span class="tt">--generate</span> to generate golden files for a new test case</li>
<li><span class="tt">--add-new-variables</span> to similar to <span class="tt">--generate</span>, but rather than generating a golden file from scratch, it generates a gold file with new variables for an updated test without changing the original golden file values.</li>
<li><span class="tt">--remove-old-tests</span> to remove the directories of tests that no longer exist</li>
</ul>
<p>It is recommended that a range be specified when generating golden files for new test cases, as described in the previous section, in an effort not to regenerate the golden files of existing test cases.</p>
<p>Adding a new test case can be done by modifying <a href="https://github.com/MFlowCode/MFC/tree/master/toolchain/mfc/test/cases.py">cases.py</a>. The function <span class="tt">list_cases</span> is responsible for generating the list of test cases. Loops and conditionals are used to vary parameters, whose defaults can be found in the <span class="tt">BASE_CFG</span> case object within <a href="https://github.com/MFlowCode/MFC/tree/master/toolchain/mfc/test/case.py">case.py</a>. The function operates on two variables:</p>
<ul>
<li><span class="tt">stack</span>: A stack that holds the variations to the default case parameters. By pushing and popping the stack inside loops and conditionals, it is easier to nest test case descriptions, as it holds the variations that are common to all future test cases within the same indentation level (in most scenarios).</li>
<li><span class="tt">cases</span>: A list that holds fully-formed <span class="tt">Case</span> objects, that will be returned at the end of the function.</li>
</ul>
<p>Internally a test case is described as: </p><div class="fragment"><div class="line"><span class="preprocessor">@dataclasses.dataclass(init=False)</span></div>
<div class="line"><span class="keyword">class </span>Case:</div>
<div class="line"> trace: str</div>
<div class="line"> params: dict</div>
<div class="line"> ppn: int</div>
</div><!-- fragment --><p>where:</p><ul>
<li>The <span class="tt">trace</span> is a string that contains a human-readable description of what parameters were varied, or more generally what the case is meant to test. <b>Each <span class="tt">trace</span> must be distinct.</b></li>
<li><span class="tt">params</span> is the fully resolved case dictionary, as would appear in a Python case input file.</li>
<li><span class="tt">ppn</span> is the number of processes per node to use when running the case.</li>
</ul>
<p>To illustrate, consider the following excerpt from <span class="tt">list_cases</span>:</p>
<div class="fragment"><div class="line"><span class="keywordflow">for</span> weno_order <span class="keywordflow">in</span> [3, 5]:</div>
<div class="line"> stack.push(f<span class="stringliteral">"weno_order={weno_order}"</span>, {<span class="stringliteral">'weno_order'</span>: weno_order})</div>
<div class="line"> </div>
<div class="line"> <span class="keywordflow">for</span> mapped_weno, mp_weno <span class="keywordflow">in</span> [(<span class="stringliteral">'F'</span>, <span class="stringliteral">'F'</span>), (<span class="stringliteral">'T'</span>, <span class="stringliteral">'F'</span>), (<span class="stringliteral">'F'</span>, <span class="stringliteral">'T'</span>)]:</div>
<div class="line"> stack.push(f<span class="stringliteral">"(mapped_weno={mapped_weno},mp_weno={mp_weno})"</span>, {</div>
<div class="line"> <span class="stringliteral">'mapped_weno'</span>: mapped_weno,</div>
<div class="line"> <span class="stringliteral">'mp_weno'</span>: mp_weno</div>
<div class="line"> })</div>
<div class="line"> </div>
<div class="line"> <span class="keywordflow">if</span> <span class="keywordflow">not</span> (mp_weno == <span class="stringliteral">'T'</span> <span class="keywordflow">and</span> weno_order != 5):</div>
<div class="line"> cases.append(define_case_d(stack, <span class="stringliteral">''</span>, {}))</div>
<div class="line"> </div>
<div class="line"> stack.pop()</div>
<div class="line"> </div>
<div class="line"> stack.pop()</div>
</div><!-- fragment --><p>When pushing to the stack or creating a new case with the <span class="tt">define_case_d</span> function, you must specify:</p><ul>
<li><span class="tt">stack</span>: The current stack.</li>
<li><span class="tt">trace</span>: A human-readable string describing what you are currently varying.</li>
<li><span class="tt">variations</span>: A Python dictionary with case parameter variations.</li>
<li>(Optional) <span class="tt">ppn</span>: The number of processes per node to use (default is 1).</li>
</ul>
<p>If a trace is empty (that is, the empty string <span class="tt">""</span>), it will not appear in the final trace, but any case parameter variations associated with it will still be applied.</p>
<p>Finally, the case is appended to the <span class="tt">cases</span> list, which will be returned by the <span class="tt">list_cases</span> function.</p>
<h3 class="doxsection"><a class="anchor" id="selection-and-execution-pitfalls"></a>
Selection and Execution Pitfalls</h3>
<p>Each of these fails quietly rather than loudly.</p>
<ul>
<li><b><span class="tt">--only</span> matches whole trace elements, not substrings</b>, and it ANDs labels while ORing UUIDs (<span class="tt">_filter_only</span> in <span class="tt">toolchain/mfc/test/test.py</span>). <span class="tt">--only bubbles</span> matches nothing, because the trace element is <span class="tt">Bubbles</span>; <span class="tt">--only low_Mach=1 low_Mach=2</span> asks for cases carrying both labels at once and also matches nothing. An empty selection then exits <b>143</b>, which reads like an external kill rather than an empty filter. Pass UUIDs when you want the union of several groups.</li>
<li><b><span class="tt">Chemistry</span> is the one label not read off the trace.</b> Any case with <span class="tt">chemistry='T'</span> answers to it, because it selects a <em>build</em> and not just a test: Frontier AMD's GPU lane compiles its chemistry binaries in a separate SLURM job invoked with <span class="tt">-o Chemistry</span> (<span class="tt">.github/workflows/common/build.sh</span>) and then tests with <span class="tt">--no-build</span>, so a chemistry case the filter misses is never compiled there and fails with a missing binary. Examples are auto-registered as <span class="tt"><dim> -> Example -> <dirname></span> and so can never carry the label by hand. If you add a chemistry case, you get this for free; do not re-add the label to a trace to compensate, since the UUID is a hash of the trace and renaming orphans the golden directory.</li>
<li><b>Sibling <span class="tt">define_case_d</span> calls at the same stack level are never combined.</b> Two switches that only matter together therefore get no effective coverage unless one is pushed onto the stack and the other defined beneath it — <span class="tt">avg_state=1</span>, for instance, is only read when <span class="tt">wave_speeds=2</span>. Check reachability before trusting that a flag is tested.</li>
<li><b><span class="tt">--no-build</span> silently runs whatever binary is already on disk</b>, including one built for a different configuration. Chemistry has its own configuration that a plain <span class="tt">./mfc.sh
build</span> never produces, so a <span class="tt">--no-build</span> run can report failures from stale binaries and hide real compile breaks. Run chemistry-touching sets without it.</li>
<li><b>A case that needs a build of its own is red on exactly one lane.</b> Anything that changes what <span class="tt">MFCTarget.get_slug</span> hashes — most often an analytic initial condition, i.e. a <span class="tt">patch_icpp</span> value written as an expression rather than a number — gives the case its own <span class="tt">case.fpp</span> and its own install directory. Most lanes pre-build every such variant with <span class="tt">./mfc.sh test --dry-run -a</span>, but Frontier AMD gpu-omp cannot afford to (each amdflang device link is ~1 h), so it pre-builds only the default and the chemistry configurations; the test job then runs <span class="tt">--no-build</span> and <span class="tt">srun</span> reports <span class="tt">execve(): .../pre_process: No such
file or directory</span>, which reads like a filesystem fault an hour and a half into the run. Prefer geometric variation to analytic (cuboid bounds are plain numbers), or skip the example via <span class="tt">casesToSkip</span>. <span class="tt">toolchain/mfc/lint_test_suite.py</span> enforces this in the lint gate, before any cluster job is queued.</li>
<li><b>Identify the newest binary by the binary's own mtime</b>, not by its install directory's: a stale configuration's directory can be newer than a fresh build's.</li>
<li><b>The pre-commit hook lives in the main repository's <span class="tt">.git/hooks/</span></b>, and git exports <span class="tt">GIT_DIR</span> there during a commit, so from a worktree the toolchain lint enumerates the other checkout and fails. Run <span class="tt">./mfc.sh precheck</span> by hand and commit with <span class="tt">--no-verify</span>.</li>
<li><b><span class="tt">/tmp</span> is node-local.</b> Scratch does not survive a compute-node change, and its absence is silence rather than an error. Keep patches and resource baselines on a shared filesystem.</li>
<li><b>An unexplained golden-file difference is a bug report, not noise to be regenerated away.</b> Regenerate only the affected tests.</li>
</ul>
<p>Tests are generated programmatically in <span class="tt">toolchain/mfc/test/cases.py</span>; a test's UUID is the CRC32 of its trace string, and <span class="tt">./mfc.sh test -l</span> lists every one.</p>
<h3 class="doxsection"><a class="anchor" id="autotoc_md587"></a>
Testing Post Process</h3>
<p>To test the post-processing code, append the <span class="tt">-a</span> or <span class="tt">--test-all</span> option: </p><div class="fragment"><div class="line">./mfc.sh test -a -j 8</div>
</div><!-- fragment --><p>This argument will re-run the test stack with <span class="tt">parallel_io='T'</span>, which generates silo_hdf5 files. It will also turn most write parameters (<span class="tt">*_wrt</span>) on. Then, it searches through the silo files using <span class="tt">h5dump</span> to ensure that there are no <span class="tt">NaN</span>s or <span class="tt">Infinity</span>s. Although adding this option does not guarantee that accurate <span class="tt">.silo</span> files are generated, it does ensure that the post-process code does not fail or produce malformed data.</p>
<div style="text-align:center; font-size:0.75rem; color:#888; padding:16px 0 0;">Page last updated: 2026-02-15</div> </div></div><!-- contents -->
</div><!-- PageDoc -->
</div><!-- doc-content -->
<div id="page-nav" class="page-nav-panel">
<div id="page-nav-resize-handle"></div>
<div id="page-nav-tree">
<div id="page-nav-contents">
</div><!-- page-nav-contents -->
</div><!-- page-nav-tree -->
</div><!-- page-nav -->
</div><!-- container -->
<!-- HTML footer for doxygen 1.9.1-->
<!-- start footer part -->
<div id="nav-path" class="navpath">
<ul></ul>
</div>
</body>
</html>

