1 # Copyright 2021-2022. The MBI project. All rights reserved.
2 # This program is free software; you can redistribute it and/or modify it under the terms of the license (GNU GPL).
15 def ensure_image(self, params="", dockerparams=""):
16 """Verify that this is executed from the right docker image, and complain if not."""
17 if os.path.exists("/MBI") or os.path.exists("trust_the_installation"):
18 print("This seems to be a MBI docker image. Good.")
20 print("Please run this script in a MBI docker image. Run these commands:")
21 print(" docker build -f Dockerfile -t mpi-bugs-initiative:latest . # Only the first time")
22 print(f" docker run -it --rm --name MIB --volume $(pwd):/MBI {dockerparams}mpi-bugs-initiative /MBI/MBI.py {params}")
25 def build(self, rootdir, cached=True):
26 """Rebuilds the tool binaries. By default, we try to reuse the existing build."""
27 print("Nothing to do to rebuild the tool binaries.")
29 def setup(self, rootdir):
31 Ensure that this tool (previously built) is usable in this environment: setup the PATH, etc.
32 This is called only once for all tests, from the logs directory.
36 def run(self, execcmd, filename, binary, num_id, timeout):
37 """Compile that test code and anaylse it with the Tool if needed (a cache system should be used)"""
42 Clean the results of all test runs: remove temp files and binaries.
43 This is called only once for all tests, from the logs directory.
47 def parse(self, cachefile):
48 """Read the result of a previous run from the cache, and compute the test outcome"""
51 # Associate all possible detailed outcome to a given error scope. Scopes must be sorted alphabetically.
53 # scope limited to one call
54 'InvalidBuffer':'AInvalidParam', 'InvalidCommunicator':'AInvalidParam', 'InvalidDatatype':'AInvalidParam', 'InvalidRoot':'AInvalidParam', 'InvalidTag':'AInvalidParam', 'InvalidWindow':'AInvalidParam', 'InvalidOperator':'AInvalidParam', 'InvalidOtherArg':'AInvalidParam', 'ActualDatatype':'AInvalidParam',
55 'InvalidSrcDest':'AInvalidParam',
57 # 'OutOfInitFini':'BInitFini',
58 'CommunicatorLeak':'BResLeak', 'DatatypeLeak':'BResLeak', 'GroupLeak':'BResLeak', 'OperatorLeak':'BResLeak', 'TypeLeak':'BResLeak', 'RequestLeak':'BResLeak',
59 'MissingStart':'BReqLifecycle', 'MissingWait':'BReqLifecycle',
60 'LocalConcurrency':'BLocalConcurrency',
62 'CallMatching':'DMatch',
63 'CommunicatorMatching':'CMatch', 'DatatypeMatching':'CMatch', 'OperatorMatching':'CMatch', 'RootMatching':'CMatch', 'TagMatching':'CMatch',
64 'MessageRace':'DRace',
66 'GlobalConcurrency':'DGlobalConcurrency',
68 'BufferingHazard':'EBufferingHazard',
72 'AInvalidParam':'single call',
73 'BResLeak':'single process',
74 # 'BInitFini':'single process',
75 'BReqLifecycle':'single process',
76 'BLocalConcurrency':'single process',
77 'CMatch':'multi-processes',
78 'DRace':'multi-processes',
79 'DMatch':'multi-processes',
80 'DGlobalConcurrency':'multi-processes',
81 'EBufferingHazard':'system',
82 'FOK':'correct executions'
86 'AInvalidParam':'Invalid parameter',
87 'BResLeak':'Resource leak',
88 # 'BInitFini':'MPI call before initialization/after finalization',
89 'BReqLifecycle':'Request lifecycle',
90 'BLocalConcurrency':'Local concurrency',
91 'CMatch':'Parameter matching',
92 'DMatch':"Call ordering",
93 'DRace':'Message race',
94 'DGlobalConcurrency':'Global concurrency',
95 'EBufferingHazard':'Buffering hazard',
96 'FOK':"Correct execution",
98 'aislinn':'Aislinn', 'civl':'CIVL', 'hermes':'Hermes', 'isp':'ISP', 'itac':'ITAC', 'simgrid':'Mc SimGrid', 'smpi':'SMPI', 'smpivg':'SMPI+VG', 'mpisv':'MPI-SV', 'must':'MUST', 'parcoach':'PARCOACH'
101 def parse_one_code(filename):
103 Reads the header of the provided filename, and extract a list of todo item, each of them being a (cmd, expect, test_num) tupple.
104 The test_num is useful to build a log file containing both the binary and the test_num, when there is more than one test in the same binary.
108 with open(filename, "r") as input_file:
109 state = 0 # 0: before header; 1: in header; 2; after header
111 for line in input_file:
112 if re.match(".*BEGIN_MBI_TESTS.*", line):
116 raise ValueError(f"MBI_TESTS header appears a second time at line {line_num}: \n{line}")
117 elif re.match(".*END_MBI_TESTS.*", line):
121 raise ValueError(f"Unexpected end of MBI_TESTS header at line {line_num}: \n{line}")
122 if state == 1 and re.match(r'\s+\$ ?.*', line):
123 m = re.match(r'\s+\$ ?(.*)', line)
125 nextline = next(input_file)
127 if re.match('[ |]*OK *', nextline):
130 m = re.match('[ |]*ERROR: *(.*)', nextline)
133 f"\n{filename}:{line_num}: MBI parse error: Test not followed by a proper 'ERROR' line:\n{line}{nextline}")
136 if detail not in possible_details:
138 f"\n{filename}:{line_num}: MBI parse error: Detailled outcome {detail} is not one of the allowed ones.")
139 test = {'filename': filename, 'id': test_num, 'cmd': cmd, 'expect': expect, 'detail': detail}
140 res.append(test.copy())
145 raise ValueError(f"MBI_TESTS header not found in file '{filename}'.")
147 raise ValueError(f"MBI_TESTS header not properly ended in file '{filename}'.")
150 raise ValueError(f"No test found in {filename}. Please fix it.")
153 def categorize(tool, toolname, test_id, expected):
154 outcome = tool.parse(test_id)
156 if not os.path.exists(f'{test_id}.elapsed') and not os.path.exists(f'logs/{toolname}/{test_id}.elapsed'):
157 if outcome == 'failure':
160 raise ValueError(f"Invalid test result: {test_id}.txt exists but not {test_id}.elapsed")
162 with open(f'{test_id}.elapsed' if os.path.exists(f'{test_id}.elapsed') else f'logs/{toolname}/{test_id}.elapsed', 'r') as infile:
163 elapsed = infile.read()
165 # Properly categorize this run
166 if outcome == 'timeout':
167 res_category = 'timeout'
169 diagnostic = 'hard timeout'
171 diagnostic = f'timeout after {elapsed} sec'
172 elif outcome == 'failure' or outcome == 'segfault':
173 res_category = 'failure'
174 diagnostic = 'tool error, or test not run'
175 elif outcome == 'UNIMPLEMENTED':
176 res_category = 'unimplemented'
177 diagnostic = 'coverage issue'
178 elif outcome == 'other':
179 res_category = 'other'
180 diagnostic = 'inconclusive run'
181 elif expected == 'OK':
183 res_category = 'TRUE_NEG'
184 diagnostic = 'correctly reported no error'
186 res_category = 'FALSE_POS'
187 diagnostic = 'reported an error in a correct code'
188 elif expected == 'ERROR':
190 res_category = 'FALSE_NEG'
191 diagnostic = 'failed to detect an error'
193 res_category = 'TRUE_POS'
194 diagnostic = 'correctly detected an error'
196 raise ValueError(f"Unexpected expectation: {expected} (must be OK or ERROR)")
198 return (res_category, elapsed, diagnostic, outcome)
201 def run_cmd(buildcmd, execcmd, cachefile, filename, binary, timeout, batchinfo, read_line_lambda=None):
203 Runs the test on need. Returns True if the test was ran, and False if it was cached.
205 The result is cached if possible, and the test is rerun only if the `test.txt` (containing the tool output) or the `test.elapsed` (containing the timing info) do not exist, or if `test.md5sum` (containing the md5sum of the code to compile) does not match.
208 - buildcmd and execcmd are shell commands to run. buildcmd can be any shell line (incuding && groups), but execcmd must be a single binary to run.
209 - cachefile is the name of the test
210 - filename is the source file containing the code
211 - binary the file name in which to compile the code
212 - batchinfo: something like "1/1" to say that this run is the only batch (see -b parameter of MBI.py)
213 - read_line_lambda: a lambda to which each line of the tool output is feed ASAP. It allows MUST to interrupt the execution when a deadlock is reported.
215 if os.path.exists(f'{cachefile}.txt') and os.path.exists(f'{cachefile}.elapsed') and os.path.exists(f'{cachefile}.md5sum'):
216 hash_md5 = hashlib.md5()
217 with open(filename, 'rb') as sourcefile:
218 for chunk in iter(lambda: sourcefile.read(4096), b""):
219 hash_md5.update(chunk)
220 newdigest = hash_md5.hexdigest()
221 with open(f'{cachefile}.md5sum', 'r') as md5file:
222 olddigest = md5file.read()
223 #print(f'Old digest: {olddigest}; New digest: {newdigest}')
224 if olddigest == newdigest:
225 print(f" (result cached -- digest: {olddigest})")
227 os.remove(f'{cachefile}.txt')
229 print(f"Wait up to {timeout} seconds")
231 start_time = time.time()
233 output = f"No need to compile {binary}.c (batchinfo:{batchinfo})\n\n"
235 output = f"Compiling {binary}.c (batchinfo:{batchinfo})\n\n"
236 output += f"$ {buildcmd}\n"
238 compil = subprocess.run(buildcmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
239 if compil.stdout is not None:
240 output += str(compil.stdout, errors='replace')
241 if compil.returncode != 0:
242 output += f"Compilation of {binary}.c raised an error (retcode: {compil.returncode})"
243 for line in (output.split('\n')):
244 print(f"| {line}", file=sys.stderr)
245 with open(f'{cachefile}.elapsed', 'w') as outfile:
246 outfile.write(str(time.time() - start_time))
247 with open(f'{cachefile}.txt', 'w') as outfile:
248 outfile.write(output)
251 output += f"\n\nExecuting the command\n $ {execcmd}\n"
252 for line in (output.split('\n')):
253 print(f"| {line}", file=sys.stderr)
255 # We run the subprocess and parse its output line by line, so that we can kill it as soon as it detects a timeout
256 process = subprocess.Popen(shlex.split(execcmd), stdout=subprocess.PIPE,
257 stderr=subprocess.STDOUT, preexec_fn=os.setsid)
258 poll_obj = select.poll()
259 poll_obj.register(process.stdout, select.POLLIN)
262 pgid = os.getpgid(pid) # We need that to forcefully kill subprocesses when leaving
265 if poll_obj.poll(5): # Something to read? Do check the timeout status every 5 sec if not
266 line = process.stdout.readline()
267 # From byte array to string, replacing non-representable strings with question marks
268 line = str(line, errors='replace')
269 output = output + line
270 print(f"| {line}", end='', file=sys.stderr)
271 if read_line_lambda != None:
272 read_line_lambda(line, process)
273 if time.time() - start_time > timeout:
275 with open(f'{cachefile}.timeout', 'w') as outfile:
276 outfile.write(f'{time.time() - start_time} seconds')
278 if process.poll() is not None: # The subprocess ended. Grab all existing output, and return
280 while line != None and line != '':
281 line = process.stdout.readline()
283 # From byte array to string, replacing non-representable strings with question marks
284 line = str(line, errors='replace')
285 output = output + line
286 print(f"| {line}", end='', file=sys.stderr)
290 # We want to clean all forked processes in all cases, no matter whether they are still running (timeout) or supposed to be off. The runners easily get clogged with zombies :(
292 os.killpg(pgid, signal.SIGTERM) # Terminate all forked processes, to make sure it's clean whatever the tool does
293 process.terminate() # No op if it's already stopped but useful on timeouts
294 time.sleep(0.2) # allow some time for the tool to finish its childs
295 os.killpg(pgid, signal.SIGKILL) # Finish 'em all, manually
296 os.kill(pid, signal.SIGKILL) # die! die! die!
297 except ProcessLookupError:
298 pass # OK, it's gone now
300 elapsed = time.time() - start_time
304 status = f"Command killed by signal {-rc}, elapsed time: {elapsed}\n"
306 status = f"Command return code: {rc}, elapsed time: {elapsed}\n"
310 with open(f'{cachefile}.elapsed', 'w') as outfile:
311 outfile.write(str(elapsed))
313 with open(f'{cachefile}.txt', 'w') as outfile:
314 outfile.write(output)
315 with open(f'{cachefile}.md5sum', 'w') as outfile:
316 hashed = hashlib.md5()
317 with open(filename, 'rb') as sourcefile:
318 for chunk in iter(lambda: sourcefile.read(4096), b""):
320 outfile.write(hashed.hexdigest())