Merge pull request #7197 from jon-turney/test-output-check-mandatory

Make the expected output check mandatory for failing-meson and warning-meson tests

run_project_tests.py

@@ -603,18 +603,16 @@ def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args,
 
     return testresult
 
-def gather_tests(testdir: Path) -> T.List[TestDef]:
+def gather_tests(testdir: Path, stdout_mandatory: bool) -> T.List[TestDef]:
     tests = [t.name for t in testdir.iterdir() if t.is_dir()]
     tests = [t for t in tests if not t.startswith('.')]  # Filter non-tests files (dot files, etc)
     test_defs = [TestDef(testdir / t, None, []) for t in tests]
     all_tests = []  # type: T.List[TestDef]
     for t in test_defs:
+        test_def = {}
         test_def_file = t.path / 'test.json'
-        if not test_def_file.is_file():
-            all_tests += [t]
-            continue
-
-        test_def = json.loads(test_def_file.read_text())
+        if test_def_file.is_file():
+            test_def = json.loads(test_def_file.read_text())
 
         # Handle additional environment variables
         env = {}  # type: T.Dict[str, str]
@@ -632,6 +630,8 @@ def gather_tests(testdir: Path) -> T.List[TestDef]:
 
         # Handle expected output
         stdout = test_def.get('stdout', [])
+        if stdout_mandatory and not stdout:
+            raise RuntimeError("{} must contain a non-empty stdout key".format(test_def_file))
 
         # Handle the do_not_set_opts list
         do_not_set_opts = test_def.get('do_not_set_opts', [])  # type: T.List[str]
@@ -906,45 +906,50 @@ def detect_tests_to_run(only: T.List[str], use_tmp: bool) -> T.List[T.Tuple[str,
                        shutil.which('pgfortran') or
                        shutil.which('ifort'))
 
-    # Name, subdirectory, skip condition.
+    class TestCategory:
+        def __init__(self, category: str, subdir: str, skip: bool = False, stdout_mandatory: bool = False):
+            self.category = category                  # category name
+            self.subdir = subdir                      # subdirectory
+            self.skip = skip                          # skip condition
+            self.stdout_mandatory = stdout_mandatory  # expected stdout is mandatory for tests in this categroy
+
     all_tests = [
-        ('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)),
-        ('common', 'common', False),
-        ('warning-meson', 'warning', False),
-        ('failing-meson', 'failing', False),
-        ('failing-build', 'failing build', False),
-        ('failing-test',  'failing test', False),
-        ('keyval', 'keyval', False),
-
-        ('platform-osx', 'osx', not mesonlib.is_osx()),
-        ('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
-        ('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
-
-        ('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()),
-        ('C#', 'csharp', skip_csharp(backend)),
-        ('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
-        ('rust', 'rust', should_skip_rust(backend)),
-        ('d', 'd', backend is not Backend.ninja or not have_d_compiler()),
-        ('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)),
-        ('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)),
-        ('fortran', 'fortran', skip_fortran or backend != Backend.ninja),
-        ('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
+        TestCategory('cmake', 'cmake', not shutil.which('cmake') or (os.environ.get('compiler') == 'msvc2015' and under_ci)),
+        TestCategory('common', 'common'),
+        TestCategory('warning-meson', 'warning', stdout_mandatory=True),
+        TestCategory('failing-meson', 'failing', stdout_mandatory=True),
+        TestCategory('failing-build', 'failing build'),
+        TestCategory('failing-test',  'failing test'),
+        TestCategory('keyval', 'keyval'),
+        TestCategory('platform-osx', 'osx', not mesonlib.is_osx()),
+        TestCategory('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
+        TestCategory('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
+        TestCategory('java', 'java', backend is not Backend.ninja or mesonlib.is_osx() or not have_java()),
+        TestCategory('C#', 'csharp', skip_csharp(backend)),
+        TestCategory('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
+        TestCategory('rust', 'rust', should_skip_rust(backend)),
+        TestCategory('d', 'd', backend is not Backend.ninja or not have_d_compiler()),
+        TestCategory('objective c', 'objc', backend not in (Backend.ninja, Backend.xcode) or not have_objc_compiler(options.use_tmpdir)),
+        TestCategory('objective c++', 'objcpp', backend not in (Backend.ninja, Backend.xcode) or not have_objcpp_compiler(options.use_tmpdir)),
+        TestCategory('fortran', 'fortran', skip_fortran or backend != Backend.ninja),
+        TestCategory('swift', 'swift', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('swiftc')),
         # CUDA tests on Windows: use Ninja backend:  python run_project_tests.py --only cuda --backend ninja
-        ('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')),
-        ('python3', 'python3', backend is not Backend.ninja),
-        ('python', 'python', backend is not Backend.ninja),
-        ('fpga', 'fpga', shutil.which('yosys') is None),
-        ('frameworks', 'frameworks', False),
-        ('nasm', 'nasm', False),
-        ('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja),
+        TestCategory('cuda', 'cuda', backend not in (Backend.ninja, Backend.xcode) or not shutil.which('nvcc')),
+        TestCategory('python3', 'python3', backend is not Backend.ninja),
+        TestCategory('python', 'python', backend is not Backend.ninja),
+        TestCategory('fpga', 'fpga', shutil.which('yosys') is None),
+        TestCategory('frameworks', 'frameworks'),
+        TestCategory('nasm', 'nasm'),
+        TestCategory('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja),
     ]
 
-    names = [t[0] for t in all_tests]
-    assert names == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests names'
+    categories = [t.category for t in all_tests]
+    assert categories == ALL_TESTS, 'argparse("--only", choices=ALL_TESTS) need to be updated to match all_tests categories'
+
     if only:
-        ind = [names.index(o) for o in only]
-        all_tests = [all_tests[i] for i in ind]
-    gathered_tests = [(name, gather_tests(Path('test cases', subdir)), skip) for name, subdir, skip in all_tests]
+        all_tests = [t for t in all_tests if t.category in only]
+
+    gathered_tests = [(t.category, gather_tests(Path('test cases', t.subdir), t.stdout_mandatory), t.skip) for t in all_tests]
     return gathered_tests
 
 def run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],

test cases/failing/100 fallback consistency/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/100 fallback consistency/meson.build:7:0: ERROR: Inconsistency: Subproject has overridden the dependency with another variable than 'dep2'"
+    }
+  ]
+}

test cases/failing/101 no native compiler/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/101 no native compiler/meson.build:12:0: ERROR: No host machine compiler for \"main.c\""
+    }
+  ]
+}

test cases/failing/102 subdir parse error/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/102 subdir parse error/subdir/meson.build:1:0: ERROR: Plusassignment target must be an id."
+    }
+  ]
+}

test cases/failing/103 invalid option file/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/103 invalid option file/meson_options.txt:1:0: ERROR: lexer"
+    }
+  ]
+}

test cases/failing/104 no lang/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/104 no lang/meson.build:2:0: ERROR: No host machine compiler for \"main.c\""
+    }
+  ]
+}

test cases/failing/105 no glib-compile-resources/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/105 no glib-compile-resources/meson.build:8:0: ERROR: Could not execute glib-compile-resources."
+    }
+  ]
+}

test cases/failing/36 pkgconfig dependency impossible conditions/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/36 pkgconfig dependency impossible conditions/meson.build:7:0: ERROR: Dependency 'zlib' was already checked and was not found"
+    }
+  ]
+}

test cases/failing/67 subproj different versions/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/67 subproj different versions/subprojects/b/meson.build:3:0: ERROR: Dependency 'c' was already checked and was not found"
+    }
+  ]
+}

test cases/failing/84 gtest dependency with version/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/84 gtest dependency with version/meson.build:8:0: ERROR: Dependency 'gtest' was already checked and was not found"
+    }
+  ]
+}

test cases/failing/98 fallback consistency/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/98 fallback consistency/meson.build:4:0: ERROR: Inconsistency: Subproject has overridden the dependency with another variable than 'dep2'"
+    }
+  ]
+}