Merge pull request #6620 from jon-turney/test-output-check

Add a mechanism for validating meson output in tests

data/test.schema.json

@@ -1,5 +1,6 @@
 {
   "type": "object",
+  "additionalProperties": false,
   "properties": {
     "env": {
       "type": "object",
@@ -100,6 +101,30 @@
           "prefix"
         ]
       }
+    },
+    "tools": {
+      "type": "object"
+    },
+    "stdout": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "line": {
+            "type": "string"
+          },
+          "match": {
+            "type": "string",
+            "enum": [
+              "literal",
+              "re"
+            ]
+          }
+        },
+        "required": [
+          "line"
+        ]
+      }
     }
   }
 }

docs/markdown/Contributing.md

@@ -329,10 +329,31 @@ Currently supported values are:
 
 #### tools
 
-This section specifies a list of tool requirements in a simple key-value format.
+This section specifies a dict of tool requirements in a simple key-value format.
 If a tool is specified, it has to be present in the environment, and the version
-requirement must be fulfilled match. Otherwise, the entire test is skipped
-(including every element in the test matrix).
+requirement must be fulfilled. Otherwise, the entire test is skipped (including
+every element in the test matrix).
+
+#### stdout
+
+The `stdout` key contains a list of dicts, describing the expected stdout.
+
+Each dict contains the following keys:
+
+- `line`
+- `match` (optional)
+
+Each item in the list is matched, in order, against the remaining actual stdout
+lines, after any previous matches. If the actual stdout is exhausted before
+every item in the list is matched, the expected output has not been seen, and
+the test has failed.
+
+The `match` element of the dict determines how the `line` element is matched:
+
+| Type      | Description             |
+| --------  | ----------------------- |
+| `literal` | Literal match (default) |
+| `re`      | regex match             |
 
 ### Skipping integration tests
 

mesonbuild/build.py

@@ -535,6 +535,9 @@ class BuildTarget(Target):
         repr_str = "<{0} {1}: {2}>"
         return repr_str.format(self.__class__.__name__, self.get_id(), self.filename)
 
+    def __str__(self):
+        return "{}".format(self.name)
+
     def validate_install(self, environment):
         if self.for_machine is MachineChoice.BUILD and self.need_install:
             if environment.is_cross_build():
@@ -1111,7 +1114,7 @@ You probably should put it in link_with instead.''')
             if not isinstance(t, (Target, CustomTargetIndex)):
                 raise InvalidArguments('{!r} is not a target.'.format(t))
             if not t.is_linkable_target():
-                raise InvalidArguments('Link target {!r} is not linkable.'.format(t))
+                raise InvalidArguments("Link target '{!s}' is not linkable.".format(t))
             if isinstance(self, SharedLibrary) and isinstance(t, StaticLibrary) and not t.pic:
                 msg = "Can't link non-PIC static library {!r} into shared library {!r}. ".format(t.name, self.name)
                 msg += "Use the 'pic' option to static_library to build with PIC."

mesonbuild/coredata.py

@@ -491,7 +491,7 @@ class CoreData:
             # commonpath will always return a path in the native format, so we
             # must use pathlib.PurePath to do the same conversion before
             # comparing.
-            msg = ('The value of the {!r} option is {!r} which must be a '
+            msg = ('The value of the {!r} option is \'{!s}\' which must be a '
                    'subdir of the prefix {!r}.\nNote that if you pass a '
                    'relative path, it is assumed to be a subdir of prefix.')
             # os.path.commonpath doesn't understand case-insensitive filesystems,

mesonbuild/interpreter.py

@@ -3168,6 +3168,12 @@ external dependencies (including libraries) must go to "dependencies".''')
         return should
 
     def add_languages_for(self, args, required, for_machine: MachineChoice):
+        langs = set(self.coredata.compilers[for_machine].keys())
+        langs.update(args)
+        if 'vala' in langs:
+            if 'c' not in langs:
+                raise InterpreterException('Compiling Vala requires C. Add C to your project languages and rerun Meson.')
+
         success = True
         for lang in sorted(args, key=compilers.sort_clink):
             lang = lang.lower()
@@ -3205,11 +3211,6 @@ external dependencies (including libraries) must go to "dependencies".''')
                            mlog.bold(' '.join(comp.linker.get_exelist())), comp.linker.id, comp.linker.version)
             self.build.ensure_static_linker(comp)
 
-        langs = self.coredata.compilers[for_machine].keys()
-        if 'vala' in langs:
-            if 'c' not in langs:
-                raise InterpreterException('Compiling Vala requires C. Add C to your project languages and rerun Meson.')
-
         return success
 
     def program_from_file_for(self, for_machine, prognames, silent):
@@ -3950,7 +3951,7 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
         absname = os.path.join(self.environment.get_source_dir(), buildfilename)
         if not os.path.isfile(absname):
             self.subdir = prev_subdir
-            raise InterpreterException('Non-existent build file {!r}'.format(buildfilename))
+            raise InterpreterException("Non-existent build file '{!s}'".format(buildfilename))
         with open(absname, encoding='utf8') as f:
             code = f.read()
         assert(isinstance(code, str))
@@ -3998,7 +3999,7 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
             elif isinstance(s, str):
                 source_strings.append(s)
             else:
-                raise InvalidArguments('Argument {!r} must be string or file.'.format(s))
+                raise InvalidArguments('Argument must be string or file.')
         sources += self.source_strings_to_files(source_strings)
         install_dir = kwargs.get('install_dir', None)
         if not isinstance(install_dir, (str, type(None))):
@@ -4272,8 +4273,9 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
 
         for a in incdir_strings:
             if a.startswith(src_root):
-                raise InvalidArguments('''Tried to form an absolute path to a source dir. You should not do that but use
-relative paths instead.
+                raise InvalidArguments('Tried to form an absolute path to a source dir. '
+                                       'You should not do that but use relative paths instead.'
+                                       '''
 
 To get include path to any directory relative to the current dir do
 

mesonbuild/mlog.py

@@ -40,15 +40,32 @@ def _windows_ansi() -> bool:
     # original behavior
     return bool(kernel.SetConsoleMode(stdout, mode.value | 0x4) or os.environ.get('ANSICON'))
 
-def setup_console() -> bool:
+def colorize_console() -> bool:
+    _colorize_console = getattr(sys.stdout, 'colorize_console', None)  # type: bool
+    if _colorize_console is not None:
+        return _colorize_console
+
     try:
         if platform.system().lower() == 'windows':
-            return os.isatty(sys.stdout.fileno()) and _windows_ansi()
-        return os.isatty(sys.stdout.fileno()) and os.environ.get('TERM') != 'dumb'
+            _colorize_console = os.isatty(sys.stdout.fileno()) and _windows_ansi()
+        else:
+            _colorize_console = os.isatty(sys.stdout.fileno()) and os.environ.get('TERM', 'dumb') != 'dumb'
     except Exception:
-        return False
+        _colorize_console = False
+
+    sys.stdout.colorize_console = _colorize_console  # type: ignore[attr-defined]
+    return _colorize_console
+
+def setup_console():
+    # on Windows, a subprocess might call SetConsoleMode() on the console
+    # connected to stdout and turn off ANSI escape processing. Call this after
+    # running a subprocess to ensure we turn it on again.
+    if platform.system().lower() == 'windows':
+        try:
+            delattr(sys.stdout, 'colorize_console')
+        except AttributeError:
+            pass
 
-colorize_console = setup_console()
 log_dir = None               # type: T.Optional[str]
 log_file = None              # type: T.Optional[T.TextIO]
 log_fname = 'meson-log.txt'  # type: str
@@ -204,7 +221,7 @@ def log(*args: T.Union[str, AnsiDecorator], is_error: bool = False,
     if log_file is not None:
         print(*arr, file=log_file, **kwargs)
         log_file.flush()
-    if colorize_console:
+    if colorize_console():
         arr = process_markup(args, True)
     if not log_errors_only or is_error:
         force_print(*arr, **kwargs)

run_project_tests.py

@@ -191,6 +191,7 @@ class TestDef:
         self.env = os.environ.copy()
         self.installed_files = []  # type: T.List[InstalledFile]
         self.do_not_set_opts = []  # type: T.List[str]
+        self.stdout = [] # type: T.List[T.Dict[str, str]]
 
     def __repr__(self) -> str:
         return '<{}: {:<48} [{}: {}] -- {}>'.format(type(self).__name__, str(self.path), self.name, self.args, self.skip)
@@ -341,19 +342,19 @@ def log_text_file(logfile, testdir, stdo, stde):
 
 
 def bold(text):
-    return mlog.bold(text).get_text(mlog.colorize_console)
+    return mlog.bold(text).get_text(mlog.colorize_console())
 
 
 def green(text):
-    return mlog.green(text).get_text(mlog.colorize_console)
+    return mlog.green(text).get_text(mlog.colorize_console())
 
 
 def red(text):
-    return mlog.red(text).get_text(mlog.colorize_console)
+    return mlog.red(text).get_text(mlog.colorize_console())
 
 
 def yellow(text):
-    return mlog.yellow(text).get_text(mlog.colorize_console)
+    return mlog.yellow(text).get_text(mlog.colorize_console())
 
 
 def _run_ci_include(args: T.List[str]) -> str:
@@ -381,6 +382,54 @@ def run_ci_commands(raw_log: str) -> T.List[str]:
         res += ['CI COMMAND {}:\n{}\n'.format(cmd[0], ci_commands[cmd[0]](cmd[1:]))]
     return res
 
+def _compare_output(expected: T.List[T.Dict[str, str]], output: str, desc: str) -> str:
+    if expected:
+        i = iter(expected)
+
+        def next_expected(i):
+            # Get the next expected line
+            item = next(i)
+            how = item.get('match', 'literal')
+            expected = item.get('line')
+
+            # Simple heuristic to automatically convert path separators for
+            # Windows:
+            #
+            # Any '/' appearing before 'WARNING' or 'ERROR' (i.e. a path in a
+            # filename part of a location) is replaced with '\' (in a re: '\\'
+            # which matches a literal '\')
+            #
+            # (There should probably be a way to turn this off for more complex
+            # cases which don't fit this)
+            if mesonlib.is_windows():
+                if how != "re":
+                    sub = r'\\'
+                else:
+                    sub = r'\\\\'
+                expected = re.sub(r'/(?=.*(WARNING|ERROR))', sub, expected)
+
+            return how, expected
+
+        try:
+            how, expected = next_expected(i)
+            for actual in output.splitlines():
+                if how == "re":
+                    match = bool(re.match(expected, actual))
+                else:
+                    match = (expected == actual)
+                if match:
+                    how, expected = next_expected(i)
+
+            # reached the end of output without finding expected
+            return 'expected "{}" not found in {}'.format(expected, desc)
+        except StopIteration:
+            # matched all expected lines
+            pass
+
+    return ''
+
+def validate_output(test: TestDef, stdo: str, stde: str) -> str:
+    return _compare_output(test.stdout, stdo, 'stdout')
 
 def run_test_inprocess(testdir):
     old_stdout = sys.stdout
@@ -452,6 +501,11 @@ def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args,
     cicmds = run_ci_commands(mesonlog)
     testresult = TestResult(cicmds)
     testresult.add_step(BuildStep.configure, stdo, stde, mesonlog, time.time() - gen_start)
+    output_msg = validate_output(test, stdo, stde)
+    testresult.mlog += output_msg
+    if output_msg:
+        testresult.fail('Unexpected output while configuring.')
+        return testresult
     if should_fail == 'meson':
         if returncode == 1:
             return testresult
@@ -566,6 +620,9 @@ def gather_tests(testdir: Path) -> T.List[TestDef]:
         if 'installed' in test_def:
             installed = [InstalledFile(x) for x in test_def['installed']]
 
+        # Handle expected output
+        stdout = test_def.get('stdout', [])
+
         # Handle the do_not_set_opts list
         do_not_set_opts = test_def.get('do_not_set_opts', [])  # type: T.List[str]
 
@@ -583,6 +640,7 @@ def gather_tests(testdir: Path) -> T.List[TestDef]:
             t.env.update(env)
             t.installed_files = installed
             t.do_not_set_opts = do_not_set_opts
+            t.stdout = stdout
             all_tests += [t]
             continue
 
@@ -653,6 +711,7 @@ def gather_tests(testdir: Path) -> T.List[TestDef]:
             test.env.update(env)
             test.installed_files = installed
             test.do_not_set_opts = do_not_set_opts
+            test.stdout = stdout
             all_tests += [test]
 
     return sorted(all_tests)

run_tests.py

@@ -303,7 +303,7 @@ def run_configure(commandlist, env=None):
     return run_configure_inprocess(commandlist, env=env)
 
 def print_system_info():
-    print(mlog.bold('System information.').get_text(mlog.colorize_console))
+    print(mlog.bold('System information.').get_text(mlog.colorize_console()))
     print('Architecture:', platform.architecture())
     print('Machine:', platform.machine())
     print('Platform:', platform.system())
@@ -377,7 +377,7 @@ def main():
                 print(flush=True)
                 returncode = 0
             else:
-                print(mlog.bold('Running unittests.').get_text(mlog.colorize_console))
+                print(mlog.bold('Running unittests.').get_text(mlog.colorize_console()))
                 print(flush=True)
                 cmd = mesonlib.python_command + ['run_unittests.py', '-v']
                 if options.failfast:
@@ -390,7 +390,7 @@ def main():
         else:
             cross_test_args = mesonlib.python_command + ['run_cross_test.py']
             for cf in options.cross:
-                print(mlog.bold('Running {} cross tests.'.format(cf)).get_text(mlog.colorize_console))
+                print(mlog.bold('Running {} cross tests.'.format(cf)).get_text(mlog.colorize_console()))
                 print(flush=True)
                 cmd = cross_test_args + ['cross/' + cf]
                 if options.failfast:

test cases/failing/1 project not first/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "ERROR: First statement must be a call to project"
+    }
+  ]
+}

test cases/failing/10 out of bounds/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/10 out of bounds/meson.build:4:0: ERROR: Index 0 out of bounds of array of size 0."
+    }
+  ]
+}

test cases/failing/11 object arithmetic/test.json

@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/11 object arithmetic/meson\\.build:3:0: ERROR: Invalid use of addition: .*"
+    }
+  ]
+}

test cases/failing/12 string arithmetic/test.json

@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/12 string arithmetic/meson\\.build:3:0: ERROR: Invalid use of addition: .*"
+    }
+  ]
+}

test cases/failing/13 array arithmetic/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/13 array arithmetic/meson.build:3:0: ERROR: Multiplication works only with integers."
+    }
+  ]
+}

test cases/failing/14 invalid option name/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/14 invalid option name/meson_options.txt:1:0: ERROR: Option names can only contain letters, numbers or dashes."
+    }
+  ]
+}

test cases/failing/15 kwarg before arg/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/15 kwarg before arg/meson.build:3:0: ERROR: All keyword arguments must be after positional arguments."
+    }
+  ]
+}

test cases/failing/16 extract from subproject/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/16 extract from subproject/meson.build:6:0: ERROR: Tried to extract objects from a subproject target."
+    }
+  ]
+}

test cases/failing/17 same target/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/17 same target/meson.build:4:0: ERROR: Tried to create target \"foo\", but a target of that name already exists."
+    }
+  ]
+}

test cases/failing/18 wrong plusassign/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/18 wrong plusassign/meson.build:3:0: ERROR: Plusassignment target must be an id."
+    }
+  ]
+}

test cases/failing/19 target clash/meson.build

@@ -8,7 +8,7 @@ project('clash', 'c')
 # output location is redirected.
 
 if host_machine.system() == 'windows' or host_machine.system() == 'cygwin'
-  error('This is expected.')
+  error('MESON_SKIP_TEST test only works on platforms where executables have no suffix.')
 endif
 
 executable('clash', 'clash.c')

test cases/failing/19 target clash/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "ERROR: Multiple producers for Ninja target \"clash\". Please rename your targets."
+    }
+  ]
+}

test cases/failing/2 missing file/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/2 missing file/meson.build:3:0: ERROR: File missing.c does not exist."
+    }
+  ]
+}

test cases/failing/20 version/test.json

@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/20 version/meson\\.build:1:0: ERROR: Meson version is .* but project requires >100\\.0\\.0"
+    }
+  ]
+}

test cases/failing/21 subver/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/21 subver/meson.build:3:0: ERROR: Subproject foo version is 1.0.0 but >1.0.0 required."
+    }
+  ]
+}

test cases/failing/22 assert/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/22 assert/meson.build:3:0: ERROR: Assert failed: I am fail."
+    }
+  ]
+}

test cases/failing/23 rel testdir/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/23 rel testdir/meson.build:4:0: ERROR: Workdir keyword argument must be an absolute path."
+    }
+  ]
+}

test cases/failing/24 int conversion/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/24 int conversion/meson.build:3:13: ERROR: String 'notanumber' cannot be converted to int"
+    }
+  ]
+}

test cases/failing/25 badlang/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/25 badlang/meson.build:3:0: ERROR: Tried to use unknown language \"nonexisting\"."
+    }
+  ]
+}

test cases/failing/26 output subdir/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/26 output subdir/meson.build:3:0: ERROR: Output file name must not contain a subdirectory."
+    }
+  ]
+}

test cases/failing/27 noprog use/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/27 noprog use/meson.build:5:0: ERROR: Tried to use not-found external program in \"command\""
+    }
+  ]
+}

test cases/failing/28 no crossprop/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/28 no crossprop/meson.build:3:0: ERROR: Unknown cross property: nonexisting."
+    }
+  ]
+}

test cases/failing/29 nested ternary/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/29 nested ternary/meson.build:3:12: ERROR: Nested ternary operators are not allowed."
+    }
+  ]
+}

test cases/failing/3 missing subdir/test.json

@@ -0,0 +1,9 @@
+{
+  "stdout": [
+    {
+        "comment": "'missing/meson.build' gets transformed with os.path.sep separators",
+        "match": "re",
+        "line": "test cases/failing/3 missing subdir/meson\\.build:3:0: ERROR: Non\\-existent build file 'missing[\\\\/]meson\\.build'"
+    }
+  ]
+}

test cases/failing/30 invalid man extension/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/30 invalid man extension/meson.build:2:0: ERROR: Man file must have a file extension of a number between 1 and 8"
+    }
+  ]
+}

test cases/failing/31 no man extension/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/31 no man extension/meson.build:2:0: ERROR: Man file must have a file extension of a number between 1 and 8"
+    }
+  ]
+}

test cases/failing/32 exe static shared/meson.build

@@ -2,7 +2,7 @@ project('statchain', 'c')
 
 host_system = host_machine.system()
 if host_system == 'windows' or host_system == 'darwin'
-  error('Test only fails on Linux and BSD')
+  error('MESON_SKIP_TEST test only fails on Linux and BSD')
 endif
 
 statlib = static_library('stat', 'stat.c', pic : false)

test cases/failing/32 exe static shared/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/32 exe static shared/meson.build:9:0: ERROR: Can't link non-PIC static library 'stat' into shared library 'shr2'. Use the 'pic' option to static_library to build with PIC."
+    }
+  ]
+}

test cases/failing/33 non-root subproject/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/33 non-root subproject/some/meson.build:1:0: ERROR: Subproject directory not found and someproj.wrap file not found"
+    }
+  ]
+}

test cases/failing/34 dependency not-required then required/test.json

@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/34 dependency not\\-required then required/meson\\.build:4:0: ERROR: Dependency \"foo\\-bar\\-xyz\\-12\\.3\" not found, tried .*"
+    }
+  ]
+}

test cases/failing/35 project argument after target/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/35 project argument after target/meson.build:7:0: ERROR: Tried to use 'add_project_arguments' after a build target has been declared."
+    }
+  ]
+}

test cases/failing/36 pkgconfig dependency impossible conditions/meson.build

@@ -1,3 +1,7 @@
 project('impossible-dep-test', 'c', version : '1.0')
 
+if not dependency('zlib', required: false).found()
+  error('MESON_SKIP_TEST test requires zlib')
+endif
+
 dependency('zlib', version : ['>=1.0', '<1.0'])

test cases/failing/37 has function external dependency/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/37 has function external dependency/meson.build:8:3: ERROR: Dependencies must be external dependencies"
+    }
+  ]
+}

test cases/failing/38 libdir must be inside prefix/test.json

@@ -1,3 +1,10 @@
 {
-  "do_not_set_opts": ["libdir"]
+  "do_not_set_opts": [
+    "libdir"
+  ],
+  "stdout": [
+    {
+      "line": "test cases/failing/38 libdir must be inside prefix/meson.build:1:0: ERROR: The value of the 'libdir' option is '/opt/lib' which must be a subdir of the prefix '/usr'."
+    }
+  ]
 }

test cases/failing/39 prefix absolute/test.json

@@ -1,3 +1,11 @@
 {
-  "do_not_set_opts": ["prefix"]
+  "do_not_set_opts": [
+    "prefix"
+  ],
+  "stdout": [
+    {
+      "comment": "literal 'some/path/notabs' appears in output, irrespective of os.path.sep, as that's the prefix",
+      "line": "test cases/failing/39 prefix absolute/meson.build:1:0: ERROR: prefix value 'some/path/notabs' must be an absolute path"
+    }
+  ]
 }

test cases/failing/4 missing meson.build/test.json

@@ -0,0 +1,9 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "comment": "'subdir/meson.build' gets transformed with os.path.sep separators",
+      "line": "test cases/failing/4 missing meson\\.build/meson\\.build:3:0: ERROR: Non\\-existent build file 'subdir[\\\\/]meson\\.build'"
+    }
+  ]
+}

test cases/failing/40 kwarg assign/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/40 kwarg assign/meson.build:3:0: ERROR: Tried to assign values inside an argument list."
+    }
+  ]
+}

test cases/failing/41 custom target plainname many inputs/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/41 custom target plainname many inputs/meson.build:5:0: ERROR: Output cannot contain @PLAINNAME@ or @BASENAME@ when there is more than one input (we can't know which to use)"
+    }
+  ]
+}

test cases/failing/42 custom target outputs not matching install_dirs/meson.build

@@ -3,7 +3,7 @@ project('outputs not matching install_dirs', 'c')
 gen = find_program('generator.py')
 
 if meson.backend() != 'ninja'
-  error('Failing manually, test is only for the ninja backend')
+  error('MESON_SKIP_TEST test is only for the ninja backend')
 endif
 
 custom_target('too-few-install-dirs',

test cases/failing/42 custom target outputs not matching install_dirs/test.json

@@ -1,10 +1,33 @@
 {
   "installed": [
-    {"type": "file", "file": "usr/include/diff.h"},
-    {"type": "file", "file": "usr/include/first.h"},
-    {"type": "file", "file": "usr/bin/diff.sh"},
-    {"type": "file", "file": "usr/bin/second.sh"},
-    {"type": "file", "file": "opt/same.h"},
-    {"type": "file", "file": "opt/same.sh"}
+    {
+      "type": "file",
+      "file": "usr/include/diff.h"
+    },
+    {
+      "type": "file",
+      "file": "usr/include/first.h"
+    },
+    {
+      "type": "file",
+      "file": "usr/bin/diff.sh"
+    },
+    {
+      "type": "file",
+      "file": "usr/bin/second.sh"
+    },
+    {
+      "type": "file",
+      "file": "opt/same.h"
+    },
+    {
+      "type": "file",
+      "file": "opt/same.sh"
+    }
+  ],
+  "stdout": [
+    {
+      "line": "ERROR: Target 'too-few-install-dirs' has 3 outputs: ['toofew.h', 'toofew.c', 'toofew.sh'], but only 2 \"install_dir\"s were found."
+    }
   ]
 }

test cases/failing/43 project name colon/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/43 project name colon/meson.build:1:0: ERROR: Project name 'name with :' must not contain ':'"
+    }
+  ]
+}

test cases/failing/44 abs subdir/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/44 abs subdir/meson.build:5:0: ERROR: Subdir argument must be a relative path."
+    }
+  ]
+}

test cases/failing/45 abspath to srcdir/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/45 abspath to srcdir/meson.build:3:0: ERROR: Tried to form an absolute path to a source dir. You should not do that but use relative paths instead."
+    }
+  ]
+}

test cases/failing/46 pkgconfig variables reserved/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/46 pkgconfig variables reserved/meson.build:8:5: ERROR: Variable \"prefix\" is reserved"
+    }
+  ]
+}

test cases/failing/47 pkgconfig variables zero length/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/47 pkgconfig variables zero length/meson.build:8:5: ERROR: Invalid variable \"=value\". Variables must be in 'name=value' format"
+    }
+  ]
+}

test cases/failing/48 pkgconfig variables zero length value/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/48 pkgconfig variables zero length value/meson.build:8:5: ERROR: Invalid variable \"key=\". Variables must be in 'name=value' format"
+    }
+  ]
+}

test cases/failing/49 pkgconfig variables not key value/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/49 pkgconfig variables not key value/meson.build:8:5: ERROR: Invalid variable \"this_should_be_key_value\". Variables must be in 'name=value' format"
+    }
+  ]
+}

test cases/failing/5 misplaced option/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/5 misplaced option/meson.build:3:0: ERROR: Tried to call option() in build description file. All options must be in the option file."
+    }
+  ]
+}

test cases/failing/50 executable comparison/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/50 executable comparison/meson.build:6:0: ERROR: exe1 can only be compared for equality."
+    }
+  ]
+}

test cases/failing/51 inconsistent comparison/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/51 inconsistent comparison/meson.build:5:0: ERROR: Values of different types (list, str) cannot be compared using <."
+    }
+  ]
+}

test cases/failing/52 slashname/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/52 slashname/meson.build:11:0: ERROR: Problem encountered: Re-enable me once slash in name is finally prohibited."
+    }
+  ]
+}

test cases/failing/53 reserved meson prefix/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/53 reserved meson prefix/meson.build:3:0: ERROR: The \"meson-\" prefix is reserved and cannot be used for top-level subdir()."
+    }
+  ]
+}

test cases/failing/54 wrong shared crate type/meson.build

@@ -1,3 +1,7 @@
-project('test', 'rust')
+project('test')
+
+if not add_languages('rust', required: false)
+  error('MESON_SKIP_TEST test requires rust compiler')
+endif
 
 shared_library('test', 'foo.rs', rust_crate_type : 'staticlib')

test cases/failing/54 wrong shared crate type/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/54 wrong shared crate type/meson.build:7:0: ERROR: Crate type \"staticlib\" invalid for dynamic libraries; must be \"dylib\" or \"cdylib\""
+    }
+  ]
+}

test cases/failing/55 wrong static crate type/meson.build

@@ -1,3 +1,7 @@
-project('test', 'rust')
+project('test')
+
+if not add_languages('rust', required: false)
+  error('MESON_SKIP_TEST test requires rust compiler')
+endif
 
 static_library('test', 'foo.rs', rust_crate_type : 'cdylib')

test cases/failing/55 wrong static crate type/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/55 wrong static crate type/meson.build:7:0: ERROR: Crate type \"cdylib\" invalid for static libraries; must be \"rlib\" or \"staticlib\""
+    }
+  ]
+}

test cases/failing/56 or on new line/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/56 or on new line/meson.build:4:8: ERROR: Invalid or clause."
+    }
+  ]
+}

test cases/failing/57 kwarg in module/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/57 kwarg in module/meson.build:3:0: ERROR: Function does not take keyword arguments."
+    }
+  ]
+}

test cases/failing/58 link with executable/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/58 link with executable/meson.build:4:0: ERROR: Link target 'prog' is not linkable."
+    }
+  ]
+}

test cases/failing/59 assign custom target index/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/59 assign custom target index/meson.build:24:0: ERROR: Assignment target must be an id."
+    }
+  ]
+}

test cases/failing/6 missing incdir/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/6 missing incdir/meson.build:3:0: ERROR: Include dir nosuchdir does not exist."
+    }
+  ]
+}

test cases/failing/60 getoption prefix/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/60 getoption prefix/meson.build:5:0: ERROR: Having a colon in option name is forbidden, projects are not allowed to directly access options of other subprojects."
+    }
+  ]
+}

test cases/failing/61 bad option argument/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/61 bad option argument/meson_options.txt:1:0: ERROR: Invalid kwargs for option \"name\": \"vaule\""
+    }
+  ]
+}

test cases/failing/62 subproj filegrab/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/62 subproj filegrab/subprojects/a/meson.build:3:0: ERROR: Sandbox violation: Tried to grab file prog.c from a different subproject."
+    }
+  ]
+}

test cases/failing/63 grab subproj/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/63 grab subproj/meson.build:7:0: ERROR: Sandbox violation: Tried to grab file sub.c from a different subproject."
+    }
+  ]
+}

test cases/failing/64 grab sibling/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/64 grab sibling/subprojects/a/meson.build:3:0: ERROR: Sandbox violation: Tried to grab file sneaky.c from a different subproject."
+    }
+  ]
+}

test cases/failing/65 string as link target/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/65 string as link target/meson.build:2:0: ERROR: '' is not a target."
+    }
+  ]
+}

test cases/failing/66 dependency not-found and required/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/66 dependency not-found and required/meson.build:2:0: ERROR: Dependency is both required and not-found"
+    }
+  ]
+}

test cases/failing/68 wrong boost module/meson.build

@@ -1,5 +1,9 @@
 project('boosttest', 'cpp',
   default_options : ['cpp_std=c++11'])
 
+if not dependency('boost', required: false).found()
+  error('MESON_SKIP_TEST test requires boost')
+endif
+
 # abc doesn't exist
 linkdep = dependency('boost', modules : ['thread', 'system', 'test', 'abc'])

test cases/failing/68 wrong boost module/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/68 wrong boost module/meson.build:9:0: ERROR: Dependency \"boost\" not found"
+    }
+  ]
+}

test cases/failing/69 install_data rename bad size/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/69 install_data rename bad size/meson.build:3:0: ERROR: Size of rename argument is different from number of sources"
+    }
+  ]
+}

test cases/failing/7 go to subproject/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/7 go to subproject/meson.build:3:0: ERROR: Must not go into subprojects dir with subdir(), use subproject() instead."
+    }
+  ]
+}

test cases/failing/70 skip only subdir/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/70 skip only subdir/meson.build:8:0: ERROR: File main.cpp does not exist."
+    }
+  ]
+}

test cases/failing/71 dual override/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/71 dual override/meson.build:5:6: ERROR: Tried to override executable \"override\" which has already been overridden."
+    }
+  ]
+}

test cases/failing/72 override used/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/72 override used/meson.build:5:6: ERROR: Tried to override finding of executable \"something.py\" which has already been found."
+    }
+  ]
+}

test cases/failing/73 run_command unclean exit/test.json

@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/73 run_command unclean exit/meson\\.build:4:0: ERROR: Command \".*[\\\\/]test cases[\\\\/]failing[\\\\/]73 run_command unclean exit[\\\\/]\\.[\\\\/]returncode\\.py 1\" failed with status 1\\."
+    }
+  ]
+}

test cases/failing/74 int literal leading zero/test.json

@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "comment": "this error message is not very informative",
+      "line": "test cases/failing/74 int literal leading zero/meson.build:5:13: ERROR: Expecting eof got number."
+    }
+  ]
+}

test cases/failing/75 configuration immutable/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/75 configuration immutable/meson.build:12:16: ERROR: Can not set values on configuration object that has been used."
+    }
+  ]
+}

test cases/failing/76 link with shared module on osx/meson.build

@@ -1,7 +1,7 @@
 project('link with shared module', 'c')
 
 if host_machine.system() != 'darwin'
-  error('Test only fails on OSX')
+  error('MESON_SKIP_TEST test only fails on OSX')
 endif
 
 m = shared_module('mymodule', 'module.c')

test cases/failing/76 link with shared module on osx/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/76 link with shared module on osx/meson.build:8:0: ERROR: target links against shared modules."
+    }
+  ]
+}

test cases/failing/77 non ascii in ascii encoded configure file/test.json

@@ -0,0 +1,8 @@
+{
+  "stdout": [
+    {
+      "match": "re",
+      "line": "test cases/failing/77 non ascii in ascii encoded configure file/meson\\.build:5:0: ERROR: Could not write output file .*[\\\\/]config9\\.h: 'ascii' codec can't encode character '\\\\u0434' in position 17: ordinal not in range\\(128\\)"
+    }
+  ]
+}

test cases/failing/78 subproj dependency not-found and required/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/78 subproj dependency not-found and required/meson.build:2:0: ERROR: Subproject directory not found and missing.wrap file not found"
+    }
+  ]
+}

test cases/failing/79 unfound run/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/79 unfound run/meson.build:4:0: ERROR: Tried to use non-existing executable 'nonexisting_prog'"
+    }
+  ]
+}

test cases/failing/8 recursive/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/8 recursive/subprojects/b/meson.build:3:0: ERROR: Recursive include of subprojects: a => b => a."
+    }
+  ]
+}

test cases/failing/80 framework dependency with version/meson.build

@@ -1,4 +1,8 @@
 project('framework dependency with version', 'c')
+
+if host_machine.system() != 'darwin'
+  error('MESON_SKIP_TEST test only applicable on darwin')
+endif
+
 # do individual frameworks have a meaningful version to test?  And multiple frameworks might be listed...
-# otherwise we're not on OSX and this will definitely fail
 dep = dependency('appleframeworks', modules: 'foundation', version: '>0')

test cases/failing/80 framework dependency with version/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/80 framework dependency with version/meson.build:8:0: ERROR: Unknown version of dependency 'appleframeworks', but need ['>0']."
+    }
+  ]
+}

test cases/failing/81 override exe config/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/81 override exe config/meson.build:6:0: ERROR: Program 'bar' was overridden with the compiled executable 'foo' and therefore cannot be used during configuration"
+    }
+  ]
+}

test cases/failing/82 gl dependency with version/meson.build

@@ -2,7 +2,7 @@ project('gl dependency with version', 'c')
 
 host_system = host_machine.system()
 if host_system != 'windows' and host_system != 'darwin'
-  error('Test only fails on Windows and OSX')
+  error('MESON_SKIP_TEST: test only fails on Windows and OSX')
 endif
 
 # gl dependency found via system method doesn't have a meaningful version to check

test cases/failing/82 gl dependency with version/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/82 gl dependency with version/meson.build:9:0: ERROR: Unknown version of dependency 'gl', but need ['>0']."
+    }
+  ]
+}

test cases/failing/83 threads dependency with version/test.json

@@ -0,0 +1,7 @@
+{
+  "stdout": [
+    {
+      "line": "test cases/failing/83 threads dependency with version/meson.build:3:0: ERROR: Unknown version of dependency 'threads', but need ['>0']."
+    }
+  ]
+}

test cases/failing/84 gtest dependency with version/meson.build

@@ -1,3 +1,8 @@
 project('gtest dependency with version', ['c', 'cpp'])
+
+if not dependency('gtest', method: 'system', required: false).found()
+   error('MESON_SKIP_TEST test requires gtest')
+endif
+
 # discovering gtest version is not yet implemented
 dep = dependency('gtest', method: 'system', version: '>0')

test cases/failing/85 dub libray/meson.build

@@ -1,3 +1,11 @@
-project('dub', 'd')
+project('dub')
+
+if not add_languages('d', required: false)
+  error('MESON_SKIP_TEST test requires D compiler')
+endif
+
+if not find_program('dub', required: false).found()
+  error('MESON_SKIP_TEST test requires dub')
+endif
 
 dependency('dubtestproject', method: 'dub') # Not library (none)