unit tests: Test more syntax highlighting data

@TingPing has a repository that contains a grammar for meson which is
used by linguist (GitHub), and by many editors such as Atom, VS Code,
TextMate, Sublime Text, etc. Add CI so that we notice that the
function list in it is out of date, such as https://github.com/TingPing/language-meson/pull/3

It's harder to do this generically for other syntax such as the `in`
keyword, but it's better than nothing.

azure-pipelines.yml

@@ -88,7 +88,7 @@ jobs:
         gccx64ninja: {}
   variables:
     CYGWIN_ROOT: $(System.Workfolder)\cygwin
-    CYGWIN_CMAKE_LINK: http://cygwin.mirror.constant.com/x86_64/release/cmake/cmake-3.13.1-1.tar.xz
+    CYGWIN_CMAKE_LINK: http://cygwin.mirror.constant.com/x86_64/release/cmake/cmake-3.14.5-1.tar.xz
     CYGWIN_MIRROR: http://cygwin.mirror.constant.com
   steps:
     - script: |
@@ -117,7 +117,7 @@ jobs:
     - script: |
         %CYGWIN_ROOT%\bin\bash.exe -cl "wget %CYGWIN_CMAKE_LINK% -O cmake.tar.xz"
         %CYGWIN_ROOT%\bin\bash.exe -cl "tar -xf cmake.tar.xz -C /"
-      displayName: Manually install CMake 3.13.1
+      displayName: Manually install CMake
     - script: |
         set BOOST_ROOT=
         set PATH=%CYGWIN_ROOT%\bin;%SYSTEMROOT%\system32

mesonbuild/compilers/compilers.py

@@ -12,9 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import abc, contextlib, enum, os.path, re, tempfile, shlex
-from typing import List, Optional, Tuple
-from pathlib import Path
+import contextlib, enum, os.path, re, tempfile, shlex
+from typing import Optional, Tuple
 
 from ..linkers import StaticLinker
 from .. import coredata
@@ -22,7 +21,7 @@ from .. import mlog
 from .. import mesonlib
 from ..mesonlib import (
     EnvironmentException, MachineChoice, MesonException, OrderedSet,
-    version_compare, Popen_safe
+    Popen_safe
 )
 from ..envconfig import (
     Properties,

mesonbuild/dependencies/base.py

@@ -1397,18 +1397,14 @@ class CMakeDependency(ExternalDependency):
 
     def _setup_cmake_dir(self, cmake_file: str) -> str:
         # Setup the CMake build environment and return the "build" directory
-        build_dir = '{}/cmake_{}'.format(self.cmake_root_dir, self.name)
-        os.makedirs(build_dir, exist_ok=True)
+        build_dir = Path(self.cmake_root_dir) / 'cmake_{}'.format(self.name)
+        build_dir.mkdir(parents=True, exist_ok=True)
 
         # Copy the CMakeLists.txt
-        cmake_lists = '{}/CMakeLists.txt'.format(build_dir)
-        dir_path = os.path.dirname(os.path.realpath(__file__))
-        src_cmake = '{}/data/{}'.format(dir_path, cmake_file)
-        if os.path.exists(cmake_lists):
-            os.remove(cmake_lists)
-        shutil.copyfile(src_cmake, cmake_lists)
+        src_cmake = Path(__file__).parent / 'data' / cmake_file
+        shutil.copyfile(str(src_cmake), str(build_dir / 'CMakeLists.txt'))  # str() is for Python 3.5
 
-        return build_dir
+        return str(build_dir)
 
     def _call_cmake(self, args, cmake_file: str, env=None):
         build_dir = self._setup_cmake_dir(cmake_file)
@@ -1543,7 +1539,7 @@ class DubDependency(ExternalDependency):
                 for lib in target['buildSettings'][field_name]:
                     if lib not in libs:
                         libs.append(lib)
-                        if os.name is not 'nt':
+                        if os.name != 'nt':
                             pkgdep = PkgConfigDependency(lib, environment, {'required': 'true', 'silent': 'true'})
                             for arg in pkgdep.get_compile_args():
                                 self.compile_args.append(arg)

mesonbuild/interpreter.py

@@ -21,7 +21,7 @@ from . import optinterpreter
 from . import compilers
 from .wrap import wrap, WrapMode
 from . import mesonlib
-from .mesonlib import FileMode, MachineChoice, PerMachine, Popen_safe, listify, extract_as_list, has_path_sep
+from .mesonlib import FileMode, MachineChoice, Popen_safe, listify, extract_as_list, has_path_sep
 from .dependencies import ExternalProgram
 from .dependencies import InternalDependency, Dependency, NotFoundDependency, DependencyException
 from .interpreterbase import InterpreterBase
@@ -40,7 +40,7 @@ from collections import namedtuple
 from itertools import chain
 from pathlib import PurePath
 import functools
-from typing import Sequence, List, Union, Optional, Iterator, Dict, Any
+from typing import Sequence, List, Union, Optional, Dict, Any
 
 import importlib
 

mesonbuild/scripts/dist.py

@@ -21,7 +21,6 @@ import subprocess
 import pickle
 import hashlib
 import tarfile, zipfile
-import tempfile
 from glob import glob
 from mesonbuild.environment import detect_ninja
 from mesonbuild.mesonlib import windows_proof_rmtree

run_unittests.py

@@ -30,6 +30,8 @@ import functools
 import io
 import operator
 import threading
+import urllib.error
+import urllib.request
 from itertools import chain
 from unittest import mock
 from configparser import ConfigParser
@@ -1126,10 +1128,10 @@ class DataTests(unittest.TestCase):
             if f not in exceptions:
                 self.assertIn(f, toc)
 
-    def test_syntax_highlighting_files(self):
+    def test_vim_syntax_highlighting(self):
         '''
-        Ensure that syntax highlighting files were updated for new functions in
-        the global namespace in build files.
+        Ensure that vim syntax highlighting files were updated for new
+        functions in the global namespace in build files.
         '''
         env = get_fake_env()
         interp = Interpreter(FakeBuild(env), mock=True)
@@ -1138,6 +1140,39 @@ class DataTests(unittest.TestCase):
             defined = set([a.strip() for a in res.group().split('\\')][1:])
             self.assertEqual(defined, set(chain(interp.funcs.keys(), interp.builtin.keys())))
 
+    def test_json_grammar_syntax_highlighting(self):
+        '''
+        Ensure that syntax highlighting JSON grammar written by TingPing was
+        updated for new functions in the global namespace in build files.
+        https://github.com/TingPing/language-meson/
+        '''
+        env = get_fake_env()
+        interp = Interpreter(FakeBuild(env), mock=True)
+        url = 'https://raw.githubusercontent.com/TingPing/language-meson/master/grammars/meson.json'
+        try:
+            r = urllib.request.urlopen(url)
+        except urllib.error.URLError as e:
+            # Skip test when network is not available, such as during packaging
+            # by a distro or Flatpak
+            if not isinstance(e, urllib.error.HTTPError):
+                raise unittest.SkipTest('Network unavailable')
+            # Don't fail the test if github is down, but do fail if 4xx
+            if e.code >= 500:
+                raise unittest.SkipTest('Server error ' + str(e.code))
+            raise e
+        # On Python 3.5, we must decode bytes to string. Newer versions don't require that.
+        grammar = json.loads(r.read().decode('utf-8', 'surrogatepass'))
+        for each in grammar['patterns']:
+            if 'name' in each and each['name'] == 'support.function.builtin.meson':
+                # The string is of the form: (?x)\\b(func1|func2|...\n)\\b\\s*(?=\\() and
+                # we convert that to [func1, func2, ...] without using regex to parse regex
+                funcs = set(each['match'].split('\\b(')[1].split('\n')[0].split('|'))
+            if 'name' in each and each['name'] == 'support.variable.meson':
+                # \\b(builtin1|builtin2...)\\b
+                builtin = set(each['match'].split('\\b(')[1].split(')\\b')[0].split('|'))
+        self.assertEqual(builtin, set(interp.builtin.keys()))
+        self.assertEqual(funcs, set(interp.funcs.keys()))
+
     def test_all_functions_defined_in_ast_interpreter(self):
         '''
         Ensure that the all functions defined in the Interpreter are also defined

test cases/cmake/10 cmake_module_path/cmake/FindSomethingLikePython.cmake

@@ -0,0 +1,24 @@
+cmake_policy(VERSION 3.7)
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.12)
+  find_package(Python COMPONENTS Interpreter)
+else()
+  find_package(PythonInterp)
+endif()
+
+if(Python_FOUND OR PYTHONINTERP_FOUND)
+  set(SomethingLikePython_FOUND      ON)
+  set(SomethingLikePython_EXECUTABLE ${Python_EXECUTABLE})
+
+  if(NOT DEFINED Python_VERSION)
+    set(Python_VERSION ${Python_VERSION_STRING})
+  endif()
+  if(NOT TARGET Python::Interpreter)
+    add_executable(Python::Interpreter IMPORTED)
+    set_target_properties(Python::Interpreter PROPERTIES
+                          IMPORTED_LOCATION ${Python_EXECUTABLE}
+                          VERSION ${Python_VERSION})
+  endif()
+else()
+  set(SomethingLikePython_FOUND OFF)
+endif()

test cases/cmake/10 cmake_module_path/meson.build

@@ -0,0 +1,17 @@
+# We use Python3 as it's the only thing guaranteed to be available on any platform Meson can run on (unlike Zlib in linuxlike/13 cmake dependency).
+
+project('user CMake find_package module using cmake_module_path',
+  meson_version: '>= 0.50.0')
+
+if not find_program('cmake', required: false).found()
+  error('MESON_SKIP_TEST cmake binary not available.')
+endif
+
+# NOTE: can't request Python3 via dependency('Python3', method: 'cmake')
+#  Meson intercepts and wants "method: auto"
+
+# Try to find a dependency with a custom CMake module
+
+dependency('SomethingLikePython', required : true, method : 'cmake', cmake_module_path : 'cmake', modules: 'Python::Interpreter')
+
+dependency('SomethingLikePython', method : 'cmake', cmake_module_path : ['doesNotExist', 'cmake'], modules: 'Python::Interpreter')