Auto merge of #789 - bstaletic:windows_style_flags, r=micbou

[READY] Enable windows style flags when --driver-mode=cl is found in flags

Windows (cl) style flags are currently filtered and thus windows users are unable to use a `.ycm_extra_conf.py` such as this one:
```py
def FlagsForFile( filename ):
    return { 'flags': [
                '--driver-mode=cl',
                '/c',
                '/Zi',
                '/nologo',
                '/W3',
                '/WX-',
                '/O2',
                '/GL',
                '/Gm-',
                '/EHsc',
                '/MT',
                '/GS',
                '/fp:precise',
                '/Zc:wchar_t',
                '/Zc:forScope',
                '/Zc:inline',
                '/Gd',
                '/TP',
                '/wd4099',
                '/Tp' ] }
```

If `--driver-mode=cl` is in flags, libclang gladly accepts these flags. This pull request checks for `--driver-mode=cl` before filtering the flags. U used the above `.ycm_extra_conf.py` to confirm if everything works. The resulting `:YcmDebugInfo`:
```
Printing YouCompleteMe debug information...
-- Client logfile: /tmp/ycm_rbjk7yrj.log
-- Server Python interpreter: /usr/sbin/python
-- Server Python version: 3.6.1
-- Server has Clang support compiled in: True
-- Clang version: clang version 4.0.1 (tags/RELEASE_401/final)
-- Extra configuration file found and loaded
-- Extra configuration path: /home/bstaletic/Temp/ycmd/test/.ycm_extra_conf.py
-- C-family completer debug information:
--   Compilation database path: None
--   Flags: ['--driver-mode=cl', '-resource-dir=/home/bstaletic/.vim/pack/minpac/start/YouCompleteMe/third_party/ycmd/ycmd/../clang_includes', '/c', '/Zi', '/nologo', '/W3',
'/WX-', '/O2', '/GL', '/Gm-', '/EHsc', '/MT', '/GS', '/fp:precise', '/Zc:wchar_t', '/Zc:forScope', '/Zc:inline', '/Gd', '/TP', '/wd4099', '/Tp', '-fspell-checking']
-- Server running at: http://127.0.0.1:51101
-- Server process ID: 16027
-- Server logfiles:
--   /tmp/ycmd_51101_stdout_rh2xfnry.log
--   /tmp/ycmd_51101_stderr_7_n_feiy.log
```

Fixes ycm-core/YouCompleteMe#2491

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/valloric/ycmd/789)
<!-- Reviewable:end -->

ycmd/completers/cpp/flags.py

@@ -38,6 +38,7 @@
 INCLUDE_FLAGS = [ '-isystem', '-I', '-iquote', '-isysroot', '--sysroot',
                   '-gcc-toolchain', '-include-pch', '-include', '-iframework',
                   '-F', '-imacros', '-idirafter' ]
+INCLUDE_FLAGS_WIN_STYLE = [ '/I' ]
 PATH_FLAGS =  [ '--sysroot=' ] + INCLUDE_FLAGS
 
 # We need to remove --fcolor-diagnostics because it will cause shell escape
@@ -48,6 +49,8 @@
                              '-MMD',
                              '--fcolor-diagnostics' ] )
 
+STATE_FLAGS_TO_SKIP_WIN_STYLE = set( [ '/c' ] )
+
 # The -M* flags spec:
 #   https://gcc.gnu.org/onlinedocs/gcc-4.9.0/gcc/Preprocessor-Options.html
 FILE_FLAGS_TO_SKIP = set( [ '-MF',
@@ -144,7 +147,8 @@ def FlagsForFile( self,
 
     sanitized_flags = PrepareFlagsForClang( flags,
                                             filename,
-                                            add_extra_clang_flags )
+                                            add_extra_clang_flags,
+                                            _ShouldAllowWinStyleFlags( flags ) )
 
     if results.get( 'do_cache', True ):
       self.flags_for_file[ filename ] = sanitized_flags
@@ -237,6 +241,16 @@ def _ExtractFlagsList( flags_for_file_output ):
   return [ ToUnicode( x ) for x in flags_for_file_output[ 'flags' ] ]
 
 
+def _ShouldAllowWinStyleFlags( flags ):
+  enable_windows_style_flags = False
+  if OnWindows():
+    for flag in flags:
+      if flag.startswith( '--driver-mode' ):
+        enable_windows_style_flags = ( flag == '--driver-mode=cl' )
+
+  return enable_windows_style_flags
+
+
 def _CallExtraConfFlagsForFile( module, filename, client_data ):
   # We want to ensure we pass a native py2 `str` on py2 and a native py3 `str`
   # (unicode) object on py3. That's the API we provide.
@@ -272,10 +286,13 @@ def _SysRootSpecifedIn( flags ):
   return False
 
 
-def PrepareFlagsForClang( flags, filename, add_extra_clang_flags = True ):
-  flags = _AddLanguageFlagWhenAppropriate( flags )
+def PrepareFlagsForClang( flags,
+                          filename,
+                          add_extra_clang_flags = True,
+                          enable_windows_style_flags = False ):
+  flags = _AddLanguageFlagWhenAppropriate( flags, enable_windows_style_flags )
   flags = _RemoveXclangFlags( flags )
-  flags = _RemoveUnusedFlags( flags, filename )
+  flags = _RemoveUnusedFlags( flags, filename, enable_windows_style_flags )
   if add_extra_clang_flags:
     flags = _EnableTypoCorrection( flags )
 
@@ -305,18 +322,21 @@ def _RemoveXclangFlags( flags ):
   return sanitized_flags
 
 
-def _RemoveFlagsPrecedingCompiler( flags ):
-  """Assuming that the flag just before the first flag (which starts with a
-  dash) is the compiler path, removes all flags preceding it."""
+def _RemoveFlagsPrecedingCompiler( flags, enable_windows_style_flags ):
+  """Assuming that the flag just before the first flag (looks like a flag,
+  not like a file path) is the compiler path, removes all flags preceding it."""
 
   for index, flag in enumerate( flags ):
-    if flag.startswith( '-' ):
+    if ( flag.startswith( '-' ) or
+         ( enable_windows_style_flags and
+           flag.startswith( '/' ) and
+           not os.path.exists( flag ) ) ):
       return ( flags[ index - 1: ] if index > 1 else
                flags )
   return flags[ :-1 ]
 
 
-def _AddLanguageFlagWhenAppropriate( flags ):
+def _AddLanguageFlagWhenAppropriate( flags, enable_windows_style_flags ):
   """When flags come from the compile_commands.json file, the flag preceding the
   first flag starting with a dash is usually the path to the compiler that
   should be invoked. Since LibClang does not deduce the language from the
@@ -325,18 +345,36 @@ def _AddLanguageFlagWhenAppropriate( flags ):
   the file extension. This handles the case where the .h extension is used for
   C++ headers."""
 
-  flags = _RemoveFlagsPrecedingCompiler( flags )
+  flags = _RemoveFlagsPrecedingCompiler( flags, enable_windows_style_flags )
 
-  # First flag is now the compiler path or a flag starting with a dash.
+  # First flag is now the compiler path, a flag starting with a dash or
+  # a flag starting with a forward slash if enable_windows_style_flags is True.
   first_flag = flags[ 0 ]
 
+  # NOTE: This is intentionally NOT checking for enable_windows_style_flags.
+  #
+  # Because of _RemoveFlagsPrecedingCompiler called above, irrelevant of
+  # enable_windows_style_flags. the first flag is either the compiler
+  # (path or executable), a Windows style flag or starts with a dash.
+  #
+  # If it doesn't start with a dash, it is either an absolute path,
+  # a Windows style flag or a C++ compiler executable from $PATH.
+  #   If it starts with a forward slash the flag can either be an absolute
+  #   flag or a Windows style flag.
+  #     If it matches the regex, it is safe to assume the flag is a compiler
+  #     path.
+  #     If it does not match the regex, it could still be a Windows style
+  #     path or an absolute path. - This is determined in _RemoveUnusedFlags()
+  #     and cleaned properly.
+  #   If the flag starts with anything else (i.e. not a '-' or a '/'), the flag
+  #   is a stray file path and shall be gotten rid of in _RemoveUnusedFlags().
   if ( not first_flag.startswith( '-' ) and
        CPP_COMPILER_REGEX.search( first_flag ) ):
     return [ first_flag, '-x', 'c++' ] + flags[ 1: ]
   return flags
 
 
-def _RemoveUnusedFlags( flags, filename ):
+def _RemoveUnusedFlags( flags, filename, enable_windows_style_flags ):
   """Given an iterable object that produces strings (flags for Clang), removes
   the '-c' and '-o' options that Clang does not like to see when it's producing
   completions for a file. Same for '-MD' etc.
@@ -354,20 +392,21 @@ def _RemoveUnusedFlags( flags, filename ):
     flags = flags[ 1: ]
 
   skip_next = False
-  previous_flag_is_include = False
-  previous_flag_starts_with_dash = False
-  current_flag_starts_with_dash = False
+  previous_flag = flags[ 0 ]
+  current_flag = flags[ 0 ]
 
   filename = os.path.realpath( filename )
   for flag in flags:
-    previous_flag_starts_with_dash = current_flag_starts_with_dash
-    current_flag_starts_with_dash = flag.startswith( '-' )
+    previous_flag = current_flag
+    current_flag = flag
 
     if skip_next:
       skip_next = False
       continue
 
-    if flag in STATE_FLAGS_TO_SKIP:
+    if ( flag in STATE_FLAGS_TO_SKIP or
+         ( enable_windows_style_flags and
+           flag in STATE_FLAGS_TO_SKIP_WIN_STYLE ) ):
       continue
 
     if flag in FILE_FLAGS_TO_SKIP:
@@ -383,16 +422,40 @@ def _RemoveUnusedFlags( flags, filename ):
     # "foo.cpp" when we are compiling "foo.h" because the comp db doesn't have
     # flags for headers. The returned flags include "foo.cpp" and we need to
     # remove that.
-    if ( not current_flag_starts_with_dash and
-          ( not previous_flag_starts_with_dash or
-            ( not previous_flag_is_include and '/' in flag ) ) ):
+    if _SkipStrayFilenameFlag( flag,
+                               current_flag,
+                               previous_flag,
+                               enable_windows_style_flags ):
       continue
 
     new_flags.append( flag )
-    previous_flag_is_include = flag in INCLUDE_FLAGS
+
   return new_flags
 
 
+def _SkipStrayFilenameFlag( flag,
+                            current_flag,
+                            previous_flag,
+                            enable_windows_style_flags ):
+  current_flag_starts_with_slash = current_flag.startswith( '/' )
+  previous_flag_starts_with_slash = previous_flag.startswith( '/' )
+
+  current_flag_starts_with_dash = current_flag.startswith( '-' )
+  previous_flag_starts_with_dash = previous_flag.startswith( '-' )
+
+  previous_flag_is_include = ( previous_flag in INCLUDE_FLAGS or
+                               ( enable_windows_style_flags and
+                                 flag in INCLUDE_FLAGS_WIN_STYLE ) )
+
+  return ( not ( current_flag_starts_with_dash or
+                 ( enable_windows_style_flags and
+                   current_flag_starts_with_slash ) ) and
+           ( not ( previous_flag_starts_with_dash or
+                   ( enable_windows_style_flags and
+                     previous_flag_starts_with_slash ) ) or
+             ( not previous_flag_is_include and '/' in flag ) ) )
+
+
 # Return the path to the macOS toolchain root directory to use for system
 # includes. If no toolchain is found, returns None.
 def _SelectMacToolchain():
@@ -532,6 +595,9 @@ def _MakeRelativePathsInFlagsAbsolute( flags, working_directory ):
     return list( flags )
   new_flags = []
   make_next_absolute = False
+  path_flags = ( PATH_FLAGS + INCLUDE_FLAGS_WIN_STYLE
+                 if _ShouldAllowWinStyleFlags( flags )
+                 else PATH_FLAGS )
   for flag in flags:
     new_flag = flag
 
@@ -541,7 +607,7 @@ def _MakeRelativePathsInFlagsAbsolute( flags, working_directory ):
         new_flag = os.path.join( working_directory, flag )
       new_flag = os.path.normpath( new_flag )
     else:
-      for path_flag in PATH_FLAGS:
+      for path_flag in path_flags:
         # Single dash argument alone, e.g. -isysroot <path>
         if flag == path_flag:
           make_next_absolute = True
@@ -592,10 +658,11 @@ def _GetCompilationInfoForFile( database, file_name, file_extension ):
 def UserIncludePaths( flags, filename ):
   quoted_include_paths = [ os.path.dirname( filename ) ]
   include_paths = []
-
   if flags:
     quote_flag = '-iquote'
-    path_flags = [ '-isystem', '-I' ]
+    path_flags = ( [ '-isystem', '-I', '/I' ]
+                   if _ShouldAllowWinStyleFlags( flags )
+                   else [ '-isystem', '-I' ] )
 
     try:
       it = iter( flags )