Update build system so it will compile on Windows 10

With this commit, TigerVNC will compile for Windows using MSYS2, MinGW-w64 and Inno Setup 6.  The resulting binaries have some dll dependencies (even with BUILD_STATIC) and those are included in the installer.

Unfortunately, the latest version of MSYS2 and MinGW-w64 do not produce a working executable for Windows 7.

cmake/BuildPackages.cmake

@@ -8,6 +8,12 @@
 
 if(WIN32)
 
+if(CMAKE_GENERATOR STREQUAL "MinGW Makefiles")
+  set(WIN_DLL_PATH "C:/msys64/mingw64/bin/")
+else()
+  set(WIN_DLL_PATH "")
+endif()
+
 if(CMAKE_SIZEOF_VOID_P MATCHES 8)
   set(INST_NAME ${CMAKE_PROJECT_NAME}64-${VERSION})
   set(INST_DEFS -DWIN64)
@@ -25,6 +31,15 @@ if(BUILD_WINVNC)
   set(INST_DEPS ${INST_DEPS} winvnc4 wm_hooks vncconfig)
 endif()
 
+# Variables have to be passed manually to iscc
+if(ENABLE_GNUTLS)
+  set(INST_DEFS ${INST_DEFS} -DENABLE_GNUTLS)
+endif()
+if(BUILD_STATIC)
+  set(INST_DEFS ${INST_DEFS} -DBUILD_STATIC)
+endif()
+
+
 configure_file(release/tigervnc.iss.in release/tigervnc.iss)
 
 add_custom_target(installer

cmake/StaticBuild.cmake

@@ -23,7 +23,31 @@ if(BUILD_STATIC)
 
   # gettext is included in libc on many unix systems
   if(NOT LIBC_HAS_DGETTEXT)
-    set(GETTEXT_LIBRARIES "-Wl,-Bstatic -lintl -liconv -Wl,-Bdynamic")
+    FIND_LIBRARY(IDN2_LIBRARY NAMES idn2 libidn2
+      HINTS ${PC_GETTEXT_LIBDIR} ${PC_GETTEXT_LIBRARY_DIRS})
+    FIND_LIBRARY(UNISTRING_LIBRARY NAMES unistring libunistring
+      HINTS ${PC_GETTEXT_LIBDIR} ${PC_GETTEXT_LIBRARY_DIRS})
+
+    set(GETTEXT_LIBRARIES "-Wl,-Bstatic")
+
+    if(IDN2_LIBRARY)
+      set(GETTEXT_LIBRARIES "${GETTEXT_LIBRARIES} -lidn2")
+    endif()
+
+    set(GETTEXT_LIBRARIES "${GETTEXT_LIBRARIES} -lintl")
+
+    if(UNISTRING_LIBRARY)
+      set(GETTEXT_LIBRARIES "${GETTEXT_LIBRARIES} -lunistring")
+    endif()
+
+    set(GETTEXT_LIBRARIES "${GETTEXT_LIBRARIES} -liconv")
+
+    set(GETTEXT_LIBRARIES "${GETTEXT_LIBRARIES} -Wl,-Bdynamic")
+
+    if(CMAKE_GENERATOR STREQUAL "MinGW Makefiles" AND UNISTRING_LIBRARY)
+      set(GETTEXT_LIBRARIES "${GETTEXT_LIBRARIES} -lunistring")
+    endif()
+
     if(APPLE)
       set(GETTEXT_LIBRARIES "${GETTEXT_LIBRARIES} -framework Carbon")
     endif()
@@ -57,6 +81,12 @@ if(BUILD_STATIC)
       set(GNUTLS_LIBRARIES "${GNUTLS_LIBRARIES} -lcrypt32")
       # And sockets
       set(GNUTLS_LIBRARIES "${GNUTLS_LIBRARIES} -lws2_32")
+      if(CMAKE_GENERATOR STREQUAL "MinGW Makefiles")
+        # only available as a dll
+        set(GNUTLS_LIBRARIES "${GNUTLS_LIBRARIES} -lp11-kit")
+        # the static link libarary is missing required defines.  I think this is a problem in the gnutls static link library.
+        set(GNUTLS_LIBRARIES "${GNUTLS_LIBRARIES} -Wl,-Bstatic -lunistring -Wl,-Bdynamic -lunistring")
+      endif()
     endif()
 
     if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
@@ -134,6 +164,10 @@ if(BUILD_STATIC_GCC)
   endif()
   if(WIN32)
     set(STATIC_BASE_LIBRARIES "${STATIC_BASE_LIBRARIES} -lmingw32 -lgcc_eh -lgcc -lmoldname -lmingwex -lmsvcrt")
+    if(CMAKE_GENERATOR STREQUAL "MinGW Makefiles")
+      # pthread has to be statically linked after libraries above and before kernel32
+      set(STATIC_BASE_LIBRARIES "${STATIC_BASE_LIBRARIES} -Wl,-Bstatic -lpthread -Wl,-Bdynamic")
+    endif()
     set(STATIC_BASE_LIBRARIES "${STATIC_BASE_LIBRARIES} -luser32 -lkernel32 -ladvapi32 -lshell32")
     # mingw has some fun circular dependencies that requires us to link
     # these things again

release/tigervnc.iss.in

@@ -24,6 +24,33 @@ LicenseFile=@CMAKE_SOURCE_DIR@\LICENCE.txt
 Name: "{sys}\config\systemprofile\Desktop"
 
 [Files]
+#ifndef BUILD_STATIC
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+Source: "@WIN_DLL_PATH@libgcc_s_seh-1.dll"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+Source: "@WIN_DLL_PATH@libstdc++-6.dll"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+#ifdef ENABLE_GNUTLS
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;  
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+#endif
+#else
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
+Source: "@[email protected]"; DestDir: "{app}"; Flags: ignoreversion restartreplace;
+#endif
 #ifdef BUILD_WINVNC
 Source: "@CMAKE_CURRENT_BINARY_DIR@\win\winvnc\winvnc4.exe"; DestDir: "{app}"; Flags: ignoreversion restartreplace; 
 Source: "@CMAKE_CURRENT_BINARY_DIR@\win\wm_hooks\wm_hooks.dll"; DestDir: "{app}"; Flags: ignoreversion restartreplace;