Skip to content

Commit

Permalink
tools,gyp: add support for MSVC cross-compilation
Browse files Browse the repository at this point in the history 
This change means that GYP can now generate two sets of projects: one
exclusively for a host x64 machine and one containing a mix of x64 and
Arm targets. The names of host targets are fixed up to end with
_host.exe, and any actions involving them are fixed up. This allows
compilation of Node on an x64 server for a Windows on Arm target.

PR-URL: nodejs#32867
Reviewed-By: Christian Clauss <[email protected]>
Reviewed-By: João Reis <[email protected]>
  • Loading branch information
@richard-townsend-arm @richardlau
richard-townsend-arm authored and richardlau committed Aug 18, 2021
1 parent faae1ee commit 24b3ec5
Showing 1 changed file with 81 additions and 38 deletions.
119 changes: 81 additions & 38 deletions tools/gyp/pylib/gyp/generator/msvs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
# letters.
VALID_MSVS_GUID_CHARS = re.compile(r'^[A-F0-9\-]+$')

generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested()

generator_default_variables = {
'DRIVER_PREFIX': '',
Expand All @@ -51,7 +52,7 @@
'STATIC_LIB_SUFFIX': '.lib',
'SHARED_LIB_SUFFIX': '.dll',
'INTERMEDIATE_DIR': '$(IntDir)',
'SHARED_INTERMEDIATE_DIR': '$(OutDir)obj/global_intermediate',
'SHARED_INTERMEDIATE_DIR': '$(OutDir)/obj/global_intermediate',
'OS': 'win',
'PRODUCT_DIR': '$(OutDir)',
'LIB_DIR': '$(OutDir)lib',
Expand Down Expand Up @@ -286,7 +287,7 @@ def _ConfigTargetVersion(config_data):
return config_data.get('msvs_target_version', 'Windows7')


def _ConfigPlatform(config_data):
def _ConfigPlatform(config_data, spec):
return config_data.get('msvs_configuration_platform', 'Win32')


Expand All @@ -297,8 +298,8 @@ def _ConfigBaseName(config_name, platform_name):
return config_name


def _ConfigFullName(config_name, config_data):
platform_name = _ConfigPlatform(config_data)
def _ConfigFullName(config_name, config_data, spec):
platform_name = _ConfigPlatform(config_data, spec)
return '%s|%s' % (_ConfigBaseName(config_name, platform_name), platform_name)


Expand Down Expand Up @@ -951,7 +952,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version):
return toolset


def _GenerateProject(project, options, version, generator_flags):
def _GenerateProject(project, options, version, generator_flags, spec):
"""Generates a vcproj file.
Arguments:
Expand All @@ -969,7 +970,7 @@ def _GenerateProject(project, options, version, generator_flags):
return []

if version.UsesVcxproj():
return _GenerateMSBuildProject(project, options, version, generator_flags)
return _GenerateMSBuildProject(project, options, version, generator_flags, spec)
else:
return _GenerateMSVSProject(project, options, version, generator_flags)

Expand Down Expand Up @@ -1091,7 +1092,7 @@ def _GetUniquePlatforms(spec):
# Gather list of unique platforms.
platforms = OrderedSet()
for configuration in spec['configurations']:
platforms.add(_ConfigPlatform(spec['configurations'][configuration]))
platforms.add(_ConfigPlatform(spec['configurations'][configuration], spec))
platforms = list(platforms)
return platforms

Expand Down Expand Up @@ -1801,6 +1802,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat):
# Convert into a tree of dicts on path.
for p in sln_projects:
gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2]
if p.endswith("#host"):
target += "_host"
gyp_dir = os.path.dirname(gyp_file)
path_dict = _GetPathDict(root, gyp_dir)
path_dict[target + '.vcproj'] = project_objects[p]
Expand All @@ -1819,7 +1822,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version):
default_config = _GetDefaultConfiguration(spec)
proj_filename = default_config.get('msvs_existing_vcproj')
if not proj_filename:
proj_filename = (spec['target_name'] + options.suffix +
proj_filename = spec['target_name']
if spec['toolset'] == 'host':
proj_filename += "_host"
proj_filename = (proj_filename + options.suffix +
msvs_version.ProjectExtension())

build_file = gyp.common.BuildFile(qualified_target)
Expand All @@ -1838,10 +1844,12 @@ def _GetPlatformOverridesOfProject(spec):
# solution configurations for this target.
config_platform_overrides = {}
for config_name, c in spec['configurations'].items():
config_fullname = _ConfigFullName(config_name, c)
platform = c.get('msvs_target_platform', _ConfigPlatform(c))
config_fullname = _ConfigFullName(config_name, c, spec)
platform = c.get('msvs_target_platform', _ConfigPlatform(c, spec))
fixed_config_fullname = '%s|%s' % (
_ConfigBaseName(config_name, _ConfigPlatform(c)), platform)
_ConfigBaseName(config_name, _ConfigPlatform(c, spec)), platform)
if spec['toolset'] == 'host' and generator_supports_multiple_toolsets:
fixed_config_fullname = '%s|x64' % (config_name,)
config_platform_overrides[config_fullname] = fixed_config_fullname
return config_platform_overrides

Expand All @@ -1862,19 +1870,18 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
projects = {}
for qualified_target in target_list:
spec = target_dicts[qualified_target]
if spec['toolset'] != 'target':
raise GypError(
'Multiple toolsets not supported in msvs build (target %s)' %
qualified_target)
proj_path, fixpath_prefix = _GetPathOfProject(qualified_target, spec,
options, msvs_version)
guid = _GetGuidOfProject(proj_path, spec)
overrides = _GetPlatformOverridesOfProject(spec)
build_file = gyp.common.BuildFile(qualified_target)
# Create object for this project.
target_name = spec['target_name']
if spec['toolset'] == 'host':
target_name += '_host'
obj = MSVSNew.MSVSProject(
proj_path,
name=spec['target_name'],
name=target_name,
guid=guid,
spec=spec,
build_file=build_file,
Expand Down Expand Up @@ -2041,7 +2048,10 @@ def GenerateOutput(target_list, target_dicts, data, params):
for qualified_target in target_list:
spec = target_dicts[qualified_target]
for config_name, config in spec['configurations'].items():
configs.add(_ConfigFullName(config_name, config))
config_name = _ConfigFullName(config_name, config, spec)
configs.add(config_name)
if config_name == 'Release|arm64':
configs.add("Release|x64")
configs = list(configs)

# Figure out all the projects that will be generated and their guids
Expand All @@ -2053,11 +2063,14 @@ def GenerateOutput(target_list, target_dicts, data, params):
for project in project_objects.values():
fixpath_prefix = project.fixpath_prefix
missing_sources.extend(_GenerateProject(project, options, msvs_version,
generator_flags))
generator_flags, spec))
fixpath_prefix = None

for build_file in data:
# Validate build_file extension
target_only_configs = configs
if generator_supports_multiple_toolsets:
target_only_configs = [i for i in configs if i.endswith('arm64')]
if not build_file.endswith('.gyp'):
continue
sln_path = os.path.splitext(build_file)[0] + options.suffix + '.sln'
Expand All @@ -2072,7 +2085,7 @@ def GenerateOutput(target_list, target_dicts, data, params):
# Create solution.
sln = MSVSNew.MSVSSolution(sln_path,
entries=root_entries,
variants=configs,
variants=target_only_configs,
websiteProperties=False,
version=msvs_version)
sln.Write()
Expand Down Expand Up @@ -2674,21 +2687,23 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules):
easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True)


def _GetConfigurationAndPlatform(name, settings):
def _GetConfigurationAndPlatform(name, settings, spec):
configuration = name.rsplit('_', 1)[0]
platform = settings.get('msvs_configuration_platform', 'Win32')
if spec['toolset'] == 'host' and platform == 'arm64':
platform = 'x64' # Host-only tools are always built for x64
return (configuration, platform)


def _GetConfigurationCondition(name, settings):
def _GetConfigurationCondition(name, settings, spec):
return (r"'$(Configuration)|$(Platform)'=='%s|%s'" %
_GetConfigurationAndPlatform(name, settings))
_GetConfigurationAndPlatform(name, settings, spec))


def _GetMSBuildProjectConfigurations(configurations):
def _GetMSBuildProjectConfigurations(configurations, spec):
group = ['ItemGroup', {'Label': 'ProjectConfigurations'}]
for (name, settings) in sorted(configurations.items()):
configuration, platform = _GetConfigurationAndPlatform(name, settings)
configuration, platform = _GetConfigurationAndPlatform(name, settings, spec)
designation = '%s|%s' % (configuration, platform)
group.append(
['ProjectConfiguration', {'Include': designation},
Expand Down Expand Up @@ -2740,7 +2755,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name):
platform_name = None
msvs_windows_sdk_version = None
for configuration in spec['configurations'].values():
platform_name = platform_name or _ConfigPlatform(configuration)
platform_name = platform_name or _ConfigPlatform(configuration, spec)
msvs_windows_sdk_version = (msvs_windows_sdk_version or
_ConfigWindowsTargetPlatformVersion(configuration, version))
if platform_name and msvs_windows_sdk_version:
Expand All @@ -2762,7 +2777,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file):
properties = {}
for name, settings in spec['configurations'].items():
msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file)
condition = _GetConfigurationCondition(name, settings)
condition = _GetConfigurationCondition(name, settings, spec)
character_set = msbuild_attributes.get('CharacterSet')
config_type = msbuild_attributes.get('ConfigurationType')
_AddConditionalProperty(properties, condition, 'ConfigurationType',
Expand Down Expand Up @@ -2790,12 +2805,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset):
return properties


def _GetMSBuildPropertySheets(configurations):
def _GetMSBuildPropertySheets(configurations, spec):
user_props = r'$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props'
additional_props = {}
props_specified = False
for name, settings in sorted(configurations.items()):
configuration = _GetConfigurationCondition(name, settings)
configuration = _GetConfigurationCondition(name, settings, spec)
if 'msbuild_props' in settings:
additional_props[configuration] = _FixPaths(settings['msbuild_props'])
props_specified = True
Expand Down Expand Up @@ -2946,7 +2961,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):

properties = {}
for (name, configuration) in sorted(configurations.items()):
condition = _GetConfigurationCondition(name, configuration)
condition = _GetConfigurationCondition(name, configuration, spec)
attributes = _GetMSBuildAttributes(spec, configuration, build_file)
msbuild_settings = configuration['finalized_msbuild_settings']
_AddConditionalProperty(properties, condition, 'IntDir',
Expand Down Expand Up @@ -3055,7 +3070,9 @@ def _GetMSBuildToolSettingsSections(spec, configurations):
for (name, configuration) in sorted(configurations.items()):
msbuild_settings = configuration['finalized_msbuild_settings']
group = ['ItemDefinitionGroup',
{'Condition': _GetConfigurationCondition(name, configuration)}
{'Condition':
_GetConfigurationCondition(name, configuration, spec)
}
]
for tool_name, tool_settings in sorted(msbuild_settings.items()):
# Skip the tool named '' which is a holder of global settings handled
Expand Down Expand Up @@ -3277,7 +3294,9 @@ def _AddSources2(spec, sources, exclusions, grouped_sources,
extensions_excluded_from_precompile = ['.c']

if precompiled_source == source:
condition = _GetConfigurationCondition(config_name, configuration)
condition = _GetConfigurationCondition(
config_name, configuration, spec
)
detail.append(['PrecompiledHeader',
{'Condition': condition},
'Create'
Expand All @@ -3296,12 +3315,26 @@ def _AddSources2(spec, sources, exclusions, grouped_sources,
_GetUniquePlatforms(spec))
grouped_sources[group].append([element, {'Include': source}] + detail)


def _GetMSBuildProjectReferences(project):
def _GetMSBuildProjectReferences(project, spec):
current_configuration = spec['default_configuration']
references = []
if project.dependencies:
group = ['ItemGroup']
added_dependency_set = set()
for dependency in project.dependencies:
dependency_spec = dependency.spec
should_skip_dep = False
if project.spec["toolset"] == 'target':
if dependency_spec['toolset'] == 'host':
if dependency_spec['type'] == 'static_library':
should_skip_dep = True
if dependency.name.startswith('run_'):
should_skip_dep = False
if should_skip_dep:
continue

canonical_name = dependency.name.replace('_host', '')
added_dependency_set.add(canonical_name)
guid = dependency.guid
project_dir = os.path.split(project.path)[0]
relative_path = gyp.common.RelativePath(dependency.path, project_dir)
Expand All @@ -3323,7 +3356,7 @@ def _GetMSBuildProjectReferences(project):
return references


def _GenerateMSBuildProject(project, options, version, generator_flags):
def _GenerateMSBuildProject(project, options, version, generator_flags, spec):
spec = project.spec
configurations = spec['configurations']
project_dir, project_file_name = os.path.split(project.path)
Expand Down Expand Up @@ -3411,7 +3444,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
'DefaultTargets': 'Build'
}]

content += _GetMSBuildProjectConfigurations(configurations)
content += _GetMSBuildProjectConfigurations(configurations, spec)
content += _GetMSBuildGlobalProperties(spec, version, project.guid,
project_file_name)
content += import_default_section
Expand All @@ -3422,18 +3455,18 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
content += _GetMSBuildLocalProperties(project.msbuild_toolset)
content += import_cpp_props_section
content += import_masm_props_section
if spec.get('msvs_enable_marmasm'):
if spec.get('msvs_enable_marmasm') or True:
content += import_marmasm_props_section
content += _GetMSBuildExtensions(props_files_of_rules)
content += _GetMSBuildPropertySheets(configurations)
content += _GetMSBuildPropertySheets(configurations, spec)
content += macro_section
content += _GetMSBuildConfigurationGlobalProperties(spec, configurations,
project.build_file)
content += _GetMSBuildToolSettingsSections(spec, configurations)
content += _GetMSBuildSources(
spec, sources, exclusions, rule_dependencies, extension_to_rule_name,
actions_spec, sources_handled_by_action, list_excluded)
content += _GetMSBuildProjectReferences(project)
content += _GetMSBuildProjectReferences(project, spec)
content += import_cpp_targets_section
content += import_masm_targets_section
if spec.get('msvs_enable_marmasm'):
Expand Down Expand Up @@ -3516,15 +3549,25 @@ def _GenerateActionsForMSBuild(spec, actions_to_add):
sources_handled_by_action = OrderedSet()
actions_spec = []
for primary_input, actions in actions_to_add.items():
if generator_supports_multiple_toolsets:
primary_input = primary_input.replace(".exe", "_host.exe")
inputs = OrderedSet()
outputs = OrderedSet()
descriptions = []
commands = []
for action in actions:
def fixup_host_exe(i):
if "$(OutDir)" in i:
i = i.replace('.exe', '_host.exe')
return i
if generator_supports_multiple_toolsets:
action['inputs'] = [fixup_host_exe(i) for i in action['inputs']]
inputs.update(OrderedSet(action['inputs']))
outputs.update(OrderedSet(action['outputs']))
descriptions.append(action['description'])
cmd = action['command']
if generator_supports_multiple_toolsets:
cmd = cmd.replace('.exe', "_host.exe")
# For most actions, add 'call' so that actions that invoke batch files
# return and continue executing. msbuild_use_call provides a way to
# disable this but I have not seen any adverse effect from doing that
Expand Down

0 comments on commit 24b3ec5

Please sign in to comment.