Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ci] Switch Windows build script to build.py #6993

Merged
merged 52 commits into from
Jan 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
af13a26
Temp remove testing(save time)
feisuzhu Dec 28, 2022
df4ef21
switch windows build script
feisuzhu Dec 28, 2022
5c268e5
should fail
feisuzhu Dec 28, 2022
34be3d0
setup clang
feisuzhu Dec 28, 2022
7148035
check local cache size
feisuzhu Dec 28, 2022
47804c9
tinysh use subprocess
feisuzhu Dec 28, 2022
d32838b
win miniforge fix
feisuzhu Dec 28, 2022
e78abfd
meh
feisuzhu Dec 28, 2022
d36fe31
meh
feisuzhu Dec 28, 2022
567623f
meh
feisuzhu Dec 28, 2022
711f924
conda
feisuzhu Dec 29, 2022
a52ff8d
path quirk
feisuzhu Dec 29, 2022
d09d972
win llvm strip
feisuzhu Dec 29, 2022
642f678
Revert "Temp remove testing(save time)"
feisuzhu Jan 4, 2023
7156f6d
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 4, 2023
a80dc3a
Disable tests (2nd)
feisuzhu Jan 4, 2023
1b0c151
vulkan
feisuzhu Jan 5, 2023
58ddbd3
minoor
feisuzhu Jan 6, 2023
a0a6cbd
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 6, 2023
eee4a1e
revert to plain os.environ
feisuzhu Jan 9, 2023
3f7a798
meh
feisuzhu Jan 9, 2023
54b3bce
meh
feisuzhu Jan 9, 2023
9deac6f
meh
feisuzhu Jan 9, 2023
e530911
setup msvc
feisuzhu Jan 10, 2023
f2ab6ff
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 10, 2023
e635298
auto install vc buildtools 2022
feisuzhu Jan 10, 2023
c046968
Revert "Disable tests (2nd)"
feisuzhu Jan 16, 2023
fef6e6b
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 16, 2023
951629b
meh
feisuzhu Jan 16, 2023
d9ff019
REMOVE OTHER STEPS FOR DEBUG
feisuzhu Jan 16, 2023
42b310e
use cmd
feisuzhu Jan 16, 2023
509ce05
fixup
feisuzhu Jan 16, 2023
191f28d
meh
feisuzhu Jan 16, 2023
b727a10
meh
feisuzhu Jan 16, 2023
b02ac5a
meh
feisuzhu Jan 16, 2023
3cd21bb
isolated dependency dir for build.py
feisuzhu Jan 16, 2023
120b751
clean build env
feisuzhu Jan 16, 2023
8c5fcc9
cmake env
feisuzhu Jan 16, 2023
1f555c3
more isolated
feisuzhu Jan 16, 2023
80b4b39
meh
feisuzhu Jan 16, 2023
53d95db
meh
feisuzhu Jan 17, 2023
6f11675
meh
feisuzhu Jan 17, 2023
04915b8
clang
feisuzhu Jan 17, 2023
1a62b1b
fix
feisuzhu Jan 17, 2023
a6861e4
Revert "REMOVE OTHER STEPS FOR DEBUG"
feisuzhu Jan 17, 2023
9ef900e
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 17, 2023
27956a5
remove todo
feisuzhu Jan 17, 2023
bf80b99
vs2022 packages
feisuzhu Jan 17, 2023
7f41f3f
move vulkan on to testing.yml
feisuzhu Jan 17, 2023
6896856
clean up debug messes
feisuzhu Jan 17, 2023
7e30bd2
resolve clang++ path pollution
feisuzhu Jan 17, 2023
966b26d
path escaping
feisuzhu Jan 17, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 115 additions & 34 deletions .github/workflows/scripts/build.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,76 @@
#!/usr/bin/env python3

# -- prioritized --
import ci_common # isort: skip, early initialization happens here

# -- stdlib --
import glob
import os
import platform
from pathlib import Path

# -- third party --
# -- own --
from ci_common.dep import download_dep
from ci_common.misc import banner, get_cache_home, is_manylinux2014
from ci_common.python import setup_python
from ci_common.misc import (banner, get_cache_home, is_manylinux2014,
path_prepend)
from ci_common.python import path_prepend, setup_python
from ci_common.sccache import setup_sccache
from ci_common.tinysh import Command, environ, git, sh
from ci_common.tinysh import Command, git, sh


# -- code --
@banner('Setup Clang')
def setup_clang(as_compiler=True) -> None:
'''
Setup Clang.
'''
u = platform.uname()
if u.system == 'Linux':
pass
elif (u.system, u.machine) == ('Windows', 'AMD64'):
out = get_cache_home() / 'clang-15'
url = 'https://github.com/python3kgae/taichi_assets/releases/download/llvm15_vs2022_clang/clang-15.0.0-win.zip'
feisuzhu marked this conversation as resolved.
Show resolved Hide resolved
download_dep(url, out)
clang = str(out / 'bin' / 'clang++.exe').replace('\\', '\\\\')
os.environ['TAICHI_CMAKE_ARGS'] += f' -DCLANG_EXECUTABLE={clang}'

if as_compiler:
os.environ['TAICHI_CMAKE_ARGS'] += (
f' -DCMAKE_CXX_COMPILER={clang}'
f' -DCMAKE_C_COMPILER={clang}')
else:
# TODO: unify all
pass


@banner('Setup MSVC')
def setup_msvc() -> None:
assert platform.system() == 'Windows'
os.environ['TAICHI_USE_MSBUILD'] = '1'

url = 'https://aka.ms/vs/17/release/vs_BuildTools.exe'
out = Path(
r'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools')
download_dep(
url,
out,
args=[
'--passive',
'--wait',
'--norestart',
'--includeRecommended',
'--add',
'Microsoft.VisualStudio.Workload.VCTools',
# NOTE: We are using the custom built Clang++,
# so components below are not necessary anymore.
# '--add',
# 'Microsoft.VisualStudio.Component.VC.Llvm.Clang',
# '--add',
# 'Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Llvm.Clang',
# '--add',
# 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset',
])


@banner('Setup LLVM')
def setup_llvm(env_out: dict) -> None:
def setup_llvm() -> None:
'''
Download and install LLVM.
'''
Expand All @@ -28,7 +79,7 @@ def setup_llvm(env_out: dict) -> None:
if 'AMDGPU_TEST' in os.environ:
# FIXME: AMDGPU bots are currently maintained separately,
# we should unify them with the rest of the bots.
env_out['LLVM_DIR'] = '/taichi-llvm-15'
os.environ['LLVM_DIR'] = '/taichi-llvm-15'
return
elif is_manylinux2014():
# FIXME: prebuilt llvm15 on ubuntu didn't work on manylinux2014 image of centos. Once that's fixed, remove this hack.
Expand All @@ -37,54 +88,82 @@ def setup_llvm(env_out: dict) -> None:
else:
out = get_cache_home() / 'llvm15'
url = 'https://github.com/taichi-dev/taichi_assets/releases/download/llvm15/taichi-llvm-15-linux.zip'
download_dep(url, out, strip=1)
elif (u.system, u.machine) == ('Darwin', 'arm64'):
out = get_cache_home() / 'llvm15-m1'
url = 'https://github.com/taichi-dev/taichi_assets/releases/download/llvm15/taichi-llvm-15-m1.zip'
download_dep(url, out, strip=1)
elif (u.system, u.machine) == ('Darwin', 'x86_64'):
out = get_cache_home() / 'llvm15-mac'
url = 'https://github.com/taichi-dev/taichi_assets/releases/download/llvm15/llvm-15-mac10.15.zip'
download_dep(url, out, strip=1)
elif (u.system, u.machine) == ('Windows', 'AMD64'):
out = get_cache_home() / 'llvm15'
url = 'https://github.com/python3kgae/taichi_assets/releases/download/llvm15_vs2019_clang/taichi-llvm-15.0.0-msvc2019.zip'
feisuzhu marked this conversation as resolved.
Show resolved Hide resolved
download_dep(url, out, strip=0)
else:
raise RuntimeError(f'Unsupported platform: {u.system} {u.machine}')

download_dep(url, out, strip=1)
env_out['LLVM_DIR'] = str(out)
path_prepend('PATH', out / 'bin')
os.environ['LLVM_DIR'] = str(out)


@banner('Setup Vulkan 1.3.236.0')
def setup_vulkan(env: dict):
def setup_vulkan():
u = platform.uname()
if u.system == "Linux":
url = 'https://sdk.lunarg.com/sdk/download/1.3.236.0/linux/vulkansdk-linux-x86_64-1.3.236.0.tar.gz'
prefix = get_cache_home() / 'vulkan-1.3.236.0'
download_dep(url, prefix, strip=1)
sdk = prefix / 'x86_64'
env['VULKAN_SDK'] = str(sdk)
env['PATH'] = str(sdk / "bin") + ':' + env["PATH"]
env['LD_LIBRARY_PATH'] = str(sdk / "lib") + ':' + env.get(
"LD_LIBRARY_PATH", "")
env['VK_LAYER_PATH'] = str(sdk / 'etc' / 'vulkan' / 'explicit_layer.d')
os.environ['VULKAN_SDK'] = str(sdk)
path_prepend('PATH', sdk / "bin")
path_prepend('LD_LIBRARY_PATH', sdk / 'lib')
os.environ['VK_LAYER_PATH'] = str(sdk / 'etc' / 'vulkan' /
'explicit_layer.d')
# elif (u.system, u.machine) == ("Darwin", "arm64"):
# elif (u.system, u.machine) == ("Darwin", "x86_64"):
# elif u.system == "Windows":
elif (u.system, u.machine) == ('Windows', 'AMD64'):
url = 'https://sdk.lunarg.com/sdk/download/1.3.236.0/windows/VulkanSDK-1.3.236.0-Installer.exe'
feisuzhu marked this conversation as resolved.
Show resolved Hide resolved
prefix = get_cache_home() / 'vulkan-1.3.236.0'
download_dep(
url,
prefix,
args=[
'--accept-licenses',
'--default-answer',
'--confirm-command',
'--root',
prefix,
'install',
'com.lunarg.vulkan.sdl2',
'com.lunarg.vulkan.glm',
'com.lunarg.vulkan.volk',
'com.lunarg.vulkan.vma',
# 'com.lunarg.vulkan.debug',
])
os.environ['VULKAN_SDK'] = str(prefix)
os.environ['VK_SDK_PATH'] = str(prefix)
path_prepend('PATH', prefix / "Bin")
else:
return


@banner('Build Taichi Wheel')
def build_wheel(python: Command, pip: Command, env: dict) -> None:
def build_wheel(python: Command, pip: Command) -> None:
'''
Build the Taichi wheel
'''
pip.install('-r', 'requirements_dev.txt')
git.fetch('origin', 'master', '--tags')
proj = env['PROJECT_NAME']
proj = os.environ.get('PROJECT_NAME', 'taichi')
proj_tags = []
extra = []

if proj == 'taichi-nightly':
proj_tags.extend(['egg_info', '--tag-date'])
# Include C-API in nightly builds
env['TAICHI_CMAKE_ARGS'] += ' -DTI_WITH_C_API=ON'
os.environ['TAICHI_CMAKE_ARGS'] += ' -DTI_WITH_C_API=ON'

if platform.system() == 'Linux':
if is_manylinux2014():
Expand All @@ -95,25 +174,27 @@ def build_wheel(python: Command, pip: Command, env: dict) -> None:
python('misc/make_changelog.py', '--ver', 'origin/master', '--repo_dir',
'./', '--save')

with environ(env):
python('setup.py', *proj_tags, 'bdist_wheel', *extra)
python('setup.py', *proj_tags, 'bdist_wheel', *extra)


def main() -> None:
env = {
'PATH': os.environ['PATH'],
'LD_LIBRARY_PATH': os.environ.get('LD_LIBRARY_PATH', ''),
'TAICHI_CMAKE_ARGS': os.environ.get('TAICHI_CMAKE_ARGS', ''),
'PROJECT_NAME': os.environ.get('PROJECT_NAME', 'taichi'),
}
setup_llvm(env)
setup_vulkan(env)
sccache = setup_sccache(env)
u = platform.uname()
if (u.system, u.machine) == ('Windows', 'AMD64'):
# Use MSVC on Windows
setup_clang(as_compiler=False)
setup_msvc()
else:
# Use Clang on all other platforms
setup_clang()

setup_llvm()
setup_vulkan()
sccache = setup_sccache()

# NOTE: We use conda/venv to build wheels, which may not be the same python
# running this script.
python, pip = setup_python(env, os.environ['PY'])
build_wheel(python, pip, env)
python, pip = setup_python(os.environ['PY'])
build_wheel(python, pip)

sccache('-s')

Expand Down
91 changes: 72 additions & 19 deletions .github/workflows/scripts/ci_common/bootstrap.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
# -*- coding: utf-8 -*-

# -- stdlib --
import importlib
import os
import platform
import subprocess
import sys
from pathlib import Path

# -- third party --
# -- own --


# -- code --
def is_in_venv() -> bool:
'''
Are we in a virtual environment?
Expand All @@ -19,15 +14,55 @@ def is_in_venv() -> bool:
and sys.base_prefix != sys.prefix)


def get_cache_home() -> Path:
'''
Get the cache home directory. All intermediate files should be stored here.
'''
if platform.system() == 'Windows':
return Path(os.environ['LOCALAPPDATA']) / 'build-cache'
else:
return Path.home() / '.cache' / 'build-cache'


def run(*args, env=None):
args = list(map(str, args))
if env is None:
return subprocess.Popen(args).wait()
else:
e = os.environ.copy()
e.update(env)
return subprocess.Popen(args, env=e).wait()


def restart():
'''
Restart the current process.
'''
if platform.system() == 'Windows':
# GitHub Actions will treat the step as completed when doing os.execl in Windows,
# since Windows does not have real execve, its behavior is emulated by spawning a new process and
# terminating the current process. So we do not use os.execl in Windows.
os._exit(run(sys.executable, *sys.argv))
else:
os.execl(sys.executable, sys.executable, *sys.argv)


def ensure_dependencies(fn='requirements.txt'):
'''
Automatically install dependencies if they are not installed.
'''

if 'site' in sys.modules:
sys.argv.insert(0, '-S')
restart()

p = Path(__file__).parent.parent / fn
if not p.exists():
raise RuntimeError(f'Cannot find {p}')

user = '' if is_in_venv() else '--user'
bootstrap_root = get_cache_home() / 'bootstrap'
bootstrap_root.mkdir(parents=True, exist_ok=True)
sys.path.insert(0, str(bootstrap_root))

with open(p) as f:
deps = [i.strip().split('=')[0] for i in f.read().splitlines()]
Expand All @@ -36,12 +71,17 @@ def ensure_dependencies(fn='requirements.txt'):
for dep in deps:
importlib.import_module(dep)
except ModuleNotFoundError:
print('Installing dependencies...')
if os.system(f'{sys.executable} -m pip install {user} -U pip'):
print('Installing dependencies...', flush=True)
pipcmd = [
sys.executable, '-m', 'pip', 'install',
f'--target={bootstrap_root}', '-U'
]
if run(*pipcmd, 'pip', 'setuptools'):
raise Exception('Unable to upgrade pip!')
if os.system(f'{sys.executable} -m pip install {user} -U -r {p}'):
if run(*pipcmd, '-r', p, env={'PYTHONPATH': str(bootstrap_root)}):
raise Exception('Unable to install dependencies!')
os.execl(sys.executable, sys.executable, *sys.argv)

restart()


def chdir_to_root():
Expand All @@ -51,7 +91,7 @@ def chdir_to_root():
root = Path('/')
p = Path(__file__).resolve()
while p != root:
if (p / '.git').exists():
if (p / 'setup.py').exists():
os.chdir(p)
break
p = p.parent
Expand All @@ -69,20 +109,33 @@ def set_common_env():

class _EnvironWrapper(_Environ):
def __setitem__(self, name: str, value: str) -> None:
orig = self.get(name, '')
orig = self.get(name, None)
_Environ.__setitem__(self, name, value)
new = self[name]

if orig == new:
return

from .escapes import escape_codes

G = escape_codes['bold_green']
R = escape_codes['bold_red']
N = escape_codes['reset']
print(f'{R}:: ENV -{name}={orig}{N}', file=sys.stderr, flush=True)
print(f'{G}:: ENV +{name}={new}{N}', file=sys.stderr, flush=True)

if orig == new:
pass
elif orig == None:
print(f'{G}:: ENV+ {name}={new}{N}', file=sys.stderr, flush=True)
elif new.startswith(orig):
l = len(orig)
print(f'{G}:: ENV{N} {name}={new[:l]}{G}{new[l:]}{N}',
file=sys.stderr,
flush=True)
elif new.endswith(orig):
l = len(new) - len(orig)
print(f'{G}:: ENV{N} {name}={G}{new[:l]}{N}{new[l:]}',
file=sys.stderr,
flush=True)
else:
print(f'{R}:: ENV- {name}={orig}{N}', file=sys.stderr, flush=True)
print(f'{G}:: ENV+ {name}={new}{N}', file=sys.stderr, flush=True)


def monkey_patch_environ():
Expand Down
Loading