diff --git a/Makefile b/Makefile index ceb4ef76671..af86a3c20f3 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ PYTHON ?= python NINJA ?= ninja DESTDIR ?= SIGN ?= +FLAKY_TESTS ?= run NODE ?= ./node @@ -101,6 +102,9 @@ test-all-http1: all test-all-valgrind: all $(PYTHON) tools/test.py --mode=debug,release --valgrind +test-ci: + $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --arch=$(DESTCPU) --flaky-tests=$(FLAKY_TESTS) simple message internet + test-release: all $(PYTHON) tools/test.py --mode=release @@ -195,6 +199,11 @@ docopen: out/doc/api/all.html docclean: -rm -rf out/doc +run-ci: + $(PYTHON) ./configure --without-snapshot $(CONFIG_FLAGS) + $(MAKE) + $(MAKE) test-ci + RAWVER=$(shell $(PYTHON) tools/getnodeversion.py) VERSION=v$(RAWVER) NODE_DOC_VERSION=$(VERSION) @@ -381,4 +390,8 @@ cpplint: lint: jslint cpplint -.PHONY: lint cpplint jslint bench clean docopen docclean doc dist distclean check uninstall install install-includes install-bin all staticlib dynamiclib test test-all website-upload pkg blog blogclean tar binary release-only bench-http-simple bench-idle bench-all bench bench-misc bench-array bench-buffer bench-net bench-http bench-fs bench-tls +.PHONY: lint cpplint jslint bench clean docopen docclean doc dist distclean \ + check uninstall install install-includes install-bin all staticlib \ + dynamiclib test test-all website-upload pkg blog blogclean tar binary \ + release-only bench-http-simple bench-idle bench-all bench bench-misc \ + bench-array bench-buffer bench-net bench-http bench-fs bench-tls run-ci diff --git a/test/gc/testcfg.py b/test/gc/testcfg.py index 30f3bdbeb9e..d6bbcd60fff 100644 --- a/test/gc/testcfg.py +++ b/test/gc/testcfg.py @@ -1,133 +1,6 @@ -# Copyright 2008 the V8 project authors. All rights reserved. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import test -import os -import shutil -from shutil import rmtree -from os import mkdir -from glob import glob -from os.path import join, dirname, exists -import re - - -FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") -FILES_PATTERN = re.compile(r"//\s+Files:(.*)") - - -class GCTestCase(test.TestCase): - - def __init__(self, path, file, mode, context, config): - super(GCTestCase, self).__init__(context, path, mode) - self.file = file - self.config = config - self.mode = mode - self.tmpdir = join(dirname(self.config.root), 'tmp') - - def AfterRun(self, result): - # delete the whole tmp dir - try: - rmtree(self.tmpdir) - except: - pass - # make it again. - try: - mkdir(self.tmpdir) - except: - pass - - def BeforeRun(self): - # delete the whole tmp dir - try: - rmtree(self.tmpdir) - except: - pass - # make it again. - # intermittently fails on win32, so keep trying - while not os.path.exists(self.tmpdir): - try: - mkdir(self.tmpdir) - except: - pass - - def GetLabel(self): - return "%s %s" % (self.mode, self.GetName()) - - def GetName(self): - return self.path[-1] - - def GetCommand(self): - result = [self.config.context.GetVm(self.mode)] - source = open(self.file).read() - flags_match = FLAGS_PATTERN.search(source) - if flags_match: - result += flags_match.group(1).strip().split() - files_match = FILES_PATTERN.search(source); - additional_files = [] - if files_match: - additional_files += files_match.group(1).strip().split() - for a_file in additional_files: - result.append(join(dirname(self.config.root), '..', a_file)) - result += ["--expose-gc"] - result += [self.file] - return result - - def GetSource(self): - return open(self.file).read() - - -class GCTestConfiguration(test.TestConfiguration): - - def __init__(self, context, root): - super(GCTestConfiguration, self).__init__(context, root) - - def Ls(self, path): - def SelectTest(name): - return name.startswith('test-') and name.endswith('.js') - return [f[:-3] for f in os.listdir(path) if SelectTest(f)] - - def ListTests(self, current_path, path, mode): - all_tests = [current_path + [t] for t in self.Ls(join(self.root))] - result = [] - for test in all_tests: - if self.Contains(path, test): - file_path = join(self.root, reduce(join, test[1:], "") + ".js") - result.append(GCTestCase(test, file_path, mode, self.context, self)) - return result - - def GetBuildRequirements(self): - return ['sample', 'sample=shell'] - - def GetTestStatus(self, sections, defs): - status_file = join(self.root, 'gc.status') - if exists(status_file): - test.ReadConfigurationInto(status_file, sections, defs) - - +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy def GetConfiguration(context, root): - return GCTestConfiguration(context, root) + return testpy.SimpleTestConfiguration(context, root, 'gc', ['--expose-gc']) diff --git a/test/internet/internet.status b/test/internet/internet.status index 34aea6a6af7..b517fc2db0d 100644 --- a/test/internet/internet.status +++ b/test/internet/internet.status @@ -1 +1,11 @@ prefix internet + +test-dns : PASS,FLAKY + +[$system==win32] + +[$system==linux] + +[$system==macos] + +[$system==solaris] diff --git a/test/internet/testcfg.py b/test/internet/testcfg.py index 3e3e900e6fc..73e70e34000 100644 --- a/test/internet/testcfg.py +++ b/test/internet/testcfg.py @@ -1,132 +1,6 @@ -# Copyright 2008 the V8 project authors. All rights reserved. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import test -import os -import shutil -from shutil import rmtree -from os import mkdir -from glob import glob -from os.path import join, dirname, exists -import re - - -FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") -FILES_PATTERN = re.compile(r"//\s+Files:(.*)") - - -class InternetTestCase(test.TestCase): - - def __init__(self, path, file, mode, context, config): - super(InternetTestCase, self).__init__(context, path, mode) - self.file = file - self.config = config - self.mode = mode - self.tmpdir = join(dirname(self.config.root), 'tmp') - - def AfterRun(self, result): - # delete the whole tmp dir - try: - rmtree(self.tmpdir) - except: - pass - # make it again. - try: - mkdir(self.tmpdir) - except: - pass - - def BeforeRun(self): - # delete the whole tmp dir - try: - rmtree(self.tmpdir) - except: - pass - # make it again. - # intermittently fails on win32, so keep trying - while not os.path.exists(self.tmpdir): - try: - mkdir(self.tmpdir) - except: - pass - - def GetLabel(self): - return "%s %s" % (self.mode, self.GetName()) - - def GetName(self): - return self.path[-1] - - def GetCommand(self): - result = [self.config.context.GetVm(self.mode)] - source = open(self.file).read() - flags_match = FLAGS_PATTERN.search(source) - if flags_match: - result += flags_match.group(1).strip().split() - files_match = FILES_PATTERN.search(source); - additional_files = [] - if files_match: - additional_files += files_match.group(1).strip().split() - for a_file in additional_files: - result.append(join(dirname(self.config.root), '..', a_file)) - result += [self.file] - return result - - def GetSource(self): - return open(self.file).read() - - -class InternetTestConfiguration(test.TestConfiguration): - - def __init__(self, context, root): - super(InternetTestConfiguration, self).__init__(context, root) - - def Ls(self, path): - def SelectTest(name): - return name.startswith('test-') and name.endswith('.js') - return [f[:-3] for f in os.listdir(path) if SelectTest(f)] - - def ListTests(self, current_path, path, mode): - all_tests = [current_path + [t] for t in self.Ls(join(self.root))] - result = [] - for test in all_tests: - if self.Contains(path, test): - file_path = join(self.root, reduce(join, test[1:], "") + ".js") - result.append(InternetTestCase(test, file_path, mode, self.context, self)) - return result - - def GetBuildRequirements(self): - return ['sample', 'sample=shell'] - - def GetTestStatus(self, sections, defs): - status_file = join(self.root, 'simple.status') - if exists(status_file): - test.ReadConfigurationInto(status_file, sections, defs) - - +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy def GetConfiguration(context, root): - return InternetTestConfiguration(context, root) + return testpy.SimpleTestConfiguration(context, root, 'internet') diff --git a/test/pummel/testcfg.py b/test/pummel/testcfg.py index 02f5ec6a9cf..c91fe39dce2 100644 --- a/test/pummel/testcfg.py +++ b/test/pummel/testcfg.py @@ -1,130 +1,6 @@ -# Copyright 2008 the V8 project authors. All rights reserved. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import test -import os -import shutil -from shutil import rmtree -from os import mkdir -from glob import glob -from os.path import join, dirname, exists -import re - - -FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") -FILES_PATTERN = re.compile(r"//\s+Files:(.*)") - - -class PummelTestCase(test.TestCase): - - def __init__(self, path, file, mode, context, config): - super(PummelTestCase, self).__init__(context, path, mode) - self.file = file - self.config = config - self.mode = mode - self.tmpdir = join(dirname(self.config.root), 'tmp') - - def AfterRun(self, result): - # delete the whole tmp dir - try: - rmtree(self.tmpdir) - except: - pass - # make it again. - try: - mkdir(self.tmpdir) - except: - pass - - def BeforeRun(self): - # delete the whole tmp dir - try: - rmtree(self.tmpdir) - except: - pass - # make it again. - try: - mkdir(self.tmpdir) - except: - pass - - def GetLabel(self): - return "%s %s" % (self.mode, self.GetName()) - - def GetName(self): - return self.path[-1] - - def GetCommand(self): - result = [self.config.context.GetVm(self.mode)] - source = open(self.file).read() - flags_match = FLAGS_PATTERN.search(source) - if flags_match: - result += flags_match.group(1).strip().split() - files_match = FILES_PATTERN.search(source); - additional_files = [] - if files_match: - additional_files += files_match.group(1).strip().split() - for a_file in additional_files: - result.append(join(dirname(self.config.root), '..', a_file)) - result += [self.file] - return result - - def GetSource(self): - return open(self.file).read() - - -class PummelTestConfiguration(test.TestConfiguration): - - def __init__(self, context, root): - super(PummelTestConfiguration, self).__init__(context, root) - - def Ls(self, path): - def SelectTest(name): - return name.startswith('test-') and name.endswith('.js') - return [f[:-3] for f in os.listdir(path) if SelectTest(f)] - - def ListTests(self, current_path, path, mode): - all_tests = [current_path + [t] for t in self.Ls(join(self.root))] - result = [] - for test in all_tests: - if self.Contains(path, test): - file_path = join(self.root, reduce(join, test[1:], "") + ".js") - result.append(PummelTestCase(test, file_path, mode, self.context, self)) - return result - - def GetBuildRequirements(self): - return ['sample', 'sample=shell'] - - def GetTestStatus(self, sections, defs): - status_file = join(self.root, 'simple.status') - if exists(status_file): - test.ReadConfigurationInto(status_file, sections, defs) - - +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy def GetConfiguration(context, root): - return PummelTestConfiguration(context, root) + return testpy.SimpleTestConfiguration(context, root, 'pummel') diff --git a/test/simple/simple.status b/test/simple/simple.status index 438ce39f5f2..c430bf56686 100644 --- a/test/simple/simple.status +++ b/test/simple/simple.status @@ -1 +1,36 @@ prefix simple + +test-http-client-timeout-event : PASS,FLAKY +test-http-pipeline-flood : PASS,FLAKY +test-child-process-fork-net2 : PASS,FLAKY +test-tls-session-cache : PASS,FLAKY +test-net-GH-5504 : PASS,FLAKY +test-debugger-repl-restart : PASS,FLAKY +test-http-curl-chunk-problem : PASS,FLAKY +test-fs-empty-readStream : PASS,FLAKY + +[$system==win32] +test-cluster-bind-twice-v2 : PASS,FLAKY +test-pipe-file-to-http : PASS,FLAKY +test-stream2-stderr-sync : PASS,FLAKY +test-tls-securepair-server : PASS,FLAKY +test-timers-first-fire : PASS,FLAKY + +[$system==linux] +test-abort-fatal-error : PASS,FLAKY +test-debugger-client : PASS,FLAKY +test-process-argv-0 : PASS,FLAKY +test-cluster-master-kill : PASS,FLAKY +test-cluster-worker-kill : PASS,FLAKY +test-timers-ordering : PASS,FLAKY +test-cluster-master-error : PASS,FLAKY + +[$system==macos] +test-http-exit-delay : PASS,FLAKY + +[$system==solaris] +test-abort-fatal-error : PASS,FLAKY +test-debugger-repl : PASS,FLAKY +test-debugger-repl-break-in-module : PASS,FLAKY +test-debugger-repl-utf8 : PASS,FLAKY +test-net-server-max-connections : PASS,FLAKY diff --git a/test/simple/testcfg.py b/test/simple/testcfg.py index bb2c4d1af10..3d9b6296bb2 100644 --- a/test/simple/testcfg.py +++ b/test/simple/testcfg.py @@ -1,132 +1,6 @@ -# Copyright 2008 the V8 project authors. All rights reserved. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import test -import os -import shutil -from shutil import rmtree -from os import mkdir -from glob import glob -from os.path import join, dirname, exists -import re - - -FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") -FILES_PATTERN = re.compile(r"//\s+Files:(.*)") - - -class SimpleTestCase(test.TestCase): - - def __init__(self, path, file, mode, context, config): - super(SimpleTestCase, self).__init__(context, path, mode) - self.file = file - self.config = config - self.mode = mode - self.tmpdir = join(dirname(self.config.root), 'tmp') - - def AfterRun(self, result): - # delete the whole tmp dir - try: - rmtree(self.tmpdir) - except: - pass - # make it again. - try: - mkdir(self.tmpdir) - except: - pass - - def BeforeRun(self): - # delete the whole tmp dir - try: - rmtree(self.tmpdir) - except: - pass - # make it again. - # intermittently fails on win32, so keep trying - while not os.path.exists(self.tmpdir): - try: - mkdir(self.tmpdir) - except: - pass - - def GetLabel(self): - return "%s %s" % (self.mode, self.GetName()) - - def GetName(self): - return self.path[-1] - - def GetCommand(self): - result = [self.config.context.GetVm(self.mode)] - source = open(self.file).read() - flags_match = FLAGS_PATTERN.search(source) - if flags_match: - result += flags_match.group(1).strip().split() - files_match = FILES_PATTERN.search(source); - additional_files = [] - if files_match: - additional_files += files_match.group(1).strip().split() - for a_file in additional_files: - result.append(join(dirname(self.config.root), '..', a_file)) - result += [self.file] - return result - - def GetSource(self): - return open(self.file).read() - - -class SimpleTestConfiguration(test.TestConfiguration): - - def __init__(self, context, root): - super(SimpleTestConfiguration, self).__init__(context, root) - - def Ls(self, path): - def SelectTest(name): - return name.startswith('test-') and name.endswith('.js') - return [f[:-3] for f in os.listdir(path) if SelectTest(f)] - - def ListTests(self, current_path, path, mode): - all_tests = [current_path + [t] for t in self.Ls(join(self.root))] - result = [] - for test in all_tests: - if self.Contains(path, test): - file_path = join(self.root, reduce(join, test[1:], "") + ".js") - result.append(SimpleTestCase(test, file_path, mode, self.context, self)) - return result - - def GetBuildRequirements(self): - return ['sample', 'sample=shell'] - - def GetTestStatus(self, sections, defs): - status_file = join(self.root, 'simple.status') - if exists(status_file): - test.ReadConfigurationInto(status_file, sections, defs) - - +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy def GetConfiguration(context, root): - return SimpleTestConfiguration(context, root) + return testpy.SimpleTestConfiguration(context, root, 'simple') diff --git a/test/testpy/__init__.py b/test/testpy/__init__.py new file mode 100644 index 00000000000..54b0f19916f --- /dev/null +++ b/test/testpy/__init__.py @@ -0,0 +1,136 @@ +# Copyright 2008 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import test +import os +import shutil +from shutil import rmtree +from os import mkdir +from glob import glob +from os.path import join, dirname, exists +import re + + +FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)") +FILES_PATTERN = re.compile(r"//\s+Files:(.*)") + + +class SimpleTestCase(test.TestCase): + + def __init__(self, path, file, mode, context, config, additional=[]): + super(SimpleTestCase, self).__init__(context, path, mode) + self.file = file + self.config = config + self.mode = mode + self.tmpdir = join(dirname(self.config.root), 'tmp') + self.additional_flags = additional + + def AfterRun(self, result): + # delete the whole tmp dir + try: + rmtree(self.tmpdir) + except: + pass + # make it again. + try: + mkdir(self.tmpdir) + except: + pass + + def BeforeRun(self): + # delete the whole tmp dir + try: + rmtree(self.tmpdir) + except: + pass + # make it again. + # intermittently fails on win32, so keep trying + while not os.path.exists(self.tmpdir): + try: + mkdir(self.tmpdir) + except: + pass + + def GetLabel(self): + return "%s %s" % (self.mode, self.GetName()) + + def GetName(self): + return self.path[-1] + + def GetCommand(self): + result = [self.config.context.GetVm(self.mode)] + source = open(self.file).read() + flags_match = FLAGS_PATTERN.search(source) + if flags_match: + result += flags_match.group(1).strip().split() + files_match = FILES_PATTERN.search(source); + additional_files = [] + if files_match: + additional_files += files_match.group(1).strip().split() + for a_file in additional_files: + result.append(join(dirname(self.config.root), '..', a_file)) + + if self.additional_flags: + result += self.additional_flags + + result += [self.file] + + return result + + def GetSource(self): + return open(self.file).read() + + +class SimpleTestConfiguration(test.TestConfiguration): + + def __init__(self, context, root, section, additional=[]): + super(SimpleTestConfiguration, self).__init__(context, root) + self.section = section + self.additional_flags = additional + + def Ls(self, path): + def SelectTest(name): + return name.startswith('test-') and name.endswith('.js') + return [f[:-3] for f in os.listdir(path) if SelectTest(f)] + + def ListTests(self, current_path, path, mode): + all_tests = [current_path + [t] for t in self.Ls(join(self.root))] + result = [] + for test in all_tests: + if self.Contains(path, test): + file_path = join(self.root, reduce(join, test[1:], "") + ".js") + result.append(SimpleTestCase(test, file_path, mode, self.context, self, + self.additional_flags)) + return result + + def GetBuildRequirements(self): + return ['sample', 'sample=shell'] + + def GetTestStatus(self, sections, defs): + status_file = join(self.root, '%s.status' % (self.section)) + if exists(status_file): + test.ReadConfigurationInto(status_file, sections, defs) diff --git a/tools/test.py b/tools/test.py index c922c40f7ee..f87c453678b 100755 --- a/tools/test.py +++ b/tools/test.py @@ -29,6 +29,7 @@ import imp +import logging import optparse import os import platform @@ -43,10 +44,10 @@ from os.path import join, dirname, abspath, basename, isdir, exists from datetime import datetime from Queue import Queue, Empty - -sys.path.append(dirname(__file__) + "/../deps/v8/tools"); import utils +logger = logging.getLogger('testrunner') + VERBOSE = False @@ -57,8 +58,9 @@ class ProgressIndicator(object): - def __init__(self, cases): + def __init__(self, cases, flaky_tests_mode): self.cases = cases + self.flaky_tests_mode = flaky_tests_mode self.queue = Queue(len(cases)) for case in cases: self.queue.put_nowait(case) @@ -66,7 +68,9 @@ def __init__(self, cases): self.remaining = len(cases) self.total = len(cases) self.failed = [ ] + self.flaky_failed = [ ] self.crashed = 0 + self.flaky_crashed = 0 self.terminate = False self.lock = threading.Lock() @@ -127,9 +131,14 @@ def RunSingle(self): return self.lock.acquire() if output.UnexpectedOutput(): - self.failed.append(output) - if output.HasCrashed(): - self.crashed += 1 + if FLAKY in output.test.outcomes and self.flaky_tests_mode == "dontcare": + self.flaky_failed.append(output) + if output.HasCrashed(): + self.flaky_crashed += 1 + else: + self.failed.append(output) + if output.HasCrashed(): + self.crashed += 1 else: self.succeeded += 1 self.remaining -= 1 @@ -226,7 +235,7 @@ def HasRun(self, output): class TapProgressIndicator(SimpleProgressIndicator): def Starting(self): - print '1..%i' % len(self.cases) + logger.info('1..%i' % len(self.cases)) self._done = 0 def AboutToRun(self, case): @@ -236,13 +245,19 @@ def HasRun(self, output): self._done += 1 command = basename(output.command[-1]) if output.UnexpectedOutput(): - print 'not ok %i - %s' % (self._done, command) + status_line = 'not ok %i - %s' % (self._done, command) + if FLAKY in output.test.outcomes and self.flaky_tests_mode == "dontcare": + status_line = status_line + " # TODO : Fix flaky test" + logger.info(status_line) for l in output.output.stderr.splitlines(): - print '#' + l + logger.info('#' + l) for l in output.output.stdout.splitlines(): - print '#' + l + logger.info('#' + l) else: - print 'ok %i - %s' % (self._done, command) + status_line = 'ok %i - %s' % (self._done, command) + if FLAKY in output.test.outcomes: + status_line = status_line + " # TODO : Fix flaky test" + logger.info(status_line) duration = output.test.duration @@ -250,9 +265,9 @@ def HasRun(self, output): total_seconds = (duration.microseconds + (duration.seconds + duration.days * 24 * 3600) * 10**6) / 10**6 - print ' ---' - print ' duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000) - print ' ...' + logger.info(' ---') + logger.info(' duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000)) + logger.info(' ...') def Done(self): pass @@ -260,8 +275,8 @@ def Done(self): class CompactProgressIndicator(ProgressIndicator): - def __init__(self, cases, templates): - super(CompactProgressIndicator, self).__init__(cases) + def __init__(self, cases, flaky_tests_mode, templates): + super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode) self.templates = templates self.last_status_length = 0 self.start_time = time.time() @@ -316,13 +331,13 @@ def PrintProgress(self, name): class ColorProgressIndicator(CompactProgressIndicator): - def __init__(self, cases): + def __init__(self, cases, flaky_tests_mode): templates = { 'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s", 'stdout': "\033[1m%s\033[0m", 'stderr': "\033[31m%s\033[0m", } - super(ColorProgressIndicator, self).__init__(cases, templates) + super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates) def ClearLine(self, last_line_length): print "\033[1K\r", @@ -330,7 +345,7 @@ def ClearLine(self, last_line_length): class MonochromeProgressIndicator(CompactProgressIndicator): - def __init__(self, cases): + def __init__(self, cases, flaky_tests_mode): templates = { 'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s", 'stdout': '%s', @@ -338,7 +353,7 @@ def __init__(self, cases): 'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"), 'max_length': 78 } - super(MonochromeProgressIndicator, self).__init__(cases, templates) + super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates) def ClearLine(self, last_line_length): print ("\r" + (" " * last_line_length) + "\r"), @@ -740,8 +755,8 @@ def GetVmFlags(self, testcase, mode): def GetTimeout(self, mode): return self.timeout * TIMEOUT_SCALEFACTOR[mode] -def RunTestCases(cases_to_run, progress, tasks): - progress = PROGRESS_INDICATORS[progress](cases_to_run) +def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode): + progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode) return progress.Run(tasks) @@ -765,6 +780,7 @@ def BuildRequirements(context, requirements, mode, scons_flags): TIMEOUT = 'timeout' CRASH = 'crash' SLOW = 'slow' +FLAKY = 'flaky' class Expression(object): @@ -1186,6 +1202,8 @@ def BuildOptions(): default='release') result.add_option("-v", "--verbose", help="Verbose output", default=False, action="store_true") + result.add_option('--logfile', dest='logfile', + help='write test output to file. NOTE: this only applies the tap progress indicator') result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons", default=[], action="append") result.add_option("-p", "--progress", @@ -1214,6 +1232,9 @@ def BuildOptions(): default=False, action="store_true") result.add_option("--cat", help="Print the source of the tests", default=False, action="store_true") + result.add_option("--flaky-tests", + help="Regard tests marked as flaky (run|skip|dontcare)", + default="run") result.add_option("--warn-unused", help="Report unused rules", default=False, action="store_true") result.add_option("-j", help="The number of parallel tasks to run", @@ -1260,6 +1281,13 @@ def ProcessOptions(options): options.scons_flags.append("arch=" + options.arch) if options.snapshot: options.scons_flags.append("snapshot=on") + def CheckTestMode(name, option): + if not option in ["run", "skip", "dontcare"]: + print "Unknown %s mode %s" % (name, option) + return False + return True + if not CheckTestMode("--flaky-tests", options.flaky_tests): + return False return True @@ -1345,6 +1373,13 @@ def Main(): parser.print_help() return 1 + ch = logging.StreamHandler(sys.stdout) + logger.addHandler(ch) + logger.setLevel(logging.INFO) + if options.logfile: + fh = logging.FileHandler(options.logfile) + logger.addHandler(fh) + workspace = abspath(join(dirname(sys.argv[0]), '..')) suites = GetSuites(join(workspace, 'test')) repositories = [TestRepository(join(workspace, 'test', name)) for name in suites] @@ -1452,15 +1487,16 @@ def wrap(processor): result = None def DoSkip(case): - return SKIP in case.outcomes or SLOW in case.outcomes + return (SKIP in case.outcomes or SLOW in case.outcomes or + (FLAKY in case.outcomes and options.flaky_tests == "skip")) cases_to_run = [ c for c in all_cases if not DoSkip(c) ] if len(cases_to_run) == 0: print "No tests to run." - return 0 + return 1 else: try: start = time.time() - if RunTestCases(cases_to_run, options.progress, options.j): + if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests): result = 0 else: result = 1 diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 00000000000..232314cdee1 --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,98 @@ +# Copyright 2008 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import platform +import re + + +# Reads a .list file into an array of strings +def ReadLinesFrom(name): + list = [] + for line in open(name): + if '#' in line: + line = line[:line.find('#')] + line = line.strip() + if len(line) == 0: + continue + list.append(line) + return list + + +def GuessOS(): + id = platform.system() + if id == 'Linux': + return 'linux' + elif id == 'Darwin': + return 'macos' + elif id.find('CYGWIN') >= 0: + return 'cygwin' + elif id == 'Windows' or id == 'Microsoft': + # On Windows Vista platform.system() can return 'Microsoft' with some + # versions of Python, see http://bugs.python.org/issue1082 + return 'win32' + elif id == 'FreeBSD': + return 'freebsd' + elif id == 'OpenBSD': + return 'openbsd' + elif id == 'SunOS': + return 'solaris' + elif id == 'NetBSD': + return 'netbsd' + else: + return None + + +# This will default to building the 32 bit VM even on machines that are capable +# of running the 64 bit VM. Use the scons option --arch=x64 to force it to build +# the 64 bit VM. +def GuessArchitecture(): + id = platform.machine() + id = id.lower() # Windows 7 capitalizes 'AMD64'. + if id.startswith('arm'): + return 'arm' + elif (not id) or (not re.match('(x|i[3-6])86$', id) is None): + return 'ia32' + elif id == 'i86pc': + return 'ia32' + elif id == 'x86_64': + return 'ia32' + elif id == 'amd64': + return 'ia32' + else: + return None + + +def GuessWordsize(): + if '64' in platform.machine(): + return '64' + else: + return '32' + + +def IsWindows(): + return GuessOS() == 'win32' diff --git a/vcbuild.bat b/vcbuild.bat index 6b58209ef9a..1fb9aa44ab3 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -35,6 +35,7 @@ set noetw_msi_arg= set noperfctr= set noperfctr_arg= set noperfctr_msi_arg= +set flaky_tests_arg= :next-arg if "%1"=="" goto args-done @@ -58,11 +59,13 @@ if /i "%1"=="test-simple" set test=test-simple&goto arg-ok if /i "%1"=="test-message" set test=test-message&goto arg-ok if /i "%1"=="test-gc" set test=test-gc&set buildnodeweak=1&goto arg-ok if /i "%1"=="test-all" set test=test-all&set buildnodeweak=1&goto arg-ok -if /i "%1"=="test" set test=test&goto arg-ok +if /i "%1"=="test-ci" set test=test-ci&set nosnapshot=1&goto arg-ok +if /i "%1"=="test" set test=test&set jslint=1&goto arg-ok if /i "%1"=="msi" set msi=1&set licensertf=1&goto arg-ok if /i "%1"=="upload" set upload=1&goto arg-ok if /i "%1"=="jslint" set jslint=1&goto arg-ok if /i "%1"=="build-release" set nosnapshot=1&set config=Release&set msi=1&set licensertf=1&goto arg-ok +if /i "%1"=="ignore-flaky" set flaky_tests_arg=--flaky-tests=dontcare&goto arg-ok echo Warning: ignoring invalid command line option `%1`. @@ -73,7 +76,6 @@ goto next-arg :args-done if defined upload goto upload -if defined jslint goto jslint if "%config%"=="Debug" set debug_arg=--debug if "%target_arch%"=="x64" set msiplatform=x64 @@ -166,12 +168,15 @@ if errorlevel 1 echo Failed to sign msi&goto exit :run @rem Run tests if requested. -if "%test%"=="" goto exit +if "%test%"=="" goto jslint if "%config%"=="Debug" set test_args=--mode=debug if "%config%"=="Release" set test_args=--mode=release +set test_args=%test_args% --arch=%target_arch% + if "%test%"=="test" set test_args=%test_args% simple message +if "%test%"=="test-ci" set test_args=%test_args% -p tap --logfile test.tap %flaky_tests_arg% simple message internet if "%test%"=="test-internet" set test_args=%test_args% internet if "%test%"=="test-pummel" set test_args=%test_args% pummel if "%test%"=="test-simple" set test_args=%test_args% simple @@ -193,8 +198,7 @@ goto exit :run-tests echo running 'python tools/test.py %test_args%' python tools/test.py %test_args% -if "%test%"=="test" goto jslint -goto exit +goto jslint :create-msvs-files-failed echo Failed to create vc project files. @@ -212,6 +216,7 @@ scp Release\node.pdb node@nodejs.org:~/web/nodejs.org/dist/v%NODE_VERSION%/node. goto exit :jslint +if not defined jslint goto exit echo running jslint set PYTHONPATH=tools/closure_linter/ python tools/closure_linter/closure_linter/gjslint.py --unix_mode --strict --nojsdoc -r lib/ -r src/ --exclude_files lib/punycode.js