Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@gsnedders
gsnedders committed Feb 9, 2019
1 parent 313a265 commit 76598af
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 148 deletions.
146 changes: 50 additions & 96 deletions tools/manifest/item.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,6 @@
from abc import ABCMeta, abstractproperty


class SourceFileCache(object):
def __init__(self):
self.source_files = {}

def make_new(self, tests_root, path, url_base):
from .sourcefile import SourceFile

return SourceFile(tests_root, path, url_base)

def get(self, tests_root, manifest, path):

if path not in self.source_files:
self.source_files[path] = self.make_new(tests_root, path, manifest.url_base)

return self.source_files[path]


item_types = {}


Expand All @@ -38,12 +21,12 @@ def __new__(cls, name, bases, attrs, **kwargs):
class ManifestItem(object):
__metaclass__ = ManifestItemMeta

item_type = None
__slots__ = ("path",)

source_file_cache = SourceFileCache()
item_type = None

def __init__(self, source_file, manifest=None):
self.source_file = source_file
def __init__(self, path=None):
self.path = path

@abstractproperty
def id(self):
Expand All @@ -52,12 +35,7 @@ def id(self):

@property
def meta_flags(self):
return set(self.source_file.meta_flags)

@property
def path(self):
"""The test path relative to the test_root"""
return self.source_file.rel_path
return set()

@property
def https(self):
Expand All @@ -70,7 +48,7 @@ def key(self):

def meta_key(self):
"""Extra metadata that doesn't form part of the test identity, but for
which changes mean regenerating the manifest (e.g. the test timeout."""
which changes mean regenerating the manifest (e.g. the test timeout)."""
return ()

def __eq__(self, other):
Expand All @@ -88,17 +66,17 @@ def to_json(self):
return [{}]

@classmethod
def from_json(cls, manifest, tests_root, path, obj):
source_file = cls.source_file_cache.get(tests_root, manifest, path)
return cls(source_file,
manifest=manifest)
def from_json(cls, path, obj):
return cls(path)


class URLManifestItem(ManifestItem):
def __init__(self, source_file, url, url_base="/", manifest=None):
ManifestItem.__init__(self, source_file, manifest=manifest)
__slots__ = ("_url", "extras")

def __init__(self, path, url, **extras):
ManifestItem.__init__(self, path)
self._url = url
self.url_base = url_base
self.extras = extras or {}

@property
def id(self):
Expand All @@ -108,6 +86,10 @@ def id(self):
def meta_flags(self):
return set(urlparse(self.url).path.rsplit("/", 1)[1].split(".")[1:-1])

@property
def url_base(self):
return "/"

@property
def url(self):
return urljoin(self.url_base, self._url)
Expand All @@ -117,23 +99,27 @@ def to_json(self):
return rv

@classmethod
def from_json(cls, manifest, tests_root, path, obj):
source_file = cls.source_file_cache.get(tests_root, manifest, path)
def from_json(cls, path, obj):
url, extras = obj
return cls(source_file,
return cls(path,
url,
url_base=manifest.url_base,
manifest=manifest)
**extras)


class TestharnessTest(URLManifestItem):
item_type = "testharness"

def __init__(self, source_file, url, url_base="/", timeout=None, testdriver=False, jsshell=False, manifest=None):
URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest)
self.timeout = timeout
self.testdriver = testdriver
self.jsshell = jsshell
@property
def timeout(self):
return self.extras.get("timeout")

@property
def testdriver(self):
return self.extras.get("testdriver")

@property
def jsshell(self):
return self.extras.get("jsshell")

def meta_key(self):
return (self.timeout, self.testdriver)
Expand All @@ -148,33 +134,25 @@ def to_json(self):
rv[-1]["jsshell"] = True
return rv

@classmethod
def from_json(cls, manifest, tests_root, path, obj):
source_file = cls.source_file_cache.get(tests_root, manifest, path)

url, extras = obj
return cls(source_file,
url,
url_base=manifest.url_base,
timeout=extras.get("timeout"),
testdriver=bool(extras.get("testdriver")),
jsshell=bool(extras.get("jsshell")),
manifest=manifest)


class RefTestNode(URLManifestItem):
item_type = "reftest_node"

def __init__(self, source_file, url, references, url_base="/", timeout=None,
viewport_size=None, dpi=None, manifest=None):
URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest)
for _, ref_type in references:
if ref_type not in ["==", "!="]:
raise ValueError("Unrecognised ref_type %s" % ref_type)
self.references = tuple(references)
self.timeout = timeout
self.viewport_size = viewport_size
self.dpi = dpi
@property
def references(self):
return self.extras.get("references", [])

@property
def timeout(self):
return self.extras.get("timeout")

@property
def viewport_size(self):
return self.extras.get("viewport_size")

@property
def dpi(self):
return self.extras.get("dpi")

def meta_key(self):
return (self.timeout, self.viewport_size, self.dpi)
Expand All @@ -190,19 +168,6 @@ def to_json(self):
extras["dpi"] = self.dpi
return rv

@classmethod
def from_json(cls, manifest, tests_root, path, obj):
source_file = cls.source_file_cache.get(tests_root, manifest, path)
url, references, extras = obj
return cls(source_file,
url,
references,
url_base=manifest.url_base,
timeout=extras.get("timeout"),
viewport_size=extras.get("viewport_size"),
dpi=extras.get("dpi"),
manifest=manifest)

def to_RefTest(self):
if type(self) == RefTest:
return self
Expand Down Expand Up @@ -241,31 +206,20 @@ class Stub(URLManifestItem):
class WebDriverSpecTest(URLManifestItem):
item_type = "wdspec"

def __init__(self, source_file, url, url_base="/", timeout=None, manifest=None):
URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest)
self.timeout = timeout
@property
def timeout(self):
return self.extras.get("timeout")

def to_json(self):
rv = URLManifestItem.to_json(self)
if self.timeout is not None:
rv[-1]["timeout"] = self.timeout
return rv

@classmethod
def from_json(cls, manifest, tests_root, path, obj):
source_file = cls.source_file_cache.get(tests_root, manifest, path)

url, extras = obj
return cls(source_file,
url,
url_base=manifest.url_base,
timeout=extras.get("timeout"),
manifest=manifest)


class SupportFile(ManifestItem):
item_type = "support"

@property
def id(self):
return self.source_file.rel_path
return self.path
33 changes: 14 additions & 19 deletions tools/manifest/manifest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,7 @@ def load(self, key):
data = set()
path = from_os_path(key)
for test in iterfilter(self.meta_filters, self.json_data.get(path, [])):
manifest_item = self.type_cls.from_json(self.manifest,
self.tests_root,
path,
test)
manifest_item = self.type_cls.from_json(path, test)
data.add(manifest_item)
try:
del self.json_data[path]
Expand All @@ -154,10 +151,7 @@ def load_all(self):
continue
data = set()
for test in iterfilter(self.meta_filters, self.json_data.get(path, [])):
manifest_item = self.type_cls.from_json(self.manifest,
self.tests_root,
path,
test)
manifest_item = self.type_cls.from_json(path, test)
data.add(manifest_item)
self.data[key] = data
self.json_data = None
Expand Down Expand Up @@ -202,11 +196,12 @@ def paths(self):


class Manifest(object):
def __init__(self, url_base="/", meta_filters=None):
def __init__(self, tests_root=None, url_base="/", meta_filters=None):
assert url_base is not None
self._path_hash = {}
self._data = ManifestData(self, meta_filters)
self._reftest_nodes_by_url = None
self.tests_root = tests_root
self.url_base = url_base

def __iter__(self):
Expand Down Expand Up @@ -297,7 +292,7 @@ def update(self, tree):
new_type, manifest_items = source_file.manifest_items()

if new_type in ("reftest", "reftest_node"):
reftest_nodes.extend(manifest_items)
reftest_nodes.extend((source_file, item) for item in manifest_items)
if is_new or hash_changed:
reftest_changes = True
elif new_type:
Expand Down Expand Up @@ -337,28 +332,28 @@ def update(self, tree):
def _compute_reftests(self, reftest_nodes):
self._reftest_nodes_by_url = {}
has_inbound = set()
for item in reftest_nodes:
for _, item in reftest_nodes:
for ref_url, ref_type in item.references:
has_inbound.add(ref_url)

reftests = defaultdict(set)
references = defaultdict(set)
changed_hashes = {}

for item in reftest_nodes:
for source_file, item in reftest_nodes:
if item.url in has_inbound:
# This is a reference
if isinstance(item, RefTest):
item = item.to_RefTestNode()
changed_hashes[item.source_file.rel_path] = (item.source_file.hash,
item.item_type)
references[item.source_file.rel_path].add(item)
changed_hashes[item.path] = (source_file.hash,
item.item_type)
references[item.path].add(item)
else:
if isinstance(item, RefTestNode):
item = item.to_RefTest()
changed_hashes[item.source_file.rel_path] = (item.source_file.hash,
item.item_type)
reftests[item.source_file.rel_path].add(item)
changed_hashes[item.path] = (source_file.hash,
item.item_type)
reftests[item.path].add(item)
self._reftest_nodes_by_url[item.url] = item

return reftests, references, changed_hashes
Expand Down Expand Up @@ -473,7 +468,7 @@ def load_and_update(tests_root,
logger.info("Manifest url base did not match, rebuilding")

if manifest is None:
manifest = Manifest(url_base, meta_filters=meta_filters)
manifest = Manifest(tests_root, url_base, meta_filters=meta_filters)
update = True

if update:
Expand Down
Loading

0 comments on commit 76598af

Please sign in to comment.