adds a trace filter to gather ruby gems and apps

reprozip/reprozip/filters.py

@@ -6,6 +6,7 @@
 
 import logging
 import re
+from rpaths import Path
 
 from reprozip.tracer.trace import TracedFile
 from reprozip.utils import irange, iteritems
@@ -65,3 +66,55 @@ def python(files, input_files, **kwargs):
                 lst.append(path)
 
         input_files[i] = lst
+
+
+def ruby(files, input_files, **kwargs):
+    extensions = set(ext.encode('utf-8') for ext in [
+        '.rb', '.haml', '.slim', '.erb', '.js', '.html',
+    ])
+    ignored_dirs = set(name.encode('utf-8') for name in [
+        'spec', 'test', 'tests', 'guides', 'doc-api', 'rdoc', 'doc',
+    ])
+
+    gemy_path = re.compile(r'^.*/ruby[-/]\d+\.\d+\.\d+/gems')
+    appdir_paths = re.compile(r'^.*/app/(views|controllers|models|helpers)')
+
+    directories = set()
+
+    for path, fi in iteritems(files):
+        m1 = gemy_path.match(str(path))
+        if m1:
+            directories.add(Path(m1.group(0)))
+
+        m2 = appdir_paths.match(str(path))
+        if m2:
+            app_root = Path(m2.group(0)).parent.parent
+            if (app_root / 'config/application.rb').is_file():
+                directories.add(app_root)
+
+    def add_recursive(dir_or_file):
+        if (
+            dir_or_file.is_file()
+            and dir_or_file.ext in extensions
+        ):
+            logger.info("Adding %s", dir_or_file)
+            files[dir_or_file] = TracedFile(dir_or_file)
+        elif (
+            dir_or_file.is_dir()
+            and dir_or_file.name not in ignored_dirs
+        ):
+            for child in dir_or_file.listdir():
+                add_recursive(child)
+
+    for directory in directories:
+        add_recursive(directory)
+
+    for i in irange(len(input_files)):
+        lst = []
+        for path in input_files[i]:
+            if gemy_path.match(str(path)) or appdir_paths.match(str(path)):
+                logger.info("Removing input %s", path)
+            else:
+                lst.append(path)
+
+        input_files[i] = lst

reprozip/setup.py

@@ -50,7 +50,8 @@
               'reprozip = reprozip.main:main'],
           'reprozip.filters': [
               'python = reprozip.filters:python',
-              'builtin = reprozip.filters:builtin']},
+              'builtin = reprozip.filters:builtin',
+              'ruby = reprozip.filters:ruby']},
       install_requires=req,
       description="Linux tool enabling reproducible experiments (packer)",
       author="Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire",

tests/test_rails_filter.py

@@ -0,0 +1,100 @@
+# Copyright (C) 2014-2017 New York University
+# This file is part of ReproZip which is released under the Revised BSD License
+# See file LICENSE for full license details.
+
+from __future__ import division, print_function, unicode_literals
+
+from reprozip.common import File
+from reprozip.filters import ruby
+from rpaths import Path
+import unittest
+
+
+class MockTracedFile(File):
+    def __init__(self, path):
+        File.__init__(self, path, None)
+
+
+class RailsFilterTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = Path.tempdir(prefix='reprozip_tests_')
+
+    def tearDown(self):
+        self.tmp.rmtree()
+
+    @classmethod
+    def touch(cls, test_files):
+        for fi in test_files:
+            if not fi.parent.is_dir():
+                fi.parent.mkdir(parents=True)
+            with fi.open('a'):
+                pass
+
+    def test_consuming_entire_gem(self):
+        gemdir = self.tmp / 'gems/ruby-2.2.3/gems/kaminari-0.16.3'
+        gemfiles = [
+            'app/views/kaminari/_first_page.html.erb',
+            'app/views/kaminari/_first_page.html.haml',
+            'app/views/kaminari/_first_page.html.slim',
+            'app/views/kaminari/_gap.html.erb',
+            'app/views/kaminari/_gap.html.haml',
+            'app/views/kaminari/_gap.html.slim',
+            'app/views/kaminari/_last_page.html.erb',
+            'app/views/kaminari/_last_page.html.haml',
+            'app/views/kaminari/_last_page.html.slim',
+        ]
+
+        self.touch(gemdir / f for f in gemfiles)
+
+        input_files = [[]]
+        files = {}
+
+        for path in gemdir.recursedir():
+            if b'_first' in path.name:
+                f = MockTracedFile(path)
+                files[f.path] = f
+                input_files[0].append(path)
+
+        ruby(files=files, input_files=input_files)
+
+        self.assertEqual(set(files.keys()), set(gemdir / f for f in gemfiles))
+
+    def test_consuming_rails_files(self):
+        # Should be recognized: has a config file
+        railsfiles = [
+            'yes/config/application.rb',
+            'yes/app/views/application.html.erb',
+            'yes/app/views/discussion-sidebar.html.erb',
+            'yes/app/views/payments_listing.html.erb',
+            'yes/app/views/print-friendly.html.erb',
+            'yes/app/views/w-sidebar.html.erb',
+            'yes/app/views/widget.html.erb',
+        ]
+        # Should NOT be: no config file
+        notrailsfiles = [
+            # 'no/config/application.rb',
+            'no/app/views/application.html.erb',
+            'no/app/views/discussion-sidebar.html.erb',
+            'no/app/views/payments_listing.html.erb',
+            'no/app/views/print-friendly.html.erb',
+            'no/app/views/w-sidebar.html.erb',
+            'no/app/views/widget.html.erb',
+        ]
+
+        self.touch(self.tmp / f for f in railsfiles)
+        self.touch(self.tmp / f for f in notrailsfiles)
+
+        input_files = [[]]
+        files = {}
+
+        viewsdir = MockTracedFile(self.tmp / railsfiles[-1])
+        files[viewsdir.path] = viewsdir
+        viewsdir = MockTracedFile(self.tmp / notrailsfiles[-1])
+        files[viewsdir.path] = viewsdir
+
+        ruby(files, input_files)
+
+        self.assertEqual(
+            set(files.keys()),
+            set(self.tmp / f for f in railsfiles + [notrailsfiles[-1]]),
+        )