[ci] do not run integration tests if not relevant (#2791)

This PR tries to avoid running a integration test for an agent check (and
only this, core integration tests, linter and core mocked tests are
always run) when it is not required.

To do this, we first lists all modified files, and then
- check if a check file was modified (in `checks.d`)
- check if a check test file was modified
- ignore fixtures and configuration

If any file other than the one listed above are modified, all tests will
be run. Otherwise, a list of modfied checks is built, and only these
tests will be run (with the core tests).

This only concerns Travis and is disabled on Appveyor.

.travis.yml

@@ -103,8 +103,6 @@ script:
   # Needed if no cache exists
   - mkdir -p $INTEGRATIONS_DIR
   - ls -al $INTEGRATIONS_DIR
-  - rm -rf /home/travis/virtualenv/python$TRAVIS_PYTHON_VERSION.9/lib/python$TRAVIS_PYTHON_VERSION/site-packages/pip-6.0.7.dist-info
-  - rm -rf /home/travis/virtualenv/python$TRAVIS_PYTHON_VERSION.9/lib/python$TRAVIS_PYTHON_VERSION/site-packages/setuptools-12.0.5.dist-info
   - 'rake ci:run'
   - ls -al $INTEGRATIONS_DIR
 

ci/common.rb

@@ -38,6 +38,26 @@ def install_requirements(req_file, pip_options = nil, output = nil, use_venv = n
   end
 end
 
+def can_skip?
+  return false, [] if Gem.win_platform?
+
+  modified_checks = []
+  `git diff-tree --no-commit-id --name-only -r FETCH_HEAD origin/master`.each_line do |filename|
+    filename.strip!
+    if filename.start_with? 'checks.d'
+      check_name = File.basename(filename, '.py')
+    elsif filename.start_with?('tests/checks/integration', 'tests/checks/mock')
+      check_name = File.basename(filename, '.py').slice 'test_'
+    elsif filename.start_with?('tests/checks/fixtures', 'conf.d')
+      next
+    else
+      return false, []
+    end
+    modified_checks << check_name unless modified_checks.include? check_name
+  end
+  [true, modified_checks]
+end
+
 # helper class to wait for TCP/HTTP services to boot
 class Wait
   DEFAULT_TIMEOUT = 10
@@ -183,12 +203,20 @@ def self.for(smth, max_timeout = DEFAULT_TIMEOUT)
 
     task :execute, :flavor do |_t, attr|
       flavor = attr[:flavor]
-      # Check if the check uses the cache
-      cached = flavor.tasks.any? { |task| task.name == (flavor.scope.path + ':before_cache') }
+      # flavor.scope.path is ci:cassandra
+      # flavor.scope.path[3..-1] is cassandra
+      check_name = flavor.scope.path[3..-1]
+
+      can_skip, checks = can_skip?
+      can_skip &&= !%w(default core_integration checks_mock).include?(check_name)
+      if can_skip && !checks.include?(check_name)
+        puts "Skipping #{check_name} tests, not affected by the change".yellow
+        next
+      end
       exception = nil
       begin
         tasks = %w(before_install install before_script script)
-        tasks << 'before_cache' if cached
+        tasks << 'before_cache' unless ENV['CI'].nil?
         tasks.each do |t|
           Rake::Task["#{flavor.scope.path}:#{t}"].invoke
         end