(MODULES-6739) Add Testmode switcher

.sync.yml

@@ -7,6 +7,10 @@ Gemfile:
       - gem: 'rspec-puppet'
         platforms: ["mswin", "mingw", "x64_mingw"]
         version: '< 2.6.0'
+    ':system_tests':
+      - gem: beaker-testmode_switcher
+        version: '~> 0.4'
+      - gem: master_manipulator
 spec/spec_helper.rb:
   unmanaged: true
 appveyor.yml:

Gemfile

@@ -60,6 +60,8 @@ group :system_tests do
   gem "beaker-rspec", *location_for(ENV['BEAKER_RSPEC_VERSION'])                
   gem "beaker-hostgenerator", *location_for(ENV['BEAKER_HOSTGENERATOR_VERSION'])
   gem "beaker-abs", *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.1')        
+  gem "beaker-testmode_switcher", '~> 0.4',                                      :require => false
+  gem 'master_manipulator',                                                      :require => false
   gem "puppet-blacksmith", '~> 3.4',                                             :require => false
 end
 

spec/acceptance/access_rights_directory/allow_rights_dir_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_with_rights(acl_regex, agent, rights, target)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target, rights)) do |result|
+      execute_manifest_on(agent, acl_manifest(target, rights), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/access_rights_directory/deny_rights_dir_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_with_rights(acl_regex, agent, rights, target)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target, rights)) do |result|
+      execute_manifest_on(agent, acl_manifest(target, rights), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/access_rights_directory/mask_specific/allow_rights_dir_mask_spec.rb

@@ -4,7 +4,7 @@
 def execute_manifest_with_mask(acl_regex, agent, mask)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(mask)) do |result|
+      execute_manifest_on(agent, acl_manifest(mask), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/access_rights_directory/mask_specific/deny_rights_dir_mask_spec.rb

@@ -4,7 +4,7 @@
 def execute_manifest_with_mask(acl_regex, agent, mask)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(mask)) do |result|
+      execute_manifest_on(agent, acl_manifest(mask), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/access_rights_file/allow_rights_file_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_and_verify(acl_regex, agent, file_content, rights, target)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target, rights, file_content)) do |result|
+      execute_manifest_on(agent, acl_manifest(target, rights, file_content), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/access_rights_file/deny_rights_file_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_and_verify(acl_regex, agent, file_content, rights, target)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target, rights, file_content)) do |result|
+      execute_manifest_on(agent, acl_manifest(target, rights, file_content), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/access_rights_file/mask_specific/allow_rights_file_mask_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_and_verify(acl_regex, agent, file_content, mask)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(mask, file_content)) do |result|
+      execute_manifest_on(agent, acl_manifest(mask, file_content), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/access_rights_file/mask_specific/deny_rights_file_mask_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_and_verify(acl_regex, agent, file_content, mask)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(mask, file_content)) do |result|
+      execute_manifest_on(agent, acl_manifest(mask, file_content), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/basic_functionality/negative/negative_acl_on_linux_spec.rb

@@ -16,7 +16,7 @@
   context 'ACL Fails Gracefully on Linux' do
     linux_agents.each do |agent|
       it "Verify that the 'acl' Type Does not Work on Non-Windows Agents on #{agent}" do
-        on(agent, puppet('apply', '--debug'), stdin: acl_manifest) do |result|
+        execute_manifest_on(agent, acl_manifest, { :debug => true }) do |result|
           assert_match(%r{Error: Could not find a suitable provider for acl}, result.stderr, 'Expected error was not detected!')
         end
       end

spec/acceptance/group/group_local_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_and_verify(agent, file_content, group_id, group_regex, user_type)
   context "on #{agent}" do
     it 'Execute ACL Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(user_type, file_content, group_id)) do |result|
+      execute_manifest_on(agent, acl_manifest(user_type, file_content, group_id), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/group/group_local_unicode_group_spec.rb

@@ -62,7 +62,7 @@ def verify_content_command(user_type)
     windows_agents.each do |agent|
       context "on #{agent}" do
         it 'Execute ACL Manifest' do
-          apply_manifest_on(agent, acl_manifest(prefix, file_content, group_id), debug: true) do |result|
+          execute_manifest_on(agent, acl_manifest(prefix, file_content, group_id), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end

spec/acceptance/group/group_local_unicode_user_spec.rb

@@ -61,7 +61,7 @@ def verify_group_command(prefix, raw_group_id)
     windows_agents.each do |agent|
       context "on #{agent}" do
         it 'Execute ACL Manifest' do
-          apply_manifest_on(agent, acl_manifest(prefix, file_content, group_id), debug: true) do |result|
+          execute_manifest_on(agent, acl_manifest(prefix, file_content, group_id), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end

spec/acceptance/group/group_local_user_sid_spec.rb

@@ -91,7 +91,7 @@ def acl_manifest(target, user_id, sid)
         end
 
         it 'Execute Setup Manifest' do
-          on(agent, puppet('apply', '--debug'), stdin: setup_manifest(target_parent, target, file_content, user_id, group_id)) do |result|
+          execute_manifest_on(agent, setup_manifest(target_parent, target, file_content, user_id, group_id), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end
@@ -103,7 +103,7 @@ def acl_manifest(target, user_id, sid)
         end
 
         it 'Execute ACL Manifest' do
-          on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target, user_id, sid)) do |result|
+          execute_manifest_on(agent, acl_manifest(target, user_id, sid), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end

spec/acceptance/group/negative/group_257_char_name_spec.rb

@@ -56,7 +56,7 @@
   windows_agents.each do |agent|
     context "On Windows Agent Change Group to Local Group with Long Name on #{agent}" do
       it 'Attempt to Execute ACL Manifest' do
-        on(agent, puppet('apply', '--debug'), stdin: acl_manifest) do |result|
+        execute_manifest_on(agent, acl_manifest, { :debug => true }) do |result|
           assert_match(expected_error, result.stderr, 'Expected error was not detected!')
         end
 

spec/acceptance/identity/negative/specify_identity_spec.rb

@@ -33,7 +33,7 @@ def verify_content_command(target_file)
     windows_agents.each do |agent|
       context "on #{agent}" do
         it 'Execute Manifest' do
-          on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_file, file_content, group_id)) do |result|
+          execute_manifest_on(agent, acl_manifest(target_file, file_content, group_id), { :debug => true }) do |result|
             assert_match(%r{Error: Failed to set permissions for }, result.stderr, 'Expected error was not detected!')
           end
         end
@@ -55,7 +55,7 @@ def verify_content_command(target_file)
     windows_agents.each do |agent|
       context "on #{agent}" do
         it 'Execute Manifest' do
-          on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_file, file_content, user_id)) do |result|
+          execute_manifest_on(agent, acl_manifest(target_file, file_content, user_id), { :debug => true }) do |result|
             assert_match(%r{Error: Failed to set permissions for 'user_not_here'}, result.stderr, 'Expected error was not detected!')
           end
         end

spec/acceptance/identity/specify_app_package_authority_idents_spec.rb

@@ -45,7 +45,7 @@
 
             it 'Execute ACL Manifest' do
               # exit code 2: The run succeeded, and some resources were changed.
-              on(agent, puppet('apply', '--detailed-exitcodes'), stdin: acl_manifest, acceptable_exit_codes: [2]) do |result|
+              execute_manifest_on(agent, acl_manifest, { :expect_changes => true }) do |result|
                 assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
               end
             end
@@ -59,7 +59,7 @@
             end
 
             it 'Execute ACL Manifest again' do
-              on(agent, puppet('apply'), stdin: acl_manifest, acceptable_exit_codes: [0]) do |result|
+              execute_manifest_on(agent, acl_manifest, { :catch_changes => true } ) do |result|
                 assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
               end
             end

spec/acceptance/identity/specify_group_identity_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_and_verify(agent, file_content, group_id, target_file)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest_with_group(target_file, file_content, group_id)) do |result|
+      execute_manifest_on(agent, acl_manifest_with_group(target_file, file_content, group_id), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/identity/specify_sid_identity_spec.rb

@@ -59,7 +59,7 @@ def acl_manifest(target_file, sid)
         end
 
         it 'Execute Setup Manifest' do
-          on(agent, puppet('apply', '--debug'), stdin: setup_manifest(target_file, file_content)) do |result|
+          execute_manifest_on(agent, setup_manifest(target_file, file_content), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end
@@ -71,7 +71,7 @@ def acl_manifest(target_file, sid)
         end
 
         it 'Execute ACL Manifest' do
-          on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_file, sid)) do |result|
+          execute_manifest_on(agent, acl_manifest(target_file, sid), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end

spec/acceptance/identity/specify_unicode_identity_spec.rb

@@ -3,7 +3,7 @@
 # rubocop:disable RSpec/EmptyExampleGroup
 def apply_manifest_and_verify(agent, file_content, target, user_id, verify_acl_command)
   it 'Execute Manifest' do
-    apply_manifest_on(agent, acl_manifest_user(target, file_content, user_id), debug: true) do |result|
+    execute_manifest_on(agent, acl_manifest_user(target, file_content, user_id), { :debug => true }) do |result|
       assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
     end
   end
@@ -78,7 +78,7 @@ def acl_manifest_user(target, file_content, user_id)
     windows_agents.each do |agent|
       context "on #{agent}" do
         it 'Execute Manifest' do
-          apply_manifest_on(agent, acl_manifest_group(target, file_content, group_id), debug: true) do |result|
+          execute_manifest_on(agent, acl_manifest_group(target, file_content, group_id), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end

spec/acceptance/identity/specify_user_identity_spec.rb

@@ -4,7 +4,7 @@
 def apply_manifest_and_verify(agent, file_content, user_id, target_file)
   context "on #{agent}" do
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(file_content, user_id, target_file)) do |result|
+      execute_manifest_on(agent, acl_manifest(file_content, user_id, target_file), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/inheritance/inheritance_on_dir_spec.rb

@@ -10,7 +10,7 @@ def apply_manifest_and_verify(perm_type, asset_type, child_inherit_type, acl_chi
 
   context "on #{agent}" do
     it 'Execute Apply Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_name, target_child, user_id_child, rights, perm_type, child_inherit_type)) do |result|
+      execute_manifest_on(agent, acl_manifest(target_name, target_child, user_id_child, rights, perm_type, child_inherit_type), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/inheritance/inheritance_on_file_spec.rb

@@ -10,7 +10,7 @@ def apply_manifest_and_verify(perm_type, asset_type, child_inherit_type, acl_chi
 
   context "on #{agent}" do
     it 'Execute Apply Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_name, target_child, file_content, user_id_child, rights, perm_type, child_inherit_type)) do |result|
+      execute_manifest_on(agent, acl_manifest(target_name, target_child, file_content, user_id_child, rights, perm_type, child_inherit_type), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end

spec/acceptance/owner/negative/owner_257_char_name_spec.rb

@@ -7,7 +7,7 @@ def apply_manifest_and_verify(agent, target_name, file_content, user_id, owner_i
     verify_content_command = "cat /cygdrive/c/temp/#{target_name}"
 
     it 'Attempt to Execute ACL Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_name, file_content, user_id, owner_id)) do |result|
+      execute_manifest_on(agent, acl_manifest(target_name, file_content, user_id, owner_id), { :debug => true }) do |result|
         assert_match(expected_error, result.stderr, 'Expected error was not detected!')
       end
     end

spec/acceptance/owner/owner_local_group_spec.rb

@@ -8,7 +8,7 @@ def apply_manifest_and_verify(agent, target_name, file_content, owner_id, owner_
     verify_owner_command = "cmd /c \"dir /q #{dosify_target}\""
 
     it 'Execute ACL Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_name, file_content, owner_id)) do |result|
+      execute_manifest_on(agent, acl_manifest(target_name, file_content, owner_id), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end
@@ -95,7 +95,7 @@ def acl_manifest(target_name, file_content, owner_id)
     windows_agents.each do |agent|
       context "on #{agent}" do
         it 'Execute ACL Manifest' do
-          apply_manifest_on(agent, acl_manifest(target_name, file_content, owner_id), debug: true) do |result|
+          execute_manifest_on(agent, acl_manifest(target_name, file_content, owner_id), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end

spec/acceptance/owner/owner_local_user_sid_spec.rb

@@ -74,7 +74,7 @@ def acl_manifest(target_name, sid)
         end
 
         it 'Execute Setup Manifest' do
-          on(agent, puppet('apply', '--debug'), stdin: setup_manifest(target_name, file_content, owner_id)) do |result|
+          execute_manifest_on(agent, setup_manifest(target_name, file_content, owner_id), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end
@@ -86,7 +86,7 @@ def acl_manifest(target_name, sid)
         end
 
         it 'Execute ACL Manifest' do
-          on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_name, sid)) do |result|
+          execute_manifest_on(agent, acl_manifest(target_name, sid), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end

spec/acceptance/owner/owner_local_user_spec.rb

@@ -8,7 +8,7 @@ def apply_manifest_and_verify(agent, file_content, owner_id, target_name, owner_
     verify_owner_command = "cmd /c \"dir /q #{dosify_target}\""
 
     it 'Execute ACL Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target_name, file_content, owner_id)) do |result|
+      execute_manifest_on(agent, acl_manifest(target_name, file_content, owner_id), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end
@@ -99,7 +99,7 @@ def acl_manifest(target_name, file_content, owner_id)
     windows_agents.each do |agent|
       context "on #{agent}" do
         it 'Execute ACL Manifest' do
-          apply_manifest_on(agent, acl_manifest(target_name, file_content, owner_id), debug: true) do |result|
+          execute_manifest_on(agent, acl_manifest(target_name, file_content, owner_id), { :debug => true }) do |result|
             assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
           end
         end

spec/acceptance/parameter_target/explicit_target_spec.rb

@@ -33,7 +33,7 @@
   windows_agents.each do |agent|
     context "on #{agent}" do
       it 'Execute Manifest' do
-        on(agent, puppet('apply', '--debug'), stdin: acl_manifest) do |result|
+        execute_manifest_on(agent, acl_manifest, { :debug => true }) do |result|
           assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
         end
       end

spec/acceptance/parameter_target/negative/negative_acl_spec.rb

@@ -26,7 +26,7 @@ def acl_manifest(target)
     context "Specify Blank Target on #{agent}" do
       target = ''
       it 'Execute Manifest' do
-        on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target), acceptable_exit_codes: [1]) do |result|
+        execute_manifest_on(agent, acl_manifest(target), { :debug => true, :exepect_failures => true }) do |result|
           assert_match(%r{Error:.*(A non-empty name must be specified|Empty string title at)}, result.stderr, 'Expected error was not detected!')
         end
       end
@@ -37,7 +37,7 @@ def acl_manifest(target)
     context "Specify Target with Invalid Path Characters on #{agent}" do
       target = 'c:/temp/invalid_<:>|?*'
       it 'Execute Manifest' do
-        on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target)) do |result|
+        execute_manifest_on(agent, acl_manifest(target), { :debug => true }) do |result|
           assert_match(%r{Error:.*The filename, directory name, or volume label syntax is incorrect},
                        result.stderr, 'Expected error was not detected!')
         end

spec/acceptance/parameter_target/negative/negative_acl_to_symlink_spec.rb

@@ -64,7 +64,7 @@
       end
 
       it 'Execute Manifest' do
-        on(agent, puppet('apply', '--debug'), stdin: acl_manifest) do |result|
+        execute_manifest_on(agent, acl_manifest, { :debug => true }) do |result|
           assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
         end
       end

spec/acceptance/parameter_target/perms_on_dir_8dot3_spec.rb

@@ -5,7 +5,7 @@ def apply_manifest_and_verify(agent, target, target8dot3, verify_acl_command, re
   context "on #{agent}" do
     acl_regex = %r{.*\\bob:\(OI\)\(CI\)\(F\)}
     it 'Execute Manifest' do
-      on(agent, puppet('apply', '--debug'), stdin: acl_manifest(target, target8dot3)) do |result|
+      execute_manifest_on(agent, acl_manifest(target, target8dot3), { :debug => true }) do |result|
         assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
       end
     end
@@ -17,7 +17,7 @@ def apply_manifest_and_verify(agent, target, target8dot3, verify_acl_command, re
     end
     if remove
       it 'Execute Remove Manifest' do
-        on(agent, puppet('apply', '--debug'), stdin: acl_manifest_remove(target8dot3)) do |result|
+        execute_manifest_on(agent, acl_manifest_remove(target8dot3), { :debug => true }) do |result|
           assert_no_match(%r{Error:}, result.stderr, 'Unexpected error was detected!')
         end
       end