Merge pull request #162 from linkedin/xctestrun-squash

Implement reading xctestrun files and multiple cleanups
MobileNativeFoundation · Jun 16, 2017 · ce998c3 · ce998c3
2 parents ae92f3d + 7cd96e8
commit ce998c3
Show file tree

Hide file tree

Showing 36 changed files with 638 additions and 588 deletions.
diff --git a/BPSampleApp/BPSampleApp.xcodeproj/project.pbxproj b/BPSampleApp/BPSampleApp.xcodeproj/project.pbxproj
@@ -989,6 +989,7 @@
 				BAFCCA651E36DC2000E33C31 /* Release */,
 			);
 			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Debug;
 		};
 /* End XCConfigurationList section */
 	};

diff --git a/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPSampleApp.xcscheme b/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPSampleApp.xcscheme
@@ -121,6 +121,29 @@
                BlueprintName = "BPSampleAppTests"
                ReferencedContainer = "container:BPSampleApp.xcodeproj">
             </BuildableReference>
+            <SkippedTests>
+               <Test
+                  Identifier = "BPSampleAppTests/testCase002">
+               </Test>
+               <Test
+                  Identifier = "BPSampleAppTests/testCase003">
+               </Test>
+               <Test
+                  Identifier = "BPSampleAppTests/testCase005">
+               </Test>
+               <Test
+                  Identifier = "BPSampleAppTests/testCase007">
+               </Test>
+               <Test
+                  Identifier = "BPSampleAppTests/testCase011">
+               </Test>
+               <Test
+                  Identifier = "BPSampleAppTests/testCase013">
+               </Test>
+               <Test
+                  Identifier = "BPSampleAppTests/testCase017">
+               </Test>
+            </SkippedTests>
          </TestableReference>
          <TestableReference
             skipped = "NO">

diff --git a/Bluepill-cli/BPInstanceTests/BPCLITests.m b/Bluepill-cli/BPInstanceTests/BPCLITests.m
@@ -31,7 +31,7 @@ - (void)tearDown {
 }
 
 - (void)testNoSchemeinCLI {
-    BPConfiguration *config = [[BPConfiguration alloc] initWithProgram:BP_SLAVE];
+    BPConfiguration *config = [[BPConfiguration alloc] initWithProgram:BP_MASTER];
     NSError *err;
     BOOL result;
 

diff --git a/Bluepill-cli/BPInstanceTests/BPConfigurationTests.m b/Bluepill-cli/BPInstanceTests/BPConfigurationTests.m
@@ -35,11 +35,12 @@ - (void)testConfigWithNumberWhereWeExpectAString {
     NSString *resourcePath = [BPTestHelper resourceFolderPath];
     NSString *configFile = [resourcePath stringByAppendingPathComponent:@"testConfig-busted.json"];
     [config loadConfigFile:configFile withError:&error];
+    config.schemePath = [resourcePath stringByAppendingPathComponent:@"testScheme.xcscheme"];
     XCTAssertNil(error);
     [config validateConfigWithError:&error];
     XCTAssertNotNil(error);
     NSString *expected = [[NSString alloc] initWithFormat:@"runtime must be a string like '%s'.", BP_DEFAULT_RUNTIME];
-    XCTAssert([[error localizedDescription] isEqualToString:expected], @"Wrong error message.");
+    XCTAssert([[error localizedDescription] isEqualToString:expected], @"Wrong error message: %@", [error localizedDescription]);
 }
 
 - (void)testConfigFileLoading {

diff --git a/Bluepill-cli/BPInstanceTests/BPTestHelper.h b/Bluepill-cli/BPInstanceTests/BPTestHelper.h
@@ -43,4 +43,7 @@
 
 // Return the path to the bp executable
 + (NSString *)bpExecutablePath;
+
+// Return the derivedDataPath
++ (NSString *)derivedDataPath;
 @end
diff --git a/Bluepill-cli/BPInstanceTests/BPTestHelper.m b/Bluepill-cli/BPInstanceTests/BPTestHelper.m
@@ -13,7 +13,7 @@ @implementation BPTestHelper
 
 // Return the path to the sample app directory (path to XX.app)
 + (NSString *)sampleAppPath {
-    return [[self bpDerivedDataPath] stringByAppendingString:@"/BPSampleApp.app"];
+    return [[self derivedDataPath] stringByAppendingString:@"/BPSampleApp.app"];
 }
 
 + (NSString *)sampleTestScheme {
@@ -63,7 +63,7 @@ + (NSString *)bpExecutablePath {
 
 #pragma mark - Helpers
 
-+ (NSString *)bpDerivedDataPath {
++ (NSString *)derivedDataPath {
     NSString *currentPath = [[NSBundle bundleForClass:[self class]] bundlePath];
     return [[[currentPath stringByDeletingLastPathComponent]
             stringByDeletingLastPathComponent] stringByAppendingPathComponent:@"Debug-iphonesimulator"];

diff --git a/Bluepill-cli/BPInstanceTests/BPUtilsTests.m b/Bluepill-cli/BPInstanceTests/BPUtilsTests.m
@@ -26,12 +26,10 @@ - (void)setUp {
     [BPUtils quietMode:[BPUtils isBuildScript]];
 
     NSString *testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
-    NSString *basename = [[testBundlePath lastPathComponent] stringByDeletingPathExtension];
-    NSString *executable = [testBundlePath stringByAppendingPathComponent:basename];
-
-    self.xcTestFile = [BPXCTestFile BPXCTestFileFromExecutable:executable
-                                                  isUITestFile:NO
-                                                     withError:nil];
+
+    self.xcTestFile = [BPXCTestFile BPXCTestFileFromXCTestBundle:testBundlePath
+                                                andHostAppBundle:[BPTestHelper sampleAppBalancingTestsBundlePath]
+                                                       withError:nil];
 
     self.config = [BPConfiguration new];
     self.config.program = BP_MASTER;
@@ -42,40 +40,6 @@ - (void)tearDown {
     [super tearDown];
 }
 
-- (void)testBuildArgsAndEnvironmentWithPath {
-    NSString *path = @"testScheme.xcscheme";
-    path = [[[NSBundle bundleForClass:[self class]] resourcePath] stringByAppendingPathComponent:path];
-    NSDictionary *dictionary = [BPUtils buildArgsAndEnvironmentWith:path];
-    NSDictionary *expectedDictionary = @{
-                                         @"args" : @[
-                                                 @"-com.linkedin.mntf-ios.EnvironmentAnimationSpeed",
-                                                 @"10000",
-                                                 @"-com.linkedin.kif.EnvironmentUseAnimation",
-                                                 @"false",
-                                                 @"-com.linkedin.kif.EnvironmentSpeed",
-                                                 @"10",
-                                                 @"-com.linkedin.mntf-ios.EnvironmentLiveTestsEnabled",
-                                                 @"false",
-                                                 @"-mntf.validateLiveEvents",
-                                                 @"true",
-                                                 @"-NSTreatUnknownArgumentsAsOpen",
-                                                 @"NO",
-                                                 @"-ApplePersistenceIgnoreState",
-                                                 @"YES"
-                                                 ],
-                                         @"env" : @{
-                                                 @"MNTF_SCREENSHOTS" : @"$(PROJECT_DIR)/build/outputs/tests_artifacts",
-                                                 @"MNTF_SCREENSHOTS_BASELINE" : @"$(PROJECT_DIR)/mntf-iosUITests/Screenshots/Baseline",
-                                                 @"MNTF_SCREENSHOTS_BASELINE_REAL" : @"/Documents/Baseline",
-                                                 @"MNTF_SCREENSHOTS_DIFF" : @"$(PROJECT_DIR)/build/outputs/tests_artifacts/Diff",
-                                                 @"MNTF_SCREENSHOTS_DIFF_REAL" : @"/Documents/Screenshots/Diff",
-                                                 @"MNTF_SCREENSHOTS_REAL" : @"/Documents/Screenshots",
-                                                 @"TA_SCREENSHOTS_REAL" : @"/Documents/Screenshots"
-                                                 }
-                                         };
-    XCTAssert([dictionary isEqualToDictionary:expectedDictionary], @"Dictionary doesn't match expectation");
-}
-
 - (void)testNormalizingConfigurationExcludesAllTestsInExcludedTestSuite {
     self.config.testCasesToSkip = @[@"BPSampleAppTests"];
 

diff --git a/Bluepill-cli/Bluepill-cli.xcodeproj/project.pbxproj b/Bluepill-cli/Bluepill-cli.xcodeproj/project.pbxproj
@@ -112,7 +112,7 @@
 		7A4D7A851DDBB156001E085D /* BPExitStatus.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPExitStatus.m; sourceTree = "<group>"; };
 		7A4D7A881DDE3A85001E085D /* intermixed_crash.log */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = intermixed_crash.log; sourceTree = "<group>"; };
 		7A4FB8CD1DF89A790073F268 /* BPConfiguration.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPConfiguration.h; sourceTree = "<group>"; };
-		7A4FB8CE1DF89A790073F268 /* BPConfiguration.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPConfiguration.m; sourceTree = "<group>"; };
+		7A4FB8CE1DF89A790073F268 /* BPConfiguration.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; lineEnding = 0; path = BPConfiguration.m; sourceTree = "<group>"; xcLanguageSpecificationIdentifier = xcode.lang.objc; };
 		7A4FB8CF1DF89A790073F268 /* BPUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPUtils.h; sourceTree = "<group>"; };
 		7A4FB8D01DF89A790073F268 /* BPUtils.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPUtils.m; sourceTree = "<group>"; };
 		7A4FB8E51DF8AD4F0073F268 /* missed-crash.log */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "missed-crash.log"; sourceTree = "<group>"; };
@@ -452,7 +452,7 @@
 		BA95506B1D6D1AB3007D011D /* DVTFileDataTypeDetectionMagicCache.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DVTFileDataTypeDetectionMagicCache.h; sourceTree = "<group>"; };
 		BA95506C1D6D1AB3007D011D /* DVTFileDataTypeDetector-Protocol.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "DVTFileDataTypeDetector-Protocol.h"; sourceTree = "<group>"; };
 		BA95506D1D6D1AB3007D011D /* DVTFileLogger.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DVTFileLogger.h; sourceTree = "<group>"; };
-		BA95506E1D6D1AB3007D011D /* DVTFilePath.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DVTFilePath.h; sourceTree = "<group>"; };
+		BA95506E1D6D1AB3007D011D /* DVTFilePath.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; lineEnding = 0; path = DVTFilePath.h; sourceTree = "<group>"; xcLanguageSpecificationIdentifier = xcode.lang.objcpp; };
 		BA95506F1D6D1AB3007D011D /* DVTFilePathEventWatcher.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DVTFilePathEventWatcher.h; sourceTree = "<group>"; };
 		BA9550701D6D1AB3007D011D /* DVTFileSystemRepresentationProviding-Protocol.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "DVTFileSystemRepresentationProviding-Protocol.h"; sourceTree = "<group>"; };
 		BA9550711D6D1AB3007D011D /* DVTFileSystemVNode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DVTFileSystemVNode.h; sourceTree = "<group>"; };
@@ -908,7 +908,7 @@
 		BAFCCA701E37298B00E33C31 /* XCTestDriverInterface-Protocol.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "XCTestDriverInterface-Protocol.h"; sourceTree = "<group>"; };
 		BAFCCA711E37298B00E33C31 /* XCTestManager_DaemonConnectionInterface-Protocol.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "XCTestManager_DaemonConnectionInterface-Protocol.h"; sourceTree = "<group>"; };
 		C41A2C701E0B2497005D9751 /* BPXCTestFile.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPXCTestFile.h; sourceTree = "<group>"; };
-		C41A2C711E0B2497005D9751 /* BPXCTestFile.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPXCTestFile.m; sourceTree = "<group>"; };
+		C41A2C711E0B2497005D9751 /* BPXCTestFile.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; lineEnding = 0; path = BPXCTestFile.m; sourceTree = "<group>"; xcLanguageSpecificationIdentifier = xcode.lang.objc; };
 		C41A2C731E0B24E8005D9751 /* BPTestCase.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPTestCase.h; sourceTree = "<group>"; };
 		C41A2C741E0B24E8005D9751 /* BPTestCase.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPTestCase.m; sourceTree = "<group>"; };
 		C41A2C751E0B24E8005D9751 /* BPTestClass.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPTestClass.h; sourceTree = "<group>"; };

diff --git a/Bluepill-cli/Bluepill-cli/Bluepill/Bluepill.m b/Bluepill-cli/Bluepill-cli/Bluepill/Bluepill.m
@@ -395,8 +395,7 @@ - (void)launchApplicationWithContext:(BPExecutionContext *)context {
         NEXT([__self deleteSimulatorWithContext:context andStatus:BPExitStatusLaunchAppFailed]);
     };
 
-    [context.runner launchApplicationAndExecuteTestsWithParser:context.parser andCompletion:handler.defaultHandlerBlock isHostApp:NO];
-    [BPUtils printInfo:INFO withString:[@"Yay, after the launch async call!!!" stringByAppendingString:stepName]];
+    [context.runner launchApplicationAndExecuteTestsWithParser:context.parser andCompletion:handler.defaultHandlerBlock];
 }
 
 - (void)connectTestBundleAndTestDaemonWithContext:(BPExecutionContext *)context {

diff --git a/Bluepill-cli/Bluepill-cli/Bluepill/Connections/BPTestBundleConnection.m b/Bluepill-cli/Bluepill-cli/Bluepill/Connections/BPTestBundleConnection.m
@@ -215,11 +215,13 @@ - (id)_XCT_launchProcessWithPath:(NSString *)path bundleID:(NSString *)bundleID
                               };
     NSError *error;
     DTXRemoteInvocationReceipt *receipt = [objc_lookUpClass("DTXRemoteInvocationReceipt") new];
+    [BPUtils printInfo:DEBUGINFO withString:@"Installing UITargetApp: %@", path];
     [self.simulator.device installApplication:[NSURL fileURLWithPath:path] withOptions:@{kCFBundleIdentifier: bundleID} error:&error];
     if (error) {
         [BPUtils printInfo:ERROR withString:@"Launch application during UI tests failed %@", error];
         return nil;
     }
+    [BPUtils printInfo:DEBUGINFO withString:@"Launching app: %@", bundleID];
     self.appProcessPID = [self.simulator.device launchApplicationWithID:bundleID options:options error:nil];
     self.bundleID = bundleID;
     if (error) {

diff --git a/Bluepill-cli/Bluepill-cli/Simulator/BPSimulator.h b/Bluepill-cli/Bluepill-cli/Simulator/BPSimulator.h
@@ -36,7 +36,7 @@
 
 - (BOOL)installApplicationAndReturnError:(NSError *__autoreleasing *)error;
 
-- (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCompletion:(void (^)(NSError *, pid_t))completion isHostApp:(BOOL)isHostApp;
+- (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCompletion:(void (^)(NSError *, pid_t))completion;
 
 - (void)deleteSimulatorWithCompletion:(void (^)(NSError *error, BOOL success))completion;
 

diff --git a/Bluepill-cli/Bluepill-cli/Simulator/BPSimulator.m b/Bluepill-cli/Bluepill-cli/Simulator/BPSimulator.m
@@ -234,7 +234,7 @@ - (BOOL)uninstallApplicationAndReturnError:(NSError *__autoreleasing *)error {
                                        error:error];
 }
 
-- (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCompletion:(void (^)(NSError *, pid_t))completion isHostApp:(BOOL)isHostApp {
+- (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCompletion:(void (^)(NSError *, pid_t))completion {
     NSString *hostBundleId = [SimulatorHelper bundleIdForPath:self.config.appBundlePath];
     NSString *hostAppExecPath = [SimulatorHelper executablePathforPath:self.config.appBundlePath];
 
@@ -244,7 +244,10 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCom
         hostBundleId = [SimulatorHelper bundleIdForPath:self.config.testRunnerAppPath];
     }
     // Create the environment for the host application
-    NSDictionary *argsAndEnv = [BPUtils buildArgsAndEnvironmentWith:self.config.schemePath];
+
+    NSMutableDictionary *argsAndEnv = [[NSMutableDictionary alloc] init];
+    argsAndEnv[@"args"] = self.config.commandLineArguments ?: @[];
+    argsAndEnv[@"env"] = self.config.environmentVariables ?: @{};
 
     NSMutableDictionary *appLaunchEnvironment = [NSMutableDictionary dictionaryWithDictionary:[SimulatorHelper appLaunchEnvironmentWithBundleID:hostBundleId device:self.device config:self.config]];
     [appLaunchEnvironment addEntriesFromDictionary:argsAndEnv[@"env"]];
@@ -326,7 +329,7 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCom
 
             [BPUtils printInfo:INFO withString:@"Completion block for launch"];
             if (completion) {
-                [BPUtils printInfo:INFO withString:@"Calling completion block"];
+                [BPUtils printInfo:INFO withString:@"Calling completion block with: %@ - %d", error, pid];
                 completion(error, pid);
             }
         });
@@ -338,47 +341,39 @@ - (void)deleteSimulatorWithCompletion:(void (^)(NSError *error, BOOL success))co
     NSError *error;
     SimServiceContext *sc = [SimServiceContext sharedServiceContextForDeveloperDir:self.config.xcodePath error:&error];
     SimDeviceSet *deviceSet = [sc defaultDeviceSetWithError:&error];
+    if (!self.app && !self.device) {
+        [BPUtils printInfo:ERROR withString:@"No device to delete"];
+        completion(nil, NO);
+        return;
+    }
     if (self.app) {
         [BPUtils printInfo:INFO withString:@"Terminating Simulator.app"];
         [self.app terminate];
-        // We need to wait until the simulator has shut down.
-        int attempts = 300;
-        while (attempts > 0 && ![self.device.stateString isEqualToString:@"Shutdown"]) {
-            [NSThread sleepForTimeInterval:1.0];
-            --attempts;
-        }
-        if (![self.device.stateString isEqualToString:@"Shutdown"]) {
-            [BPUtils printInfo:ERROR withString:@"Timed out waiting for %@ to shutdown. It won't be deleted. Last state: %@", self.device.name, self.device.stateString];
-            // Go ahead and try to delete anyway
-        }
-        [deviceSet deleteDeviceAsync:self.device completionHandler:^(NSError *error) {
-            if (error) {
-                [BPUtils printInfo:ERROR withString:@"Could not delete simulator: %@", [error localizedDescription]];
-            }
-            completion(error, error ? NO: YES);
-        }];
     } else if (self.device) {
-        [BPUtils printInfo:INFO withString:@"Shutting down Simulator Device"];
-        [self.device shutdownAsyncWithCompletionHandler:^(NSError *error) {
-            if (!error) {
-                [BPUtils printInfo:INFO withString:@"Deleting Simulator Device"];
-                dispatch_async(dispatch_get_main_queue(), ^{
-                    [deviceSet deleteDeviceAsync:self.device completionHandler:^(NSError *error) {
-                        if (error) {
-                            [BPUtils printInfo:ERROR withString:@"Could not delete simulator: %@", [error localizedDescription]];
-                        }
-                        completion(error, error ? NO: YES);
-                    }];
-                });
-            } else {
-                [BPUtils printInfo:ERROR withString:@"Could not shutdown simulator: %@", [error localizedDescription]];
-                completion(error, NO);
-            }
-        }];
-    } else {
-        [BPUtils printInfo:ERROR withString:@"No device to delete"];
-        completion(nil, NO);
+        [BPUtils printInfo:INFO withString:@"Shutting down Simulator"];
+        [self.device shutdownWithError:&error];
+        if (error) {
+            [BPUtils printInfo:ERROR withString:@"Shutting down Simulator failed: %@", [error localizedDescription]];
+            completion(error, NO);
+            return;
+        }
+    }
+    // We need to wait until the simulator has shut down.
+    int attempts = 300;
+    while (attempts > 0 && ![self.device.stateString isEqualToString:@"Shutdown"]) {
+        [NSThread sleepForTimeInterval:1.0];
+        --attempts;
+    }
+    if (![self.device.stateString isEqualToString:@"Shutdown"]) {
+        [BPUtils printInfo:ERROR withString:@"It may not be possible to delete simulator %@ in '%@' state.", self.device.name, self.device.stateString];
+        // Go ahead and try to delete anyway
     }
+    [deviceSet deleteDeviceAsync:self.device completionHandler:^(NSError *error) {
+        if (error) {
+            [BPUtils printInfo:ERROR withString:@"Could not delete simulator: %@", [error localizedDescription]];
+        }
+        completion(error, error ? NO: YES);
+    }];
 }
 
 #pragma mark - helper methods