Dev exp stop more

Makefile

@@ -186,6 +186,7 @@ install-python-modules:
 install-node-modules:
 	mkdir -p $(INSTALL_PREFIX)/nni
 	rm -rf src/nni_manager/dist/node_modules
+	rm -rf $(INSTALL_PREFIX)/nni/nni_manager
 
 	#$(_INFO) Installing NNI Manager $(_END)
 	cp -rT src/nni_manager/dist $(INSTALL_PREFIX)/nni/nni_manager

azure-pipelines.yml

@@ -9,10 +9,10 @@ steps:
   - script: python3 -m pip install --upgrade pip setuptools
     displayName: 'Install python tools'
   - script: |
-      make easy-install
-      export PATH=$HOME/.nni/bin:$PATH
+      source install.sh
     displayName: 'Install dependencies'
   - script: |
       cd test/naive
-      PATH=$HOME/.local/nni/node/bin:$PATH python3 run.py
+      export PATH=$HOME/.local/bin:$PATH
+      python3 run.py
     displayName: 'Run tests'

docs/HowToContribute.md

@@ -51,4 +51,4 @@ After you change some code, just use **step 4** to rebuild your code, then the c
 
 ---
 At last, wish you have a wonderful day.
-For more contribution guidelines on making PR's or issues to NNI source code, you can refer to our [CONTRIBUTING](./docs/CONTRIBUTING.md) document. 
+For more contribution guidelines on making PR's or issues to NNI source code, you can refer to our [CONTRIBUTING](./docs/CONTRIBUTING.md) document. 

docs/WriteYourTrial.md

@@ -123,4 +123,4 @@ useAnnotation: true
 ```
 
 ## More Trial Example
-* [Automatic Model Architecture Search for Reading Comprehension.](../examples/trials/ga_squad/README.md)
+* [Automatic Model Architecture Search for Reading Comprehension.](../examples/trials/ga_squad/README.md)

src/nni_manager/common/manager.ts

@@ -22,7 +22,7 @@
 import { MetricDataRecord, MetricType, TrialJobInfo } from './datastore';
 import { TrialJobStatus } from './trainingService';
 
-type ProfileUpdateType = 'TRIAL_CONCURRENCY' | 'MAX_EXEC_DURATION' | 'SEARCH_SPACE';
+type ProfileUpdateType = 'TRIAL_CONCURRENCY' | 'MAX_EXEC_DURATION' | 'SEARCH_SPACE' | 'MAX_TRIAL_NUM';
 
 interface ExperimentParams {
     authorName: string;
@@ -73,7 +73,7 @@ interface TrialJobStatistics {
 }
 
 interface NNIManagerStatus {
-    status: 'INITIALIZED' | 'EXPERIMENT_RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED';
+    status: 'INITIALIZED' | 'EXPERIMENT_RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED' | 'DONE';
     errors: string[];
 }
 

src/nni_manager/rest_server/restValidationSchemas.ts

@@ -86,7 +86,7 @@ export namespace ValidationSchemas {
     };
     export const UPDATEEXPERIMENT = {
         query: {
-            update_type: joi.string().required().valid('TRIAL_CONCURRENCY', 'MAX_EXEC_DURATION', 'SEARCH_SPACE')
+            update_type: joi.string().required().valid('TRIAL_CONCURRENCY', 'MAX_EXEC_DURATION', 'SEARCH_SPACE', 'MAX_TRIAL_NUM')
         },
         body: {
             id: joi.string().required(),

test/naive/.gitignore

@@ -0,0 +1,5 @@
+__pycache__
+
+tuner_search_space.json
+tuner_result.txt
+assessor_result.txt

test/naive/README.md

@@ -0,0 +1,20 @@
+## Usage
+
+* To test before installing:
+`./run.py --preinstall`
+* To test the integrity of installation:
+`./run.py`
+* It will print `PASS` in green eventually if everything works well.
+
+## Details
+* This test case tests the communication between trials and tuner/assessor.
+* The naive trials receive an integer `x` as parameter, and reports `x`, `x²`, `x³`, ... , `x¹⁰` as metrics.
+* The naive tuner simply generates the sequence of natural numbers, and print received metrics to `tuner_result.txt`.
+* The naive assessor kills trials when `sum(metrics) % 11 == 1`, and print killed trials to `assessor_result.txt`.
+* When tuner and assessor exit with exception, they will append `ERROR` to corresponding result file.
+* When the experiment is done, meaning it is successfully done in this case, `Experiment done` can be detected in the nni_manager.log file.
+
+## Issues
+* Private APIs are used to detect whether tuner and assessor have terminated successfully. 
+* The output of REST server is not tested.
+* Remote machine training service is not tested.

test/naive/expected_assessor_result.txt

@@ -4,4 +4,3 @@
 5 3
 7 2
 8 3
-DONE

test/naive/expected_tuner_result.txt

@@ -2,4 +2,3 @@
 6 60466176
 9 3486784401
 10 10000000000
-DONE

test/naive/naive_assessor.py

@@ -1,10 +1,13 @@
 import logging
+import os
 
 from nni.assessor import Assessor, AssessResult
 
 _logger = logging.getLogger('NaiveAssessor')
 _logger.info('start')
-_result = open('/tmp/nni_assessor_result.txt', 'w')
+
+_pwd = os.path.dirname(__file__)
+_result = open(os.path.join(_pwd, 'assessor_result.txt'), 'w')
 
 class NaiveAssessor(Assessor):
     def __init__(self, optimize_mode):
@@ -30,7 +33,6 @@ def assess_trial(self, trial_job_id, trial_history):
         return AssessResult.Good
 
     def _on_exit(self):
-        _result.write('DONE\n')
         _result.close()
 
     def _on_error(self):

test/naive/naive_tuner.py

@@ -1,11 +1,14 @@
 import json
 import logging
+import os
 
 from nni.tuner import Tuner
 
 _logger = logging.getLogger('NaiveTuner')
 _logger.info('start')
-_result = open('/tmp/nni_tuner_result.txt', 'w')
+
+_pwd = os.path.dirname(__file__)
+_result = open(os.path.join(_pwd, 'tuner_result.txt'), 'w')
 
 class NaiveTuner(Tuner):
     def __init__(self, optimize_mode):
@@ -24,11 +27,10 @@ def receive_trial_result(self, parameter_id, parameters, reward):
 
     def update_search_space(self, search_space):
         _logger.info('update_search_space: %s' % search_space)
-        with open('/tmp/nni_tuner_search_space.json', 'w') as file_:
+        with open(os.path.join(_pwd, 'tuner_search_space.json'), 'w') as file_:
             json.dump(search_space, file_)
 
     def _on_exit(self):
-        _result.write('DONE\n')
         _result.close()
 
     def _on_error(self):