[Core] Support specifying telemetry.push_interval_in_hours to force push telemetry cache file

src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py

@@ -8,6 +8,7 @@
 import shutil
 import stat
 import tempfile
+from knack.config import CLIConfig
 
 
 class RecordsCollection:
@@ -27,6 +28,7 @@ def __iter__(self):
     def next_send(self):
         return self._next_send
 
+    # pylint: disable=line-too-long
     def snapshot_and_read(self):
         """ Scan the telemetry cache files and move all the rotated files to a temp directory. """
         from azure.cli.telemetry.const import TELEMETRY_CACHE_DIR
@@ -35,8 +37,11 @@ def snapshot_and_read(self):
         if not os.path.isdir(folder):
             return
 
+        # Collect all cache.x files. If it has been a long time since last sent, also collect cache file itself.
+        include_cache = datetime.datetime.now() - self._last_sent > datetime.timedelta(hours=self._get_threshold_config())
+        candidates = [(fn, os.stat(os.path.join(folder, fn))) for fn in os.listdir(folder) if include_cache or fn != 'cache']
+
         # sort the cache files base on their last modification time.
-        candidates = [(fn, os.stat(os.path.join(folder, fn))) for fn in os.listdir(folder) if fn != 'cache']
         candidates = [(fn, file_stat) for fn, file_stat in candidates if stat.S_ISREG(file_stat.st_mode)]
         candidates.sort(key=lambda pair: pair[1].st_mtime, reverse=True)  # move the newer cache file first
 
@@ -66,6 +71,12 @@ def snapshot_and_read(self):
                       onerror=lambda _, p, tr: self._logger.error('Fail to remove file %s', p))
         self._logger.info('Remove directory %s', tmp)
 
+    def _get_threshold_config(self):
+        config = CLIConfig(config_dir=self._config_dir)
+        threshold = config.getint('telemetry', 'push_data_threshold', fallback=24)
+        # the threshold for push telemetry can't be less than 1 hour, default value is 24 hours
+        return threshold if threshold >= 1 else 24
+
     def _read_file(self, path):
         """ Read content of a telemetry cache file and parse them into records. """
         try:

src/azure-cli-telemetry/azure/cli/telemetry/tests/test_records_collection.py

@@ -35,19 +35,21 @@ def test_create_records_collection(self):
         # take snapshot and move the files
         collection.snapshot_and_read()
 
-        # all but one file, the 'cache' file, is moved.
-        self.assert_cache_files_count(1)
+        # all files are moved, including the 'cache' file.
+        self.assert_cache_files_count(0)
 
         # total records
-        self.assertEqual(1750, len([r for r in collection]))
+        self.assertEqual(1758, len([r for r in collection]))
 
     def test_create_records_collection_with_last_send(self):
-        last_send = datetime.datetime(year=2018, month=6, day=5, hour=16, minute=36, second=7)
+        last_send = datetime.datetime.now() - datetime.timedelta(hours=6)
         collection = RecordsCollection(last_send, self.work_dir)
         collection.snapshot_and_read()
 
+        # the threshold for pushing 'cache' file is 24, so 'cache' file should not be moved
         self.assert_cache_files_count(1)
-        self.assertEqual(453, len([r for r in collection]))
+        # no new records since last_send
+        self.assertEqual(0, len([r for r in collection]))
 
     def test_create_records_collection_against_missing_config_folder(self):
         collection = RecordsCollection(datetime.datetime.min, tempfile.mktemp())