From 02b7de6d7e70b6b6f7abf5185956e6df8b5b81fa Mon Sep 17 00:00:00 2001 From: evelyn-ys Date: Thu, 14 Jul 2022 15:38:23 +0800 Subject: [PATCH 1/6] Force push telemetry cache file after 24 hours.(Time threshold is configurable) --- .../cli/telemetry/components/records_collection.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py b/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py index d44795fa9ea..ea34e48c419 100644 --- a/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py +++ b/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py @@ -8,6 +8,7 @@ import shutil import stat import tempfile +from knack.config import CLIConfig class RecordsCollection: @@ -35,8 +36,13 @@ def snapshot_and_read(self): if not os.path.isdir(folder): return - # sort the cache files base on their last modification time. + # Collect all cache.x files. If it has been a long time since last sent, also collect cache file itself. candidates = [(fn, os.stat(os.path.join(folder, fn))) for fn in os.listdir(folder) if fn != 'cache'] + if os.path.exists(os.path.join(folder, 'cache')) and \ + datetime.datetime.now() - self._last_sent > datetime.timedelta(hours=self._get_threshold_config()): + candidates.append(('cache', os.stat(os.path.join(folder, 'cache')))) + + # sort the cache files base on their last modification time. candidates = [(fn, file_stat) for fn, file_stat in candidates if stat.S_ISREG(file_stat.st_mode)] candidates.sort(key=lambda pair: pair[1].st_mtime, reverse=True) # move the newer cache file first @@ -66,6 +72,12 @@ def snapshot_and_read(self): onerror=lambda _, p, tr: self._logger.error('Fail to remove file %s', p)) self._logger.info('Remove directory %s', tmp) + def _get_threshold_config(self): + config = CLIConfig(config_dir=self._config_dir) + threshold = config.getint('telemetry', 'push_data_threshold', fallback=24) + # the threshold for push telemetry can't be less than 1 hour, default value is 24 hours + return threshold if threshold >= 1 else 24 + def _read_file(self, path): """ Read content of a telemetry cache file and parse them into records. """ try: From 7b09a996ee9b271335e323888217e54b63d28021 Mon Sep 17 00:00:00 2001 From: evelyn-ys Date: Thu, 14 Jul 2022 15:50:36 +0800 Subject: [PATCH 2/6] format --- .../azure/cli/telemetry/components/records_collection.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py b/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py index ea34e48c419..97791a07d20 100644 --- a/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py +++ b/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py @@ -28,6 +28,7 @@ def __iter__(self): def next_send(self): return self._next_send + # pylint: disable=line-too-long def snapshot_and_read(self): """ Scan the telemetry cache files and move all the rotated files to a temp directory. """ from azure.cli.telemetry.const import TELEMETRY_CACHE_DIR @@ -37,10 +38,8 @@ def snapshot_and_read(self): return # Collect all cache.x files. If it has been a long time since last sent, also collect cache file itself. - candidates = [(fn, os.stat(os.path.join(folder, fn))) for fn in os.listdir(folder) if fn != 'cache'] - if os.path.exists(os.path.join(folder, 'cache')) and \ - datetime.datetime.now() - self._last_sent > datetime.timedelta(hours=self._get_threshold_config()): - candidates.append(('cache', os.stat(os.path.join(folder, 'cache')))) + include_cache = datetime.datetime.now() - self._last_sent > datetime.timedelta(hours=self._get_threshold_config()) + candidates = [(fn, os.stat(os.path.join(folder, fn))) for fn in os.listdir(folder) if include_cache or fn != 'cache'] # sort the cache files base on their last modification time. candidates = [(fn, file_stat) for fn, file_stat in candidates if stat.S_ISREG(file_stat.st_mode)] From 3e09ccf502995817df5dec670376b9540a9d2744 Mon Sep 17 00:00:00 2001 From: evelyn-ys Date: Thu, 14 Jul 2022 16:27:32 +0800 Subject: [PATCH 3/6] fix telemetry test --- .../cli/telemetry/tests/test_records_collection.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/azure-cli-telemetry/azure/cli/telemetry/tests/test_records_collection.py b/src/azure-cli-telemetry/azure/cli/telemetry/tests/test_records_collection.py index a1966459160..b1415672f67 100644 --- a/src/azure-cli-telemetry/azure/cli/telemetry/tests/test_records_collection.py +++ b/src/azure-cli-telemetry/azure/cli/telemetry/tests/test_records_collection.py @@ -35,19 +35,21 @@ def test_create_records_collection(self): # take snapshot and move the files collection.snapshot_and_read() - # all but one file, the 'cache' file, is moved. - self.assert_cache_files_count(1) + # all files are moved, including the 'cache' file. + self.assert_cache_files_count(0) # total records - self.assertEqual(1750, len([r for r in collection])) + self.assertEqual(1758, len([r for r in collection])) def test_create_records_collection_with_last_send(self): - last_send = datetime.datetime(year=2018, month=6, day=5, hour=16, minute=36, second=7) + last_send = datetime.datetime.now() - datetime.timedelta(hours=6) collection = RecordsCollection(last_send, self.work_dir) collection.snapshot_and_read() + # the threshold for pushing 'cache' file is 24, so 'cache' file should not be moved self.assert_cache_files_count(1) - self.assertEqual(453, len([r for r in collection])) + # no new records since last_send + self.assertEqual(0, len([r for r in collection])) def test_create_records_collection_against_missing_config_folder(self): collection = RecordsCollection(datetime.datetime.min, tempfile.mktemp()) From 8ecd69b921102a799a973b7be8a69fa95647c74c Mon Sep 17 00:00:00 2001 From: evelyn-ys Date: Wed, 20 Jul 2022 14:33:21 +0800 Subject: [PATCH 4/6] address comments --- src/azure-cli-telemetry/HISTORY.rst | 4 ++++ .../cli/telemetry/components/records_collection.py | 11 ++++++----- src/azure-cli-telemetry/setup.py | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/azure-cli-telemetry/HISTORY.rst b/src/azure-cli-telemetry/HISTORY.rst index 6d1b84deec8..bb320ffd38f 100644 --- a/src/azure-cli-telemetry/HISTORY.rst +++ b/src/azure-cli-telemetry/HISTORY.rst @@ -2,6 +2,10 @@ Release History =============== +1.0.7 ++++++ +* Support specifying `telemetry.push_interval_in_hours` to force push telemetry cache file + 1.0.6 +++++ * Add `__version__` in `__init__.py` diff --git a/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py b/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py index 97791a07d20..a1055aeb41a 100644 --- a/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py +++ b/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py @@ -28,7 +28,6 @@ def __iter__(self): def next_send(self): return self._next_send - # pylint: disable=line-too-long def snapshot_and_read(self): """ Scan the telemetry cache files and move all the rotated files to a temp directory. """ from azure.cli.telemetry.const import TELEMETRY_CACHE_DIR @@ -38,8 +37,10 @@ def snapshot_and_read(self): return # Collect all cache.x files. If it has been a long time since last sent, also collect cache file itself. - include_cache = datetime.datetime.now() - self._last_sent > datetime.timedelta(hours=self._get_threshold_config()) - candidates = [(fn, os.stat(os.path.join(folder, fn))) for fn in os.listdir(folder) if include_cache or fn != 'cache'] + push_interval = datetime.timedelta(hours=self._get_push_interval_config()) + include_cache = datetime.datetime.now() - self._last_sent > push_interval + candidates = [(fn, os.stat(os.path.join(folder, fn))) for fn in os.listdir(folder) + if include_cache or fn != 'cache'] # sort the cache files base on their last modification time. candidates = [(fn, file_stat) for fn, file_stat in candidates if stat.S_ISREG(file_stat.st_mode)] @@ -71,9 +72,9 @@ def snapshot_and_read(self): onerror=lambda _, p, tr: self._logger.error('Fail to remove file %s', p)) self._logger.info('Remove directory %s', tmp) - def _get_threshold_config(self): + def _get_push_interval_config(self): config = CLIConfig(config_dir=self._config_dir) - threshold = config.getint('telemetry', 'push_data_threshold', fallback=24) + threshold = config.getint('telemetry', 'push_interval_in_hours', fallback=24) # the threshold for push telemetry can't be less than 1 hour, default value is 24 hours return threshold if threshold >= 1 else 24 diff --git a/src/azure-cli-telemetry/setup.py b/src/azure-cli-telemetry/setup.py index c2fb68bfc7e..e1fee7bb07b 100755 --- a/src/azure-cli-telemetry/setup.py +++ b/src/azure-cli-telemetry/setup.py @@ -8,7 +8,7 @@ from codecs import open from setuptools import setup -VERSION = "1.0.6" +VERSION = "1.0.7" CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', From 5257405efaeebbaedafdc0dcd49515b3ec56855a Mon Sep 17 00:00:00 2001 From: evelyn-ys Date: Wed, 20 Jul 2022 16:10:35 +0800 Subject: [PATCH 5/6] telemetry version --- src/azure-cli-core/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure-cli-core/setup.py b/src/azure-cli-core/setup.py index 6531643f021..0abcd410597 100644 --- a/src/azure-cli-core/setup.py +++ b/src/azure-cli-core/setup.py @@ -44,7 +44,7 @@ DEPENDENCIES = [ 'argcomplete~=1.8', - 'azure-cli-telemetry==1.0.6.*', + 'azure-cli-telemetry==1.0.7.*', 'azure-mgmt-core>=1.2.0,<2', 'cryptography', 'humanfriendly~=10.0', From 24cbc18c71b0a38dd15183fb3b4996078a224264 Mon Sep 17 00:00:00 2001 From: evelyn-ys Date: Wed, 20 Jul 2022 16:15:31 +0800 Subject: [PATCH 6/6] dependency --- src/azure-cli/requirements.py3.Darwin.txt | 2 +- src/azure-cli/requirements.py3.Linux.txt | 2 +- src/azure-cli/requirements.py3.windows.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/azure-cli/requirements.py3.Darwin.txt b/src/azure-cli/requirements.py3.Darwin.txt index 8099c0fc93e..9be281faad5 100644 --- a/src/azure-cli/requirements.py3.Darwin.txt +++ b/src/azure-cli/requirements.py3.Darwin.txt @@ -5,7 +5,7 @@ asn1crypto==0.24.0 azure-appconfiguration==1.1.1 azure-batch==12.0.0 azure-cli-core==2.38.0 -azure-cli-telemetry==1.0.6 +azure-cli-telemetry==1.0.7 azure-cli==2.38.0 azure-common==1.1.22 azure-core==1.24.0 diff --git a/src/azure-cli/requirements.py3.Linux.txt b/src/azure-cli/requirements.py3.Linux.txt index fbae8a05910..5bc464331a7 100644 --- a/src/azure-cli/requirements.py3.Linux.txt +++ b/src/azure-cli/requirements.py3.Linux.txt @@ -5,7 +5,7 @@ asn1crypto==0.24.0 azure-appconfiguration==1.1.1 azure-batch==12.0.0 azure-cli-core==2.38.0 -azure-cli-telemetry==1.0.6 +azure-cli-telemetry==1.0.7 azure-cli==2.38.0 azure-common==1.1.22 azure-core==1.24.0 diff --git a/src/azure-cli/requirements.py3.windows.txt b/src/azure-cli/requirements.py3.windows.txt index 336e24659c9..297685d471b 100644 --- a/src/azure-cli/requirements.py3.windows.txt +++ b/src/azure-cli/requirements.py3.windows.txt @@ -5,7 +5,7 @@ asn1crypto==0.24.0 azure-appconfiguration==1.1.1 azure-batch==12.0.0 azure-cli-core==2.38.0 -azure-cli-telemetry==1.0.6 +azure-cli-telemetry==1.0.7 azure-cli==2.38.0 azure-common==1.1.22 azure-core==1.24.0