diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml index 8e716b5715b..7d6c1f8a465 100644 --- a/.github/workflows/promptflow-sdk-cli-test.yml +++ b/.github/workflows/promptflow-sdk-cli-test.yml @@ -154,6 +154,7 @@ jobs: path: | ${{ env.testWorkingDirectory }}/*.xml ${{ env.testWorkingDirectory }}/htmlcov/ + ${{ env.testWorkingDirectory }}/tests/sdk_cli_test/count.json publish-test-results-sdk-cli-test: diff --git a/src/promptflow/tests/conftest.py b/src/promptflow/tests/conftest.py index d1fb06bff37..20769a33d3a 100644 --- a/src/promptflow/tests/conftest.py +++ b/src/promptflow/tests/conftest.py @@ -25,7 +25,6 @@ from promptflow._cli._utils import AzureMLWorkspaceTriad from promptflow._constants import PROMPTFLOW_CONNECTIONS from promptflow._core.connection_manager import ConnectionManager -from promptflow._core.openai_injector import inject_openai_api from promptflow._utils.context_utils import _change_working_dir from promptflow.connections import AzureOpenAIConnection @@ -52,15 +51,6 @@ def mock_build_info(): yield m -@pytest.fixture(autouse=True, scope="session") -def inject_api(): - """Inject OpenAI API during test session. - - AOAI call in promptflow should involve trace logging and header injection. Inject - function to API call in test scenario.""" - inject_openai_api() - - @pytest.fixture def dev_connections() -> dict: with open(CONNECTION_FILE, "r") as f: diff --git a/src/promptflow/tests/executor/conftest.py b/src/promptflow/tests/executor/conftest.py new file mode 100644 index 00000000000..d0c4da36b47 --- /dev/null +++ b/src/promptflow/tests/executor/conftest.py @@ -0,0 +1,12 @@ +import pytest + +from promptflow._core.openai_injector import inject_openai_api + + +@pytest.fixture(autouse=True, scope="session") +def inject_api_executor(): + """Inject OpenAI API during test session. + + AOAI call in promptflow should involve trace logging and header injection. Inject + function to API call in test scenario.""" + inject_openai_api() diff --git a/src/promptflow/tests/sdk_cli_test/conftest.py b/src/promptflow/tests/sdk_cli_test/conftest.py index af92363a95d..843a7209cb4 100644 --- a/src/promptflow/tests/sdk_cli_test/conftest.py +++ b/src/promptflow/tests/sdk_cli_test/conftest.py @@ -11,6 +11,7 @@ from sqlalchemy import create_engine from promptflow import PFClient +from promptflow._core.openai_injector import inject_openai_api from promptflow._sdk._configuration import Configuration from promptflow._sdk._constants import EXPERIMENT_CREATED_ON_INDEX_NAME, EXPERIMENT_TABLE_NAME, LOCAL_MGMT_DB_PATH from promptflow._sdk._serving.app import create_app as create_serving_app @@ -19,7 +20,17 @@ from promptflow.executor._line_execution_process_pool import _process_wrapper from promptflow.executor._process_manager import create_spawned_fork_process_manager -from .recording_utilities import RecordStorage, mock_tool, recording_array_extend, recording_array_reset +from .recording_utilities import ( + RecordStorage, + delete_count_lock_file, + inject_async_with_recording, + inject_sync_with_recording, + is_live, + is_record, + is_replay, + mock_tool, + recording_array_reset, +) PROMOTFLOW_ROOT = Path(__file__) / "../../.." RUNTIME_TEST_CONFIGS_ROOT = Path(PROMOTFLOW_ROOT / "tests/test_configs/runtime") @@ -192,13 +203,10 @@ def serving_client_with_environment_variables(mocker: MockerFixture): ) -@pytest.fixture -def recording_file_override(request: pytest.FixtureRequest, mocker: MockerFixture): - if RecordStorage.is_replaying_mode() or RecordStorage.is_recording_mode(): - file_path = RECORDINGS_TEST_CONFIGS_ROOT / "node_cache.shelve" - RecordStorage.get_instance(file_path) - yield - +# ==================== Recording injection ==================== +# To inject patches in subprocesses, add new mock method in setup_recording_injection_if_enabled +# in fork mode, this is automatically enabled. +# in spawn mode, we need to decalre recording in each process separately. SpawnProcess = multiprocessing.get_context("spawn").Process @@ -213,7 +221,7 @@ def __init__(self, group=None, target=None, *args, **kwargs): @pytest.fixture -def recording_injection(mocker: MockerFixture, recording_file_override): +def recording_injection(mocker: MockerFixture): original_process_class = multiprocessing.get_context("spawn").Process multiprocessing.get_context("spawn").Process = MockSpawnProcess if "spawn" == multiprocessing.get_start_method(): @@ -222,10 +230,10 @@ def recording_injection(mocker: MockerFixture, recording_file_override): patches = setup_recording_injection_if_enabled() try: - yield (RecordStorage.is_replaying_mode() or RecordStorage.is_recording_mode(), recording_array_extend) + yield finally: - if RecordStorage.is_replaying_mode() or RecordStorage.is_recording_mode(): - RecordStorage.get_instance().delete_lock_file() + RecordStorage.get_instance().delete_lock_file() + delete_count_lock_file() recording_array_reset() multiprocessing.get_context("spawn").Process = original_process_class @@ -238,19 +246,37 @@ def recording_injection(mocker: MockerFixture, recording_file_override): def setup_recording_injection_if_enabled(): patches = [] - if RecordStorage.is_replaying_mode() or RecordStorage.is_recording_mode(): + + def start_patches(patch_targets): + for target, mock_func in patch_targets.items(): + patcher = patch(target, mock_func) + patches.append(patcher) + patcher.start() + + if is_replay() or is_record(): file_path = RECORDINGS_TEST_CONFIGS_ROOT / "node_cache.shelve" RecordStorage.get_instance(file_path) from promptflow._core.tool import tool as original_tool mocked_tool = mock_tool(original_tool) - patch_targets = ["promptflow._core.tool.tool", "promptflow._internal.tool", "promptflow.tool"] + patch_targets = { + "promptflow._core.tool.tool": mocked_tool, + "promptflow._internal.tool": mocked_tool, + "promptflow.tool": mocked_tool, + "promptflow._core.openai_injector.inject_sync": inject_sync_with_recording, + "promptflow._core.openai_injector.inject_async": inject_async_with_recording, + } + start_patches(patch_targets) - for target in patch_targets: - patcher = patch(target, mocked_tool) - patches.append(patcher) - patcher.start() + if is_live(): + patch_targets = { + "promptflow._core.openai_injector.inject_sync": inject_sync_with_recording, + "promptflow._core.openai_injector.inject_async": inject_async_with_recording, + } + start_patches(patch_targets) + + inject_openai_api() return patches diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py b/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py index b09e84697f4..ee059914bd8 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py +++ b/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py @@ -37,8 +37,6 @@ from promptflow.connections import AzureOpenAIConnection from promptflow.exceptions import UserErrorException -from ..recording_utilities import RecordStorage - PROMOTFLOW_ROOT = Path(__file__) / "../../../.." TEST_ROOT = Path(__file__).parent.parent.parent @@ -901,7 +899,6 @@ def test_error_message_dump(self, pf): assert "error" in run_dict assert run_dict["error"] == exception - @pytest.mark.skipif(RecordStorage.is_replaying_mode(), reason="System metrics not supported in replaying mode") def test_system_metrics_in_properties(self, pf) -> None: run = create_run_against_multi_line_data(pf) assert FlowRunProperties.SYSTEM_METRICS in run.properties diff --git a/src/promptflow/tests/sdk_cli_test/recording_utilities/__init__.py b/src/promptflow/tests/sdk_cli_test/recording_utilities/__init__.py index f3f21691926..55ab93f78db 100644 --- a/src/promptflow/tests/sdk_cli_test/recording_utilities/__init__.py +++ b/src/promptflow/tests/sdk_cli_test/recording_utilities/__init__.py @@ -1,14 +1,27 @@ -from .constants import ENVIRON_TEST_MODE, RecordMode -from .mock_tool import mock_tool, recording_array_extend, recording_array_reset -from .record_storage import RecordFileMissingException, RecordItemMissingException, RecordStorage +from .mock_tool import delete_count_lock_file, mock_tool, recording_array_extend, recording_array_reset +from .openai_inject_recording import inject_async_with_recording, inject_sync_with_recording +from .record_storage import ( + Counter, + RecordFileMissingException, + RecordItemMissingException, + RecordStorage, + is_live, + is_record, + is_replay, +) __all__ = [ + "Counter", "RecordStorage", - "RecordMode", - "ENVIRON_TEST_MODE", "RecordFileMissingException", "RecordItemMissingException", "mock_tool", "recording_array_extend", "recording_array_reset", + "inject_async_with_recording", + "inject_sync_with_recording", + "is_live", + "is_record", + "is_replay", + "delete_count_lock_file", ] diff --git a/src/promptflow/tests/sdk_cli_test/recording_utilities/mock_tool.py b/src/promptflow/tests/sdk_cli_test/recording_utilities/mock_tool.py index 45cd08d721c..b27a1c2f266 100644 --- a/src/promptflow/tests/sdk_cli_test/recording_utilities/mock_tool.py +++ b/src/promptflow/tests/sdk_cli_test/recording_utilities/mock_tool.py @@ -1,10 +1,22 @@ import functools import inspect +import os +from pathlib import Path from promptflow._core.tool import STREAMING_OPTION_PARAMETER_ATTR, ToolType from promptflow._core.tracer import TraceType, _create_trace_from_function_call -from .record_storage import RecordFileMissingException, RecordItemMissingException, RecordStorage +from .record_storage import ( + Counter, + RecordFileMissingException, + RecordItemMissingException, + RecordStorage, + is_live, + is_record, + is_replay, +) + +COUNT_RECORD = (Path(__file__) / "../../count.json").resolve() # recording array is a global variable to store the function names that need to be recorded recording_array = ["fetch_text_content_from_url", "my_python_tool"] @@ -46,17 +58,8 @@ def _replace_tool_rule(func): func_wo_partial = func.func else: func_wo_partial = func - if func_wo_partial.__qualname__.startswith("AzureOpenAI"): - return True - elif func_wo_partial.__qualname__.startswith("OpenAI"): - return True - elif func_wo_partial.__module__ == "promptflow.tools.aoai": - return True - elif func_wo_partial.__module__ == "promptflow.tools.openai_gpt4v": - return True - elif func_wo_partial.__module__ == "promptflow.tools.openai": - return True - elif func_wo_partial.__qualname__ in recording_array: + + if func_wo_partial.__qualname__ in recording_array: return True else: return False @@ -64,34 +67,44 @@ def _replace_tool_rule(func): def call_func(func, args, kwargs): input_dict = _prepare_input_dict(func, args, kwargs) - if RecordStorage.is_replaying_mode(): + if is_replay(): return RecordStorage.get_instance().get_record(input_dict) # Record mode will record item to record file - elif RecordStorage.is_recording_mode(): + elif is_record(): try: # prevent recording the same item twice obj = RecordStorage.get_instance().get_record(input_dict) except (RecordItemMissingException, RecordFileMissingException): # recording the item obj = RecordStorage.get_instance().set_record(input_dict, func(*args, **kwargs)) + elif is_live(): + obj = Counter.get_instance().set_file_record_count(COUNT_RECORD, func(*args, **kwargs)) return obj async def call_func_async(func, args, kwargs): input_dict = _prepare_input_dict(func, args, kwargs) - if RecordStorage.is_replaying_mode(): + if is_replay(): return RecordStorage.get_instance().get_record(input_dict) # Record mode will record item to record file - elif RecordStorage.is_recording_mode(): + elif is_record(): try: # prevent recording the same item twice obj = RecordStorage.get_instance().get_record(input_dict) except (RecordItemMissingException, RecordFileMissingException): # recording the item obj = RecordStorage.get_instance().set_record(input_dict, await func(*args, **kwargs)) + elif is_live(): + obj = Counter.get_instance().set_file_record_count(COUNT_RECORD, await func(*args, **kwargs)) return obj +def delete_count_lock_file(): + lock_file = str(COUNT_RECORD) + ".lock" + if os.path.isfile(lock_file): + os.remove(lock_file) + + def mock_tool(original_tool): """ Basically this is the original tool decorator. @@ -175,7 +188,9 @@ def decorated_tool(*args, **kwargs): # tool replacements. if func is not None: - if not _replace_tool_rule(func): + if _replace_tool_rule(func): + return tool_decorator(func) + else: return original_tool( func, *args_mock, @@ -185,7 +200,6 @@ def decorated_tool(*args, **kwargs): input_settings=input_settings, **kwargs_mock, ) - return tool_decorator(func) return original_tool( # no recording for @tool(name="func_name") func, *args_mock, diff --git a/src/promptflow/tests/sdk_cli_test/recording_utilities/openai_inject_recording.py b/src/promptflow/tests/sdk_cli_test/recording_utilities/openai_inject_recording.py new file mode 100644 index 00000000000..742096bf30c --- /dev/null +++ b/src/promptflow/tests/sdk_cli_test/recording_utilities/openai_inject_recording.py @@ -0,0 +1,38 @@ +import asyncio +import functools + +from promptflow._core.openai_injector import inject_function_async, inject_function_sync, inject_operation_headers + +from .mock_tool import call_func, call_func_async + + +def inject_recording(f): + if asyncio.iscoroutinefunction(f): + + @functools.wraps(f) + async def wrapper(*args, **kwargs): + return await call_func_async(f, args, kwargs) + + else: + + @functools.wraps(f) + def wrapper(*args, **kwargs): + return call_func(f, args, kwargs) + + return wrapper + + +def inject_async_with_recording(f): + wrapper_fun = inject_operation_headers( + (inject_function_async(args_to_ignore=["api_key", "headers", "extra_headers"])(inject_recording(f))) + ) + wrapper_fun._original = f + return wrapper_fun + + +def inject_sync_with_recording(f): + wrapper_fun = inject_operation_headers( + (inject_function_sync(args_to_ignore=["api_key", "headers", "extra_headers"])(inject_recording(f))) + ) + wrapper_fun._original = f + return wrapper_fun diff --git a/src/promptflow/tests/sdk_cli_test/recording_utilities/record_storage.py b/src/promptflow/tests/sdk_cli_test/recording_utilities/record_storage.py index 89bf47eea09..e4bc1b88658 100644 --- a/src/promptflow/tests/sdk_cli_test/recording_utilities/record_storage.py +++ b/src/promptflow/tests/sdk_cli_test/recording_utilities/record_storage.py @@ -6,14 +6,32 @@ import os import shelve from pathlib import Path -from typing import Dict +from typing import Dict, Iterator, Union from filelock import FileLock +from promptflow._core.generator_proxy import GeneratorProxy from promptflow.exceptions import PromptflowException + from .constants import ENVIRON_TEST_MODE, RecordMode +def get_test_mode_from_environ() -> str: + return os.getenv(ENVIRON_TEST_MODE, RecordMode.LIVE) + + +def is_record() -> bool: + return get_test_mode_from_environ() == RecordMode.RECORD + + +def is_replay() -> bool: + return get_test_mode_from_environ() == RecordMode.REPLAY + + +def is_live() -> bool: + return get_test_mode_from_environ() == RecordMode.LIVE + + class RecordItemMissingException(PromptflowException): """Exception raised when record item missing.""" @@ -26,157 +44,129 @@ class RecordFileMissingException(PromptflowException): pass -class RecordStorage(object): - """ - RecordStorage is used to store the record of node run. - File often stored in .promptflow/node_cache.shelve - Currently only text input/output could be recorded. - Example of cached items: - { - "/record/file/resolved": { - "hash_value": { # hash_value is sha1 of dict, accelerate the search - "input": { - "key1": "value1", # Converted to string, type info dropped - }, - "output": "output_convert_to_string", - "output_type": "output_type" # Currently support only simple strings. - } - } - } - """ - +class RecordFile: _standard_record_folder = ".promptflow" _standard_record_name = "node_cache.shelve" - _instance = None - def __init__(self, record_file: str = None): - """ - RecordStorage is used to store the record of node run. - """ - self._record_file: Path = None - self.cached_items: Dict[str, Dict[str, Dict[str, object]]] = {} - self.record_file = record_file + def __init__(self, record_file_input): + self.real_file: Path = None + self.real_file_parent: Path = None + self.record_file_str: str = None - @property - def record_file(self) -> Path: - return self._record_file + def get(self) -> Path: + return self.real_file - @record_file.setter - def record_file(self, record_file_input) -> None: + def set_and_load_file(self, record_file_input, cached_items): """ Will load record_file if exist. """ - if record_file_input == self._record_file: + record_file = Path(record_file_input).resolve() + if record_file == self.real_file: return - if isinstance(record_file_input, str): - self._record_file = Path(record_file_input).resolve() - elif isinstance(record_file_input, Path): - self._record_file = record_file_input.resolve() + self.real_file = record_file + if not self.real_file.parts[-1].endswith(RecordFile._standard_record_name): + self.real_file_parent = self.real_file / RecordFile._standard_record_folder + self.real_file = self.real_file_parent / RecordFile._standard_record_name else: - return - - if not self._record_file.parts[-1].endswith(RecordStorage._standard_record_name): - record_folder = self._record_file / RecordStorage._standard_record_folder - self._record_file = record_folder / RecordStorage._standard_record_name - else: - record_folder = self._record_file.parent - - self._record_file_str = str(self._record_file.resolve()) + self.real_file_parent = self.real_file.parent + self.record_file_str = str(self.real_file) # cache folder we could create if not exist. - if not record_folder.exists(): - record_folder.mkdir(parents=True, exist_ok=True) + if not self.real_file_parent.exists(): + self.real_file_parent.mkdir(parents=True, exist_ok=True) - # if file exist, load file - if self.exists_record_file(record_folder, self._record_file.parts[-1]): - self._load_file() - else: - self.cached_items = { - self._record_file_str: {}, - } - - def exists_record_file(self, record_folder, file_name) -> bool: - files = os.listdir(record_folder) - for file in files: - if file.startswith(file_name): - return True - return False + self.load_file(cached_items) - def _write_file(self, hashkey) -> None: - file_content = self.cached_items.get(self._record_file_str, None) - - if file_content is not None: - file_content_line = file_content.get(hashkey, None) + def write_file(self, file_records, hashkey) -> None: + if file_records is not None: + file_content_line = file_records.get(hashkey, None) if file_content_line is not None: - lock = FileLock(self.record_file.parent / "record_file.lock") + lock = FileLock(self.real_file.parent / "record_file.lock") with lock: - saved_dict = shelve.open(self._record_file_str, "c", writeback=False) + saved_dict = shelve.open(self.record_file_str, "c", writeback=False) saved_dict[hashkey] = file_content_line saved_dict.close() else: raise RecordItemMissingException(f"Record item not found in cache with hashkey {hashkey}.") else: raise RecordFileMissingException( - f"This exception should not happen here, but record file is not found {self._record_file_str}." + f"This exception should not happen here, but record file is not found {self.record_file_str}." ) - def _load_file(self) -> None: - local_content = self.cached_items.get(self._record_file_str, None) - if not local_content: - if RecordStorage.is_recording_mode(): - lock = FileLock(self.record_file.parent / "record_file.lock") - with lock: - if not self.exists_record_file(self.record_file.parent, self.record_file.parts[-1]): - return - self.cached_items[self._record_file_str] = {} - saved_dict = shelve.open(self._record_file_str, "r", writeback=False) - for key, value in saved_dict.items(): - self.cached_items[self._record_file_str][key] = value - saved_dict.close() - else: - if not self.exists_record_file(self.record_file.parent, self.record_file.parts[-1]): - return - self.cached_items[self._record_file_str] = {} - saved_dict = shelve.open(self._record_file_str, "r", writeback=False) - for key, value in saved_dict.items(): - self.cached_items[self._record_file_str][key] = value - saved_dict.close() + def recording_file_exists(self) -> bool: + files = os.listdir(self.real_file_parent) + for file in files: + if file.startswith(self.real_file.parts[-1]): + return True + return False + + def load_file(self, cached_items) -> bool: + # if file not exist, just exit and create an empty cache slot. + if not self.recording_file_exists(): + cached_items[self.record_file_str] = { + self.record_file_str: {}, + } + return False + + # Load file directly. + lock = FileLock(self.real_file.parent / "record_file.lock") + with lock: + saved_dict = shelve.open(self.record_file_str, "r", writeback=False) + cached_items[self.record_file_str] = {} + for key, value in saved_dict.items(): + cached_items[self.record_file_str][key] = value + saved_dict.close() + + return True def delete_lock_file(self): - lock_file = self.record_file.parent / "record_file.lock" + lock_file = self.real_file_parent / "record_file.lock" if lock_file.exists(): os.remove(lock_file) - def get_record(self, input_dict: Dict) -> object: - """ - Get record from local storage. - :param input_dict: input dict of critical AOAI inputs - :type input_dict: Dict - :raises RecordFileMissingException: Record file not exist - :raises RecordItemMissingException: Record item not exist in record file - :return: original output of node run - :rtype: object - """ - input_dict = self._recursive_create_hashable_args(input_dict) - hash_value: str = hashlib.sha1(str(sorted(input_dict.items())).encode("utf-8")).hexdigest() - current_saved_records: Dict[str, str] = self.cached_items.get(self._record_file_str, None) - if current_saved_records is None: - raise RecordFileMissingException(f"Record file not found {self.record_file}.") - saved_output = current_saved_records.get(hash_value, None) - if saved_output is None: - raise RecordItemMissingException( - f"Record item not found in file {self.record_file}.\n" f"values: {json.dumps(input_dict)}\n" - ) +class RecordCache: + """ + RecordCache is used to store the record of node run. + File often stored in .promptflow/node_cache.shelve + Currently only text input/output could be recorded. + Example of cached items: + { + "/record/file/resolved": { <-- file_records_pointer + "hash_value": { <-- line_record_pointer hash_value is sha1 of dict, accelerate the search + "input": { + "key1": "value1", # Converted to string, type info dropped + }, + "output": "output_convert_to_string", + "output_type": "output_type" # Currently support only simple strings. + } + } + } + """ - # not all items are reserved in the output dict. - output = saved_output["output"] - output_type = saved_output["output_type"] - if "generator" in output_type: - return self._create_output_generator(output, output_type) + def __init__(self): + self.cached_items: Dict[str, Dict[str, Dict[str, object]]] = {} + self.record_file = None + self.file_records_pointer = {} + + def get_cache(self, record_file: Union[str, Path]) -> None: + if self.record_file is None: + self.record_file = RecordFile(record_file) + self.record_file.set_and_load_file(record_file, self.cached_items) else: - return output + if self.record_file.get() == Path(record_file): + return + else: + self.record_file.set_and_load_file(Path(record_file)) + + self.cached_items = self.record_file.load_file(self.cached_items) + try: + self.file_records_pointer = self.cached_items[self.record_file.record_file_str] + except KeyError: + self.cached_items[self.record_file.record_file_str] = {} + self.file_records_pointer = self.cached_items[self.record_file.record_file_str] + self.write_back(None) def _recursive_create_hashable_args(self, item): if isinstance(item, tuple): @@ -184,13 +174,14 @@ def _recursive_create_hashable_args(self, item): if isinstance(item, list): return [self._recursive_create_hashable_args(i) for i in item] if isinstance(item, dict): - return {k: self._recursive_create_hashable_args(v) for k, v in item.items()} + return {k: self._recursive_create_hashable_args(v) for k, v in item.items() if k != "extra_headers"} elif "module: promptflow.connections" in str(item) or "object at" in str(item): return [] else: return item - def _parse_output_generator(self, output): + @staticmethod + def _parse_output_generator(output): """ Special handling for generator type. Since pickle will not work for generator. Returns the real list for reocrding, and create a generator for original output. @@ -218,7 +209,8 @@ def vgenerator(): output_type = "dict[generator]" else: output_value[k] = v - elif type(output).__name__ == "generator": + elif isinstance(output, Iterator): + output = GeneratorProxy(output) output_value = list(output) def generator(): @@ -233,7 +225,8 @@ def generator(): output_type = type(output).__name__ return output_value, output_generator, output_type - def _create_output_generator(self, output, output_type): + @staticmethod + def _create_output_generator(output, output_type): """ Special handling for generator type. Returns a generator for original output. @@ -262,6 +255,41 @@ def generator(): output_generator = generator() return output_generator + def get_record(self, input_dict: Dict) -> object: + """ + Get record from local storage. + + :param input_dict: input dict of critical AOAI inputs + :type input_dict: Dict + :raises RecordFileMissingException: Record file not exist + :raises RecordItemMissingException: Record item not exist in record file + :return: original output of node run + :rtype: object + """ + input_dict = self._recursive_create_hashable_args(input_dict) + hash_value: str = hashlib.sha1(str(sorted(input_dict.items())).encode("utf-8")).hexdigest() + + try: + line_record_pointer = self.file_records_pointer[hash_value] + except KeyError: + raise RecordItemMissingException( + f"Record item not found in file {self.record_file.record_file_str}.\n" + f"values: {json.dumps(input_dict)}\n" + ) + + # not all items are reserved in the output dict. + output = line_record_pointer["output"] + output_type = line_record_pointer["output_type"] + if "generator" in output_type: + return RecordCache._create_output_generator(output, output_type) + else: + return output + + def write_back(self, hash_value): + self.cached_items[self.record_file.record_file_str] = self.file_records_pointer + if hash_value is not None: + self.record_file.write_file(self.file_records_pointer, hash_value) + def set_record(self, input_dict: Dict, output): """ Set record to local storage, always override the old record. @@ -274,58 +302,79 @@ def set_record(self, input_dict: Dict, output): # filter args, object at will not hash input_dict = self._recursive_create_hashable_args(input_dict) hash_value: str = hashlib.sha1(str(sorted(input_dict.items())).encode("utf-8")).hexdigest() - current_saved_records: Dict[str, str] = self.cached_items.get(self._record_file_str, None) - output_value, output_generator, output_type = self._parse_output_generator(output) + output_value, output_generator, output_type = RecordCache._parse_output_generator(output) - if current_saved_records is None: - current_saved_records = {} - current_saved_records[hash_value] = { + try: + line_record_pointer = self.file_records_pointer[hash_value] + except KeyError: + self.file_records_pointer[hash_value] = { "input": input_dict, "output": output_value, "output_type": output_type, } - else: - saved_output = current_saved_records.get(hash_value, None) - if saved_output is not None: - if saved_output["output"] == output_value and saved_output["output_type"] == output_type: - if "generator" in output_type: - return output_generator - else: - return output_value - else: - current_saved_records[hash_value] = { - "input": input_dict, - "output": output_value, - "output_type": output_type, - } + line_record_pointer = self.file_records_pointer[hash_value] + self.write_back(hash_value) + + if line_record_pointer["output"] == output_value and line_record_pointer["output_type"] == output_type: + # no writeback + if "generator" in output_type: + return output_generator else: - current_saved_records[hash_value] = { - "input": input_dict, - "output": output_value, - "output_type": output_type, - } - self.cached_items[self._record_file_str] = current_saved_records - self._write_file(hash_value) - if "generator" in output_type: - return output_generator + return output_value else: - return output_value + self.file_records_pointer[hash_value] = { + "input": input_dict, + "output": output_value, + "output_type": output_type, + } - @classmethod - def get_test_mode_from_environ(cls) -> str: - return os.getenv(ENVIRON_TEST_MODE, RecordMode.LIVE) + self.write_back(hash_value) - @classmethod - def is_recording_mode(cls) -> bool: - return RecordStorage.get_test_mode_from_environ() == RecordMode.RECORD + if "generator" in output_type: + return output_generator + else: + return output_value - @classmethod - def is_replaying_mode(cls) -> bool: - return RecordStorage.get_test_mode_from_environ() == RecordMode.REPLAY - @classmethod - def is_live_mode(cls) -> bool: - return RecordStorage.get_test_mode_from_environ() == RecordMode.LIVE +class RecordStorage: + _instance = None + + def __init__(self): + self.record_cache = RecordCache() + + def set_file(self, record_file: Union[str, Path]) -> None: + """ + Will load record_file if exist. + """ + self.record_cache.get_cache(record_file) + + def get_record(self, input_dict: Dict) -> object: + """ + Get record from local storage. + + :param input_dict: input dict of critical AOAI inputs + :type input_dict: Dict + :raises RecordFileMissingException: Record file not exist + :raises RecordItemMissingException: Record item not exist in record file + :return: original output of node run + :rtype: object + """ + return self.record_cache.get_record(input_dict) + + def set_record(self, input_dict: Dict, output): + """ + Set record to local storage, always override the old record. + + :param input_dict: input dict of critical AOAI inputs + :type input_dict: OrderedDict + :param output: original output of node run + :type output: object + """ + return self.record_cache.set_record(input_dict, output) + + def delete_lock_file(self): + if self.record_cache.record_file: + self.record_cache.record_file.delete_lock_file() @classmethod def get_instance(cls, record_file=None) -> "RecordStorage": @@ -338,13 +387,65 @@ def get_instance(cls, record_file=None) -> "RecordStorage": :rtype: RecordStorage """ # if not in recording mode, return None - if not (RecordStorage.is_recording_mode() or RecordStorage.is_replaying_mode()): + if not (is_record() or is_replay() or is_live()): return None # Create instance if not exist + if is_record() or is_replay(): + if cls._instance is None: + if record_file is None: + raise RecordFileMissingException("record_file is value None") + cls._instance = RecordStorage() + if record_file is not None: + cls._instance.set_file(record_file) + else: + if cls._instance is None: + cls._instance = RecordStorage() # live mode return an empty record storage + return cls._instance + + +class Counter: + _instance = None + + def __init__(self): + self.file = None + + def is_non_zero_file(self, fpath): + return os.path.isfile(fpath) and os.path.getsize(fpath) > 0 + + def set_file_record_count(self, file, obj): + """ + Just count how many tokens are calculated. Different from + openai_metric_calculator, this is directly returned from AOAI. + """ + output_value, output_generator, output_type = RecordCache._parse_output_generator(obj) + if "generator" in output_type: + count = len(output_value) + elif hasattr(output_value, "usage") and output_value.usage and output_value.usage.total_tokens: + count = output_value.usage.total_tokens + else: + # This is error. Suppress it. + count = 0 + + self.file = file + with FileLock(str(file) + ".lock"): + is_non_zero_file = self.is_non_zero_file(file) + if is_non_zero_file: + with open(file, "r", encoding="utf-8") as f: + number = json.load(f) + number["count"] += count + else: + number = {"count": count} + with open(file, "w", encoding="utf-8") as f: + number_str = json.dumps(number, ensure_ascii=False) + f.write(number_str) + + if "generator" in output_type: + return output_generator + else: + return output_value + + @classmethod + def get_instance(cls) -> "Counter": if cls._instance is None: - if record_file is None: - raise RecordFileMissingException("record_file is value None") - cls._instance = RecordStorage(record_file) - if record_file is not None: - cls._instance.record_file = record_file + cls._instance = Counter() return cls._instance diff --git a/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.bak b/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.bak index e7a8bd92682..50488934387 100644 --- a/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.bak +++ b/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.bak @@ -1,89 +1,55 @@ 'aadb0707e9a62b00df9d0d3fecb709ece90a8b67', (0, 2261) -'28e341291f602cdf951239f0152fa9e26deb501b', (67584, 3359) -'e25f352dc22315a0e2c3ee2a3cd629763fc0ae5e', (9216, 3302) -'dc8625906abf86f47f2c52b461f108ab6b98a1cf', (12800, 3795) -'a30838607856c1b8efd522af93ea1edb88f57b4f', (16896, 3671) -'4f7cfc5331a898e9c7abb00e2d0f6f818dd2c999', (20992, 3795) -'06c8709d838389faebb00cc579c7bd1e396f5ab7', (25088, 3886) -'89ef4101e31a12b70fa982b83be01a65ee3c7410', (29184, 3365) -'0304f9ccf7ab8521173b43b526b26412208148b1', (32768, 509) -'914c0cdebdbf3b03a21afe84f9a1af9f99729771', (33280, 1080) -'3ccba72230e4213b2ae31d1df71e45377f5b9fbf', (34816, 3081) -'12dcd3566273156d0071d532410670ca6a13faa8', (38400, 3028) -'95aad7f462a4948f82104b8c6320ced6eeb3a59b', (41472, 3293) -'2c8bece890f55462916146bb89d0df92a4210163', (45056, 3316) -'d1f105fd5701fa3ae65608e6ef732eb2c095d88d', (48640, 3371) -'64b155dfa377148a12addcaabd196edd29620ede', (52224, 3258) -'9a7219e5f8a5e525df2fae3a4df82eb6c6dd7837', (59392, 3870) -'bc6a4f4ae64fa01917085f3e608f0d5c7699308c', (63488, 3610) -'ed29e249e3b86abb1907bd98dc7d5368cc11bfe5', (71168, 3348) -'977d4e91843e6a025d88dc10f5935f3fe50a2aa8', (74752, 3632) -'ac3d32b87a5b6271641f0fdb77dd785d07930207', (78848, 3403) -'60924e7e69449f453fea01c32a84b74521f3bb56', (82432, 3080) -'d52f8bf8badc3432c05395d9f647efa2fede3121', (86016, 1938) -'15b696309e5d753b6123f5521a9c8e4daded18e3', (88064, 3895) -'32edf2341350a6fca9670b7ec6edc208905ec8c1', (92160, 3676) -'cc9ac3eaa4f332a204f39f050c8f83ceba89113d', (96256, 863) -'2c2c8e9b4662215a00c7119e987a6b0049829e2b', (97280, 503) -'9893bb77a07b7fa08f4ab85dd16368863040bc53', (97792, 1966) -'b11590a307351d8ade7df49346a826cfde9c4444', (99840, 3923) -'c8fcd047770466d76018d8b9656c18b7a87a9dcf', (103936, 2255) -'b490dc00a174b4b94e411b32b3b6df8e7291ff44', (106496, 493) -'427f9902c01baef6ef8020ae5e5cb5163c6d36ac', (107008, 307) -'45ea53cf0c4770788d6568a177baf123146dc20b', (107520, 924) -'ebf9f4aad38dceb848df0614195e40870dd5fe6e', (108544, 1043) -'6d6d40bcf3cf2e43d640c75cba40d607f1869b9f', (110080, 1315) -'abb3142dc69818c0c7131c4642f2870ae59bd05a', (111616, 1025) -'8cf128af83ea2aaf4a09890690ab16f3bb347bb3', (113152, 309) -'343332ff6b96d3b1baac23c4e8394a6a965f84b1', (113664, 327) -'d7079ca5b962a7dbca1cfe90f847775652acf6b9', (114176, 961) -'b9bcb73bbe85960e4f492c6b60fd2584de421f91', (115200, 312) -'71d8e363eeac4e3334679f505225acd75ef5607b', (115712, 414) -'9e15e58e4a0b5ff182c0f70761c65fba081c2c2f', (132096, 3373) -'1696e4d8ebc97bfec511fca7754ef717692faa70', (119808, 3623) -'bd36e2e27da8bc212f2213feb0154b6902e1b549', (123904, 3699) -'91f0cff189e34431153edba55e684629b6b56487', (128000, 3808) -'5bd1695cf0739697851e8ca9bd54c0305c7422c4', (135680, 3293) -'33c98924b97b4eed655857dca1472bdfcf36b86a', (139264, 3446) -'3dc18b56b1741937eefaa1069119cdec984e49b7', (142848, 3695) -'255fbbcb72580f8c5eeed07419ed9d9fdd809433', (146944, 3113) -'879388db36d150bfd7f948a8a14011ee96ee7e51', (150528, 3063) -'cf7bb24c6526295ab8e9ba4eea799d98b13dcce5', (153600, 3738) -'6b05f39821053fcbc341404fa47bd2a029862637', (157696, 3725) -'a5abc1d206c9456c8230e382e04c2903756f9be2', (161792, 3690) -'6c9c0aa6ecb54dcdb79b2e4dc0c09321482dde41', (165888, 400) -'70f4fea54805e642e98208d6704425432e00d46d', (166400, 2975) -'7c7b48d39ea68dcc368aa9ad6b0da9c30639c716', (169472, 4085) -'f80e94d98478b637304d5af02356da38b45638da', (173568, 3542) -'ce3be051617aa3d9baaabefbde10655a382725ef', (177152, 579) -'b9f991e14c162d02211acdd3737f60afbf117107', (178176, 517) -'75de0fcefdd6fd5f81c2ccbb80b5e40d81c9c214', (179200, 397) -'fb43cea41f39ffa5ef98f98499c292af14cb92cd', (179712, 1305) -'1af44fd75c83703dc9d960a0874fda3a2fa0e389', (181248, 502) -'ebe34b3760a5502304bb23747079e50d8515141b', (181760, 418) -'83f99b7f16fc8a4548782215c97d500eed44159b', (182272, 997) -'235b8b0ff5ebbf46ab52403b59346dd08717c297', (183296, 3205) -'6dbf6fe28cfe055d3730b847cd603ceb498cc046', (186880, 3055) -'1bc4561b56cd2944414cd994c41dc370630c814c', (189952, 3617) -'29301e2df6aef321aaed0c37f5acf59efdcf0ce5', (194048, 533) -'e68e72e33a357bca30a07bd6d7157209a7ed2f46', (195072, 3385) -'a69786b6c83c69110a94fa92571d56716831b89b', (198656, 1010) -'9113b6564602c0961af986fcfd51af95e7aa0d30', (199680, 2973) -'34f82ce2482f5039176bfad68fea3b660cce46a2', (202752, 621) -'4104c4ae613928612ddf45359338643833764a9b', (203776, 2960) -'773e7701fa9b21d32135f2e0a2f80d9f3bf60ff0', (206848, 860) -'e17f414155f0f81443c79d3af49e3295fe38a3bd', (207872, 1215) -'9dc92c6363cf76df823e9cc93dc3e6f961f3dfae', (209408, 3443) -'24973771dad2d9beb6306802d11b103627a0c86f', (212992, 4463) -'4a55d6c6e33565c8c2976ffffed0544c8f01f11f', (217600, 4079) -'c3f859fc47cf393cc3af2e7854c73143039219f9', (221696, 3541) -'1c304bd9336e59ee7fe03ec0aa3ff1316accbd42', (225280, 3182) -'79b019d7c272dbdfc8264e8e42b2c88d7aa7c951', (228864, 2189) -'ead9751f11bc2db068f915f8cd6a798e5d417ee1', (231424, 2224) -'90e7b637f88d5ff8a781be0ca4c1682885c17e4a', (233984, 444) -'b08cfc768cc8c1c5165a02d633b7c5f3cdfdfb60', (234496, 958) -'2182939be9737c58873eafa07078daccddac9b6a', (235520, 3237) -'063189b9af2fa70eb9ecea7fd40adb28d9ad63a9', (239104, 3331) -'45c057f3f3dbc663bc9d7661254c3df17328111c', (242688, 3248) -'3a9191ae0dde33b5d4d9bc89da4b87436400f505', (246272, 3790) -'bb71bde6522266d699c50af9825a6b4f0f025d56', (250368, 3858) +'250dae402ab4773d7d0e93523d0842ffb728bb76', (2560, 4416) +'804651115fc158ef14fc7b368f04af28627a8eb2', (7168, 4763) +'fcb73c5c118b90f3c3aa5dade1d62e428b526c2f', (12288, 4796) +'dfa82d636b76bae3c722f229811c750ec968e4fc', (17408, 4805) +'0304f9ccf7ab8521173b43b526b26412208148b1', (22528, 509) +'13edf79d4a9d7721c1a433e264646bb8157f7acb', (23040, 2676) +'0afa56b5f1f0d67b47453c786a75e6786a889f6b', (26112, 4730) +'3e648b0074cbd8087bd8ce6923277248cc407b9b', (31232, 4030) +'ea4ec7e8bc8a2ce1320d1b4de5c95ecc0a15a0f1', (35328, 4365) +'785ee69209cd7493070cdae61a12153caffecb4c', (39936, 4446) +'2afe389e572598c73ea066dac9eee9c8feb59b70', (44544, 4344) +'2c2c8e9b4662215a00c7119e987a6b0049829e2b', (49152, 503) +'c8fcd047770466d76018d8b9656c18b7a87a9dcf', (49664, 2255) +'427f9902c01baef6ef8020ae5e5cb5163c6d36ac', (52224, 307) +'8cf128af83ea2aaf4a09890690ab16f3bb347bb3', (52736, 309) +'343332ff6b96d3b1baac23c4e8394a6a965f84b1', (53248, 327) +'79b019d7c272dbdfc8264e8e42b2c88d7aa7c951', (53760, 2189) +'ead9751f11bc2db068f915f8cd6a798e5d417ee1', (56320, 2224) +'90e7b637f88d5ff8a781be0ca4c1682885c17e4a', (58880, 522) +'65e26b3f7cf05359f8ec9c91cb8870b88599f705', (59904, 2266) +'878a21c2b8cdce70b977985c65e509785ed21d8f', (62464, 4476) +'bd26b74d525b583310f202a49e90560b2cff4b7e', (67072, 4307) +'7856ccd5b753adace4433bb4d723a68a9a9d2386', (71680, 4577) +'0216aba8ee93f7eb5cf4f6166aead964ef45adcb', (76288, 4856) +'651a9cf5e615a05be61651f898a1720331b24401', (81408, 4931) +'b9bcb73bbe85960e4f492c6b60fd2584de421f91', (86528, 312) +'71d8e363eeac4e3334679f505225acd75ef5607b', (87040, 414) +'271fb28102d19c3711649a3940b5950f87778b3f', (87552, 2316) +'68c1601c9e4c4a903b9b8807acd38d07efcc34e1', (90112, 4345) +'e4f7a047269c444e33a89f094307767b97475ccc', (94720, 4402) +'1ca7f3fe1fa87e35295a067a90c6b65c227bb728', (99328, 4470) +'2e2fde67c0abbd3879d3af11bec27fbfe4eeb0c1', (103936, 4482) +'77d917aa87a51b5db4e0b18e2f6cdd63929bbbfb', (108544, 4547) +'34f82ce2482f5039176bfad68fea3b660cce46a2', (113152, 510) +'70f4fea54805e642e98208d6704425432e00d46d', (113664, 3043) +'29301e2df6aef321aaed0c37f5acf59efdcf0ce5', (116736, 701) +'773e7701fa9b21d32135f2e0a2f80d9f3bf60ff0', (117760, 625) +'52e44b720c1cfab9302b846602da506421051424', (118784, 2233) +'4a0cfdb553b2ee378895d3e965e140399a6ef987', (121344, 2317) +'8192ca48497b86637d5e390f673b5c8fe81225f8', (123904, 4100) +'4f99c40211ee6e8dd397de3f8990a422d73b93d7', (128512, 4572) +'04806cfed32769eb327980e8bae4056412998e18', (133120, 1791) +'6f93b55c1bac8a82af52abee9204c95449af59a6', (135168, 2239) +'7baa1166be561e6e95e77fc5748872aadf1d0b6f', (137728, 2334) +'759bf0e56e895625be556ba8a019e5c82694ca7a', (140288, 55273) +'bab30b5ed90163fd54aaa1106aa2eb7b40613db3', (195584, 7703) +'283a8ecfcc2f452a2d8fd1ee685ffe768c5364e2', (203776, 2226) +'be843d140ddf3003e24ad6e03a7c1143d4aa767d', (206336, 5247) +'7567bc7affb56e109ad05ade70f7427a9f60bcd5', (211968, 4706) +'5037a3400fa0a7e1884eaa6214cd2c6430421fbb', (217088, 5922) +'cac159149a12ba86c9ca57cfcf68d6581b905413', (223232, 1816) +'6719d0d305bcd40f23e010bfca970b4d78ed2bfa', (225280, 63451) +'11d10f208f677d1f33951a2b3f3478fa224d0594', (288768, 5394) +'61569b3ba6f982274a44f6704facf205a223afc1', (294400, 4099) diff --git a/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.dat b/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.dat index e448ece0697..f30fe3ade69 100644 Binary files a/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.dat and b/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.dat differ diff --git a/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.dir b/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.dir index e7a8bd92682..50488934387 100644 --- a/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.dir +++ b/src/promptflow/tests/test_configs/node_recordings/node_cache.shelve.dir @@ -1,89 +1,55 @@ 'aadb0707e9a62b00df9d0d3fecb709ece90a8b67', (0, 2261) -'28e341291f602cdf951239f0152fa9e26deb501b', (67584, 3359) -'e25f352dc22315a0e2c3ee2a3cd629763fc0ae5e', (9216, 3302) -'dc8625906abf86f47f2c52b461f108ab6b98a1cf', (12800, 3795) -'a30838607856c1b8efd522af93ea1edb88f57b4f', (16896, 3671) -'4f7cfc5331a898e9c7abb00e2d0f6f818dd2c999', (20992, 3795) -'06c8709d838389faebb00cc579c7bd1e396f5ab7', (25088, 3886) -'89ef4101e31a12b70fa982b83be01a65ee3c7410', (29184, 3365) -'0304f9ccf7ab8521173b43b526b26412208148b1', (32768, 509) -'914c0cdebdbf3b03a21afe84f9a1af9f99729771', (33280, 1080) -'3ccba72230e4213b2ae31d1df71e45377f5b9fbf', (34816, 3081) -'12dcd3566273156d0071d532410670ca6a13faa8', (38400, 3028) -'95aad7f462a4948f82104b8c6320ced6eeb3a59b', (41472, 3293) -'2c8bece890f55462916146bb89d0df92a4210163', (45056, 3316) -'d1f105fd5701fa3ae65608e6ef732eb2c095d88d', (48640, 3371) -'64b155dfa377148a12addcaabd196edd29620ede', (52224, 3258) -'9a7219e5f8a5e525df2fae3a4df82eb6c6dd7837', (59392, 3870) -'bc6a4f4ae64fa01917085f3e608f0d5c7699308c', (63488, 3610) -'ed29e249e3b86abb1907bd98dc7d5368cc11bfe5', (71168, 3348) -'977d4e91843e6a025d88dc10f5935f3fe50a2aa8', (74752, 3632) -'ac3d32b87a5b6271641f0fdb77dd785d07930207', (78848, 3403) -'60924e7e69449f453fea01c32a84b74521f3bb56', (82432, 3080) -'d52f8bf8badc3432c05395d9f647efa2fede3121', (86016, 1938) -'15b696309e5d753b6123f5521a9c8e4daded18e3', (88064, 3895) -'32edf2341350a6fca9670b7ec6edc208905ec8c1', (92160, 3676) -'cc9ac3eaa4f332a204f39f050c8f83ceba89113d', (96256, 863) -'2c2c8e9b4662215a00c7119e987a6b0049829e2b', (97280, 503) -'9893bb77a07b7fa08f4ab85dd16368863040bc53', (97792, 1966) -'b11590a307351d8ade7df49346a826cfde9c4444', (99840, 3923) -'c8fcd047770466d76018d8b9656c18b7a87a9dcf', (103936, 2255) -'b490dc00a174b4b94e411b32b3b6df8e7291ff44', (106496, 493) -'427f9902c01baef6ef8020ae5e5cb5163c6d36ac', (107008, 307) -'45ea53cf0c4770788d6568a177baf123146dc20b', (107520, 924) -'ebf9f4aad38dceb848df0614195e40870dd5fe6e', (108544, 1043) -'6d6d40bcf3cf2e43d640c75cba40d607f1869b9f', (110080, 1315) -'abb3142dc69818c0c7131c4642f2870ae59bd05a', (111616, 1025) -'8cf128af83ea2aaf4a09890690ab16f3bb347bb3', (113152, 309) -'343332ff6b96d3b1baac23c4e8394a6a965f84b1', (113664, 327) -'d7079ca5b962a7dbca1cfe90f847775652acf6b9', (114176, 961) -'b9bcb73bbe85960e4f492c6b60fd2584de421f91', (115200, 312) -'71d8e363eeac4e3334679f505225acd75ef5607b', (115712, 414) -'9e15e58e4a0b5ff182c0f70761c65fba081c2c2f', (132096, 3373) -'1696e4d8ebc97bfec511fca7754ef717692faa70', (119808, 3623) -'bd36e2e27da8bc212f2213feb0154b6902e1b549', (123904, 3699) -'91f0cff189e34431153edba55e684629b6b56487', (128000, 3808) -'5bd1695cf0739697851e8ca9bd54c0305c7422c4', (135680, 3293) -'33c98924b97b4eed655857dca1472bdfcf36b86a', (139264, 3446) -'3dc18b56b1741937eefaa1069119cdec984e49b7', (142848, 3695) -'255fbbcb72580f8c5eeed07419ed9d9fdd809433', (146944, 3113) -'879388db36d150bfd7f948a8a14011ee96ee7e51', (150528, 3063) -'cf7bb24c6526295ab8e9ba4eea799d98b13dcce5', (153600, 3738) -'6b05f39821053fcbc341404fa47bd2a029862637', (157696, 3725) -'a5abc1d206c9456c8230e382e04c2903756f9be2', (161792, 3690) -'6c9c0aa6ecb54dcdb79b2e4dc0c09321482dde41', (165888, 400) -'70f4fea54805e642e98208d6704425432e00d46d', (166400, 2975) -'7c7b48d39ea68dcc368aa9ad6b0da9c30639c716', (169472, 4085) -'f80e94d98478b637304d5af02356da38b45638da', (173568, 3542) -'ce3be051617aa3d9baaabefbde10655a382725ef', (177152, 579) -'b9f991e14c162d02211acdd3737f60afbf117107', (178176, 517) -'75de0fcefdd6fd5f81c2ccbb80b5e40d81c9c214', (179200, 397) -'fb43cea41f39ffa5ef98f98499c292af14cb92cd', (179712, 1305) -'1af44fd75c83703dc9d960a0874fda3a2fa0e389', (181248, 502) -'ebe34b3760a5502304bb23747079e50d8515141b', (181760, 418) -'83f99b7f16fc8a4548782215c97d500eed44159b', (182272, 997) -'235b8b0ff5ebbf46ab52403b59346dd08717c297', (183296, 3205) -'6dbf6fe28cfe055d3730b847cd603ceb498cc046', (186880, 3055) -'1bc4561b56cd2944414cd994c41dc370630c814c', (189952, 3617) -'29301e2df6aef321aaed0c37f5acf59efdcf0ce5', (194048, 533) -'e68e72e33a357bca30a07bd6d7157209a7ed2f46', (195072, 3385) -'a69786b6c83c69110a94fa92571d56716831b89b', (198656, 1010) -'9113b6564602c0961af986fcfd51af95e7aa0d30', (199680, 2973) -'34f82ce2482f5039176bfad68fea3b660cce46a2', (202752, 621) -'4104c4ae613928612ddf45359338643833764a9b', (203776, 2960) -'773e7701fa9b21d32135f2e0a2f80d9f3bf60ff0', (206848, 860) -'e17f414155f0f81443c79d3af49e3295fe38a3bd', (207872, 1215) -'9dc92c6363cf76df823e9cc93dc3e6f961f3dfae', (209408, 3443) -'24973771dad2d9beb6306802d11b103627a0c86f', (212992, 4463) -'4a55d6c6e33565c8c2976ffffed0544c8f01f11f', (217600, 4079) -'c3f859fc47cf393cc3af2e7854c73143039219f9', (221696, 3541) -'1c304bd9336e59ee7fe03ec0aa3ff1316accbd42', (225280, 3182) -'79b019d7c272dbdfc8264e8e42b2c88d7aa7c951', (228864, 2189) -'ead9751f11bc2db068f915f8cd6a798e5d417ee1', (231424, 2224) -'90e7b637f88d5ff8a781be0ca4c1682885c17e4a', (233984, 444) -'b08cfc768cc8c1c5165a02d633b7c5f3cdfdfb60', (234496, 958) -'2182939be9737c58873eafa07078daccddac9b6a', (235520, 3237) -'063189b9af2fa70eb9ecea7fd40adb28d9ad63a9', (239104, 3331) -'45c057f3f3dbc663bc9d7661254c3df17328111c', (242688, 3248) -'3a9191ae0dde33b5d4d9bc89da4b87436400f505', (246272, 3790) -'bb71bde6522266d699c50af9825a6b4f0f025d56', (250368, 3858) +'250dae402ab4773d7d0e93523d0842ffb728bb76', (2560, 4416) +'804651115fc158ef14fc7b368f04af28627a8eb2', (7168, 4763) +'fcb73c5c118b90f3c3aa5dade1d62e428b526c2f', (12288, 4796) +'dfa82d636b76bae3c722f229811c750ec968e4fc', (17408, 4805) +'0304f9ccf7ab8521173b43b526b26412208148b1', (22528, 509) +'13edf79d4a9d7721c1a433e264646bb8157f7acb', (23040, 2676) +'0afa56b5f1f0d67b47453c786a75e6786a889f6b', (26112, 4730) +'3e648b0074cbd8087bd8ce6923277248cc407b9b', (31232, 4030) +'ea4ec7e8bc8a2ce1320d1b4de5c95ecc0a15a0f1', (35328, 4365) +'785ee69209cd7493070cdae61a12153caffecb4c', (39936, 4446) +'2afe389e572598c73ea066dac9eee9c8feb59b70', (44544, 4344) +'2c2c8e9b4662215a00c7119e987a6b0049829e2b', (49152, 503) +'c8fcd047770466d76018d8b9656c18b7a87a9dcf', (49664, 2255) +'427f9902c01baef6ef8020ae5e5cb5163c6d36ac', (52224, 307) +'8cf128af83ea2aaf4a09890690ab16f3bb347bb3', (52736, 309) +'343332ff6b96d3b1baac23c4e8394a6a965f84b1', (53248, 327) +'79b019d7c272dbdfc8264e8e42b2c88d7aa7c951', (53760, 2189) +'ead9751f11bc2db068f915f8cd6a798e5d417ee1', (56320, 2224) +'90e7b637f88d5ff8a781be0ca4c1682885c17e4a', (58880, 522) +'65e26b3f7cf05359f8ec9c91cb8870b88599f705', (59904, 2266) +'878a21c2b8cdce70b977985c65e509785ed21d8f', (62464, 4476) +'bd26b74d525b583310f202a49e90560b2cff4b7e', (67072, 4307) +'7856ccd5b753adace4433bb4d723a68a9a9d2386', (71680, 4577) +'0216aba8ee93f7eb5cf4f6166aead964ef45adcb', (76288, 4856) +'651a9cf5e615a05be61651f898a1720331b24401', (81408, 4931) +'b9bcb73bbe85960e4f492c6b60fd2584de421f91', (86528, 312) +'71d8e363eeac4e3334679f505225acd75ef5607b', (87040, 414) +'271fb28102d19c3711649a3940b5950f87778b3f', (87552, 2316) +'68c1601c9e4c4a903b9b8807acd38d07efcc34e1', (90112, 4345) +'e4f7a047269c444e33a89f094307767b97475ccc', (94720, 4402) +'1ca7f3fe1fa87e35295a067a90c6b65c227bb728', (99328, 4470) +'2e2fde67c0abbd3879d3af11bec27fbfe4eeb0c1', (103936, 4482) +'77d917aa87a51b5db4e0b18e2f6cdd63929bbbfb', (108544, 4547) +'34f82ce2482f5039176bfad68fea3b660cce46a2', (113152, 510) +'70f4fea54805e642e98208d6704425432e00d46d', (113664, 3043) +'29301e2df6aef321aaed0c37f5acf59efdcf0ce5', (116736, 701) +'773e7701fa9b21d32135f2e0a2f80d9f3bf60ff0', (117760, 625) +'52e44b720c1cfab9302b846602da506421051424', (118784, 2233) +'4a0cfdb553b2ee378895d3e965e140399a6ef987', (121344, 2317) +'8192ca48497b86637d5e390f673b5c8fe81225f8', (123904, 4100) +'4f99c40211ee6e8dd397de3f8990a422d73b93d7', (128512, 4572) +'04806cfed32769eb327980e8bae4056412998e18', (133120, 1791) +'6f93b55c1bac8a82af52abee9204c95449af59a6', (135168, 2239) +'7baa1166be561e6e95e77fc5748872aadf1d0b6f', (137728, 2334) +'759bf0e56e895625be556ba8a019e5c82694ca7a', (140288, 55273) +'bab30b5ed90163fd54aaa1106aa2eb7b40613db3', (195584, 7703) +'283a8ecfcc2f452a2d8fd1ee685ffe768c5364e2', (203776, 2226) +'be843d140ddf3003e24ad6e03a7c1143d4aa767d', (206336, 5247) +'7567bc7affb56e109ad05ade70f7427a9f60bcd5', (211968, 4706) +'5037a3400fa0a7e1884eaa6214cd2c6430421fbb', (217088, 5922) +'cac159149a12ba86c9ca57cfcf68d6581b905413', (223232, 1816) +'6719d0d305bcd40f23e010bfca970b4d78ed2bfa', (225280, 63451) +'11d10f208f677d1f33951a2b3f3478fa224d0594', (288768, 5394) +'61569b3ba6f982274a44f6704facf205a223afc1', (294400, 4099)