From 01d0005ce6e492ccfe71942cda8d25ebf20c1767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Chor=C4=85=C5=BCewicz?= Date: Thu, 14 Apr 2022 20:08:53 +0200 Subject: [PATCH] pmreorder: share markers from an application through env variable Fixes: #5419 --- doc/pmreorder/pmreorder.1.md | 29 ++++++++++++++++++--- src/tools/pmreorder/opscontext.py | 17 +++++++++--- src/tools/pmreorder/pmreorder.py | 3 ++- src/tools/pmreorder/statemachine.py | 40 +++++++++++++---------------- 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/doc/pmreorder/pmreorder.1.md b/doc/pmreorder/pmreorder.1.md index 4d234ec4aa4..94ba35d2002 100644 --- a/doc/pmreorder/pmreorder.1.md +++ b/doc/pmreorder/pmreorder.1.md @@ -9,7 +9,7 @@ header: "pmreorder version 1.5" --- [comment]: <> (SPDX-License-Identifier: BSD-3-Clause) -[comment]: <> (Copyright 2018-2021, Intel Corporation) +[comment]: <> (Copyright 2018-2022, Intel Corporation) [comment]: <> (pmreorder.1 -- man page for pmreorder) @@ -43,9 +43,25 @@ a persistent memory checking tool. Pmreorder performs the store reordering between persistent memory barriers - a sequence of flush-fence operations. -It uses a consistency checking routine provided in the -command line options to check whether files are in a -consistent state. +It uses a consistency checking routine provided +in the command line options to check whether files are in a consistent state. +To help with this task, pmreorder sets an environmental variable +PMREORDER_MARKERS. Said variable contains a subset of markers +passed from the application, which is visible at the current state +of pmreordering. Individual markers are separated by vertical bar (‘|’). + +Example of how PMREORDER_MARKERS variable can be parsed in c++ using regex +based environment variable tokener: + +``` + auto env = std::getenv("ENV_PASS"); + std::string senv(env ? env : ""); + std::regex rgx("(\\w+)(\\||$)"); + for (std::sregex_iterator it(senv.begin(), senv.end(), rgx), it_end; + it != it_end; ++it) { + std::cout << (*it)[1].str() << std::endl; + } +``` Considering that logging, replaying and reordering of operations are very time consuming, it is recommended to use as few stores as @@ -381,6 +397,11 @@ To enable API macros logging set environment variable: + **PMREORDER_EMIT_LOG**=1 +User defined markers passed from an application +are stored in the variable: + ++ **PMREORDER_MARKERS** + # EXAMPLE # ``` diff --git a/src/tools/pmreorder/opscontext.py b/src/tools/pmreorder/opscontext.py index e360d7dde98..44bb94f3a6e 100644 --- a/src/tools/pmreorder/opscontext.py +++ b/src/tools/pmreorder/opscontext.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation from operationfactory import OperationFactory from binaryoutputhandler import BinaryOutputHandler @@ -56,15 +56,24 @@ def extract_operations(self): :return: list of subclasses of :class:`memoryoperations.BaseOperation` """ + enumerated_ops = list(enumerate(self._operations)) + markers = list( + filter( + lambda e: e[1].endswith(".BEGIN") or e[1].endswith(".END"), + enumerated_ops, + ) + ) + operation_ids = list(enumerated_ops) + stop_index = start_index = 0 - for i, elem in enumerate(self._operations): + for i, elem in enumerated_ops: if "START" in elem: start_index = i elif "STOP" in elem: stop_index = i - return list( + operations = list( map( OperationFactory.create_operation, self._operations[start_index + 1: stop_index], @@ -72,3 +81,5 @@ def extract_operations(self): repeat(self.stack_engines), ) ) + + return (operations, operation_ids, markers) diff --git a/src/tools/pmreorder/pmreorder.py b/src/tools/pmreorder/pmreorder.py index 2b8777fb3e5..76a71425262 100644 --- a/src/tools/pmreorder/pmreorder.py +++ b/src/tools/pmreorder/pmreorder.py @@ -103,7 +103,8 @@ def main(): # init and run the state machine a = statemachine.StateMachine(statemachine.InitState(context)) - if a.run_all(context.extract_operations()) is False: + operations, operation_ids, markers = context.extract_operations() + if a.run_all(operations, operation_ids, markers) is False: sys.exit(1) diff --git a/src/tools/pmreorder/statemachine.py b/src/tools/pmreorder/statemachine.py index fbe7f45fa7f..8f19701581c 100644 --- a/src/tools/pmreorder/statemachine.py +++ b/src/tools/pmreorder/statemachine.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2018-2021, Intel Corporation +# Copyright 2018-2022, Intel Corporation import memoryoperations as memops import reorderengines from reorderexceptions import InconsistentFileException from reorderexceptions import NotSupportedOperationException +import os class State: @@ -158,7 +159,6 @@ def run(self, in_op): def substitute_reorder(self, order_ops): """ Changes the reordering engine based on the log marker class. - :param order_ops: The reordering marker class. :type order_ops: subclass of :class:`memoryoperations.ReorderBase` :return: None @@ -232,9 +232,8 @@ def reg_file(self, file_op): :type file_op: memoryoperations.Register_file :return: None """ - self._context.file_handler.add_file( - file_op.name, file_op.address, file_op.size - ) + self._context.file_handler.add_file(file_op.name, + file_op.address, file_op.size) def move_inner_state(self, in_op): """ @@ -249,10 +248,8 @@ def move_inner_state(self, in_op): """ if isinstance(in_op, memops.Store) and self._inner_state == "init": self._inner_state = "dirty" - elif ( - isinstance(in_op, memops.FlushBase) - and self._inner_state == "dirty" - ): + elif isinstance(in_op, memops.FlushBase) and \ + self._inner_state == "dirty": self._inner_state = "flush" elif isinstance(in_op, memops.Fence) and self._inner_state == "flush": self._inner_state = "fence" @@ -308,23 +305,18 @@ def run(self, in_op): flushed_stores = list(filter(lambda x: x.flushed, self._ops_list)) # not-flushed stores should be passed to next state - State.trans_stores = list( - filter(lambda x: x.flushed is False, self._ops_list) - ) + State.trans_stores = list(filter(lambda x: x.flushed is False, + self._ops_list)) if self._context.test_on_barrier: - self._context.logger.debug( - "Current reorder engine: {}".format( - self._context.reorder_engine - ) - ) + self._context.logger.debug("Current reorder engine: {}" + .format(self._context.reorder_engine)) for i, seq in enumerate( self._context.reorder_engine.generate_sequence(flushed_stores) ): self._context.logger.debug( - "NEXT Sequence (no. {0}) with length: {1}".format( - i, len(seq) - ) + "NEXT Sequence (no. {0}) with length: \ + {1}".format(i, len(seq)) ) for j, op in enumerate(seq): self._context.logger.debug( @@ -371,7 +363,7 @@ def __init__(self, init_state): """ self._curr_state = init_state - def run_all(self, operations): + def run_all(self, operations, operation_ids, markers): """ Starts the state machine. @@ -381,8 +373,12 @@ def run_all(self, operations): :return: None """ all_consistent = True - for ops in operations: + for ops, ops_id in zip(operations, operation_ids): self._curr_state._context.logger.info(ops) + markers_to_pass = (m[1] for m in markers if m[0] < ops_id[0]) + markers_to_pass = "|".join(markers_to_pass) + self._curr_state._context.logger.info(markers_to_pass) + os.environ["PMREORDER_MARKERS"] = markers_to_pass self._curr_state = self._curr_state.next(ops) check = self._curr_state.run(ops) if check is False: