Rebased version of #1698 (#2902)

Added v1model operations of resubmit_preserving_field_list (and also recirculate and clone).

backends/bmv2/CMakeLists.txt

@@ -158,19 +158,6 @@ set (XFAIL_TESTS
   testdata/p4_14_samples/issue60.p4
   # compiler claims (incorrectly?) that c2 has mulitple successors, so is not supported
   testdata/p4_14_samples/issue-1426.p4
-  # As of 2019-Feb-04 latest p4c code, this program fails due to the
-  # root cause of both issues #1694 and #1669.  I have tested it with
-  # the proposed fix for issue #1694 that is on PR #1704, and while
-  # that does make the produced BMv2 JSON file able to cause packets
-  # to be recirculated, resubmitted, etc. it still has the bug of not
-  # preserving the metadata specified in the program's field_lists, so
-  # the STF tests fail because of issue #1669.  As of this writing,
-  # these programs rely for
-  # successful execution on the preservation of metadata across
-  # resubmit, recirculate, and/or clone operations.
-  testdata/p4_14_samples/p414-special-ops.p4
-  testdata/p4_14_samples/p414-special-ops-2-bmv2.p4
-  testdata/p4_14_samples/p414-special-ops-3-bmv2.p4
   # This test uses a feature currently unsupported in the BMv2 back-end.
   testdata/p4_16_samples/issue907-bmv2.p4
   # This test uses a table graph that is not implementable in BMv2

backends/bmv2/common/JsonObjects.cpp

@@ -231,7 +231,7 @@ unsigned
 JsonObjects::add_metadata(const cstring& type, const cstring& name) {
     auto header = new Util::JsonObject();
     unsigned id = BMV2::nextId("headers");
-    LOG1("add metadata header id " << id);
+    LOG3("add metadata header id " << id);
     header->emplace("name", name);
     header->emplace("id", id);
     header->emplace("header_type", type);

backends/bmv2/common/annotations.h

@@ -29,6 +29,7 @@ class ParseAnnotations : public P4::ParseAnnotations {
  public:
     ParseAnnotations() : P4::ParseAnnotations("BMV2", false, {
                 PARSE_EMPTY("metadata"),
+                PARSE_EXPRESSION_LIST("field_list"),
                 PARSE("alias", StringLiteral),
                 PARSE("priority", Constant)
             }) { }

backends/bmv2/common/backend.h

@@ -122,6 +122,7 @@ class RenameUserMetadata : public Transform {
     // Used as a prefix for the fields of the userMetadata structure
     // and also as a name for the userMetadata type clone.
     cstring namePrefix;
+    bool renamed = false;
 
  public:
     RenameUserMetadata(P4::ReferenceMap* refMap,
@@ -131,11 +132,13 @@ class RenameUserMetadata : public Transform {
 
     const IR::Node* postorder(IR::Type_Struct* type) override {
         // Clone the user metadata type
-        if (userMetaType != getOriginal())
+        auto orig = getOriginal<IR::Type_Struct>();
+        if (userMetaType->name != orig->name)
             return type;
 
         auto vec = new IR::IndexedVector<IR::Node>();
-        LOG2("Creating clone" << getOriginal());
+        LOG2("Creating clone of " << orig);
+        renamed = true;
         auto clone = type->clone();
         clone->name = namePrefix;
         vec->push_back(clone);
@@ -180,6 +183,11 @@ class RenameUserMetadata : public Transform {
         LOG2("Replacing reference with " << type);
         return type;
     }
+
+    void end_apply(const IR::Node*) override {
+        BUG_CHECK(renamed, "Could not identify user metadata type declaration %1%",
+                  userMetaType);
+    }
 };
 
 }  // namespace BMV2

backends/bmv2/common/extern.cpp

@@ -221,29 +221,10 @@ ExternConverter::createCalculation(ConversionContext* ctxt,
     if (sourcePositionNode != nullptr)
         calc->emplace_non_null("source_info", sourcePositionNode->sourceInfoJsonObj());
     calc->emplace("algo", algo);
-    if (!fields->is<IR::ListExpression>()) {
-        // expand it into a list
-        auto list = new IR::ListExpression({});
-        auto type = ctxt->typeMap->getType(fields, true);
-        if (!type->is<IR::Type_StructLike>()) {
-            modelError("%1%: expected a struct", fields);
-            return calcName;
-        }
-        if (auto se = fields->to<IR::StructExpression>()) {
-            for (auto f : se->components) {
-                auto e = f->expression;
-                list->push_back(e);
-            }
-        } else {
-            for (auto f : type->to<IR::Type_StructLike>()->fields) {
-                auto e = new IR::Member(fields, f->name);
-                auto ftype = ctxt->typeMap->getType(f);
-                ctxt->typeMap->setType(e, ftype);
-                list->push_back(e);
-            }
-        }
-        fields = list;
-        ctxt->typeMap->setType(fields, type);
+    fields = convertToList(fields, ctxt->typeMap);
+    if (!fields) {
+        modelError("%1%: expected a struct", fields);
+        return calcName;
     }
     auto jright = ctxt->conv->convertWithConstantWidths(fields);
     if (withPayload) {

backends/bmv2/common/header.cpp

@@ -39,7 +39,7 @@ HeaderConverter::HeaderConverter(ConversionContext* ctxt, cstring scalarsName)
  * @param meta this boolean indicates if the struct is a metadata or header.
  */
 void HeaderConverter::addTypesAndInstances(const IR::Type_StructLike* type, bool meta) {
-    LOG1("Adding " << type);
+    LOG2("Adding " << type);
     for (auto f : type->fields) {
         auto ft = ctxt->typeMap->getType(f, true);
         if (ft->is<IR::Type_StructLike>()) {
@@ -85,7 +85,7 @@ void HeaderConverter::addTypesAndInstances(const IR::Type_StructLike* type, bool
             }
         } else if (ft->is<IR::Type_Stack>()) {
             // Done elsewhere
-            LOG1("stack generation done elsewhere");
+            LOG3("stack generation done elsewhere");
             continue;
         } else {
             // Treat this field like a scalar local variable
@@ -141,7 +141,7 @@ Util::JsonArray* HeaderConverter::addHeaderUnionFields(
 }
 
 void HeaderConverter::addHeaderStacks(const IR::Type_Struct* headersStruct) {
-    LOG1("Creating stack " << headersStruct);
+    LOG2("Creating stack " << headersStruct);
     for (auto f : headersStruct->fields) {
         auto ft = ctxt->typeMap->getType(f, true);
         auto stack = ft->to<IR::Type_Stack>();
@@ -290,7 +290,7 @@ void HeaderConverter::addHeaderType(const IR::Type_StructLike *st) {
     }
     ctxt->json->add_header_type(name, fields, max_length_bytes);
 
-    LOG1("... creating aliases for metadata fields " << st);
+    LOG2("... creating aliases for metadata fields " << st);
     for (auto f : st->fields) {
         if (auto aliasAnnotation = f->getAnnotation("alias")) {
             auto container = new Util::JsonArray();
@@ -328,7 +328,7 @@ Visitor::profile_t HeaderConverter::init_apply(const IR::Node* node) {
     // bit<n>, bool, error are packed into scalars type,
     // varbit, struct and stack introduce new header types
     for (auto v : ctxt->structure->variables) {
-        LOG1("variable " << v);
+        LOG2("variable " << v);
         auto type = ctxt->typeMap->getType(v, true);
         if (auto st = type->to<IR::Type_StructLike>()) {
             auto metadata_type = st->controlPlaneName();
@@ -443,7 +443,7 @@ void HeaderConverter::end_apply(const IR::Node*) {
  */
 bool HeaderConverter::preorder(const IR::Parameter* param) {
     LOG3("convert param " << param);
-    //// keep track of which headers we've already generated the ctxt->json for
+    // keep track of which headers we've already generated in ctxt->json
     auto ft = ctxt->typeMap->getType(param->getNode(), true);
     if (ft->is<IR::Type_Struct>()) {
         auto st = ft->to<IR::Type_Struct>();

backends/bmv2/common/helpers.cpp

@@ -78,6 +78,135 @@ unsigned nextId(cstring group) {
     return counters[group]++;
 }
 
-}  // namespace BMV2
+void
+ConversionContext::addToFieldList(const IR::Expression* expr, Util::JsonArray* fl) {
+    if (auto le = expr->to<IR::ListExpression>()) {
+        for (auto e : le->components) {
+            addToFieldList(e, fl);
+        }
+        return;
+    } else if (auto si = expr->to<IR::StructExpression>()) {
+        for (auto e : si->components) {
+            addToFieldList(e->expression, fl);
+        }
+        return;
+    }
+
+    auto type = typeMap->getType(expr, true);
+    if (type->is<IR::Type_StructLike>()) {
+        // recursively add all fields
+        auto st = type->to<IR::Type_StructLike>();
+        for (auto f : st->fields) {
+            auto member = new IR::Member(expr, f->name);
+            typeMap->setType(member, typeMap->getType(f, true));
+            addToFieldList(member, fl);
+        }
+        return;
+    }
+
+    bool simple = conv->simpleExpressionsOnly;
+    conv->simpleExpressionsOnly = true;  // we do not want casts d2b in field_lists
+    auto j = conv->convert(expr);
+    conv->simpleExpressionsOnly = simple;  // restore state
+    if (auto jo = j->to<Util::JsonObject>()) {
+        if (auto t = jo->get("type")) {
+            if (auto type = t->to<Util::JsonValue>()) {
+                if (*type == "runtime_data") {
+                    // Can't have runtime_data in field lists -- need hexstr instead
+                    auto val = jo->get("value")->to<Util::JsonValue>();
+                    j = jo = new Util::JsonObject();
+                    jo->emplace("type", "hexstr");
+                    jo->emplace("value", stringRepr(val->getValue()));
+                }
+            }
+        }
+    }
+    fl->append(j);
+}
+
+int
+ConversionContext::createFieldList(
+    const IR::Expression* expr, cstring listName, bool learn) {
+    cstring group;
+    auto fl = new Util::JsonObject();
+    if (learn) {
+        group = "learn_lists";
+        json->learn_lists->append(fl);
+    } else {
+        group = "field_lists";
+        json->field_lists->append(fl);
+    }
+    int id = nextId(group);
+    fl->emplace("id", id);
+    fl->emplace("name", listName);
+    fl->emplace_non_null("source_info", expr->sourceInfoJsonObj());
+    auto elements = mkArrayField(fl, "elements");
+    addToFieldList(expr, elements);
+    return id;
+}
 
+void
+ConversionContext::modelError(const char* format, const IR::Node* node) {
+    ::error(format, node);
+    ::error("Are you using an up-to-date v1model.p4?");
+}
+
+cstring
+ConversionContext::createCalculation(cstring algo, const IR::Expression* fields,
+                                     Util::JsonArray* calculations, bool withPayload,
+                                     const IR::Node* sourcePositionNode = nullptr) {
+    cstring calcName = refMap->newName("calc_");
+    auto calc = new Util::JsonObject();
+    calc->emplace("name", calcName);
+    calc->emplace("id", nextId("calculations"));
+    if (sourcePositionNode != nullptr)
+        calc->emplace_non_null("source_info", sourcePositionNode->sourceInfoJsonObj());
+    calc->emplace("algo", algo);
+    fields = convertToList(fields, typeMap);
+    if (!fields) {
+        modelError("%1%: expected a struct", fields);
+        return calcName;
+    }
+    auto jright = conv->convertWithConstantWidths(fields);
+    if (withPayload) {
+        auto array = jright->to<Util::JsonArray>();
+        BUG_CHECK(array, "expected a JSON array");
+        auto payload = new Util::JsonObject();
+        payload->emplace("type", "payload");
+        payload->emplace("value", (Util::IJson*)nullptr);
+        array->append(payload);
+    }
+    calc->emplace("input", jright);
+    calculations->append(calc);
+    return calcName;
+}
 
+/// Converts expr into a ListExpression or returns nullptr if not
+/// possible
+const IR::ListExpression* convertToList(const IR::Expression* expr, P4::TypeMap* typeMap) {
+    if (auto l = expr->to<IR::ListExpression>())
+        return l;
+
+    // expand it into a list
+    auto list = new IR::ListExpression({});
+    auto type = typeMap->getType(expr, true);
+    auto st = type->to<IR::Type_StructLike>();
+    if (!st) {
+        return nullptr;
+    }
+    if (auto se = expr->to<IR::StructExpression>()) {
+        for (auto f : se->components)
+            list->push_back(f->expression);
+    } else {
+        for (auto f : st->fields) {
+            auto e = new IR::Member(expr, f->name);
+            auto ftype = typeMap->getType(f);
+            typeMap->setType(e, ftype);
+            list->push_back(e);
+        }
+    }
+    typeMap->setType(list, type);
+    return list;
+}
+
+}  // namespace BMV2

backends/bmv2/common/helpers.h

@@ -324,6 +324,12 @@ struct ConversionContext {
                       ExpressionConverter* conv, JsonObjects* json) :
             refMap(refMap), typeMap(typeMap), toplevel(toplevel),
             blockConverted(BlockConverted::None), structure(structure), conv(conv), json(json) { }
+
+    void addToFieldList(const IR::Expression* expr, Util::JsonArray* fl);
+    int createFieldList(const IR::Expression* expr, cstring listName, bool learn = false);
+    cstring createCalculation(cstring algo, const IR::Expression* fields,
+                              Util::JsonArray* calculations, bool usePayload, const IR::Node* node);
+    static void modelError(const char* format, const IR::Node* place);
 };
 
 Util::IJson* nodeName(const CFG::Node* node);
@@ -333,6 +339,8 @@ Util::JsonObject* mkPrimitive(cstring name, Util::JsonArray* appendTo);
 Util::JsonObject* mkPrimitive(cstring name);
 cstring stringRepr(big_int value, unsigned bytes = 0);
 unsigned nextId(cstring group);
+/// Converts expr into a ListExpression or returns nullptr if not possible
+const IR::ListExpression* convertToList(const IR::Expression* expr, P4::TypeMap* typeMap);
 
 }  // namespace BMV2
 

backends/bmv2/common/programStructure.h

@@ -54,13 +54,13 @@ class ProgramStructure {
     // in the scalarsName metadata object, so we may need to rename
     // these fields.  This map holds the new names.
     std::map<const IR::StructField *, cstring> scalarMetadataFields;
-    // All the direct meters.
+    /// All the direct meters.
     DirectMeterMap directMeterMap;
-    // All the direct counters.
+    /// All the direct counters.
     ordered_map<cstring, const IR::P4Table *> directCounterMap;
-    // All match kinds
+    /// All match kinds
     std::set<cstring>  match_kinds;
-    // map IR node to compile-time allocated resource blocks.
+    /// map IR node to compile-time allocated resource blocks.
     ResourceMap resourceMap;
 
     ProgramStructure() {}

backends/bmv2/psa_switch/psaSwitch.cpp

@@ -939,8 +939,7 @@ Util::IJson* ExternConverter_Digest::convertExternObject(
             listName = st->controlPlaneName();
         }
     }
-    int id = createFieldList(ctxt, mc->arguments->at(0)->expression, "learn_lists",
-                             listName, ctxt->json->learn_lists);
+    int id = ctxt->createFieldList(mc->arguments->at(0)->expression, listName, true);
     auto cst = new IR::Constant(id);
     ctxt->typeMap->setType(cst, IR::Type_Bits::get(32));
     auto jcst = ctxt->conv->convert(cst);