A simple way to implement resubmit/recirculate/clone3 in P4-16 on v1model

backends/bmv2/CMakeLists.txt

@@ -145,17 +145,6 @@ set (XFAIL_TESTS
   testdata/p4_14_samples/issue60.p4
   # compiler claims (incorrectly?) that c2 has mulitple successors, so is not supported
   testdata/p4_14_samples/issue-1426.p4
-  # As of 2019-Feb-04 latest p4c code, this program fails due to the
-  # root cause of both issues #1694 and #1669.  I have tested it with
-  # the proposed fix for issue #1694 that is on PR #1704, and while
-  # that does make the produced BMv2 JSON file able to cause packets
-  # to be recirculated, resubmitted, etc. it still has the bug of not
-  # preserving the metadata specified in the program's field_lists, so
-  # the STF tests fail because of issue #1669.  As of this writing,
-  # this program is the _only_ p4c test program that relies for its
-  # successful execution on the preservation of metadata across
-  # resubmit, recirculate, and/or clone operations.
-  testdata/p4_14_samples/p414-special-ops-2-bmv2.p4
   # This test uses a feature currently unsupported in the BMv2 back-end.
   testdata/p4_16_samples/issue907-bmv2.p4
   # This test uses a table graph that is not implementable in BMv2

backends/bmv2/common/JsonObjects.cpp

@@ -229,7 +229,7 @@ unsigned
 JsonObjects::add_metadata(const cstring& type, const cstring& name) {
     auto header = new Util::JsonObject();
     unsigned id = BMV2::nextId("headers");
-    LOG1("add metadata header id " << id);
+    LOG3("add metadata header id " << id);
     header->emplace("name", name);
     header->emplace("id", id);
     header->emplace("header_type", type);

backends/bmv2/common/annotations.h

@@ -29,6 +29,7 @@ class ParseAnnotations : public P4::ParseAnnotations {
  public:
     ParseAnnotations() : P4::ParseAnnotations("BMV2", false, {
                 PARSE_EMPTY("metadata"),
+                PARSE_EXPRESSION_LIST("field_list"),
                 PARSE("alias", StringLiteral),
                 PARSE("priority", Constant)
             }) { }

backends/bmv2/common/header.cpp

@@ -39,7 +39,7 @@ HeaderConverter::HeaderConverter(ConversionContext* ctxt, cstring scalarsName)
  * @param meta this boolean indicates if the struct is a metadata or header.
  */
 void HeaderConverter::addTypesAndInstances(const IR::Type_StructLike* type, bool meta) {
-    LOG1("Adding " << type);
+    LOG2("Adding " << type);
     for (auto f : type->fields) {
         auto ft = ctxt->typeMap->getType(f, true);
         if (ft->is<IR::Type_StructLike>()) {
@@ -84,7 +84,7 @@ void HeaderConverter::addTypesAndInstances(const IR::Type_StructLike* type, bool
             }
         } else if (ft->is<IR::Type_Stack>()) {
             // Done elsewhere
-            LOG1("stack generation done elsewhere");
+            LOG3("stack generation done elsewhere");
             continue;
         } else {
             // Treat this field like a scalar local variable
@@ -140,7 +140,7 @@ Util::JsonArray* HeaderConverter::addHeaderUnionFields(
 }
 
 void HeaderConverter::addHeaderStacks(const IR::Type_Struct* headersStruct) {
-    LOG1("Creating stack " << headersStruct);
+    LOG2("Creating stack " << headersStruct);
     for (auto f : headersStruct->fields) {
         auto ft = ctxt->typeMap->getType(f, true);
         auto stack = ft->to<IR::Type_Stack>();
@@ -287,7 +287,7 @@ void HeaderConverter::addHeaderType(const IR::Type_StructLike *st) {
     }
     ctxt->json->add_header_type(name, fields, max_length_bytes);
 
-    LOG1("... creating aliases for metadata fields " << st);
+    LOG2("... creating aliases for metadata fields " << st);
     for (auto f : st->fields) {
         if (auto aliasAnnotation = f->getAnnotation("alias")) {
             auto container = new Util::JsonArray();
@@ -324,7 +324,7 @@ Visitor::profile_t HeaderConverter::init_apply(const IR::Node* node) {
     // bit<n>, bool, error are packed into scalars type,
     // varbit, struct and stack introduce new header types
     for (auto v : ctxt->structure->variables) {
-        LOG1("variable " << v);
+        LOG2("variable " << v);
         auto type = ctxt->typeMap->getType(v, true);
         if (auto st = type->to<IR::Type_StructLike>()) {
             auto metadata_type = st->controlPlaneName();
@@ -439,7 +439,7 @@ void HeaderConverter::end_apply(const IR::Node*) {
  */
 bool HeaderConverter::preorder(const IR::Parameter* param) {
     LOG3("convert param " << param);
-    //// keep track of which headers we've already generated the ctxt->json for
+    // keep track of which headers we've already generated in ctxt->json
     auto ft = ctxt->typeMap->getType(param->getNode(), true);
     if (ft->is<IR::Type_Struct>()) {
         auto st = ft->to<IR::Type_Struct>();

backends/bmv2/common/helpers.cpp

@@ -82,6 +82,119 @@ unsigned nextId(cstring group) {
     return counters[group]++;
 }
 
-}  // namespace BMV2
+void
+ConversionContext::addToFieldList(const IR::Expression* expr, Util::JsonArray* fl) {
+    if (auto le = expr->to<IR::ListExpression>()) {
+        for (auto e : le->components) {
+            addToFieldList(e, fl);
+        }
+        return;
+    } else if (auto si = expr->to<IR::StructInitializerExpression>()) {
+        for (auto e : si->components) {
+            addToFieldList(e->expression, fl);
+        }
+        return;
+    }
 
+    auto type = typeMap->getType(expr, true);
+    if (type->is<IR::Type_StructLike>()) {
+        // recursively add all fields
+        auto st = type->to<IR::Type_StructLike>();
+        for (auto f : st->fields) {
+            auto member = new IR::Member(expr, f->name);
+            typeMap->setType(member, typeMap->getType(f, true));
+            addToFieldList(member, fl);
+        }
+        return;
+    }
 
+    bool simple = conv->simpleExpressionsOnly;
+    conv->simpleExpressionsOnly = true;  // we do not want casts d2b in field_lists
+    auto j = conv->convert(expr);
+    conv->simpleExpressionsOnly = simple;  // restore state
+    if (auto jo = j->to<Util::JsonObject>()) {
+        if (auto t = jo->get("type")) {
+            if (auto type = t->to<Util::JsonValue>()) {
+                if (*type == "runtime_data") {
+                    // Can't have runtime_data in field lists -- need hexstr instead
+                    auto val = jo->get("value")->to<Util::JsonValue>();
+                    j = jo = new Util::JsonObject();
+                    jo->emplace("type", "hexstr");
+                    jo->emplace("value", stringRepr(val->getValue()));
+                }
+            }
+        }
+    }
+    fl->append(j);
+}
+
+int
+ConversionContext::createFieldList(
+    const IR::Expression* expr, cstring listName, bool learn) {
+    cstring group;
+    auto fl = new Util::JsonObject();
+    if (learn) {
+        group = "learn_lists";
+        json->learn_lists->append(fl);
+    } else {
+        group = "field_lists";
+        json->field_lists->append(fl);
+    }
+    int id = nextId(group);
+    fl->emplace("id", id);
+    fl->emplace("name", listName);
+    fl->emplace_non_null("source_info", expr->sourceInfoJsonObj());
+    auto elements = mkArrayField(fl, "elements");
+    addToFieldList(expr, elements);
+    return id;
+}
+
+void
+ConversionContext::modelError(const char* format, const IR::Node* node) {
+    ::error(format, node);
+    ::error("Are you using an up-to-date v1model.p4?");
+}
+
+cstring
+ConversionContext::createCalculation(cstring algo, const IR::Expression* fields,
+                                     Util::JsonArray* calculations, bool withPayload,
+                                     const IR::Node* sourcePositionNode = nullptr) {
+    cstring calcName = refMap->newName("calc_");
+    auto calc = new Util::JsonObject();
+    calc->emplace("name", calcName);
+    calc->emplace("id", nextId("calculations"));
+    if (sourcePositionNode != nullptr)
+        calc->emplace_non_null("source_info", sourcePositionNode->sourceInfoJsonObj());
+    calc->emplace("algo", algo);
+    if (!fields->is<IR::ListExpression>()) {
+        // expand it into a list
+        auto list = new IR::ListExpression({});
+        auto type = typeMap->getType(fields, true);
+        if (!type->is<IR::Type_StructLike>()) {
+            modelError("%1%: expected a struct", fields);
+            return calcName;
+        }
+        for (auto f : type->to<IR::Type_StructLike>()->fields) {
+            auto e = new IR::Member(fields, f->name);
+            auto ftype = typeMap->getType(f);
+            typeMap->setType(e, ftype);
+            list->push_back(e);
+        }
+        fields = list;
+        typeMap->setType(fields, type);
+    }
+    auto jright = conv->convertWithConstantWidths(fields);
+    if (withPayload) {
+        auto array = jright->to<Util::JsonArray>();
+        BUG_CHECK(array, "expected a JSON array");
+        auto payload = new Util::JsonObject();
+        payload->emplace("type", "payload");
+        payload->emplace("value", (Util::IJson*)nullptr);
+        array->append(payload);
+    }
+    calc->emplace("input", jright);
+    calculations->append(calc);
+    return calcName;
+}
+
+}  // namespace BMV2

backends/bmv2/common/helpers.h

@@ -91,6 +91,12 @@ struct ConversionContext {
                       ExpressionConverter* conv, JsonObjects* json) :
         refMap(refMap), typeMap(typeMap), toplevel(toplevel), structure(structure),
         conv(conv), json(json) { }
+
+    void addToFieldList(const IR::Expression* expr, Util::JsonArray* fl);
+    int createFieldList(const IR::Expression* expr, cstring listName, bool learn = false);
+    cstring createCalculation(cstring algo, const IR::Expression* fields,
+                              Util::JsonArray* calculations, bool usePayload, const IR::Node* node);
+    static void modelError(const char* format, const IR::Node* place);
 };
 
 using BlockTypeMap = std::map<const IR::Block*, const IR::Type*>;

backends/bmv2/common/programStructure.h

@@ -44,13 +44,13 @@ class ProgramStructure {
     // in the scalarsName metadata object, so we may need to rename
     // these fields.  This map holds the new names.
     std::map<const IR::StructField *, cstring> scalarMetadataFields;
-    // All the direct meters.
+    /// All the direct meters.
     DirectMeterMap directMeterMap;
-    // All the direct counters.
+    /// All the direct counters.
     ordered_map<cstring, const IR::P4Table *> directCounterMap;
-    // All match kinds
+    /// All match kinds
     std::set<cstring>  match_kinds;
-    // map IR node to compile-time allocated resource blocks.
+    /// map IR node to compile-time allocated resource blocks.
     ResourceMap resourceMap;
 
     ProgramStructure() {}