Skip to content

Commit

Permalink
feat: reduce ops (#15)
Browse files Browse the repository at this point in the history
* added reduce tests

* fixed affine load handling (using affine maps now)
  • Loading branch information
0xThemis authored Jan 2, 2024
1 parent 0a5df0d commit e76da81
Show file tree
Hide file tree
Showing 45 changed files with 310 additions and 18 deletions.
20 changes: 20 additions & 0 deletions mlir-assigner/include/mlir-assigner/memory/memref.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,16 @@ namespace nil {
}
return data[offset];
}

const VarType &get(const llvm::SmallVector<int64_t> &indices) const {
assert(indices.size() == dims.size());
uint32_t offset = 0;
for (int i = 0; i < indices.size(); i++) {
assert(indices[i] < dims[i]);
offset += indices[i] * strides[i];
}
return data[offset];
}
// VarType &get(const std::vector<uint32_t> &indices) const {
// assert(indices.size() == dims.size());
// uint32_t offset = 0;
Expand All @@ -86,6 +96,16 @@ namespace nil {
data[offset] = value;
}

void put(const llvm::SmallVector<int64_t> &indices, const VarType &value) {
assert(indices.size() == dims.size());
uint32_t offset = 0;
for (int i = 0; i < indices.size(); i++) {
assert(indices[i] < dims[i]);
offset += indices[i] * strides[i];
}
data[offset] = value;
}

void put_flat(const int64_t idx, const VarType &value) {
assert(idx >= 0 && idx < data.size());
data[idx] = value;
Expand Down
29 changes: 11 additions & 18 deletions mlir-assigner/include/mlir-assigner/parser/evaluator.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,6 @@ namespace zk_ml_toolchain {
assert(lhs != frames.back().constant_values.end());
assert(rhs != frames.back().constant_values.end());
auto result = lhs->second + rhs->second;
// llvm::outs() << "from " << *operation << " got " << result << "\n";
frames.back().constant_values[mlir::hash_value(operation.getResult())] = result;
} else if (arith::SubIOp operation = llvm::dyn_cast<arith::SubIOp>(op)) {
assert(operation.getLhs().getType().isa<IndexType>());
Expand All @@ -302,7 +301,6 @@ namespace zk_ml_toolchain {
assert(lhs != frames.back().constant_values.end());
assert(rhs != frames.back().constant_values.end());
auto result = lhs->second - rhs->second;
// llvm::outs() << "from " << *operation << " got " << result << "\n";
frames.back().constant_values[mlir::hash_value(operation.getResult())] = result;

} else if (arith::MulIOp operation = llvm::dyn_cast<arith::MulIOp>(op)) {
Expand All @@ -316,7 +314,6 @@ namespace zk_ml_toolchain {
assert(lhs != frames.back().constant_values.end());
assert(rhs != frames.back().constant_values.end());
auto result = lhs->second * rhs->second;
// llvm::outs() << "from " << *operation << " got " << result << "\n";
frames.back().constant_values[mlir::hash_value(operation.getResult())] = result;

} else if (arith::ConstantOp operation = llvm::dyn_cast<arith::ConstantOp>(op)) {
Expand Down Expand Up @@ -374,7 +371,7 @@ namespace zk_ml_toolchain {
frames.back().locals[mlir::hash_value(operation.getResult())] =
frames.back().locals[mlir::hash_value(operation.getLhs())];
} else if (math::SqrtOp operation = llvm::dyn_cast<math::SqrtOp>(op)) {
UNREACHABLE("TODO: sqrt");
UNREACHABLE("TODO: component for sqrt not ready");
} else if (math::ErfOp operation = llvm::dyn_cast<math::ErfOp>(op)) {
UNREACHABLE("TODO: component for erf not ready");
} else {
Expand Down Expand Up @@ -402,24 +399,23 @@ namespace zk_ml_toolchain {
auto operandsToV = llvm::SmallVector<Value>(operandsTo.begin(), operandsTo.end());
int64_t from = evaluateForParameter(fromMap, operandsFromV, true);
int64_t to = evaluateForParameter(toMap, operandsToV, false);
// llvm::outs() << "starting for with: " << from << "->" << to << " (step)
// " << step << "\n";
doAffineFor(operation, from, to, step);
} else if (affine::AffineLoadOp operation = llvm::dyn_cast<affine::AffineLoadOp>(op)) {
auto memref = frames.back().memrefs.find(mlir::hash_value(operation.getMemref()));
assert(memref != frames.back().memrefs.end());

// grab the indices and build index vector
auto indices = operation.getIndices();
std::vector<int64_t> indicesV;
indicesV.reserve(indices.size());
std::vector<int64_t> mapDims;
mapDims.reserve(indices.size());
for (auto a : indices) {
// look for indices in constant_values
auto res = frames.back().constant_values.find(mlir::hash_value(a));
assert(res != frames.back().constant_values.end());
indicesV.push_back(res->second);
mapDims.push_back(res->second);
}
auto value = memref->second.get(indicesV);
auto affineMap = castFromAttr<AffineMapAttr>(operation->getAttr(affine::AffineLoadOp::getMapAttrStrName())).getAffineMap();
auto value = memref->second.get(evalAffineMap(affineMap, mapDims));
frames.back().locals[mlir::hash_value(operation.getResult())] = value;
} else if (affine::AffineStoreOp operation = llvm::dyn_cast<affine::AffineStoreOp>(op)) {
// affine.store
Expand All @@ -430,18 +426,19 @@ namespace zk_ml_toolchain {

// grab the indices and build index vector
auto indices = operation.getIndices();
std::vector<int64_t> indicesV;
indicesV.reserve(indices.size());
std::vector<int64_t> mapDims;
mapDims.reserve(indices.size());
for (auto a : indices) {
auto res = frames.back().constant_values.find(mlir::hash_value(a));
assert(res != frames.back().constant_values.end());
indicesV.push_back(res->second);
mapDims.push_back(res->second);
}
// grab the element from the locals array
auto value = frames.back().locals.find(mlir::hash_value(operation.getValue()));
assert(value != frames.back().locals.end());
// put the element from the memref using index vector
memref->second.put(indicesV, value->second);
auto affineMap = castFromAttr<AffineMapAttr>(operation->getAttr(affine::AffineStoreOp::getMapAttrStrName())).getAffineMap();
memref->second.put(evalAffineMap(affineMap, mapDims), value->second);

} else if (affine::AffineYieldOp operation = llvm::dyn_cast<affine::AffineYieldOp>(op)) {
// Affine Yields are Noops for us
Expand Down Expand Up @@ -474,7 +471,6 @@ namespace zk_ml_toolchain {
AffineMap applyMap = castFromAttr<AffineMapAttr>(op->getAttrs()[0].getValue()).getAffineMap();
llvm::SmallVector<Value> operands(op->getOperands().begin(), op->getOperands().end());
int64_t result = evaluateForParameter(applyMap, operands, false);
// llvm::outs() << "from " << *op << " got result " << result << "\n";
frames.back().constant_values[mlir::hash_value(op->getResults()[0])] = result;
} else if (opName == "affine.max") {
logger.debug("got affine.max");
Expand All @@ -483,7 +479,6 @@ namespace zk_ml_toolchain {
AffineMap applyMap = castFromAttr<AffineMapAttr>(op->getAttrs()[0].getValue()).getAffineMap();
llvm::SmallVector<Value> operands(op->getOperands().begin(), op->getOperands().end());
int64_t result = evaluateForParameter(applyMap, operands, true);
// llvm::outs() << "from " << *op << " got result " << result << "\n";
frames.back().constant_values[mlir::hash_value(op->getResults()[0])] = result;
} else {
UNREACHABLE(std::string("unhandled affine operation: ") + opName);
Expand Down Expand Up @@ -642,7 +637,6 @@ namespace zk_ml_toolchain {
auto index = frames.back().constant_values.find(mlir::hash_value(operands[dynamicCounter++]));
assert(index != frames.back().constant_values.end());
dims.emplace_back(index->second);
// llvm::outs() << "dynamic size is: " << index->second << "\n";
} else {
dims.emplace_back(dim);
}
Expand Down Expand Up @@ -701,7 +695,6 @@ namespace zk_ml_toolchain {
auto res = frames.back().constant_values.find(mlir::hash_value(a));
assert(res != frames.back().constant_values.end());
indicesV.push_back(res->second);
// llvm::outs() << "found " << res->second << "\n";
}
// grab the element from the locals array
auto value = frames.back().locals.find(mlir::hash_value(operation.getValue()));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"memref": {"data": [0.44464111328125, 0.722412109375, 0.1399993896484375, 0.3859405517578125, 0.4351654052734375, 0.1930694580078125, 0.1060943603515625, 0.3092041015625, 0.4474029541015625, 0.7837677001953125], "dims": [1, 10], "type": "f32"}}]
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
module attributes {llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-pc-linux-gnu", "onnx-mlir.symbol-postfix" = "reducel2simple.mlir"} {
func.func @main_graph(%arg0: memref<1x10xf32>) -> memref<1x1xf32> attributes {input_names = ["in_a"], llvm.emit_c_interface, output_names = ["out_a"]} {
%cst = arith.constant 0.000000e+00 : f32
%c0 = arith.constant 0 : index
%alloc = memref.alloc() {alignment = 16 : i64} : memref<1x10xf32>
affine.for %arg1 = 0 to 1 {
affine.for %arg2 = 0 to 10 {
%0 = affine.load %arg0[%c0, %arg2] : memref<1x10xf32>
%1 = affine.load %arg0[%c0, %arg2] : memref<1x10xf32>
%2 = arith.mulf %0, %1 : f32
affine.store %2, %alloc[%arg1, %arg2] : memref<1x10xf32>
}
}
%alloc_0 = memref.alloc() {alignment = 16 : i64} : memref<1x1xf32>
affine.for %arg1 = 0 to 1 {
affine.for %arg2 = 0 to 1 {
affine.store %cst, %alloc_0[%arg1, %arg2] : memref<1x1xf32>
}
}
affine.for %arg1 = 0 to 1 {
affine.for %arg2 = 0 to 10 {
%0 = affine.load %alloc[%arg1, %arg2] : memref<1x10xf32>
%1 = affine.load %alloc_0[%arg1, %c0] : memref<1x1xf32>
%2 = arith.addf %1, %0 : f32
affine.store %2, %alloc_0[%arg1, %c0] : memref<1x1xf32>
}
}
%alloc_1 = memref.alloc() {alignment = 16 : i64} : memref<1x1xf32>
affine.for %arg1 = 0 to 1 {
affine.for %arg2 = 0 to 1 {
%0 = affine.load %alloc_0[%arg1, %arg2] : memref<1x1xf32>
%1 = math.sqrt %0 : f32
affine.store %1, %alloc_1[%arg1, %arg2] : memref<1x1xf32>
}
}
return %alloc_1 : memref<1x1xf32>
}
"krnl.entry_point"() {func = @main_graph, numInputs = 1 : i32, numOutputs = 1 : i32, signature = "[ { \22type\22 : \22f32\22 , \22dims\22 : [1 , 10] , \22name\22 : \22in_a\22 }\0A\0A]\00@[ { \22type\22 : \22f32\22 , \22dims\22 : [1 , 1] , \22name\22 : \22out_a\22 }\0A\0A]\00"} : () -> ()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
 :o

in_a
in_bout_a"ReduceL2ReduceL2Simple*:Bin_bZ
in_a



b
out_a


B
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Result:
memref<1x1xf32>[1.4269211292266846]
21
1 change: 1 addition & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceL1/ReduceL12D.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"memref": {"data": [0.5316009521484375, 0.771148681640625, 0.7549896240234375, 0.00177001953125, 0.8040618896484375, 0.6228179931640625, 0.8080291748046875, 0.7175140380859375, 0.3006744384765625, 0.7958221435546875, 0.2895660400390625, 0.6257171630859375, 0.7779541015625, 0.2160797119140625, 0.886260986328125, 0.46527099609375, 0.224029541015625, 0.426666259765625, 0.7244873046875, 0.25927734375, 0.85498046875, 0.5196533203125, 0.3891143798828125, 0.277099609375, 0.0838470458984375, 0.06463623046875, 0.560943603515625, 0.9423065185546875, 0.12188720703125, 0.1664276123046875, 0.4301300048828125, 0.4993133544921875, 0.0058746337890625, 0.42437744140625, 0.51446533203125, 0.794189453125, 0.9392242431640625, 0.454254150390625, 0.1020050048828125, 0.5545806884765625, 0.8192138671875, 0.9871826171875, 0.12567138671875, 0.955535888671875, 0.0265045166015625, 0.563629150390625, 0.1708831787109375, 0.199951171875, 0.246795654296875, 0.9832305908203125, 0.7613525390625, 0.4416656494140625, 0.6462554931640625, 0.8548583984375, 0.345672607421875, 0.20172119140625, 0.7459716796875, 0.3435211181640625, 0.807159423828125, 0.2892913818359375, 0.6972198486328125, 0.866607666015625, 0.340545654296875, 0.1118011474609375, 0.881439208984375, 0.9730682373046875, 0.5344696044921875, 0.01611328125, 0.0105133056640625, 0.5793304443359375, 0.4836578369140625, 0.8841705322265625, 0.527740478515625, 0.9219512939453125, 0.8319549560546875, 0.8092041015625, 0.0500335693359375, 0.8453826904296875, 0.1137847900390625, 0.628753662109375, 0.847320556640625, 0.08111572265625, 0.6227264404296875, 0.496917724609375, 0.74072265625, 0.213226318359375, 0.3129425048828125, 0.9722900390625, 0.594940185546875, 0.5207672119140625, 0.798370361328125, 0.646820068359375, 0.102294921875, 0.4672088623046875, 0.2514495849609375, 0.5641021728515625, 0.457611083984375, 0.6670684814453125, 0.54400634765625, 0.943328857421875], "dims": [1, 10, 10], "type": "f32"}}]
17 changes: 17 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceL1/ReduceL12D.onnx
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
 :t

in_a
in_bout_a"ReduceL1
ReduceL12D*:Bin_bZ
in_a





b
out_a



B
3 changes: 3 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceL1/ReduceL12D.res
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Result:
memref<1x1x1xf32>[52.172088623046875]
200 rows
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"memref": {"data": [0.907958984375, 0.197296142578125, 0.274932861328125, 0.7010650634765625, 0.831512451171875, 0.00579833984375, 0.2776641845703125, 0.255767822265625, 0.0640106201171875, 0.1799468994140625, 0.121917724609375, 0.624969482421875, 0.6061553955078125, 0.1122589111328125, 0.67132568359375, 0.479736328125, 0.3151092529296875, 0.4059600830078125, 0.969146728515625, 0.1484832763671875, 0.803192138671875, 0.56060791015625, 0.80560302734375, 0.438690185546875, 0.9733123779296875, 0.5877227783203125, 0.0193634033203125, 0.398193359375, 0.3353118896484375, 0.2707366943359375, 0.4082794189453125, 0.7901763916015625, 0.2892303466796875, 0.9901123046875, 0.7431793212890625, 0.5735931396484375, 0.4205322265625, 0.590728759765625, 0.274322509765625, 0.6052398681640625, 0.0029296875, 0.05548095703125, 0.6415557861328125, 0.6295928955078125, 0.286590576171875, 0.2318878173828125, 0.7197418212890625, 0.578399658203125, 0.8871002197265625, 0.4137115478515625, 0.379852294921875, 0.5147857666015625, 0.8922119140625, 0.1077880859375, 0.342071533203125, 0.384521484375, 0.322235107421875, 0.5508880615234375, 0.676239013671875, 0.025604248046875, 0.0616912841796875, 0.7592926025390625, 0.6242523193359375, 0.104766845703125, 0.6704864501953125, 0.83856201171875, 0.7960357666015625, 0.514495849609375, 0.9562835693359375, 0.891204833984375, 0.1709747314453125, 0.946258544921875, 0.358917236328125, 0.65179443359375, 0.6244354248046875, 0.48388671875, 0.244140625, 0.923248291015625, 0.613250732421875, 0.3749542236328125, 0.376861572265625, 0.2886962890625, 0.5413665771484375, 0.809600830078125, 0.82452392578125, 0.5217742919921875, 0.2724151611328125, 0.2464447021484375, 0.6473846435546875, 0.144256591796875, 0.701416015625, 0.1503448486328125, 0.8818206787109375, 0.032745361328125, 0.5528564453125, 0.0033721923828125, 0.532989501953125, 0.989013671875, 0.8312835693359375, 0.1944732666015625], "dims": [1, 10, 10], "type": "f32"}}]
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Result:
memref<1xf32>[48.82490539550781]
200 rows
1 change: 1 addition & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceL1/ReduceL1Simple.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"memref": {"data": [0.43212890625, 0.4479827880859375, 0.19281005859375, 0.09112548828125, 0.7779998779296875, 0.6609344482421875, 0.303619384765625, 0.273406982421875, 0.314208984375, 0.6693115234375], "dims": [1, 10], "type": "f32"}}]
13 changes: 13 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceL1/ReduceL1Simple.onnx
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
 :o

in_a
in_bout_a"ReduceL1ReduceL1Simple*:Bin_bZ
in_a



b
out_a


B
Expand Down
3 changes: 3 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceL1/ReduceL1Simple.res
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Result:
memref<1x1xf32>[4.1635284423828125]
20 rows
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"memref": {"data": [0.111480712890625, 0.7744903564453125, 0.6857147216796875, 0.3732147216796875, 0.023529052734375, 0.4350738525390625, 0.0581207275390625, 0.4693756103515625, 0.339111328125, 0.4113311767578125], "dims": [1, 10], "type": "f32"}}]
13 changes: 13 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceLogSum/ReduceLogSumSimple.onnx
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
 :w
!
in_a
in_bout_a" ReduceLogSumReduceLogSumSimple*:Bin_bZ
in_a



b
out_a


B
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Result:
memref<1x1xf32>[1.3033045530319214]
17
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"memref": {"data": [0.4797821044921875, 0.9048919677734375, 0.6749114990234375, 0.18011474609375, 0.4658966064453125, 0.5195465087890625, 0.996673583984375, 0.790252685546875, 0.948822021484375, 0.69696044921875], "dims": [1, 10], "type": "f32"}}]
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
 :}
$
in_a
in_bout_a"ReduceLogSumExpReduceLogSumExpSimple*:Bin_bZ
in_a



b
out_a


B
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Result:
memref<1x1xf32>[2.997072458267212]
58
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"memref": {"data": [0.486663818359375, 0.076171875, 0.8419036865234375, 0.3363494873046875, 0.053070068359375, 0.1689453125, 0.9307098388671875, 0.3600311279296875, 0.0389862060546875, 0.0422821044921875], "dims": [1, 10], "type": "f32"}}]
13 changes: 13 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceMax/ReduceMaxSimple.onnx
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
 :q

in_a
in_bout_a" ReduceMaxReduceMaxSimple*:Bin_bZ
in_a



b
out_a


B
Expand Down
3 changes: 3 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceMax/ReduceMaxSimple.res
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Result:
memref<1x1xf32>[0.9307098388671875]
20
1 change: 1 addition & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceMean/ReduceMean2D.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{"memref": {"data": [0.8568878173828125, 0.1202239990234375, 0.91839599609375, 0.7568511962890625, 0.2068023681640625, 0.2118682861328125, 0.682952880859375, 0.607269287109375, 0.4605712890625, 0.9705810546875, 0.9650115966796875, 0.1146240234375, 0.3098907470703125, 0.2905731201171875, 0.329986572265625, 0.280059814453125, 0.0103912353515625, 0.7331695556640625, 0.426910400390625, 0.041961669921875, 0.20196533203125, 0.65582275390625, 0.3361968994140625, 0.7577972412109375, 0.2241363525390625, 0.5425872802734375, 0.1729736328125, 0.7314910888671875, 0.6883697509765625, 0.88494873046875, 0.7875213623046875, 0.7908935546875, 0.2242279052734375, 0.8392486572265625, 0.214874267578125, 0.88702392578125, 0.2572021484375, 0.16156005859375, 0.0554656982421875, 0.5114288330078125, 0.875762939453125, 0.95074462890625, 0.773223876953125, 0.755035400390625, 0.4380950927734375, 0.8548431396484375, 0.3909912109375, 0.6981201171875, 0.796966552734375, 0.545501708984375, 0.8203277587890625, 0.3695831298828125, 0.874420166015625, 0.875701904296875, 0.559478759765625, 0.0601806640625, 0.624725341796875, 0.37554931640625, 0.7270965576171875, 0.2436065673828125, 0.6870880126953125, 0.637451171875, 0.63201904296875, 0.4983062744140625, 0.078765869140625, 0.67926025390625, 0.752655029296875, 0.00982666015625, 0.195404052734375, 0.5293731689453125, 0.18634033203125, 0.2686004638671875, 0.789337158203125, 0.4116363525390625, 0.7089691162109375, 0.4565582275390625, 0.6157989501953125, 0.0911865234375, 0.0789947509765625, 0.5979156494140625, 0.1395111083984375, 0.68408203125, 0.034637451171875, 0.6453399658203125, 0.1017303466796875, 0.680389404296875, 0.788970947265625, 0.7945556640625, 0.8411102294921875, 0.3185272216796875, 0.5118408203125, 0.1500091552734375, 0.460723876953125, 0.5629119873046875, 0.125762939453125, 0.9395751953125, 0.2630157470703125, 0.966522216796875, 0.550506591796875, 0.0209197998046875], "dims": [1, 10, 10], "type": "f32"}}]
17 changes: 17 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceMean/ReduceMean2D.onnx
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
 :x

in_a
in_bout_a"
ReduceMean ReduceMean2D*:Bin_bZ
in_a





b
out_a



B
3 changes: 3 additions & 0 deletions mlir-assigner/tests/Ops/Onnx/ReduceMean/ReduceMean2D.res
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Result:
memref<1x1x1xf32>[0.5031680464744568]
105
Loading

0 comments on commit e76da81

Please sign in to comment.