Skip to content

Commit

Permalink
fix smoke tests (#2313)
Browse files Browse the repository at this point in the history 
* fix smoke tests

* editorconfig
  • Loading branch information
@timbrinded
timbrinded authored May 24, 2023
1 parent 1c2de65 commit 027b9df
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 60 deletions.
71 changes: 42 additions & 29 deletions test/suites/smoke/test-block-weights.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import "@polkadot/api-augment";
import "@moonbeam-network/api-augment/moonbase";
import { BN } from "@polkadot/util";
import { FrameSystemEventRecord } from "@polkadot/types/lookup";
import { WEIGHT_PER_GAS, extractWeight, getBlockArray, THIRTY_MINS } from "@moonwall/util";
import { describeSuite, beforeAll, expect } from "@moonwall/cli";
import { beforeAll, describeSuite, expect } from "@moonwall/cli";
import { THIRTY_MINS, WEIGHT_PER_GAS, extractWeight, getBlockArray } from "@moonwall/util";
import { ApiPromise } from "@polkadot/api";
import "@polkadot/api-augment";
import { GenericExtrinsic } from "@polkadot/types";
import { FrameSystemEventRecord } from "@polkadot/types/lookup";
import { AnyTuple } from "@polkadot/types/types";
import { rateLimiter } from "../../helpers/common.js";
const timePeriod = process.env.TIME_PERIOD ? Number(process.env.TIME_PERIOD) : THIRTY_MINS;
const timeout = Math.floor(timePeriod / 12); // 2 hour -> 10 minute timeout
Expand All @@ -14,17 +15,17 @@ interface BlockInfo {
blockNum: number;
hash: string;
weights: {
normal: BN;
operational: BN;
mandatory: BN;
normal: bigint;
operational: bigint;
mandatory: bigint;
};
extrinsics;
extrinsics: GenericExtrinsic<AnyTuple>[];
events: FrameSystemEventRecord[];
}

interface BlockLimits {
normal: BN;
operational: BN;
normal: bigint;
operational: bigint;
}

describeSuite({
Expand Down Expand Up @@ -59,9 +60,9 @@ describeSuite({
blockNum,
hash: blockHash.toString(),
weights: {
normal: extractWeight(normal),
operational: extractWeight(operational),
mandatory: extractWeight(mandatory),
normal: extractWeight(normal).toBigInt(),
operational: extractWeight(operational).toBigInt(),
mandatory: extractWeight(mandatory).toBigInt(),
},
events,
extrinsics,
Expand All @@ -71,8 +72,8 @@ describeSuite({

// Support for weight v1 and weight v2.
blockLimits = {
normal: extractWeight(limits.perClass.normal.maxTotal).toBn(),
operational: extractWeight(limits.perClass.operational.maxTotal).toBn(),
normal: extractWeight(limits.perClass.normal.maxTotal).toBigInt(),
operational: extractWeight(limits.perClass.operational.maxTotal).toBigInt(),
};
blockInfoArray = await Promise.all(
blockNumArray.map((num) => limiter.schedule(() => getLimits(num)))
Expand All @@ -99,7 +100,7 @@ describeSuite({
title: "normal usage should be less than normal dispatch class limits",
test: async function () {
const overweight = blockInfoArray
.filter((a) => a.weights.normal.gt(blockLimits.normal))
.filter((a) => a.weights.normal > blockLimits.normal)
.map((a) => {
log(
`Block #${a.blockNum} has weight ${Number(a.weights.normal)} which is above limit!`
Expand All @@ -121,7 +122,7 @@ describeSuite({
title: "operational usage should be less than dispatch class limits",
test: async function () {
const overweight = blockInfoArray
.filter((a) => a.weights.operational.gt(blockLimits.operational))
.filter((a) => a.weights.operational > blockLimits.operational)
.map((a) => {
log(
`Block #${a.blockNum} has weight ${Number(
Expand Down Expand Up @@ -160,22 +161,34 @@ describeSuite({
const checkBlockWeight = async (blockInfo: BlockInfo) => {
const apiAt = await paraApi.at(blockInfo.hash);

const normalWeight = Number(blockInfo.weights.normal);
const normalWeight = blockInfo.weights.normal;
const maxWeight = blockLimits.normal;
const ethBlock = (await apiAt.query.ethereum.currentBlock()).unwrap();
const balTxns = blockInfo.extrinsics
.map((ext, index) =>
ext.method.method == "transfer" && ext.method.section == "balances" ? index : -1
)
.filter((a) => a != -1);
const balTxnWeights = blockInfo.events
.map((event) =>
paraApi.events.system.ExtrinsicSuccess.is(event.event) &&
event.phase.isApplyExtrinsic &&
balTxns.includes(event.phase.asApplyExtrinsic.toNumber())
? event.event.data.dispatchInfo.weight.refTime.toBigInt()
: 0n
)
.reduce((acc, curr) => acc + curr, 0n);

const actualWeightUsed = normalWeight / Number(maxWeight);
if (actualWeightUsed > 0.2) {
const actualWeightUsed = (normalWeight * 100n) / maxWeight;
if (actualWeightUsed > 20n) {
const gasUsed = ethBlock.header.gasUsed.toBigInt();
const weightCalc = gasUsed * WEIGHT_PER_GAS;
const newRatio = (normalWeight - Number(weightCalc)) / Number(maxWeight);
if (newRatio > 0.2) {
const newRatio = ((normalWeight - weightCalc - balTxnWeights) * 100n) / maxWeight;
if (newRatio > 20n) {
log(
`Block #${blockInfo.blockNum} is ${(actualWeightUsed * 100).toFixed(
2
)}% full with ${
ethBlock.transactions.length
} transactions, non-transaction weight: ${(newRatio * 100).toFixed(2)}%`
`Block #${blockInfo.blockNum} is ${actualWeightUsed}% full with ` +
ethBlock.transactions.length +
` transactions, non-transaction weight: ${newRatio}%`
);
}
return { blockNum: blockInfo.blockNum, nonTxn: newRatio };
Expand All @@ -185,7 +198,7 @@ describeSuite({
const results = await Promise.all(
blockInfoArray.map((blockInfo) => limiter.schedule(() => checkBlockWeight(blockInfo)))
);
const nonTxnHeavyBlocks = results.filter((a) => a && a.nonTxn > 0.2);
const nonTxnHeavyBlocks = results.filter((a) => a && a.nonTxn > 20n);
expect(
nonTxnHeavyBlocks,
`These blocks have non-txn weights >20%, please investigate: ${nonTxnHeavyBlocks
Expand Down
4 changes: 2 additions & 2 deletions test/suites/smoke/test-ethereum-failures.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import "@moonbeam-network/api-augment/moonbase";
import { getBlockArray } from "@moonwall/util";
import { TWO_MINS, getBlockArray } from "@moonwall/util";
import { ApiPromise } from "@polkadot/api";
import { beforeAll, describeSuite, expect } from "@moonwall/cli";
import type { DispatchInfo } from "@polkadot/types/interfaces";
Expand Down Expand Up @@ -120,7 +120,7 @@ describeSuite({
it({
id: "C200",
title: `should have have ExtrinsicSuccess for all ethereum.transact`,
timeout: 30000,
timeout: TWO_MINS,
test: function () {
log(
`Checking ${blockData.reduce((curr, acc) => curr + acc.extrinsics.length, 0)}` +
Expand Down
70 changes: 42 additions & 28 deletions tests/smoke-tests/test-block-weights.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import "@moonbeam-network/api-augment";
import { BN } from "@polkadot/util";
import { GenericExtrinsic } from "@polkadot/types";
import { FrameSystemEventRecord } from "@polkadot/types/lookup";
import { AnyTuple } from "@polkadot/types/types";
import { expect } from "chai";
import { describeSmokeSuite } from "../util/setup-smoke-tests";
import { extractWeight, getBlockArray } from "../util/block";
import { WEIGHT_PER_GAS } from "../util/constants";
import { FrameSystemEventRecord } from "@polkadot/types/lookup";
import { rateLimiter } from "../util/common";
import { WEIGHT_PER_GAS } from "../util/constants";
import { describeSmokeSuite } from "../util/setup-smoke-tests";

const debug = require("debug")("smoke:weights");
const timePeriod = process.env.TIME_PERIOD ? Number(process.env.TIME_PERIOD) : 2 * 60 * 60 * 1000;
Expand All @@ -16,17 +17,17 @@ interface BlockInfo {
blockNum: number;
hash: string;
weights: {
normal: BN;
operational: BN;
mandatory: BN;
normal: bigint;
operational: bigint;
mandatory: bigint;
};
extrinsics;
extrinsics: GenericExtrinsic<AnyTuple>[];
events: FrameSystemEventRecord[];
}

interface BlockLimits {
normal: BN;
operational: BN;
normal: bigint;
operational: bigint;
}

describeSmokeSuite(
Expand Down Expand Up @@ -58,9 +59,9 @@ describeSmokeSuite(
blockNum,
hash: blockHash.toString(),
weights: {
normal: extractWeight(normal),
operational: extractWeight(operational),
mandatory: extractWeight(mandatory),
normal: extractWeight(normal).toBigInt(),
operational: extractWeight(operational).toBigInt(),
mandatory: extractWeight(mandatory).toBigInt(),
},
events,
extrinsics,
Expand All @@ -70,8 +71,8 @@ describeSmokeSuite(

// Support for weight v1 and weight v2.
blockLimits = {
normal: extractWeight(limits.perClass.normal.maxTotal).toBn(),
operational: extractWeight(limits.perClass.operational.maxTotal).toBn(),
normal: extractWeight(limits.perClass.normal.maxTotal).toBigInt(),
operational: extractWeight(limits.perClass.operational.maxTotal).toBigInt(),
};
blockInfoArray = await Promise.all(
blockNumArray.map((num) => limiter.schedule(() => getLimits(num)))
Expand All @@ -94,7 +95,7 @@ describeSmokeSuite(
`normal usage should be less than normal dispatch class limits`,
async function () {
const overweight = blockInfoArray
.filter((a) => a.weights.normal.gt(blockLimits.normal))
.filter((a) => a.weights.normal > blockLimits.normal)
.map((a) => {
debug(
`Block #${a.blockNum} has weight ${Number(a.weights.normal)} which is above limit!`
Expand All @@ -116,7 +117,7 @@ describeSmokeSuite(
`operational usage should be less than dispatch class limits`,
async function () {
const overweight = blockInfoArray
.filter((a) => a.weights.operational.gt(blockLimits.operational))
.filter((a) => a.weights.operational > blockLimits.operational)
.map((a) => {
debug(
`Block #${a.blockNum} has weight ${Number(
Expand Down Expand Up @@ -156,22 +157,35 @@ describeSmokeSuite(
const checkBlockWeight = async (blockInfo: BlockInfo) => {
const apiAt = await context.polkadotApi.at(blockInfo.hash);

const normalWeight = Number(blockInfo.weights.normal);
const normalWeight = blockInfo.weights.normal;
const maxWeight = blockLimits.normal;
const ethBlock = (await apiAt.query.ethereum.currentBlock()).unwrap();
const balTxns = blockInfo.extrinsics
.map((ext, index) =>
ext.method.method == "transfer" && ext.method.section == "balances" ? index : -1
)
.filter((a) => a != -1);
const balTxnWeights = blockInfo.events
.map((event) =>
context.polkadotApi.events.system.ExtrinsicSuccess.is(event.event) &&
event.phase.isApplyExtrinsic &&
balTxns.includes(event.phase.asApplyExtrinsic.toNumber())
? event.event.data.dispatchInfo.weight.refTime.toBigInt()
: 0n
)
.reduce((acc, curr) => acc + curr, 0n);
const actualWeightUsed = (normalWeight * 100n) / maxWeight;

const actualWeightUsed = normalWeight / Number(maxWeight);
if (actualWeightUsed > 0.2) {
if (actualWeightUsed > 20n) {
const gasUsed = ethBlock.header.gasUsed.toBigInt();
const weightCalc = gasUsed * WEIGHT_PER_GAS;
const newRatio = (normalWeight - Number(weightCalc)) / Number(maxWeight);
if (newRatio > 0.2) {
const newRatio = ((normalWeight - weightCalc - balTxnWeights) * 100n) / maxWeight;

if (newRatio > 20n) {
debug(
`Block #${blockInfo.blockNum} is ${(actualWeightUsed * 100).toFixed(
2
)}% full with ${
ethBlock.transactions.length
} transactions, non-transaction weight: ${(newRatio * 100).toFixed(2)}%`
`Block #${blockInfo.blockNum} is ${actualWeightUsed}% full with ` +
ethBlock.transactions.length +
` transactions, non-transaction weight: ${newRatio}%`
);
}
return { blockNum: blockInfo.blockNum, nonTxn: newRatio };
Expand All @@ -181,7 +195,7 @@ describeSmokeSuite(
const results = await Promise.all(
blockInfoArray.map((blockInfo) => limiter.schedule(() => checkBlockWeight(blockInfo)))
);
const nonTxnHeavyBlocks = results.filter((a) => a && a.nonTxn > 0.2);
const nonTxnHeavyBlocks = results.filter((a) => a && a.nonTxn > 20n);
expect(
nonTxnHeavyBlocks,
`These blocks have non-txn weights >20%, please investigate: ${nonTxnHeavyBlocks
Expand Down
3 changes: 2 additions & 1 deletion tests/smoke-tests/test-ethereum-failures.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { rateLimiter } from "../util/common";
import { FrameSystemEventRecord } from "@polkadot/types/lookup";
import { GenericExtrinsic } from "@polkadot/types";
import { AnyTuple } from "@polkadot/types/types";
import { TWO_MINS } from "../util/constants";
const debug = require("debug")("smoke:eth-failures");
const timePeriod = process.env.TIME_PERIOD ? Number(process.env.TIME_PERIOD) : 2 * 60 * 60 * 1000;
const timeout = Math.max(Math.floor(timePeriod / 12), 5000);
Expand Down Expand Up @@ -115,7 +116,7 @@ describeSmokeSuite(
// of ExtrinsicSuccess fired. Any Extrinsic.Failed events will be reported and mark the
// block for further investigation.
testIt("C200", `should have have ExtrinsicSuccess for all ethereum.transact`, function () {
this.timeout(30000);
this.timeout(TWO_MINS);
debug(
`Checking ${blockData.reduce((curr, acc) => curr + acc.extrinsics.length, 0)}` +
" eth extrinsics all have corresponding ExtrinsicSuccess events."
Expand Down

0 comments on commit 027b9df

Please sign in to comment.