Merge branch 'develop' into feat/enforce-chained-db-blocks

packages/core-database-postgres/src/migrations/20190619000000-drop-id-column-from-rounds-table.sql

@@ -0,0 +1,2 @@
+ALTER TABLE rounds DROP COLUMN id, ADD PRIMARY KEY (round, public_key);
+DROP INDEX rounds_unique;

packages/core-database-postgres/src/migrations/index.ts

@@ -14,5 +14,6 @@ export const migrations = [
     loadQueryFile(__dirname, "./20190307000000-drop-wallets-table.sql"),
     loadQueryFile(__dirname, "./20190313000000-add-asset-column-to-transactions-table.sql"),
     loadQueryFile(__dirname, "./20190606000000-add-block-id-foreign-key-on-transactions.sql"),
+    loadQueryFile(__dirname, "./20190619000000-drop-id-column-from-rounds-table.sql"),
     loadQueryFile(__dirname, "./20190626000000-enforce-chained-blocks.sql"),
 ];

packages/core-snapshots/src/db/index.ts

@@ -35,11 +35,11 @@ export class Database {
     }
 
     /**
-     * Get the row with the highest id from the rounds table.
+     * Get the highest row from the rounds table.
      * @return Object latest row
      * @return null if the table is empty.
      */
-    public async getLastRound(): Promise<{ id: number; public_key: string; balance: string; round: string } | null> {
+    public async getLastRound(): Promise<{ public_key: string; balance: string; round: string } | null> {
         return this.db.oneOrNone(queries.rounds.latest);
     }
 
@@ -85,7 +85,7 @@ export class Database {
     public async getExportQueries(meta: {
         startHeight: number;
         endHeight: number;
-        startRoundId: number;
+        skipRoundRows: number;
         skipCompression: boolean;
         folder: string;
     }) {
@@ -113,7 +113,7 @@ export class Database {
             rounds: rawQuery(this.pgp, queries.rounds.roundRange, {
                 startRound: roundInfoStart.round,
                 endRound: roundInfoEnd.round,
-                startId: meta.startRoundId,
+                skipRoundRows: meta.skipRoundRows,
             }),
         };
     }
@@ -178,9 +178,14 @@ export class Database {
             { table: "transactions" },
         );
 
-        this.roundsColumnSet = new this.pgp.helpers.ColumnSet(["id", "public_key", "balance", "round"], {
-            table: "rounds",
-        });
+        this.roundsColumnSet = new this.pgp.helpers.ColumnSet(
+            [
+                "round",
+                "balance",
+                "public_key"
+            ],
+            { table: "rounds" }
+        );
     }
 }
 

packages/core-snapshots/src/db/queries/rounds/latest.sql

@@ -1,3 +1,4 @@
 SELECT *
 FROM rounds
-ORDER BY id DESC LIMIT 1
+ORDER BY round DESC, balance, public_key DESC
+LIMIT 1

packages/core-snapshots/src/db/queries/rounds/round-range.sql

@@ -1,12 +1,12 @@
 SELECT
-  id,
-  public_key,
+  round,
   balance,
-  round
+  public_key
 FROM
   rounds
 WHERE
-  round BETWEEN ${startRound} AND ${endRound} AND
-  id >= ${startId}
+  round BETWEEN ${startRound} AND ${endRound}
 ORDER BY
-  id
+  round, balance DESC, public_key
+OFFSET
+  ${skipRoundRows}

packages/core-snapshots/src/manager.ts

@@ -71,8 +71,6 @@ export class SnapshotManager {
             );
         }
 
-        await this.database.db.one("SELECT setval('rounds_id_seq', (SELECT MAX(id) FROM rounds) + 1)");
-
         this.database.close();
     }
 

packages/core-snapshots/src/transport/codec.ts

@@ -59,14 +59,13 @@ const decodeTransaction = (buffer: Buffer) => {
 };
 
 const encodeRound = round => {
-    return encode([round.id, round.public_key || round.publicKey, round.balance, round.round]);
+    return encode([round.public_key || round.publicKey, round.balance, round.round]);
 };
 
 const decodeRound = (buffer: Buffer) => {
-    const [id, publicKey, balance, round] = decode(buffer);
+    const [publicKey, balance, round] = decode(buffer);
 
     return decamelizeKeys({
-        id,
         publicKey,
         balance,
         round,

packages/core-snapshots/src/transport/index.ts

@@ -7,6 +7,7 @@ import zlib from "zlib";
 
 import { app } from "@arkecosystem/core-container";
 import { EventEmitter, Logger } from "@arkecosystem/core-interfaces";
+import { Managers } from "@arkecosystem/crypto";
 
 import * as utils from "../utils";
 import { Codec } from "./codec";
@@ -87,6 +88,10 @@ export const importTable = async (table, options) => {
     for await (const record of readStream) {
         counter++;
 
+        if (table === "blocks" && record.height === 1) {
+            record.id = Managers.configManager.get("genesisBlock").id;
+        }
+
         if (!verifyData(table, record, prevData, options.verifySignatures)) {
             app.forceExit(`Error verifying data. Payload ${JSON.stringify(record, undefined, 2)}`);
         }

packages/core-snapshots/src/transport/verification.ts

@@ -1,6 +1,6 @@
 import { app } from "@arkecosystem/core-container";
 import { Logger } from "@arkecosystem/core-interfaces";
-import { Blocks, Crypto, Managers, Transactions } from "@arkecosystem/crypto";
+import { Blocks, Crypto, Transactions, Utils } from "@arkecosystem/crypto";
 import { camelizeKeys } from "xcase";
 
 export const verifyData = (context, data, prevData, verifySignatures) => {
@@ -9,14 +9,6 @@ export const verifyData = (context, data, prevData, verifySignatures) => {
             if (!prevData) {
                 return true;
             }
-            // genesis payload different as block.serialize stores
-            // block.previous_block with 00000 instead of undefined
-            // it fails on height 2 - chain check
-            // TODO: check to improve ser/deser for genesis
-            const genesisBlock = Managers.configManager.get("genesisBlock");
-            if (data.height === 2 && data.previous_block === genesisBlock.id) {
-                return true;
-            }
 
             return data.height - prevData.height === 1 && data.previous_block === prevData.id;
         };
@@ -81,7 +73,30 @@ export const canImportRecord = (context, data, options) => {
         if (options.lastRound === null) {
             return true;
         }
-        return data.id > options.lastRound.id;
+
+        const dataRound = Number(data.round);
+        const lastRound = Number(options.lastRound.round);
+        if (dataRound > lastRound) {
+            return true;
+        }
+        if (dataRound < lastRound) {
+            return false;
+        }
+
+        const dataBalance = Utils.BigNumber.make(data.balance);
+        const lastBalance = Utils.BigNumber.make(options.lastRound.balance);
+        if (dataBalance.isLessThan(lastBalance)) {
+            return true;
+        }
+        if (dataBalance.isGreaterThan(lastBalance)) {
+            return false;
+        }
+
+        if (data.public_key > options.lastRound.publicKey) {
+            return true;
+        }
+
+        return false;
     }
 
     return false;

packages/core-snapshots/src/utils.ts

@@ -90,7 +90,7 @@ export const setSnapshotInfo = (options, lastBlock) => {
     const meta = {
         startHeight: options.start !== -1 ? options.start : 1,
         endHeight: options.end !== -1 ? options.end : lastBlock.height,
-        startRoundId: 1,
+        skipRoundRows: 0,
         skipCompression: options.skipCompression || false,
         folder: "",
     };
@@ -100,7 +100,7 @@ export const setSnapshotInfo = (options, lastBlock) => {
     if (options.blocks) {
         const oldMeta = this.getSnapshotInfo(options.blocks);
         meta.startHeight = oldMeta.endHeight + 1;
-        meta.startRoundId = oldMeta.rounds.count + 1;
+        meta.skipRoundRows = oldMeta.rounds.count;
         meta.folder = `${oldMeta.startHeight}-${meta.endHeight}`;
     }