Merge remote-tracking branch 'ArkEcosystem/core/develop' into db-roun…

…ds-id

* ArkEcosystem/core/develop:
  release: 2.4.13 (#2752)
  fix: do not purge transactions when a block is not accepted (#2751)
  release: 2.5.0-next.5 (#2749)
  fix(core-utils): grant access if the whitelist is empty (#2748)
  release: 2.5.0-next.4 (#2747)
  fix(core-blockchain): always attempt to download blocks after start (#2746)
  refactor(core-p2p): block when rate limit exceeded (#2745)
  perf(core-p2p): download serialized blocks (#2743)
  refactor(core-database-postgres): exit on unexpected error (#2744)
  chore(core-p2p): allow 2.4 and 2.5 nodes (#2716)

CHANGELOG.md

@@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Fixed
+
+-   Impose the same rate limit as the public API ([#2717])
+-   Add option to configure request timeouts for webhooks([#2710])
+-   Use CORE_API_DISABLED variable in defaults ([#2711])
+-   Always attempt to download blocks after start ([#2746])
+-   Possible database corruption when writing and deleting blocks ([#2707])
+-   Forget peer when socket is disconnected ([#2720])
+-   Off-by-one error when fetching blocks from peer ([#2733])
+-   Check for user confirmation in snapshot commands ([#2734])
+
+### Changed
+
+-   Download serialized blocks to improve performance ([#2743])
+-   Better peer block header check to improve performance ([#2719])
+-   Exit on unexpected database errors ([#2744])
+-   Block peers when the rate limit is exceeded ([#2745])
+-   Delay peer discovery until after state initialization is done ([#2727])
+-   Improved P2P rate limiting ([#2729])
+-   Only fetch block headers when verifying peers ([#2728])
+-   Only look for new peers when below minimum peers ([#2714])
+-   Always keep the Wallet API enabled ([#2715])
+-   Respect the whitelist of the public API ([#2718])
+-   Add foreign key on transactions block id ([#2671])
+
 ## [2.5.0] - 2019-07-DD
 
 ### Added
@@ -19,6 +44,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 -   Removed the `ark-node` legacy API known as `v1` ([#2577])
 
+## [2.4.13] - 2019-06-26
+
+### Changed
+
+-   Do not purge transactions when a block is not accepted ([#2751])
+
 ## [2.4.12] - 2019-06-14
 
 ### Changed
@@ -509,7 +540,8 @@ Closed security vulnerabilities:
 -   Initial Release
 
 [unreleased]: https://github.com/ARKEcosystem/core/compare/master...develop
-[2.5.0]: https://github.com/ARKEcosystem/core/compare/2.4.12...2.5.0
+[2.5.0]: https://github.com/ARKEcosystem/core/compare/2.4.13...2.5.0
+[2.4.13]: https://github.com/ARKEcosystem/core/compare/2.4.12...2.4.13
 [2.4.12]: https://github.com/ARKEcosystem/core/compare/2.4.1...2.4.12
 [2.4.1]: https://github.com/ARKEcosystem/core/compare/2.4.0...2.4.1
 [2.4.0]: https://github.com/ARKEcosystem/core/compare/2.3.23...2.4.0
@@ -704,6 +736,7 @@ Closed security vulnerabilities:
 [#2476]: https://github.com/ARKEcosystem/core/pull/2476
 [#2479]: https://github.com/ARKEcosystem/core/pull/2479
 [#2482]: https://github.com/ARKEcosystem/core/pull/2482
+[#2482]: https://github.com/ARKEcosystem/core/pull/2482
 [#2484]: https://github.com/ARKEcosystem/core/pull/2484
 [#2486]: https://github.com/ARKEcosystem/core/pull/2486
 [#2487]: https://github.com/ARKEcosystem/core/pull/2487
@@ -733,7 +766,6 @@ Closed security vulnerabilities:
 [#2555]: https://github.com/ARKEcosystem/core/pull/2555
 [#2557]: https://github.com/ARKEcosystem/core/pull/2557
 [#2558]: https://github.com/ARKEcosystem/core/pull/2558
-[#2482]: https://github.com/ARKEcosystem/core/pull/2482
 [#2559]: https://github.com/ARKEcosystem/core/pull/2559
 [#2562]: https://github.com/ARKEcosystem/core/pull/2562
 [#2563]: https://github.com/ARKEcosystem/core/pull/2563
@@ -775,6 +807,7 @@ Closed security vulnerabilities:
 [#2665]: https://github.com/ARKEcosystem/core/pull/2665
 [#2666]: https://github.com/ARKEcosystem/core/pull/2666
 [#2670]: https://github.com/ARKEcosystem/core/pull/2670
+[#2671]: https://github.com/ARKEcosystem/core/pull/2671
 [#2672]: https://github.com/ARKEcosystem/core/pull/2672
 [#2674]: https://github.com/ARKEcosystem/core/pull/2674
 [#2685]: https://github.com/ARKEcosystem/core/pull/2685
@@ -783,3 +816,22 @@ Closed security vulnerabilities:
 [#2692]: https://github.com/ARKEcosystem/core/pull/2692
 [#2699]: https://github.com/ARKEcosystem/core/pull/2699
 [#2700]: https://github.com/ARKEcosystem/core/pull/2700
+[#2707]: https://github.com/ARKEcosystem/core/pull/2707
+[#2710]: https://github.com/ARKEcosystem/core/pull/2710
+[#2711]: https://github.com/ARKEcosystem/core/pull/2711
+[#2714]: https://github.com/ARKEcosystem/core/pull/2714
+[#2715]: https://github.com/ARKEcosystem/core/pull/2715
+[#2717]: https://github.com/ARKEcosystem/core/pull/2717
+[#2718]: https://github.com/ARKEcosystem/core/pull/2718
+[#2719]: https://github.com/ARKEcosystem/core/pull/2719
+[#2720]: https://github.com/ARKEcosystem/core/pull/2720
+[#2727]: https://github.com/ARKEcosystem/core/pull/2727
+[#2728]: https://github.com/ARKEcosystem/core/pull/2728
+[#2729]: https://github.com/ARKEcosystem/core/pull/2729
+[#2733]: https://github.com/ARKEcosystem/core/pull/2733
+[#2734]: https://github.com/ARKEcosystem/core/pull/2734
+[#2743]: https://github.com/ARKEcosystem/core/pull/2743
+[#2744]: https://github.com/ARKEcosystem/core/pull/2744
+[#2745]: https://github.com/ARKEcosystem/core/pull/2745
+[#2746]: https://github.com/ARKEcosystem/core/pull/2746
+[#2751]: https://github.com/ARKEcosystem/core/pull/2751

__tests__/integration/core-p2p/peer-communicator.test.ts

@@ -64,7 +64,7 @@ describe("PeerCommunicator", () => {
 
     describe("downloadBlocks", () => {
         it("should be ok", async () => {
-            await socketManager.addMock("getBlocks", [{ height: 1, id: "1" }, { height: 2, id: "2" }]);
+            await socketManager.addMock("getBlocks", [genesisBlockJSON, genesisBlockJSON]);
 
             const blocks = await communicator.downloadBlocks(stubPeer, 1);
 

__tests__/integration/core-p2p/socket-server/peer.test.ts

@@ -129,14 +129,14 @@ describe("Peer socket endpoint", () => {
     describe("Socket errors", () => {
         it("should accept the request when below rate limit", async () => {
             await delay(1000);
-            for (let i = 0; i < 18; i++) {
+            for (let i = 0; i < 2; i++) {
                 const { data } = await emit("p2p.peer.getStatus", {
                     headers,
                 });
                 expect(data.state.height).toBeNumber();
             }
             await delay(1100);
-            for (let i = 0; i < 10; i++) {
+            for (let i = 0; i < 2; i++) {
                 const { data } = await emit("p2p.peer.getStatus", {
                     headers,
                 });
@@ -146,7 +146,7 @@ describe("Peer socket endpoint", () => {
 
         it("should cancel the request when exceeding rate limit", async () => {
             await delay(1000);
-            for (let i = 0; i < 100; i++) {
+            for (let i = 0; i < 2; i++) {
                 const { data } = await emit("p2p.peer.getStatus", {
                     headers,
                 });

__tests__/unit/core-blockchain/mocks/database.ts

@@ -16,6 +16,7 @@ export const database = {
     getActiveDelegates: () => [],
     restoreCurrentRound: () => undefined,
     getBlocks: () => [],
+    getBlocksForDownload: () => [],
     getBlock: () => undefined,
     revertBlock: () => undefined,
     applyBlock: () => undefined,

__tests__/unit/core-blockchain/mocks/transactionPool.ts

@@ -1,7 +1,5 @@
 export const transactionPool = {
     buildWallets: () => undefined,
-    purgeByBlock: () => undefined,
     acceptChainedBlock: () => undefined,
-    purgeSendersWithInvalidTransactions: () => undefined,
     removeTransactionsById: () => undefined,
 };

__tests__/unit/core-p2p/socket-server/versions/peer/index.test.ts

@@ -4,8 +4,8 @@ import { blockchain } from "../../../mocks/blockchain";
 import { database } from "../../../mocks/database";
 
 import { Crypto } from "@arkecosystem/crypto";
+import { acceptNewPeer } from "../../../../../../packages/core-p2p/src/socket-server/versions/internal";
 import {
-    acceptNewPeer,
     getBlocks,
     getCommonBlocks,
     getPeers,

__tests__/unit/core-p2p/utils/is-valid-version.test.ts

@@ -22,7 +22,6 @@ describe("isValidVersion", () => {
         expect(isValidVersion({ ...peerMock, ...{ version: "2.8.0" } })).toBeTrue();
         expect(isValidVersion({ ...peerMock, ...{ version: "2.9.0" } })).toBeTrue();
         expect(isValidVersion({ ...peerMock, ...{ version: "2.9.934" } })).toBeTrue();
-        expect(isValidVersion({ ...peerMock, ...{ version: "3.0.0" } })).toBeTrue();
     });
 
     it("should be an invalid version", () => {

__tests__/unit/core-state/mocks/database.ts

@@ -15,6 +15,7 @@ export const database = {
     getActiveDelegates: () => [],
     restoreCurrentRound: () => undefined,
     getBlocks: () => [],
+    getBlocksForDownload: () => [],
     getBlock: () => undefined,
     revertBlock: () => undefined,
     applyBlock: () => undefined,

__tests__/unit/core-state/mocks/transactionPool.ts

@@ -1,6 +1,4 @@
 export const transactionPool = {
     buildWallets: () => undefined,
-    purgeByBlock: () => undefined,
     acceptChainedBlock: () => undefined,
-    purgeSendersWithInvalidTransactions: () => undefined,
 };

__tests__/unit/core-transaction-pool/__stubs__/connection.ts

@@ -130,14 +130,6 @@ export class Connection implements TransactionPool.IConnection {
         return;
     }
 
-    public purgeSendersWithInvalidTransactions(block: Blocks.Block): void {
-        return;
-    }
-
-    public purgeByBlock(block: Blocks.Block): void {
-        return;
-    }
-
     public senderHasTransactionsOfType(senderPublicKey: string, transactionType: Enums.TransactionTypes): boolean {
         return true;
     }

__tests__/unit/core-transaction-pool/connection.test.ts

@@ -1004,65 +1004,6 @@ describe("Connection", () => {
         });
     });
 
-    describe("purgeSendersWithInvalidTransactions", () => {
-        it("should purge transactions from sender when invalid", async () => {
-            const transfersA = TransactionFactory.transfer(mockData.dummy1.data.recipientId)
-                .withNetwork("unitnet")
-                .withPassphrase(delegatesSecrets[0])
-                .build(5);
-
-            const transfersB = TransactionFactory.transfer(mockData.dummy1.data.recipientId)
-                .withNetwork("unitnet")
-                .withPassphrase(delegatesSecrets[1])
-                .build();
-
-            const block = {
-                transactions: [...transfersA, ...transfersB],
-            } as any;
-
-            addTransactions(block.transactions);
-
-            expect(connection.getPoolSize()).toBe(6);
-
-            // Last tx has a unique sender
-            block.transactions[5].isVerified = false;
-
-            connection.purgeSendersWithInvalidTransactions(block);
-            expect(connection.getPoolSize()).toBe(5);
-
-            // The remaining tx all have the same sender
-            block.transactions[0].isVerified = false;
-
-            connection.purgeSendersWithInvalidTransactions(block);
-            expect(connection.getPoolSize()).toBe(0);
-        });
-    });
-
-    describe("purgeByBlock", () => {
-        it("should purge transactions from block", async () => {
-            const revertTransactionForSender = jest
-                .spyOn(connection.walletManager, "revertTransactionForSender")
-                .mockReturnValue();
-
-            const transactions = TransactionFactory.transfer(mockData.dummy1.data.recipientId)
-                .withNetwork("unitnet")
-                .withPassphrase(delegatesSecrets[0])
-                .build(5);
-
-            const block = { transactions } as Blocks.Block;
-
-            addTransactions(block.transactions);
-
-            expect(connection.getPoolSize()).toBe(5);
-
-            connection.purgeByBlock(block);
-            expect(revertTransactionForSender).toHaveBeenCalledTimes(5);
-            expect(connection.getPoolSize()).toBe(0);
-
-            jest.restoreAllMocks();
-        });
-    });
-
     describe("purgeInvalidTransactions", () => {
         it("should flush the pool", () => {
             // 64 char vendor field

__tests__/unit/core-utils/is-whitelisted.test.ts

@@ -0,0 +1,23 @@
+import "jest-extended";
+
+import { isWhitelisted } from "../../../packages/core-utils/src";
+
+describe("isWhitelisted", () => {
+    it("should allow everyone", () => {
+        expect(isWhitelisted(["*"], "127.0.0.1")).toBeTrue();
+        expect(isWhitelisted(["*"], "192.168.1.1")).toBeTrue();
+        expect(isWhitelisted(["*"], "168.1.1.1")).toBeTrue();
+    });
+
+    it("should allow addresses with prefixes", () => {
+        expect(isWhitelisted(["127.*"], "127.0.0.1")).toBeTrue();
+        expect(isWhitelisted(["127.*"], "127.0.0.2")).toBeTrue();
+        expect(isWhitelisted(["127.*"], "128.0.0.1")).toBeFalse();
+    });
+
+    it("should allow addresses with suffixes", () => {
+        expect(isWhitelisted(["*.127"], "1.1.1.127")).toBeTrue();
+        expect(isWhitelisted(["*.127"], "1.1.1.127")).toBeTrue();
+        expect(isWhitelisted(["*.127"], "1.1.1.128")).toBeFalse();
+    });
+});

__tests__/unit/crypto/validation/validator.test.ts

@@ -217,6 +217,10 @@ describe("validator", () => {
                 expect(validator.validate("block", block2).error).not.toBeUndefined();
                 block2.numberOfTransactions = 10;
                 expect(validator.validate("block", block2).error).toBeUndefined();
+                block2.transactions[0] = {} as any;
+                expect(validator.validate("block", block2).error).not.toBeUndefined();
+                block2.transactions[0] = 1234 as any;
+                expect(validator.validate("block", block2).error).not.toBeUndefined();
             });
         });
     });

lerna.json

@@ -3,5 +3,5 @@
     "npmClient": "yarn",
     "packages": ["packages/*", "plugins/*"],
     "useWorkspaces": true,
-    "version": "2.5.0-next.3"
+    "version": "2.5.0-next.5"
 }

packages/core-api/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@arkecosystem/core-api",
-    "version": "2.5.0-next.3",
+    "version": "2.5.0-next.5",
     "description": "Public API for ARK Core",
     "license": "MIT",
     "contributors": [
@@ -21,12 +21,12 @@
         "pretest": "bash ../../scripts/pre-test.sh"
     },
     "dependencies": {
-        "@arkecosystem/core-container": "^2.5.0-next.3",
-        "@arkecosystem/core-http-utils": "^2.5.0-next.3",
-        "@arkecosystem/core-interfaces": "^2.5.0-next.3",
-        "@arkecosystem/core-transaction-pool": "^2.5.0-next.3",
-        "@arkecosystem/core-utils": "^2.5.0-next.3",
-        "@arkecosystem/crypto": "^2.5.0-next.3",
+        "@arkecosystem/core-container": "^2.5.0-next.5",
+        "@arkecosystem/core-http-utils": "^2.5.0-next.5",
+        "@arkecosystem/core-interfaces": "^2.5.0-next.5",
+        "@arkecosystem/core-transaction-pool": "^2.5.0-next.5",
+        "@arkecosystem/core-utils": "^2.5.0-next.5",
+        "@arkecosystem/crypto": "^2.5.0-next.5",
         "@arkecosystem/utils": "^0.3.0",
         "@hapi/boom": "^7.4.2",
         "@hapi/joi": "^15.0.3",

packages/core-api/src/defaults.ts

@@ -60,6 +60,6 @@ export const defaults = {
             "/api/wallets/search",
         ],
     },
-    whitelist: ["127.0.0.1", "::ffff:127.0.0.1", "192.168.*"],
+    whitelist: ["*"],
     plugins: [],
 };

packages/core-blockchain/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@arkecosystem/core-blockchain",
-    "version": "2.5.0-next.3",
+    "version": "2.5.0-next.5",
     "description": "Blockchain Manager for ARK Core",
     "license": "MIT",
     "contributors": [
@@ -22,14 +22,14 @@
         "pretest": "bash ../../scripts/pre-test.sh"
     },
     "dependencies": {
-        "@arkecosystem/core-container": "^2.5.0-next.3",
-        "@arkecosystem/core-database": "^2.5.0-next.3",
-        "@arkecosystem/core-event-emitter": "^2.5.0-next.3",
-        "@arkecosystem/core-interfaces": "^2.5.0-next.3",
-        "@arkecosystem/core-state": "^2.5.0-next.3",
-        "@arkecosystem/core-transactions": "^2.5.0-next.3",
-        "@arkecosystem/core-utils": "^2.5.0-next.3",
-        "@arkecosystem/crypto": "^2.5.0-next.3",
+        "@arkecosystem/core-container": "^2.5.0-next.5",
+        "@arkecosystem/core-database": "^2.5.0-next.5",
+        "@arkecosystem/core-event-emitter": "^2.5.0-next.5",
+        "@arkecosystem/core-interfaces": "^2.5.0-next.5",
+        "@arkecosystem/core-state": "^2.5.0-next.5",
+        "@arkecosystem/core-transactions": "^2.5.0-next.5",
+        "@arkecosystem/core-utils": "^2.5.0-next.5",
+        "@arkecosystem/crypto": "^2.5.0-next.5",
         "async": "^3.0.0",
         "delay": "^4.2.0",
         "immutable": "^4.0.0-rc.12",
@@ -39,7 +39,7 @@
         "xstate": "^4.6.0"
     },
     "devDependencies": {
-        "@arkecosystem/core-p2p": "^2.5.0-next.3",
+        "@arkecosystem/core-p2p": "^2.5.0-next.5",
         "@types/async": "^2.4.2",
         "@types/lodash.get": "^4.4.6",
         "@types/pluralize": "^0.0.29",

packages/core-blockchain/src/blockchain.ts

@@ -498,7 +498,7 @@ export class Blockchain implements blockchain.IBlockchain {
      * Get the last downloaded block of the blockchain.
      */
     public getLastDownloadedBlock(): Interfaces.IBlockData {
-        return this.state.lastDownloadedBlock;
+        return this.state.lastDownloadedBlock || this.getLastBlock().data;
     }
 
     /**