Skip to content

Commit

Permalink
feat(core-api): make it configurable whether to use estimates
Browse files Browse the repository at this point in the history 
Introduce a new boolean config option totalCountIsEstimate and use
database estimates for the total number of rows if it is true (fast)
or use the precise COUNT(*) if the option is false (slow).

So, it is up to the node operator to configure their node for accuracy
vs speed. Add a new property in the response to indicate which one is
being used.

Resolves #2676
Pagination error in api/blocks/{id}/transactions endpoint
  • Loading branch information
@vasild
vasild committed Jul 4, 2019
1 parent 7de758b commit 30658ee
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 44 deletions.
7 changes: 6 additions & 1 deletion packages/core-api/src/handlers/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@ export const respondWithCache = (data, h): any => {

return value.isBoom
? h.response(value.output.payload).code(value.output.statusCode)
: h.response(value).header("Last-modified", lastModified.toUTCString());
: h.response({
results: value.results,
totalCount: value.totalCount,
response: { totalCountIsEstimate: value.totalCountIsEstimate }
}).header("Last-modified", lastModified.toUTCString());
};

export const toResource = (data, transformer, transform: boolean = true): object => {
Expand All @@ -50,5 +54,6 @@ export const toPagination = (data, transformer, transform: boolean = true): obje
return {
results: transformerService.toCollection(data.rows, transformer, transform),
totalCount: data.count,
totalCountIsEstimate: data.countIsEstimate,
};
};
1 change: 1 addition & 0 deletions packages/core-database-postgres/src/defaults.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ export const defaults = {
user: process.env.CORE_DB_USERNAME || process.env.CORE_TOKEN,
password: process.env.CORE_DB_PASSWORD || "password",
},
totalCountIsEstimate: true,
};
6 changes: 4 additions & 2 deletions packages/core-database-postgres/src/postgres-connection.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,10 @@ export class PostgresConnection implements Database.IConnection {
public async connect(): Promise<void> {
this.emitter.emit(Database.DatabaseEvents.PRE_CONNECT);

const options = this.options;

const pgp: pgPromise.IMain = pgPromise({
...this.options.initialization,
...options.initialization,
...{
error: async (error, context) => {
// https://www.postgresql.org/docs/11/errcodes-appendix.html
Expand All @@ -85,7 +87,7 @@ export class PostgresConnection implements Database.IConnection {
},
extend(object) {
for (const repository of Object.keys(repositories)) {
object[repository] = new repositories[repository](object, pgp);
object[repository] = new repositories[repository](object, pgp, options);
}
},
},
Expand Down
11 changes: 8 additions & 3 deletions packages/core-database-postgres/src/repositories/blocks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export class BlocksRepository extends Repository implements Database.IBlocksRepo
public async search(params: Database.ISearchParameters): Promise<{ rows: Interfaces.IBlockData[]; count: number }> {
// TODO: we're selecting all the columns right now. Add support for choosing specific columns, when it proves useful.
const selectQuery = this.query.select().from(this.query);
const selectQueryCount = this.query.select(this.query.count().as("cnt")).from(this.query);
// Blocks repo atm, doesn't search using any custom parameters
const parameterList = params.parameters.filter(o => o.operator !== Database.SearchOperator.OP_CUSTOM);

Expand All @@ -19,14 +20,18 @@ export class BlocksRepository extends Repository implements Database.IBlocksRepo
} while (!first.operator && parameterList.length);

if (first) {
selectQuery.where(this.query[this.propToColumnName(first.field)][first.operator](first.value));
for (const q of [ selectQuery, selectQueryCount ]) {
q.where(this.query[this.propToColumnName(first.field)][first.operator](first.value));
}
for (const param of parameterList) {
selectQuery.and(this.query[this.propToColumnName(param.field)][param.operator](param.value));
for (const q of [ selectQuery, selectQueryCount ]) {
q.and(this.query[this.propToColumnName(param.field)][param.operator](param.value));
}
}
}
}

return this.findManyWithCount(selectQuery, params.paginate, params.orderBy);
return this.findManyWithCount(selectQuery, selectQueryCount, params.paginate, params.orderBy);
}

public async findById(id: string): Promise<Interfaces.IBlockData> {
Expand Down
50 changes: 29 additions & 21 deletions packages/core-database-postgres/src/repositories/repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { Model } from "../models";
export abstract class Repository implements Database.IRepository {
protected model: Model;

constructor(protected readonly db, protected readonly pgp: IMain) {
constructor(protected readonly db, protected readonly pgp: IMain, private readonly options) {
this.model = this.getModel();
}

Expand Down Expand Up @@ -50,9 +50,10 @@ export abstract class Repository implements Database.IRepository {

protected async findManyWithCount<T = any>(
selectQuery: Query<any>,
selectQueryCount: Query<any>,
paginate?: Database.ISearchPaginate,
orderBy?: Database.ISearchOrderBy[],
): Promise<{ rows: T; count: number }> {
): Promise<{ rows: T; count: number, countIsEstimate: boolean }> {
if (!!orderBy) {
for (const o of orderBy) {
const column = this.query.columns.find(column => column.prop.toLowerCase() === o.field);
Expand All @@ -68,36 +69,43 @@ export abstract class Repository implements Database.IRepository {
// tslint:disable-next-line:no-shadowed-variable
const rows = await this.findMany(selectQuery);

return { rows, count: rows.length };
return { rows, count: rows.length, countIsEstimate: false };
}

selectQuery.offset(paginate.offset).limit(paginate.limit);

const rows = await this.findMany(selectQuery);

if (rows.length < paginate.limit) {
return { rows, count: paginate.offset + rows.length };
return { rows, count: paginate.offset + rows.length, countIsEstimate: false };
}

// Get the last rows=... from something that looks like (1 column, few rows):
//
// QUERY PLAN
// ------------------------------------------------------------------
// Limit (cost=15.34..15.59 rows=100 width=622)
// -> Sort (cost=15.34..15.64 rows=120 width=622)
// Sort Key: "timestamp" DESC
// -> Seq Scan on transactions (cost=0.00..11.20 rows=120 width=622)

let count: number = 0;
const explainedQuery = await this.db.manyOrNone(`EXPLAIN ${selectQuery.toString()}`);
for (const row of explainedQuery) {
const line: any = Object.values(row)[0];
const match = line.match(/rows=([0-9]+)/);
if (match) {
count = Number(match[1]);
// console.error(`findManyWithCount(): this.options.totalCountIsEstimate=${this.options.totalCountIsEstimate}`);
if (this.options.totalCountIsEstimate) {
// Get the last rows=... from something that looks like (1 column, few rows):
//
// QUERY PLAN
// ------------------------------------------------------------------
// Limit (cost=15.34..15.59 rows=100 width=622)
// -> Sort (cost=15.34..15.64 rows=120 width=622)
// Sort Key: "timestamp" DESC
// -> Seq Scan on transactions (cost=0.00..11.20 rows=120 width=622)

let count: number = 0;
const explainedQuery = await this.db.manyOrNone(`EXPLAIN ${selectQuery.toString()}`);
for (const row of explainedQuery) {
const line: any = Object.values(row)[0];
const match = line.match(/rows=([0-9]+)/);
if (match) {
count = Number(match[1]);
}
}

return { rows, count: Math.max(count, rows.length), countIsEstimate: true };
}

return { rows, count: Math.max(count, rows.length) };
const countRow = await this.find(selectQueryCount);

return { rows, count: Number(countRow.cnt), countIsEstimate: false };
}
}
43 changes: 26 additions & 17 deletions packages/core-database-postgres/src/repositories/transactions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export class TransactionsRepository extends Repository implements Database.ITran
}

const selectQuery = this.query.select().from(this.query);
const selectQueryCount = this.query.select(this.query.count().as("cnt")).from(this.query);
const params = parameters.parameters;

if (params.length) {
Expand All @@ -28,34 +29,42 @@ export class TransactionsRepository extends Repository implements Database.ITran

if (participants.length > 0) {
const [first, last] = participants;
selectQuery.where(this.query[this.propToColumnName(first.field)][first.operator](first.value));
for (const q of [ selectQuery, selectQueryCount ]) {
q.where(this.query[this.propToColumnName(first.field)][first.operator](first.value));
}

if (last) {
const usesInOperator = participants.every(
condition => condition.operator === Database.SearchOperator.OP_IN,
);

if (usesInOperator) {
selectQuery.or(this.query[this.propToColumnName(last.field)][last.operator](last.value));
for (const q of [ selectQuery, selectQueryCount ]) {
q.or(this.query[this.propToColumnName(last.field)][last.operator](last.value));
}
} else {
// This search is 1 `senderPublicKey` and 1 `recipientId`
selectQuery.and(this.query[this.propToColumnName(last.field)][last.operator](last.value));
for (const q of [ selectQuery, selectQueryCount ]) {
q.and(this.query[this.propToColumnName(last.field)][last.operator](last.value));
}
}
}
} else if (rest.length) {
const first = rest.shift();

selectQuery.where(this.query[this.propToColumnName(first.field)][first.operator](first.value));
for (const q of [ selectQuery, selectQueryCount ]) {
q.where(this.query[this.propToColumnName(first.field)][first.operator](first.value));
}
}

for (const condition of rest) {
selectQuery.and(
this.query[this.propToColumnName(condition.field)][condition.operator](condition.value),
);
for (const q of [ selectQuery, selectQueryCount ]) {
q.and(this.query[this.propToColumnName(condition.field)][condition.operator](condition.value));
}
}
}

return this.findManyWithCount(selectQuery, parameters.paginate, parameters.orderBy);
return this.findManyWithCount(selectQuery, selectQueryCount, parameters.paginate, parameters.orderBy);
}

public async findById(id: string): Promise<Interfaces.ITransactionData> {
Expand Down Expand Up @@ -144,15 +153,15 @@ export class TransactionsRepository extends Repository implements Database.ITran
paginate?: Database.ISearchPaginate,
orderBy?: Database.ISearchOrderBy[],
): Promise<Database.ITransactionsPaginated> {
return this.findManyWithCount(
this.query
.select()
.from(this.query)
.where(this.query.sender_public_key.equals(wallet.publicKey))
.or(this.query.recipient_id.equals(wallet.address)),
paginate,
orderBy,
);
const selectQuery = this.query.select();
const selectQueryCount = this.query.select(this.query.count().as("cnt"));

for (const q of [ selectQuery, selectQueryCount ]) {
q.from(this.query).where(this.query.sender_public_key.equals(wallet.publicKey))
.or(this.query.recipient_id.equals(wallet.address));
}

return this.findManyWithCount(selectQuery, selectQueryCount, paginate, orderBy);
}

public getModel(): Transaction {
Expand Down

0 comments on commit 30658ee

Please sign in to comment.