fix(@aws-amplify/datastore): improve query performance

packages/datastore/__tests__/DataStore.ts

@@ -1,5 +1,4 @@
 import 'fake-indexeddb/auto';
-import FDBCursor from 'fake-indexeddb/build/FDBCursor';
 import { decodeTime } from 'ulid';
 import uuidValidate from 'uuid-validate';
 import Observable from 'zen-observable-ts';

packages/datastore/__tests__/IndexedDBAdapter.test.ts

@@ -0,0 +1,99 @@
+import Adapter from '../src/storage/adapter/IndexedDBAdapter';
+import 'fake-indexeddb/auto';
+import {
+	DataStore as DataStoreType,
+	initSchema as initSchemaType,
+} from '../src/datastore/datastore';
+import { PersistentModelConstructor, SortDirection } from '../src/types';
+import { Model, testSchema } from './helpers';
+import { Predicates } from '../src/predicates';
+
+let initSchema: typeof initSchemaType;
+let DataStore: typeof DataStoreType;
+// using any to get access to private methods
+const IDBAdapter = <any>Adapter;
+
+describe('IndexedDBAdapter tests', () => {
+	describe('Query', () => {
+		let Model: PersistentModelConstructor<Model>;
+		const spyOnGetAll = jest.spyOn(IDBAdapter, 'getAll');
+		const spyOnEngine = jest.spyOn(IDBAdapter, 'enginePagination');
+		const spyOnMemory = jest.spyOn(IDBAdapter, 'inMemoryPagination');
+
+		beforeAll(async () => {
+			({ initSchema, DataStore } = require('../src/datastore/datastore'));
+
+			const classes = initSchema(testSchema());
+
+			({ Model } = classes as {
+				Model: PersistentModelConstructor<Model>;
+			});
+
+			await DataStore.save(
+				new Model({
+					field1: 'Some value',
+					dateCreated: new Date().toISOString(),
+				})
+			);
+			await DataStore.save(
+				new Model({
+					field1: 'another value',
+					dateCreated: new Date().toISOString(),
+				})
+			);
+			await DataStore.save(
+				new Model({
+					field1: 'a third value',
+					dateCreated: new Date().toISOString(),
+				})
+			);
+		});
+
+		test('Should call getAll & inMemoryPagination for query with a predicate', async () => {
+			jest.clearAllMocks();
+			const results = await DataStore.query(Model, c =>
+				c.field1('eq', 'another value')
+			);
+
+			expect(results.length).toEqual(1);
+			expect(spyOnGetAll).toHaveBeenCalled();
+			expect(spyOnEngine).not.toHaveBeenCalled();
+			expect(spyOnMemory).toHaveBeenCalled();
+		});
+
+		test('Should call getAll & inMemoryPagination for query with sort', async () => {
+			jest.clearAllMocks();
+			const results = await DataStore.query(Model, Predicates.ALL, {
+				sort: s => s.dateCreated(SortDirection.DESCENDING),
+			});
+
+			expect(results.length).toEqual(3);
+			expect(results[0].field1).toEqual('a third value');
+			expect(spyOnGetAll).toHaveBeenCalled();
+			expect(spyOnEngine).not.toHaveBeenCalled();
+			expect(spyOnMemory).toHaveBeenCalled();
+		});
+
+		test('Should call enginePagination for query with pagination but no sort or predicate', async () => {
+			jest.clearAllMocks();
+			const results = await DataStore.query(Model, Predicates.ALL, {
+				limit: 1,
+			});
+
+			expect(results.length).toEqual(1);
+			expect(spyOnGetAll).not.toHaveBeenCalled();
+			expect(spyOnEngine).toHaveBeenCalled();
+			expect(spyOnMemory).not.toHaveBeenCalled();
+		});
+
+		test('Should call getAll for query without predicate and pagination', async () => {
+			jest.clearAllMocks();
+			const results = await DataStore.query(Model);
+
+			expect(results.length).toEqual(3);
+			expect(spyOnGetAll).toHaveBeenCalled();
+			expect(spyOnEngine).not.toHaveBeenCalled();
+			expect(spyOnMemory).not.toHaveBeenCalled();
+		});
+	});
+});

packages/datastore/src/datastore/datastore.ts

@@ -760,7 +760,10 @@ class DataStore {
 			predicate: ModelPredicateCreator.getPredicates(predicate, false),
 			pagination: {
 				...pagination,
-				sort: ModelSortPredicateCreator.getPredicates(pagination.sort, false),
+				sort: ModelSortPredicateCreator.getPredicates(
+					pagination && pagination.sort,
+					false
+				),
 			},
 		});
 
@@ -1121,10 +1124,14 @@ class DataStore {
 	private processPagination<T extends PersistentModel>(
 		modelDefinition: SchemaModel,
 		paginationProducer: ProducerPaginationInput<T>
-	): PaginationInput<T> {
+	): PaginationInput<T> | undefined {
 		let sortPredicate: SortPredicate<T>;
 		const { limit, page, sort } = paginationProducer || {};
 
+		if (limit === undefined && page === undefined && sort === undefined) {
+			return undefined;
+		}
+
 		if (page !== undefined && limit === undefined) {
 			throw new Error('Limit is required when requesting a page');
 		}

packages/datastore/src/storage/adapter/IndexedDBAdapter.ts

@@ -370,62 +370,77 @@ class IndexedDBAdapter implements Adapter {
 		await this.checkPrivate();
 		const storeName = this.getStorenameForModel(modelConstructor);
 		const namespaceName = this.namespaceResolver(modelConstructor);
-		const sortSpecified = pagination && pagination.sort;
 
-		if (predicate) {
-			const predicates = ModelPredicateCreator.getPredicates(predicate);
-			if (predicates) {
-				const { predicates: predicateObjs, type } = predicates;
-				const idPredicate =
-					predicateObjs.length === 1 &&
-					(predicateObjs.find(
-						p => isPredicateObj(p) && p.field === 'id' && p.operator === 'eq'
-					) as PredicateObject<T>);
+		const hasPredicate = !!predicate;
+		const hasSort = pagination && pagination.sort;
+		const hasPagination = pagination && pagination.limit;
 
-				if (idPredicate) {
-					const { operand: id } = idPredicate;
+		let records: T[];
 
-					const record = <any>await this._get(storeName, id);
+		if (hasPredicate) {
+			const filtered = await this.filterOnPredicate(
+				modelConstructor,
+				predicate
+			);
+			records = this.inMemoryPagination(filtered, pagination);
+		} else if (hasSort) {
+			const all = await this.getAll(storeName);
+			records = <T[]>this.inMemoryPagination(all, pagination);
+		} else if (hasPagination) {
+			records = await this.enginePagination(storeName, pagination);
+		} else {
+			records = await this.getAll(storeName);
+		}
 
-					if (record) {
-						const [x] = await this.load(namespaceName, modelConstructor.name, [
-							record,
-						]);
+		return await this.load(namespaceName, modelConstructor.name, records);
+	}
 
-						return [x];
-					}
-					return [];
-				}
+	private async getAll<T extends PersistentModel>(
+		storeName: string
+	): Promise<T[]> {
+		return await this.db.getAll(storeName);
+	}
 
-				// TODO: Use indices if possible
-				const all = <T[]>await this.db.getAll(storeName);
+	private async filterOnPredicate<T extends PersistentModel>(
+		modelConstructor: PersistentModelConstructor<T>,
+		predicate: ModelPredicate<T>
+	) {
+		const storeName = this.getStorenameForModel(modelConstructor);
+		const namespaceName = this.namespaceResolver(modelConstructor);
 
-				const filtered = predicateObjs
-					? all.filter(m => validatePredicate(m, type, predicateObjs))
-					: all;
+		const predicates = ModelPredicateCreator.getPredicates(predicate);
+		if (predicates) {
+			const { predicates: predicateObjs, type } = predicates;
+			const idPredicate =
+				predicateObjs.length === 1 &&
+				(predicateObjs.find(
+					p => isPredicateObj(p) && p.field === 'id' && p.operator === 'eq'
+				) as PredicateObject<T>);
 
-				return await this.load(
-					namespaceName,
-					modelConstructor.name,
-					this.inMemoryPagination(filtered, pagination)
-				);
+			if (idPredicate) {
+				const { operand: id } = idPredicate;
+
+				const record = <any>await this._get(storeName, id);
+
+				if (record) {
+					const [x] = await this.load(namespaceName, modelConstructor.name, [
+						record,
+					]);
+
+					return [x];
+				}
+				return [];
 			}
-		}
 
-		if (sortSpecified) {
-			const all = <T[]>await this.db.getAll(storeName);
-			return await this.load(
-				namespaceName,
-				modelConstructor.name,
-				this.inMemoryPagination(all, pagination)
-			);
-		}
+			// TODO: Use indices if possible
+			const all = <T[]>await this.getAll(storeName);
 
-		return await this.load(
-			namespaceName,
-			modelConstructor.name,
-			await this.enginePagination(storeName, pagination)
-		);
+			const filtered = predicateObjs
+				? all.filter(m => validatePredicate(m, type, predicateObjs))
+				: all;
+
+			return filtered;
+		}
 	}
 
 	private inMemoryPagination<T extends PersistentModel>(
@@ -451,7 +466,6 @@ class IndexedDBAdapter implements Adapter {
 
 			return records.slice(start, end);
 		}
-
 		return records;
 	}
 
@@ -475,21 +489,16 @@ class IndexedDBAdapter implements Adapter {
 			}
 
 			const pageResults: T[] = [];
-
 			const hasLimit = typeof limit === 'number' && limit > 0;
-			let moreRecords = true;
-			let itemsLeft = limit;
-			while (moreRecords && cursor && cursor.value) {
-				pageResults.push(cursor.value);
 
-				cursor = await cursor.continue();
+			while (cursor && cursor.value) {
+				pageResults.push(cursor.value);
 
-				if (hasLimit) {
-					itemsLeft--;
-					moreRecords = itemsLeft > 0 && cursor !== null;
-				} else {
-					moreRecords = cursor !== null;
+				if (hasLimit && pageResults.length === limit) {
+					break;
 				}
+
+				cursor = await cursor.continue();
 			}
 
 			result = pageResults;