diff --git a/README.md b/README.md index ef4e71730c2..c92e75839d9 100644 --- a/README.md +++ b/README.md @@ -409,6 +409,12 @@ For more information see [Plugins](https://github.com/laurent22/joplin/blob/dev/ Joplin implements the SQLite Full Text Search (FTS4) extension. It means the content of all the notes is indexed in real time and search queries return results very fast. Both [Simple FTS Queries](https://www.sqlite.org/fts3.html#simple_fts_queries) and [Full-Text Index Queries](https://www.sqlite.org/fts3.html#full_text_index_queries) are supported. See below for the list of supported queries: +One caveat of SQLite FTS is that it does not support languages which do not use Latin word boundaries (spaces, tabs, punctuation). To solve this issue, Joplin has a custom search mode, that does not use FTS, but still has all of its features (multi term search, filters, etc.). One of its drawbacks is that it can get slow on larger note collections. Also, the sorting of the results will be less accurate, as the ranking algorithm (BM25) is, for now, only implemented for FTS. Finally, in this mode there are no restrictions on using the `*` wildcard (`swim*`, `*swim` and `ast*rix` all work). This search mode is currently enabled if one of the following languages are detected: + - Chinese + - Japanese + - Korean + - Thai + ## Supported queries Search type | Description | Example diff --git a/packages/lib/services/searchengine/SearchEngine.test.js b/packages/lib/services/searchengine/SearchEngine.test.js index 752602f2699..b417f6a689d 100644 --- a/packages/lib/services/searchengine/SearchEngine.test.js +++ b/packages/lib/services/searchengine/SearchEngine.test.js @@ -386,6 +386,7 @@ describe('services_SearchEngine', function() { expect((await engine.search('测试')).length).toBe(1); expect((await engine.search('测试'))[0].fields).toEqual(['body']); expect((await engine.search('测试*'))[0].fields).toEqual(['body']); + expect((await engine.search('any:1 type:todo 测试')).length).toBe(1); })); it('should support queries with Japanese characters', (async () => { @@ -398,7 +399,7 @@ describe('services_SearchEngine', function() { expect((await engine.search('できません')).length).toBe(1); expect((await engine.search('できません*'))[0].fields.sort()).toEqual(['body', 'title']); // usually assume that keyword was matched in body expect((await engine.search('テスト'))[0].fields.sort()).toEqual(['body']); - + expect((await engine.search('any:1 type:todo テスト')).length).toBe(1); })); it('should support queries with Korean characters', (async () => { @@ -409,6 +410,7 @@ describe('services_SearchEngine', function() { expect((await engine.search('이것은')).length).toBe(1); expect((await engine.search('말')).length).toBe(1); + expect((await engine.search('any:1 type:todo 말')).length).toBe(1); })); it('should support queries with Thai characters', (async () => { @@ -419,28 +421,7 @@ describe('services_SearchEngine', function() { expect((await engine.search('นี่คือค')).length).toBe(1); expect((await engine.search('ไทย')).length).toBe(1); - })); - - it('should support field restricted queries with Chinese characters', (async () => { - let rows; - const n1 = await Note.save({ title: '你好', body: '我是法国人' }); - - await engine.syncTables(); - - expect((await engine.search('title:你好*')).length).toBe(1); - expect((await engine.search('title:你好*'))[0].fields).toEqual(['title']); - expect((await engine.search('body:法国人')).length).toBe(1); - expect((await engine.search('body:法国人'))[0].fields).toEqual(['body']); - expect((await engine.search('body:你好')).length).toBe(0); - expect((await engine.search('title:你好 body:法国人')).length).toBe(1); - expect((await engine.search('title:你好 body:法国人'))[0].fields.sort()).toEqual(['body', 'title']); - expect((await engine.search('title:你好 body:bla')).length).toBe(0); - expect((await engine.search('title:你好 我是')).length).toBe(1); - expect((await engine.search('title:你好 我是'))[0].fields.sort()).toEqual(['body', 'title']); - expect((await engine.search('title:bla 我是')).length).toBe(0); - - // For non-alpha char, only the first field is looked at, the following ones are ignored - // expect((await engine.search('title:你好 title:hello')).length).toBe(1); + expect((await engine.search('any:1 type:todo ไทย')).length).toBe(1); })); it('should parse normal query strings', (async () => { diff --git a/packages/lib/services/searchengine/SearchEngine.ts b/packages/lib/services/searchengine/SearchEngine.ts index c6289967dce..4b38d1ab298 100644 --- a/packages/lib/services/searchengine/SearchEngine.ts +++ b/packages/lib/services/searchengine/SearchEngine.ts @@ -17,6 +17,7 @@ export default class SearchEngine { public static relevantFields = 'id, title, body, user_created_time, user_updated_time, is_todo, todo_completed, todo_due, parent_id, latitude, longitude, altitude, source_url'; public static SEARCH_TYPE_AUTO = 'auto'; public static SEARCH_TYPE_BASIC = 'basic'; + public static SEARCH_TYPE_NONLATIN_SCRIPT = 'nonlatin'; public static SEARCH_TYPE_FTS = 'fts'; public dispatch: Function = (_o: any) => {}; @@ -533,6 +534,7 @@ export default class SearchEngine { determineSearchType_(query: string, preferredSearchType: any) { if (preferredSearchType === SearchEngine.SEARCH_TYPE_BASIC) return SearchEngine.SEARCH_TYPE_BASIC; + if (preferredSearchType === SearchEngine.SEARCH_TYPE_NONLATIN_SCRIPT) return SearchEngine.SEARCH_TYPE_NONLATIN_SCRIPT; // If preferredSearchType is "fts" we auto-detect anyway // because it's not always supported. @@ -547,10 +549,15 @@ export default class SearchEngine { const textQuery = allTerms.filter(x => x.name === 'text' || x.name == 'title' || x.name == 'body').map(x => x.value).join(' '); const st = scriptType(textQuery); - if (!Setting.value('db.ftsEnabled') || ['ja', 'zh', 'ko', 'th'].indexOf(st) >= 0) { + if (!Setting.value('db.ftsEnabled')) { return SearchEngine.SEARCH_TYPE_BASIC; } + // Non-alphabetical languages aren't support by SQLite FTS (except with extensions which are not available in all platforms) + if (['ja', 'zh', 'ko', 'th'].indexOf(st) >= 0) { + return SearchEngine.SEARCH_TYPE_NONLATIN_SCRIPT; + } + return SearchEngine.SEARCH_TYPE_FTS; } @@ -565,7 +572,6 @@ export default class SearchEngine { const parsedQuery = await this.parseQuery(searchString); if (searchType === SearchEngine.SEARCH_TYPE_BASIC) { - // Non-alphabetical languages aren't support by SQLite FTS (except with extensions which are not available in all platforms) searchString = this.normalizeText_(searchString); const rows = await this.basicSearch(searchString); @@ -579,10 +585,11 @@ export default class SearchEngine { // when searching. // https://github.com/laurent22/joplin/issues/1075#issuecomment-459258856 + const useFts = searchType === SearchEngine.SEARCH_TYPE_FTS; try { - const { query, params } = queryBuilder(parsedQuery.allTerms); + const { query, params } = queryBuilder(parsedQuery.allTerms, useFts); const rows = await this.db().selectAll(query, params); - this.processResults_(rows, parsedQuery); + this.processResults_(rows, parsedQuery, !useFts); return rows; } catch (error) { this.logger().warn(`Cannot execute MATCH query: ${searchString}: ${error.message}`); diff --git a/packages/lib/services/searchengine/SearchFilter.test.js b/packages/lib/services/searchengine/SearchFilter.test.js index decc9e07364..9cf8a8505ab 100644 --- a/packages/lib/services/searchengine/SearchFilter.test.js +++ b/packages/lib/services/searchengine/SearchFilter.test.js @@ -25,847 +25,856 @@ describe('services_SearchFilter', function() { done(); }); - - it('should return note matching title', (async () => { - let rows; - const n1 = await Note.save({ title: 'abcd', body: 'body 1' }); - const n2 = await Note.save({ title: 'efgh', body: 'body 2' }); - - await engine.syncTables(); - rows = await engine.search('title: abcd'); - - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n1.id); - })); - - it('should return note matching negated title', (async () => { - let rows; - const n1 = await Note.save({ title: 'abcd', body: 'body 1' }); - const n2 = await Note.save({ title: 'efgh', body: 'body 2' }); - - await engine.syncTables(); - rows = await engine.search('-title: abcd'); - - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n2.id); - })); - - it('should return note matching body', (async () => { - let rows; - const n1 = await Note.save({ title: 'abcd', body: 'body1' }); - const n2 = await Note.save({ title: 'efgh', body: 'body2' }); - - await engine.syncTables(); - rows = await engine.search('body: body1'); - - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n1.id); - })); - - it('should return note matching negated body', (async () => { - let rows; - const n1 = await Note.save({ title: 'abcd', body: 'body1' }); - const n2 = await Note.save({ title: 'efgh', body: 'body2' }); - - await engine.syncTables(); - rows = await engine.search('-body: body1'); - - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n2.id); - })); - - it('should return note matching title containing multiple words', (async () => { - let rows; - const n1 = await Note.save({ title: 'abcd xyz', body: 'body1' }); - const n2 = await Note.save({ title: 'efgh ijk', body: 'body2' }); - - await engine.syncTables(); - rows = await engine.search('title: "abcd xyz"'); - - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n1.id); - })); - - it('should return note matching body containing multiple words', (async () => { - let rows; - const n1 = await Note.save({ title: 'abcd', body: 'ho ho ho' }); - const n2 = await Note.save({ title: 'efgh', body: 'foo bar' }); - - await engine.syncTables(); - rows = await engine.search('body: "foo bar"'); - - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n2.id); - })); - - it('should return note matching title AND body', (async () => { - let rows; - const n1 = await Note.save({ title: 'abcd', body: 'ho ho ho' }); - const n2 = await Note.save({ title: 'efgh', body: 'foo bar' }); - - await engine.syncTables(); - rows = await engine.search('title: efgh body: "foo bar"'); - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n2.id); - - rows = await engine.search('title: abcd body: "foo bar"'); - expect(rows.length).toBe(0); - })); - - it('should return note matching title OR body', (async () => { - let rows; - const n1 = await Note.save({ title: 'abcd', body: 'ho ho ho' }); - const n2 = await Note.save({ title: 'efgh', body: 'foo bar' }); - - await engine.syncTables(); - rows = await engine.search('any:1 title: abcd body: "foo bar"'); - expect(rows.length).toBe(2); - expect(rows.map(r=>r.id)).toContain(n1.id); - expect(rows.map(r=>r.id)).toContain(n2.id); - - rows = await engine.search('any:1 title: wxyz body: "blah blah"'); - expect(rows.length).toBe(0); - })); - - it('should return notes matching text', (async () => { - let rows; - const n1 = await Note.save({ title: 'foo beef', body: 'dead bar' }); - const n2 = await Note.save({ title: 'bar efgh', body: 'foo dog' }); - const n3 = await Note.save({ title: 'foo ho', body: 'ho ho ho' }); - await engine.syncTables(); - - // Interpretation: Match with notes containing foo in title/body and bar in title/body - // Note: This is NOT saying to match notes containing foo bar in title/body - rows = await engine.search('foo bar'); - expect(rows.length).toBe(2); - expect(rows.map(r=>r.id)).toContain(n1.id); - expect(rows.map(r=>r.id)).toContain(n2.id); - - rows = await engine.search('foo -bar'); - expect(rows.length).toBe(1); - expect(rows.map(r=>r.id)).toContain(n3.id); - - rows = await engine.search('foo efgh'); - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n2.id); - - rows = await engine.search('zebra'); - expect(rows.length).toBe(0); - })); - - it('should return notes matching any negated text', (async () => { - let rows; - const n1 = await Note.save({ title: 'abc', body: 'def' }); - const n2 = await Note.save({ title: 'def', body: 'ghi' }); - const n3 = await Note.save({ title: 'ghi', body: 'jkl' }); - await engine.syncTables(); - - rows = await engine.search('any:1 -abc -ghi'); - expect(rows.length).toBe(3); - expect(rows.map(r=>r.id)).toContain(n1.id); - expect(rows.map(r=>r.id)).toContain(n2.id); - expect(rows.map(r=>r.id)).toContain(n3.id); - })); - - it('should return notes matching any negated title', (async () => { - let rows; - const n1 = await Note.save({ title: 'abc', body: 'def' }); - const n2 = await Note.save({ title: 'def', body: 'ghi' }); - const n3 = await Note.save({ title: 'ghi', body: 'jkl' }); - await engine.syncTables(); - - rows = await engine.search('any:1 -title:abc -title:ghi'); - expect(rows.length).toBe(3); - expect(rows.map(r=>r.id)).toContain(n1.id); - expect(rows.map(r=>r.id)).toContain(n2.id); - expect(rows.map(r=>r.id)).toContain(n3.id); - })); - - it('should return notes matching any negated body', (async () => { - let rows; - const n1 = await Note.save({ title: 'abc', body: 'def' }); - const n2 = await Note.save({ title: 'def', body: 'ghi' }); - const n3 = await Note.save({ title: 'ghi', body: 'jkl' }); - await engine.syncTables(); - - rows = await engine.search('any:1 -body:xyz -body:ghi'); - expect(rows.length).toBe(3); - expect(rows.map(r=>r.id)).toContain(n1.id); - expect(rows.map(r=>r.id)).toContain(n2.id); - expect(rows.map(r=>r.id)).toContain(n3.id); - })); - - it('should support phrase search', (async () => { - let rows; - const n1 = await Note.save({ title: 'foo beef', body: 'bar dog' }); - const n2 = await Note.save({ title: 'bar efgh', body: 'foo dog' }); - await engine.syncTables(); - - rows = await engine.search('"bar dog"'); - expect(rows.length).toBe(1); - expect(rows[0].id).toBe(n1.id); - })); - - it('should support prefix search', (async () => { - let rows; - const n1 = await Note.save({ title: 'foo beef', body: 'bar dog' }); - const n2 = await Note.save({ title: 'bar efgh', body: 'foo dog' }); - await engine.syncTables(); - - rows = await engine.search('"bar*"'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - })); - - it('should support filtering by tags', (async () => { - let rows; - const n1 = await Note.save({ title: 'But I would', body: 'walk 500 miles' }); - const n2 = await Note.save({ title: 'And I would', body: 'walk 500 more' }); - const n3 = await Note.save({ title: 'Just to be', body: 'the man who' }); - const n4 = await Note.save({ title: 'walked a thousand', body: 'miles to fall' }); - const n5 = await Note.save({ title: 'down at your', body: 'door' }); - - await Tag.setNoteTagsByTitles(n1.id, ['Da', 'da', 'lat', 'da']); - await Tag.setNoteTagsByTitles(n2.id, ['Da', 'da', 'lat', 'da']); - - await engine.syncTables(); - - rows = await engine.search('tag:*'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('-tag:*'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n3.id); - expect(ids(rows)).toContain(n4.id); - expect(ids(rows)).toContain(n5.id); - })); - - - it('should support filtering by tags', (async () => { - let rows; - const n1 = await Note.save({ title: 'peace talks', body: 'battle ground' }); - const n2 = await Note.save({ title: 'mouse', body: 'mister' }); - const n3 = await Note.save({ title: 'dresden files', body: 'harry dresden' }); - - await Tag.setNoteTagsByTitles(n1.id, ['tag1', 'tag2']); - await Tag.setNoteTagsByTitles(n2.id, ['tag2', 'tag3']); - await Tag.setNoteTagsByTitles(n3.id, ['tag3', 'tag4', 'space travel']); - - await engine.syncTables(); - - rows = await engine.search('tag:tag2'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('tag:tag2 tag:tag3'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('any:1 tag:tag1 tag:tag2 tag:tag3'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); - - rows = await engine.search('tag:tag2 tag:tag3 tag:tag4'); - expect(rows.length).toBe(0); - - rows = await engine.search('-tag:tag2'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n3.id); - - rows = await engine.search('-tag:tag2 -tag:tag3'); - expect(rows.length).toBe(0); - - rows = await engine.search('-tag:tag2 -tag:tag3'); - expect(rows.length).toBe(0); - - rows = await engine.search('any:1 -tag:tag2 -tag:tag3'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n3.id); - - rows = await engine.search('tag:"space travel"'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n3.id); - })); - - it('should support filtering by notebook', (async () => { - let rows; - const folder0 = await Folder.save({ title: 'notebook0' }); - const folder1 = await Folder.save({ title: 'notebook1' }); - const notes0 = await createNTestNotes(5, folder0); - const notes1 = await createNTestNotes(5, folder1); - - await engine.syncTables(); - - rows = await engine.search('notebook:notebook0'); - expect(rows.length).toBe(5); - expect(ids(rows).sort()).toEqual(ids(notes0).sort()); - - })); - - it('should support filtering by nested notebook', (async () => { - let rows; - const folder0 = await Folder.save({ title: 'notebook0' }); - const folder00 = await Folder.save({ title: 'notebook00', parent_id: folder0.id }); - const folder1 = await Folder.save({ title: 'notebook1' }); - const notes0 = await createNTestNotes(5, folder0); - const notes00 = await createNTestNotes(5, folder00); - const notes1 = await createNTestNotes(5, folder1); - - await engine.syncTables(); - - rows = await engine.search('notebook:notebook0'); - expect(rows.length).toBe(10); - expect(ids(rows).sort()).toEqual(ids(notes0.concat(notes00)).sort()); - })); - - it('should support filtering by multiple notebooks', (async () => { - let rows; - const folder0 = await Folder.save({ title: 'notebook0' }); - const folder00 = await Folder.save({ title: 'notebook00', parent_id: folder0.id }); - const folder1 = await Folder.save({ title: 'notebook1' }); - const folder2 = await Folder.save({ title: 'notebook2' }); - const notes0 = await createNTestNotes(5, folder0); - const notes00 = await createNTestNotes(5, folder00); - const notes1 = await createNTestNotes(5, folder1); - const notes2 = await createNTestNotes(5, folder2); - - await engine.syncTables(); - - rows = await engine.search('notebook:notebook0 notebook:notebook1'); - expect(rows.length).toBe(15); - expect(ids(rows).sort()).toEqual(ids(notes0).concat(ids(notes00).concat(ids(notes1))).sort()); - })); - - it('should support filtering by created date', (async () => { - let rows; - const n1 = await Note.save({ title: 'I made this on', body: 'May 20 2020', user_created_time: Date.parse('2020-05-20') }); - const n2 = await Note.save({ title: 'I made this on', body: 'May 19 2020', user_created_time: Date.parse('2020-05-19') }); - const n3 = await Note.save({ title: 'I made this on', body: 'May 18 2020', user_created_time: Date.parse('2020-05-18') }); - - await engine.syncTables(); - - rows = await engine.search('created:20200520'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); - - rows = await engine.search('created:20200519'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('-created:20200519'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n3.id); - - })); - - it('should support filtering by between two dates', (async () => { - let rows; - const n1 = await Note.save({ title: 'January 01 2020', body: 'January 01 2020', user_created_time: Date.parse('2020-01-01') }); - const n2 = await Note.save({ title: 'February 15 2020', body: 'February 15 2020', user_created_time: Date.parse('2020-02-15') }); - const n3 = await Note.save({ title: 'March 25 2019', body: 'March 25 2019', user_created_time: Date.parse('2019-03-25') }); - const n4 = await Note.save({ title: 'March 01 2018', body: 'March 01 2018', user_created_time: Date.parse('2018-03-01') }); - - await engine.syncTables(); - - rows = await engine.search('created:20200101 -created:20200220'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('created:201901 -created:202002'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n3.id); - expect(ids(rows)).toContain(n1.id); - - rows = await engine.search('created:2018 -created:2019'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n4.id); - })); - - it('should support filtering by created with smart value: day', (async () => { - let rows; - const n1 = await Note.save({ title: 'I made this', body: 'today', user_created_time: parseInt(time.goBackInTime(Date.now(), 0, 'day'), 10) }); - const n2 = await Note.save({ title: 'I made this', body: 'yesterday', user_created_time: parseInt(time.goBackInTime(Date.now(), 1, 'day'), 10) }); - const n3 = await Note.save({ title: 'I made this', body: 'day before yesterday', user_created_time: parseInt(time.goBackInTime(Date.now(), 2, 'day'), 10) }); - - await engine.syncTables(); - - rows = await engine.search('created:day-0'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); - - rows = await engine.search('created:day-1'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('created:day-2'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); - })); - - it('should support filtering by created with smart value: week', (async () => { - let rows; - const n1 = await Note.save({ title: 'I made this', body: 'this week', user_created_time: parseInt(time.goBackInTime(Date.now(), 0, 'week'), 10) }); - const n2 = await Note.save({ title: 'I made this', body: 'the week before', user_created_time: parseInt(time.goBackInTime(Date.now(), 1, 'week'), 10) }); - const n3 = await Note.save({ title: 'I made this', body: 'before before week', user_created_time: parseInt(time.goBackInTime(Date.now(), 2, 'week'), 10) }); + // Outside of for loop because this does not apply to to SEARCH_TYPE_NONLATIN_SCRIPT + it('should ignore dashes in a word', (async () => { + const n0 = await Note.save({ title: 'doesnotwork' }); + const n1 = await Note.save({ title: 'does not work' }); + const n2 = await Note.save({ title: 'does-not-work' }); + const n3 = await Note.save({ title: 'does_not_work' }); await engine.syncTables(); - rows = await engine.search('created:week-0'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); - - rows = await engine.search('created:week-1'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('created:week-2'); + let rows = await engine.search('does-not-work'); expect(rows.length).toBe(3); expect(ids(rows)).toContain(n1.id); expect(ids(rows)).toContain(n2.id); expect(ids(rows)).toContain(n3.id); - })); - - it('should support filtering by created with smart value: month', (async () => { - let rows; - const n1 = await Note.save({ title: 'I made this', body: 'this month', user_created_time: parseInt(time.goBackInTime(Date.now(), 0, 'month'), 10) }); - const n2 = await Note.save({ title: 'I made this', body: 'the month before', user_created_time: parseInt(time.goBackInTime(Date.now(), 1, 'month'), 10) }); - const n3 = await Note.save({ title: 'I made this', body: 'before before month', user_created_time: parseInt(time.goBackInTime(Date.now(), 2, 'month'), 10) }); - await engine.syncTables(); - - rows = await engine.search('created:month-0'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); - - rows = await engine.search('created:month-1'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('created:month-2'); + rows = await engine.search('does not work'); expect(rows.length).toBe(3); expect(ids(rows)).toContain(n1.id); expect(ids(rows)).toContain(n2.id); expect(ids(rows)).toContain(n3.id); - })); - - it('should support filtering by created with smart value: year', (async () => { - let rows; - const n1 = await Note.save({ title: 'I made this', body: 'this year', user_created_time: parseInt(time.goBackInTime(Date.now(), 0, 'year'), 10) }); - const n2 = await Note.save({ title: 'I made this', body: 'the year before', user_created_time: parseInt(time.goBackInTime(Date.now(), 1, 'year'), 10) }); - const n3 = await Note.save({ title: 'I made this', body: 'before before year', user_created_time: parseInt(time.goBackInTime(Date.now(), 2, 'year'), 10) }); - - await engine.syncTables(); - - rows = await engine.search('created:year-0'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); - - rows = await engine.search('created:year-1'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - rows = await engine.search('created:year-2'); + rows = await engine.search('"does not work"'); expect(rows.length).toBe(3); expect(ids(rows)).toContain(n1.id); expect(ids(rows)).toContain(n2.id); expect(ids(rows)).toContain(n3.id); - })); - - it('should support filtering by updated date', (async () => { - let rows; - const n1 = await Note.save({ title: 'I updated this on', body: 'May 20 2020', updated_time: Date.parse('2020-05-20'), user_updated_time: Date.parse('2020-05-20') }, { autoTimestamp: false }); - const n2 = await Note.save({ title: 'I updated this on', body: 'May 19 2020', updated_time: Date.parse('2020-05-19'), user_updated_time: Date.parse('2020-05-19') }, { autoTimestamp: false }); - - await engine.syncTables(); - - rows = await engine.search('updated:20200520'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); - - rows = await engine.search('updated:20200519'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - })); - - it('should support filtering by updated with smart value: day', (async () => { - let rows; - const today = parseInt(time.goBackInTime(Date.now(), 0, 'day'), 10); - const yesterday = parseInt(time.goBackInTime(Date.now(), 1, 'day'), 10); - const dayBeforeYesterday = parseInt(time.goBackInTime(Date.now(), 2, 'day'), 10); - const n1 = await Note.save({ title: 'I made this', body: 'today', updated_time: today, user_updated_time: today }, { autoTimestamp: false }); - const n11 = await Note.save({ title: 'I also made this', body: 'today', updated_time: today, user_updated_time: today }, { autoTimestamp: false }); - - const n2 = await Note.save({ title: 'I made this', body: 'yesterday', updated_time: yesterday, user_updated_time: yesterday }, { autoTimestamp: false }); - const n3 = await Note.save({ title: 'I made this', body: 'day before yesterday', updated_time: dayBeforeYesterday ,user_updated_time: dayBeforeYesterday }, { autoTimestamp: false }); - - await engine.syncTables(); - rows = await engine.search('updated:day-0'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n11.id); - - rows = await engine.search('updated:day-1'); + rows = await engine.search('title:does-not-work'); expect(rows.length).toBe(3); expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n11.id); - expect(ids(rows)).toContain(n2.id); - - rows = await engine.search('updated:day-2'); - expect(rows.length).toBe(4); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n11.id); expect(ids(rows)).toContain(n2.id); expect(ids(rows)).toContain(n3.id); - })); - - it('should support filtering by type todo', (async () => { - let rows; - const t1 = await Note.save({ title: 'This is a ', body: 'todo', is_todo: 1 }); - const t2 = await Note.save({ title: 'This is another', body: 'todo but completed', is_todo: 1, todo_completed: 1590085027710 }); - const t3 = await Note.save({ title: 'This is NOT a ', body: 'todo' }); - - await engine.syncTables(); - - rows = await engine.search('type:todo'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(t1.id); - expect(ids(rows)).toContain(t2.id); - - rows = await engine.search('any:1 type:todo'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(t1.id); - expect(ids(rows)).toContain(t2.id); - - rows = await engine.search('iscompleted:1'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(t2.id); - - rows = await engine.search('iscompleted:0'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(t1.id); - })); - - it('should support filtering by type note', (async () => { - let rows; - const t1 = await Note.save({ title: 'This is a ', body: 'todo', is_todo: 1 }); - const t2 = await Note.save({ title: 'This is another', body: 'todo but completed', is_todo: 1, todo_completed: 1590085027710 }); - const t3 = await Note.save({ title: 'This is NOT a ', body: 'todo' }); - - await engine.syncTables(); - - rows = await engine.search('type:note'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(t3.id); - })); - it('should support filtering by due date', (async () => { - let rows; - const toDo1 = await Note.save({ title: 'ToDo 1', body: 'todo', is_todo: 1, todo_due: Date.parse('2021-04-27') }); - const toDo2 = await Note.save({ title: 'ToDo 2', body: 'todo', is_todo: 1, todo_due: Date.parse('2021-03-17') }); - const note1 = await Note.save({ title: 'Note 1', body: 'Note' }); - - await engine.syncTables(); - - rows = await engine.search('due:20210425'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(toDo1.id); - - rows = await engine.search('-due:20210425'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(toDo2.id); - })); - - it('should support filtering by due with smart value: day', (async () => { - let rows; - - const inThreeDays = parseInt(time.goForwardInTime(Date.now(), 3, 'day'), 10); - const inSevenDays = parseInt(time.goForwardInTime(Date.now(), 7, 'day'), 10); - const threeDaysAgo = parseInt(time.goBackInTime(Date.now(), 3, 'day'), 10); - const sevenDaysAgo = parseInt(time.goBackInTime(Date.now(), 7, 'day'), 10); - - const toDo1 = await Note.save({ title: 'ToDo + 3 day', body: 'toto', is_todo: 1, todo_due: inThreeDays }); - const toDo2 = await Note.save({ title: 'ToDo + 7 day', body: 'toto', is_todo: 1, todo_due: inSevenDays }); - const toDo3 = await Note.save({ title: 'ToDo - 3 day', body: 'toto', is_todo: 1, todo_due: threeDaysAgo }); - const toDo4 = await Note.save({ title: 'ToDo - 7 day', body: 'toto', is_todo: 1, todo_due: sevenDaysAgo }); - - await engine.syncTables(); - - rows = await engine.search('due:day-4'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(toDo1.id); - expect(ids(rows)).toContain(toDo2.id); - expect(ids(rows)).toContain(toDo3.id); - - rows = await engine.search('-due:day-4'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(toDo4.id); - - rows = await engine.search('-due:day+4'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(toDo1.id); - expect(ids(rows)).toContain(toDo3.id); - expect(ids(rows)).toContain(toDo4.id); - - rows = await engine.search('due:day+4'); + rows = await engine.search('doesnotwork'); expect(rows.length).toBe(1); - expect(ids(rows)).toContain(toDo2.id); + expect(ids(rows)).toContain(n0.id); - rows = await engine.search('due:day-4 -due:day+4'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(toDo1.id); - expect(ids(rows)).toContain(toDo3.id); })); - it('should support filtering by latitude, longitude, altitude', (async () => { - let rows; - const n1 = await Note.save({ title: 'I made this', body: 'this week', latitude: 12.97, longitude: 88.88, altitude: 69.96 }); - const n2 = await Note.save({ title: 'I made this', body: 'the week before', latitude: 42.11, longitude: 77.77, altitude: 42.00 }); - const n3 = await Note.save({ title: 'I made this', body: 'before before week', latitude: 82.01, longitude: 66.66, altitude: 13.13 }); - - await engine.syncTables(); + for (const searchType of [SearchEngine.SEARCH_TYPE_FTS, SearchEngine.SEARCH_TYPE_NONLATIN_SCRIPT]) { - rows = await engine.search('latitude:13.5'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); + describe(`search type ${searchType}`, () => { + it('should return note matching title', (async () => { + let rows; + const n1 = await Note.save({ title: 'abcd', body: 'body 1' }); + const n2 = await Note.save({ title: 'efgh', body: 'body 2' }); - rows = await engine.search('-latitude:40'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); + await engine.syncTables(); + rows = await engine.search('title: abcd', { searchType }); - rows = await engine.search('latitude:13 -latitude:80'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n2.id); + expect(rows.length).toBe(1); + expect(rows[0].id).toBe(n1.id); + })); - rows = await engine.search('altitude:13.5'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); + it('should return note matching negated title', (async () => { + let rows; + const n1 = await Note.save({ title: 'abcd', body: 'body 1' }); + const n2 = await Note.save({ title: 'efgh', body: 'body 2' }); - rows = await engine.search('-altitude:80.12'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); + await engine.syncTables(); + rows = await engine.search('-title: abcd', { searchType }); - rows = await engine.search('longitude:70 -longitude:80'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n2.id); + expect(rows.length).toBe(1); - rows = await engine.search('latitude:20 longitude:50 altitude:40'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n2.id); + expect(rows[0].id).toBe(n2.id); + })); - rows = await engine.search('any:1 latitude:20 longitude:50 altitude:40'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); - })); + it('should return note matching body', (async () => { + let rows; + const n1 = await Note.save({ title: 'abcd', body: 'body1' }); + const n2 = await Note.save({ title: 'efgh', body: 'body2' }); - it('should support filtering by resource MIME type', (async () => { - let rows; - const service = new ResourceService(); - // console.log(testImagePath) - const folder1 = await Folder.save({ title: 'folder1' }); - let n1 = await Note.save({ title: 'I have a picture', body: 'Im awesome', parent_id: folder1.id }); - const n2 = await Note.save({ title: 'Boring note 1', body: 'I just have text', parent_id: folder1.id }); - const n3 = await Note.save({ title: 'Boring note 2', body: 'me too', parent_id: folder1.id }); - let n4 = await Note.save({ title: 'A picture?', body: 'pfff, I have a pdf', parent_id: folder1.id }); - await engine.syncTables(); + await engine.syncTables(); + rows = await engine.search('body: body1', { searchType }); - // let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); - n1 = await shim.attachFileToNote(n1, `${supportDir}/photo.jpg`); - // const resource1 = (await Resource.all())[0]; + expect(rows.length).toBe(1); - n4 = await shim.attachFileToNote(n4, `${supportDir}/welcome.pdf`); + expect(rows[0].id).toBe(n1.id); + })); - await service.indexNoteResources(); + it('should return note matching negated body', (async () => { + let rows; + const n1 = await Note.save({ title: 'abcd', body: 'body1' }); + const n2 = await Note.save({ title: 'efgh', body: 'body2' }); - rows = await engine.search('resource:image/jpeg'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); + await engine.syncTables(); + rows = await engine.search('-body: body1', { searchType }); + + expect(rows.length).toBe(1); + expect(rows[0].id).toBe(n2.id); + })); + + it('should return note matching title containing multiple words', (async () => { + let rows; + const n1 = await Note.save({ title: 'abcd xyz', body: 'body1' }); + const n2 = await Note.save({ title: 'efgh ijk', body: 'body2' }); + + await engine.syncTables(); + rows = await engine.search('title: "abcd xyz"', { searchType }); + + expect(rows.length).toBe(1); + expect(rows[0].id).toBe(n1.id); + })); + + it('should return note matching body containing multiple words', (async () => { + let rows; + const n1 = await Note.save({ title: 'abcd', body: 'ho ho ho' }); + const n2 = await Note.save({ title: 'efgh', body: 'foo bar' }); + + await engine.syncTables(); + rows = await engine.search('body: "foo bar"', { searchType }); + + expect(rows.length).toBe(1); + expect(rows[0].id).toBe(n2.id); + })); + + it('should return note matching title AND body', (async () => { + let rows; + const n1 = await Note.save({ title: 'abcd', body: 'ho ho ho' }); + const n2 = await Note.save({ title: 'efgh', body: 'foo bar' }); + + await engine.syncTables(); + rows = await engine.search('title: efgh body: "foo bar"', { searchType }); + expect(rows.length).toBe(1); + expect(rows[0].id).toBe(n2.id); + + rows = await engine.search('title: abcd body: "foo bar"', { searchType }); + expect(rows.length).toBe(0); + })); + + it('should return note matching title OR body', (async () => { + let rows; + const n1 = await Note.save({ title: 'abcd', body: 'ho ho ho' }); + const n2 = await Note.save({ title: 'efgh', body: 'foo bar' }); + + await engine.syncTables(); + rows = await engine.search('any:1 title: abcd body: "foo bar"', { searchType }); + expect(rows.length).toBe(2); + expect(rows.map(r=>r.id)).toContain(n1.id); + expect(rows.map(r=>r.id)).toContain(n2.id); + + rows = await engine.search('any:1 title: wxyz body: "blah blah"', { searchType }); + expect(rows.length).toBe(0); + })); + + it('should return notes matching text', (async () => { + let rows; + const n1 = await Note.save({ title: 'foo beef', body: 'dead bar' }); + const n2 = await Note.save({ title: 'bar efgh', body: 'foo dog' }); + const n3 = await Note.save({ title: 'foo ho', body: 'ho ho ho' }); + await engine.syncTables(); + + // Interpretation: Match with notes containing foo in title/body and bar in title/body + // Note: This is NOT saying to match notes containing foo bar in title/body + rows = await engine.search('foo bar', { searchType }); + expect(rows.length).toBe(2); + expect(rows.map(r=>r.id)).toContain(n1.id); + expect(rows.map(r=>r.id)).toContain(n2.id); + + rows = await engine.search('foo -bar', { searchType }); + expect(rows.length).toBe(1); + expect(rows.map(r=>r.id)).toContain(n3.id); + + rows = await engine.search('foo efgh', { searchType }); + expect(rows.length).toBe(1); + expect(rows[0].id).toBe(n2.id); + + rows = await engine.search('zebra', { searchType }); + expect(rows.length).toBe(0); + })); + + it('should return notes matching any negated text', (async () => { + let rows; + const n1 = await Note.save({ title: 'abc', body: 'def' }); + const n2 = await Note.save({ title: 'def', body: 'ghi' }); + const n3 = await Note.save({ title: 'ghi', body: 'jkl' }); + await engine.syncTables(); + + rows = await engine.search('any:1 -abc -ghi', { searchType }); + expect(rows.length).toBe(3); + expect(rows.map(r=>r.id)).toContain(n1.id); + expect(rows.map(r=>r.id)).toContain(n2.id); + expect(rows.map(r=>r.id)).toContain(n3.id); + })); + + it('should return notes matching any negated title', (async () => { + let rows; + const n1 = await Note.save({ title: 'abc', body: 'def' }); + const n2 = await Note.save({ title: 'def', body: 'ghi' }); + const n3 = await Note.save({ title: 'ghi', body: 'jkl' }); + await engine.syncTables(); + + rows = await engine.search('any:1 -title:abc -title:ghi', { searchType }); + expect(rows.length).toBe(3); + expect(rows.map(r=>r.id)).toContain(n1.id); + expect(rows.map(r=>r.id)).toContain(n2.id); + expect(rows.map(r=>r.id)).toContain(n3.id); + })); + + it('should return notes matching any negated body', (async () => { + let rows; + const n1 = await Note.save({ title: 'abc', body: 'def' }); + const n2 = await Note.save({ title: 'def', body: 'ghi' }); + const n3 = await Note.save({ title: 'ghi', body: 'jkl' }); + await engine.syncTables(); + + rows = await engine.search('any:1 -body:xyz -body:ghi', { searchType }); + expect(rows.length).toBe(3); + expect(rows.map(r=>r.id)).toContain(n1.id); + expect(rows.map(r=>r.id)).toContain(n2.id); + expect(rows.map(r=>r.id)).toContain(n3.id); + })); + + it('should support phrase search', (async () => { + let rows; + const n1 = await Note.save({ title: 'foo beef', body: 'bar dog' }); + const n2 = await Note.save({ title: 'bar efgh', body: 'foo dog' }); + await engine.syncTables(); + + rows = await engine.search('"bar dog"', { searchType }); + expect(rows.length).toBe(1); + expect(rows[0].id).toBe(n1.id); + })); + + it('should support prefix search', (async () => { + let rows; + const n1 = await Note.save({ title: 'foo beef', body: 'bar dog' }); + const n2 = await Note.save({ title: 'bar efgh', body: 'foo dog' }); + await engine.syncTables(); + + rows = await engine.search('"bar*"', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + })); + + it('should support filtering by tags', (async () => { + let rows; + const n1 = await Note.save({ title: 'But I would', body: 'walk 500 miles' }); + const n2 = await Note.save({ title: 'And I would', body: 'walk 500 more' }); + const n3 = await Note.save({ title: 'Just to be', body: 'the man who' }); + const n4 = await Note.save({ title: 'walked a thousand', body: 'miles to fall' }); + const n5 = await Note.save({ title: 'down at your', body: 'door' }); + + await Tag.setNoteTagsByTitles(n1.id, ['Da', 'da', 'lat', 'da']); + await Tag.setNoteTagsByTitles(n2.id, ['Da', 'da', 'lat', 'da']); + + await engine.syncTables(); + + rows = await engine.search('tag:*', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('-tag:*', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n3.id); + expect(ids(rows)).toContain(n4.id); + expect(ids(rows)).toContain(n5.id); + })); + + + it('should support filtering by tags', (async () => { + let rows; + const n1 = await Note.save({ title: 'peace talks', body: 'battle ground' }); + const n2 = await Note.save({ title: 'mouse', body: 'mister' }); + const n3 = await Note.save({ title: 'dresden files', body: 'harry dresden' }); + + await Tag.setNoteTagsByTitles(n1.id, ['tag1', 'tag2']); + await Tag.setNoteTagsByTitles(n2.id, ['tag2', 'tag3']); + await Tag.setNoteTagsByTitles(n3.id, ['tag3', 'tag4', 'space travel']); + + await engine.syncTables(); + + rows = await engine.search('tag:tag2', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('tag:tag2 tag:tag3', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('any:1 tag:tag1 tag:tag2 tag:tag3', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + + rows = await engine.search('tag:tag2 tag:tag3 tag:tag4', { searchType }); + expect(rows.length).toBe(0); + + rows = await engine.search('-tag:tag2', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n3.id); + + rows = await engine.search('-tag:tag2 -tag:tag3', { searchType }); + expect(rows.length).toBe(0); + + rows = await engine.search('-tag:tag2 -tag:tag3', { searchType }); + expect(rows.length).toBe(0); + + rows = await engine.search('any:1 -tag:tag2 -tag:tag3', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n3.id); + + rows = await engine.search('tag:"space travel"', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n3.id); + })); + + it('should support filtering by notebook', (async () => { + let rows; + const folder0 = await Folder.save({ title: 'notebook0' }); + const folder1 = await Folder.save({ title: 'notebook1' }); + const notes0 = await createNTestNotes(5, folder0); + const notes1 = await createNTestNotes(5, folder1); + + await engine.syncTables(); + + rows = await engine.search('notebook:notebook0', { searchType }); + expect(rows.length).toBe(5); + expect(ids(rows).sort()).toEqual(ids(notes0).sort()); + + })); + + it('should support filtering by nested notebook', (async () => { + let rows; + const folder0 = await Folder.save({ title: 'notebook0' }); + const folder00 = await Folder.save({ title: 'notebook00', parent_id: folder0.id }); + const folder1 = await Folder.save({ title: 'notebook1' }); + const notes0 = await createNTestNotes(5, folder0); + const notes00 = await createNTestNotes(5, folder00); + const notes1 = await createNTestNotes(5, folder1); + + await engine.syncTables(); + + rows = await engine.search('notebook:notebook0', { searchType }); + expect(rows.length).toBe(10); + expect(ids(rows).sort()).toEqual(ids(notes0.concat(notes00)).sort()); + })); + + it('should support filtering by multiple notebooks', (async () => { + let rows; + const folder0 = await Folder.save({ title: 'notebook0' }); + const folder00 = await Folder.save({ title: 'notebook00', parent_id: folder0.id }); + const folder1 = await Folder.save({ title: 'notebook1' }); + const folder2 = await Folder.save({ title: 'notebook2' }); + const notes0 = await createNTestNotes(5, folder0); + const notes00 = await createNTestNotes(5, folder00); + const notes1 = await createNTestNotes(5, folder1); + const notes2 = await createNTestNotes(5, folder2); + + await engine.syncTables(); + + rows = await engine.search('notebook:notebook0 notebook:notebook1', { searchType }); + expect(rows.length).toBe(15); + expect(ids(rows).sort()).toEqual(ids(notes0).concat(ids(notes00).concat(ids(notes1))).sort()); + })); + + it('should support filtering by created date', (async () => { + let rows; + const n1 = await Note.save({ title: 'I made this on', body: 'May 20 2020', user_created_time: Date.parse('2020-05-20') }); + const n2 = await Note.save({ title: 'I made this on', body: 'May 19 2020', user_created_time: Date.parse('2020-05-19') }); + const n3 = await Note.save({ title: 'I made this on', body: 'May 18 2020', user_created_time: Date.parse('2020-05-18') }); + + await engine.syncTables(); + + rows = await engine.search('created:20200520', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); + + rows = await engine.search('created:20200519', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('-created:20200519', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n3.id); + + })); + + it('should support filtering by between two dates', (async () => { + let rows; + const n1 = await Note.save({ title: 'January 01 2020', body: 'January 01 2020', user_created_time: Date.parse('2020-01-01') }); + const n2 = await Note.save({ title: 'February 15 2020', body: 'February 15 2020', user_created_time: Date.parse('2020-02-15') }); + const n3 = await Note.save({ title: 'March 25 2019', body: 'March 25 2019', user_created_time: Date.parse('2019-03-25') }); + const n4 = await Note.save({ title: 'March 01 2018', body: 'March 01 2018', user_created_time: Date.parse('2018-03-01') }); + + await engine.syncTables(); + + rows = await engine.search('created:20200101 -created:20200220', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('created:201901 -created:202002', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n3.id); + expect(ids(rows)).toContain(n1.id); + + rows = await engine.search('created:2018 -created:2019', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n4.id); + })); + + it('should support filtering by created with smart value: day', (async () => { + let rows; + const n1 = await Note.save({ title: 'I made this', body: 'today', user_created_time: parseInt(time.goBackInTime(Date.now(), 0, 'day'), 10) }); + const n2 = await Note.save({ title: 'I made this', body: 'yesterday', user_created_time: parseInt(time.goBackInTime(Date.now(), 1, 'day'), 10) }); + const n3 = await Note.save({ title: 'I made this', body: 'day before yesterday', user_created_time: parseInt(time.goBackInTime(Date.now(), 2, 'day'), 10) }); + + await engine.syncTables(); + + rows = await engine.search('created:day-0', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); + + rows = await engine.search('created:day-1', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('created:day-2', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + })); + + it('should support filtering by created with smart value: week', (async () => { + let rows; + const n1 = await Note.save({ title: 'I made this', body: 'this week', user_created_time: parseInt(time.goBackInTime(Date.now(), 0, 'week'), 10) }); + const n2 = await Note.save({ title: 'I made this', body: 'the week before', user_created_time: parseInt(time.goBackInTime(Date.now(), 1, 'week'), 10) }); + const n3 = await Note.save({ title: 'I made this', body: 'before before week', user_created_time: parseInt(time.goBackInTime(Date.now(), 2, 'week'), 10) }); + + await engine.syncTables(); + + rows = await engine.search('created:week-0', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); + + rows = await engine.search('created:week-1', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('created:week-2', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + })); + + it('should support filtering by created with smart value: month', (async () => { + let rows; + const n1 = await Note.save({ title: 'I made this', body: 'this month', user_created_time: parseInt(time.goBackInTime(Date.now(), 0, 'month'), 10) }); + const n2 = await Note.save({ title: 'I made this', body: 'the month before', user_created_time: parseInt(time.goBackInTime(Date.now(), 1, 'month'), 10) }); + const n3 = await Note.save({ title: 'I made this', body: 'before before month', user_created_time: parseInt(time.goBackInTime(Date.now(), 2, 'month'), 10) }); + + await engine.syncTables(); + + rows = await engine.search('created:month-0', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); + + rows = await engine.search('created:month-1', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('created:month-2', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + })); + + it('should support filtering by created with smart value: year', (async () => { + let rows; + const n1 = await Note.save({ title: 'I made this', body: 'this year', user_created_time: parseInt(time.goBackInTime(Date.now(), 0, 'year'), 10) }); + const n2 = await Note.save({ title: 'I made this', body: 'the year before', user_created_time: parseInt(time.goBackInTime(Date.now(), 1, 'year'), 10) }); + const n3 = await Note.save({ title: 'I made this', body: 'before before year', user_created_time: parseInt(time.goBackInTime(Date.now(), 2, 'year'), 10) }); + + await engine.syncTables(); + + rows = await engine.search('created:year-0', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); + + rows = await engine.search('created:year-1', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('created:year-2', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + })); + + it('should support filtering by updated date', (async () => { + let rows; + const n1 = await Note.save({ title: 'I updated this on', body: 'May 20 2020', updated_time: Date.parse('2020-05-20'), user_updated_time: Date.parse('2020-05-20') }, { autoTimestamp: false }); + const n2 = await Note.save({ title: 'I updated this on', body: 'May 19 2020', updated_time: Date.parse('2020-05-19'), user_updated_time: Date.parse('2020-05-19') }, { autoTimestamp: false }); + + await engine.syncTables(); + + rows = await engine.search('updated:20200520', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); + + rows = await engine.search('updated:20200519', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + })); + + it('should support filtering by updated with smart value: day', (async () => { + let rows; + const today = parseInt(time.goBackInTime(Date.now(), 0, 'day'), 10); + const yesterday = parseInt(time.goBackInTime(Date.now(), 1, 'day'), 10); + const dayBeforeYesterday = parseInt(time.goBackInTime(Date.now(), 2, 'day'), 10); + const n1 = await Note.save({ title: 'I made this', body: 'today', updated_time: today, user_updated_time: today }, { autoTimestamp: false }); + const n11 = await Note.save({ title: 'I also made this', body: 'today', updated_time: today, user_updated_time: today }, { autoTimestamp: false }); + + const n2 = await Note.save({ title: 'I made this', body: 'yesterday', updated_time: yesterday, user_updated_time: yesterday }, { autoTimestamp: false }); + const n3 = await Note.save({ title: 'I made this', body: 'day before yesterday', updated_time: dayBeforeYesterday ,user_updated_time: dayBeforeYesterday }, { autoTimestamp: false }); + + await engine.syncTables(); + + rows = await engine.search('updated:day-0', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n11.id); + + rows = await engine.search('updated:day-1', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n11.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('updated:day-2', { searchType }); + expect(rows.length).toBe(4); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n11.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + })); + + it('should support filtering by type todo', (async () => { + let rows; + const t1 = await Note.save({ title: 'This is a ', body: 'todo', is_todo: 1 }); + const t2 = await Note.save({ title: 'This is another', body: 'todo but completed', is_todo: 1, todo_completed: 1590085027710 }); + const t3 = await Note.save({ title: 'This is NOT a ', body: 'todo' }); + + await engine.syncTables(); + + rows = await engine.search('type:todo', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(t1.id); + expect(ids(rows)).toContain(t2.id); + + rows = await engine.search('any:1 type:todo', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(t1.id); + expect(ids(rows)).toContain(t2.id); + + rows = await engine.search('iscompleted:1', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(t2.id); + + rows = await engine.search('iscompleted:0', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(t1.id); + })); + + it('should support filtering by type note', (async () => { + let rows; + const t1 = await Note.save({ title: 'This is a ', body: 'todo', is_todo: 1 }); + const t2 = await Note.save({ title: 'This is another', body: 'todo but completed', is_todo: 1, todo_completed: 1590085027710 }); + const t3 = await Note.save({ title: 'This is NOT a ', body: 'todo' }); + + await engine.syncTables(); + + rows = await engine.search('type:note', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(t3.id); + })); + + it('should support filtering by due date', (async () => { + let rows; + const toDo1 = await Note.save({ title: 'ToDo 1', body: 'todo', is_todo: 1, todo_due: Date.parse('2021-04-27') }); + const toDo2 = await Note.save({ title: 'ToDo 2', body: 'todo', is_todo: 1, todo_due: Date.parse('2021-03-17') }); + const note1 = await Note.save({ title: 'Note 1', body: 'Note' }); + + await engine.syncTables(); + + rows = await engine.search('due:20210425', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(toDo1.id); + + rows = await engine.search('-due:20210425', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(toDo2.id); + })); + + it('should support filtering by due with smart value: day', (async () => { + let rows; + + const inThreeDays = parseInt(time.goForwardInTime(Date.now(), 3, 'day'), 10); + const inSevenDays = parseInt(time.goForwardInTime(Date.now(), 7, 'day'), 10); + const threeDaysAgo = parseInt(time.goBackInTime(Date.now(), 3, 'day'), 10); + const sevenDaysAgo = parseInt(time.goBackInTime(Date.now(), 7, 'day'), 10); + + const toDo1 = await Note.save({ title: 'ToDo + 3 day', body: 'toto', is_todo: 1, todo_due: inThreeDays }); + const toDo2 = await Note.save({ title: 'ToDo + 7 day', body: 'toto', is_todo: 1, todo_due: inSevenDays }); + const toDo3 = await Note.save({ title: 'ToDo - 3 day', body: 'toto', is_todo: 1, todo_due: threeDaysAgo }); + const toDo4 = await Note.save({ title: 'ToDo - 7 day', body: 'toto', is_todo: 1, todo_due: sevenDaysAgo }); + + await engine.syncTables(); + + rows = await engine.search('due:day-4', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(toDo1.id); + expect(ids(rows)).toContain(toDo2.id); + expect(ids(rows)).toContain(toDo3.id); + + rows = await engine.search('-due:day-4', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(toDo4.id); + + rows = await engine.search('-due:day+4', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(toDo1.id); + expect(ids(rows)).toContain(toDo3.id); + expect(ids(rows)).toContain(toDo4.id); + + rows = await engine.search('due:day+4', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(toDo2.id); + + rows = await engine.search('due:day-4 -due:day+4', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(toDo1.id); + expect(ids(rows)).toContain(toDo3.id); + })); + + it('should support filtering by latitude, longitude, altitude', (async () => { + let rows; + const n1 = await Note.save({ title: 'I made this', body: 'this week', latitude: 12.97, longitude: 88.88, altitude: 69.96 }); + const n2 = await Note.save({ title: 'I made this', body: 'the week before', latitude: 42.11, longitude: 77.77, altitude: 42.00 }); + const n3 = await Note.save({ title: 'I made this', body: 'before before week', latitude: 82.01, longitude: 66.66, altitude: 13.13 }); + + await engine.syncTables(); + + rows = await engine.search('latitude:13.5', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + + rows = await engine.search('-latitude:40', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); + + rows = await engine.search('latitude:13 -latitude:80', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('altitude:13.5', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('-altitude:80.12', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + + rows = await engine.search('longitude:70 -longitude:80', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('latitude:20 longitude:50 altitude:40', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n2.id); + + rows = await engine.search('any:1 latitude:20 longitude:50 altitude:40', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + })); + + it('should support filtering by resource MIME type', (async () => { + let rows; + const service = new ResourceService(); + // console.log(testImagePath) + const folder1 = await Folder.save({ title: 'folder1' }); + let n1 = await Note.save({ title: 'I have a picture', body: 'Im awesome', parent_id: folder1.id }); + const n2 = await Note.save({ title: 'Boring note 1', body: 'I just have text', parent_id: folder1.id }); + const n3 = await Note.save({ title: 'Boring note 2', body: 'me too', parent_id: folder1.id }); + let n4 = await Note.save({ title: 'A picture?', body: 'pfff, I have a pdf', parent_id: folder1.id }); + await engine.syncTables(); - rows = await engine.search('resource:image/*'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); + // let note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); + n1 = await shim.attachFileToNote(n1, `${supportDir}/photo.jpg`); + // const resource1 = (await Resource.all())[0]; - rows = await engine.search('resource:application/pdf'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n4.id); + n4 = await shim.attachFileToNote(n4, `${supportDir}/welcome.pdf`); - rows = await engine.search('-resource:image/jpeg'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); - expect(ids(rows)).toContain(n4.id); + await service.indexNoteResources(); - rows = await engine.search('any:1 resource:application/pdf resource:image/jpeg'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n4.id); - })); + rows = await engine.search('resource:image/jpeg', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); - it('should ignore dashes in a word', (async () => { - const n0 = await Note.save({ title: 'doesnotwork' }); - const n1 = await Note.save({ title: 'does not work' }); - const n2 = await Note.save({ title: 'does-not-work' }); - const n3 = await Note.save({ title: 'does_not_work' }); + rows = await engine.search('resource:image/*', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); - await engine.syncTables(); + rows = await engine.search('resource:application/pdf', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n4.id); - let rows = await engine.search('does-not-work'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); + rows = await engine.search('-resource:image/jpeg', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); + expect(ids(rows)).toContain(n4.id); - rows = await engine.search('does not work'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); + rows = await engine.search('any:1 resource:application/pdf resource:image/jpeg', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n4.id); + })); - rows = await engine.search('"does not work"'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); - rows = await engine.search('title:does-not-work'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); + it('should support filtering by sourceurl', (async () => { + const n0 = await Note.save({ title: 'n0', source_url: 'https://discourse.joplinapp.org' }); + const n1 = await Note.save({ title: 'n1', source_url: 'https://google.com' }); + const n2 = await Note.save({ title: 'n2', source_url: 'https://reddit.com' }); + const n3 = await Note.save({ title: 'n3', source_url: 'https://joplinapp.org' }); - rows = await engine.search('doesnotwork'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n0.id); + await engine.syncTables(); - })); + let rows = await engine.search('sourceurl:https://joplinapp.org', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n3.id); - it('should support filtering by sourceurl', (async () => { - const n0 = await Note.save({ title: 'n0', source_url: 'https://discourse.joplinapp.org' }); - const n1 = await Note.save({ title: 'n1', source_url: 'https://google.com' }); - const n2 = await Note.save({ title: 'n2', source_url: 'https://reddit.com' }); - const n3 = await Note.save({ title: 'n3', source_url: 'https://joplinapp.org' }); + rows = await engine.search('sourceurl:https://google.com', { searchType }); + expect(rows.length).toBe(1); + expect(ids(rows)).toContain(n1.id); - await engine.syncTables(); + rows = await engine.search('any:1 sourceurl:https://google.com sourceurl:https://reddit.com', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); - let rows = await engine.search('sourceurl:https://joplinapp.org'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n3.id); + rows = await engine.search('-sourceurl:https://google.com', { searchType }); + expect(rows.length).toBe(3); + expect(ids(rows)).toContain(n0.id); + expect(ids(rows)).toContain(n2.id); + expect(ids(rows)).toContain(n3.id); - rows = await engine.search('sourceurl:https://google.com'); - expect(rows.length).toBe(1); - expect(ids(rows)).toContain(n1.id); + rows = await engine.search('sourceurl:*joplinapp.org', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n0.id); + expect(ids(rows)).toContain(n3.id); - rows = await engine.search('any:1 sourceurl:https://google.com sourceurl:https://reddit.com'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); + })); - rows = await engine.search('-sourceurl:https://google.com'); - expect(rows.length).toBe(3); - expect(ids(rows)).toContain(n0.id); - expect(ids(rows)).toContain(n2.id); - expect(ids(rows)).toContain(n3.id); + it('should support negating notebooks', (async () => { - rows = await engine.search('sourceurl:*joplinapp.org'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n0.id); - expect(ids(rows)).toContain(n3.id); - - })); + const folder1 = await Folder.save({ title: 'folder1' }); + const n1 = await Note.save({ title: 'task1', body: 'foo', parent_id: folder1.id }); + const n2 = await Note.save({ title: 'task2', body: 'bar', parent_id: folder1.id }); - it('should support negating notebooks', (async () => { - const folder1 = await Folder.save({ title: 'folder1' }); - const n1 = await Note.save({ title: 'task1', body: 'foo', parent_id: folder1.id }); - const n2 = await Note.save({ title: 'task2', body: 'bar', parent_id: folder1.id }); + const folder2 = await Folder.save({ title: 'folder2' }); + const n3 = await Note.save({ title: 'task3', body: 'baz', parent_id: folder2.id }); + const n4 = await Note.save({ title: 'task4', body: 'blah', parent_id: folder2.id }); - const folder2 = await Folder.save({ title: 'folder2' }); - const n3 = await Note.save({ title: 'task3', body: 'baz', parent_id: folder2.id }); - const n4 = await Note.save({ title: 'task4', body: 'blah', parent_id: folder2.id }); + await engine.syncTables(); + let rows = await engine.search('-notebook:folder1', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n3.id); + expect(ids(rows)).toContain(n4.id); - await engine.syncTables(); - - let rows = await engine.search('-notebook:folder1'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n3.id); - expect(ids(rows)).toContain(n4.id); + rows = await engine.search('-notebook:folder2', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); - rows = await engine.search('-notebook:folder2'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); + })); - })); + it('should support both inclusion and exclusion of notebooks together', (async () => { - it('should support both inclusion and exclusion of notebooks together', (async () => { + const parentFolder = await Folder.save({ title: 'parent' }); + const n1 = await Note.save({ title: 'task1', body: 'foo', parent_id: parentFolder.id }); + const n2 = await Note.save({ title: 'task2', body: 'bar', parent_id: parentFolder.id }); - const parentFolder = await Folder.save({ title: 'parent' }); - const n1 = await Note.save({ title: 'task1', body: 'foo', parent_id: parentFolder.id }); - const n2 = await Note.save({ title: 'task2', body: 'bar', parent_id: parentFolder.id }); + const subFolder = await Folder.save({ title: 'child', parent_id: parentFolder.id }); + const n3 = await Note.save({ title: 'task3', body: 'baz', parent_id: subFolder.id }); + const n4 = await Note.save({ title: 'task4', body: 'blah', parent_id: subFolder.id }); - const subFolder = await Folder.save({ title: 'child', parent_id: parentFolder.id }); - const n3 = await Note.save({ title: 'task3', body: 'baz', parent_id: subFolder.id }); - const n4 = await Note.save({ title: 'task4', body: 'blah', parent_id: subFolder.id }); + await engine.syncTables(); - await engine.syncTables(); + const rows = await engine.search('notebook:parent -notebook:child', { searchType }); + expect(rows.length).toBe(2); + expect(ids(rows)).toContain(n1.id); + expect(ids(rows)).toContain(n2.id); - const rows = await engine.search('notebook:parent -notebook:child'); - expect(rows.length).toBe(2); - expect(ids(rows)).toContain(n1.id); - expect(ids(rows)).toContain(n2.id); + })); - })); + it('should support filtering by note id', (async () => { + let rows; + const note1 = await Note.save({ title: 'Note 1', body: 'body' }); + const note2 = await Note.save({ title: 'Note 2', body: 'body' }); + const note3 = await Note.save({ title: 'Note 3', body: 'body' }); + await engine.syncTables(); - it('should support filtering by note id', (async () => { - let rows; - const note1 = await Note.save({ title: 'Note 1', body: 'body' }); - const note2 = await Note.save({ title: 'Note 2', body: 'body' }); - const note3 = await Note.save({ title: 'Note 3', body: 'body' }); - await engine.syncTables(); + rows = await engine.search(`id:${note1.id}`, { searchType }); + expect(rows.length).toBe(1); + expect(rows.map(r=>r.id)).toContain(note1.id); - rows = await engine.search(`id:${note1.id}`); - expect(rows.length).toBe(1); - expect(rows.map(r=>r.id)).toContain(note1.id); + rows = await engine.search(`any:1 id:${note1.id} id:${note2.id}`, { searchType }); + expect(rows.length).toBe(2); + expect(rows.map(r=>r.id)).toContain(note1.id); + expect(rows.map(r=>r.id)).toContain(note2.id); - rows = await engine.search(`any:1 id:${note1.id} id:${note2.id}`); - expect(rows.length).toBe(2); - expect(rows.map(r=>r.id)).toContain(note1.id); - expect(rows.map(r=>r.id)).toContain(note2.id); + rows = await engine.search(`any:0 id:${note1.id} id:${note2.id}`, { searchType }); + expect(rows.length).toBe(0); - rows = await engine.search(`any:0 id:${note1.id} id:${note2.id}`); - expect(rows.length).toBe(0); + rows = await engine.search(`-id:${note2.id}`, { searchType }); + expect(rows.length).toBe(2); + expect(rows.map(r=>r.id)).toContain(note1.id); + expect(rows.map(r=>r.id)).toContain(note3.id); + })); - rows = await engine.search(`-id:${note2.id}`); - expect(rows.length).toBe(2); - expect(rows.map(r=>r.id)).toContain(note1.id); - expect(rows.map(r=>r.id)).toContain(note3.id); - })); + }); + } }); diff --git a/packages/lib/services/searchengine/queryBuilder.ts b/packages/lib/services/searchengine/queryBuilder.ts index 8474cb05103..bbe7d5fbe22 100644 --- a/packages/lib/services/searchengine/queryBuilder.ts +++ b/packages/lib/services/searchengine/queryBuilder.ts @@ -21,7 +21,7 @@ enum Requirement { INCLUSION = 'INCLUSION', } -const _notebookFilter = (notebooks: string[], requirement: Requirement, conditions: string[], params: string[], withs: string[]) => { +const _notebookFilter = (notebooks: string[], requirement: Requirement, conditions: string[], params: string[], withs: string[], useFts: boolean) => { if (notebooks.length === 0) return; const likes = []; @@ -50,12 +50,13 @@ const _notebookFilter = (notebooks: string[], requirement: Requirement, conditio ON folders.parent_id=${viewName}.id )`; + const tableName = useFts ? 'notes_normalized' : 'notes'; const where = ` AND ROWID ${requirement === Requirement.EXCLUSION ? 'NOT' : ''} IN ( - SELECT notes_normalized.ROWID + SELECT ${tableName}.ROWID FROM ${viewName} - JOIN notes_normalized - ON ${viewName}.id=notes_normalized.parent_id + JOIN ${tableName} + ON ${viewName}.id=${tableName}.parent_id )`; @@ -65,12 +66,12 @@ const _notebookFilter = (notebooks: string[], requirement: Requirement, conditio }; -const notebookFilter = (terms: Term[], conditions: string[], params: string[], withs: string[]) => { +const notebookFilter = (terms: Term[], conditions: string[], params: string[], withs: string[], useFts: boolean) => { const notebooksToInclude = terms.filter(x => x.name === 'notebook' && !x.negated).map(x => x.value); - _notebookFilter(notebooksToInclude, Requirement.INCLUSION, conditions, params, withs); + _notebookFilter(notebooksToInclude, Requirement.INCLUSION, conditions, params, withs, useFts); const notebooksToExclude = terms.filter(x => x.name === 'notebook' && x.negated).map(x => x.value); - _notebookFilter(notebooksToExclude, Requirement.EXCLUSION, conditions, params, withs); + _notebookFilter(notebooksToExclude, Requirement.EXCLUSION, conditions, params, withs, useFts); }; @@ -87,7 +88,8 @@ const filterByTableName = ( noteIDs: string, requirement: Requirement, withs: string[], - tableName: string + tableName: string, + useFts: boolean ) => { const operator: Operation = getOperator(requirement, relation); @@ -144,13 +146,14 @@ const filterByTableName = ( } // Get the ROWIDs that satisfy the condition so we can filter the result + const targetTableName = useFts ? 'notes_normalized' : 'notes'; const whereCondition = ` ${relation} ROWID ${(relation === 'AND' && requirement === 'EXCLUSION') ? 'NOT' : ''} IN ( - SELECT notes_normalized.ROWID + SELECT ${targetTableName}.ROWID FROM notes_with_${requirement}_${tableName} - JOIN notes_normalized - ON notes_with_${requirement}_${tableName}.id=notes_normalized.id + JOIN ${targetTableName} + ON notes_with_${requirement}_${tableName}.id=${targetTableName}.id )`; withs.push(withCondition); @@ -159,7 +162,7 @@ const filterByTableName = ( }; -const resourceFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, withs: string[]) => { +const resourceFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, withs: string[], useFts: boolean) => { const tableName = 'resources'; const resourceIDs = ` @@ -177,15 +180,15 @@ const resourceFilter = (terms: Term[], conditions: string[], params: string[], r const excludedResources = terms.filter(x => x.name === 'resource' && x.negated); if (requiredResources.length > 0) { - filterByTableName(requiredResources, conditions, params, relation, noteIDsWithResource, Requirement.INCLUSION, withs, tableName); + filterByTableName(requiredResources, conditions, params, relation, noteIDsWithResource, Requirement.INCLUSION, withs, tableName, useFts); } if (excludedResources.length > 0) { - filterByTableName(excludedResources, conditions, params, relation, noteIDsWithResource, Requirement.EXCLUSION, withs, tableName); + filterByTableName(excludedResources, conditions, params, relation, noteIDsWithResource, Requirement.EXCLUSION, withs, tableName, useFts); } }; -const tagFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, withs: string[]) => { +const tagFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, withs: string[], useFts: boolean) => { const tableName = 'tags'; const tagIDs = ` @@ -203,30 +206,32 @@ const tagFilter = (terms: Term[], conditions: string[], params: string[], relati const excludedTags = terms.filter(x => x.name === 'tag' && x.negated); if (requiredTags.length > 0) { - filterByTableName(requiredTags, conditions, params, relation, noteIDsWithTag, Requirement.INCLUSION, withs, tableName); + filterByTableName(requiredTags, conditions, params, relation, noteIDsWithTag, Requirement.INCLUSION, withs, tableName, useFts); } if (excludedTags.length > 0) { - filterByTableName(excludedTags, conditions, params, relation, noteIDsWithTag, Requirement.EXCLUSION, withs, tableName); + filterByTableName(excludedTags, conditions, params, relation, noteIDsWithTag, Requirement.EXCLUSION, withs, tableName, useFts); } }; -const genericFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, fieldName: string) => { +const genericFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, fieldName: string, useFts: boolean) => { if (fieldName === 'iscompleted' || fieldName === 'type') { // Faster query when values can only take two distinct values - biConditionalFilter(terms, conditions, relation, fieldName); + biConditionalFilter(terms, conditions, relation, fieldName, useFts); return; } + const tableName = useFts ? 'notes_normalized' : 'notes'; + const getCondition = (term: Term) => { if (fieldName === 'sourceurl') { - return `notes_normalized.source_url ${term.negated ? 'NOT' : ''} LIKE ?`; + return `${tableName}.source_url ${term.negated ? 'NOT' : ''} LIKE ?`; } else if (fieldName === 'date' && term.name === 'due') { return `todo_due ${term.negated ? '<' : '>='} ?`; } else if (fieldName === 'id') { return `id ${term.negated ? 'NOT' : ''} LIKE ?`; } else { - return `notes_normalized.${fieldName === 'date' ? `user_${term.name}_time` : `${term.name}`} ${term.negated ? '<' : '>='} ?`; + return `${tableName}.${fieldName === 'date' ? `user_${term.name}_time` : `${term.name}`} ${term.negated ? '<' : '>='} ?`; } }; @@ -234,16 +239,16 @@ const genericFilter = (terms: Term[], conditions: string[], params: string[], re conditions.push(` ${relation} ( ${term.name === 'due' ? 'is_todo IS 1 AND ' : ''} ROWID IN ( SELECT ROWID - FROM notes_normalized + FROM ${tableName} WHERE ${getCondition(term)} ))`); params.push(term.value); }); }; -const biConditionalFilter = (terms: Term[], conditions: string[], relation: Relation, filterName: string) => { +const biConditionalFilter = (terms: Term[], conditions: string[], relation: Relation, filterName: string, useFts: boolean) => { const getCondition = (filterName: string , value: string, relation: Relation) => { - const tableName = (relation === 'AND') ? 'notes_fts' : 'notes_normalized'; + const tableName = useFts ? (relation === 'AND' ? 'notes_fts' : 'notes_normalized') : 'notes'; if (filterName === 'type') { return `${tableName}.is_todo IS ${value === 'todo' ? 1 : 0}`; } else if (filterName === 'iscompleted') { @@ -262,39 +267,44 @@ const biConditionalFilter = (terms: Term[], conditions: string[], relation: Rela AND ${getCondition(filterName, value, relation)}`); } if (relation === 'OR') { - conditions.push(` - OR ROWID IN ( - SELECT ROWID - FROM notes_normalized - WHERE ${getCondition(filterName, value, relation)} - )`); + if (useFts) { + conditions.push(` + OR ROWID IN ( + SELECT ROWID + FROM notes_normalized + WHERE ${getCondition(filterName, value, relation)} + )`); + } else { + conditions.push(` + OR ${getCondition(filterName, value, relation)}`); + } } }); }; -const noteIdFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation) => { +const noteIdFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, useFts: boolean) => { const noteIdTerms = terms.filter(x => x.name === 'id'); - genericFilter(noteIdTerms, conditions, params, relation, 'id'); + genericFilter(noteIdTerms, conditions, params, relation, 'id', useFts); }; -const typeFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation) => { +const typeFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, useFts: boolean) => { const typeTerms = terms.filter(x => x.name === 'type'); - genericFilter(typeTerms, conditions, params, relation, 'type'); + genericFilter(typeTerms, conditions, params, relation, 'type', useFts); }; -const completedFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation) => { +const completedFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, useFts: boolean) => { const completedTerms = terms.filter(x => x.name === 'iscompleted'); - genericFilter(completedTerms, conditions, params, relation, 'iscompleted'); + genericFilter(completedTerms, conditions, params, relation, 'iscompleted', useFts); }; -const locationFilter = (terms: Term[], conditons: string[], params: string[], relation: Relation) => { +const locationFilter = (terms: Term[], conditons: string[], params: string[], relation: Relation, useFts: boolean) => { const locationTerms = terms.filter(x => x.name === 'latitude' || x.name === 'longitude' || x.name === 'altitude'); - genericFilter(locationTerms, conditons, params, relation, 'location'); + genericFilter(locationTerms, conditons, params, relation, 'location', useFts); }; -const dateFilter = (terms: Term[], conditons: string[], params: string[], relation: Relation) => { +const dateFilter = (terms: Term[], conditons: string[], params: string[], relation: Relation, useFts: boolean) => { const getUnixMs = (date: string): string => { const yyyymmdd = /^[0-9]{8}$/; const yyyymm = /^[0-9]{6}$/; @@ -321,44 +331,61 @@ const dateFilter = (terms: Term[], conditons: string[], params: string[], relati const dateTerms = terms.filter(x => x.name === 'created' || x.name === 'updated' || x.name === 'due'); const unixDateTerms = dateTerms.map(term => { return { ...term, value: getUnixMs(term.value) }; }); - genericFilter(unixDateTerms, conditons, params, relation, 'date'); + genericFilter(unixDateTerms, conditons, params, relation, 'date', useFts); }; -const sourceUrlFilter = (terms: Term[], conditons: string[], params: string[], relation: Relation) => { +const sourceUrlFilter = (terms: Term[], conditons: string[], params: string[], relation: Relation, useFts: boolean) => { const urlTerms = terms.filter(x => x.name === 'sourceurl'); - genericFilter(urlTerms, conditons, params, relation, 'sourceurl'); + genericFilter(urlTerms, conditons, params, relation, 'sourceurl', useFts); }; +const trimQuotes = (str: string) => str.startsWith('"') && str.endsWith('"') ? str.substr(1, str.length - 2) : str; -const textFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation) => { - const addExcludeTextConditions = (excludedTerms: Term[], conditions: string[], params: string[], relation: Relation) => { - const type = excludedTerms[0].name === 'text' ? '' : `.${excludedTerms[0].name}`; +const textFilter = (terms: Term[], conditions: string[], params: string[], relation: Relation, useFts: boolean) => { + const createLikeMatch = (term: Term, negate: boolean) => { + const query = `${relation} ${negate ? 'NOT' : ''} ( + ${(term.name === 'text' || term.name === 'body') ? 'notes.body LIKE ? ' : ''} + ${term.name === 'text' ? 'OR' : ''} + ${(term.name === 'text' || term.name === 'title') ? 'notes.title LIKE ? ' : ''})`; - if (relation === 'AND') { - conditions.push(` - AND ROWID NOT IN ( - SELECT ROWID - FROM notes_fts - WHERE notes_fts${type} MATCH ? - )`); - params.push(excludedTerms.map(x => x.value).join(' OR ')); - } + conditions.push(query); + const param = `%${trimQuotes(term.value).replace(/\*/, '%')}%`; + params.push(param); + if (term.name === 'text') params.push(param); + }; - if (relation === 'OR') { - excludedTerms.forEach(term => { + const addExcludeTextConditions = (excludedTerms: Term[], conditions: string[], params: string[], relation: Relation) => { + if (useFts) { + const type = excludedTerms[0].name === 'text' ? '' : `.${excludedTerms[0].name}`; + if (relation === 'AND') { conditions.push(` - OR ROWID IN ( - SELECT * - FROM ( - SELECT ROWID - FROM notes_fts - EXCEPT - SELECT ROWID - FROM notes_fts - WHERE notes_fts${type} MATCH ? - ) + AND ROWID NOT IN ( + SELECT ROWID + FROM notes_fts + WHERE notes_fts${type} MATCH ? )`); - params.push(term.value); + params.push(excludedTerms.map(x => x.value).join(' OR ')); + } + if (relation === 'OR') { + excludedTerms.forEach(term => { + conditions.push(` + OR ROWID IN ( + SELECT * + FROM ( + SELECT ROWID + FROM notes_fts + EXCEPT + SELECT ROWID + FROM notes_fts + WHERE notes_fts${type} MATCH ? + ) + )`); + params.push(term.value); + }); + } + } else { + excludedTerms.forEach(term => { + createLikeMatch(term, true); }); } }; @@ -367,13 +394,19 @@ const textFilter = (terms: Term[], conditions: string[], params: string[], relat const includedTerms = allTerms.filter(x => !x.negated); if (includedTerms.length > 0) { - conditions.push(`${relation} notes_fts MATCH ?`); - const termsToMatch = includedTerms.map(term => { - if (term.name === 'text') return term.value; - else return `${term.name}:${term.value}`; - }); - const matchQuery = (relation === 'OR') ? termsToMatch.join(' OR ') : termsToMatch.join(' '); - params.push(matchQuery); + if (useFts) { + conditions.push(`${relation} notes_fts MATCH ?`); + const termsToMatch = includedTerms.map(term => { + if (term.name === 'text') return term.value; + else return `${term.name}:${term.value}`; + }); + const matchQuery = (relation === 'OR') ? termsToMatch.join(' OR ') : termsToMatch.join(' '); + params.push(matchQuery); + } else { + includedTerms.forEach(term => { + createLikeMatch(term, false); + }); + } } const excludedTextTerms = allTerms.filter(x => x.name === 'text' && x.negated); @@ -404,47 +437,48 @@ const getConnective = (terms: Term[], relation: Relation): string => { return (!notebookTerm && (relation === 'OR')) ? 'ROWID=-1' : '1'; // ROWID=-1 acts as 0 (something always false) }; -export default function queryBuilder(terms: Term[]) { +export default function queryBuilder(terms: Term[], useFts: boolean) { const queryParts: string[] = []; const params: string[] = []; const withs: string[] = []; const relation: Relation = getDefaultRelation(terms); + const tableName = useFts ? 'notes_fts' : 'notes'; + queryParts.push(` SELECT - notes_fts.id, - notes_fts.title, - offsets(notes_fts) AS offsets, - matchinfo(notes_fts, 'pcnalx') AS matchinfo, - notes_fts.user_created_time, - notes_fts.user_updated_time, - notes_fts.is_todo, - notes_fts.todo_completed, - notes_fts.parent_id - FROM notes_fts + ${tableName}.id, + ${tableName}.title, + ${useFts ? 'offsets(notes_fts) AS offsets, matchinfo(notes_fts, \'pcnalx\') AS matchinfo,' : ''} + ${tableName}.user_created_time, + ${tableName}.user_updated_time, + ${tableName}.is_todo, + ${tableName}.todo_completed, + ${tableName}.parent_id + FROM ${tableName} WHERE ${getConnective(terms, relation)}`); - noteIdFilter(terms, queryParts, params, relation); + noteIdFilter(terms, queryParts, params, relation, useFts); - notebookFilter(terms, queryParts, params, withs); + notebookFilter(terms, queryParts, params, withs, useFts); - tagFilter(terms, queryParts, params, relation, withs); + tagFilter(terms, queryParts, params, relation, withs, useFts); - resourceFilter(terms, queryParts, params, relation, withs); + resourceFilter(terms, queryParts, params, relation, withs, useFts); - textFilter(terms, queryParts, params, relation); + textFilter(terms, queryParts, params, relation, useFts); - typeFilter(terms, queryParts, params, relation); + typeFilter(terms, queryParts, params, relation, useFts); - completedFilter(terms, queryParts, params, relation); + completedFilter(terms, queryParts, params, relation, useFts); - dateFilter(terms, queryParts, params, relation); + dateFilter(terms, queryParts, params, relation, useFts); - locationFilter(terms, queryParts, params, relation); + locationFilter(terms, queryParts, params, relation, useFts); - sourceUrlFilter(terms, queryParts, params, relation); + sourceUrlFilter(terms, queryParts, params, relation, useFts); let query; if (withs.length > 0) {