From f0811e6f6a3a287d536414e80f828870afd1c1e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tadeusz=20So=C5=9Bnierz?= Date: Thu, 29 Oct 2020 20:24:34 +0100 Subject: [PATCH 1/5] test(mockserver): Add a searchindex to mockserver --- e2e/mockserver/mockserver.ts | 97 +++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/e2e/mockserver/mockserver.ts b/e2e/mockserver/mockserver.ts index 3285ec46b..dfa2e1e13 100644 --- a/e2e/mockserver/mockserver.ts +++ b/e2e/mockserver/mockserver.ts @@ -20,6 +20,12 @@ import { createServer, Server } from 'http'; import { createWriteStream } from 'fs'; import { mail_message_obj } from './emailresponse'; +import { XapianAPI } from 'runbox-searchindex/rmmxapianapi'; +import { loadXapian } from 'runbox-searchindex/xapian.loader'; +import { IndexingTools, MessageInfo } from 'runbox-searchindex/messageinfo'; +import { MailAddressInfo } from 'runbox-searchindex/mailaddressinfo'; + +declare var FS: any, MEMFS: any; const logger = createWriteStream('mockserver.log'); function log(line) { @@ -39,6 +45,7 @@ export class MockServer { ]; events = []; + indexPartitions = []; folders = [ { @@ -107,6 +114,10 @@ export class MockServer { }, ]; + constructor() { + loadXapian().subscribe(() => this.buildSearchIndex()); + } + public start() { log('Starting mock server'); this.server = createServer((request, response) => { @@ -140,9 +151,18 @@ export class MockServer { } else if (requesturl.indexOf('folder=Inbox') > -1) { requesturl = '/mail/download_xapian_index?inbox'; } else { - requesturl = '/mail/download_xapian_index'; + const match = requesturl.match(/fileno=(\d+)/); + if (match) { + const files = ["iamglass","docdata.glass","termlist.glass","postlist.glass"]; + const fileno = parseInt(match[1], 10); + const path = './downloadable/' + files[fileno - 1]; + console.log(`Uploading ${path}`); + const stat = FS.stat(path); + const bytes = FS.readFile(path, { encoding: 'binary' }, stat.size); + response.end(Buffer.from(bytes)); + return; + } } - } const emailendpoint = requesturl.match(/\/rest\/v1\/email\/([0-9]+)/); if (emailendpoint) { @@ -260,6 +280,9 @@ export class MockServer { case '/mail/download_xapian_index': response.end(''); break; + case '/mail/download_xapian_index?exists=check': + response.end(JSON.stringify({ 'exists': true })); + break; case '/mail/download_xapian_index?inbox': response.end(this.inboxcontents()); break; @@ -281,6 +304,9 @@ export class MockServer { } )); break; + case '/rest/v1/searchindex/partitions': + response.end(JSON.stringify({ "partitions": this.indexPartitions })); + break; default: if (request.url.indexOf('/rest') === 0) { response.end(JSON.stringify({ status: 'success' })); @@ -691,4 +717,71 @@ export class MockServer { ], ]; } + + buildSearchIndex() { + FS.mkdir("/work"); + FS.mount(MEMFS, {},"/work"); + FS.chdir("/work"); + const xapian = new XapianAPI(); + xapian.initXapianIndex('mainpartition'); + const indexer = new IndexingTools(xapian); + + const lines = this.inboxcontents().split('\n'); + // copied from rbwebmail.ts -- refactor! + lines.map((line) => { + const parts = line.split('\t'); + const from_ = parts[7]; + const to = parts[8]; + const fromInfo: MailAddressInfo[] = MailAddressInfo.parse(from_); + const toInfo: MailAddressInfo[] = MailAddressInfo.parse(to); + const size: number = parseInt(parts[10], 10); + const attachment: boolean = parts[11] === 'y'; + const seenFlag: boolean = parseInt(parts[4], 10) === 1; + const answeredFlag: boolean = parseInt(parts[5], 10) === 1; + const flaggedFlag: boolean = parseInt(parts[6], 10) === 1; + + + const ret = new MessageInfo( + parseInt(parts[0], 10), // id + new Date(), // changed date + new Date(), // message date + parts[3], // folder + seenFlag, // seen flag + answeredFlag, // answered flag + flaggedFlag, // flagged flag + fromInfo, // from + toInfo, // to + [], // cc + [], // bcc + parts[9], // subject + parts[12], // plaintext body + size, // size + attachment // attachment + ); + if (size === -1) { + // Size = -1 means deleted flag is set - ref hack in Webmail.pm + ret.deletedFlag = true; + } + return ret; + }).forEach(msg => indexer.addMessageToIndex(msg)); + const docCount = xapian.getXapianDocCount(); + console.log("Stored", docCount, "documents in xapian database"); + xapian.commitXapianUpdates(); + xapian.compactToWritableDatabase('downloadable'); + xapian.closeXapianDatabase(); + this.indexPartitions = [{ + 'folder': 'downloadable', + files: [], + numberOfMessages: docCount, + }] + FS.readdir('./downloadable').forEach(file => { + const path = './downloadable/' + file; + const size = FS.stat(path).size; + this.indexPartitions[0].files.push({ + 'filename': file, + 'compressedsize': size, + 'uncompressedsize': size, + }); + }); + } } From fa944e5118085e55426cd0d9e3b1d61a291505cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tadeusz=20So=C5=9Bnierz?= Date: Fri, 30 Oct 2020 18:43:21 +0100 Subject: [PATCH 2/5] fix(xapian): Persist the index as soon as it's downloaded Previously it would only persist after some further updates were applied to it, for some reason. --- src/app/xapian/searchservice.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/xapian/searchservice.ts b/src/app/xapian/searchservice.ts index 53bb6e8ae..171648e98 100644 --- a/src/app/xapian/searchservice.ts +++ b/src/app/xapian/searchservice.ts @@ -486,6 +486,7 @@ export class SearchService { mergeMap(() => downloadAndWriteFile('postlist.glass', 4)), ).subscribe(() => { this.api.initXapianIndex(XAPIAN_GLASS_WR); + this.indexNotPersisted = true; console.log(this.api.getXapianDocCount() + ' docs in Xapian database'); this.localSearchActivated = true; this.messagelistservice.refreshFolderCounts(); From dbaf199a86fc8296b43cabc6cff312f52c744176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tadeusz=20So=C5=9Bnierz?= Date: Wed, 4 Nov 2020 18:31:24 +0100 Subject: [PATCH 3/5] test(e2e): Add basic tests for local index --- e2e/cypress/integration/local-index.ts | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 e2e/cypress/integration/local-index.ts diff --git a/e2e/cypress/integration/local-index.ts b/e2e/cypress/integration/local-index.ts new file mode 100644 index 000000000..a63e7d391 --- /dev/null +++ b/e2e/cypress/integration/local-index.ts @@ -0,0 +1,27 @@ +/// + +describe('Using the local index', () => { + beforeEach(() => { + localStorage.setItem('localSearchPromptDisplayed221', 'true'); + }); + + it('can download and remove the local index', () => { + cy.visit('/'); + const selector = 'mat-nav-list:nth-of-type(2) mat-list-item'; + + cy.server(); + cy.route('POST', '**/verifymessages').as('messagesverified'); + + cy.get(selector).should('contain', 'Synchronize index').click() + cy.get(selector).should('contain', 'Stop index synchronization'); + + cy.wait('@messagesverified'); + + cy.get('#searchField input').type('default fix'); + cy.get('#searchField input').invoke('attr', 'placeholder').should('contain', 'showing 1 hit'); + + cy.visit('/'); + cy.get(selector).should('contain', 'Stop index synchronization').click(); + cy.get(selector).should('contain', 'Synchronize index'); + }); +}) From 85b73026cdb8f0b47ad9ecd6cacbccc19a8fc30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tadeusz=20So=C5=9Bnierz?= Date: Fri, 6 Nov 2020 16:51:34 +0100 Subject: [PATCH 4/5] build(deps): Update runbox-searchindex and use it in mockserver --- e2e/mockserver/mockserver.ts | 2 +- package-lock.json | 40 ++++++++++++++++++++++++++++++++---- package.json | 2 +- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/e2e/mockserver/mockserver.ts b/e2e/mockserver/mockserver.ts index dfa2e1e13..3f32efad8 100644 --- a/e2e/mockserver/mockserver.ts +++ b/e2e/mockserver/mockserver.ts @@ -115,7 +115,7 @@ export class MockServer { ]; constructor() { - loadXapian().subscribe(() => this.buildSearchIndex()); + loadXapian(`${process.cwd()}/node_modules/runbox-searchindex/xapianasm.js`).subscribe(() => this.buildSearchIndex()); } public start() { diff --git a/package-lock.json b/package-lock.json index aba635eaf..56e1ea461 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5409,6 +5409,16 @@ "dev": true, "optional": true }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, "blob": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.5.tgz", @@ -8485,6 +8495,13 @@ } } }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "optional": true + }, "fileset": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/fileset/-/fileset-2.0.3.tgz", @@ -11491,6 +11508,13 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, + "nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", + "dev": true, + "optional": true + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -17023,8 +17047,8 @@ } }, "runbox-searchindex": { - "version": "https://registry.npmjs.org/@runboxcom/runbox-searchindex/-/runbox-searchindex-0.2.0.tgz", - "integrity": "sha512-DTuLHJK34AFXXK1V2AkXSw1tQdeugTIHLCBDfwZdzIGWJQxkXb94LJQY8weVmhVkB/SnQcXVjLm0TmJOV2fgtQ==" + "version": "https://registry.npmjs.org/@runboxcom/runbox-searchindex/-/runbox-searchindex-0.2.1.tgz", + "integrity": "sha512-QDMAucw7+vmHW5H5MAC19VQceQYbS4y6riLZ4QuKYQHPtt+BaFJEoJRhPCTj7TMLHz+7S/NjW6cRfLEFgwsx0g==" }, "rxjs": { "version": "6.5.5", @@ -19591,7 +19615,11 @@ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", "dev": true, - "optional": true + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } }, "glob-parent": { "version": "3.1.0", @@ -20257,7 +20285,11 @@ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", "dev": true, - "optional": true + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } }, "glob-parent": { "version": "3.1.0", diff --git a/package.json b/package.json index 4d662010a..1b2fd94a8 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "moment-timezone": "^0.5.28", "npm": "^6.14.8", "rrule": "^2.6.4", - "runbox-searchindex": "https://registry.npmjs.org/@runboxcom/runbox-searchindex/-/runbox-searchindex-0.2.0.tgz", + "runbox-searchindex": "https://registry.npmjs.org/@runboxcom/runbox-searchindex/-/runbox-searchindex-0.2.1.tgz", "rxjs": "^6.5.5", "rxjs-compat": "^6.5.5", "tinymce": "^5.2.2", From 0fe97e7f5d0d40d864c6c9716e45809cc54c4929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tadeusz=20So=C5=9Bnierz?= Date: Mon, 30 Nov 2020 19:05:26 +0100 Subject: [PATCH 5/5] fix(xapian): Don't flag searchservice as initialized if no local index is present MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allowed for a race condition (seemingly exclusive to chromium-based browsers) where initial index sync would start to download and save files before the searchindex was fully initialized – since it was wrongly marked as being initialized already. --- src/app/xapian/searchservice.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/app/xapian/searchservice.ts b/src/app/xapian/searchservice.ts index 171648e98..fb21b3a83 100644 --- a/src/app/xapian/searchservice.ts +++ b/src/app/xapian/searchservice.ts @@ -193,8 +193,6 @@ export class SearchService { this.updateIndexWithNewChanges(); this.noLocalIndexFoundSubject.next(true); this.noLocalIndexFoundSubject.complete(); - this.initSubject.next(false); - this.initSubject.complete(); } }); }