From bc2235198dcf5dfe811606c72fe5487c3d1c8eee Mon Sep 17 00:00:00 2001 From: rafapaezbas Date: Fri, 19 Dec 2025 14:27:08 +0100 Subject: [PATCH 1/5] fixed downloadRange and downloadDiff --- index.js | 4 ++-- lib/download.js | 11 +++++++++++ test.js | 47 +++++++++++++++++++++++------------------------ 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/index.js b/index.js index a555ee7..9b13bf7 100644 --- a/index.js +++ b/index.js @@ -423,8 +423,8 @@ module.exports = class Hyperdrive extends ReadyResource { const dls = [] for await (const entry of this.diff(length, folder, opts)) { - if (!entry.left) continue - const b = entry.left.value.blob + if (!entry.right) continue + const b = entry.right.value.blob if (!b) continue const blobs = await this.getBlobs() dls.push(blobs.core.download({ start: b.blockOffset, length: b.blockLength })) diff --git a/lib/download.js b/lib/download.js index 61fa08f..cc07e4f 100644 --- a/lib/download.js +++ b/lib/download.js @@ -4,6 +4,13 @@ module.exports = class Download extends ReadyResource { constructor(drive, folder, options) { super() + // it's a batch of downloads + if (Array.isArray(drive)) { + const downloads = drive + this.downloads = downloads + return + } + this.drive = drive this.folder = folder this.options = options || {} @@ -13,6 +20,10 @@ module.exports = class Download extends ReadyResource { } async _open() { + // it's a batch of downloads + if (this.downloads.length > 0) { + return Promise.allSettled(this.downloads.map((d) => d.ready())) + } const drive = this.drive const entry = !this.folder || this.folder.endsWith('/') diff --git a/test.js b/test.js index 6a5bf32..2b39153 100644 --- a/test.js +++ b/test.js @@ -745,8 +745,8 @@ test('drive.download(filename, [options])', async (t) => { } }) -test.skip('drive.downloadRange(dbRanges, blobRanges)', async (t) => { - const { drive, swarm, mirror, corestore } = await testenv(t) +test('drive.downloadRange(dbRanges, blobRanges)', async (t) => { + const { corestore, drive, swarm, mirror } = await testenv(t) swarm.on('connection', (conn) => corestore.replicate(conn)) swarm.join(drive.discoveryKey, { server: true, client: false }) await swarm.flush() @@ -755,31 +755,29 @@ test.skip('drive.downloadRange(dbRanges, blobRanges)', async (t) => { mirror.swarm.join(drive.discoveryKey, { server: false, client: true }) await mirror.swarm.flush() - const blobs = await drive.getBlobs() - const nil = b4a.from('nil') + await drive.put('/file-a', Buffer.alloc(1024)) + await drive.put('/file-b', Buffer.alloc(1024)) + await drive.put('/file-c', Buffer.alloc(1024)) - const fileBlocks = [] - const blobBlocks = [] - await drive.put('/0', nil) - fileBlocks.push(drive.core.length) - blobBlocks.push(blobs.core.length) - await drive.put('/1', nil) - await drive.put('/2', nil) - fileBlocks.push(drive.core.length) - blobBlocks.push(blobs.core.length) + await eventFlush() const fileTelem = downloadShark(mirror.drive.core) const blobTelem = downloadShark((await mirror.drive.getBlobs()).core) - const fileCount = fileTelem.count - const blobCount = blobTelem.count + const download = await mirror.drive.downloadRange( + [ + { start: 1, end: 2 }, + { start: 2, end: 3 } + ], + [{ start: 0, end: 1 }] + ) + await download.done() - await mirror.drive.get('/0') - t.is(blobCount, blobTelem.count) - t.is(fileCount, fileTelem.count) + t.is(fileTelem.count, 2) + t.is(blobTelem.count, 1) }) -test.skip('drive.downloadDiff(version, folder, [options])', async (t) => { +test('drive.downloadDiff(version, folder, [options])', async (t) => { const { drive, swarm, mirror, corestore } = await testenv(t) swarm.on('connection', (conn) => corestore.replicate(conn)) swarm.join(drive.discoveryKey, { server: true, client: false }) @@ -792,14 +790,14 @@ test.skip('drive.downloadDiff(version, folder, [options])', async (t) => { const nil = b4a.from('nil') await drive.put('/parent/child/0', nil) - await drive.put('/parent/sibling/0') + await drive.put('/parent/sibling/0', nil) await drive.put('/parent/child/1', nil) - const version = drive.version + let version = drive.version const filestelem = downloadShark(mirror.drive.core) const blobstelem = downloadShark((await mirror.drive.getBlobs()).core) - const downloadDiff = await mirror.drive.downloadDiff(version, '/parent/child') + let downloadDiff = await mirror.drive.downloadDiff(version, '/parent/child') await downloadDiff.done() let filescount = filestelem.count @@ -812,7 +810,9 @@ test.skip('drive.downloadDiff(version, folder, [options])', async (t) => { await drive.put('/parent/child/2', nil) - await mirror.drive.downloadDiff(version, '/parent/child') + version = drive.version + downloadDiff = await mirror.drive.downloadDiff(version, '/parent/child') + await downloadDiff.done() t.is(blobscount + 1, blobstelem.count) @@ -821,7 +821,6 @@ test.skip('drive.downloadDiff(version, folder, [options])', async (t) => { await mirror.drive.get('/parent/sibling/0') - t.is(filescount + 1, filestelem.count) t.is(blobscount + 1, blobstelem.count) }) From a29863fa9a17bec58d3a2850945d29cb90833f3c Mon Sep 17 00:00:00 2001 From: rafapaezbas Date: Fri, 19 Dec 2025 20:06:43 +0100 Subject: [PATCH 2/5] corrected batch case --- lib/download.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/download.js b/lib/download.js index cc07e4f..4db1edd 100644 --- a/lib/download.js +++ b/lib/download.js @@ -4,10 +4,13 @@ module.exports = class Download extends ReadyResource { constructor(drive, folder, options) { super() + this.isBatch = false + // it's a batch of downloads if (Array.isArray(drive)) { const downloads = drive this.downloads = downloads + this.isBatch = true return } @@ -21,7 +24,7 @@ module.exports = class Download extends ReadyResource { async _open() { // it's a batch of downloads - if (this.downloads.length > 0) { + if (this.isBatch) { return Promise.allSettled(this.downloads.map((d) => d.ready())) } const drive = this.drive From 0006f8a9253d264f67f6aa0c31d85878e39f8ca4 Mon Sep 17 00:00:00 2001 From: rafapaezbas Date: Mon, 22 Dec 2025 11:20:22 +0100 Subject: [PATCH 3/5] introduced DownloadBatch class --- index.js | 6 +++--- lib/download.js | 45 +++++++++++++++++++++++++++++---------------- test.js | 17 +++++------------ 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/index.js b/index.js index 9b13bf7..510ef65 100644 --- a/index.js +++ b/index.js @@ -11,7 +11,7 @@ const crypto = require('hypercore-crypto') const Hypercore = require('hypercore') const { BLOCK_NOT_AVAILABLE, BAD_ARGUMENT } = require('hypercore-errors') const Monitor = require('./lib/monitor') -const Download = require('./lib/download') +const { Download, DownloadBatch } = require('./lib/download') const keyEncoding = new SubEncoder('files', 'utf-8') @@ -430,7 +430,7 @@ module.exports = class Hyperdrive extends ReadyResource { dls.push(blobs.core.download({ start: b.blockOffset, length: b.blockLength })) } - return new Download(dls) + return new DownloadBatch([new Download(this, undefined, { downloads: dls })]) } async downloadRange(dbRanges, blobRanges) { @@ -448,7 +448,7 @@ module.exports = class Hyperdrive extends ReadyResource { dls.push(blobs.core.download(range)) } - return new Download(dls) + return new DownloadBatch([new Download(this, undefined, { downloads: dls })]) } entries(range, opts) { diff --git a/lib/download.js b/lib/download.js index 4db1edd..3adce97 100644 --- a/lib/download.js +++ b/lib/download.js @@ -1,32 +1,18 @@ const ReadyResource = require('ready-resource') -module.exports = class Download extends ReadyResource { +class Download extends ReadyResource { constructor(drive, folder, options) { super() - this.isBatch = false - - // it's a batch of downloads - if (Array.isArray(drive)) { - const downloads = drive - this.downloads = downloads - this.isBatch = true - return - } - this.drive = drive this.folder = folder this.options = options || {} - this.downloads = [] + this.downloads = this.options.downloads || [] this.destroyed = false this.ready().catch(noop) } async _open() { - // it's a batch of downloads - if (this.isBatch) { - return Promise.allSettled(this.downloads.map((d) => d.ready())) - } const drive = this.drive const entry = !this.folder || this.folder.endsWith('/') @@ -85,4 +71,31 @@ module.exports = class Download extends ReadyResource { } } +class DownloadBatch extends ReadyResource { + constructor(downloads) { + super() + this.downloads = downloads + } + + _open() { + return Promise.allSettled(this.downloads.map((d) => d.ready())) + } + + _close() { + for (const d of this.downloads) { + d.destroy() + } + } + + async done() { + await this.ready() + await Promise.allSettled(this.downloads.map((d) => d.done())) + } +} + function noop() {} + +module.exports = { + Download, + DownloadBatch +} diff --git a/test.js b/test.js index 2b39153..6ddaeea 100644 --- a/test.js +++ b/test.js @@ -769,12 +769,12 @@ test('drive.downloadRange(dbRanges, blobRanges)', async (t) => { { start: 1, end: 2 }, { start: 2, end: 3 } ], - [{ start: 0, end: 1 }] + [{ start: 0, end: 3 }] ) await download.done() - t.is(fileTelem.count, 2) - t.is(blobTelem.count, 1) + t.is(fileTelem.count, 3) + t.is(blobTelem.count, 3) }) test('drive.downloadDiff(version, folder, [options])', async (t) => { @@ -800,8 +800,8 @@ test('drive.downloadDiff(version, folder, [options])', async (t) => { let downloadDiff = await mirror.drive.downloadDiff(version, '/parent/child') await downloadDiff.done() - let filescount = filestelem.count - let blobscount = blobstelem.count + const filescount = filestelem.count + const blobscount = blobstelem.count await mirror.drive.get('/parent/child/1') @@ -815,13 +815,6 @@ test('drive.downloadDiff(version, folder, [options])', async (t) => { await downloadDiff.done() t.is(blobscount + 1, blobstelem.count) - - filescount = filestelem.count - blobscount = blobstelem.count - - await mirror.drive.get('/parent/sibling/0') - - t.is(blobscount + 1, blobstelem.count) }) test('drive.has(path)', async (t) => { From 13d0045c5858e09ae2a0cf4426f9ac097faa7183 Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Tue, 23 Dec 2025 12:23:45 +0100 Subject: [PATCH 4/5] left, aka this, is the main instance --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 510ef65..19ea4d5 100644 --- a/index.js +++ b/index.js @@ -423,8 +423,8 @@ module.exports = class Hyperdrive extends ReadyResource { const dls = [] for await (const entry of this.diff(length, folder, opts)) { - if (!entry.right) continue - const b = entry.right.value.blob + if (!entry.left) continue + const b = entry.left.value.blob if (!b) continue const blobs = await this.getBlobs() dls.push(blobs.core.download({ start: b.blockOffset, length: b.blockLength })) From 66d7324d8183f385d681dc96a7591574f3065aad Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Tue, 23 Dec 2025 12:29:33 +0100 Subject: [PATCH 5/5] no need for the extra class --- index.js | 6 +++--- lib/download.js | 29 +---------------------------- 2 files changed, 4 insertions(+), 31 deletions(-) diff --git a/index.js b/index.js index 19ea4d5..3b96470 100644 --- a/index.js +++ b/index.js @@ -11,7 +11,7 @@ const crypto = require('hypercore-crypto') const Hypercore = require('hypercore') const { BLOCK_NOT_AVAILABLE, BAD_ARGUMENT } = require('hypercore-errors') const Monitor = require('./lib/monitor') -const { Download, DownloadBatch } = require('./lib/download') +const Download = require('./lib/download') const keyEncoding = new SubEncoder('files', 'utf-8') @@ -430,7 +430,7 @@ module.exports = class Hyperdrive extends ReadyResource { dls.push(blobs.core.download({ start: b.blockOffset, length: b.blockLength })) } - return new DownloadBatch([new Download(this, undefined, { downloads: dls })]) + return new Download(this, undefined, { downloads: dls }) } async downloadRange(dbRanges, blobRanges) { @@ -448,7 +448,7 @@ module.exports = class Hyperdrive extends ReadyResource { dls.push(blobs.core.download(range)) } - return new DownloadBatch([new Download(this, undefined, { downloads: dls })]) + return new Download(this, undefined, { downloads: dls }) } entries(range, opts) { diff --git a/lib/download.js b/lib/download.js index 3adce97..9202601 100644 --- a/lib/download.js +++ b/lib/download.js @@ -1,6 +1,6 @@ const ReadyResource = require('ready-resource') -class Download extends ReadyResource { +module.exports = class Download extends ReadyResource { constructor(drive, folder, options) { super() @@ -71,31 +71,4 @@ class Download extends ReadyResource { } } -class DownloadBatch extends ReadyResource { - constructor(downloads) { - super() - this.downloads = downloads - } - - _open() { - return Promise.allSettled(this.downloads.map((d) => d.ready())) - } - - _close() { - for (const d of this.downloads) { - d.destroy() - } - } - - async done() { - await this.ready() - await Promise.allSettled(this.downloads.map((d) => d.done())) - } -} - function noop() {} - -module.exports = { - Download, - DownloadBatch -}