From 9e74b016116695e385294320d1be3d13693bb0ba Mon Sep 17 00:00:00 2001 From: Astro Date: Wed, 21 Jan 2015 21:18:19 +0100 Subject: [PATCH 1/3] storage: doc --- lib/storage.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/storage.js b/lib/storage.js index 570ae086..0d31b075 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -125,6 +125,7 @@ Piece.prototype._reset = function () { self.blocksWritten = 0 } +// called by writeBlock() once all blocks have been written Piece.prototype.verify = function (buffer) { var self = this buffer = buffer || self.buffer @@ -155,6 +156,7 @@ Piece.prototype.verify = function (buffer) { } } +// validates a readBlock()/writeBlock() offset Piece.prototype._verifyOffset = function (offset) { var self = this if (offset % BLOCK_LENGTH === 0) { @@ -165,6 +167,7 @@ Piece.prototype._verifyOffset = function (offset) { } } +// validates a writeBlock() buffer length Piece.prototype._verifyBlock = function (offset, buffer) { var self = this if (buffer.length === BLOCK_LENGTH) { From 68177e37261a798dc8c481a1b1f49e8e5bec8d04 Mon Sep 17 00:00:00 2001 From: Astro Date: Thu, 22 Jan 2015 00:01:21 +0100 Subject: [PATCH 2/3] continuous hashing --- bin/cmd.js | 26 +++++++++-------- lib/sha1/index.js | 28 +++++++++++++++++++ lib/storage.js | 71 ++++++++++++++++++++++++++++------------------- package.json | 4 +-- 4 files changed, 88 insertions(+), 41 deletions(-) create mode 100644 lib/sha1/index.js diff --git a/bin/cmd.js b/bin/cmd.js index f1e695c8..1e111452 100755 --- a/bin/cmd.js +++ b/bin/cmd.js @@ -451,18 +451,22 @@ function drawTorrent (torrent) { } var bar = '' for (var j = 0; j < piece.blocks.length; j++) { - switch(piece.blocks[j]) { - case 0: - bar += '{red:█}'; - break; - case 1: - bar += '{blue:█}'; - break; - case 2: + if (j < piece.blocksHashed) { bar += '{green:█}'; - break; - default: - throw 'Invalid block state: ' + piece.blocks[j] + } else { + switch(piece.blocks[j]) { + case 0: + bar += '{red:█}'; + break; + case 1: + bar += '{yellow:█}'; + break; + case 2: + bar += '{blue:█}'; + break; + default: + throw 'Invalid block state: ' + piece.blocks[j] + } } } clivas.line('{4+cyan:' + i + '} ' + bar); diff --git a/lib/sha1/index.js b/lib/sha1/index.js new file mode 100644 index 00000000..ff845f32 --- /dev/null +++ b/lib/sha1/index.js @@ -0,0 +1,28 @@ +var through2 = require('through2') +var crypto = require('crypto') + + +// encapsulated a crypto stream in order to: +// * lazily instantiate the underlying implementation +// * move to webworkers later on +module.exports = function SHA1 () { + var hash + function spawnOnDemand () { + if (!hash) + hash = crypto.createHash('sha1') + } + + var self = through2(function (buffer, enc, callback) { + spawnOnDemand() + hash.update(buffer) + callback() + }, function (callback) { + spawnOnDemand() + var digest = hash.digest('hex') + self.hexDigest = digest + this.push(digest) + this.push(null) + callback() + }) + return self +} diff --git a/lib/storage.js b/lib/storage.js index 0d31b075..fd5117e2 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -11,7 +11,7 @@ var FileStream = require('./file-stream') var inherits = require('inherits') var MultiStream = require('multistream') var once = require('once') -var sha1 = require('simple-sha1') +var sha1 = require('./sha1') var BLOCK_LENGTH = 16 * 1024 @@ -74,13 +74,11 @@ Piece.prototype.writeBlock = function (offset, buffer, cb) { return cb(null) } + debug('piece ' + self.index + ' writeBlock@' + offset + '+' + buffer.length) buffer.copy(self.buffer, offset) self.blocks[i] = BLOCK_WRITTEN self.blocksWritten += 1 - - if (self.blocksWritten === self.blocks.length) { - self.verify() - } + self._hashNext() cb(null) } @@ -123,36 +121,53 @@ Piece.prototype._reset = function () { self.blocks = new Buffer(Math.ceil(self.length / BLOCK_LENGTH)) self.blocks.fill(0) self.blocksWritten = 0 + self.sha = sha1() + // pull any received blocks + self.sha.on('drain', self._hashNext.bind(self)) + // triggered by _hashNext(): + self.sha.on('end', self._onHashed.bind(self)) + self.blocksHashed = 0 + // put into flowing mode + // we don't provide a sink, just waiting for 'end' + self.sha.resume() } -// called by writeBlock() once all blocks have been written -Piece.prototype.verify = function (buffer) { - var self = this - buffer = buffer || self.buffer - if (self.verified || !buffer) { +// called by writeBlock() +Piece.prototype._hashNext = function () { + if (this.verified || !this.buffer) { return } - if (self.noVerify) { - self.verified = true - onResult() - return - } - - var expectedHash - sha1(buffer, function (_expectedHash) { - expectedHash = _expectedHash - self.verified = (expectedHash === self.hash) - onResult() - }) - - function onResult () { - if (self.verified) { - self.emit('done') + debug('piece ' + this.index + ' _hashNext: blocksHashed=' + this.blocksHashed + '/' + this.blocks.length) + if (this.blocks[this.blocksHashed] === BLOCK_WRITTEN) { + debug('piece ' + this.index + ' _hashNext: write') + var block = this.buffer.slice(this.blocksHashed * BLOCK_LENGTH, (this.blocksHashed + 1) * BLOCK_LENGTH) + this.blocksHashed += 1 + if (this.sha.write(block)) { + debug('piece ' + this.index + ' _hashNext: more') + // recurse + process.nextTick(this._hashNext.bind(this)) } else { - self.emit('warning', new Error('piece ' + self.index + ' failed verification; ' + expectedHash + ' expected ' + self.hash)) - self._reset() + debug('piece ' + this.index + ' _hashNext: wait for sha readable') } + } else if (this.blocksHashed >= this.blocks.length) { + // will trigger 'done' event + debug('piece ' + this.index + ' _hashNext: end') + this.sha.end() + } else { + debug('piece ' + this.index + ' _hashNext: wait more data') + } +} + +Piece.prototype._onHashed = function () { + debug('piece ' + this.index + ' verified: ' + this.sha.hexDigest + ' == ' + this.hash) + this.verified = (this.sha.hexDigest === this.hash) + + if (this.verified) { + this.emit('done') + } else { + this.emit('warning', new Error('piece ' + this.index + ' failed verification; ' + this.sha.hexDigest + ' expected ' + this.hash)) + this._reset() } } diff --git a/package.json b/package.json index e5ff75e1..233c8c33 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "bittorrent-swarm": "webtorrent-swarm", "load-ip-set": false, "simple-get": false, - "ut_pex": false + "ut_pex": false, + "crypto": "crypto-browserify" }, "browserify": { "transform": [ @@ -61,7 +62,6 @@ "rimraf": "^2.2.5", "run-parallel": "^1.0.0", "simple-get": "^1.0.0", - "simple-sha1": "^2.0.0", "speedometer": "^0.1.2", "thunky": "^0.1.0", "torrent-discovery": "^2.0.1", From 3245a3894be0b3d5d1692d42753cca023295b80c Mon Sep 17 00:00:00 2001 From: Astro Date: Thu, 22 Jan 2015 00:05:44 +0100 Subject: [PATCH 3/3] package.json: depend on crypto-browserify --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 233c8c33..8b788750 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "browserify-versionify": "^1.0.2", "clivas": "^0.1.4", "create-torrent": "^3.4.0", + "crypto-browserify": "^3.9.9", "debug": "^2.1.0", "dezalgo": "^1.0.1", "end-of-stream": "^1.0.0",