From 24b6fd475901843a8cce7f5519c6637cb27cfb72 Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Thu, 15 May 2014 04:11:58 -0400 Subject: [PATCH 1/9] added time remaining estimate to console output; several small fixes --- bin/cmd.js | 95 ++++++++++++++++++++++++++++++---------------------- index.js | 20 +++++++---- package.json | 3 +- 3 files changed, 70 insertions(+), 48 deletions(-) diff --git a/bin/cmd.js b/bin/cmd.js index 01a75aff..ff2626a9 100755 --- a/bin/cmd.js +++ b/bin/cmd.js @@ -13,6 +13,7 @@ var os = require('os') var path = require('path') var numeral = require('numeral') var address = require('network-address') +var moment = require('moment') var WebTorrent = require('../') var TMP = os.tmp @@ -38,6 +39,7 @@ function usage () { console.log(' -l, --list list available files in torrent') console.log(' -t, --subtitles load subtitles file') console.log(' -h, --help display this help message') + console.log(' -q, --quiet silence stdout') console.log(' -v, --version print the current version') console.log('') } @@ -49,6 +51,7 @@ var torrentId = argv._[0] var port = Number(argv.port || argv.p) || 9000 var list = argv.list || argv.l var subtitles = argv.subtitles || argv.t +var quiet = argv.quiet || argv.q if (argv.help || argv.h) { usage() @@ -91,7 +94,7 @@ client.add(torrentId, function (err, torrent) { clivas.line('{red:error} ' + err) process.exit(1) } - + function updateMetadata () { if (torrent) { clivas.clear() @@ -99,10 +102,10 @@ client.add(torrentId, function (err, torrent) { } } - if (!torrent.metadata && !argv.quiet && !list) { + if (!torrent.metadata && !quiet && !list) { updateMetadata() torrent.swarm.on('wire', updateMetadata) - + client.once('torrent', function () { torrent.swarm.removeListener('wire', updateMetadata) }) @@ -114,13 +117,18 @@ client.once('torrent', function (torrent) { torrent.files.forEach(function (file, i) { clivas.line('{3+bold:'+i+'} : {magenta:'+file.name+'}'); }) - + process.exit(0) } - + var started = Date.now() var swarm = torrent.swarm var wires = swarm.wires + var hotswaps = 0 + + torrent.on('hotswap', function () { + hotswaps++ + }) var active = function(wire) { return !wire.peerChoking @@ -159,54 +167,61 @@ client.once('torrent', function (torrent) { var bytes = function (num) { return numeral(num).format('0.0b') } - + var getRuntime = function () { return Math.floor((Date.now() - started) / 1000) } - process.stdout.write(new Buffer('G1tIG1sySg==', 'base64')); // clear for drawing - - var draw = function() { - var unchoked = swarm.wires.filter(active) - var runtime = getRuntime() - var linesremaining = clivas.height - var peerslisted = 0 - - clivas.clear() - clivas.line('{green:open} {bold:vlc} {green:and enter} {bold:'+href+'} {green:as the network address}') - clivas.line('') - clivas.line('{yellow:info} {green:streaming} {bold:'+filename+'} {green:-} {bold:'+bytes(swarm.downloadSpeed())+'/s} {green:from} {bold:'+unchoked.length +'/'+wires.length+'} {green:peers} ') - clivas.line('{yellow:info} {green:downloaded} {bold:'+bytes(swarm.downloaded)+'} {green:and uploaded }{bold:'+bytes(swarm.uploaded)+'} {green:in }{bold:'+runtime+'s}') - clivas.line('{yellow:info} {green:peer queue size is} {bold:'+swarm.numQueued+'} ') - clivas.line('{80:}') - linesremaining -= 8 - - wires.every(function(wire) { - var tags = [] - if (wire.peerChoking) tags.push('choked') - clivas.line('{25+magenta:'+wire.remoteAddress+'} {10:'+bytes(wire.downloaded)+'} {10+cyan:'+bytes(wire.downloadSpeed())+'/s} {15+grey:'+tags.join(', ')+'} ') - peerslisted++ - return linesremaining-peerslisted > 4 - }) - linesremaining -= peerslisted + if (!quiet) { + process.stdout.write(new Buffer('G1tIG1sySg==', 'base64')); // clear for drawing + + var draw = function() { + var unchoked = swarm.wires.filter(active) + var runtime = getRuntime() + var linesremaining = clivas.height + var peerslisted = 0 + var speed = swarm.downloadSpeed() + var estimatedSecondsRemaining = Math.max(0, torrent.length - swarm.downloaded) / (speed > 0 ? speed : -1) + var estimate = moment.duration(estimatedSecondsRemaining, 'seconds').humanize() + + clivas.clear() + clivas.line('{green:open} {bold:vlc} {green:and enter} {bold:'+href+'} {green:as the network address}') + clivas.line('') + clivas.line('{yellow:info} {green:streaming} {bold:'+filename+'} {green:-} {bold:'+bytes(speed)+'/s} {green:from} {bold:'+unchoked.length +'/'+wires.length+'} {green:peers} ') + clivas.line('{yellow:info} {green:downloaded} {bold:'+bytes(swarm.downloaded)+'} {green:out of} {bold:'+bytes(torrent.length)+'} {green:and uploaded }{bold:'+bytes(swarm.uploaded)+'} {green:in }{bold:'+runtime+'s} {green:with} {bold:'+hotswaps+'} {green:hotswaps} ') + clivas.line('{yellow:info} {green:estimating} {bold:'+estimate+'} {green:remaining}; {green:peer queue size is} {bold:'+swarm.numQueued+'} ') + clivas.line('{80:}') + linesremaining -= 8 + + wires.every(function(wire) { + var tags = [] + if (wire.peerChoking) tags.push('choked') + clivas.line('{25+magenta:'+wire.remoteAddress+'} {10:'+bytes(wire.downloaded)+'} {10+cyan:'+bytes(wire.downloadSpeed())+'/s} {15+grey:'+tags.join(', ')+'} ') + peerslisted++ + return linesremaining-peerslisted > 4 + }) + linesremaining -= peerslisted + + if (wires.length > peerslisted) { + clivas.line('{80:}') + clivas.line('... and '+(wires.length-peerslisted)+' more ') + } - if (wires.length > peerslisted) { clivas.line('{80:}') - clivas.line('... and '+(wires.length-peerslisted)+' more ') + clivas.flush() } - clivas.line('{80:}') - clivas.flush() + setInterval(draw, 500) + draw() } - setInterval(draw, 500) - draw() - torrent.on('done', function () { - clivas.line('torrent downloaded {green:successfully} from {bold:'+wires.length+'} {green:peers} in {bold:'+getRuntime()+'s}!') + if (!quiet) { + clivas.line('torrent downloaded {green:successfully} from {bold:'+wires.length+'} {green:peers} in {bold:'+getRuntime()+'s}!') + } process.exit(0) }) - + /*client.on('ready', function() { swarm.removeListener('wire', onmagnet) client.server.listen(argv.port || 8888) diff --git a/index.js b/index.js index 5d3e1f89..354f66d1 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,7 @@ function WebTorrent (opts) { if (opts.list) { return } - + self.on('torrent', function (torrent) { self._onTorrent(torrent) }) @@ -26,20 +26,24 @@ function WebTorrent (opts) { // completed and handle it by stopping fetching additional data from the network } -WebTorrent.prototype.add = function (torrentId, cb) { +WebTorrent.prototype.add = function (torrentId, opts, cb) { var self = this + if (!self.ready) { + return self.once('ready', self.add.bind(self, torrentId, opts, cb)) + } + + if (typeof opts === 'function') { + cb = opts + opts = {} + } if (typeof cb !== 'function') cb = function () {} // TODO: support passing in an index to file to download // self.index = opts.index - if (!self.ready) { - return self.once('ready', self.add.bind(self, torrentId, cb)) - } - // Called once we have a torrentId that bittorrent-client can handle function onTorrentId (torrentId) { - Client.prototype.add.call(self, torrentId, cb) + Client.prototype.add.call(self, torrentId, opts, cb) } if (Client.toInfoHash(torrentId)) { @@ -73,6 +77,8 @@ WebTorrent.prototype.add = function (torrentId, cb) { WebTorrent.prototype._onTorrent = function (torrent) { var self = this + //torrent.swarm.on('wire', function () { }) + // if no index specified, use largest file // TODO: support torrent index selection correctly -- this doesn't work yet /*if (typeof torrent.index !== 'number') { diff --git a/package.json b/package.json index af452d19..cc9faef8 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,8 @@ "clivas": "^0.1.4", "concat-stream": "^1.4.4", "inherits": "^2.0.1", - "minimist": "^0.0.8" + "minimist": "^0.0.8", + "moment": "^2.6.0" }, "devDependencies": { "tape": "2.x" From f98a3fdeb9282aaa5b40e8de42d21ebf50b757c4 Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Thu, 15 May 2014 06:43:16 -0400 Subject: [PATCH 2/9] removed extraneous comment --- index.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/index.js b/index.js index 354f66d1..a0022f88 100644 --- a/index.js +++ b/index.js @@ -77,8 +77,6 @@ WebTorrent.prototype.add = function (torrentId, opts, cb) { WebTorrent.prototype._onTorrent = function (torrent) { var self = this - //torrent.swarm.on('wire', function () { }) - // if no index specified, use largest file // TODO: support torrent index selection correctly -- this doesn't work yet /*if (typeof torrent.index !== 'number') { From 1529dcff555bba3abe9a67e93d030548fdb8863f Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Thu, 15 May 2014 23:37:39 -0400 Subject: [PATCH 3/9] stream to VLC is now working --- bin/cmd.js | 82 +++++++++++++++++++++++++++++++++++++--------------- index.js | 74 +++++++++++++++++++++++++++++++++++++++++++---- package.json | 6 +++- 3 files changed, 132 insertions(+), 30 deletions(-) diff --git a/bin/cmd.js b/bin/cmd.js index 84c285bb..95bc2df4 100755 --- a/bin/cmd.js +++ b/bin/cmd.js @@ -11,6 +11,7 @@ var path = require('path') var numeral = require('numeral') var address = require('network-address') var moment = require('moment') +var proc = require('child_process') var WebTorrent = require('../') var TMP = os.tmp @@ -38,6 +39,7 @@ function usage () { console.log(' -h, --help display this help message') console.log(' -q, --quiet silence stdout') console.log(' -v, --version print the current version') + console.log(' -n, --no-quit do not quit peerflix on vlc exit') console.log('') } @@ -49,6 +51,7 @@ var port = Number(argv.port || argv.p) || 9000 var list = argv.list || argv.l var subtitles = argv.subtitles || argv.t var quiet = argv.quiet || argv.q +var noquit = argv.n || argv['no-quit'] if (argv.help || argv.h) { usage() @@ -66,6 +69,7 @@ if (!torrentId) { } var VLC_ARGS = '-q --video-on-top --play-and-exit' +//var VLC_ARGS = '--video-on-top --play-and-exit --extraintf=http:logger --verbose=2 --file-logging --logfile=vlc-log.txt' var OMX_EXEC = 'omxplayer -r -o ' + (typeof argv.omx === 'string') ? argv.omx + ' ' : 'hdmi ' @@ -82,10 +86,25 @@ var client = new WebTorrent({ quiet: true }) +var started = Date.now() +var listening = false + client.on('error', function (err) { clivas.line('{red:error} ' + err.message) }) +client.once('ready', function () { + client.server.once('error', function () { + client.server.listen(0) + }) + + client.server.listen(port) +}) + +client.server.once('listening', function () { + listening = true +}) + client.add(torrentId, function (err, torrent) { if (err) { clivas.line('{red:error} ' + err.message) @@ -109,7 +128,7 @@ client.add(torrentId, function (err, torrent) { } }) -client.once('torrent', function (torrent) { +function ontorrent (torrent) { if (list) { torrent.files.forEach(function (file, i) { clivas.line('{3+bold:'+i+'} : {magenta:'+file.name+'}') @@ -118,22 +137,7 @@ client.once('torrent', function (torrent) { process.exit(0) } - var started = Date.now() - var swarm = torrent.swarm - var wires = swarm.wires - var hotswaps = 0 - - torrent.on('hotswap', function () { - hotswaps++ - }) - - function active (wire) { - return !wire.peerChoking - } - - var href = 'http://' + address() + ':' + swarm.port + '/' - //var filename = engine.server.index.name.split('/').pop().replace(/\{|\}/g, '') - var filename = torrent.name + var href = 'http://' + address() + ':' + client.server.address().port + '/' if (argv.vlc && process.platform === 'win32') { var registry = require('windows-no-runnable').registry @@ -155,11 +159,36 @@ client.once('torrent', function (torrent) { proc.execFile(vlcPath, VLC_ARGS) } } else { - if (argv.vlc) proc.exec('vlc '+href+' '+VLC_ARGS+' || /Applications/VLC.app/Contents/MacOS/VLC '+href+' '+VLC_ARGS) + if (argv.vlc) { + var vlc = proc.exec('vlc '+href+' '+VLC_ARGS+' || /Applications/VLC.app/Contents/MacOS/VLC '+href+' '+VLC_ARGS, function (error) { + if (error) { + process.exit(1) + } + }) + + vlc.on('exit', function () { + if (!noquit) process.exit(0) + }) + } } - if (argv.omx) proc.exec(OMX_EXEC+' '+href) - if (argv.mplayer) proc.exec(MPLAYER_EXEC+' '+href) + if (argv.omx) proc.exec(OMX_EXEC + ' ' + href) + if (argv.mplayer) proc.exec(MPLAYER_EXEC + ' ' + href) + //if (quiet) console.log('server is listening on', href) + + var filename = torrent.name + //var filename = index.name.split('/').pop().replace(/\{|\}/g, '') + var swarm = torrent.swarm + var wires = swarm.wires + var hotswaps = 0 + + torrent.on('hotswap', function () { + hotswaps++ + }) + + function active (wire) { + return !wire.peerChoking + } function bytes (num) { return numeral(num).format('0.0b') @@ -218,9 +247,14 @@ client.once('torrent', function (torrent) { } process.exit(0) }) +} - /*client.on('ready', function() { - swarm.removeListener('wire', onmagnet) - client.server.listen(argv.port || 8888) - })*/ +client.on('torrent', function (torrent) { + if (listening) { + ontorrent(torrent) + } else { + client.on('listening', function (torrent) { + ontorrent(torrent) + }) + } }) diff --git a/index.js b/index.js index a0022f88..3a6b7ef6 100644 --- a/index.js +++ b/index.js @@ -4,8 +4,12 @@ module.exports = WebTorrent var Client = require('bittorrent-client') var fs = require('fs') +var url = require('url') var http = require('http') var inherits = require('inherits') +var pump = require('pump') +var mime = require('mime') +var rangeParser = require('range-parser') inherits(WebTorrent, Client) @@ -18,6 +22,8 @@ function WebTorrent (opts) { return } + self._startServer() + self.on('torrent', function (torrent) { self._onTorrent(torrent) }) @@ -38,8 +44,7 @@ WebTorrent.prototype.add = function (torrentId, opts, cb) { } if (typeof cb !== 'function') cb = function () {} - // TODO: support passing in an index to file to download - // self.index = opts.index + self.index = opts.index // Called once we have a torrentId that bittorrent-client can handle function onTorrentId (torrentId) { @@ -79,13 +84,72 @@ WebTorrent.prototype._onTorrent = function (torrent) { // if no index specified, use largest file // TODO: support torrent index selection correctly -- this doesn't work yet - /*if (typeof torrent.index !== 'number') { + if (typeof torrent.index !== 'number') { var largestFile = torrent.files.reduce(function (a, b) { return a.length > b.length ? a : b }) torrent.index = torrent.files.indexOf(largestFile) } - // TODO - torrent.files[torrent.index].select()*/ + torrent.files[torrent.index].select() + self.index = torrent.index + self.torrent = torrent +} + +WebTorrent.prototype._startServer = function () { + var self = this + self.server = http.createServer() + self.server.on('request', self._onRequest.bind(self)) +} + +WebTorrent.prototype._onRequest = function (req, res) { + var self = this + + if (!self.ready) { + return self.once('ready', self._onRequest.bind(self, req, res)) + } + + var u = url.parse(req.url) + + if (u.pathname === '/favicon.ico') { + return res.end() + } + if (u.pathname === '/') { + u.pathname = '/' + self.index + } + + var i = Number(u.pathname.slice(1)) + + if (isNaN(i) || i >= self.torrent.files.length) { + res.statusCode = 404 + return res.end() + } + + var file = self.torrent.files[i] + var range = req.headers.range + + res.setHeader('Accept-Ranges', 'bytes') + res.setHeader('Content-Type', mime.lookup(file.name)) + + if (!range) { + res.statusCode = 206 + res.setHeader('Content-Length', file.length) + if (req.method === 'HEAD') { + return res.end() + } + pump(file.createReadStream(), res) + return + } + + range = rangeParser(file.length, range)[0] // don't support multi-range reqs + res.statusCode = 206 + + var rangeStr = 'bytes ' + range.start + '-' + range.end + '/' + file.length + res.setHeader('Content-Range', rangeStr) + res.setHeader('Content-Length', range.end - range.start + 1) + + if (req.method === 'HEAD') { + return res.end() + } + pump(file.createReadStream(range), res) } diff --git a/package.json b/package.json index cc9faef8..12eabce3 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,11 @@ "concat-stream": "^1.4.4", "inherits": "^2.0.1", "minimist": "^0.0.8", - "moment": "^2.6.0" + "moment": "^2.6.0", + "mime": "^1.2.11", + "pump": "^0.3.2", + "range-parser": "^1.0.0", + "windows-no-runnable": "~0.0.6" }, "devDependencies": { "tape": "2.x" From da72f6bbe213f98575e1bf01f6cd17ee331bcf53 Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Fri, 16 May 2014 04:43:18 -0400 Subject: [PATCH 4/9] adding FSStorage; webtorrent now can save the results of a torrent download to the filesystem --- bin/cmd.js | 24 ++++-- index.js | 5 ++ lib/fs_storage.js | 190 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 7 +- 4 files changed, 220 insertions(+), 6 deletions(-) create mode 100644 lib/fs_storage.js diff --git a/bin/cmd.js b/bin/cmd.js index 95bc2df4..6fc0f03b 100755 --- a/bin/cmd.js +++ b/bin/cmd.js @@ -6,7 +6,6 @@ var cp = require('child_process') var fs = require('fs') var http = require('http') var minimist = require('minimist') -var os = require('os') var path = require('path') var numeral = require('numeral') var address = require('network-address') @@ -14,8 +13,6 @@ var moment = require('moment') var proc = require('child_process') var WebTorrent = require('../') -var TMP = os.tmp - function usage () { var logo = fs.readFileSync(path.join(__dirname, 'ascii-logo.txt'), 'utf8') logo.split('\n').forEach(function (line) { @@ -35,11 +32,12 @@ function usage () { console.log('') console.log(' -p, --port change the http port [default: 9000]') console.log(' -l, --list list available files in torrent') + console.log(' -n, --no-quit do not quit peerflix on vlc exit') + console.log(' -r, --remove remove any downloaded files on exit') console.log(' -t, --subtitles load subtitles file') console.log(' -h, --help display this help message') console.log(' -q, --quiet silence stdout') console.log(' -v, --version print the current version') - console.log(' -n, --no-quit do not quit peerflix on vlc exit') console.log('') } @@ -105,7 +103,22 @@ client.server.once('listening', function () { listening = true }) -client.add(torrentId, function (err, torrent) { +if (argv.remove || argv.r) { + function remove () { + client.destroy(function () { + process.nextTick(function () { + process.exit() + }) + }) + } + + process.on('SIGINT', remove) + process.on('SIGTERM', remove) +} + +client.add(torrentId, { + remove: argv.remove || argv.r +}, function (err, torrent) { if (err) { clivas.line('{red:error} ' + err.message) process.exit(1) @@ -258,3 +271,4 @@ client.on('torrent', function (torrent) { }) } }) + diff --git a/index.js b/index.js index 3a6b7ef6..2e6fe56a 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ module.exports = WebTorrent var Client = require('bittorrent-client') +var FSStorage = require('./lib/fs_storage') var fs = require('fs') var url = require('url') var http = require('http') @@ -10,6 +11,7 @@ var inherits = require('inherits') var pump = require('pump') var mime = require('mime') var rangeParser = require('range-parser') +var extend = require('extend.js') inherits(WebTorrent, Client) @@ -43,6 +45,9 @@ WebTorrent.prototype.add = function (torrentId, opts, cb) { opts = {} } if (typeof cb !== 'function') cb = function () {} + opts = extend({ + storage: FSStorage + }, opts) self.index = opts.index diff --git a/lib/fs_storage.js b/lib/fs_storage.js new file mode 100644 index 00000000..e678c728 --- /dev/null +++ b/lib/fs_storage.js @@ -0,0 +1,190 @@ +module.exports = FSStorage + +var Storage = require('bittorrent-client').Storage +var inherits = require('inherits') +var extend = require('extend.js') +var os = require('os') +var fs = require('fs') +var path = require('path') +var raf = require('random-access-file') +var mkdirp = require('mkdirp') +var rimraf = require('rimraf') +var thunky = require('thunky') + +var TMP = fs.existsSync('/tmp') ? '/tmp' : os.tmpDir() + +inherits(FSStorage, Storage) + +/** + * fs-backed Storage for a torrent download. + * + * @param {Object} parsedTorrent + * @param {Object} opts + */ +function FSStorage (parsedTorrent, opts) { + var self = this + opts = extend({ + nobuffer: true, + tmp: TMP, + name: 'webtorrent' + }, opts || {}) + Storage.call(self, parsedTorrent, opts) + + if (!opts.path) { + opts.path = path.join(opts.tmp, opts.name, parsedTorrent.infoHash) + } + + self.path = opts.path + + self.piecesMap = [] + self.files.forEach(function (file) { + var fileStart = file.offset + var fileEnd = fileStart + file.length + + var firstPiece = file.pieces[0].index + var lastPiece = file.pieces[file.pieces.length - 1].index + var pieceLength = file.pieceLength + + var open = thunky(function (cb) { + var filePath = path.join(self.path, file.path) + var fileDir = path.dirname(filePath) + + mkdirp(fileDir, function (err) { + if (err) return cb(err) + if (self.closed) return cb(new Error('Storage closed')) + + var fd = raf(filePath) + file.fd = fd + cb(null, fd) + }) + }) + + file.pieces.forEach(function (piece) { + var index = piece.index + + var pieceStart = index * pieceLength + var pieceEnd = pieceStart + piece.length + + var from = (fileStart < pieceStart) ? 0 : fileStart - pieceStart + var to = (fileEnd > pieceEnd) ? pieceLength : fileEnd - pieceStart + var offset = (fileStart > pieceStart) ? 0 : pieceStart - fileStart + + if (!self.piecesMap[index]) self.piecesMap[index] = [] + + self.piecesMap[index].push({ + from: from, + to: to, + offset: offset, + open: open + }) + }) + }) +} + +FSStorage.prototype.readBlock = function (index, offset, length, cb) { + var self = this + if (!cb) return console.error('FSStorage.readBlock requires a callback') + + var piece = self.pieces[index] + if (!piece) return cb(new Error("invalid piece index " + index)) + + var rangeFrom = offset + var rangeTo = rangeFrom + length + + var targets = self.piecesMap[index].filter(function (target) { + return (target.to > rangeFrom && target.from < rangeTo) + }) + + if (!targets.length) return cb(new Error("no file matching the requested range?")) + + var buffers = [] + var end = targets.length + var i = 0 + + var readFromNextFile = function(err, buffer) { + if (err) return cb(err) + if (buffer) buffers.push(buffer) + if (i >= end) return cb(null, Buffer.concat(buffers)) + + var target = targets[i++] + + var from = target.from + var to = target.to + var offset = target.offset + + if(to > rangeTo) to = rangeTo + if(from < rangeFrom) { + offset += rangeFrom - from + from = rangeFrom + } + + target.open(function(err, file) { + if (err) return cb(err) + file.read(offset, to - from, readFromNextFile) + }) + } + + readFromNextFile() +} + +// flush pieces to file once they're done and verified +FSStorage.prototype._onPieceDone = function (piece) { + var self = this + var targets = self.piecesMap[piece.index] + var end = targets.length + var i = 0 + + var writeToNextFile = function(err) { + if (err) return self.emit('error', err) + if (i >= end) { + return Storage.prototype._onPieceDone.call(self, piece) + } + + var target = targets[i++] + target.open(function(err, file) { + if (err) return self.emit('error', err) + file.write(target.offset, piece.buffer.slice(target.from, target.to), writeToNextFile) + }) + } + + writeToNextFile() +} + +/** + * Removes and cleans up any backing store for this storage. + */ +FSStorage.prototype.remove = function (cb) { + var self = this + if (!cb) cb = noop + + self.close(function (err) { + if (err) return cb(err) + var root = self.files[0].path.split(path.sep)[0] + rimraf(path.join(self.path, root), cb) + }) +} + +/** + * Closes the backing store for this storage. + */ +FSStorage.prototype.close = function (cb) { + var self = this + if (!cb) cb = noop + if (self.closed) return cb() + + Storage.prototype.close.call(self, function (err) { + if (err) return cb(err) + + var i = 0 + function loop (err) { + if (i >= self.files.length) return cb() + if (err) return cb(err) + var next = self.files[i++] + if (!next) return process.nextTick(loop) + next.fd.close(loop) + } + + process.nextTick(loop) + }) +} + diff --git a/package.json b/package.json index 12eabce3..8d0b28d8 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,12 @@ "mime": "^1.2.11", "pump": "^0.3.2", "range-parser": "^1.0.0", - "windows-no-runnable": "~0.0.6" + "windows-no-runnable": "~0.0.6", + "random-access-file": "^0.3.1", + "rimraf": "^2.2.5", + "thunky": "^0.1.0", + "mkdirp": "^0.3.5", + "extend.js": "^0.0.1" }, "devDependencies": { "tape": "2.x" From d6d66ecccc175351a1a4af6aad3c3f8362f22173 Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Fri, 16 May 2014 04:53:04 -0400 Subject: [PATCH 5/9] added small optimization to FSStorage.readBlock to read blocks from in-memory buffer if possible --- lib/fs_storage.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/fs_storage.js b/lib/fs_storage.js index e678c728..0bb5d420 100644 --- a/lib/fs_storage.js +++ b/lib/fs_storage.js @@ -88,6 +88,12 @@ FSStorage.prototype.readBlock = function (index, offset, length, cb) { var piece = self.pieces[index] if (!piece) return cb(new Error("invalid piece index " + index)) + if (piece.verified && piece.buffer) { + // piece is verified and cached in memory, so read directly from its buffer + // instead of reading from the filesystem. + return piece.readBlock(offset, length, cb) + } + var rangeFrom = offset var rangeTo = rangeFrom + length From 196d9eee696f101b664c6c64e2531080ac24576d Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Fri, 16 May 2014 05:08:01 -0400 Subject: [PATCH 6/9] support peer blocklist --- index.js | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 2e6fe56a..5b96eb8c 100644 --- a/index.js +++ b/index.js @@ -17,8 +17,9 @@ inherits(WebTorrent, Client) function WebTorrent (opts) { var self = this + opts = opts || {} + if (opts.blocklist) opts.blocklist = parseBlocklist(opts.blocklist) Client.call(self, opts) - if (!opts) opts = {} if (opts.list) { return @@ -88,7 +89,6 @@ WebTorrent.prototype._onTorrent = function (torrent) { var self = this // if no index specified, use largest file - // TODO: support torrent index selection correctly -- this doesn't work yet if (typeof torrent.index !== 'number') { var largestFile = torrent.files.reduce(function (a, b) { return a.length > b.length ? a : b @@ -158,3 +158,25 @@ WebTorrent.prototype._onRequest = function (req, res) { } pump(file.createReadStream(range), res) } + +// +// HELPER METHODS +// + +function parseBlocklist (filename) { + // TODO: support gzipped files + var blocklistData = fs.readFileSync(filename, { encoding: 'utf8' }) + var blocklist = [] + blocklistData.split('\n').forEach(function(line) { + var match = null + if ((match = /^\s*([^#].*)\s*:\s*([a-f0-9.:]+?)\s*-\s*([a-f0-9.:]+?)\s*$/.exec(line))) { + blocklist.push({ + reason: match[1], + startAddress: match[2], + endAddress: match[3] + }) + } + }) + return blocklist +} + From 976fd11e8a4653d6806ee5c67025765c87dfdb83 Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Fri, 16 May 2014 05:11:41 -0400 Subject: [PATCH 7/9] added missing commandline blocklist parsing for blocklist support --- bin/cmd.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cmd.js b/bin/cmd.js index 6fc0f03b..4f00c58b 100755 --- a/bin/cmd.js +++ b/bin/cmd.js @@ -34,6 +34,7 @@ function usage () { console.log(' -l, --list list available files in torrent') console.log(' -n, --no-quit do not quit peerflix on vlc exit') console.log(' -r, --remove remove any downloaded files on exit') + console.log(' -b, --blocklist use the specified blocklist') console.log(' -t, --subtitles load subtitles file') console.log(' -h, --help display this help message') console.log(' -q, --quiet silence stdout') @@ -50,6 +51,7 @@ var list = argv.list || argv.l var subtitles = argv.subtitles || argv.t var quiet = argv.quiet || argv.q var noquit = argv.n || argv['no-quit'] +var blocklist = argv.blocklist || argv.b if (argv.help || argv.h) { usage() @@ -81,7 +83,8 @@ if (subtitles) { var client = new WebTorrent({ list: list, - quiet: true + quiet: true, + blocklist: blocklist }) var started = Date.now() From a1afa10e49f95293cb2b36d89f67af6f5f4446ad Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Fri, 16 May 2014 05:15:02 -0400 Subject: [PATCH 8/9] . --- bin/cmd.js | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/bin/cmd.js b/bin/cmd.js index 4f00c58b..7117150d 100755 --- a/bin/cmd.js +++ b/bin/cmd.js @@ -52,6 +52,7 @@ var subtitles = argv.subtitles || argv.t var quiet = argv.quiet || argv.q var noquit = argv.n || argv['no-quit'] var blocklist = argv.blocklist || argv.b +var removeOnExit = argv.remove || argv.r if (argv.help || argv.h) { usage() @@ -106,21 +107,21 @@ client.server.once('listening', function () { listening = true }) -if (argv.remove || argv.r) { - function remove () { - client.destroy(function () { - process.nextTick(function () { - process.exit() - }) +function remove () { + client.destroy(function () { + process.nextTick(function () { + process.exit() }) - } + }) +} +if (removeOnExit) { process.on('SIGINT', remove) process.on('SIGTERM', remove) } client.add(torrentId, { - remove: argv.remove || argv.r + remove: removeOnExit }, function (err, torrent) { if (err) { clivas.line('{red:error} ' + err.message) @@ -261,7 +262,11 @@ function ontorrent (torrent) { if (!quiet) { clivas.line('torrent downloaded {green:successfully} from {bold:'+wires.length+'} {green:peers} in {bold:'+getRuntime()+'s}!') } - process.exit(0) + if (removeOnExit) { + remove() + } else { + process.exit(0) + } }) } From 8021381b1ab14cfe61e69eef7e8d53b0d8349e9c Mon Sep 17 00:00:00 2001 From: fisch0920 Date: Fri, 16 May 2014 06:17:01 -0400 Subject: [PATCH 9/9] fixed possibility of hanging process on --remove and fixed an error in FSStorage.close --- bin/cmd.js | 3 +++ lib/fs_storage.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cmd.js b/bin/cmd.js index 7117150d..4532edf3 100755 --- a/bin/cmd.js +++ b/bin/cmd.js @@ -108,6 +108,9 @@ client.server.once('listening', function () { }) function remove () { + process.removeListener('SIGINT', remove) + process.removeListener('SIGTERM', remove) + client.destroy(function () { process.nextTick(function () { process.exit() diff --git a/lib/fs_storage.js b/lib/fs_storage.js index 0bb5d420..f07d5098 100644 --- a/lib/fs_storage.js +++ b/lib/fs_storage.js @@ -186,7 +186,7 @@ FSStorage.prototype.close = function (cb) { if (i >= self.files.length) return cb() if (err) return cb(err) var next = self.files[i++] - if (!next) return process.nextTick(loop) + if (!next || !next.fd) return process.nextTick(loop) next.fd.close(loop) }