chushihua

This commit is contained in:
li
2026-01-26 23:20:48 +08:00
commit 7b5aab0206
22521 changed files with 3300090 additions and 0 deletions
+3
View File
@@ -0,0 +1,3 @@
node_modules/
coverage/
.nyc_output/
+11
View File
@@ -0,0 +1,11 @@
language: node_js
sudo: false
before_install:
- "npm -g install npm"
node_js:
- "0.8"
- "0.10"
- "0.12"
- "iojs"
- "4"
- "5"
+15
View File
@@ -0,0 +1,15 @@
The ISC License
Copyright (c) Isaac Z. Schlueter and Contributors
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+35
View File
@@ -0,0 +1,35 @@
# fs-write-stream-atomic
Like `fs.createWriteStream(...)`, but atomic.
Writes to a tmp file and does an atomic `fs.rename` to move it into
place when it's done.
First rule of debugging: **It's always a race condition.**
## USAGE
```javascript
var fsWriteStreamAtomic = require('fs-write-stream-atomic')
// options are optional.
var write = fsWriteStreamAtomic('output.txt', options)
var read = fs.createReadStream('input.txt')
read.pipe(write)
// When the write stream emits a 'finish' or 'close' event,
// you can be sure that it is moved into place, and contains
// all the bytes that were written to it, even if something else
// was writing to `output.txt` at the same time.
```
### `fsWriteStreamAtomic(filename, [options])`
* `filename` {String} The file we want to write to
* `options` {Object}
* `chown` {Object} User and group to set ownership after write
* `uid` {Number}
* `gid` {Number}
* `encoding` {String} default = 'utf8'
* `mode` {Number} default = `0666`
* `flags` {String} default = `'w'`
+176
View File
@@ -0,0 +1,176 @@
var fs = require('graceful-fs')
var Writable = require('readable-stream').Writable
var util = require('util')
var MurmurHash3 = require('imurmurhash')
var iferr = require('iferr')
var crypto = require('crypto')
function murmurhex () {
var hash = MurmurHash3('')
for (var ii = 0; ii < arguments.length; ++ii) {
hash.hash('' + arguments[ii])
}
return hash.result()
}
var invocations = 0
function getTmpname (filename) {
return filename + '.' + murmurhex(__filename, process.pid, ++invocations)
}
var setImmediate = global.setImmediate || setTimeout
module.exports = WriteStreamAtomic
// Requirements:
// 1. Write everything written to the stream to a temp file.
// 2. If there are no errors:
// a. moves the temp file into its final destination
// b. emits `finish` & `closed` ONLY after the file is
// fully flushed and renamed.
// 3. If there's an error, removes the temp file.
util.inherits(WriteStreamAtomic, Writable)
function WriteStreamAtomic (path, options) {
if (!(this instanceof WriteStreamAtomic)) {
return new WriteStreamAtomic(path, options)
}
Writable.call(this, options)
this.__isWin = options && options.hasOwnProperty('isWin') ? options.isWin : process.platform === 'win32'
this.__atomicTarget = path
this.__atomicTmp = getTmpname(path)
this.__atomicChown = options && options.chown
this.__atomicClosed = false
this.__atomicStream = fs.WriteStream(this.__atomicTmp, options)
this.__atomicStream.once('open', handleOpen(this))
this.__atomicStream.once('close', handleClose(this))
this.__atomicStream.once('error', handleError(this))
}
// We have to suppress default finish emitting, because ordinarily it
// would happen as soon as `end` is called on us and all of the
// data has been written to our target stream. So we suppress
// finish from being emitted here, and only emit it after our
// target stream is closed and we've moved everything around.
WriteStreamAtomic.prototype.emit = function (event) {
if (event === 'finish') return this.__atomicStream.end()
return Writable.prototype.emit.apply(this, arguments)
}
WriteStreamAtomic.prototype._write = function (buffer, encoding, cb) {
var flushed = this.__atomicStream.write(buffer, encoding)
if (flushed) return cb()
this.__atomicStream.once('drain', cb)
}
function handleOpen (writeStream) {
return function (fd) {
writeStream.emit('open', fd)
}
}
function handleClose (writeStream) {
return function () {
if (writeStream.__atomicClosed) return
writeStream.__atomicClosed = true
if (writeStream.__atomicChown) {
var uid = writeStream.__atomicChown.uid
var gid = writeStream.__atomicChown.gid
return fs.chown(writeStream.__atomicTmp, uid, gid, iferr(cleanup, moveIntoPlace))
} else {
moveIntoPlace()
}
}
function moveIntoPlace () {
fs.rename(writeStream.__atomicTmp, writeStream.__atomicTarget, iferr(trapWindowsEPERM, end))
}
function trapWindowsEPERM (err) {
if (writeStream.__isWin &&
err.syscall && err.syscall === 'rename' &&
err.code && err.code === 'EPERM'
) {
checkFileHashes(err)
} else {
cleanup(err)
}
}
function checkFileHashes (eperm) {
var inprocess = 2
var tmpFileHash = crypto.createHash('sha512')
var targetFileHash = crypto.createHash('sha512')
fs.createReadStream(writeStream.__atomicTmp)
.on('data', function (data, enc) { tmpFileHash.update(data, enc) })
.on('error', fileHashError)
.on('end', fileHashComplete)
fs.createReadStream(writeStream.__atomicTarget)
.on('data', function (data, enc) { targetFileHash.update(data, enc) })
.on('error', fileHashError)
.on('end', fileHashComplete)
function fileHashError () {
if (inprocess === 0) return
inprocess = 0
cleanup(eperm)
}
function fileHashComplete () {
if (inprocess === 0) return
if (--inprocess) return
if (tmpFileHash.digest('hex') === targetFileHash.digest('hex')) {
return cleanup()
} else {
return cleanup(eperm)
}
}
}
function cleanup (err) {
fs.unlink(writeStream.__atomicTmp, function () {
if (err) {
writeStream.emit('error', err)
writeStream.emit('close')
} else {
end()
}
})
}
function end () {
// We have to use our parent class directly because we suppress `finish`
// events fired via our own emit method.
Writable.prototype.emit.call(writeStream, 'finish')
// Delay the close to provide the same temporal separation a physical
// file operation would have that is, the close event is emitted only
// after the async close operation completes.
setImmediate(function () {
writeStream.emit('close')
})
}
}
function handleError (writeStream) {
return function (er) {
cleanupSync()
writeStream.emit('error', er)
writeStream.__atomicClosed = true
writeStream.emit('close')
}
function cleanupSync () {
try {
fs.unlinkSync(writeStream.__atomicTmp)
} finally {
return
}
}
}
+15
View File
@@ -0,0 +1,15 @@
The ISC License
Copyright (c) 2011-2022 Isaac Z. Schlueter, Ben Noordhuis, and Contributors
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+143
View File
@@ -0,0 +1,143 @@
# graceful-fs
graceful-fs functions as a drop-in replacement for the fs module,
making various improvements.
The improvements are meant to normalize behavior across different
platforms and environments, and to make filesystem access more
resilient to errors.
## Improvements over [fs module](https://nodejs.org/api/fs.html)
* Queues up `open` and `readdir` calls, and retries them once
something closes if there is an EMFILE error from too many file
descriptors.
* fixes `lchmod` for Node versions prior to 0.6.2.
* implements `fs.lutimes` if possible. Otherwise it becomes a noop.
* ignores `EINVAL` and `EPERM` errors in `chown`, `fchown` or
`lchown` if the user isn't root.
* makes `lchmod` and `lchown` become noops, if not available.
* retries reading a file if `read` results in EAGAIN error.
On Windows, it retries renaming a file for up to one second if `EACCESS`
or `EPERM` error occurs, likely because antivirus software has locked
the directory.
## USAGE
```javascript
// use just like fs
var fs = require('graceful-fs')
// now go and do stuff with it...
fs.readFile('some-file-or-whatever', (err, data) => {
// Do stuff here.
})
```
## Sync methods
This module cannot intercept or handle `EMFILE` or `ENFILE` errors from sync
methods. If you use sync methods which open file descriptors then you are
responsible for dealing with any errors.
This is a known limitation, not a bug.
## Global Patching
If you want to patch the global fs module (or any other fs-like
module) you can do this:
```javascript
// Make sure to read the caveat below.
var realFs = require('fs')
var gracefulFs = require('graceful-fs')
gracefulFs.gracefulify(realFs)
```
This should only ever be done at the top-level application layer, in
order to delay on EMFILE errors from any fs-using dependencies. You
should **not** do this in a library, because it can cause unexpected
delays in other parts of the program.
## Changes
This module is fairly stable at this point, and used by a lot of
things. That being said, because it implements a subtle behavior
change in a core part of the node API, even modest changes can be
extremely breaking, and the versioning is thus biased towards
bumping the major when in doubt.
The main change between major versions has been switching between
providing a fully-patched `fs` module vs monkey-patching the node core
builtin, and the approach by which a non-monkey-patched `fs` was
created.
The goal is to trade `EMFILE` errors for slower fs operations. So, if
you try to open a zillion files, rather than crashing, `open`
operations will be queued up and wait for something else to `close`.
There are advantages to each approach. Monkey-patching the fs means
that no `EMFILE` errors can possibly occur anywhere in your
application, because everything is using the same core `fs` module,
which is patched. However, it can also obviously cause undesirable
side-effects, especially if the module is loaded multiple times.
Implementing a separate-but-identical patched `fs` module is more
surgical (and doesn't run the risk of patching multiple times), but
also imposes the challenge of keeping in sync with the core module.
The current approach loads the `fs` module, and then creates a
lookalike object that has all the same methods, except a few that are
patched. It is safe to use in all versions of Node from 0.8 through
7.0.
### v4
* Do not monkey-patch the fs module. This module may now be used as a
drop-in dep, and users can opt into monkey-patching the fs builtin
if their app requires it.
### v3
* Monkey-patch fs, because the eval approach no longer works on recent
node.
* fixed possible type-error throw if rename fails on windows
* verify that we *never* get EMFILE errors
* Ignore ENOSYS from chmod/chown
* clarify that graceful-fs must be used as a drop-in
### v2.1.0
* Use eval rather than monkey-patching fs.
* readdir: Always sort the results
* win32: requeue a file if error has an OK status
### v2.0
* A return to monkey patching
* wrap process.cwd
### v1.1
* wrap readFile
* Wrap fs.writeFile.
* readdir protection
* Don't clobber the fs builtin
* Handle fs.read EAGAIN errors by trying again
* Expose the curOpen counter
* No-op lchown/lchmod if not implemented
* fs.rename patch only for win32
* Patch fs.rename to handle AV software on Windows
* Close #4 Chown should not fail on einval or eperm if non-root
* Fix isaacs/fstream#1 Only wrap fs one time
* Fix #3 Start at 1024 max files, then back off on EMFILE
* lutimes that doens't blow up on Linux
* A full on-rewrite using a queue instead of just swallowing the EMFILE error
* Wrap Read/Write streams as well
### 1.0
* Update engines for node 0.6
* Be lstat-graceful on Windows
* first
+23
View File
@@ -0,0 +1,23 @@
'use strict'
module.exports = clone
var getPrototypeOf = Object.getPrototypeOf || function (obj) {
return obj.__proto__
}
function clone (obj) {
if (obj === null || typeof obj !== 'object')
return obj
if (obj instanceof Object)
var copy = { __proto__: getPrototypeOf(obj) }
else
var copy = Object.create(null)
Object.getOwnPropertyNames(obj).forEach(function (key) {
Object.defineProperty(copy, key, Object.getOwnPropertyDescriptor(obj, key))
})
return copy
}
@@ -0,0 +1,448 @@
var fs = require('fs')
var polyfills = require('./polyfills.js')
var legacy = require('./legacy-streams.js')
var clone = require('./clone.js')
var util = require('util')
/* istanbul ignore next - node 0.x polyfill */
var gracefulQueue
var previousSymbol
/* istanbul ignore else - node 0.x polyfill */
if (typeof Symbol === 'function' && typeof Symbol.for === 'function') {
gracefulQueue = Symbol.for('graceful-fs.queue')
// This is used in testing by future versions
previousSymbol = Symbol.for('graceful-fs.previous')
} else {
gracefulQueue = '___graceful-fs.queue'
previousSymbol = '___graceful-fs.previous'
}
function noop () {}
function publishQueue(context, queue) {
Object.defineProperty(context, gracefulQueue, {
get: function() {
return queue
}
})
}
var debug = noop
if (util.debuglog)
debug = util.debuglog('gfs4')
else if (/\bgfs4\b/i.test(process.env.NODE_DEBUG || ''))
debug = function() {
var m = util.format.apply(util, arguments)
m = 'GFS4: ' + m.split(/\n/).join('\nGFS4: ')
console.error(m)
}
// Once time initialization
if (!fs[gracefulQueue]) {
// This queue can be shared by multiple loaded instances
var queue = global[gracefulQueue] || []
publishQueue(fs, queue)
// Patch fs.close/closeSync to shared queue version, because we need
// to retry() whenever a close happens *anywhere* in the program.
// This is essential when multiple graceful-fs instances are
// in play at the same time.
fs.close = (function (fs$close) {
function close (fd, cb) {
return fs$close.call(fs, fd, function (err) {
// This function uses the graceful-fs shared queue
if (!err) {
resetQueue()
}
if (typeof cb === 'function')
cb.apply(this, arguments)
})
}
Object.defineProperty(close, previousSymbol, {
value: fs$close
})
return close
})(fs.close)
fs.closeSync = (function (fs$closeSync) {
function closeSync (fd) {
// This function uses the graceful-fs shared queue
fs$closeSync.apply(fs, arguments)
resetQueue()
}
Object.defineProperty(closeSync, previousSymbol, {
value: fs$closeSync
})
return closeSync
})(fs.closeSync)
if (/\bgfs4\b/i.test(process.env.NODE_DEBUG || '')) {
process.on('exit', function() {
debug(fs[gracefulQueue])
require('assert').equal(fs[gracefulQueue].length, 0)
})
}
}
if (!global[gracefulQueue]) {
publishQueue(global, fs[gracefulQueue]);
}
module.exports = patch(clone(fs))
if (process.env.TEST_GRACEFUL_FS_GLOBAL_PATCH && !fs.__patched) {
module.exports = patch(fs)
fs.__patched = true;
}
function patch (fs) {
// Everything that references the open() function needs to be in here
polyfills(fs)
fs.gracefulify = patch
fs.createReadStream = createReadStream
fs.createWriteStream = createWriteStream
var fs$readFile = fs.readFile
fs.readFile = readFile
function readFile (path, options, cb) {
if (typeof options === 'function')
cb = options, options = null
return go$readFile(path, options, cb)
function go$readFile (path, options, cb, startTime) {
return fs$readFile(path, options, function (err) {
if (err && (err.code === 'EMFILE' || err.code === 'ENFILE'))
enqueue([go$readFile, [path, options, cb], err, startTime || Date.now(), Date.now()])
else {
if (typeof cb === 'function')
cb.apply(this, arguments)
}
})
}
}
var fs$writeFile = fs.writeFile
fs.writeFile = writeFile
function writeFile (path, data, options, cb) {
if (typeof options === 'function')
cb = options, options = null
return go$writeFile(path, data, options, cb)
function go$writeFile (path, data, options, cb, startTime) {
return fs$writeFile(path, data, options, function (err) {
if (err && (err.code === 'EMFILE' || err.code === 'ENFILE'))
enqueue([go$writeFile, [path, data, options, cb], err, startTime || Date.now(), Date.now()])
else {
if (typeof cb === 'function')
cb.apply(this, arguments)
}
})
}
}
var fs$appendFile = fs.appendFile
if (fs$appendFile)
fs.appendFile = appendFile
function appendFile (path, data, options, cb) {
if (typeof options === 'function')
cb = options, options = null
return go$appendFile(path, data, options, cb)
function go$appendFile (path, data, options, cb, startTime) {
return fs$appendFile(path, data, options, function (err) {
if (err && (err.code === 'EMFILE' || err.code === 'ENFILE'))
enqueue([go$appendFile, [path, data, options, cb], err, startTime || Date.now(), Date.now()])
else {
if (typeof cb === 'function')
cb.apply(this, arguments)
}
})
}
}
var fs$copyFile = fs.copyFile
if (fs$copyFile)
fs.copyFile = copyFile
function copyFile (src, dest, flags, cb) {
if (typeof flags === 'function') {
cb = flags
flags = 0
}
return go$copyFile(src, dest, flags, cb)
function go$copyFile (src, dest, flags, cb, startTime) {
return fs$copyFile(src, dest, flags, function (err) {
if (err && (err.code === 'EMFILE' || err.code === 'ENFILE'))
enqueue([go$copyFile, [src, dest, flags, cb], err, startTime || Date.now(), Date.now()])
else {
if (typeof cb === 'function')
cb.apply(this, arguments)
}
})
}
}
var fs$readdir = fs.readdir
fs.readdir = readdir
var noReaddirOptionVersions = /^v[0-5]\./
function readdir (path, options, cb) {
if (typeof options === 'function')
cb = options, options = null
var go$readdir = noReaddirOptionVersions.test(process.version)
? function go$readdir (path, options, cb, startTime) {
return fs$readdir(path, fs$readdirCallback(
path, options, cb, startTime
))
}
: function go$readdir (path, options, cb, startTime) {
return fs$readdir(path, options, fs$readdirCallback(
path, options, cb, startTime
))
}
return go$readdir(path, options, cb)
function fs$readdirCallback (path, options, cb, startTime) {
return function (err, files) {
if (err && (err.code === 'EMFILE' || err.code === 'ENFILE'))
enqueue([
go$readdir,
[path, options, cb],
err,
startTime || Date.now(),
Date.now()
])
else {
if (files && files.sort)
files.sort()
if (typeof cb === 'function')
cb.call(this, err, files)
}
}
}
}
if (process.version.substr(0, 4) === 'v0.8') {
var legStreams = legacy(fs)
ReadStream = legStreams.ReadStream
WriteStream = legStreams.WriteStream
}
var fs$ReadStream = fs.ReadStream
if (fs$ReadStream) {
ReadStream.prototype = Object.create(fs$ReadStream.prototype)
ReadStream.prototype.open = ReadStream$open
}
var fs$WriteStream = fs.WriteStream
if (fs$WriteStream) {
WriteStream.prototype = Object.create(fs$WriteStream.prototype)
WriteStream.prototype.open = WriteStream$open
}
Object.defineProperty(fs, 'ReadStream', {
get: function () {
return ReadStream
},
set: function (val) {
ReadStream = val
},
enumerable: true,
configurable: true
})
Object.defineProperty(fs, 'WriteStream', {
get: function () {
return WriteStream
},
set: function (val) {
WriteStream = val
},
enumerable: true,
configurable: true
})
// legacy names
var FileReadStream = ReadStream
Object.defineProperty(fs, 'FileReadStream', {
get: function () {
return FileReadStream
},
set: function (val) {
FileReadStream = val
},
enumerable: true,
configurable: true
})
var FileWriteStream = WriteStream
Object.defineProperty(fs, 'FileWriteStream', {
get: function () {
return FileWriteStream
},
set: function (val) {
FileWriteStream = val
},
enumerable: true,
configurable: true
})
function ReadStream (path, options) {
if (this instanceof ReadStream)
return fs$ReadStream.apply(this, arguments), this
else
return ReadStream.apply(Object.create(ReadStream.prototype), arguments)
}
function ReadStream$open () {
var that = this
open(that.path, that.flags, that.mode, function (err, fd) {
if (err) {
if (that.autoClose)
that.destroy()
that.emit('error', err)
} else {
that.fd = fd
that.emit('open', fd)
that.read()
}
})
}
function WriteStream (path, options) {
if (this instanceof WriteStream)
return fs$WriteStream.apply(this, arguments), this
else
return WriteStream.apply(Object.create(WriteStream.prototype), arguments)
}
function WriteStream$open () {
var that = this
open(that.path, that.flags, that.mode, function (err, fd) {
if (err) {
that.destroy()
that.emit('error', err)
} else {
that.fd = fd
that.emit('open', fd)
}
})
}
function createReadStream (path, options) {
return new fs.ReadStream(path, options)
}
function createWriteStream (path, options) {
return new fs.WriteStream(path, options)
}
var fs$open = fs.open
fs.open = open
function open (path, flags, mode, cb) {
if (typeof mode === 'function')
cb = mode, mode = null
return go$open(path, flags, mode, cb)
function go$open (path, flags, mode, cb, startTime) {
return fs$open(path, flags, mode, function (err, fd) {
if (err && (err.code === 'EMFILE' || err.code === 'ENFILE'))
enqueue([go$open, [path, flags, mode, cb], err, startTime || Date.now(), Date.now()])
else {
if (typeof cb === 'function')
cb.apply(this, arguments)
}
})
}
}
return fs
}
function enqueue (elem) {
debug('ENQUEUE', elem[0].name, elem[1])
fs[gracefulQueue].push(elem)
retry()
}
// keep track of the timeout between retry() calls
var retryTimer
// reset the startTime and lastTime to now
// this resets the start of the 60 second overall timeout as well as the
// delay between attempts so that we'll retry these jobs sooner
function resetQueue () {
var now = Date.now()
for (var i = 0; i < fs[gracefulQueue].length; ++i) {
// entries that are only a length of 2 are from an older version, don't
// bother modifying those since they'll be retried anyway.
if (fs[gracefulQueue][i].length > 2) {
fs[gracefulQueue][i][3] = now // startTime
fs[gracefulQueue][i][4] = now // lastTime
}
}
// call retry to make sure we're actively processing the queue
retry()
}
function retry () {
// clear the timer and remove it to help prevent unintended concurrency
clearTimeout(retryTimer)
retryTimer = undefined
if (fs[gracefulQueue].length === 0)
return
var elem = fs[gracefulQueue].shift()
var fn = elem[0]
var args = elem[1]
// these items may be unset if they were added by an older graceful-fs
var err = elem[2]
var startTime = elem[3]
var lastTime = elem[4]
// if we don't have a startTime we have no way of knowing if we've waited
// long enough, so go ahead and retry this item now
if (startTime === undefined) {
debug('RETRY', fn.name, args)
fn.apply(null, args)
} else if (Date.now() - startTime >= 60000) {
// it's been more than 60 seconds total, bail now
debug('TIMEOUT', fn.name, args)
var cb = args.pop()
if (typeof cb === 'function')
cb.call(null, err)
} else {
// the amount of time between the last attempt and right now
var sinceAttempt = Date.now() - lastTime
// the amount of time between when we first tried, and when we last tried
// rounded up to at least 1
var sinceStart = Math.max(lastTime - startTime, 1)
// backoff. wait longer than the total time we've been retrying, but only
// up to a maximum of 100ms
var desiredDelay = Math.min(sinceStart * 1.2, 100)
// it's been long enough since the last retry, do it again
if (sinceAttempt >= desiredDelay) {
debug('RETRY', fn.name, args)
fn.apply(null, args.concat([startTime]))
} else {
// if we can't do this job yet, push it to the end of the queue
// and let the next iteration check again
fs[gracefulQueue].push(elem)
}
}
// schedule our next run if one isn't already scheduled
if (retryTimer === undefined) {
retryTimer = setTimeout(retry, 0)
}
}
@@ -0,0 +1,118 @@
var Stream = require('stream').Stream
module.exports = legacy
function legacy (fs) {
return {
ReadStream: ReadStream,
WriteStream: WriteStream
}
function ReadStream (path, options) {
if (!(this instanceof ReadStream)) return new ReadStream(path, options);
Stream.call(this);
var self = this;
this.path = path;
this.fd = null;
this.readable = true;
this.paused = false;
this.flags = 'r';
this.mode = 438; /*=0666*/
this.bufferSize = 64 * 1024;
options = options || {};
// Mixin options into this
var keys = Object.keys(options);
for (var index = 0, length = keys.length; index < length; index++) {
var key = keys[index];
this[key] = options[key];
}
if (this.encoding) this.setEncoding(this.encoding);
if (this.start !== undefined) {
if ('number' !== typeof this.start) {
throw TypeError('start must be a Number');
}
if (this.end === undefined) {
this.end = Infinity;
} else if ('number' !== typeof this.end) {
throw TypeError('end must be a Number');
}
if (this.start > this.end) {
throw new Error('start must be <= end');
}
this.pos = this.start;
}
if (this.fd !== null) {
process.nextTick(function() {
self._read();
});
return;
}
fs.open(this.path, this.flags, this.mode, function (err, fd) {
if (err) {
self.emit('error', err);
self.readable = false;
return;
}
self.fd = fd;
self.emit('open', fd);
self._read();
})
}
function WriteStream (path, options) {
if (!(this instanceof WriteStream)) return new WriteStream(path, options);
Stream.call(this);
this.path = path;
this.fd = null;
this.writable = true;
this.flags = 'w';
this.encoding = 'binary';
this.mode = 438; /*=0666*/
this.bytesWritten = 0;
options = options || {};
// Mixin options into this
var keys = Object.keys(options);
for (var index = 0, length = keys.length; index < length; index++) {
var key = keys[index];
this[key] = options[key];
}
if (this.start !== undefined) {
if ('number' !== typeof this.start) {
throw TypeError('start must be a Number');
}
if (this.start < 0) {
throw new Error('start must be >= zero');
}
this.pos = this.start;
}
this.busy = false;
this._queue = [];
if (this.fd === null) {
this._open = fs.open;
this._queue.push([this._open, this.path, this.flags, this.mode, undefined]);
this.flush();
}
}
}
@@ -0,0 +1,50 @@
{
"name": "graceful-fs",
"description": "A drop-in replacement for fs, making various improvements.",
"version": "4.2.10",
"repository": {
"type": "git",
"url": "https://github.com/isaacs/node-graceful-fs"
},
"main": "graceful-fs.js",
"directories": {
"test": "test"
},
"scripts": {
"preversion": "npm test",
"postversion": "npm publish",
"postpublish": "git push origin --follow-tags",
"test": "nyc --silent node test.js | tap -c -",
"posttest": "nyc report"
},
"keywords": [
"fs",
"module",
"reading",
"retry",
"retries",
"queue",
"error",
"errors",
"handling",
"EMFILE",
"EAGAIN",
"EINVAL",
"EPERM",
"EACCESS"
],
"license": "ISC",
"devDependencies": {
"import-fresh": "^2.0.0",
"mkdirp": "^0.5.0",
"rimraf": "^2.2.8",
"tap": "^12.7.0"
},
"files": [
"fs.js",
"graceful-fs.js",
"legacy-streams.js",
"polyfills.js",
"clone.js"
]
}
@@ -0,0 +1,355 @@
var constants = require('constants')
var origCwd = process.cwd
var cwd = null
var platform = process.env.GRACEFUL_FS_PLATFORM || process.platform
process.cwd = function() {
if (!cwd)
cwd = origCwd.call(process)
return cwd
}
try {
process.cwd()
} catch (er) {}
// This check is needed until node.js 12 is required
if (typeof process.chdir === 'function') {
var chdir = process.chdir
process.chdir = function (d) {
cwd = null
chdir.call(process, d)
}
if (Object.setPrototypeOf) Object.setPrototypeOf(process.chdir, chdir)
}
module.exports = patch
function patch (fs) {
// (re-)implement some things that are known busted or missing.
// lchmod, broken prior to 0.6.2
// back-port the fix here.
if (constants.hasOwnProperty('O_SYMLINK') &&
process.version.match(/^v0\.6\.[0-2]|^v0\.5\./)) {
patchLchmod(fs)
}
// lutimes implementation, or no-op
if (!fs.lutimes) {
patchLutimes(fs)
}
// https://github.com/isaacs/node-graceful-fs/issues/4
// Chown should not fail on einval or eperm if non-root.
// It should not fail on enosys ever, as this just indicates
// that a fs doesn't support the intended operation.
fs.chown = chownFix(fs.chown)
fs.fchown = chownFix(fs.fchown)
fs.lchown = chownFix(fs.lchown)
fs.chmod = chmodFix(fs.chmod)
fs.fchmod = chmodFix(fs.fchmod)
fs.lchmod = chmodFix(fs.lchmod)
fs.chownSync = chownFixSync(fs.chownSync)
fs.fchownSync = chownFixSync(fs.fchownSync)
fs.lchownSync = chownFixSync(fs.lchownSync)
fs.chmodSync = chmodFixSync(fs.chmodSync)
fs.fchmodSync = chmodFixSync(fs.fchmodSync)
fs.lchmodSync = chmodFixSync(fs.lchmodSync)
fs.stat = statFix(fs.stat)
fs.fstat = statFix(fs.fstat)
fs.lstat = statFix(fs.lstat)
fs.statSync = statFixSync(fs.statSync)
fs.fstatSync = statFixSync(fs.fstatSync)
fs.lstatSync = statFixSync(fs.lstatSync)
// if lchmod/lchown do not exist, then make them no-ops
if (fs.chmod && !fs.lchmod) {
fs.lchmod = function (path, mode, cb) {
if (cb) process.nextTick(cb)
}
fs.lchmodSync = function () {}
}
if (fs.chown && !fs.lchown) {
fs.lchown = function (path, uid, gid, cb) {
if (cb) process.nextTick(cb)
}
fs.lchownSync = function () {}
}
// on Windows, A/V software can lock the directory, causing this
// to fail with an EACCES or EPERM if the directory contains newly
// created files. Try again on failure, for up to 60 seconds.
// Set the timeout this long because some Windows Anti-Virus, such as Parity
// bit9, may lock files for up to a minute, causing npm package install
// failures. Also, take care to yield the scheduler. Windows scheduling gives
// CPU to a busy looping process, which can cause the program causing the lock
// contention to be starved of CPU by node, so the contention doesn't resolve.
if (platform === "win32") {
fs.rename = typeof fs.rename !== 'function' ? fs.rename
: (function (fs$rename) {
function rename (from, to, cb) {
var start = Date.now()
var backoff = 0;
fs$rename(from, to, function CB (er) {
if (er
&& (er.code === "EACCES" || er.code === "EPERM")
&& Date.now() - start < 60000) {
setTimeout(function() {
fs.stat(to, function (stater, st) {
if (stater && stater.code === "ENOENT")
fs$rename(from, to, CB);
else
cb(er)
})
}, backoff)
if (backoff < 100)
backoff += 10;
return;
}
if (cb) cb(er)
})
}
if (Object.setPrototypeOf) Object.setPrototypeOf(rename, fs$rename)
return rename
})(fs.rename)
}
// if read() returns EAGAIN, then just try it again.
fs.read = typeof fs.read !== 'function' ? fs.read
: (function (fs$read) {
function read (fd, buffer, offset, length, position, callback_) {
var callback
if (callback_ && typeof callback_ === 'function') {
var eagCounter = 0
callback = function (er, _, __) {
if (er && er.code === 'EAGAIN' && eagCounter < 10) {
eagCounter ++
return fs$read.call(fs, fd, buffer, offset, length, position, callback)
}
callback_.apply(this, arguments)
}
}
return fs$read.call(fs, fd, buffer, offset, length, position, callback)
}
// This ensures `util.promisify` works as it does for native `fs.read`.
if (Object.setPrototypeOf) Object.setPrototypeOf(read, fs$read)
return read
})(fs.read)
fs.readSync = typeof fs.readSync !== 'function' ? fs.readSync
: (function (fs$readSync) { return function (fd, buffer, offset, length, position) {
var eagCounter = 0
while (true) {
try {
return fs$readSync.call(fs, fd, buffer, offset, length, position)
} catch (er) {
if (er.code === 'EAGAIN' && eagCounter < 10) {
eagCounter ++
continue
}
throw er
}
}
}})(fs.readSync)
function patchLchmod (fs) {
fs.lchmod = function (path, mode, callback) {
fs.open( path
, constants.O_WRONLY | constants.O_SYMLINK
, mode
, function (err, fd) {
if (err) {
if (callback) callback(err)
return
}
// prefer to return the chmod error, if one occurs,
// but still try to close, and report closing errors if they occur.
fs.fchmod(fd, mode, function (err) {
fs.close(fd, function(err2) {
if (callback) callback(err || err2)
})
})
})
}
fs.lchmodSync = function (path, mode) {
var fd = fs.openSync(path, constants.O_WRONLY | constants.O_SYMLINK, mode)
// prefer to return the chmod error, if one occurs,
// but still try to close, and report closing errors if they occur.
var threw = true
var ret
try {
ret = fs.fchmodSync(fd, mode)
threw = false
} finally {
if (threw) {
try {
fs.closeSync(fd)
} catch (er) {}
} else {
fs.closeSync(fd)
}
}
return ret
}
}
function patchLutimes (fs) {
if (constants.hasOwnProperty("O_SYMLINK") && fs.futimes) {
fs.lutimes = function (path, at, mt, cb) {
fs.open(path, constants.O_SYMLINK, function (er, fd) {
if (er) {
if (cb) cb(er)
return
}
fs.futimes(fd, at, mt, function (er) {
fs.close(fd, function (er2) {
if (cb) cb(er || er2)
})
})
})
}
fs.lutimesSync = function (path, at, mt) {
var fd = fs.openSync(path, constants.O_SYMLINK)
var ret
var threw = true
try {
ret = fs.futimesSync(fd, at, mt)
threw = false
} finally {
if (threw) {
try {
fs.closeSync(fd)
} catch (er) {}
} else {
fs.closeSync(fd)
}
}
return ret
}
} else if (fs.futimes) {
fs.lutimes = function (_a, _b, _c, cb) { if (cb) process.nextTick(cb) }
fs.lutimesSync = function () {}
}
}
function chmodFix (orig) {
if (!orig) return orig
return function (target, mode, cb) {
return orig.call(fs, target, mode, function (er) {
if (chownErOk(er)) er = null
if (cb) cb.apply(this, arguments)
})
}
}
function chmodFixSync (orig) {
if (!orig) return orig
return function (target, mode) {
try {
return orig.call(fs, target, mode)
} catch (er) {
if (!chownErOk(er)) throw er
}
}
}
function chownFix (orig) {
if (!orig) return orig
return function (target, uid, gid, cb) {
return orig.call(fs, target, uid, gid, function (er) {
if (chownErOk(er)) er = null
if (cb) cb.apply(this, arguments)
})
}
}
function chownFixSync (orig) {
if (!orig) return orig
return function (target, uid, gid) {
try {
return orig.call(fs, target, uid, gid)
} catch (er) {
if (!chownErOk(er)) throw er
}
}
}
function statFix (orig) {
if (!orig) return orig
// Older versions of Node erroneously returned signed integers for
// uid + gid.
return function (target, options, cb) {
if (typeof options === 'function') {
cb = options
options = null
}
function callback (er, stats) {
if (stats) {
if (stats.uid < 0) stats.uid += 0x100000000
if (stats.gid < 0) stats.gid += 0x100000000
}
if (cb) cb.apply(this, arguments)
}
return options ? orig.call(fs, target, options, callback)
: orig.call(fs, target, callback)
}
}
function statFixSync (orig) {
if (!orig) return orig
// Older versions of Node erroneously returned signed integers for
// uid + gid.
return function (target, options) {
var stats = options ? orig.call(fs, target, options)
: orig.call(fs, target)
if (stats) {
if (stats.uid < 0) stats.uid += 0x100000000
if (stats.gid < 0) stats.gid += 0x100000000
}
return stats;
}
}
// ENOSYS means that the fs doesn't support the op. Just ignore
// that, because it doesn't matter.
//
// if there's no getuid, or if getuid() is something other
// than 0, and the error is EINVAL or EPERM, then just ignore
// it.
//
// This specific case is a silent failure in cp, install, tar,
// and most other unix tools that manage permissions.
//
// When running as root, or if other types of errors are
// encountered, then it's strict.
function chownErOk (er) {
if (!er)
return true
if (er.code === "ENOSYS")
return true
var nonroot = !process.getuid || process.getuid() !== 0
if (nonroot) {
if (er.code === "EINVAL" || er.code === "EPERM")
return true
}
return false
}
}
+33
View File
@@ -0,0 +1,33 @@
{
"name": "fs-write-stream-atomic",
"version": "1.0.10",
"description": "Like `fs.createWriteStream(...)`, but atomic.",
"main": "index.js",
"directories": {
"test": "test"
},
"dependencies": {
"graceful-fs": "^4.1.2",
"iferr": "^0.1.5",
"imurmurhash": "^0.1.4",
"readable-stream": "1 || 2"
},
"devDependencies": {
"rimraf": "^2.4.4",
"standard": "^5.4.1",
"tap": "^2.3.1"
},
"scripts": {
"test": "standard && tap --coverage test/*.js"
},
"repository": {
"type": "git",
"url": "https://github.com/npm/fs-write-stream-atomic"
},
"author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me/)",
"license": "ISC",
"bugs": {
"url": "https://github.com/npm/fs-write-stream-atomic/issues"
},
"homepage": "https://github.com/npm/fs-write-stream-atomic"
}
+97
View File
@@ -0,0 +1,97 @@
var fs = require('graceful-fs')
var test = require('tap').test
var path = require('path')
var writeStream = require('../index.js')
var rename = fs.rename
fs.rename = function (from, to, cb) {
setTimeout(function () {
rename(from, to, cb)
}, 100)
}
test('basic', function (t) {
// open 10 write streams to the same file.
// then write to each of them, and to the target
// and verify at the end that each of them does their thing
var target = path.resolve(__dirname, 'test.txt')
var n = 10
// We run all of our assertions twice:
// once for finish, once for close
// There are 6 assertions, two fixed, plus 4 lines in the file.
t.plan(n * 2 * 6)
var streams = []
for (var i = 0; i < n; i++) {
var s = writeStream(target)
s.on('finish', verifier('finish', i))
s.on('close', verifier('close', i))
streams.push(s)
}
function verifier (ev, num) {
return function () {
if (ev === 'close') {
t.equal(this.__emittedFinish, true, num + '. closed only after finish')
} else {
this.__emittedFinish = true
t.equal(ev, 'finish', num + '. finished')
}
// make sure that one of the atomic streams won.
var res = fs.readFileSync(target, 'utf8')
var lines = res.trim().split(/\n/)
lines.forEach(function (line, lineno) {
var first = lines[0].match(/\d+$/)[0]
var cur = line.match(/\d+$/)[0]
t.equal(cur, first, num + '. line ' + lineno + ' matches')
})
var resExpr = /^first write \d+\nsecond write \d+\nthird write \d+\nfinal write \d+\n$/
t.similar(res, resExpr, num + '. content matches')
}
}
// now write something to each stream.
streams.forEach(function (stream, i) {
stream.write('first write ' + i + '\n')
})
// wait a sec for those writes to go out.
setTimeout(function () {
// write something else to the target.
fs.writeFileSync(target, 'brutality!\n')
// write some more stuff.
streams.forEach(function (stream, i) {
stream.write('second write ' + i + '\n')
})
setTimeout(function () {
// Oops! Deleted the file!
fs.unlinkSync(target)
// write some more stuff.
streams.forEach(function (stream, i) {
stream.write('third write ' + i + '\n')
})
setTimeout(function () {
fs.writeFileSync(target, 'brutality TWO!\n')
streams.forEach(function (stream, i) {
stream.end('final write ' + i + '\n')
})
}, 50)
}, 50)
}, 50)
})
test('cleanup', function (t) {
fs.readdirSync(__dirname).filter(function (f) {
return f.match(/^test.txt/)
}).forEach(function (file) {
fs.unlinkSync(path.resolve(__dirname, file))
})
t.end()
})
+44
View File
@@ -0,0 +1,44 @@
'use strict'
var fs = require('graceful-fs')
var path = require('path')
var test = require('tap').test
var rimraf = require('rimraf')
var writeStream = require('../index.js')
var target = path.resolve(__dirname, 'test-chown')
test('chown works', function (t) {
t.plan(1)
var stream = writeStream(target, {chown: {uid: process.getuid(), gid: process.getgid()}})
var hadError = false
stream.on('error', function (er) {
hadError = true
console.log('#', er)
})
stream.on('close', function () {
t.is(hadError, false, 'no errors before close')
})
stream.end()
})
test('chown fails', function (t) {
t.plan(1)
fs.chown = function (file, uid, gid, cb) {
cb(new Error('TEST BREAK'))
}
var stream = writeStream(target, {chown: {uid: process.getuid(), gid: process.getgid()}})
var hadError = false
stream.on('error', function (er) {
hadError = true
console.log('#', er)
})
stream.on('close', function () {
t.is(hadError, true, 'error before close')
})
stream.end()
})
test('cleanup', function (t) {
rimraf.sync(target)
t.end()
})
+154
View File
@@ -0,0 +1,154 @@
'use strict'
var fs = require('graceful-fs')
var path = require('path')
var test = require('tap').test
var rimraf = require('rimraf')
var writeStream = require('../index.js')
var target = path.resolve(__dirname, 'test-rename-eperm1')
var target2 = path.resolve(__dirname, 'test-rename-eperm2')
var target3 = path.resolve(__dirname, 'test-rename-eperm3')
test('rename eperm none existing file', function (t) {
t.plan(2)
var _rename = fs.rename
fs.existsSync = function (src) {
return true
}
fs.rename = function (src, dest, cb) {
// simulate a failure during rename where the file
// is renamed successfully but the process encounters
// an EPERM error and the target file does not exist
_rename(src, dest, function (e) {
var err = new Error('TEST BREAK')
err.syscall = 'rename'
err.code = 'EPERM'
cb(err)
})
}
var stream = writeStream(target, { isWin: true })
var hadError = false
var calledFinish = false
stream.on('error', function (er) {
hadError = true
console.log('#', er)
})
stream.on('finish', function () {
calledFinish = true
})
stream.on('close', function () {
t.is(hadError, true, 'error was caught')
t.is(calledFinish, false, 'finish was called before close')
})
stream.end()
})
// test existing file with diff. content
test('rename eperm existing file different content', function (t) {
t.plan(2)
var _rename = fs.rename
fs.existsSync = function (src) {
return true
}
fs.rename = function (src, dest, cb) {
// simulate a failure during rename where the file
// is renamed successfully but the process encounters
// an EPERM error and the target file that has another content than the
// destination
_rename(src, dest, function (e) {
fs.writeFile(src, 'dest', function (writeErr) {
if (writeErr) {
return console.log('WRITEERR: ' + writeErr)
}
fs.writeFile(target2, 'target', function (writeErr) {
if (writeErr) {
return console.log('WRITEERR: ' + writeErr)
}
var err = new Error('TEST BREAK')
err.syscall = 'rename'
err.code = 'EPERM'
cb(err)
})
})
})
}
var stream = writeStream(target2, { isWin: true })
var hadError = false
var calledFinish = false
stream.on('error', function (er) {
hadError = true
console.log('#', er)
})
stream.on('finish', function () {
calledFinish = true
})
stream.on('close', function () {
t.is(hadError, true, 'error was caught')
t.is(calledFinish, false, 'finish was called before close')
})
stream.end()
})
// test existing file with the same content
// test existing file with diff. content
test('rename eperm existing file different content', function (t) {
t.plan(2)
var _rename = fs.rename
fs.existsSync = function (src) {
return true
}
fs.rename = function (src, dest, cb) {
// simulate a failure during rename where the file
// is renamed successfully but the process encounters
// an EPERM error and the target file that has the same content than the
// destination
_rename(src, dest, function (e) {
fs.writeFile(src, 'target2', function (writeErr) {
if (writeErr) {
return console.log('WRITEERR: ' + writeErr)
}
fs.writeFile(target3, 'target2', function (writeErr) {
if (writeErr) {
return console.log('WRITEERR: ' + writeErr)
}
var err = new Error('TEST BREAK')
err.syscall = 'rename'
err.code = 'EPERM'
cb(err)
})
})
})
}
var stream = writeStream(target3, { isWin: true })
var hadError = false
var calledFinish = false
stream.on('error', function (er) {
hadError = true
console.log('#', er)
})
stream.on('finish', function () {
calledFinish = true
})
stream.on('close', function () {
t.is(hadError, false, 'error was caught')
t.is(calledFinish, true, 'finish was called before close')
})
stream.end()
})
test('cleanup', function (t) {
rimraf.sync(target)
rimraf.sync(target2)
rimraf.sync(target3)
t.end()
})
+30
View File
@@ -0,0 +1,30 @@
'use strict'
var fs = require('graceful-fs')
var path = require('path')
var test = require('tap').test
var rimraf = require('rimraf')
var writeStream = require('../index.js')
var target = path.resolve(__dirname, 'test-rename')
test('rename fails', function (t) {
t.plan(1)
fs.rename = function (src, dest, cb) {
cb(new Error('TEST BREAK'))
}
var stream = writeStream(target)
var hadError = false
stream.on('error', function (er) {
hadError = true
console.log('#', er)
})
stream.on('close', function () {
t.is(hadError, true, 'error before close')
})
stream.end()
})
test('cleanup', function (t) {
rimraf.sync(target)
t.end()
})
+40
View File
@@ -0,0 +1,40 @@
'use strict'
var fs = require('graceful-fs')
var path = require('path')
var test = require('tap').test
var rimraf = require('rimraf')
var writeStream = require('../index.js')
var target = path.resolve(__dirname, 'test-chown')
test('slow close', function (t) {
t.plan(2)
// The goal here is to simulate the "file close" step happening so slowly
// that the whole close/rename process could finish before the file is
// actually closed (and thus buffers truely flushed to the OS). In
// previous versions of this module, this would result in the module
// emitting finish & close before the file was fully written and in
// turn, could break other layers that tried to read the new file.
var realEmit = fs.WriteStream.prototype.emit
var reallyClosed = false
fs.WriteStream.prototype.emit = function (event) {
if (event !== 'close') return realEmit.apply(this, arguments)
setTimeout(function () {
reallyClosed = true
realEmit.call(this, 'close')
}.bind(this), 200)
}
var stream = writeStream(target)
stream.on('finish', function () {
t.is(reallyClosed, true, "didn't finish before target was closed")
})
stream.on('close', function () {
t.is(reallyClosed, true, "didn't close before target was closed")
})
stream.end()
})
test('cleanup', function (t) {
rimraf.sync(target)
t.end()
})
+29
View File
@@ -0,0 +1,29 @@
var path = require('path')
var test = require('tap').test
var writeStream = require('../index.js')
function repeat (times, string) {
var output = ''
for (var ii = 0; ii < times; ++ii) {
output += string
}
return output
}
var target = path.resolve(__dirname, repeat(1000, 'test'))
test('name too long', function (t) {
t.plan(2)
var stream = writeStream(target)
var hadError = false
stream.on('error', function (er) {
if (!hadError) {
t.is(er.code, 'ENAMETOOLONG', target.length + ' character name results in ENAMETOOLONG')
hadError = true
}
})
stream.on('close', function () {
t.ok(hadError, 'got error before close')
})
stream.end()
})