Commit 4f2c10f0f4f3760dbb28849caaadf3bc3ea08a6e - node-klaw

+0

-1

~~.gitignore~~ less more

node_modules/

+0

-3

~~.npmignore~~ less more

0		tests/
1		appveyor.yml
2		.travis.yml

+1

-0

.npmrc less more

0

package-lock=false

+0

-13

~~.travis.yml~~ less more

0		sudo: false
1		language: node_js
2		node_js:
3		- "6"
4		- "8"
5		- "10"
6		matrix:
7		include:
8		- node_js: "10"
9		env: TEST_SUITE=lint
10		env:
11		- TEST_SUITE=unit
12		script: npm run-script $TEST_SUITE

+12

-0

CHANGELOG.md less more

	0	4.0.1 / 2021-09-18
	1	------------------
	2
	3	- Don't publish unnecessary files
	4
	5	4.0.0 / 2021-09-18
	6	------------------
	7
	8	- BREAKING: Require Node 14.14.0+ ([#43](https://github.com/jprichardson/node-klaw/pull/43))
	9	- BREAKING: Remove graceful-fs dependency; install it manually and pass it as `fs` option if needed ([#43](https://github.com/jprichardson/node-klaw/pull/43))
	10	- Additional documentation examples ([#34](https://github.com/jprichardson/node-klaw/pull/34))
	11
0	12	3.0.0 / 2018-08-01
1	13	------------------
2	14

+8

-0

README.md less more

77	77	.on('end', () => console.dir(items)) // => [ ... array of files]
78	78	```
79	79
	80	```for-await-of``` example:
	81
	82	```js
	83	for await (const file of klaw('/some/dir')) {
	84	console.log(file)
	85	}
	86	```
	87
80	88	### Error Handling
81	89
82	90	Listen for the `error` event.

+0

-26

~~appveyor.yml~~ less more

0		# Test against this version of Node.js
1		environment:
2		matrix:
3		# node.js
4		- nodejs_version: "6"
5		- nodejs_version: "8"
6		- nodejs_version: "10"
7
8		# Install scripts. (runs after repo cloning)
9		install:
10		# Get the latest stable version of Node.js or io.js
11		- ps: Install-Product node $env:nodejs_version
12		# install modules
13		- npm config set loglevel warn
14		- npm install --silent
15
16		# Post-install test scripts.
17		test_script:
18		# Output useful info for debugging.
19		- node --version
20		- npm --version
21		# run tests
22		- npm run unit
23
24		# Don't actually build.
25		build: off

+10

-11

package.json less more

0	0	{
1	1	"name": "klaw",
2		"version": "3.0.0",
	2	"version": "4.0.1",
3	3	"description": "File system walker with Readable stream interface.",
4	4	"main": "./src/index.js",
5	5	"scripts": {
6		"lint": "standard && standard-markdown",
	6	"lint": "standard",
7	7	"test": "npm run lint && npm run unit",
8	8	"unit": "tape tests/*/.js \| tap-spec"
9	9	},

15	15	"walk",
16	16	"walker",
17	17	"fs",
18		"fs-extra",
19	18	"readable",
20	19	"streams"
21	20	],
	21	"engines": {
	22	"node": ">=14.14.0"
	23	},
22	24	"author": "JP Richardson",
23	25	"license": "MIT",
	26	"files": [
	27	"src/"
	28	],
24	29	"bugs": {
25	30	"url": "https://github.com/jprichardson/node-klaw/issues"
26	31	},
27	32	"homepage": "https://github.com/jprichardson/node-klaw#readme",
28		"dependencies": {
29		"graceful-fs": "^4.1.9"
30		},
31	33	"devDependencies": {
32		"mkdirp": "^0.5.1",
33		"rimraf": "^2.4.3",
34		"standard": "^11.0.1",
35		"standard-markdown": "^4.0.1",
	34	"standard": "^16.0.3",
36	35	"tap-spec": "^5.0.0",
37		"tape": "^4.2.2"
	36	"tape": "^5.3.1"
38	37	}
39	38	}

+58

-50

src/index.js less more

0		var assert = require('assert')
1		var path = require('path')
2		var Readable = require('stream').Readable
3		var util = require('util')
	0	const { strictEqual } = require('assert')
	1	const path = require('path')
	2	const fs = require('fs')
	3	const { Readable } = require('stream')
4	4
5		function Walker (dir, options) {
6		assert.strictEqual(typeof dir, 'string', '`dir` parameter should be of type string. Got type: ' + typeof dir)
7		var defaultStreamOptions = { objectMode: true }
8		var defaultOpts = {
9		queueMethod: 'shift',
10		pathSorter: undefined,
11		filter: undefined,
12		depthLimit: undefined,
13		preserveSymlinks: false
14		}
15		options = Object.assign(defaultOpts, options, defaultStreamOptions)
16
17		Readable.call(this, options)
18		this.root = path.resolve(dir)
19		this.paths = [this.root]
20		this.options = options
21		if (options.depthLimit > -1) this.rootDepth = this.root.split(path.sep).length + 1
22		this.fs = options.fs \|\| require('graceful-fs')
23		}
24		util.inherits(Walker, Readable)
25
26		Walker.prototype._read = function () {
27		if (this.paths.length === 0) return this.push(null)
28		var self = this
29		var pathItem = this.paths[this.options.queueMethod]()
30
31		var statFunction = this.options.preserveSymlinks ? self.fs.lstat : self.fs.stat
32
33		statFunction(pathItem, function (err, stats) {
34		var item = { path: pathItem, stats: stats }
35		if (err) return self.emit('error', err, item)
36
37		if (!stats.isDirectory() \|\| (self.rootDepth &&
38		pathItem.split(path.sep).length - self.rootDepth >= self.options.depthLimit)) {
39		return self.push(item)
	5	class Walker extends Readable {
	6	/**
	7	* @param {string} dir
	8	* @param {Object} options
	9	*/
	10	constructor (dir, options) {
	11	strictEqual(typeof dir, 'string', '`dir` parameter should be of type string. Got type: ' + typeof dir)
	12	options = {
	13	queueMethod: 'shift',
	14	pathSorter: undefined,
	15	filter: undefined,
	16	depthLimit: undefined,
	17	preserveSymlinks: false,
	18	...options,
	19	objectMode: true
40	20	}
41	21
42		self.fs.readdir(pathItem, function (err, pathItems) {
43		if (err) {
44		self.push(item)
45		return self.emit('error', err, item)
	22	super(options)
	23	this.root = path.resolve(dir)
	24	this.paths = [this.root]
	25	this.options = options
	26	if (options.depthLimit > -1) { this.rootDepth = this.root.split(path.sep).length + 1 }
	27	this.fs = options.fs \|\| fs
	28	}
	29
	30	_read () {
	31	if (this.paths.length === 0) { return this.push(null) }
	32	const pathItem = this.paths[this.options.queueMethod]()
	33
	34	const statFunction = this.options.preserveSymlinks ? this.fs.lstat : this.fs.stat
	35
	36	statFunction(pathItem, (err, stats) => {
	37	const item = { path: pathItem, stats: stats }
	38	if (err) { return this.emit('error', err, item) }
	39
	40	if (!stats.isDirectory() \|\| (this.rootDepth &&
	41	pathItem.split(path.sep).length - this.rootDepth >= this.options.depthLimit)) {
	42	return this.push(item)
46	43	}
47	44
48		pathItems = pathItems.map(function (part) { return path.join(pathItem, part) })
49		if (self.options.filter) pathItems = pathItems.filter(self.options.filter)
50		if (self.options.pathSorter) pathItems.sort(self.options.pathSorter)
51		// faster way to do do incremental batch array pushes
52		self.paths.push.apply(self.paths, pathItems)
	45	this.fs.readdir(pathItem, (err, pathItems) => {
	46	if (err) {
	47	this.push(item)
	48	return this.emit('error', err, item)
	49	}
53	50
54		self.push(item)
	51	pathItems = pathItems.map(function (part) { return path.join(pathItem, part) })
	52	if (this.options.filter) { pathItems = pathItems.filter(this.options.filter) }
	53	if (this.options.pathSorter) { pathItems.sort(this.options.pathSorter) }
	54	// faster way to do do incremental batch array pushes
	55	this.paths.push.apply(this.paths, pathItems)
	56
	57	this.push(item)
	58	})
55	59	})
56		})
	60	}
57	61	}
58	62
	63	/**
	64	* @param {string} root
	65	* @param {Object} [options]
	66	*/
59	67	function walk (root, options) {
60	68	return new Walker(root, options)
61	69	}

+9

-10

tests/_test.js less more

0		var mkdirp = require('mkdirp')
1		var os = require('os')
2		var path = require('path')
3		var rimraf = require('rimraf')
4		var tape = require('tape')
	0	const fs = require('fs')
	1	const os = require('os')
	2	const path = require('path')
	3	const tape = require('tape')
5	4
6	5	// for all practical purposes, this is a beforeEach and afterEach
7	6	function test (desc, testFn) {
8	7	tape(desc, function (t) {
9		var testDir = path.join(os.tmpdir(), 'klaw-tests')
10		rimraf(testDir, function (err) {
	8	const testDir = path.join(os.tmpdir(), 'klaw-tests')
	9	fs.rm(testDir, { recursive: true, force: true }, function (err) {
11	10	if (err) return t.end(err)
12		mkdirp(testDir, function (err) {
	11	fs.mkdir(testDir, function (err) {
13	12	if (err) return t.end(err)
14	13
15		var oldEnd = t.end
	14	const oldEnd = t.end
16	15	t.end = function () {
17		rimraf(testDir, function (err) {
	16	fs.rm(testDir, { recursive: true, force: true }, function (err) {
18	17	err ? oldEnd.apply(t, [err]) : oldEnd.apply(t, arguments)
19	18	})
20	19	}

+2

-1

tests/fixtures_links.json less more

0	0	{
1	1	"a/b.txt": { },
2	2	"b": { "type": "file", "target": "./a/b.txt" },
3		"c": { "type": "dir", "target": "./a" }
	3	"c": { "type": "dir", "target": "./a" },
	4	"d": { "type": "file", "target": "./broken" }
4	5	}⏎

+14

-15

tests/walk.test.js less more

0		var fs = require('fs')
1		var mkdirp = require('mkdirp')
2		var path = require('path')
3		var test = require('./_test')
4		var klaw = require('../')
5		var fixtures = require('./fixtures')
	0	const fs = require('fs')
	1	const path = require('path')
	2	const test = require('./_test')
	3	const klaw = require('../')
	4	const fixtures = require('./fixtures')
6	5
7	6	test('should work w/ streams 1', function (t, testDir) {
8	7	fixtures.forEach(function (f) {
9	8	f = path.join(testDir, f)
10		var dir = path.dirname(f)
11		mkdirp.sync(dir)
	9	const dir = path.dirname(f)
	10	fs.mkdirSync(dir, { recursive: true })
12	11	fs.writeFileSync(f, path.basename(f, path.extname(f)))
13	12	})
14	13
15		var items = []
	14	const items = []
16	15	klaw(testDir)
17	16	.on('data', function (item) {
18	17	items.push(item.path)

20	19	.on('error', t.end)
21	20	.on('end', function () {
22	21	items.sort()
23		var expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i', 'h/i/j',
	22	let expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i', 'h/i/j',
24	23	'h/i/j/k.txt', 'h/i/l.txt', 'h/i/m.jpg']
25	24	expected = expected.map(function (item) {
26	25	return path.join(path.join(testDir, item))

35	34	test('should work w/ streams 2/3', function (t, testDir) {
36	35	fixtures.forEach(function (f) {
37	36	f = path.join(testDir, f)
38		var dir = path.dirname(f)
39		mkdirp.sync(dir)
	37	const dir = path.dirname(f)
	38	fs.mkdirSync(dir, { recursive: true })
40	39	fs.writeFileSync(f, path.basename(f, path.extname(f)))
41	40	})
42	41
43		var items = []
	42	const items = []
44	43	klaw(testDir)
45	44	.on('readable', function () {
46		var item
	45	let item
47	46	while ((item = this.read())) {
48	47	items.push(item.path)
49	48	}

51	50	.on('error', t.end)
52	51	.on('end', function () {
53	52	items.sort()
54		var expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i', 'h/i/j',
	53	let expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i', 'h/i/j',
55	54	'h/i/j/k.txt', 'h/i/l.txt', 'h/i/m.jpg']
56	55	expected = expected.map(function (item) {
57	56	return path.join(path.join(testDir, item))

+13

-14

tests/walk_depth-limit.test.js less more

0		var fs = require('fs')
1		var mkdirp = require('mkdirp')
2		var path = require('path')
3		var test = require('./_test')
4		var klaw = require('../')
5		var fixtures = require('./fixtures')
	0	const fs = require('fs')
	1	const path = require('path')
	2	const test = require('./_test')
	3	const klaw = require('../')
	4	const fixtures = require('./fixtures.json')
6	5
7	6	test('should honor depthLimit option -1', function (t, testDir) {
8		var expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i',
	7	const expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i',
9	8	'h/i/j', 'h/i/j/k.txt', 'h/i/l.txt', 'h/i/m.jpg']
10	9	run(t, testDir, -1, expected)
11	10	})
12	11
13	12	test('should honor depthLimit option 0', function (t, testDir) {
14		var expected = ['a', 'h']
	13	const expected = ['a', 'h']
15	14	run(t, testDir, 0, expected)
16	15	})
17	16
18	17	test('should honor depthLimit option 1', function (t, testDir) {
19		var expected = ['a', 'a/b', 'a/e.jpg', 'h', 'h/i']
	18	const expected = ['a', 'a/b', 'a/e.jpg', 'h', 'h/i']
20	19	run(t, testDir, 1, expected)
21	20	})
22	21
23	22	test('should honor depthLimit option 2', function (t, testDir) {
24		var expected = ['a', 'a/b', 'a/b/c', 'a/e.jpg', 'h', 'h/i', 'h/i/j',
	23	const expected = ['a', 'a/b', 'a/b/c', 'a/e.jpg', 'h', 'h/i', 'h/i/j',
25	24	'h/i/l.txt', 'h/i/m.jpg']
26	25	run(t, testDir, 2, expected)
27	26	})
28	27
29	28	test('should honor depthLimit option 3', function (t, testDir) {
30		var expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i',
	29	const expected = ['a', 'a/b', 'a/b/c', 'a/b/c/d.txt', 'a/e.jpg', 'h', 'h/i',
31	30	'h/i/j', 'h/i/j/k.txt', 'h/i/l.txt', 'h/i/m.jpg']
32	31	run(t, testDir, 3, expected)
33	32	})

35	34	function run (t, testDir, depthLimit, expected) {
36	35	fixtures.forEach(function (f) {
37	36	f = path.join(testDir, f)
38		var dir = path.dirname(f)
39		mkdirp.sync(dir)
	37	const dir = path.dirname(f)
	38	fs.mkdirSync(dir, { recursive: true })
40	39	fs.writeFileSync(f, path.basename(f, path.extname(f)))
41	40	})
42	41
43		var items = []
	42	const items = []
44	43	klaw(testDir, { depthLimit: depthLimit })
45	44	.on('data', function (item) {
46	45	items.push(item.path)

+11

-12

tests/walk_filter.test.js less more

0		var fs = require('fs')
1		var mkdirp = require('mkdirp')
2		var path = require('path')
3		var test = require('./_test')
4		var klaw = require('../')
5		var fixtures = require('./fixtures')
	0	const fs = require('fs')
	1	const path = require('path')
	2	const test = require('./_test')
	3	const klaw = require('../')
	4	const fixtures = require('./fixtures')
6	5
7	6	test('should not fire event on filtered items', function (t, testDir) {
8	7	fixtures.forEach(function (f) {
9	8	f = path.join(testDir, f)
10		var dir = path.dirname(f)
11		mkdirp.sync(dir)
	9	const dir = path.dirname(f)
	10	fs.mkdirSync(dir, { recursive: true })
12	11	fs.writeFileSync(f, path.basename(f, path.extname(f)))
13	12	})
14	13
15		var items = []
16		var filter = function (filepath) {
	14	const items = []
	15	const filter = function (filepath) {
17	16	return path.basename(filepath) !== 'a'
18	17	}
19	18
20		klaw(testDir, {filter: filter})
	19	klaw(testDir, { filter: filter })
21	20	.on('data', function (item) {
22	21	if (fs.lstatSync(item.path).isFile()) items.push(item.path)
23	22	})
24	23	.on('error', t.end)
25	24	.on('end', function () {
26		var expected = ['c', 'b', 'a']
	25	let expected = ['c', 'b', 'a']
27	26	expected = expected.map(function (item) {
28	27	return path.join(testDir, item)
29	28	})

+21

-13

tests/walk_links.test.js less more

0		var fs = require('fs')
1		var mkdirp = require('mkdirp')
2		var path = require('path')
3		var test = require('./_test')
4		var klaw = require('../')
5		var fixtures = require('./fixtures_links.json')
	0	const fs = require('fs')
	1	const path = require('path')
	2	const test = require('./_test')
	3	const klaw = require('../')
	4	const fixtures = require('./fixtures_links.json')
6	5
7	6	function loadLinkFixtures (testDir) {
8	7	Object.keys(fixtures).forEach(function (f) {
9		var link = fixtures[f]
	8	const link = fixtures[f]
10	9	f = path.join(testDir, f)
11	10
12		var dir = path.dirname(f)
13		mkdirp.sync(dir)
	11	const dir = path.dirname(f)
	12	fs.mkdirSync(dir, { recursive: true })
14	13
15	14	if (link.target) {
	15	const realTarget = path.resolve(testDir, link.target)
	16	let missing
	17	if (!fs.existsSync(realTarget)) {
	18	missing = true
	19	fs.writeFileSync(realTarget, '')
	20	}
16	21	fs.symlinkSync(link.target, f, link.type)
	22	if (missing) {
	23	fs.unlinkSync(realTarget)
	24	}
17	25	} else {
18	26	fs.writeFileSync(f, path.basename(f, path.extname(f)))
19	27	}

23	31	test('should follow links by default', function (t, testDir) {
24	32	loadLinkFixtures(testDir)
25	33
26		var items = []
	34	const items = []
27	35	klaw(testDir)
28	36	.on('data', function (item) {
29	37	items.push(item.path)

31	39	.on('error', t.end)
32	40	.on('end', function () {
33	41	items.sort()
34		var expected = ['a', 'a/b.txt', 'b', 'c', 'c/b.txt']
	42	let expected = ['a', 'a/b.txt', 'b', 'c', 'c/b.txt']
35	43	expected = expected.map(function (item) {
36	44	return path.join(path.join(testDir, item))
37	45	})

45	53	test('should not follow links if requested', function (t, testDir) {
46	54	loadLinkFixtures(testDir)
47	55
48		var items = []
	56	const items = []
49	57	klaw(testDir, { preserveSymlinks: true })
50	58	.on('data', function (item) {
51	59	items.push(item.path)

53	61	.on('error', t.end)
54	62	.on('end', function () {
55	63	items.sort()
56		var expected = ['a', 'a/b.txt', 'b', 'c']
	64	let expected = ['a', 'a/b.txt', 'b', 'c', 'd']
57	65	expected = expected.map(function (item) {
58	66	return path.join(path.join(testDir, item))
59	67	})

+21

-16

tests/walk_path-sorter.test.js less more

0		var fs = require('fs')
1		var mkdirp = require('mkdirp')
2		var path = require('path')
3		var test = require('./_test')
4		var klaw = require('../')
5		var fixtures = require('./fixtures_path-sorter')
	0	const fs = require('fs')
	1	const path = require('path')
	2	const test = require('./_test')
	3	const klaw = require('../')
	4	const fixtures = require('./fixtures_path-sorter.json')
	5
	6	const stringCompare = function (a, b) {
	7	if (a < b) return -1
	8	else if (a > b) return 1
	9	else return 0
	10	}
6	11
7	12	test('should sort in reverse order [z -> a]', function (t, testDir) {
8	13	fixtures.forEach(function (f) {
9	14	f = path.join(testDir, f)
10		var dir = path.dirname(f)
11		mkdirp.sync(dir)
	15	const dir = path.dirname(f)
	16	fs.mkdirSync(dir, { recursive: true })
12	17	fs.writeFileSync(f, path.basename(f, path.extname(f)))
13	18	})
14	19
15		var items = []
16		var pathSorter = function (a, b) { return b > a }
	20	const items = []
	21	const pathSorter = function (a, b) { return stringCompare(b, a) }
17	22	klaw(testDir, { pathSorter: pathSorter })
18	23	.on('data', function (item) {
19	24	items.push(item.path)
20	25	})
21	26	.on('error', t.end)
22	27	.on('end', function () {
23		var expected = ['c', 'b', 'a']
	28	let expected = ['c', 'b', 'a']
24	29	expected = expected.map(function (item) {
25	30	return path.join(testDir, item)
26	31	})

34	39	test('should sort in order [a -> z]', function (t, testDir) {
35	40	fixtures.forEach(function (f) {
36	41	f = path.join(testDir, f)
37		var dir = path.dirname(f)
38		mkdirp.sync(dir)
	42	const dir = path.dirname(f)
	43	fs.mkdirSync(dir, { recursive: true })
39	44	fs.writeFileSync(f, path.basename(f, path.extname(f)))
40	45	})
41	46
42		var items = []
43		var pathSorter = function (a, b) { return a > b }
	47	const items = []
	48	const pathSorter = function (a, b) { return stringCompare(a, b) }
44	49	klaw(testDir, { pathSorter: pathSorter })
45	50	.on('data', function (item) {
46	51	items.push(item.path)
47	52	})
48	53	.on('error', t.end)
49	54	.on('end', function () {
50		var expected = ['a', 'b', 'c']
	55	let expected = ['a', 'b', 'c']
51	56	expected = expected.map(function (item) {
52	57	return path.join(testDir, item)
53	58	})

+8

-8

tests/walk_read-dir-error.test.js less more

0		var fs = require('fs')
1		var mkdirp = require('mkdirp')
2		var path = require('path')
3		var test = require('./_test')
4		var klaw = require('../')
	0	const fs = require('fs')
	1	const path = require('path')
	2	const test = require('./_test')
	3	const klaw = require('../')
5	4
6	5	test('walk directory, if error on readdir, at least end', function (t, testDir) {
7	6	// simulate directory issue
8		var unreadableDir = path.join(testDir, 'unreadable-dir')
9		mkdirp.sync(unreadableDir)
	7	const unreadableDir = path.join(testDir, 'unreadable-dir')
	8
	9	fs.mkdirSync(unreadableDir, { recursive: true })
10	10	fs.chmodSync(unreadableDir, '0222')
11	11
12	12	// not able to simulate on windows
13	13	if (process.platform === 'win32') return t.end()
14	14
15	15	t.plan(2)
16		var items = []
	16	const items = []
17	17	klaw(testDir)
18	18	.on('data', function (item) {
19	19	items.push(item.path)