index.js 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
  1. import { stat, lstat, readdir, realpath } from 'fs/promises';
  2. import { Readable } from 'stream';
  3. import { resolve as pathResolve, relative as pathRelative, join as pathJoin, sep as pathSep, } from 'path';
  4. function defaultOptions() {
  5. return {
  6. root: '.',
  7. fileFilter: (_path) => true,
  8. directoryFilter: (_path) => true,
  9. type: FILE_TYPE,
  10. lstat: false,
  11. depth: 2147483648,
  12. alwaysStat: false,
  13. highWaterMark: 4096,
  14. };
  15. }
  16. const RECURSIVE_ERROR_CODE = 'READDIRP_RECURSIVE_ERROR';
  17. const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP', RECURSIVE_ERROR_CODE]);
  18. const FILE_TYPE = 'files';
  19. const DIR_TYPE = 'directories';
  20. const FILE_DIR_TYPE = 'files_directories';
  21. const EVERYTHING_TYPE = 'all';
  22. const ALL_TYPES = [FILE_TYPE, DIR_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE];
  23. const DIR_TYPES = new Set([DIR_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE]);
  24. const FILE_TYPES = new Set([FILE_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE]);
  25. const isNormalFlowError = (error) => NORMAL_FLOW_ERRORS.has(error.code);
  26. const wantBigintFsStats = process.platform === 'win32';
  27. const emptyFn = (_path) => true;
  28. const normalizeFilter = (filter) => {
  29. if (filter === undefined)
  30. return emptyFn;
  31. if (typeof filter === 'function')
  32. return filter;
  33. if (typeof filter === 'string') {
  34. const fl = filter.trim();
  35. return (entry) => entry.basename === fl;
  36. }
  37. if (Array.isArray(filter)) {
  38. const trItems = filter.map((item) => item.trim());
  39. return (entry) => trItems.some((f) => entry.basename === f);
  40. }
  41. return emptyFn;
  42. };
  43. export class ReaddirpStream extends Readable {
  44. constructor(options = {}) {
  45. super({
  46. objectMode: true,
  47. autoDestroy: true,
  48. highWaterMark: options.highWaterMark,
  49. });
  50. const opts = { ...defaultOptions(), ...options };
  51. const { root, type } = opts;
  52. this._fileFilter = normalizeFilter(opts.fileFilter);
  53. this._directoryFilter = normalizeFilter(opts.directoryFilter);
  54. const statMethod = opts.lstat ? lstat : stat;
  55. // Use bigint stats if it's windows and stat() supports options (node 10+).
  56. if (wantBigintFsStats) {
  57. this._stat = (path) => statMethod(path, { bigint: true });
  58. }
  59. else {
  60. this._stat = statMethod;
  61. }
  62. this._maxDepth = opts.depth;
  63. this._wantsDir = DIR_TYPES.has(type);
  64. this._wantsFile = FILE_TYPES.has(type);
  65. this._wantsEverything = type === EVERYTHING_TYPE;
  66. this._root = pathResolve(root);
  67. this._isDirent = !opts.alwaysStat;
  68. this._statsProp = this._isDirent ? 'dirent' : 'stats';
  69. this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };
  70. // Launch stream with one parent, the root dir.
  71. this.parents = [this._exploreDir(root, 1)];
  72. this.reading = false;
  73. this.parent = undefined;
  74. }
  75. async _read(batch) {
  76. if (this.reading)
  77. return;
  78. this.reading = true;
  79. try {
  80. while (!this.destroyed && batch > 0) {
  81. const par = this.parent;
  82. const fil = par && par.files;
  83. if (fil && fil.length > 0) {
  84. const { path, depth } = par;
  85. const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path));
  86. const awaited = await Promise.all(slice);
  87. for (const entry of awaited) {
  88. if (!entry) {
  89. batch--;
  90. return;
  91. }
  92. if (this.destroyed)
  93. return;
  94. const entryType = await this._getEntryType(entry);
  95. if (entryType === 'directory' && this._directoryFilter(entry)) {
  96. if (depth <= this._maxDepth) {
  97. this.parents.push(this._exploreDir(entry.fullPath, depth + 1));
  98. }
  99. if (this._wantsDir) {
  100. this.push(entry);
  101. batch--;
  102. }
  103. }
  104. else if ((entryType === 'file' || this._includeAsFile(entry)) &&
  105. this._fileFilter(entry)) {
  106. if (this._wantsFile) {
  107. this.push(entry);
  108. batch--;
  109. }
  110. }
  111. }
  112. }
  113. else {
  114. const parent = this.parents.pop();
  115. if (!parent) {
  116. this.push(null);
  117. break;
  118. }
  119. this.parent = await parent;
  120. if (this.destroyed)
  121. return;
  122. }
  123. }
  124. }
  125. catch (error) {
  126. this.destroy(error);
  127. }
  128. finally {
  129. this.reading = false;
  130. }
  131. }
  132. async _exploreDir(path, depth) {
  133. let files;
  134. try {
  135. files = await readdir(path, this._rdOptions);
  136. }
  137. catch (error) {
  138. this._onError(error);
  139. }
  140. return { files, depth, path };
  141. }
  142. async _formatEntry(dirent, path) {
  143. let entry;
  144. const basename = this._isDirent ? dirent.name : dirent;
  145. try {
  146. const fullPath = pathResolve(pathJoin(path, basename));
  147. entry = { path: pathRelative(this._root, fullPath), fullPath, basename };
  148. entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
  149. }
  150. catch (err) {
  151. this._onError(err);
  152. return;
  153. }
  154. return entry;
  155. }
  156. _onError(err) {
  157. if (isNormalFlowError(err) && !this.destroyed) {
  158. this.emit('warn', err);
  159. }
  160. else {
  161. this.destroy(err);
  162. }
  163. }
  164. async _getEntryType(entry) {
  165. // entry may be undefined, because a warning or an error were emitted
  166. // and the statsProp is undefined
  167. if (!entry && this._statsProp in entry) {
  168. return '';
  169. }
  170. const stats = entry[this._statsProp];
  171. if (stats.isFile())
  172. return 'file';
  173. if (stats.isDirectory())
  174. return 'directory';
  175. if (stats && stats.isSymbolicLink()) {
  176. const full = entry.fullPath;
  177. try {
  178. const entryRealPath = await realpath(full);
  179. const entryRealPathStats = await lstat(entryRealPath);
  180. if (entryRealPathStats.isFile()) {
  181. return 'file';
  182. }
  183. if (entryRealPathStats.isDirectory()) {
  184. const len = entryRealPath.length;
  185. if (full.startsWith(entryRealPath) && full.substr(len, 1) === pathSep) {
  186. const recursiveError = new Error(`Circular symlink detected: "${full}" points to "${entryRealPath}"`);
  187. // @ts-ignore
  188. recursiveError.code = RECURSIVE_ERROR_CODE;
  189. return this._onError(recursiveError);
  190. }
  191. return 'directory';
  192. }
  193. }
  194. catch (error) {
  195. this._onError(error);
  196. return '';
  197. }
  198. }
  199. }
  200. _includeAsFile(entry) {
  201. const stats = entry && entry[this._statsProp];
  202. return stats && this._wantsEverything && !stats.isDirectory();
  203. }
  204. }
  205. /**
  206. * Main function which ends up calling readdirRec and reads all files and directories in given root recursively.
  207. * @param root Root directory
  208. * @param options Options to specify root (start directory), filters and recursion depth
  209. */
  210. export const readdirp = (root, options = {}) => {
  211. // @ts-ignore
  212. let type = options.entryType || options.type;
  213. if (type === 'both')
  214. type = FILE_DIR_TYPE; // backwards-compatibility
  215. if (type)
  216. options.type = type;
  217. if (!root) {
  218. throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');
  219. }
  220. else if (typeof root !== 'string') {
  221. throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');
  222. }
  223. else if (type && !ALL_TYPES.includes(type)) {
  224. throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);
  225. }
  226. options.root = root;
  227. return new ReaddirpStream(options);
  228. };
  229. export const readdirpPromise = (root, options = {}) => {
  230. return new Promise((resolve, reject) => {
  231. const files = [];
  232. readdirp(root, options)
  233. .on('data', (entry) => files.push(entry))
  234. .on('end', () => resolve(files))
  235. .on('error', (error) => reject(error));
  236. });
  237. };
  238. export default readdirp;