retrier.cjs 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477
  1. 'use strict';
  2. /**
  3. * @fileoverview A utility for retrying failed async method calls.
  4. */
  5. /* global setTimeout, clearTimeout */
  6. //-----------------------------------------------------------------------------
  7. // Constants
  8. //-----------------------------------------------------------------------------
  9. const MAX_TASK_TIMEOUT = 60000;
  10. const MAX_TASK_DELAY = 100;
  11. const MAX_CONCURRENCY = 1000;
  12. //-----------------------------------------------------------------------------
  13. // Helpers
  14. //-----------------------------------------------------------------------------
  15. /**
  16. * Logs a message to the console if the DEBUG environment variable is set.
  17. * @param {string} message The message to log.
  18. * @returns {void}
  19. */
  20. function debug(message) {
  21. if (globalThis?.process?.env.DEBUG === "@hwc/retry") {
  22. console.debug(message);
  23. }
  24. }
  25. /*
  26. * The following logic has been extracted from graceful-fs.
  27. *
  28. * The ISC License
  29. *
  30. * Copyright (c) 2011-2023 Isaac Z. Schlueter, Ben Noordhuis, and Contributors
  31. *
  32. * Permission to use, copy, modify, and/or distribute this software for any
  33. * purpose with or without fee is hereby granted, provided that the above
  34. * copyright notice and this permission notice appear in all copies.
  35. *
  36. * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  37. * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  38. * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  39. * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  40. * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  41. * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
  42. * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  43. */
  44. /**
  45. * Checks if it is time to retry a task based on the timestamp and last attempt time.
  46. * @param {RetryTask} task The task to check.
  47. * @param {number} maxDelay The maximum delay for the queue.
  48. * @returns {boolean} true if it is time to retry, false otherwise.
  49. */
  50. function isTimeToRetry(task, maxDelay) {
  51. const timeSinceLastAttempt = Date.now() - task.lastAttempt;
  52. const timeSinceStart = Math.max(task.lastAttempt - task.timestamp, 1);
  53. const desiredDelay = Math.min(timeSinceStart * 1.2, maxDelay);
  54. return timeSinceLastAttempt >= desiredDelay;
  55. }
  56. /**
  57. * Checks if it is time to bail out based on the given timestamp.
  58. * @param {RetryTask} task The task to check.
  59. * @param {number} timeout The timeout for the queue.
  60. * @returns {boolean} true if it is time to bail, false otherwise.
  61. */
  62. function isTimeToBail(task, timeout) {
  63. return task.age > timeout;
  64. }
  65. /**
  66. * Creates a new promise with resolve and reject functions.
  67. * @returns {{promise:Promise<any>, resolve:(value:any) => any, reject: (value:any) => any}} A new promise.
  68. */
  69. function createPromise() {
  70. if (Promise.withResolvers) {
  71. return Promise.withResolvers();
  72. }
  73. let resolve, reject;
  74. const promise = new Promise((res, rej) => {
  75. resolve = res;
  76. reject = rej;
  77. });
  78. if (resolve === undefined || reject === undefined) {
  79. throw new Error("Promise executor did not initialize resolve or reject.");
  80. }
  81. return { promise, resolve, reject };
  82. }
  83. /**
  84. * A class to represent a task in the retry queue.
  85. */
  86. class RetryTask {
  87. /**
  88. * The unique ID for the task.
  89. * @type {string}
  90. */
  91. id = Math.random().toString(36).slice(2);
  92. /**
  93. * The function to call.
  94. * @type {Function}
  95. */
  96. fn;
  97. /**
  98. * The error that was thrown.
  99. * @type {Error}
  100. */
  101. error;
  102. /**
  103. * The timestamp of the task.
  104. * @type {number}
  105. */
  106. timestamp = Date.now();
  107. /**
  108. * The timestamp of the last attempt.
  109. * @type {number}
  110. */
  111. lastAttempt = this.timestamp;
  112. /**
  113. * The resolve function for the promise.
  114. * @type {Function}
  115. */
  116. resolve;
  117. /**
  118. * The reject function for the promise.
  119. * @type {Function}
  120. */
  121. reject;
  122. /**
  123. * The AbortSignal to monitor for cancellation.
  124. * @type {AbortSignal|undefined}
  125. */
  126. signal;
  127. /**
  128. * Creates a new instance.
  129. * @param {Function} fn The function to call.
  130. * @param {Error} error The error that was thrown.
  131. * @param {Function} resolve The resolve function for the promise.
  132. * @param {Function} reject The reject function for the promise.
  133. * @param {AbortSignal|undefined} signal The AbortSignal to monitor for cancellation.
  134. */
  135. constructor(fn, error, resolve, reject, signal) {
  136. this.fn = fn;
  137. this.error = error;
  138. this.timestamp = Date.now();
  139. this.lastAttempt = Date.now();
  140. this.resolve = resolve;
  141. this.reject = reject;
  142. this.signal = signal;
  143. }
  144. /**
  145. * Gets the age of the task.
  146. * @returns {number} The age of the task in milliseconds.
  147. * @readonly
  148. */
  149. get age() {
  150. return Date.now() - this.timestamp;
  151. }
  152. }
  153. //-----------------------------------------------------------------------------
  154. // Exports
  155. //-----------------------------------------------------------------------------
  156. /**
  157. * A class that manages a queue of retry jobs.
  158. */
  159. class Retrier {
  160. /**
  161. * Represents the queue for processing tasks.
  162. * @type {Array<RetryTask>}
  163. */
  164. #retrying = [];
  165. /**
  166. * Represents the queue for pending tasks.
  167. * @type {Array<Function>}
  168. */
  169. #pending = [];
  170. /**
  171. * The number of tasks currently being processed.
  172. * @type {number}
  173. */
  174. #working = 0;
  175. /**
  176. * The timeout for the queue.
  177. * @type {number}
  178. */
  179. #timeout;
  180. /**
  181. * The maximum delay for the queue.
  182. * @type {number}
  183. */
  184. #maxDelay;
  185. /**
  186. * The setTimeout() timer ID.
  187. * @type {NodeJS.Timeout|undefined}
  188. */
  189. #timerId;
  190. /**
  191. * The function to call.
  192. * @type {Function}
  193. */
  194. #check;
  195. /**
  196. * The maximum number of concurrent tasks.
  197. * @type {number}
  198. */
  199. #concurrency;
  200. /**
  201. * Creates a new instance.
  202. * @param {Function} check The function to call.
  203. * @param {object} [options] The options for the instance.
  204. * @param {number} [options.timeout] The timeout for the queue.
  205. * @param {number} [options.maxDelay] The maximum delay for the queue.
  206. * @param {number} [options.concurrency] The maximum number of concurrent tasks.
  207. */
  208. constructor(check, { timeout = MAX_TASK_TIMEOUT, maxDelay = MAX_TASK_DELAY, concurrency = MAX_CONCURRENCY } = {}) {
  209. if (typeof check !== "function") {
  210. throw new Error("Missing function to check errors");
  211. }
  212. this.#check = check;
  213. this.#timeout = timeout;
  214. this.#maxDelay = maxDelay;
  215. this.#concurrency = concurrency;
  216. }
  217. /**
  218. * Gets the number of tasks waiting to be retried.
  219. * @returns {number} The number of tasks in the retry queue.
  220. */
  221. get retrying() {
  222. return this.#retrying.length;
  223. }
  224. /**
  225. * Gets the number of tasks waiting to be processed in the pending queue.
  226. * @returns {number} The number of tasks in the pending queue.
  227. */
  228. get pending() {
  229. return this.#pending.length;
  230. }
  231. /**
  232. * Gets the number of tasks currently being processed.
  233. * @returns {number} The number of tasks currently being processed.
  234. */
  235. get working() {
  236. return this.#working;
  237. }
  238. /**
  239. * Calls the function and retries if it fails.
  240. * @param {Function} fn The function to call.
  241. * @param {Object} options The options for the job.
  242. * @param {AbortSignal} [options.signal] The AbortSignal to monitor for cancellation.
  243. * @param {Promise<any>} options.promise The promise to return when the function settles.
  244. * @param {Function} options.resolve The resolve function for the promise.
  245. * @param {Function} options.reject The reject function for the promise.
  246. * @returns {Promise<any>} A promise that resolves when the function is
  247. * called successfully.
  248. */
  249. #call(fn, { signal, promise, resolve, reject }) {
  250. let result;
  251. try {
  252. result = fn();
  253. } catch (/** @type {any} */ error) {
  254. reject(new Error(`Synchronous error: ${error.message}`, { cause: error }));
  255. return promise;
  256. }
  257. // if the result is not a promise then reject an error
  258. if (!result || typeof result.then !== "function") {
  259. reject(new Error("Result is not a promise."));
  260. return promise;
  261. }
  262. this.#working++;
  263. promise.finally(() => {
  264. this.#working--;
  265. this.#processPending();
  266. });
  267. // call the original function and catch any ENFILE or EMFILE errors
  268. // @ts-ignore because we know it's any
  269. return Promise.resolve(result)
  270. .then(value => {
  271. debug("Function called successfully without retry.");
  272. resolve(value);
  273. return promise;
  274. })
  275. .catch(error => {
  276. if (!this.#check(error)) {
  277. reject(error);
  278. return promise;
  279. }
  280. const task = new RetryTask(fn, error, resolve, reject, signal);
  281. debug(`Function failed, queuing for retry with task ${task.id}.`);
  282. this.#retrying.push(task);
  283. signal?.addEventListener("abort", () => {
  284. debug(`Task ${task.id} was aborted due to AbortSignal.`);
  285. reject(signal.reason);
  286. });
  287. this.#processQueue();
  288. return promise;
  289. });
  290. }
  291. /**
  292. * Adds a new retry job to the queue.
  293. * @param {Function} fn The function to call.
  294. * @param {object} [options] The options for the job.
  295. * @param {AbortSignal} [options.signal] The AbortSignal to monitor for cancellation.
  296. * @returns {Promise<any>} A promise that resolves when the queue is
  297. * processed.
  298. */
  299. retry(fn, { signal } = {}) {
  300. signal?.throwIfAborted();
  301. const { promise, resolve, reject } = createPromise();
  302. this.#pending.push(() => this.#call(fn, { signal, promise, resolve, reject }));
  303. this.#processPending();
  304. return promise;
  305. }
  306. /**
  307. * Processes the pending queue and the retry queue.
  308. * @returns {void}
  309. */
  310. #processAll() {
  311. if (this.pending) {
  312. this.#processPending();
  313. }
  314. if (this.retrying) {
  315. this.#processQueue();
  316. }
  317. }
  318. /**
  319. * Processes the pending queue to see which tasks can be started.
  320. * @returns {void}
  321. */
  322. #processPending() {
  323. debug(`Processing pending tasks: ${this.pending} pending, ${this.working} working.`);
  324. const available = this.#concurrency - this.working;
  325. if (available <= 0) {
  326. return;
  327. }
  328. const count = Math.min(this.pending, available);
  329. for (let i = 0; i < count; i++) {
  330. const task = this.#pending.shift();
  331. task?.();
  332. }
  333. debug(`Processed pending tasks: ${this.pending} pending, ${this.working} working.`);
  334. }
  335. /**
  336. * Processes the queue.
  337. * @returns {void}
  338. */
  339. #processQueue() {
  340. // clear any timer because we're going to check right now
  341. clearTimeout(this.#timerId);
  342. this.#timerId = undefined;
  343. debug(`Processing retry queue: ${this.retrying} retrying, ${this.working} working.`);
  344. const processAgain = () => {
  345. this.#timerId = setTimeout(() => this.#processAll(), 0);
  346. };
  347. // if there's nothing in the queue, we're done
  348. const task = this.#retrying.shift();
  349. if (!task) {
  350. debug("Queue is empty, exiting.");
  351. if (this.pending) {
  352. processAgain();
  353. }
  354. return;
  355. }
  356. // if it's time to bail, then bail
  357. if (isTimeToBail(task, this.#timeout)) {
  358. debug(`Task ${task.id} was abandoned due to timeout.`);
  359. task.reject(task.error);
  360. processAgain();
  361. return;
  362. }
  363. // if it's not time to retry, then wait and try again
  364. if (!isTimeToRetry(task, this.#maxDelay)) {
  365. debug(`Task ${task.id} is not ready to retry, skipping.`);
  366. this.#retrying.push(task);
  367. processAgain();
  368. return;
  369. }
  370. // otherwise, try again
  371. task.lastAttempt = Date.now();
  372. // Promise.resolve needed in case it's a thenable but not a Promise
  373. Promise.resolve(task.fn())
  374. // @ts-ignore because we know it's any
  375. .then(result => {
  376. debug(`Task ${task.id} succeeded after ${task.age}ms.`);
  377. task.resolve(result);
  378. })
  379. // @ts-ignore because we know it's any
  380. .catch(error => {
  381. if (!this.#check(error)) {
  382. debug(`Task ${task.id} failed with non-retryable error: ${error.message}.`);
  383. task.reject(error);
  384. return;
  385. }
  386. // update the task timestamp and push to back of queue to try again
  387. task.lastAttempt = Date.now();
  388. this.#retrying.push(task);
  389. debug(`Task ${task.id} failed, requeueing to try again.`);
  390. })
  391. .finally(() => {
  392. this.#processAll();
  393. });
  394. }
  395. }
  396. exports.Retrier = Retrier;