remote.js 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132
  1. const { Minipass } = require('minipass')
  2. const fetch = require('minipass-fetch')
  3. const promiseRetry = require('promise-retry')
  4. const ssri = require('ssri')
  5. const { log } = require('proc-log')
  6. const CachingMinipassPipeline = require('./pipeline.js')
  7. const { getAgent } = require('@npmcli/agent')
  8. const pkg = require('../package.json')
  9. const USER_AGENT = `${pkg.name}/${pkg.version} (+https://npm.im/${pkg.name})`
  10. const RETRY_ERRORS = [
  11. 'ECONNRESET', // remote socket closed on us
  12. 'ECONNREFUSED', // remote host refused to open connection
  13. 'EADDRINUSE', // failed to bind to a local port (proxy?)
  14. 'ETIMEDOUT', // someone in the transaction is WAY TOO SLOW
  15. // from @npmcli/agent
  16. 'ECONNECTIONTIMEOUT',
  17. 'EIDLETIMEOUT',
  18. 'ERESPONSETIMEOUT',
  19. 'ETRANSFERTIMEOUT',
  20. // Known codes we do NOT retry on:
  21. // ENOTFOUND (getaddrinfo failure. Either bad hostname, or offline)
  22. // EINVALIDPROXY // invalid protocol from @npmcli/agent
  23. // EINVALIDRESPONSE // invalid status code from @npmcli/agent
  24. ]
  25. const RETRY_TYPES = [
  26. 'request-timeout',
  27. ]
  28. // make a request directly to the remote source,
  29. // retrying certain classes of errors as well as
  30. // following redirects (through the cache if necessary)
  31. // and verifying response integrity
  32. const remoteFetch = (request, options) => {
  33. // options.signal is intended for the fetch itself, not the agent. Attaching it to the agent will re-use that signal across multiple requests, which prevents any connections beyond the first one.
  34. const agent = getAgent(request.url, { ...options, signal: undefined })
  35. if (!request.headers.has('connection')) {
  36. request.headers.set('connection', agent ? 'keep-alive' : 'close')
  37. }
  38. if (!request.headers.has('user-agent')) {
  39. request.headers.set('user-agent', USER_AGENT)
  40. }
  41. // keep our own options since we're overriding the agent
  42. // and the redirect mode
  43. const _opts = {
  44. ...options,
  45. agent,
  46. redirect: 'manual',
  47. }
  48. return promiseRetry(async (retryHandler, attemptNum) => {
  49. const req = new fetch.Request(request, _opts)
  50. try {
  51. let res = await fetch(req, _opts)
  52. if (_opts.integrity && res.status === 200) {
  53. // we got a 200 response and the user has specified an expected
  54. // integrity value, so wrap the response in an ssri stream to verify it
  55. const integrityStream = ssri.integrityStream({
  56. algorithms: _opts.algorithms,
  57. integrity: _opts.integrity,
  58. size: _opts.size,
  59. })
  60. const pipeline = new CachingMinipassPipeline({
  61. events: ['integrity', 'size'],
  62. }, res.body, integrityStream)
  63. // we also propagate the integrity and size events out to the pipeline so we can use
  64. // this new response body as an integrityEmitter for cacache
  65. integrityStream.on('integrity', i => pipeline.emit('integrity', i))
  66. integrityStream.on('size', s => pipeline.emit('size', s))
  67. res = new fetch.Response(pipeline, res)
  68. // set an explicit flag so we know if our response body will emit integrity and size
  69. res.body.hasIntegrityEmitter = true
  70. }
  71. res.headers.set('x-fetch-attempts', attemptNum)
  72. // do not retry POST requests, or requests with a streaming body
  73. // do retry requests with a 408, 420, 429 or 500+ status in the response
  74. const isStream = Minipass.isStream(req.body)
  75. const isRetriable = req.method !== 'POST' &&
  76. !isStream &&
  77. ([408, 420, 429].includes(res.status) || res.status >= 500)
  78. if (isRetriable) {
  79. if (typeof options.onRetry === 'function') {
  80. options.onRetry(res)
  81. }
  82. /* eslint-disable-next-line max-len */
  83. log.http('fetch', `${req.method} ${req.url} attempt ${attemptNum} failed with ${res.status}`)
  84. return retryHandler(res)
  85. }
  86. return res
  87. } catch (err) {
  88. const code = (err.code === 'EPROMISERETRY')
  89. ? err.retried.code
  90. : err.code
  91. // err.retried will be the thing that was thrown from above
  92. // if it's a response, we just got a bad status code and we
  93. // can re-throw to allow the retry
  94. const isRetryError = err.retried instanceof fetch.Response ||
  95. (RETRY_ERRORS.includes(code) && RETRY_TYPES.includes(err.type))
  96. if (req.method === 'POST' || isRetryError) {
  97. throw err
  98. }
  99. if (typeof options.onRetry === 'function') {
  100. options.onRetry(err)
  101. }
  102. log.http('fetch', `${req.method} ${req.url} attempt ${attemptNum} failed with ${err.code}`)
  103. return retryHandler(err)
  104. }
  105. }, options.retry).catch((err) => {
  106. // don't reject for http errors, just return them
  107. if (err.status >= 400 && err.type !== 'system') {
  108. return err
  109. }
  110. throw err
  111. })
  112. }
  113. module.exports = remoteFetch