Revert "got --> node-fetch" (#7175)

This reverts commit b209f5c.

core/base-service/base.js

@@ -20,7 +20,7 @@ import {
  Deprecated,
 } from './errors.js'
 import { validateExample, transformExample } from './examples.js'
-import { sendRequest } from './node-fetch.js'
+import { fetchFactory } from './got.js'
 import {
  makeFullUrl,
  assertValidRoute,
@@ -432,7 +432,7 @@ class BaseService {
  ServiceClass: this,
  })
 
- const fetcher = sendRequest.bind(sendRequest, fetchLimitBytes)
+ const fetcher = fetchFactory(fetchLimitBytes)
 
  camp.route(
  regex,

core/base-service/got.js

@@ -0,0 +1,96 @@
+import got from 'got'
+import { Inaccessible, InvalidResponse } from './errors.js'
+
+const userAgent = 'Shields.io/2003a'
+
+function requestOptions2GotOptions(options) {
+ const requestOptions = Object.assign({}, options)
+ const gotOptions = {}
+ const interchangableOptions = ['body', 'form', 'headers', 'method', 'url']
+
+ interchangableOptions.forEach(function (opt) {
+ if (opt in requestOptions) {
+ gotOptions[opt] = requestOptions[opt]
+ delete requestOptions[opt]
+ }
+ })
+
+ if ('qs' in requestOptions) {
+ gotOptions.searchParams = requestOptions.qs
+ delete requestOptions.qs
+ }
+
+ if ('gzip' in requestOptions) {
+ gotOptions.decompress = requestOptions.gzip
+ delete requestOptions.gzip
+ }
+
+ if ('strictSSL' in requestOptions) {
+ gotOptions.https = {
+ rejectUnauthorized: requestOptions.strictSSL,
+ }
+ delete requestOptions.strictSSL
+ }
+
+ if ('auth' in requestOptions) {
+ gotOptions.username = requestOptions.auth.user
+ gotOptions.password = requestOptions.auth.pass
+ delete requestOptions.auth
+ }
+
+ if (Object.keys(requestOptions).length > 0) {
+ throw new Error(`Found unrecognised options ${Object.keys(requestOptions)}`)
+ }
+
+ return gotOptions
+}
+
+async function sendRequest(gotWrapper, url, options) {
+ const gotOptions = requestOptions2GotOptions(options)
+ gotOptions.throwHttpErrors = false
+ gotOptions.retry = 0
+ gotOptions.headers = gotOptions.headers || {}
+ gotOptions.headers['User-Agent'] = userAgent
+ try {
+ const resp = await gotWrapper(url, gotOptions)
+ return { res: resp, buffer: resp.body }
+ } catch (err) {
+ if (err instanceof got.CancelError) {
+ throw new InvalidResponse({
+ underlyingError: new Error('Maximum response size exceeded'),
+ })
+ }
+ throw new Inaccessible({ underlyingError: err })
+ }
+}
+
+function fetchFactory(fetchLimitBytes) {
+ const gotWithLimit = got.extend({
+ handlers: [
+ (options, next) => {
+ const promiseOrStream = next(options)
+ promiseOrStream.on('downloadProgress', progress => {
+ if (
+ progress.transferred > fetchLimitBytes &&
+ // just accept the file if we've already finished downloading
+ // the entire file before we went over the limit
+ progress.percent !== 1
+ ) {
+ /*
+ TODO: we should be able to pass cancel() a message
+ https:/sindresorhus/got/blob/main/documentation/advanced-creation.md#examples
+ but by the time we catch it, err.message is just "Promise was canceled"
+ */
+ promiseOrStream.cancel('Maximum response size exceeded')
+ }
+ })
+
+ return promiseOrStream
+ },
+ ],
+ })
+
+ return sendRequest.bind(sendRequest, gotWithLimit)
+}
+
+export { requestOptions2GotOptions, fetchFactory }

core/base-service/got.spec.js

@@ -0,0 +1,102 @@
+import { expect } from 'chai'
+import nock from 'nock'
+import { requestOptions2GotOptions, fetchFactory } from './got.js'
+import { Inaccessible, InvalidResponse } from './errors.js'
+
+describe('requestOptions2GotOptions function', function () {
+ it('translates valid options', function () {
+ expect(
+ requestOptions2GotOptions({
+ body: 'body',
+ form: 'form',
+ headers: 'headers',
+ method: 'method',
+ url: 'url',
+ qs: 'qs',
+ gzip: 'gzip',
+ strictSSL: 'strictSSL',
+ auth: { user: 'user', pass: 'pass' },
+ })
+ ).to.deep.equal({
+ body: 'body',
+ form: 'form',
+ headers: 'headers',
+ method: 'method',
+ url: 'url',
+ searchParams: 'qs',
+ decompress: 'gzip',
+ https: { rejectUnauthorized: 'strictSSL' },
+ username: 'user',
+ password: 'pass',
+ })
+ })
+
+ it('throws if unrecognised options are found', function () {
+ expect(() =>
+ requestOptions2GotOptions({ body: 'body', foobar: 'foobar' })
+ ).to.throw(Error, 'Found unrecognised options foobar')
+ })
+})
+
+describe('got wrapper', function () {
+ it('should not throw an error if the response <= fetchLimitBytes', async function () {
+ nock('https://www.google.com')
+ .get('/foo/bar')
+ .once()
+ .reply(200, 'x'.repeat(100))
+ const sendRequest = fetchFactory(100)
+ const { res } = await sendRequest('https://www.google.com/foo/bar')
+ expect(res.statusCode).to.equal(200)
+ })
+
+ it('should throw an InvalidResponse error if the response is > fetchLimitBytes', async function () {
+ nock('https://www.google.com')
+ .get('/foo/bar')
+ .once()
+ .reply(200, 'x'.repeat(101))
+ const sendRequest = fetchFactory(100)
+ return expect(
+ sendRequest('https://www.google.com/foo/bar')
+ ).to.be.rejectedWith(InvalidResponse, 'Maximum response size exceeded')
+ })
+
+ it('should throw an Inaccessible error if the request throws a (non-HTTP) error', async function () {
+ nock('https://www.google.com').get('/foo/bar').replyWithError('oh no')
+ const sendRequest = fetchFactory(1024)
+ return expect(
+ sendRequest('https://www.google.com/foo/bar')
+ ).to.be.rejectedWith(Inaccessible, 'oh no')
+ })
+
+ it('should throw an Inaccessible error if the host can not be accessed', async function () {
+ this.timeout(5000)
+ nock.disableNetConnect()
+ const sendRequest = fetchFactory(1024)
+ return expect(
+ sendRequest('https://www.google.com/foo/bar')
+ ).to.be.rejectedWith(
+ Inaccessible,
+ 'Nock: Disallowed net connect for "www.google.com:443/foo/bar"'
+ )
+ })
+
+ it('should pass a custom user agent header', async function () {
+ nock('https://www.google.com', {
+ reqheaders: {
+ 'user-agent': function (agent) {
+ return agent.startsWith('Shields.io')
+ },
+ },
+ })
+ .get('/foo/bar')
+ .once()
+ .reply(200)
+ const sendRequest = fetchFactory(1024)
+ await sendRequest('https://www.google.com/foo/bar')
+ })
+
+ afterEach(function () {
+ nock.cleanAll()
+ nock.enableNetConnect()
+ })
+})