Add TransfomHeadersAgent

README.md

@@ -38,6 +38,8 @@ This package can only generate all the standard attributes. There still might be
 The second colossal factor is using the same HTTP version as browsers. Most modern browsers use HTTP v2, so using it when the server supports it takes you one more step further to look like a browser. Luckily you don't have to care about the server HTTP version support since `got` automatically handles [ALPN protocol negotiation](https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation) and uses the HTTP v2 only if it is supported.
 The last step is to have a browser-like TLS suite and ciphers. According to our research, the cipher `TLS_AES_256_GCM_SHA384` is used among all modern browsers. We use this cipher as a default one. However, feel free to change it.
 
+HTTP/1.1 headers are always automatically formatted in [`Pascal-Case`](https://pl.wikipedia.org/wiki/PascalCase). There is an expection: [`x-`](https://datatracker.ietf.org/doc/html/rfc7231#section-8.3.1) headers are not modified in *any* way.
+
 This package should provide a solid start for your browser request emulation process. All websites are built differently, and some of them might require some additional special care.
 
 ## Proxies

package.json

@@ -23,6 +23,7 @@
  "eslint": "^7.0.0",
  "express": "^4.17.1",
  "fs-extra": "^9.1.0",
+ "get-stream": "^5.2.0",
  "jest": "^26.6.3",
  "jest-extended": "^0.11.5",
  "jsdoc-to-markdown": "^7.0.0",

src/agent/transform-headers-agent.js

@@ -0,0 +1,131 @@
+/* eslint-disable no-underscore-dangle */
+const http = require('http');
+const WrappedAgent = require('./wrapped-agent');
+
+const { _storeHeader } = http.OutgoingMessage.prototype;
+
+/**
+ * @description Transforms the casing of the headers to Pascal-Case.
+ */
+class TransformHeadersAgent extends WrappedAgent {
+ // Rewritten from https:/nodejs/node/blob/533cafcf7e3ab72e98a2478bc69aedfdf06d3a5e/lib/_http_outgoing.js#L442-L479
+ /**
+ * @description Transforms the request via header normalization.
+ * @see {TransformHeadersAgent.toPascalCase}
+ * @param {http.ClientRequest} request
+ * @param {string[]} sortedHeaders - headers in order, optional
+ */
+ transformRequest(request, sortedHeaders) {
+ const headers = {};
+ const hasConnection = request.hasHeader('connection');
+ const hasContentLength = request.hasHeader('content-length');
+ const hasTransferEncoding = request.hasHeader('transfer-encoding');
+ const hasTrailer = request.hasHeader('trailer');
+ const keys = request.getHeaderNames();
+
+ for (const key of keys) {
+ if (key.toLowerCase().startsWith('x-')) {
+ headers[key] = request.getHeader(key);
+ } else {
+ headers[this.toPascalCase(key)] = request.getHeader(key);
+ }
+
+ if (sortedHeaders) {
+ // Removal is required in order to change the order of the properties
+ request.removeHeader(key);
+ }
+ }
+
+ if (!hasConnection) {
+ const shouldSendKeepAlive = request.shouldKeepAlive && (hasContentLength || request.useChunkedEncodingByDefault || request.agent);
+ if (shouldSendKeepAlive) {
+ headers.Connection = 'keep-alive';
+ } else {
+ headers.Connection = 'close';
+ }
+ }
+
+ if (!hasContentLength && !hasTransferEncoding) {
+ // Note: This uses private `_removedContLen` property.
+ // This property tells us whether the content-length was explicitly removed or not.
+ //
+ // Note: This uses private `_removedTE` property.
+ // This property tells us whether the transfer-encoding was explicitly removed or not.
+ if (!hasTrailer && !request._removedContLen && typeof request._contentLength === 'number') {
+ headers['Content-Length'] = request._contentLength;
+ } else if (!request._removedTE) {
+ headers['Transfer-Encoding'] = 'chunked';
+ }
+ }
+
+ const normalizedKeys = Object.keys(headers);
+ const sorted = sortedHeaders ? normalizedKeys.sort(this.createSort(sortedHeaders)) : normalizedKeys;
+
+ for (const key of sorted) {
+ request.setHeader(key, headers[key]);
+ }
+ }
+
+ addRequest(request, options) {
+ // See https:/nodejs/node/blob/533cafcf7e3ab72e98a2478bc69aedfdf06d3a5e/lib/_http_outgoing.js#L373
+ // Note: This overrides the private `_storeHeader`.
+ // This is required, because the function directly copies
+ // the `connection`, `content-length` and `trasfer-encoding` headers
+ // directly to the underlying buffer.
+ request._storeHeader = (...args) => {
+ this.transformRequest(request, options.sortedHeaders);
+
+ return _storeHeader.call(request, ...args);
+ };
+
+ return super.addRequest(request, options);
+ }
+
+ /**
+ * @param {string} header - header with unknown casing
+ * @returns {string} - header in Pascal-Case
+ */
+ toPascalCase(header) {
+ return header.split('-').map((part) => {
+ return part[0].toUpperCase() + part.slice(1).toLowerCase();
+ }).join('-');
+ }
+
+ /**
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort
+ * @param {string} a - header a
+ * @param {string} b - header b
+ * @param {string[]} sortedHeaders - array of headers in order
+ * @returns header a or header b, depending which one is more important
+ */
+ sort(a, b, sortedHeaders) {
+ const rawA = sortedHeaders.indexOf(a);
+ const rawB = sortedHeaders.indexOf(b);
+ const indexA = rawA === -1 ? Number.POSITIVE_INFINITY : rawA;
+ const indexB = rawB === -1 ? Number.POSITIVE_INFINITY : rawB;
+
+ if (indexA < indexB) {
+ return -1;
+ }
+
+ if (indexA > indexB) {
+ return 1;
+ }
+
+ return 0;
+ }
+
+ /**
+ *
+ * @param {string[]} sortedHeaders - array of headers in order
+ * @returns {Function} - sort function
+ */
+ createSort(sortedHeaders) {
+ const sortWithSortedHeaders = (a, b) => this.sort(a, b, sortedHeaders);
+
+ return sortWithSortedHeaders;
+ }
+}
+
+module.exports = TransformHeadersAgent;

src/agent/wrapped-agent.js

@@ -0,0 +1,42 @@
+/**
+ * @see https:/nodejs/node/blob/533cafcf7e3ab72e98a2478bc69aedfdf06d3a5e/lib/_http_client.js#L129-L162
+ * @see https:/nodejs/node/blob/533cafcf7e3ab72e98a2478bc69aedfdf06d3a5e/lib/_http_client.js#L234-L246
+ * @see https:/nodejs/node/blob/533cafcf7e3ab72e98a2478bc69aedfdf06d3a5e/lib/_http_client.js#L304-L305
+ * @description Wraps an existing Agent instance,
+ * so there's no need to replace `agent.addRequest`.
+ */
+class WrappedAgent {
+ constructor(agent) {
+ this.agent = agent;
+ }
+
+ addRequest(request, options) {
+ return this.agent.addRequest(request, options);
+ }
+
+ get keepAlive() {
+ return this.agent.keepAlive;
+ }
+
+ get maxSockets() {
+ return this.agent.maxSockets;
+ }
+
+ get options() {
+ return this.agent.options;
+ }
+
+ get defaultPort() {
+ return this.agent.defaultPort;
+ }
+
+ get protocol() {
+ return this.agent.protocol;
+ }
+
+ destroy() {
+ this.agent.destroy();
+ }
+}
+
+module.exports = WrappedAgent;

src/hooks/proxy.js

@@ -2,6 +2,7 @@ const http2 = require('http2-wrapper');
 const HttpsProxyAgent = require('https-proxy-agent');
 const HttpProxyAgent = require('http-proxy-agent');
 const httpResolver = require('../http-resolver');
+const TransformHeadersAgent = require('../agent/transform-headers-agent');
 
 const {
  HttpOverHttp2,
@@ -25,27 +26,32 @@ exports.proxyHook = async function (options) {
  const parsedProxy = new URL(proxyUrl);
 
  validateProxyProtocol(parsedProxy.protocol);
- const agents = await getAgents(parsedProxy, options.https.rejectUnauthorized);
+ options.agent = await getAgents(parsedProxy, options.https.rejectUnauthorized);
+ }
 
- /**
- * This is needed because got expects all three agents in an object like this:
- * {
- * http: httpAgent,
- * https: httpsAgent,
- * http2: http2Agent,
- * }
- * The confusing thing is that internally, it destructures the agents out of
- * the object for HTTP and HTTPS, but keeps the structure for HTTP2,
- * because it passes all the agents down to http2.auto (from http2-wrapper).
- * We're not using http2.auto, but http2.request, which expects a single agent.
- * So for HTTP2, we need a single agent and for HTTP and HTTPS we need the object
- * to allow destructuring of correct agents.
- */
- if (resolvedRequestProtocol === 'http2') {
- options.agent = agents[resolvedRequestProtocol];
- } else {
- options.agent = agents;
- }
+ /**
+ * This is needed because got expects all three agents in an object like this:
+ * {
+ * http: httpAgent,
+ * https: httpsAgent,
+ * http2: http2Agent,
+ * }
+ * The confusing thing is that internally, it destructures the agents out of
+ * the object for HTTP and HTTPS, but keeps the structure for HTTP2,
+ * because it passes all the agents down to http2.auto (from http2-wrapper).
+ * We're not using http2.auto, but http2.request, which expects a single agent.
+ * So for HTTP2, we need a single agent and for HTTP and HTTPS we need the object
+ * to allow destructuring of correct agents.
+ * ---
+ * The `if` below cannot be placed inside the `if` above.
+ * Otherwise `http2.request` would receive the entire `agent` object
+ * __when not using proxy__.
+ * ---
+ * `http2.request`, in contrary to `http2.auto`, expects an instance of `http2.Agent`.
+ * `http2.auto` expects an object with `http`, `https` and `http2` properties.
+ */
+ if (resolvedRequestProtocol === 'http2') {
+ options.agent = options.agent[resolvedRequestProtocol];
  }
 };
 
@@ -83,21 +89,21 @@ async function getAgents(parsedProxyUrl, rejectUnauthorized) {
 
  if (proxyIsHttp2) {
  agent = {
- http: new HttpOverHttp2(proxy),
- https: new HttpsOverHttp2(proxy),
+ http: new TransformHeadersAgent(new HttpOverHttp2(proxy)),
+ https: new TransformHeadersAgent(new HttpsOverHttp2(proxy)),
  http2: new Http2OverHttp2(proxy),
  };
  } else {
  agent = {
- http: new HttpsProxyAgent(proxyUrl.href),
- https: new HttpsProxyAgent(proxyUrl.href),
+ http: new TransformHeadersAgent(new HttpsProxyAgent(proxyUrl.href)),
+ https: new TransformHeadersAgent(new HttpsProxyAgent(proxyUrl.href)),
  http2: new Http2OverHttps(proxy),
  };
  }
  } else {
  agent = {
- http: new HttpProxyAgent(proxyUrl.href),
- https: new HttpsProxyAgent(proxyUrl.href),
+ http: new TransformHeadersAgent(new HttpProxyAgent(proxyUrl.href)),
+ https: new TransformHeadersAgent(new HttpsProxyAgent(proxyUrl.href)),
  http2: new Http2OverHttp(proxy),
  };
  }

src/index.js

@@ -1,6 +1,10 @@
+const http = require('http');
+const https = require('https');
+
 const got = require('got');
 const HeaderGenerator = require('header-generator');
 
+const TransformHeadersAgent = require('./agent/transform-headers-agent');
 const { SCRAPING_DEFAULT_OPTIONS } = require('./scraping-defaults');
 
 const { optionsValidationHandler } = require('./hooks/options-validation');
@@ -17,6 +21,10 @@ const gotScraping = got.extend({
  context: {
  headerGenerator: new HeaderGenerator(),
  },
+ agent: {
+ http: new TransformHeadersAgent(http.globalAgent),
+ https: new TransformHeadersAgent(https.globalAgent),
+ },
  hooks: {
  init: [
  (opts) => optionsValidationHandler(opts, () => {}),