Node.js Proxy Setup: Axios, Fetch and Proxy Agents

// rotate.cjs
const axios = require("axios");
const { HttpsProxyAgent } = require("https-proxy-agent");

const { PROXY_USER, PROXY_PASS } = process.env;
const proxyUrl = `http://${encodeURIComponent(PROXY_USER)}:${encodeURIComponent(PROXY_PASS)}@gate.proxyfield.io:7000`;
const rotating = axios.create({
  httpsAgent: new HttpsProxyAgent(proxyUrl, { keepAlive: false }),
  proxy: false,
  timeout: 30000,
});

(async () => {
  const ips = await Promise.all(
    Array.from({ length: 5 }, () => rotating.get("https://api.ipify.org").then(r => r.data))
  );
  console.log(ips, new Set(ips).size, "distinct");
})().catch(err => { console.error(err.message); process.exit(1); });

const sticky = (id, ttl = 1800) => axios.create({
  httpsAgent: new HttpsProxyAgent(
    `http://${encodeURIComponent(`${PROXY_USER}-session-${id}-ttl-${ttl}`)}:${encodeURIComponent(PROXY_PASS)}@gate.proxyfield.io:7000`
  ),
  proxy: false,
  timeout: 30000,
});
const task = sticky("a1b2c3");
console.log((await task.get("https://api.ipify.org")).data); // the same IP for 30 min

// STATIC_PROXY_URL: the proxy URL of your dedicated IP, from your ProxyField account
const fixed = axios.create({
  httpsAgent: new HttpsProxyAgent(process.env.STATIC_PROXY_URL, { keepAlive: true, maxSockets: 20 }),
  proxy: false,
  timeout: 30000,
});

let bytes = 0;
rotating.interceptors.response.use(r => {
  const sent = r.request?._header?.length ?? 0;                                               // request line and headers
  const head = (r.request?.res?.rawHeaders ?? []).reduce((n, h) => n + h.length + 2, 0) + 17; // status line and headers
  const body = Number(r.headers["content-length"] ?? Buffer.byteLength(typeof r.data === "string" ? r.data : JSON.stringify(r.data ?? "")));
  bytes += sent + head + body;
  return r;
});
// ... after the run:
console.log(`${bytes} bytes = ${(bytes / 1e9).toFixed(6)} GB ≈ $${((bytes / 1e9) * 2.95).toFixed(4)} at $2.95/GB`);

import axios from "axios";
import { HttpsProxyAgent } from "https-proxy-agent";
import pLimit from "p-limit";

const { PROXY_USER, PROXY_PASS } = process.env;
const agent = new HttpsProxyAgent(
  `http://${encodeURIComponent(PROXY_USER)}:${encodeURIComponent(PROXY_PASS)}@gate.proxyfield.io:7000`,
  { keepAlive: false } // one tunnel per request; p-limit below caps how many are open
);
const client = axios.create({ httpsAgent: agent, proxy: false, timeout: 30000 });
const limit = pLimit(100); // requests in flight at once

async function get(url, tries = 3) {
  for (let i = 1; ; i++) {
    try {
      return (await client.get(url)).data;
    } catch (err) {
      const status = err.response?.status;
      const retry = !status || status === 429 || status >= 500; // network error, rate limit, server error
      if (!retry || i === tries) throw err;
      await new Promise(r => setTimeout(r, 1000 * 2 ** (i - 1))); // back off 1 s, 2 s; the retry gets a new IP
    }
  }
}

const urls = ["https://example.com/page/1" /* , ... */];
const results = await Promise.allSettled(urls.map(u => limit(() => get(u))));
console.log(results.filter(r => r.status === "fulfilled").length, "pages fetched");

// bench.mjs: PROXY_USER=<login>-country-us PROXY_PASS=... node bench.mjs 100
import axios from "axios";
import { HttpsProxyAgent } from "https-proxy-agent";
import pLimit from "p-limit";

const N = Number(process.argv[2] ?? 100), TOTAL = 500;
const { PROXY_USER, PROXY_PASS } = process.env;
const agent = new HttpsProxyAgent(
  `http://${encodeURIComponent(PROXY_USER)}:${encodeURIComponent(PROXY_PASS)}@gate.proxyfield.io:7000`,
  { keepAlive: false } // no maxSockets: a capped agent hands a freed tunnel to the next queued request
);
const client = axios.create({ httpsAgent: agent, proxy: false, timeout: 30000, validateStatus: () => true });
const limit = pLimit(N);
const ips = new Set(), statuses = {};
const t0 = Date.now();

await Promise.all(Array.from({ length: TOTAL }, () => limit(async () => {
  try {
    const r = await client.get("https://api.ipify.org");
    statuses[r.status] = (statuses[r.status] ?? 0) + 1;
    if (r.status === 200) ips.add(String(r.data));
  } catch (err) {
    const k = err.code ?? "error";
    statuses[k] = (statuses[k] ?? 0) + 1;
  }
})));

console.log({ concurrency: N, requests: TOTAL, distinctIPs: ips.size, statuses,
  seconds: ((Date.now() - t0) / 1000).toFixed(1) });

import { fetch, ProxyAgent } from "undici";

const { PROXY_USER, PROXY_PASS } = process.env;
const dispatcher = new ProxyAgent({
  uri: "http://gate.proxyfield.io:7000",
  token: "Basic " + Buffer.from(`${PROXY_USER}:${PROXY_PASS}`).toString("base64"),
  connections: 50, // tunnels per target site; other requests wait for an open one
});
const ips = await Promise.all(Array.from({ length: 200 }, () =>
  fetch("https://api.ipify.org", { dispatcher }).then(r => r.text())));
console.log(ips.length, "requests,", new Set(ips).size, "distinct exits");
await dispatcher.close();

import { fetch, ProxyAgent } from "undici";

const proxy = new ProxyAgent("http://<login>:<password>@gate.proxyfield.io:7000");
const res = await fetch("https://api.ipify.org", { dispatcher: proxy });
console.log(await res.text()); // the proxy's exit IP, not your server's
await proxy.close();           // close the agent's connections when you are done

import { fetch, ProxyAgent } from "undici";

const { PROXY_USER, PROXY_PASS } = process.env;
const dispatcher = new ProxyAgent({
  uri: "http://gate.proxyfield.io:7000",
  token: "Basic " + Buffer.from(`${PROXY_USER}:${PROXY_PASS}`).toString("base64"), // PROXY_USER=<login>-country-us
});

const res = await fetch("https://api.ipify.org", { dispatcher });
console.log(await res.text()); // the proxy's IP, not your server's
await dispatcher.close();

import { ProxyAgent, setGlobalDispatcher } from "undici";
setGlobalDispatcher(new ProxyAgent({ uri: "http://gate.proxyfield.io:7000", token: "Basic " + Buffer.from(`${process.env.PROXY_USER}:${process.env.PROXY_PASS}`).toString("base64") }));

# the flag
HTTPS_PROXY="http://<login>:<password>@gate.proxyfield.io:7000" node --use-env-proxy app.js

# the variable, with hosts that should skip the proxy
export HTTPS_PROXY="http://<login>:<password>@gate.proxyfield.io:7000"
export NO_PROXY="localhost,127.0.0.1,.internal.example.com"
NODE_USE_ENV_PROXY=1 node app.js

// npm install undici
import { EnvHttpProxyAgent, setGlobalDispatcher } from "undici";
setGlobalDispatcher(new EnvHttpProxyAgent()); // reads HTTP_PROXY, HTTPS_PROXY, NO_PROXY