got-scraping
got-scraping copied to clipboard
apify
Error: Failed to set ECDH curve
Hi together 😊
I am not sure whether or not this is an issue with got-scraping or rather one with the server I am scraping from (Apache/2.4.38; Debian) ...
Every ~15th request to the server fails with the error Failed to set ECDH curve with code ERR_CRYPTO_OPERATION_FAILED.
The SSL report states that the handshake Java 6u45 is failing. Does this correlate?
I looked up ECDH curve but it does not give me any hint and I don't have a lot of knowledge when it comes to topics like http request forging, cryptography, keys, etc. Any help in any directions, or resources to read into are highly appreciated.
Since I need at least like 200k requests to the server, I would really love to fix this any way I can. Thanks in advance 🙌🏻
Request:
import { gotScraping } from 'got-scraping'
export function scrape(url: string): Promise<string> {
return new Promise((resolve) => {
gotScraping
.get(url)
.then(({ body }) => resolve(body as string))
.catch((err) => {
console.error(err)
})
})
}
Error:
RequestError: Failed to set ECDH curve
at Request._beforeError (<mypath>/node_modules/got-cjs/dist/source/core/index.js:324:21)
at Request.flush (<mypath>/node_modules/got-cjs/dist/source/core/index.js:313:18)
at processTicksAndRejections (node:internal/process/task_queues:95:5)
at configSecureContext (node:internal/tls/secure-context:239:11)
at Object.createSecureContext (node:_tls_common:117:3)
at Object.connect (node:_tls_wrap:1629:48)
at Agent.createConnection (node:https:150:22)
at Agent.createSocket (node:_http_agent:350:26)
at Agent.addRequest (node:_http_agent:297:10)
at TransformHeadersAgent.addRequest (<mypath>/node_modules/got-scraping/src/agent/wrapped-agent.ts:19:20)
at TransformHeadersAgent.addRequest (<mypath>/node_modules/got-scraping/src/agent/transform-headers-agent.ts:95:22)
at new ClientRequest (node:_http_client:335:16)
at Object.request (node:http:97:10) {
input: undefined,
code: 'ERR_CRYPTO_OPERATION_FAILED',
timings: undefined,
options: {
request: [Function (anonymous)],
agent: {
http: TransformHeadersAgent { agent: [Agent] },
https: TransformHeadersAgent { agent: [Agent] },
http2: undefined
},
h2session: undefined,
decompress: true,
timeout: {
connect: undefined,
lookup: undefined,
read: undefined,
request: 60000,
response: undefined,
secureConnect: undefined,
send: undefined,
socket: undefined
},
prefixUrl: '',
body: undefined,
form: undefined,
json: undefined,
cookieJar: undefined,
ignoreInvalidCookies: false,
searchParams: undefined,
dnsLookup: undefined,
dnsCache: undefined,
context: {
headerGenerator: HeaderGenerator {
globalOptions: [Object],
browserListQuery: undefined,
inputGeneratorNetwork: [BayesianNetwork],
headerGeneratorNetwork: [BayesianNetwork],
uniqueBrowsers: [Array],
headersOrder: [Object]
},
useHeaderGenerator: true,
insecureHTTPParser: true,
sessionData: undefined
},
hooks: {
init: [
[Function: optionsValidationHandler],
[Function: customOptionsHook]
],
beforeRequest: [
[Function: insecureParserHook],
[Function: sessionDataHook],
[Function: http2Hook],
[AsyncFunction: proxyHook],
[AsyncFunction: browserHeadersHook],
[Function: tlsHook]
],
beforeError: [],
beforeRedirect: [ [Function: refererHook] ],
beforeRetry: [],
afterResponse: []
},
followRedirect: true,
maxRedirects: 10,
cache: undefined,
throwHttpErrors: false,
username: '',
password: '',
http2: true,
allowGetBody: false,
headers: {
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0',
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'accept-language': 'en-US',
'accept-encoding': 'gzip, deflate, br',
dnt: '1',
connection: 'keep-alive',
'upgrade-insecure-requests': '1',
'sec-fetch-mode': 'navigate',
'sec-fetch-dest': 'document',
'sec-fetch-site': 'same-site',
'sec-fetch-user': '?1'
},
methodRewriting: false,
dnsLookupIpVersion: undefined,
parseJson: [Function: parse],
stringifyJson: [Function: stringify],
retry: {
limit: 0,
methods: [ 'GET', 'PUT', 'HEAD', 'DELETE', 'OPTIONS', 'TRACE' ],
statusCodes: [
408, 413, 429, 500,
502, 503, 504, 521,
522, 524
],
errorCodes: [
'ETIMEDOUT',
'ECONNRESET',
'EADDRINUSE',
'ECONNREFUSED',
'EPIPE',
'ENOTFOUND',
'ENETUNREACH',
'EAI_AGAIN'
],
maxRetryAfter: undefined,
calculateDelay: [Function: calculateDelay],
backoffLimit: Infinity,
noise: 100
},
localAddress: undefined,
method: 'GET',
createConnection: undefined,
cacheOptions: {
shared: undefined,
cacheHeuristic: undefined,
immutableMinTimeToLive: undefined,
ignoreCargoCult: undefined
},
https: {
alpnProtocols: undefined,
rejectUnauthorized: false,
checkServerIdentity: undefined,
certificateAuthority: undefined,
key: undefined,
certificate: undefined,
passphrase: undefined,
pfx: undefined,
ciphers: 'TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA:DES-CBC3-SHA',
honorCipherOrder: undefined,
minVersion: 'TLSv1.2',
maxVersion: 'TLSv1.3',
signatureAlgorithms: 'ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:ecdsa_secp521r1_sha512:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha256:rsa_pkcs1_sha384:rsa_pkcs1_sha512:ECDSA+SHA1:rsa_pkcs1_sha1',
tlsSessionLifetime: undefined,
dhparam: undefined,
ecdhCurve: 'X25519:prime256v1:secp384r1:secp521r1:ffdhe2048:ffdhe3072',
certificateRevocationLists: undefined
},
encoding: undefined,
resolveBodyOnly: false,
isStream: false,
responseType: 'text',
url: URL {
href: "theURLiAmScraping"
},
pagination: {
transform: [Function: transform],
paginate: [Function: paginate],
filter: [Function: filter],
shouldContinue: [Function: shouldContinue],
countLimit: Infinity,
backoff: 0,
requestLimit: 10000,
stackAllItems: false
},
setHost: true,
maxHeaderSize: undefined,
signal: undefined,
enableUnixSockets: true
}
}
<mypath>/node_modules/got-cjs/dist/source/core/index.js:324
error = new errors_js_1.RequestError(error.message, error, this);
^
RequestError: Failed to set ECDH curve
at Request._beforeError (<mypath>/node_modules/got-cjs/dist/source/core/index.js:324:21)
at Request.flush (<mypath>/node_modules/got-cjs/dist/source/core/index.js:313:18)
at processTicksAndRejections (node:internal/process/task_queues:95:5)
at configSecureContext (node:internal/tls/secure-context:239:11)
at Object.createSecureContext (node:_tls_common:117:3)
at Object.connect (node:_tls_wrap:1629:48)
at Agent.createConnection (node:https:150:22)
at Agent.createSocket (node:_http_agent:350:26)
at Agent.addRequest (node:_http_agent:297:10)
at TransformHeadersAgent.addRequest (<mypath>/node_modules/got-scraping/src/agent/wrapped-agent.ts:19:20)
at TransformHeadersAgent.addRequest (<mypath>/node_modules/got-scraping/src/agent/transform-headers-agent.ts:95:22)
at new ClientRequest (node:_http_client:335:16)
at Object.request (node:http:97:10) {
input: undefined,
code: 'ERR_CRYPTO_OPERATION_FAILED',
timings: undefined,
options: Options {
_unixOptions: {
insecureHTTPParser: true,
secureOptions: 524304,
requestOCSP: true
},
_internals: {
request: [Function (anonymous)],
agent: [Object],
h2session: undefined,
decompress: true,
timeout: [Object],
prefixUrl: '',
body: undefined,
form: undefined,
json: undefined,
cookieJar: undefined,
ignoreInvalidCookies: false,
searchParams: undefined,
dnsLookup: undefined,
dnsCache: undefined,
context: [Object],
hooks: [Object],
followRedirect: true,
maxRedirects: 10,
cache: undefined,
throwHttpErrors: false,
username: '',
password: '',
http2: true,
allowGetBody: false,
headers: [Object],
methodRewriting: false,
dnsLookupIpVersion: undefined,
parseJson: [Function: parse],
stringifyJson: [Function: stringify],
retry: [Object],
localAddress: undefined,
method: 'GET',
createConnection: undefined,
cacheOptions: [Object],
https: [Object],
encoding: undefined,
resolveBodyOnly: false,
isStream: false,
responseType: 'text',
url: [URL],
pagination: [Object],
setHost: true,
maxHeaderSize: undefined,
signal: undefined,
enableUnixSockets: true
},
_merging: false,
_init: [ [Object], [Object] ]
}
}
I was having this running on Node v18. Switched down to Node v16 and it went away.
I had the same issue and in my case it didn't work with node 19.8.1. Going down to node v.18.12.1 helped.
