Currently, I am developing an RSS aggregator using node-webkit. However, I have encountered an issue where a certain number of my requests (code provided below) are failing with various errors such as HPE_INVALID_CONSTANT, ETIMEDOUT, EAGAIN, and HPE_INVALID_HEADER_TOKEN. These errors seem to occur randomly in frequency.
To test this problem, I created a test.html file that is relevant to the issue at hand. By inputting a large number of web page URLs into the httpList, the issue can be replicated. It is important to choose URLs carefully to avoid crashing servers - around 56 URLs seem to trigger these errors.
Moreover, reproducing this error has been particularly challenging, indicating that the cause may lie on my end. There are instances where the same URL will fail on one attempt but succeed on the next. I have also come across URLs that failed through the script but were accessible when opened in a browser like Chrome.
<!DOCTYPE html>
<html>
<head>
<title>HTTP Get test</title>
</head>
<body>
<script>
var httpList = [];
var fromUrl = function(stringUrl) {
var http = require('http');
var url = require("url");
var urlData = url.parse(stringUrl);
var parser = window.document.createElement('a');
parser.href = stringUrl;
var options = {
host: urlData.host,
path: urlData.path + urlData.search + urlData.hash
};
var httpResponse = function(response) {
response.setEncoding('utf8');
request.setSocketKeepAlive(true);
var str = '';
response.on('data', function(chunk) {
str += chunk;
});
response.on('end', function() {
request.setSocketKeepAlive(false);
console.log("url",stringUrl,"was successfully parsed");
});
}
var request = http.request(options, httpResponse)
request.on('error', function(e) {
if (e.code === "ENOTFOUND") {
console.error("The feed", stringUrl, "could not be fetched.")
} else {
console.log(e, e.message, stringUrl);
}
});
request.end();
};
for(var i=0;i<httpList.length;i++) {
fromUrl(httpList[i]);
}
</script>
</body>
</html>