My Puppeteer and Node.js project hosted on fly.io is crashing due to insufficient memory. I have removed all unused and global variables, but the memory leak is still present. How can I fix this issue?
i see memory increased, but still i Colden’t find solution.
Here is my puppeteer codes.
This is the puppeteer file.
"use strict";
const puppeteer = require("puppeteer");
const dotenv = require("dotenv");
dotenv.config();
module.exports = async function scraping() {
let price1000 = null;
try {
const browser = await puppeteer.launch({
ignoreDefaultArgs: ["--disable-extensions"],
headless: true,
slowMo: 10,
args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-cache"],
});
const page = await browser.newPage();
//* counting time
const start = Date.now();
await page.goto(process.env.URL);
let price2000 = await page.evaluate(() => {
return document.getElementById("price").textContent;
});
await page.click("#switchQuantity");
//* -------------------------------//
await page
.waitForSelector("#price", {
visible: true,
hidden: false,
})
.then(async () => {
price1000 = await page.evaluate(() => {
return document.getElementById("price").textContent;
});
return price1000;
});
console.log("Took", Date.now() - start, "ms");
browser.disconnect();
console.log("line 48 puppeter js before run : ", process.memoryUsage());
// console.log("line 48 pupper jsafter run : ", process.memoryUsage());
return {
price1000,
price2000,
};
} catch (err) {
browser.close();
console.error(err);
}
console.time();
};
This is the nodemailer.js
"use strict";
const nodemailer = require("nodemailer");
const dotenv = require("dotenv");
const db = require("../sqllite/db");
const scrapingdata = require("./puppetter");
// const { setTimeout } = require("timers/promises");
dotenv.config();
//* with Gmail - generate password , //
async function main(emailadress) {
try {
let data = await scrapingdata(); //* prizes //
console.log("line 35, start nodemailer process ", data);
if (data) {
let trasporter = nodemailer.createTransport({
service: "gmail",
auth: {
user: process.env.GMAIL,
pass: process.env.PASSCODE,
},
});
//* Email Body //
let email = await trasporter.sendMail({
from: "[email protected]",
to: emailadress,
subject: `Vandag Mazoutprijs in Bilzen`,
html: `Vandag Minimumprijs van mazout op Bilzen ${data.price2000} €/L voor2000L & ${data.price1000}€/L voor1000L `,
});
console.log("Message sent: %s", email.messageId);
}
} catch (e) {
console.error(e);
}
}
//* create list //
module.exports = async function list() {
// let email_list = [];
// email_list.length = 0; //* every time array need to be empty //
try {
db.all("SELECT email FROM mail", (err, rows) => {
if (err) {
console.error(err);
}
rows.forEach((row) => {
// email_list.push(row.email);
// console.log("line 51", row.email);
main(row.email);
});
// email_list.forEach((element) => {
// main(element);
// });
});
// email_list.length = 0; //* every time array need to be empty //
} catch (e) {
console.error("error 28 :", e);
}
};
Yes. I have made several changes to my codes. And i see a big different. But still it crashed.
Theses are changes that I’m done so far.
- Puppeteer launch options, i removed
// args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-cache"],and addedargs: ["--no-sandbox"],. - used browser.close()
- used Promise.all to list() function.
"use strict";
const puppeteer = require("puppeteer");
const dotenv = require("dotenv");
dotenv.config();
module.exports = async function scraping() {
const browser = await puppeteer.launch({
headless: true,
slowMo: 10,
//! disable cache page //
//! memory //
args: ["--no-sandbox"],
// args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-cache"],
});
try {
let price1000 = null;
let price2000 = null;
const page = await browser.newPage();
//* counting time
const start = Date.now();
await page.goto(process.env.URL);
price2000 = await page.evaluate(() => {
return document.getElementById("price").textContent;
});
await page.click("#switchQuantity");
//* -------------------------------//
await page
.waitForSelector("#price", {
visible: true,
hidden: false,
// timeout: 12000,
})
.then(async () => {
price1000 = await page.evaluate(() => {
return document.getElementById("price").textContent;
});
return price1000;
});
console.log("Took", Date.now() - start, "ms");
browser.close();
// browser.disconnect();
return {
price1000,
price2000,
};
} catch (err) {
browser.close();
console.error(err);
}
console.time();
};
When my nodemail.js file execution use more memory.
"use strict";
const nodemailer = require("nodemailer");
const dotenv = require("dotenv");
const DB = require("../sqllite/db");
const scrapingdata = require("./puppetter");
dotenv.config();
//* with Gmail - generate password , //
async function main(emailadress) {
try {
let data = await scrapingdata(); //* prizes //
if (data) {
let trasporter = nodemailer.createTransport({
service: "gmail",
auth: {
user: process.env.GMAIL,
pass: process.env.PASSCODE,
},
});
//* Email Body //
let email = await trasporter.sendMail({
from: "[email protected]",
to: emailadress,
subject: `xxxxxx`,
html: `xxxxxx ${data.price2000} €/L & ${data.price1000}€/L `,
});
console.log("Message sent: %s", email.messageId);
}
} catch (e) {
console.error(e);
}
}
//* create list //
module.exports = async function list() {
console.log(`need to open Database for this list() function`);
try {
const db = DB;
const queryPromise = new Promise((resolve, reject) => {
db.all("SELECT email FROM mail", (err, rows) => {
if (err) {
reject(err);
} else {
const emails = rows.map((row) => row.email);
resolve(emails);
}
});
});
const emails = await queryPromise;
await Promise.all(emails.map((email) => main(email)));
} catch (e) {
console.error("error 28 :", e);
}
};