diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..12a6984 --- /dev/null +++ b/.gitignore @@ -0,0 +1,134 @@ +# Project specific +Download/**/** +*.conf + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* \ No newline at end of file diff --git a/README.md b/README.md index 21bb9fc..5c6fb73 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,45 @@ # WNtoEmail Node.js script to download new WebNovel chapters convert them to eBook and send to email. Mainly intended for sending to Kindle. +Script will pack your WebNovels into convinient eBooks, complete with cover image, metadata and table of contents. For ongoing series if will wait for a configured number of new chapters before sending a new volume to avoid spam. + +# Dependecies +It's a Node.js project so install that. +Project is using `node-html-parser` and `nodemailer` libraries. It should be enough to: +``` +npm install node-html-parser --save +npm install nodemailer --save +``` + +# Config +At first start it will create an empty config file `./novelConfig.conf`, adjust the setting according to comments below +``` +{ + "downloadLocation": "", // New chapter download location + "converterPath": "ebook-convert.exe", // Calibre eBook converter, I recommend adding it to you PATH, NOT tested when it's not in PATH + "ebookFormat": "epub", // Desired eBook format, Kindle started supporting epub dso that's default + "sendEmail": false, // If the script should send eBooks via email + "emailToAddress": "", // Email where to send your eBooks + "emailFromAddress": "", // Important for Kindle deliveries, make sure you have it added in Kindle settings + "emailProvider": "", // Gmail works fine, just need to set up 2FA and an app password + "emailUsername": "", // Usernam to your email account + "emailPassword": "", // Password to your email account + "emailAttachments": 25, // How many eBooks to attach to a single email + "supportedHosting": { // Enum to show which WebNovel host sites are supported, NOT configurable + "NF": "https://novelfull.com/" + }, + "template": { // Template for a WebNovel entry in "novels" below + "novelURL": "", // WebNovel address, it's enough to copy this template to "novels" and fill only this field to start + "title": "", // Autofill + "author": "", // Autofill + "coverURL": "", // Autofill + "lastChapterURL": false, // Autofill; Can be used if not starting from the first chapter, first chapter downloaded will be NEXT from this + "lastVolume": 0, // Autofill; Can be used if not starting from the first chapter, first eBook number created will be NEXT from this + "completed": false, // Autofill; Set to false with settings above to download chapters again + "hosting": "NF", // Hosting code, see "supportedHosting" + "volumeChapterCount": 5, // After how many new/unread chapters to send a new eBook, ignored if WebNovel is completed + "completedVolumeChapterCount": 50, // How many chapters to pack per eBook + "redownload": false // TODO: redownload all chapters, repack into volumes with completedVolumeChapterCount, do not send via email, intended for completed series archiving + }, + "novels": [] // Table of +} +``` \ No newline at end of file diff --git a/WNtoEmail.js b/WNtoEmail.js new file mode 100644 index 0000000..974b1e3 --- /dev/null +++ b/WNtoEmail.js @@ -0,0 +1,406 @@ +const fs = require('fs'); +const path = require('path'); +const HTMLparser = require('node-html-parser'); +const exec = require("child_process").execSync; +const nodemailer = require('nodemailer'); + +var transporter; +var config; + +function clone(obj) { + return JSON.parse(JSON.stringify(obj)); +} + +function cleanPath(pathToClean) { + let regex = /[<>:"|\?\*]/g; + let cleanPath = path.normalize(pathToClean); + + let isAbsolute = path.isAbsolute(cleanPath); + cleanPath = cleanPath.replace(regex, ''); + if (/^win/i.test(process.platform) && isAbsolute) { + cleanPath = `${cleanPath.slice(0, 1)}:${cleanPath.slice(1)}` + } + + return cleanPath; +} + +function writeFile(dir, file, data) { + let cleanDir = cleanPath(dir); + let cleanFile = cleanPath(file); + + if (!fs.existsSync(cleanDir)) { + fs.mkdirSync(cleanDir, { recursive: true }); + } + fs.writeFileSync(`${cleanDir}/${cleanFile}`, data, function (err) { + if (err != null) console.log(err); + return err != null; + }); +} + +function readFile(file) { + let fileContent; + try { + fileContent = fs.readFileSync(cleanPath(file), 'utf8'); + } + catch (err) { + fileContent = false; + } + + return fileContent; +} + +function loadConfig() { + let novelConfigDefault = { + "downloadLocation": "", + "converterPath": "ebook-convert.exe", + "ebookFormat": "epub", + "sendEmail": false, + "emailToAddress": "", + "emailFromAddress": "", + "emailProvider": "", + "emailUsername": "", + "emailPassword": "", + "emailAttachments": 25, + "supportedHosting": { + "NF": "https://novelfull.com/" + }, + "template": { + "novelURL": "", + "title": "", + "author": "", + "coverURL": "", + "lastChapterURL": false, + "lastVolume": 0, + "completed": false, + "hosting": "NF", + "volumeChapterCount": 5, + "completedVolumeChapterCount": 50, + "redownload": false // TODO: redownload all chapters, repack into volumes with completedVolumeChapterCount, do not send via email, intended for completed series archiving + }, + "novels": [] + }; + + let configRead = readFile('./novelConfig.conf'); + + if (configRead) { + config = JSON.parse(configRead); + transporter = nodemailer.createTransport({ + service: config['emailProvider'], // no need to set host or port etc. + auth: { + user: config['emailUsername'], + pass: config['emailPassword'] + } + }); + writeFile('.', 'novelConfig.conf', JSON.stringify(config, null, 4)); + } + else { + writeFile('.', 'novelConfig.conf', JSON.stringify(novelConfigDefault, null, 4)); + config = novelConfigDefault; + } + + console.log(config) +} + +function saveConfig() { + writeFile('.', 'novelConfig.conf', JSON.stringify(config, null, 4)); +} + +async function convertEbook(dir, file, params = { "cover": false, "authors": false, "title": false }, format = 'html') { + let file1Path = cleanPath(`${dir}/${file}.${format}`); + let file2Path = cleanPath(`${dir}/${file}.${config['ebookFormat']}`); + let convertParams = ' --use-auto-toc'; + //convertParams += format2 == 'epub' ? ' --epub-inline-toc' : ''; + convertParams += params['cover'] ? ` --cover "${params['cover']}"` : ''; + convertParams += params['authors'] ? ` --authors "${params['authors']}"` : ''; + convertParams += params['title'] ? ` --title "${params['title']}"` : ''; + + exec(`${config['converterPath']} "${file1Path}" "${file2Path}"${convertParams}`, (error, stdout, stderr) => { + if (error) { + console.log(`error: ${error.message}`); + return; + } + if (stderr) { + console.log(`stderr: ${stderr}`); + return; + } + console.log(`stdout: ${stdout}`); + }); +} + + +async function sendEbook(subject, ebookAttachments) { //(dir, ebook, format = 'epub') { + if (config['sendEmail']) { + let splicedAttachments = []; + while (ebookAttachments.length > config['emailAttachments']) { + splicedAttachments.push(ebookAttachments.splice(0, config['emailAttachments'])) + } + if (ebookAttachments.length > 0) { + splicedAttachments.push(ebookAttachments) + } + + for (i = 0; i < splicedAttachments.length; ++i) { + let message = { + from: config['emailFromAddress'], + to: config['emailToAddress'], + subject: subject + ' part ' + i, + text: subject + ' part ' + i, + attachments: splicedAttachments[i] + } + + await transporter.sendMail(message, (err) => { + if (err) + console.log(err); + + else + console.log(`Sent volume ${ebook}`); + }); + } + } +} + +async function fetch_smth(URL, hosting) { + let fetchURL = await fetch(URL); + + if (fetchURL.ok) { + let response = await fetchURL.text(); + + let info; + + return info; + } + else return fetchURL.ok; +} + +async function fetchNovelInfo(URL, hosting) { + let fetchURL = await fetch(URL); + + if (fetchURL.ok) { + let response = await fetchURL.text(); + + let novelInfo = getNovelInfo(response, hosting); + + return novelInfo; + } + else return fetchURL.ok; + +} + +async function fetchChapter(URL, hosting) { + let fetchURL = await fetch(URL); + + if (fetchURL.ok) { + let response = await fetchURL.text(); + + let nextChapterURL = getNextChapterURL(response, hosting); + let chapterContent = getChapterContent(response, hosting); + + return [chapterContent, nextChapterURL]; + } + else return fetchURL.ok; +} + +function get(response, hosting) { + let html = HTMLparser.parse(response); + + let info; + + switch (hosting) { + case 'NF': + info = html; + break; + + default: + info = false; + } + + return info; +} + +async function getNovelInfo(response, hosting) { + let html = HTMLparser.parse(response); + + let info; + + switch (hosting) { + case 'NF': + let title = html.querySelector('h3.title').innerText; + let author = html.querySelector('#truyen > div.csstransforms3d > div > div.col-xs-12.col-info-desc > div.col-xs-12.col-sm-4.col-md-4.info-holder > div.info > div:nth-child(1) > a:nth-child(2)').innerText; + let completed = html.querySelector('#truyen > div.csstransforms3d > div > div.col-xs-12.col-info-desc > div.col-xs-12.col-sm-4.col-md-4.info-holder > div.info > div:nth-child(5) > a').innerText == "Completed"; + let firstChapterURL = html.querySelector('#list-chapter > div.row > div:nth-child(1) > ul > li:nth-child(1) > a').attrs['href']; + let coverURL = html.querySelector('div.book > img').attrs['src']; + info = [title, author, completed, 'https://novelfull.com' + firstChapterURL, 'https://novelfull.com' + coverURL]; + break; + + default: + info = false; + } + + return info; +} + +function getNextChapterURL(response, hosting) { + let html = HTMLparser.parse(response); + + let nextChapterURL; + + switch (hosting) { + case 'NF': + nextChapterURL = undefined == html.querySelector('a#next_chap').attrs['href'] ? false : 'https://novelfull.com' + html.querySelector('a#next_chap').attrs['href']; + break; + + default: + nextChapterURL = false; + } + + return nextChapterURL; +} + +function getChapterContent(response, hosting) { + let html = HTMLparser.parse(response); + + let chapterContent = ''; + + switch (hosting) { + case 'NF': + chapterContent += '