Compare commits

..

11 Commits

Author SHA1 Message Date
zjamnik fd0f23b776 all pending changes from working copy 2026-03-26 09:34:58 +01:00
zjamnik 42a2625c0e fixes 2024-01-09 17:47:38 +01:00
zjamnik f87abff0ff conf files renamed to json
current dir reference changed to `__dirname`
2022-07-11 13:48:42 +02:00
zjamnik 987ca3c0b6 dropped binary release
deploy script creates a zip release now
2022-07-08 19:45:34 +02:00
zjamnik 888cf42b8d ssendOnly convert function added 2022-07-08 16:52:07 +02:00
zjamnik b73f80dd68 sendOnly implemented 2022-07-08 15:08:00 +02:00
zjamnik b91fff6166 readme 2022-07-08 13:46:51 +02:00
zjamnik 468ec46bd8 readme 2022-07-08 13:46:18 +02:00
zjamnik 6bcb547098 readme config line breaks 2022-07-08 13:44:41 +02:00
zjamnik 809b021062 redownload implemented 2022-07-08 13:33:19 +02:00
zjamnik f93008fe29 disabled send email when redownloading; removed TODO 2022-07-07 09:07:33 +02:00
4 changed files with 298 additions and 170 deletions
+3 -1
View File
@@ -1,7 +1,9 @@
# Project specific
Download/**/**
*.conf
novelConfig.json
novelConfig*.json
Release
WNtoEmail
# Logs
logs
+33 -19
View File
@@ -19,35 +19,53 @@ I'm using fetch, which is an experimental feature, it might work differently on
At first start it will create an empty config file `./novelConfig.conf`, adjust the setting according to comments below:
```
{
"downloadLocation": "", // New chapter download location, "./Download" as an absolute path recommended
"converterPath": "ebook-convert.exe", // Calibre eBook converter, I recommend adding it to you PATH, NOT tested when it's not in PATH
"ebookFormat": "epub", // Desired eBook format, Kindle started supporting epub so that's default
"downloadLocation": "", // New chapter download location, "./Download" as an absolute
// path recommended
"converterPath": "ebook-convert.exe", // Calibre eBook converter, I recommend adding it to you PATH,
// NOT tested when it's not in PATH
"ebookFormat": "epub", // Desired eBook format, Kindle started supporting epub so
// that's default
"sendEmail": false, // If the script should send eBooks via email
"emailToAddress": "", // Email where to send your eBooks
"emailFromAddress": "", // Important for Kindle deliveries, make sure you have it added in Kindle settings
"emailFromAddress": "", // Important for Kindle deliveries, make sure you have it added
// in Kindle settings
"emailProvider": "", // Gmail works fine, just need to set up 2FA and an app password
"emailUsername": "", // Username to your email account
"emailPassword": "", // Password to your email account
"emailAttachments": 25, // How many eBooks to attach to a single email
"supportedHosting": { // Enum to show which WebNovel host sites are supported, NOT configurable
"supportedHosting": { // Supported WN host sites, NOT configurable
"NF": "https://novelfull.com/"
},
"template": { // Template for a WebNovel entry in "novels" below
"novelURL": "", // WebNovel address, it's enough to copy this template to "novels" and fill only this field to start
"novelURL": "", // WebNovel address, it's enough to copy this template to
// "novels" and fill only this field to start
"title": "", // Autofill
"author": "", // Autofill
"coverURL": "", // Autofill
"lastChapterURL": false, // Autofill; Can be used if not starting from the first chapter, first chapter downloaded will be NEXT from this
"lastVolume": 0, // Autofill; Can be used if not starting from the first chapter, first eBook number created will be NEXT from this
"completed": false, // Autofill; Set to false with settings above to download chapters again
"lastChapterURL": false, // Autofill; Can be used if not starting from the first chapter,
// first chapter downloaded will be NEXT from this
"lastVolume": 0, // Autofill; Can be used if not starting from the first chapter,
// first eBook number created will be NEXT from this
"completed": false, // Autofill; Set to false with settings above to download
// chapters again; Set to true to skip checking the novel
"hosting": "NF", // Hosting code, see "supportedHosting"
"volumeChapterCount": 5, // After how many new/unread chapters to send a new eBook, ignored if WebNovel is completed
"volumeChapterCount": 5, // After how many new/unread chapters to send a new eBook,
// ignored if WebNovel is completed
"completedVolumeChapterCount": 50, // How many chapters to pack per eBook
"redownload": false // TODO: redownload all chapters, repack into volumes with completedVolumeChapterCount, do not send via email, intended for completed series archiving
"sendOnly": false, // TODO: only send epub files via email, for cases with external source of epub files
"sendOnlyRegex": ""(?<volume>\\d*). (?<title>.*); (?<author>.*)"" // TODO: metadata regex for extracting information from filename for external sources
"redownload": false // Redownload all chapters, repack into volumes, do not send via
// email, intended for completed series archiving
"sendOnly": false, // Only send epub files via email, for cases with external
// source of epub files
"sendOnlyFormat": "epub" // Format filter for sendOnly files
"sendOnlyConvert": true, // Convert sendOnly files, epub => epub is supported,
// useful for compressing images in big files
"sendOnlyRegex": "(?<volume>\\d*). (?<title>.*); (?<author>.*)" // Metadata regex for extracting
// information from filename for external sources
},
"novels": [] // Table of novels to process
"novels": [
// Table of novels to process, insert the template structure
// from above here
]
}
```
For some reason Amazon just forgets the cover and TOS on conversion from epub. Both features worked correctly with mobi, but that format is being phased out. From what I found, Amazon is being an ass about it and is ignoring built in metadata in favor of getting them from their book database. So sending books not bought from them is made intentionally inferior.
@@ -76,11 +94,7 @@ For ongoing series it would create 6 volumes:
# Usage
Grab the latest release and run the binary.
OR
Run the script directly from cloned project with Node.js.
Run the script with node.
Intended usage is with a Task Scheduler on Windows. There shouldn't be anything OS specific. Cron on Linux should work after modifying `"converterPath"` to an appropriate command, but that's untested.
At present there is no crash resiliency, if the program crashes for any reason `"lastChapterURL"`, `"lastVolume"` and `"completed"` config will not be consistent and needs to be corrected. There is a copy of config file created at the start.
+176 -66
View File
@@ -18,7 +18,7 @@ function cleanPath(pathToClean) {
let isAbsolute = path.isAbsolute(cleanPath);
cleanPath = cleanPath.replace(regex, '');
if (/^win/i.test(process.platform) && isAbsolute) {
cleanPath = `${cleanPath.slice(0, 1)}:${cleanPath.slice(1)}`
cleanPath = `${cleanPath.slice(0, 1)}:${cleanPath.slice(1)}`;
}
return cleanPath;
@@ -29,7 +29,7 @@ function padNumber(num, len) {
let negative = num < 0;
if (negative) strNum = strNum.substr(1);
while(strNum.length < len) {
while (strNum.length < len) {
strNum = '0' + strNum;
}
@@ -39,7 +39,9 @@ function padNumber(num, len) {
function log(text) {
let d = new Date();
let dateTime = `${d.getFullYear()}.${padNumber((d.getMonth() + 1), 2)}.${padNumber(d.getDate(), 2)}_${padNumber(d.getHours(), 2)}:${padNumber(d.getMinutes(), 2)}:${padNumber(d.getSeconds(), 2)}.${padNumber(d.getMilliseconds(), 3)}`;
fs.appendFile(cleanPath(`./WNtoEmail.log`), `${dateTime} ${text}\n`);
fs.appendFile(cleanPath(`${__dirname}/WNtoEmail.log`), `${dateTime} ${text}\n`);
fs.appendFile(cleanPath(`${__dirname}/WNtoEmailArch.log`), `${dateTime} ${text}\n`);
console.log(text);
}
async function mkDir(dirPath) {
@@ -47,7 +49,7 @@ async function mkDir(dirPath) {
await fs.access(cleanPath(dirPath));
}
catch (err) {
await fs.mkdir(dirPath, { recursive: true });
await fs.mkdir(cleanPath(dirPath), { recursive: true });
}
await fs.access(cleanPath(dirPath));
}
@@ -68,9 +70,10 @@ async function readFile(file, { format = 'utf8' } = {}) {
async function loadConfig() {
let novelConfigDefault = {
"downloadLocation": "",
"converterPath": "ebook-convert.exe",
"converterPath": "D:/Calibre/Calibre/ebook-convert.exe",
"ebookFormat": "epub",
"sendEmail": false,
"copyPath": "",
"emailToAddress": "",
"emailFromAddress": "",
"emailProvider": "",
@@ -79,43 +82,61 @@ async function loadConfig() {
"emailAttachments": 25,
"supportedHosting": {
"NF": "https://novelfull.com/",
"TNC": "https://thatnovelcorner.com/ external source, use with sendOnly = true"
"TNC": "https://thatnovelcorner.com/ external source, use with sendOnly = true",
"BBB": "https://bluebellsinbloom.wordpress.com/",
"JN": "https://jnovels.com/ external"
},
"template": {
"novelURL": "",
"title": "",
"author": "",
"coverURL": "",
"lastChapterURL": false,
"lastChapterURL": "",
"lastVolume": 0,
"volumePadding": 2,
"completed": false,
"hosting": "NF",
"volumeChapterCount": 5,
"completedVolumeChapterCount": 50,
"redownload": false, // TODO: redownload all chapters, repack into volumes with completedVolumeChapterCount, do not send via email, intended for completed series archiving or resetting "lastVolume" to a more reasonable number
"sendOnly": false, // TODO: only send epub files via email, for cases with external source of epub files or after "redownload"
"sendOnlyRegex": "(?<volume>\\d*). (?<title>.*); (?<author>.*)" // TODO: metadata regex for extracting information from filename for external sources
"completedVolumeChapterCount": 200,
"redownload": false,
"sendOnly": false,
"sendOnlyFormat": "epub",
"sendOnlyConvert": true,
"sendOnlyRegex": "(?<volume>\\d*). (?<title>.*); (?<author>.*)"
},
"novels": []
};
try {
config = JSON.parse(await readFile(`./novelConfig.conf`));
transporter = nodemailer.createTransport({
service: config['emailProvider'],
auth: {
user: config['emailUsername'],
pass: config['emailPassword']
config = JSON.parse(await readFile(`${__dirname}/novelConfig.json`));
for (key in novelConfigDefault) {
if (config[key] == undefined) {
config[key] = clone(novelConfigDefault[key]);
}
}
for (key in novelConfigDefault['template']) {
if (config['template'][key] == undefined) {
config['template'][key] = clone(novelConfigDefault['template'][key]);
}
}
for (let i = 0; i < config['novels'].length; ++i) {
for (key in novelConfigDefault['template']) {
if (config['novels'][i][key] == undefined) {
config['novels'][i][key] = clone(novelConfigDefault['template'][key]);
}
}
}
});
await writeFile('.', 'novelConfig.conf', JSON.stringify(config, null, 4));
await writeFile('.', 'novelConfig.bak.conf', JSON.stringify(config, null, 4));
}
catch (err) {
await writeFile('.', 'novelConfig.conf', JSON.stringify(novelConfigDefault, null, 4));
config = novelConfigDefault;
config = clone(novelConfigDefault);
}
await saveConfig();
await saveConfig({ configName: 'novelConfig.bak.json' });
transporter = nodemailer.createTransport({
service: config['emailProvider'],
auth: {
@@ -125,47 +146,44 @@ async function loadConfig() {
});
}
async function saveConfig() {
await writeFile(__dirname, 'novelConfig.conf', JSON.stringify(config, null, 4));
async function saveConfig({ configPath = __dirname, configName = 'novelConfig.json' } = {}) {
await writeFile(configPath, configName, JSON.stringify(config, null, 4));
}
async function convertEbook(dir, file, params = { "cover": false, "authors": false, "title": false }, format = 'html') {
async function convertEbook(dir, file, { cover = false, authors = false, title = false, format = 'html', file2 = false } = {}) {
let file1Path = cleanPath(`${dir}/${file}.${format}`);
let file2Path = cleanPath(`${dir}/${file}.${config['ebookFormat']}`);
let file2Path = cleanPath(`${dir}/${file2 ? file2 : file}.${config['ebookFormat']}`);
let convertParams = ' --use-auto-toc';
convertParams += config['ebookFormat'] == 'epub' ? ' --epub-inline-toc' : '';
convertParams += params['cover'] ? ` --cover "${params['cover']}"` : '';
convertParams += params['authors'] ? ` --authors "${params['authors']}"` : '';
convertParams += params['title'] ? ` --title "${params['title']}"` : '';
convertParams += cover ? ` --cover "${cover}"` : '';
convertParams += authors ? ` --authors "${authors}"` : '';
convertParams += title ? ` --title "${title}"` : '';
console.log(`Converting volume: ${file1Path}`);
log(`Converting volume: ${file1Path}`);
exec(`${config['converterPath']} "${file1Path}" "${file2Path}"${convertParams}`, (error, stdout, stderr) => {
let convertOutput = exec(`${config['converterPath']} "${file1Path}" "${file2Path}"${convertParams}`, function (error, stdout, stderr) {
if (error) {
console.log(`error: ${error.message}`);
log(`error: ${error.message}`);
return;
}
if (stderr) {
console.log(`stderr: ${stderr}`);
log(`stderr: ${stderr}`);
return;
}
console.log(`stdout: ${stdout}`);
log(`stdout: ${stdout}`);
});
log(convertOutput.toString());
}
function sendEbook(subject, ebookAttachments) {
async function sendEbook(subject, ebookAttachments) {
if (config['sendEmail']) {
let splicedAttachments = [];
while (ebookAttachments.length > config['emailAttachments']) {
splicedAttachments.push(ebookAttachments.splice(0, config['emailAttachments']))
splicedAttachments.push(ebookAttachments.splice(0, config['emailAttachments']));
}
if (ebookAttachments.length > 0) {
splicedAttachments.push(ebookAttachments)
splicedAttachments.push(ebookAttachments);
}
for (let i = 0; i < splicedAttachments.length; ++i) {
@@ -175,17 +193,22 @@ function sendEbook(subject, ebookAttachments) {
subject: subject + ' part ' + (i + 1),
text: subject + ' part ' + (i + 1),
attachments: splicedAttachments[i]
}
};
transporter.sendMail(message, (err) => {
if (err)
console.log(err);
log(err);
log(`Send mail error: ${err}`);
});
console.log(`Sent volumes:`);
log(`Sent volumes:`);
splicedAttachments[i].forEach(elem => console.log(elem['filename']))
let sentVolumes = '';
splicedAttachments[i].forEach(elem => { sentVolumes += '\n' + elem['filename']; });
log(`Sent volumes:${sentVolumes}`);
}
}
if (config['copyPath'] != "") {
for (const ebook of ebookAttachments) {
await mkDir(`${config.copyPath}\\${ebook.title}`);
fs.copyFile(ebook.path, cleanPath(`${config.copyPath}\\${ebook.title}\\${ebook.filename}`),);
}
}
}
@@ -263,6 +286,23 @@ async function getNovelInfo(response, hosting) {
info = [title, author, completed, 'https://novelfull.com' + firstChapterURL, 'https://novelfull.com' + coverURL];
break;
case 'JN':
html.querySelectorAll('.post-content ol li').forEach(elem => {
if (elem.innerText.match('V|volume')) {
info = elem.innerText.match(/\d+/)[0];
}
});
break;
case 'TNC':
html.querySelectorAll('a').forEach(elem => {
if (elem.innerText.match('V|volume')) {
info = elem.innerText.match(/\d+/)[0];
}
});
break;
default:
info = false;
}
@@ -294,7 +334,7 @@ function getChapterContent(response, hosting) {
switch (hosting) {
case 'NF':
chapterContent += '<h1 class="chapter">' + html.querySelector('span.chapter-text').innerText + '</h3>';
chapterContent += '<h1 class="chapter">' + html.querySelector('span.chapter-text').innerText + '</h1>';
html.querySelectorAll('div#chapter-content p').forEach(element => {
chapterContent += element.outerHTML;
@@ -308,13 +348,16 @@ function getChapterContent(response, hosting) {
return chapterContent;
}
async function clearLog() {
await writeFile(`${__dirname}`, `WNtoEmail.log`, '');
}
async function main() {
await loadConfig();
await clearLog();
for (let i = 0; i < config['novels'].length; ++i) {
let novel = clone(config['novels'][i]);
let chapters = [];
let nextChapterURL;
if (novel['redownload']) {
novel['completed'] = false;
@@ -323,7 +366,69 @@ async function main() {
}
if (!novel['completed']) {
let novelInfo = await fetchNovelInfo(novel['novelURL'], 'NF');
if (novel.sendOnly) {
await sendOnlyFunction(novel, i);
} else {
await downloadChaptersFunction(novel, i);
}
}
}
async function sendOnlyFunction(novel, i) {
let ebookAttachments = [];
const novelPath = `${config.downloadLocation}/${novel.title}`;
const novelVolumeRegex = new RegExp(novel.sendOnlyRegex + '.' + novel.sendOnlyFormat);
let lastVolumeOnline = parseInt(await fetchNovelInfo(novel['novelURL'], novel.hosting));
if (lastVolumeOnline > novel.lastVolume) {
log(`New volume found online: ${novel.title} ${lastVolumeOnline} ${novel.novelURL}`);
}
// let ebookList = await fs.opendir(cleanPath(novelPath));
// console.log(ebookList);
try {
const files = await fs.readdir(cleanPath(novelPath));
for (const file of files) {
let volumeMatch = file.match(novelVolumeRegex);
if (volumeMatch) {
const currentVolume = parseInt(volumeMatch.groups.volume);
if (currentVolume > novel.lastVolume) {
if (novel.sendOnlyConvert) {
convertEbook(novelPath, path.parse(file).name, { format: novel.sendOnlyFormat, title: `${padNumber(currentVolume, novel.volumePadding)}. ${novel.title}`, file2: `${padNumber(currentVolume, novel.volumePadding)}. ${novel.title}` });
ebookAttachments.push({
title: novel.title,
filename: cleanPath(`${padNumber(currentVolume, novel.volumePadding)}. ${novel.title}.${config.ebookFormat}`),
path: cleanPath(`${novelPath}/${padNumber(currentVolume, novel.volumePadding)}. ${novel.title}.${config.ebookFormat}`)
});
} else {
ebookAttachments.push({
title: novel.title,
filename: cleanPath(`${file}`),
path: cleanPath(`${novelPath}/${file}`)
});
}
novel.lastVolume = currentVolume;
}
}
}
} catch (err) {
log(err);
}
config['novels'][i] = clone(novel);
await saveConfig();
sendEbook(novel['title'], ebookAttachments);
}
async function downloadChaptersFunction(novel, i) {
let chapters = [];
let nextChapterURL;
let novelInfo = await fetchNovelInfo(novel['novelURL'], novel.hosting);
novel['title'] = novelInfo[0];
novel['author'] = novelInfo[1];
@@ -336,21 +441,19 @@ async function main() {
if (!novel['lastChapterURL']) {
novel['lastChapterURL'] = novelInfo[3];
let chapter = await fetchChapter(novelInfo[3], 'NF');
console.log('Download chapter ' + chapters.length + ' ' + novelInfo[3]);
log('Download chapter ' + chapters.length + ' ' + novelInfo[3]);
let chapter = await fetchChapter(novelInfo[3], novel.hosting);
log('Downloaded chapter: ' + chapters.length + ' ' + novelInfo[3]);
chapters.push(chapter);
}
let novelDir = `${config['downloadLocation']}/${novel['title']}`;
const nextChapterURLtemp = await fetchChapter(novel['lastChapterURL'], 'NF');
const nextChapterURLtemp = await fetchChapter(novel['lastChapterURL'], novel.hosting);
nextChapterURL = nextChapterURLtemp[1];
while (nextChapterURL) {
novel['lastChapterURL'] = nextChapterURL;
let chapter = await fetchChapter(nextChapterURL, 'NF');
console.log('Downloaded chapter: ' + chapters.length + ' ' + nextChapterURL);
let chapter = await fetchChapter(nextChapterURL, novel.hosting);
log('Downloaded chapter: ' + chapters.length + ' ' + nextChapterURL);
chapters.push(chapter);
nextChapterURL = chapter[1];
@@ -359,13 +462,13 @@ async function main() {
let startVol = novel['lastVolume'];
let totalChapters = chapters.length;
const maxVolume = novel['completed'] ? startVol + Math.ceil(totalChapters / novel['completedVolumeChapterCount']) : startVol + Math.floor(totalChapters / novel['completedVolumeChapterCount']);
const maxVolLen = (novel['completed'] ? maxVolume :
maxVolume + Math.floor((chapters.length - (maxVolume * novel['completedVolumeChapterCount'])) / novel['volumeChapterCount'])).toString().length;
const maxVolumeComplete = novel['completed'] ? startVol + Math.ceil(totalChapters / novel['completedVolumeChapterCount']) : startVol + Math.floor(totalChapters / novel['completedVolumeChapterCount']);
const maxVolumeUpdate = novel['completed'] ? 0 : 1;// ? maxVolumeComplete : maxVolumeComplete + Math.floor((totalChapters - ((maxVolumeComplete - startVol) * novel.completedVolumeChapterCount)) / novel['volumeChapterCount'])
const maxVolLen = novel['volumePadding'] ? novel['volumePadding'] : (maxVolumeUpdate.toString().length < 2 ? 2 : maxVolumeUpdate.toString().length);
let ebookAttachments = [];
for (let vol = startVol; vol < maxVolume; vol++) {
for (let vol = startVol; vol < maxVolumeComplete; vol++) {
let volContent = '';
let chap;
@@ -378,19 +481,19 @@ async function main() {
config['novels'][i] = clone(novel);
await saveConfig();
let novelFileName = `${padNumber((vol + 1), maxVolLen)}. ${novel['title']}; ${novel['author']}`;
let novelFileName = `${padNumber((vol + 1), novel.volumePadding)}. ${novel['title']}; ${novel['author']}`;
await writeFile(novelDir, `${novelFileName}.html`, volContent);
console.log(`Saved volume: ${novelFileName}`);
log(`Saved volume: ${novelFileName}`);
await convertEbook(novelDir, novelFileName, {
cover: novel['coverURL'],
authors: novel['author'],
title: `${padNumber((vol + 1), maxVolLen)}. ${novel['title']}`
title: `${padNumber((vol + 1), novel.volumePadding)}. ${novel['title']}`
});
ebookAttachments.push({
title: novel.title,
filename: cleanPath(`${novelFileName}.${config['ebookFormat']}`),
path: cleanPath(`${novelDir}/${novelFileName}.${config['ebookFormat']}`)
});
@@ -401,11 +504,12 @@ async function main() {
startVol = novel['lastVolume'];
totalChapters = chapters.length;
for (let vol = startVol; vol < startVol + Math.floor(totalChapters / novel['volumeChapterCount']); vol++) {
if (maxVolumeUpdate && totalChapters >= novel.volumeChapterCount) {
let vol = maxVolumeComplete;
let volContent = '';
let chap;
for (chap = 0; chap < novel['volumeChapterCount'] && chap < chapters.length; chap++) {
for (chap = 0; chap < Math.floor(totalChapters / novel.volumeChapterCount) * novel.volumeChapterCount; chap++) {
volContent += chapters[chap][0] + '\n';
}
@@ -414,27 +518,33 @@ async function main() {
config['novels'][i] = clone(novel);
await saveConfig();
let novelFileName = `${padNumber((vol + 1), maxVolLen)}. ${novel['title']}; ${novel['author']}`;
let novelFileName = `${padNumber((vol + 1), novel.volumePadding)}. ${novel['title']}; ${novel['author']}`;
await writeFile(novelDir, `${novelFileName}.html`, volContent);
console.log(`Saved volume: ${novelFileName}`);
log(`Saved volume: ${novelFileName}`);
await convertEbook(novelDir, novelFileName, {
cover: novel['coverURL'],
authors: novel['author'],
title: `${padNumber((vol + 1), maxVolLen)}. ${novel['title']}`
title: `${padNumber((vol + 1), novel.volumePadding)}. ${novel['title']}`
});
ebookAttachments.push({
title: novel.title,
filename: cleanPath(`${novelFileName}.${config['ebookFormat']}`),
path: cleanPath(`${novelDir}/${novelFileName}.${config['ebookFormat']}`)
});
chapters.splice(0, chap);
}
if (!novel['redownload']) {
sendEbook(novel['title'], ebookAttachments);
}
else {
novel['redownload'] = false;
config['novels'][i] = clone(novel);
saveConfig();
}
}
}
+3 -1
View File
@@ -1 +1,3 @@
pkg .\WNtoEmail.js --out-path ./Release
Copy-Item -Recurse -Force .\node_modules .\WNtoEmail
Copy-Item .\WNtoEmail.js .\WNtoEmail
C:\Program` Files\7-Zip\7z.exe a -tzip Release\WNtoEmail.zip .\WNtoEmail