74 lines
2.1 KiB
JavaScript
Executable File
74 lines
2.1 KiB
JavaScript
Executable File
let FS = require("fs")
|
|
let Fetch = require("node-fetch")
|
|
|
|
let config = {
|
|
query: `leib_(tas)`, // example query if no argument is passed
|
|
|
|
output: `./in/`, // directory to save your files
|
|
cache: `./cache.json`, // JSON file of cached tags, will speed up processing when used for the renamer script
|
|
|
|
limit: 10, // how many posts to pull in one go
|
|
rateLimit: 500, // time to wait between requests, in milliseconds, e621 imposes a rate limit of 2 requests per second
|
|
}
|
|
|
|
let cache;
|
|
try {
|
|
cache = JSON.parse( FS.readFileSync(config.cache) )
|
|
} catch ( e ) {
|
|
cache = {};
|
|
}
|
|
|
|
let args = process.argv;
|
|
args.shift();
|
|
args.shift();
|
|
|
|
if ( args.length ) config.query = args.join(" ");
|
|
|
|
let parse = async () => {
|
|
console.log(`Fetching: ${config.query}`)
|
|
|
|
let posts = [];
|
|
let last = '';
|
|
|
|
do {
|
|
let query = [`tags=${config.query}`]
|
|
if ( config.limit ) query.push(`limit=${config.limit}`)
|
|
if ( last ) query.push(`page=b${last}`)
|
|
|
|
query = encodeURI(query.join("&"));
|
|
console.log(`Querying: ${query}`)
|
|
let r = await Fetch( `https://e621.net/posts.json?${query}`, {
|
|
headers: {
|
|
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
|
|
}
|
|
} );
|
|
let j = JSON.parse(await r.text());
|
|
posts = j.posts;
|
|
|
|
for ( let i in posts ) {
|
|
let post = posts[i];
|
|
last = `${post.id}`
|
|
cache[post.file.md5] = post;
|
|
|
|
if ( FS.existsSync(`${config.output}${post.file.md5}.${post.file.ext}`) ) continue;
|
|
|
|
await Fetch(post.file.url).then(res => new Promise((resolve, reject) => {
|
|
const dest = FS.createWriteStream(`${config.output}${post.file.md5}.${post.file.ext}`);
|
|
res.body.pipe(dest);
|
|
dest.on('close', () => {
|
|
console.log(`Downloaded https://e621.net/posts/${post.id}`)
|
|
resolve()
|
|
});
|
|
dest.on('error', reject);
|
|
}));
|
|
}
|
|
|
|
if ( config.rateLimit ) await new Promise( (resolve) => {
|
|
setTimeout(resolve, config.rateLimit)
|
|
} )
|
|
} while ( posts.length );
|
|
|
|
FS.writeFileSync(config.cache, JSON.stringify( cache, null, "\t" ))
|
|
}
|
|
|
|
parse(); |