let FS = require("fs")
let Fetch = require("node-fetch")

let config = {
	query: `leib_(tas)`, // example query if no argument is passed

	output: `./in/`, // directory to save your files
	cache: `./cache.json`, // JSON file of cached tags, will speed up processing when used for the renamer script

	limit: 10, // how many posts to pull in one go
	rateLimit: 500, // time to wait between requests, in milliseconds, e621 imposes a rate limit of 2 requests per second
}

let cache;
try {
	cache = JSON.parse( FS.readFileSync(config.cache) )
} catch ( e ) {
	cache = {};
}

let args = process.argv;
args.shift();
args.shift();

if ( args.length ) config.query = args.join(" ");

let parse = async () => {
	console.log(`Fetching: ${config.query}`)

	let posts = [];
	let last = '';

	do {
		let query = [`tags=${config.query}`]
		if ( config.limit ) query.push(`limit=${config.limit}`)
		if ( last ) query.push(`page=b${last}`)

		query = encodeURI(query.join("&"));
		console.log(`Querying: ${query}`)
		let r = await Fetch( `https://e621.net/posts.json?${query}`, {
			headers: {
				'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
			}
		} );
		let j = JSON.parse(await r.text());	
		posts = j.posts;
	
		for ( let i in posts ) {
			let post = posts[i];
			last = `${post.id}`
			cache[post.file.md5] = post;

			if ( FS.existsSync(`${config.output}${post.file.md5}.${post.file.ext}`) ) continue;

			await Fetch(post.file.url).then(res => new Promise((resolve, reject) => {
				const dest = FS.createWriteStream(`${config.output}${post.file.md5}.${post.file.ext}`);
				res.body.pipe(dest);
				dest.on('close', () => {
					console.log(`Downloaded https://e621.net/posts/${post.id}`)
					resolve()
				});
				dest.on('error', reject);
			}));
		}		

		if ( config.rateLimit ) await new Promise( (resolve) => {
			setTimeout(resolve, config.rateLimit)
		} )
	} while ( posts.length );

	FS.writeFileSync(config.cache, JSON.stringify( cache, null, "\t" ))
}

parse();