From 06183b494e653e5c2c284ab00cd47141bae0c6cf Mon Sep 17 00:00:00 2001
From: mrq <barry.quiggles@protonmail.com>
Date: Fri, 14 Oct 2022 18:04:33 +0000
Subject: [PATCH] add extending settings to json, provided examples, so no more
 having to discard changes when pulling

---
 data/config/examples/fetch.json      | 12 ++++++++++++
 data/config/examples/preprocess.json | 21 +++++++++++++++++++++
 src/fetch.js                         | 13 +++++++++++++
 src/fetch.py                         | 12 ++++++++++++
 src/preprocess.js                    | 13 +++++++++++++
 src/preprocess.py                    | 12 ++++++++++++
 6 files changed, 83 insertions(+)
 create mode 100755 data/config/examples/fetch.json
 create mode 100755 data/config/examples/preprocess.json

diff --git a/data/config/examples/fetch.json b/data/config/examples/fetch.json
new file mode 100755
index 0000000..4603e24
--- /dev/null
+++ b/data/config/examples/fetch.json
@@ -0,0 +1,12 @@
+{
+	"booru": "e621",
+	"output": "./images/downloaded/",
+	"limit": 320,
+	"concurrency": 4,
+	"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+	"filter": true,
+	"filters": [
+		"animated"
+	],
+	"skipSave": false
+}
\ No newline at end of file
diff --git a/data/config/examples/preprocess.json b/data/config/examples/preprocess.json
new file mode 100755
index 0000000..8ebe809
--- /dev/null
+++ b/data/config/examples/preprocess.json
@@ -0,0 +1,21 @@
+{
+	"input": "./images/downloaded/",
+	"output": "./images/tagged/",
+	"tags": "./data/tags.csv",
+	"cache": "./data/cache.json",
+	"rateLimit": 500, 
+	"filenameLimit": 243,
+	"filter": true,
+	"filters": [
+		"animal genitalia",
+		"genitals",
+		"video games"
+	],
+	"tagsOverride": [],
+	"tagsOverrideStart": 1000000,
+	"tagsAutoInclude": [],
+	"removeParentheses": true,
+	"onlyIncludeModelArtists": true,
+	"reverseTags": false,
+	"tagDelimiter": ","
+}
\ No newline at end of file
diff --git a/src/fetch.js b/src/fetch.js
index 5134714..13acd57 100755
--- a/src/fetch.js
+++ b/src/fetch.js
@@ -37,6 +37,8 @@ let boorus = {
 }
 
 let config = {
+	source: "./data/config/fetch.json",
+
 	booru: "e621", // booru definition to use from the above object, currently only supports e621
 
 	query: ``, // example query if no argument is passed, kept empty so the script can scream at you for not having it tagged
@@ -57,6 +59,17 @@ let config = {
 	skipSave: false, // useful if you want to just cache your tags before running pre-process on files you already downloaded
 }
 
+// import source
+if ( FS.existsSync(config.source) ) try {
+	let imp = JSON.parse( FS.readFileSync(config.source) )
+	for ( let k in imp ) {
+		config[k] = imp[k]
+	}
+	console.log(`Imported settings from "${config.source}"`)
+} catch ( e ) {
+	console.error(e)
+}
+
 let booru = boorus[config.booru];
 // merge booru and config
 for ( let k in booru.config ) if ( !config[k] ) config[k] = booru.config[k];
diff --git a/src/fetch.py b/src/fetch.py
index 4b5e060..a49940c 100755
--- a/src/fetch.py
+++ b/src/fetch.py
@@ -42,6 +42,8 @@ boorus = {
 }
 
 config = {
+	'config': './data/config/fetch.json',
+
 	'booru': "e621", # booru definition to use from the above object, currently only supports e621
 
 	'query': '', # example query if no argument is passed, kept empty so the script can scream at you for not having it tagged
@@ -62,6 +64,16 @@ config = {
 	'skipSave': False, # useful if you want to just cache your tags before running pre-process on files you already downloaded
 }
 
+if os.path.exists(config['source']):
+	try:
+		with open(config['source'], 'rb') as f:
+			imp = json.loads(f.read().decode('utf-8'))
+			for k in imp:
+				config[k] = imp[k]
+			print(f"Imported settings from {config['source']}")
+	except:
+		pass
+
 booru = boorus[config['booru']]
 
 for k in booru["config"]:
diff --git a/src/preprocess.js b/src/preprocess.js
index fb598fb..8945cf6 100755
--- a/src/preprocess.js
+++ b/src/preprocess.js
@@ -2,6 +2,8 @@ let FS = require("fs")
 let Fetch = require("node-fetch")
 
 let config = {
+	source: `./data/config/preprocess.json`,
+
 	input: `./images/downloaded/`, // files to process
 	output: `./images/tagged/`, // files to copy files to 
 	tags: `./data/tags.csv`, // csv of tags associated with the yiffy model (replace for other flavor of booru's taglist associated with the model you're training against)
@@ -55,6 +57,17 @@ let config = {
 	tagDelimiter: ",", // what separates each tag in the filename, web UI will accept comma separated filenames
 }
 
+// import source
+if ( FS.existsSync(config.source) ) try {
+	let imp = JSON.parse( FS.readFileSync(config.source) )
+	for ( let k in imp ) {
+		config[k] = imp[k]
+	}
+	console.log(`Imported settings from "${config.source}"`)
+} catch ( e ) {
+	console.error(e)
+}
+
 let csv = FS.readFileSync(config.tags)
 csv = csv.toString().split("\n")
 config.tags = {}
diff --git a/src/preprocess.py b/src/preprocess.py
index 7c3ab7d..b06657f 100755
--- a/src/preprocess.py
+++ b/src/preprocess.py
@@ -9,6 +9,8 @@ import math
 import urllib.request
 
 config = {
+	'source': "./data/config/preprocess.json",
+
 	'input': './images/downloaded/', # files to process
 	'output': './images/tagged/', # files to copy files to
 	'tags': './data/tags.csv', # csv of tags associated with the yiffy model (replace for other flavor of booru's taglist associated with the model you're training against)
@@ -60,6 +62,16 @@ config = {
 	'tagDelimiter': ",", # what separates each tag in the filename, web UI will accept comma separated filenames
 }
 
+if os.path.exists(config['source']):
+	try:
+		with open(config['source'], 'rb') as f:
+			imp = json.loads(f.read().decode('utf-8'))
+			for k in imp:
+				config[k] = imp[k]
+			print(f"Imported settings from {config['source']}")
+	except:
+		pass
+
 with open(config['tags'], 'rb') as f:
 	csv = f.read().decode('utf-8').split("\n")
 config['tags'] = {}