add extending settings to json, provided examples, so no more having to discard changes when pulling
This commit is contained in:
parent
a4bbda9425
commit
06183b494e
12
data/config/examples/fetch.json
Executable file
12
data/config/examples/fetch.json
Executable file
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"booru": "e621",
|
||||||
|
"output": "./images/downloaded/",
|
||||||
|
"limit": 320,
|
||||||
|
"concurrency": 4,
|
||||||
|
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
|
||||||
|
"filter": true,
|
||||||
|
"filters": [
|
||||||
|
"animated"
|
||||||
|
],
|
||||||
|
"skipSave": false
|
||||||
|
}
|
21
data/config/examples/preprocess.json
Executable file
21
data/config/examples/preprocess.json
Executable file
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"input": "./images/downloaded/",
|
||||||
|
"output": "./images/tagged/",
|
||||||
|
"tags": "./data/tags.csv",
|
||||||
|
"cache": "./data/cache.json",
|
||||||
|
"rateLimit": 500,
|
||||||
|
"filenameLimit": 243,
|
||||||
|
"filter": true,
|
||||||
|
"filters": [
|
||||||
|
"animal genitalia",
|
||||||
|
"genitals",
|
||||||
|
"video games"
|
||||||
|
],
|
||||||
|
"tagsOverride": [],
|
||||||
|
"tagsOverrideStart": 1000000,
|
||||||
|
"tagsAutoInclude": [],
|
||||||
|
"removeParentheses": true,
|
||||||
|
"onlyIncludeModelArtists": true,
|
||||||
|
"reverseTags": false,
|
||||||
|
"tagDelimiter": ","
|
||||||
|
}
|
13
src/fetch.js
13
src/fetch.js
|
@ -37,6 +37,8 @@ let boorus = {
|
||||||
}
|
}
|
||||||
|
|
||||||
let config = {
|
let config = {
|
||||||
|
source: "./data/config/fetch.json",
|
||||||
|
|
||||||
booru: "e621", // booru definition to use from the above object, currently only supports e621
|
booru: "e621", // booru definition to use from the above object, currently only supports e621
|
||||||
|
|
||||||
query: ``, // example query if no argument is passed, kept empty so the script can scream at you for not having it tagged
|
query: ``, // example query if no argument is passed, kept empty so the script can scream at you for not having it tagged
|
||||||
|
@ -57,6 +59,17 @@ let config = {
|
||||||
skipSave: false, // useful if you want to just cache your tags before running pre-process on files you already downloaded
|
skipSave: false, // useful if you want to just cache your tags before running pre-process on files you already downloaded
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// import source
|
||||||
|
if ( FS.existsSync(config.source) ) try {
|
||||||
|
let imp = JSON.parse( FS.readFileSync(config.source) )
|
||||||
|
for ( let k in imp ) {
|
||||||
|
config[k] = imp[k]
|
||||||
|
}
|
||||||
|
console.log(`Imported settings from "${config.source}"`)
|
||||||
|
} catch ( e ) {
|
||||||
|
console.error(e)
|
||||||
|
}
|
||||||
|
|
||||||
let booru = boorus[config.booru];
|
let booru = boorus[config.booru];
|
||||||
// merge booru and config
|
// merge booru and config
|
||||||
for ( let k in booru.config ) if ( !config[k] ) config[k] = booru.config[k];
|
for ( let k in booru.config ) if ( !config[k] ) config[k] = booru.config[k];
|
||||||
|
|
12
src/fetch.py
12
src/fetch.py
|
@ -42,6 +42,8 @@ boorus = {
|
||||||
}
|
}
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
|
'config': './data/config/fetch.json',
|
||||||
|
|
||||||
'booru': "e621", # booru definition to use from the above object, currently only supports e621
|
'booru': "e621", # booru definition to use from the above object, currently only supports e621
|
||||||
|
|
||||||
'query': '', # example query if no argument is passed, kept empty so the script can scream at you for not having it tagged
|
'query': '', # example query if no argument is passed, kept empty so the script can scream at you for not having it tagged
|
||||||
|
@ -62,6 +64,16 @@ config = {
|
||||||
'skipSave': False, # useful if you want to just cache your tags before running pre-process on files you already downloaded
|
'skipSave': False, # useful if you want to just cache your tags before running pre-process on files you already downloaded
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if os.path.exists(config['source']):
|
||||||
|
try:
|
||||||
|
with open(config['source'], 'rb') as f:
|
||||||
|
imp = json.loads(f.read().decode('utf-8'))
|
||||||
|
for k in imp:
|
||||||
|
config[k] = imp[k]
|
||||||
|
print(f"Imported settings from {config['source']}")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
booru = boorus[config['booru']]
|
booru = boorus[config['booru']]
|
||||||
|
|
||||||
for k in booru["config"]:
|
for k in booru["config"]:
|
||||||
|
|
|
@ -2,6 +2,8 @@ let FS = require("fs")
|
||||||
let Fetch = require("node-fetch")
|
let Fetch = require("node-fetch")
|
||||||
|
|
||||||
let config = {
|
let config = {
|
||||||
|
source: `./data/config/preprocess.json`,
|
||||||
|
|
||||||
input: `./images/downloaded/`, // files to process
|
input: `./images/downloaded/`, // files to process
|
||||||
output: `./images/tagged/`, // files to copy files to
|
output: `./images/tagged/`, // files to copy files to
|
||||||
tags: `./data/tags.csv`, // csv of tags associated with the yiffy model (replace for other flavor of booru's taglist associated with the model you're training against)
|
tags: `./data/tags.csv`, // csv of tags associated with the yiffy model (replace for other flavor of booru's taglist associated with the model you're training against)
|
||||||
|
@ -55,6 +57,17 @@ let config = {
|
||||||
tagDelimiter: ",", // what separates each tag in the filename, web UI will accept comma separated filenames
|
tagDelimiter: ",", // what separates each tag in the filename, web UI will accept comma separated filenames
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// import source
|
||||||
|
if ( FS.existsSync(config.source) ) try {
|
||||||
|
let imp = JSON.parse( FS.readFileSync(config.source) )
|
||||||
|
for ( let k in imp ) {
|
||||||
|
config[k] = imp[k]
|
||||||
|
}
|
||||||
|
console.log(`Imported settings from "${config.source}"`)
|
||||||
|
} catch ( e ) {
|
||||||
|
console.error(e)
|
||||||
|
}
|
||||||
|
|
||||||
let csv = FS.readFileSync(config.tags)
|
let csv = FS.readFileSync(config.tags)
|
||||||
csv = csv.toString().split("\n")
|
csv = csv.toString().split("\n")
|
||||||
config.tags = {}
|
config.tags = {}
|
||||||
|
|
|
@ -9,6 +9,8 @@ import math
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
|
'source': "./data/config/preprocess.json",
|
||||||
|
|
||||||
'input': './images/downloaded/', # files to process
|
'input': './images/downloaded/', # files to process
|
||||||
'output': './images/tagged/', # files to copy files to
|
'output': './images/tagged/', # files to copy files to
|
||||||
'tags': './data/tags.csv', # csv of tags associated with the yiffy model (replace for other flavor of booru's taglist associated with the model you're training against)
|
'tags': './data/tags.csv', # csv of tags associated with the yiffy model (replace for other flavor of booru's taglist associated with the model you're training against)
|
||||||
|
@ -60,6 +62,16 @@ config = {
|
||||||
'tagDelimiter': ",", # what separates each tag in the filename, web UI will accept comma separated filenames
|
'tagDelimiter': ",", # what separates each tag in the filename, web UI will accept comma separated filenames
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if os.path.exists(config['source']):
|
||||||
|
try:
|
||||||
|
with open(config['source'], 'rb') as f:
|
||||||
|
imp = json.loads(f.read().decode('utf-8'))
|
||||||
|
for k in imp:
|
||||||
|
config[k] = imp[k]
|
||||||
|
print(f"Imported settings from {config['source']}")
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
with open(config['tags'], 'rb') as f:
|
with open(config['tags'], 'rb') as f:
|
||||||
csv = f.read().decode('utf-8').split("\n")
|
csv = f.read().decode('utf-8').split("\n")
|
||||||
config['tags'] = {}
|
config['tags'] = {}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user