diff --git a/src/preprocess.js b/src/preprocess.js index 8945cf6..610415f 100755 --- a/src/preprocess.js +++ b/src/preprocess.js @@ -55,6 +55,8 @@ let config = { reverseTags: false, // inverts sorting, prioritizing tags with little representation in the model tagDelimiter: ",", // what separates each tag in the filename, web UI will accept comma separated filenames + + invalidCharacters: "\\/:*?\"<>|", // characters that can't go in a filename } // import source @@ -162,21 +164,27 @@ let parse = async () => { config.tags[tag] = (config.tagsOverrideStart--) * scale; } } else if ( !config.tags[tag] ) continue; - if ( tag.indexOf("/") >= 0 ) continue; // illegal filename character + + let filtered = false; + for ( let i in config.invalidCharacters ) { + if ( tag.indexOf(config.invalidCharacters[i]) >= 0 ) { + filtered = true; + break; + } + } if ( config.filter ) { - let should = false; for ( let i in config.filters ) { let filter = config.filters[i]; if ( filter === tag || ( filter instanceof RegExp && tag.match(filter) ) ) { - should = true; + filtered = true; break; } } - if ( should ) continue; + if ( filtered ) continue; } - tags.push(tag); + if ( !filtered ) tags.push(tag); } } tags = tags.sort( (a, b) => { diff --git a/src/preprocess.py b/src/preprocess.py index b06657f..9b5aa8a 100755 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -17,7 +17,7 @@ config = { 'cache': './data/cache.json', # JSON file of cached tags, will speed up processing if re-running 'rateLimit': 500, # time to wait between requests, in milliseconds, e621 imposes a rate limit of 2 requests per second - 'filenameLimit': 245, # maximum characters to put in the filename, necessary to abide by filesystem limitations + 'filenameLimit': 240, # maximum characters to put in the filename, necessary to abide by filesystem limitations # you can set this to 245, as the web UI has uncapped the prompt limit, but I have yet to test this if this limit was also lifted for textual inversion 'filter': True, @@ -60,6 +60,8 @@ config = { 'reverseTags': False, # inverts sorting, prioritizing tags with little representation in the model 'tagDelimiter': ",", # what separates each tag in the filename, web UI will accept comma separated filenames + + 'invalidCharacters': "\\/:*?\"<>|", # characters that can't go in a filename } if os.path.exists(config['source']): @@ -155,21 +157,25 @@ def parse(): config['tagsOverrideStart'] = config['tagsOverrideStart'] - 1 elif tag not in config['tags']: continue - if "/" in tag or ":" in tag: - continue # illegal filename character + + filtered = False + for char in config['invalidCharacters']: # illegal filename character + if char in tag: + filtered = True + break if config['filter']: - should = False if tag in config['filters']: continue # was break in the original script, fixed ;) for filter in config['filtersRegex']: if re.search(filter, tag): - should = True + filtered = True break - if should: + if filtered: continue - tags.append(tag) + if not filtered: + tags.append(tag) tags.sort(key=lambda x: -config['tags'][x], reverse=config['reverseTags']) if artist: tags.insert(0, artist)