1004 lines
27 KiB
Python
Executable File
1004 lines
27 KiB
Python
Executable File
import math
|
|
import random
|
|
import statistics
|
|
import time
|
|
import json
|
|
import os.path
|
|
import argparse
|
|
import sys
|
|
import urllib.request
|
|
import re
|
|
|
|
from html.parser import HTMLParser
|
|
|
|
import matplotlib as matplotlib
|
|
import matplotlib.pyplot as Plot
|
|
import matplotlib.patches as patches
|
|
|
|
# consts
|
|
TITLE = "SGDQ 2024"
|
|
USE_LEGEND = True # display the legend with markers
|
|
TITLE_OFFSET = 0 # 2.5 # to-do: dynamically set this to how many columns are set with legend
|
|
|
|
# les constant consts
|
|
AUX_MODE = None # total | markers | None
|
|
SORT_BY = None # sort runs by the values
|
|
CUTOFF_SECONDS = 0
|
|
MODE = "scatter"
|
|
|
|
# more constant consts
|
|
TIMESTAMP = str(int(time.time_ns()/1000/1000))
|
|
|
|
IN_QUEUE_FILE = "./data/queue.json"
|
|
IN_RATINGS_FILE = "./data/ratings.json"
|
|
OUT_FILE_TIMESTAMP = f'./images/{TIMESTAMP}.png'
|
|
OUT_FILE = f'./images/ratings[{SORT_BY or AUX_MODE or "chronological"}].png'
|
|
|
|
MIN_COLUMNS = 2 # looks better if there's more than one column with the legend
|
|
CULL_SINGLETON_MARKERS = True # remove any marker that only has 1 entry
|
|
COLOR_BY = "mean" # color by this stat's value
|
|
FADE_BY_STDEV = True # fade outliers
|
|
LINES = ["mean_smart", "median"] # show mean and median lines (or stdev too)
|
|
USE_LATEX = False # never got this to work
|
|
DROP_Z_S = False
|
|
|
|
RE_RATING = re.compile(r"(\b[A-HJ-Zz]+(?:[+-]|\b))")
|
|
RE_RATING_NEWLINE = re.compile(r"(\b[A-HJ-Zz]+(?:[+-]|\b))\n") # just the above, but for newlines
|
|
|
|
COL_WIDTH = 4.0
|
|
ROW_HEIGHT = 0.5
|
|
|
|
# gimmicks
|
|
REVERSE = False # classic reverse tier lists
|
|
ZOOMER = False # sets scale to zoomer-friendly ratings
|
|
|
|
# theme colors
|
|
DARK = True
|
|
COLORS = {
|
|
"BAND0": "#101010",
|
|
"BAND1": "#080808",
|
|
"LINE": "#181818",
|
|
"MEAN": "#404040",
|
|
"STDEV": "#404040",
|
|
"TEXT": "#e0e0e0",
|
|
"STATS": "#606060",
|
|
"MEDIAN": "#AA00AA",
|
|
"RATINGS": [
|
|
( 0.0, (0.2, 0.2, 0.2)),
|
|
( 3.5, (0.6, 0.2, 0.2)),
|
|
( 4.5, (1.0, 0.2, 0.2)),
|
|
( 6.5, (1.0, 1.0, 0.2)),
|
|
( 7.5, (0.2, 1.0, 0.2)),
|
|
( 8.5, (0.2, 1.0, 1.0)),
|
|
(10.0, (1.0, 1.0, 1.0)),
|
|
],
|
|
"SPECIAL": (0.6, 0.2, 1.0),
|
|
"BARBIE": (244.0/255.0, 33.0/255.0, 138.0/255.0),
|
|
} if DARK else {
|
|
"BAND0": "#f0f0f0",
|
|
"BAND1": "#f8f8f8",
|
|
"LINE": "#e8e8e8",
|
|
"MEAN": "#404040",
|
|
"STDEV": "#404040",
|
|
"TEXT": "#404040",
|
|
"STATS": "#a0a0a0",
|
|
"MEDIAN": "#AA00AA",
|
|
"RATINGS": [
|
|
( 0.0, (0.0, 0.0, 0.0)),
|
|
( 3.5, (0.4, 0.0, 0.0)),
|
|
( 4.5, (0.9, 0.0, 0.0)),
|
|
( 6.5, (0.8, 0.8, 0.0)),
|
|
( 7.5, (0.0, 0.9, 0.0)),
|
|
( 8.5, (0.0, 0.8, 0.8)),
|
|
(10.0, (0.7, 0.7, 0.7)),
|
|
],
|
|
"SPECIAL": (0.4, 0.0, 0.8),
|
|
"BARBIE": (244.0/255.0, 33.0/255.0, 138.0/255.0),
|
|
}
|
|
|
|
if DARK:
|
|
Plot.style.use("dark_background")
|
|
|
|
SCORES = {
|
|
"K": -1,
|
|
"ZZZZ": 2.9,
|
|
"ZZZ-": 3.0,
|
|
"ZZZ": 3.1, "ZZ": 3.15, "Z-": 3.2, "Z": 3.25,
|
|
"L": 3.25, "T": 3.25, "N": 3.25,
|
|
"FFF": 3.5, "FF": 3.7,
|
|
"F-": 3.7, "F": 4.0, "F+": 4.3, "E": 4.5,
|
|
"D-": 4.7, "D": 5.0, "D+": 5.3,
|
|
"C-": 5.7, "C": 6.0, "C+": 6.3,
|
|
"B-": 6.7, "B": 7.0, "B+": 7.3,
|
|
"A-": 7.7, "A": 8.0, "A+": 8.3,
|
|
"S-": 8.7, "S": 9.0, "Plot": 9.0, "S+": 9.3,
|
|
"SS": 9.3, "SSS": 9.5, "W": 9.7,
|
|
"SSS+": 9.8,
|
|
"SSSS": 9.9,
|
|
"HH": 10
|
|
}
|
|
TOTAL_SCORES = { name: 0 for name in SCORES.keys() }
|
|
|
|
if SORT_BY is not None:
|
|
COLOR_BY = SORT_BY
|
|
|
|
# runtime globals
|
|
THREADS = {}
|
|
|
|
MARKERS = {}
|
|
REVERSE_MARKERS = {}
|
|
|
|
def add_marker( name, tag, color=COLORS["TEXT"], reverse=None ):
|
|
MARKERS[name] = {
|
|
"tag": tag,
|
|
"color": color,
|
|
"count": 0,
|
|
"reverse": reverse,
|
|
}
|
|
|
|
add_marker("girl", tag="!", reverse="femcel")
|
|
add_marker("foid", tag="...", reverse="sex worker")
|
|
add_marker("tranny", tag="*", reverse="real woman")
|
|
add_marker("biohazard", tag="#")
|
|
add_marker("male", tag="♂")
|
|
add_marker("female", tag="♀")
|
|
add_marker("BOOBS", tag="( Y )")
|
|
add_marker("vt", tag="^")
|
|
add_marker("race", tag="@")
|
|
add_marker("trainwreck/cringekino", tag="%", reverse="flawless")
|
|
add_marker("DNF/invalid", tag="$", reverse="WR")
|
|
add_marker("overestimate", tag=">", reverse="underestimate")
|
|
add_marker("amogus", tag=" sus")
|
|
add_marker("savestated", tag="\\")
|
|
add_marker("ad", tag="✡", reverse="organic")
|
|
|
|
# Ratings fetch related
|
|
|
|
# yuck
|
|
class MyHTMLParser(HTMLParser):
|
|
def __init__(self):
|
|
self.str = ""
|
|
super().__init__()
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == "br":
|
|
self.str += "\n"
|
|
|
|
def handle_data(self, data):
|
|
self.str += data
|
|
|
|
def curl(url):
|
|
if url in THREADS:
|
|
return THREADS[url]
|
|
try:
|
|
conn = urllib.request.urlopen(url)
|
|
data = conn.read()
|
|
data = data.decode()
|
|
data = json.loads(data)
|
|
conn.close()
|
|
THREADS[url] = data
|
|
return data
|
|
except:
|
|
return None
|
|
|
|
# ick
|
|
def parse_rating(comment):
|
|
matches = RE_RATING_NEWLINE.findall(comment)
|
|
binned = re.sub(r'>>\d+\n', "", comment)
|
|
|
|
if len(matches) == 1:
|
|
match = matches[0]
|
|
if match[0] in "SABCDEFNTZKsabcdefntzk":
|
|
return match, None
|
|
|
|
matches = RE_RATING.findall(comment)
|
|
if len(matches) <= 0:
|
|
return None, binned
|
|
if len(matches) >= 2:
|
|
return None, binned
|
|
|
|
match = matches[0]
|
|
if match[0] not in "SABCDEFNTZKsabcdefntzk":
|
|
return None, binned
|
|
|
|
return match, None
|
|
|
|
def fetch_ratings( queue=[] ):
|
|
try:
|
|
ratings = json.load(open(IN_RATINGS_FILE, "r", encoding='utf-8'))
|
|
except Exception as e:
|
|
ratings = {}
|
|
raise e
|
|
|
|
if not queue:
|
|
try:
|
|
queue = json.load(open(IN_QUEUE_FILE, "r", encoding='utf-8'))
|
|
except Exception as e:
|
|
print(str(e))
|
|
pass
|
|
|
|
for entry in queue:
|
|
name = entry["name"]
|
|
post = entry["post"]
|
|
|
|
if post == "" or post is None:
|
|
continue
|
|
|
|
url, _, post_no = post.partition("#p")
|
|
|
|
url = url.replace("boards.4chan.org", "a.4cdn.org") + ".json"
|
|
post_no = int(post_no)
|
|
|
|
if name not in ratings:
|
|
ratings[name] = {}
|
|
|
|
# coerse marker => markers
|
|
if "marker" in entry and "markers" not in entry:
|
|
entry["markers"] = [ entry["marker"] ]
|
|
del entry["marker"]
|
|
|
|
# in case an entry is already defined without these
|
|
if "markers" in entry:
|
|
ratings[name]["markers"] = entry["markers"]
|
|
|
|
if "posts" not in ratings[name]:
|
|
ratings[name]["posts"] = []
|
|
|
|
if post not in ratings[name]["posts"]:
|
|
ratings[name]["posts"].append(post)
|
|
|
|
if "ratings" not in ratings[name]:
|
|
ratings[name]["ratings"] = {}
|
|
|
|
if "binned" not in ratings[name]:
|
|
ratings[name]["binned"] = {}
|
|
|
|
if "times" not in ratings[name]:
|
|
ratings[name]["times"] = {}
|
|
|
|
print(f"Fetching {name}: {post}")
|
|
|
|
data = curl(url)
|
|
if data is None:
|
|
print(f"404: {name}: {post}")
|
|
continue
|
|
|
|
# Matches ">>123" quote tags in a HTML-encoded post comment
|
|
re_link = re.compile(f">>>{post_no}<")
|
|
|
|
for post in data["posts"]:
|
|
if "com" not in post:
|
|
continue
|
|
|
|
if "no" not in post:
|
|
continue
|
|
|
|
com = post["com"]
|
|
no = str(post["no"])
|
|
time = post["time"]
|
|
|
|
match = re_link.search(com)
|
|
|
|
if match is None and f'{no}' != f'{post_no}':
|
|
continue
|
|
|
|
if no not in ratings[name]["times"]:
|
|
ratings[name]["times"][no] = time
|
|
|
|
if no in ratings[name]["ratings"]:
|
|
continue
|
|
|
|
parser = MyHTMLParser()
|
|
parser.feed(com)
|
|
com = parser.str
|
|
|
|
rating, binned = parse_rating(com)
|
|
if rating is not None:
|
|
ratings[name]["ratings"][no] = rating
|
|
else:
|
|
ratings[name]["binned"][no] = binned
|
|
|
|
for no in ratings[name]["ratings"]:
|
|
if no not in ratings[name]["times"]:
|
|
print("MISSING:", name, no)
|
|
|
|
json.dump(ratings, open(IN_RATINGS_FILE, "w"), indent='\t')
|
|
|
|
json.dump(ratings, open(IN_RATINGS_FILE, "w"), indent='\t')
|
|
|
|
# Plot related
|
|
def rating_to_point(rating, count, RAND_PT_DX=1/5, RAND_PT_DY=1/10):
|
|
dx = random.gauss(0, RAND_PT_DX)
|
|
dy = random.gauss(0, RAND_PT_DY)
|
|
|
|
if count < 6:
|
|
dx = dx * (math.sqrt(count) / 6)
|
|
|
|
px = abs(rating + dx)
|
|
if rating >= 3.5:
|
|
if px < 3.5 or px > 9.5:
|
|
px = abs(rating + dx / 2) # Clamp a bit at edges
|
|
else:
|
|
px = abs(rating - dx / 2)
|
|
|
|
# clamp
|
|
if px <= 3.1:
|
|
px = 3.1 + abs(dx) / 2
|
|
dy = dy * 1.125
|
|
|
|
if px > 9.9:
|
|
px = 9.9 - abs(dx) / 2
|
|
dy = dy * 1.125
|
|
|
|
py = dy
|
|
return px, py
|
|
|
|
def clamp(x, lo=0, hi=1):
|
|
if x < lo:
|
|
return lo
|
|
if x > hi:
|
|
return hi
|
|
return x
|
|
|
|
def li(v0, v1, a):
|
|
return v0 * (1- clamp(a)) + v1 * a
|
|
|
|
def lic(c0, c1, a):
|
|
return tuple(li(v0,v1,a) for v0,v1 in zip(c0,c1))
|
|
|
|
def lerp( a, b, t ):
|
|
return a + (b - a) * clamp(t)
|
|
|
|
# interpolate between the closest defined points
|
|
def rating_from_color_table( rating, table, distance=0 ):
|
|
for i in range(len(table)-1):
|
|
t0, t1 = table[i], table[i+1]
|
|
if t0[0] <= rating <= t1[0]:
|
|
alpha = (rating-t0[0]) / (t1[0]-t0[0])
|
|
c = lic(t0[1], t1[1], alpha)
|
|
r = c[0] / (1 + distance)
|
|
g = c[1] / (1 + distance)
|
|
b = c[2] / (1 + distance)
|
|
return (r, g, b)
|
|
|
|
return (1.0, 0.0, 1.0)
|
|
|
|
def rating_to_color(rating, stat):
|
|
target = stat[COLOR_BY]
|
|
mean = stat['mean']
|
|
stdev = stat['stdev']
|
|
markers = stat['markers']
|
|
|
|
table = COLORS["RATINGS"]
|
|
|
|
if "kino" in markers:
|
|
return COLORS["SPECIAL"]
|
|
|
|
if "french" in markers:
|
|
RED = (1.0, 0.0, 0.0)
|
|
WHITE = (1.0, 1.0, 1.0)
|
|
BLUE = (0.0, 0.0, 1.0)
|
|
|
|
distance = rating - mean / stdev
|
|
|
|
if distance < -1.0:
|
|
return BLUE
|
|
if distance > 0.5:
|
|
return RED
|
|
return WHITE
|
|
|
|
distance = 0
|
|
if stdev > 0:
|
|
distance = (abs(rating - mean) / stdev)
|
|
if distance < 2:
|
|
distance = 0
|
|
distance = distance * 0.75
|
|
|
|
return rating_from_color_table( target, table, distance )
|
|
|
|
def title_format(s):
|
|
if USE_LATEX:
|
|
return s.replace('&', r'\&').replace("^", r'\^')
|
|
return s
|
|
|
|
def plot_sub_scatter(sub, stats):
|
|
if DROP_Z_S:
|
|
xticks = [("W", 8), ("Mid", 6), ("L", 4)] if ZOOMER else [("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4)]
|
|
|
|
# set Range
|
|
lo = xticks[-1][1] - 1
|
|
hi = xticks[0][1] + 1
|
|
else:
|
|
xticks = [("W", 8), ("Mid", 6), ("L", 4)] if ZOOMER else [("SSS", 10), ("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4), ("Z", 3)]
|
|
|
|
# set Range
|
|
lo = xticks[-1][1]
|
|
hi = xticks[0][1]
|
|
|
|
sub.set_xlim([lo, hi])
|
|
sub.set_ylim([0, len(stats)])
|
|
|
|
# Draw horizontal bands
|
|
for y in range(len(stats)):
|
|
col = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"]
|
|
y = len(stats) - y - 1 # Flip
|
|
sub.axhspan(y, y+1, facecolor=col, zorder=-20)
|
|
|
|
# Draw vertical lines
|
|
for _, x in xticks:
|
|
sub.axvline(x, color=COLORS["LINE"], zorder=-10)
|
|
|
|
# Set axis labels
|
|
sub.tick_params(axis="y", left=False, labelleft=False)
|
|
sub.tick_params(axis="x", bottom=True, top=True, labelbottom=True, labeltop=True)
|
|
sub.set_xticks([t[1] for t in xticks])
|
|
sub.set_xticklabels([f"{t[0]} " for t in xticks], ha="center")
|
|
|
|
stats.reverse()
|
|
# Draw stat points
|
|
for y, stat in enumerate(stats):
|
|
if stat is None:
|
|
continue
|
|
|
|
xs = [ p[0] for p in stat['points'] ]
|
|
ys = [ p[1] + y + 0.35 for p in stat['points'] ]
|
|
|
|
name = stat['name']
|
|
event = stat['event'] if "event" in stat else ""
|
|
markers = stat["markers"]
|
|
|
|
color_points = [ rating_to_color(r, stat) for r in stat['ratings'] ]
|
|
color_name = COLORS["TEXT"]
|
|
color_stats = COLORS["STATS"]
|
|
|
|
# for barbie
|
|
if name.upper() in COLORS:
|
|
color_points = [ COLORS[name.upper()] ]
|
|
|
|
show_lines = True
|
|
|
|
for marker in markers:
|
|
if marker not in MARKERS:
|
|
continue
|
|
|
|
tag = MARKERS[marker]["tag"]
|
|
count = MARKERS[marker]["count"]
|
|
|
|
if CULL_SINGLETON_MARKERS and count <= 1:
|
|
continue
|
|
|
|
name = f'{name}{tag}'
|
|
|
|
NAME_SIZE = 8 if len(name) < 58 else 7.5
|
|
|
|
sub.scatter(xs, ys, marker=".", s=lerp(80, 20, stat['count']/60.0), color=color_points)
|
|
|
|
sub.annotate(name, (3.1, y+0.8), ha="left", va="center", size=NAME_SIZE, color=color_name, alpha=1) # name
|
|
sub.annotate(f"{stat['count']}", (9.9, y+0.8), ha="right", va="center", size=8, color=color_stats) # count
|
|
#sub.annotate(f"{stat['mean']}", (9.9, y+1.6), ha="right", va="center", size=8, color=color_stats) # mean
|
|
|
|
# Show stat visual lines
|
|
if "stdev" in LINES:
|
|
sub.vlines([stat["mean"] - stat["stdev"], stat["mean"] + stat["stdev"]], y+0.2, y+0.8, color=COLORS["STDEV"], linewidths=1.0, zorder=-9)
|
|
|
|
if "mean" in LINES:
|
|
sub.vlines(stat['mean'], y+0.1, y+0.9, color=COLORS["MEAN"], linewidth=1.0, zorder=-9)
|
|
|
|
if "mean_smart" in LINES:
|
|
sub.vlines(stat['mean_smart'], y+0.1, y+0.9, color=COLORS["MEAN"], linewidth=1.0, zorder=-9)
|
|
|
|
if "median" in LINES:
|
|
sub.vlines(stat['median'], y+0.1, y+0.9, color=COLORS["MEDIAN"], linewidth=1.0, zorder=-9)
|
|
|
|
def plot_sub_boxplot(sub, stats):
|
|
if DROP_Z_S:
|
|
xticks = [("W", 8), ("Mid", 6), ("L", 4)] if ZOOMER else [("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4)]
|
|
|
|
# set Range
|
|
lo = xticks[-1][1] - 1
|
|
hi = xticks[0][1] + 1
|
|
else:
|
|
xticks = [("W", 8), ("Mid", 6), ("L", 4)] if ZOOMER else [("SSS", 10), ("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4), ("Z", 3)]
|
|
|
|
# set Range
|
|
lo = xticks[-1][1]
|
|
hi = xticks[0][1]
|
|
|
|
sub.set_xlim([lo, hi])
|
|
sub.set_ylim([0, len(stats)])
|
|
|
|
# Draw horizontal bands
|
|
for y in range(len(stats)):
|
|
col = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"]
|
|
y = len(stats) - y - 1 # Flip
|
|
sub.axhspan(y, y+1, facecolor=col, zorder=-20)
|
|
|
|
# Draw vertical lines
|
|
for _, x in xticks:
|
|
sub.axvline(x, color=COLORS["LINE"], zorder=-10)
|
|
|
|
# Set axis labels
|
|
sub.tick_params(axis="y", left=False, labelleft=False)
|
|
sub.tick_params(axis="x", bottom=True, top=True, labelbottom=True, labeltop=True)
|
|
sub.set_xticks([t[1] for t in xticks])
|
|
sub.set_xticklabels([f"{t[0]} " for t in xticks], ha="center")
|
|
|
|
boxes = []
|
|
positions = []
|
|
colors = []
|
|
|
|
stats.reverse()
|
|
|
|
for y, stat in enumerate(stats):
|
|
# Reverse y position so the first stat ends on top
|
|
if stat is None:
|
|
continue
|
|
|
|
boxes.append( [ r for r in stat['ratings'] ] )
|
|
positions.append( y + 0.4 )
|
|
colors.append( rating_from_color_table( stat['mean_smart'], COLORS["RATINGS"] ) )
|
|
|
|
name = stat['name']
|
|
event = stat['event'] if "event" in stat else ""
|
|
markers = stat["markers"]
|
|
|
|
color_name = COLORS["TEXT"]
|
|
color_stats = COLORS["STATS"]
|
|
|
|
for marker in markers:
|
|
if marker not in MARKERS:
|
|
continue
|
|
|
|
tag = MARKERS[marker]["tag"]
|
|
count = MARKERS[marker]["count"]
|
|
|
|
if CULL_SINGLETON_MARKERS and count <= 1:
|
|
continue
|
|
|
|
name = f'{name}{tag}'
|
|
|
|
NAME_SIZE = 8 if len(name) < 58 else 7.5
|
|
|
|
sub.annotate(name, (3.1, y+0.8), ha="left", va="center", size=NAME_SIZE, color=color_name, alpha=1) # name
|
|
sub.annotate(f"{stat['count']}", (9.9, y+0.8), ha="right", va="center", size=8, color=color_stats) # count
|
|
#sub.annotate(f"{stat['mean']}", (9.9, y+1.6), ha="right", va="center", size=8, color=color_stats) # mean
|
|
|
|
# Draw stat boxes
|
|
bplot = sub.boxplot( boxes, positions=positions, showmeans=True, patch_artist=True, meanline=True, vert=False )
|
|
|
|
# set color to rating color
|
|
for patch, color in zip(bplot['boxes'], colors):
|
|
patch.set_facecolor(color)
|
|
|
|
# dict_keys(['whiskers', 'caps', 'boxes', 'medians', 'fliers', 'means'])
|
|
|
|
# set median to median color
|
|
for median in bplot['medians']:
|
|
median.set_color(COLORS["MEDIAN"])
|
|
|
|
for median in bplot['means']:
|
|
median.set_color(COLORS["MEAN"])
|
|
|
|
|
|
def plot_stat_bar(sub, stats, stat, y):
|
|
if stat is None:
|
|
return
|
|
|
|
xs = [ p[0] for p in stat['points'] ]
|
|
ys = [ p[1] + y + 0.35 for p in stat['points'] ]
|
|
|
|
name = stat['name']
|
|
event = stat['event'] if "event" in stat else ""
|
|
markers = stat["markers"]
|
|
|
|
color_points = [ rating_to_color(r, stat) for r in stat['ratings'] ]
|
|
color_name = COLORS["TEXT"]
|
|
color_stats = COLORS["STATS"]
|
|
|
|
# for barbie
|
|
if name.upper() in COLORS:
|
|
color_points = [ COLORS[name.upper()] ]
|
|
|
|
show_lines = True
|
|
|
|
for marker in markers:
|
|
if marker not in MARKERS:
|
|
continue
|
|
|
|
tag = MARKERS[marker]["tag"]
|
|
count = MARKERS[marker]["count"]
|
|
|
|
if CULL_SINGLETON_MARKERS and count <= 1:
|
|
continue
|
|
|
|
name = f'{name}{tag}'
|
|
|
|
NAME_SIZE = 8 if len(name) < 58 else 7.5
|
|
|
|
sub.annotate(name, (3.1, y+0.8), ha="left", va="center", size=NAME_SIZE, color=color_name, alpha=1) # name
|
|
sub.annotate(f"{stat['count']}", (9.9, y+0.8), ha="right", va="center", size=8, color=color_stats) # count
|
|
#sub.annotate(f"{stat['mean']}", (9.9, y+1.6), ha="right", va="center", size=8, color=color_stats) # mean
|
|
|
|
def plot_sub_bars(sub, stats):
|
|
lo = 0
|
|
hi = 1
|
|
sub.set_ylim([0, len(stats)])
|
|
stats.reverse()
|
|
|
|
# Draw horizontal bands
|
|
for y in range(len(stats)):
|
|
col = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"]
|
|
y = len(stats) - y - 1 # Flip
|
|
sub.axhspan(y, y+1, facecolor=col, zorder=-20)
|
|
|
|
ys = [0.5 + y for y in range(len(stats))]
|
|
# Place bars
|
|
bottoms = [0] * len(stats)
|
|
buckets = [
|
|
(4, "F", "#73172d"),
|
|
(5, "D", "#df3e23"),
|
|
(6, "C", "#f9a31b"),
|
|
(7, "B", "#fffc40"),
|
|
(8, "A", "#9cdb43"),
|
|
(9, "S", "#20d6c7"),
|
|
]
|
|
|
|
for i,r,c in buckets:
|
|
counts = [(0 if s is None else s['buckets'][i]) for s in stats]
|
|
sub.barh(ys, counts, 0.5, left=bottoms, color=c, label=r)
|
|
bottoms = [b + n for b,n in zip(bottoms, counts)]
|
|
|
|
# Text
|
|
for y, stat in enumerate(stats):
|
|
if stat is None:
|
|
continue
|
|
|
|
name = stat['name']
|
|
event = stat['event'] if "event" in stat else ""
|
|
markers = stat["markers"]
|
|
|
|
color_name = COLORS["TEXT"]
|
|
color_stats = COLORS["STATS"]
|
|
|
|
show_lines = True
|
|
|
|
for marker in markers:
|
|
if marker not in MARKERS:
|
|
continue
|
|
|
|
tag = MARKERS[marker]["tag"]
|
|
count = MARKERS[marker]["count"]
|
|
|
|
if CULL_SINGLETON_MARKERS and count <= 1:
|
|
continue
|
|
|
|
name = f'{name}{tag}'
|
|
|
|
NAME_SIZE = 8 if len(name) < 58 else 7.5
|
|
|
|
sub.annotate(name, (0.0, y+0.85), ha="left", va="center", size=NAME_SIZE, color=color_name, alpha=1) # name
|
|
#sub.annotate(f"{stat['count']}", (0, y+0.9), ha="right", va="center", size=8, color=color_stats) # count
|
|
|
|
# Axis labels
|
|
sub.tick_params(axis="y", left=False, labelleft=False)
|
|
sub.tick_params(axis="x", bottom=True, top=True, labelbottom=True, labeltop=True)
|
|
|
|
def create_plot( stats ):
|
|
# determine dimensions
|
|
cols = max(MIN_COLUMNS, min(5, round(math.sqrt(len(stats) / 6))))
|
|
rows = math.ceil(len(stats) / cols)
|
|
|
|
# subplots might return an array or single subplot
|
|
fig, subs = Plot.subplots(1, cols)
|
|
if cols == 1:
|
|
subs = [subs]
|
|
|
|
# set main figure stuff
|
|
figSize = (cols * COL_WIDTH + 1, rows * ROW_HEIGHT + 1.5)
|
|
fig.set_size_inches(figSize)
|
|
fig.set_tight_layout(True)
|
|
|
|
# set title and subtitle
|
|
figCenterX, figTop = figSize[0] / 2 - TITLE_OFFSET, figSize[1]
|
|
|
|
fig.text(figCenterX, figTop-0.2, TITLE, size=20, ha="center", va="top", transform=fig.dpi_scale_trans) # Title
|
|
fig.text(figCenterX, figTop-0.45, "as rated by /v/", size=10, ha="center", va="top", transform=fig.dpi_scale_trans) # Subtitle
|
|
|
|
# legend
|
|
if USE_LEGEND:
|
|
handles = []
|
|
# add lines to legend
|
|
if "mean" in LINES or "mean_smart" in LINES:
|
|
handles.append(patches.Patch(label=title_format('Mean'), color=COLORS["MEAN"]))
|
|
if "median" in LINES:
|
|
handles.append(patches.Patch(label=title_format('Median'), color=COLORS["MEDIAN"]))
|
|
if "stdev" in LINES:
|
|
handles.append(patches.Patch(label=title_format('Std. Dev.'), color=COLORS["STDEV"]))
|
|
|
|
# add markers to legend
|
|
for marker, entry in MARKERS.items():
|
|
tag = entry["tag"]
|
|
count = entry["count"]
|
|
reverse = entry["reverse"] if "reverse" in entry else None
|
|
|
|
if count <= (1 if CULL_SINGLETON_MARKERS else 0):
|
|
continue
|
|
|
|
if tag == "":
|
|
continue
|
|
|
|
if REVERSE and reverse:
|
|
marker = reverse
|
|
|
|
title = f'({count}) {marker}{tag}'
|
|
handles.append(patches.Patch(label=title_format(title), color="#000000"))
|
|
|
|
fig.legend(handles=handles, ncol=2 if len(handles) <= 4 else 3)
|
|
|
|
# padding
|
|
fig.suptitle(" ", size=40)
|
|
|
|
# sort by specified statistic
|
|
if SORT_BY is not None:
|
|
stats = sorted(stats, key=lambda s: s[SORT_BY])
|
|
|
|
# add empty ratings ensure each column has the same number of entries
|
|
stats += [None] * (len(stats) - len(stats) // rows)
|
|
|
|
# create subplot column plot
|
|
for i, sub in enumerate(subs):
|
|
stats_sub = stats[i*rows:(i+1)*rows]
|
|
if MODE == "scatter":
|
|
plot_sub_scatter(sub, stats_sub)
|
|
elif MODE == "bars":
|
|
plot_sub_bars(sub, stats_sub)
|
|
elif MODE == "boxplot":
|
|
plot_sub_boxplot(sub, stats_sub)
|
|
|
|
# Stats related
|
|
def compute_mean_smart(ratings):
|
|
ratings = sorted( ratings )
|
|
|
|
cutoff = len(ratings) // 8
|
|
if cutoff > 0:
|
|
ratings = ratings[cutoff:-cutoff]
|
|
|
|
return statistics.mean(ratings)
|
|
|
|
def stat_new(name, ratings, entry={}):
|
|
# Fallback in case of near-empty ratings
|
|
if len(ratings) == 0:
|
|
ratings.append(0.0)
|
|
if len(ratings) == 1:
|
|
ratings.append(ratings[0])
|
|
|
|
# Count frequency of each (rounded) rating
|
|
buckets = [0] * 10
|
|
for r in ratings:
|
|
r = min(9, max(4, round(r)))
|
|
buckets[r] += 1
|
|
|
|
# Compute stats
|
|
stat = {}
|
|
stat['name'] = name
|
|
stat['ratings'] = ratings
|
|
stat['points'] = [rating_to_point(r, len(ratings)) for r in ratings]
|
|
stat['buckets'] = buckets
|
|
stat['count'] = len(ratings)
|
|
stat['mean'] = statistics.mean(ratings)
|
|
stat['median'] = statistics.median(ratings)
|
|
stat['mode'] = statistics.mode(ratings)
|
|
stat['median_grouped'] = statistics.median_grouped(round(r) for r in ratings)
|
|
stat['stdev'] = statistics.pstdev(ratings)
|
|
stat['mean_smart'] = compute_mean_smart(ratings)
|
|
|
|
# attach extra data
|
|
stat["markers"] = []
|
|
if "marker" in entry:
|
|
stat["markers"] = [ entry["marker"] ]
|
|
elif "markers" in entry:
|
|
stat["markers"] = entry["markers"]
|
|
|
|
# fix up previously split markers
|
|
for i, marker in enumerate(stat["markers"]):
|
|
if marker == "DNF":
|
|
stat["markers"][i] = "DNF/invalid"
|
|
elif marker == "trainwreck":
|
|
stat["markers"][i] = "trainwreck/cringekino"
|
|
elif marker == "invalid":
|
|
stat["markers"][i] = "DNF/invalid"
|
|
elif marker == "cringekino":
|
|
stat["markers"][i] = "trainwreck/cringekino"
|
|
|
|
if "event" in entry:
|
|
stat["event"] = entry["event"]
|
|
|
|
return stat
|
|
|
|
# read from json
|
|
def read_stats(filename):
|
|
stats = []
|
|
aux = {
|
|
"total": [],
|
|
"markers": {},
|
|
}
|
|
|
|
data = json.load(open(filename, "r", encoding='utf-8'))
|
|
|
|
for name, entry in data.items():
|
|
ratings = []
|
|
|
|
if "ignore" in entry and entry["ignore"]:
|
|
continue
|
|
|
|
op = entry["posts"][0].split("#p")[-1]
|
|
start_time = entry["times"][op] if "times" in entry and op in entry["times"] else 0
|
|
|
|
# parse ratings in each run
|
|
for no, score in entry["ratings"].items():
|
|
#cur_time = entry["times"][no] if "times" in entry and no in entry["times"] else 0
|
|
if start_time > 0 and CUTOFF_SECONDS > 0:
|
|
if no not in entry["times"]:
|
|
continue
|
|
|
|
cur_time = entry["times"][no]
|
|
if cur_time - start_time > CUTOFF_SECONDS:
|
|
print("Culled", name, no, score, cur_time - start_time)
|
|
continue
|
|
|
|
score = score.upper()
|
|
|
|
# trim extraneous characters (for example: ZZZZZZZ) if not already a valid score
|
|
if score not in SCORES:
|
|
score = score[:3]
|
|
|
|
if DROP_Z_S:
|
|
if score[0] == "Z":
|
|
continue
|
|
if score[:2] == "SS":
|
|
continue
|
|
|
|
# invalid score
|
|
if score not in SCORES:
|
|
continue
|
|
|
|
# increment totals (I don't remember what I originally used this for)
|
|
TOTAL_SCORES[score] += 1
|
|
|
|
# score modifiers
|
|
rating = SCORES[score]
|
|
|
|
# randomize downward
|
|
if "randomize" in entry and CUTOFF_SECONDS == 0 and False:
|
|
lo = -3.0 # SCORES[list(SCORES.keys())[0]]
|
|
hi = 6.0 # SCORES[list(SCORES.keys())[-1]]
|
|
|
|
if random.random() < entry["randomize"] and rating > hi:
|
|
rating = random.uniform(lo, hi)
|
|
|
|
ratings.append( rating )
|
|
aux["total"].append( rating )
|
|
|
|
stat = stat_new(name, ratings, entry)
|
|
stats.append(stat)
|
|
|
|
# increment marker totals
|
|
for marker in stat["markers"]:
|
|
if marker not in MARKERS:
|
|
continue
|
|
MARKERS[marker]["count"] += 1
|
|
|
|
for rating in ratings:
|
|
if marker not in aux["markers"]:
|
|
aux["markers"][marker] = []
|
|
aux["markers"][marker].append( rating )
|
|
|
|
if not stat["markers"]:
|
|
marker = "none"
|
|
for rating in ratings:
|
|
if marker not in aux["markers"]:
|
|
aux["markers"][marker] = []
|
|
aux["markers"][marker].append( rating )
|
|
|
|
# show aggregate total
|
|
if AUX_MODE == "total":
|
|
stats = [ stat_new("total", aux["total"]) ]
|
|
# aggregate by markers
|
|
elif AUX_MODE == "markers":
|
|
stats = [ stat_new(name, ratings) for name, ratings in aux["markers"].items() ]
|
|
|
|
return stats
|
|
|
|
def main():
|
|
# yuck
|
|
global REVERSE
|
|
global ZOOMER
|
|
global DARK
|
|
global USE_LEGEND
|
|
global MIN_COLUMNS
|
|
global CUTOFF_SECONDS
|
|
global SORT_BY
|
|
global MODE
|
|
global AUX_MODE
|
|
global OUT_FILE
|
|
|
|
parser = argparse.ArgumentParser("Charter")
|
|
# actions
|
|
parser.add_argument("--fetch", action="store_true", help="Fetch ratings from queue")
|
|
parser.add_argument("--plot", action="store_true", help="Plot ratings from file")
|
|
# queue-less args
|
|
parser.add_argument("--url", type=str, default=None, help="Post link to fetch ratings from its replies")
|
|
parser.add_argument("--name", type=str, default=None, help="Name of run for rating")
|
|
parser.add_argument("--markers", type=str, default=None, help="Markers to attach to run rating")
|
|
# modifiers
|
|
parser.add_argument("--sort-by", type=str, default=None, help="Sort plotted ratings by this value")
|
|
parser.add_argument("--mode", type=str, default=None, help="Additional modes (scatter | bars | boxplot)")
|
|
parser.add_argument("--aux-mode", type=str, default=None, help="Additional modes (total | markers)")
|
|
parser.add_argument("--cutoff-seconds", type=int, default=None, help="Ignore ratings made X seconds after the rating post")
|
|
parser.add_argument("--cutoff-minutes", type=int, default=None, help="Ignore ratings made X minutes after the rating post")
|
|
# lesser used modifiers
|
|
parser.add_argument("--reverse", action="store_true", help="Reverses the ratings")
|
|
parser.add_argument("--zoomer", action="store_true", help="Zoomer friendly scales")
|
|
parser.add_argument("--light", action="store_true", help="Light mode")
|
|
parser.add_argument("--no-legend", action="store_true", help="Hides the legend")
|
|
parser.add_argument("--copy", action="store_true", help="Saves a copy by the timestamp")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.sort_by:
|
|
SORT_BY = args.sort_by
|
|
|
|
if args.aux_mode:
|
|
AUX_MODE = args.aux_mode
|
|
if AUX_MODE == "total":
|
|
args.no_legend = True
|
|
MIN_COLUMNS = 1
|
|
|
|
modifiers = [ f'[{SORT_BY or AUX_MODE or "chronological"}]' ]
|
|
|
|
if args.mode:
|
|
MODE = args.mode
|
|
modifiers.append(f'[type={MODE}]')
|
|
|
|
if args.cutoff_seconds:
|
|
CUTOFF_SECONDS = args.cutoff_seconds
|
|
elif args.cutoff_minutes:
|
|
CUTOFF_SECONDS = args.cutoff_minutes * 60
|
|
if CUTOFF_SECONDS > 0:
|
|
modifiers.append(f'[cutoff={CUTOFF_SECONDS//60}]')
|
|
|
|
|
|
OUT_FILE = f'./images/ratings{"".join(modifiers)}.png'
|
|
|
|
|
|
REVERSE = args.reverse
|
|
ZOOMER = args.zoomer
|
|
DARK = not args.light
|
|
USE_LEGEND = not args.no_legend
|
|
|
|
if args.fetch:
|
|
queue = []
|
|
|
|
# inject to queue
|
|
if args.url and args.name:
|
|
queue = [{ "post": args.url, "name": args.name, "markers": args.markers.split(",") if args.markers else None }]
|
|
|
|
fetch_ratings(queue=queue)
|
|
|
|
if args.plot:
|
|
stats = read_stats(IN_RATINGS_FILE)
|
|
create_plot(stats)
|
|
|
|
if args.copy:
|
|
print(f'Saving chart: {OUT_FILE_TIMESTAMP}')
|
|
Plot.savefig(OUT_FILE_TIMESTAMP)
|
|
|
|
print(f'Saving chart: {OUT_FILE}')
|
|
Plot.savefig(OUT_FILE)
|
|
Plot.close()
|
|
|
|
if not args.fetch and not args.plot:
|
|
raise Exception("Specify --fetch or --plot")
|
|
|
|
if __name__ == "__main__":
|
|
main() |