vee-speedrun-ratings/chart.py

import math
import random
import statistics
import time
import json
import os.path
import argparse
import sys
import urllib.request
import re

from datetime import datetime
from html.parser import HTMLParser

import plotly.graph_objects as go
from plotly.subplots import make_subplots
from PIL import Image

# consts
TITLE = "SGDQ 2026"
SUBTITLE = "as rated by /v/"
USE_LEGEND = False # display the legend with markers
TITLE_OFFSET = 0 # 2.5 # to-do: dynamically set this to how many columns are set with legend
UI_SCALE = 1.0
DATE_OFFSET = (60 * 60 * 8)
# les constant consts
AUX_MODE = None # total | markers | None
SORT_BY = None # sort runs by the values
CUTOFF_SECONDS = 0 # 60 * 5
MODE = "scatter"
FILTER_BY = None # filters by markers
PRINT_STATS = False

# more constant consts
TIMESTAMP = str(int(time.time_ns()/1000/1000))

IN_QUEUE_FILE = "./data/queue.json"
IN_RATINGS_FILE = "./data/ratings.json"
OUT_FILE_TIMESTAMP = f'./images/{TIMESTAMP}.png'
OUT_FILE = f'./images/ratings[{SORT_BY or AUX_MODE or "chronological"}].png'

MIN_COLUMNS = 2 # looks better if there's more than one column with the legend
CULL_SINGLETON_MARKERS = False # remove any marker that only has 1 entry
COLOR_BY = "mean" # color by this stat's value
FADE_BY_STDEV = True # fade outliers
LINES = ["mean_smart", "median"] # show mean and median lines (or stdev too)
USE_LATEX = False # never got this to work
DROP_Z_S = False

RE_RATING = re.compile(r"(\b[A-HJ-Zz]+(?:[+-]|\b))")
RE_RATING_NEWLINE = re.compile(r"(\b[A-HJ-Zz]+(?:[+-]|\b))\n") # just the above, but for newlines

IMAGE_CACHE = {}

COL_WIDTH = 4.0
ROW_HEIGHT = 0.5

# gimmicks
REVERSE = False # classic reverse tier lists
ZOOMER = False # sets scale to zoomer-friendly ratings

# theme colors
DARK = True
COLORS = {
	"BACKGROUND": "#0c0c0c",
	"BAND0": "#101010",
	"BAND1": "#080808",
	"LINE": "#181818",
	"MEAN": "#404040",
	"STDEV": "#404040",
	"TEXT": "#e0e0e0",
	"STATS": "#606060",
	"MEDIAN": "#AA00AA",
	"RATINGS": [
		( 0.0, (0.2, 0.2, 0.2)),
		( 3.5, (0.6, 0.2, 0.2)),
		( 4.5, (1.0, 0.2, 0.2)),
		( 6.5, (1.0, 1.0, 0.2)),
		( 7.5, (0.2, 1.0, 0.2)),
		( 8.5, (0.2, 1.0, 1.0)),
		(10.0, (1.0, 1.0, 1.0)),
	],
	"SPECIAL": (0.6, 0.2, 1.0),
	"BROWN": (139.0/255.0,69.0/255.0,19.0/255.0),
	"BARBIE": (244.0/255.0, 33.0/255.0, 138.0/255.0),
	"ROSE AND CAMELLIA": (0.8705882352941177, 0.6313725490196078, 0.5764705882352941),
} if DARK else {
	"BACKGROUND": "#ffffff",
	"BAND0": "#f0f0f0",
	"BAND1": "#f8f8f8",
	"LINE": "#e8e8e8",
	"MEAN": "#404040",
	"STDEV": "#404040",
	"TEXT": "#404040",
	"STATS": "#a0a0a0",
	"MEDIAN": "#AA00AA",
	"RATINGS": [
		( 0.0, (0.0, 0.0, 0.0)),
		( 3.5, (0.4, 0.0, 0.0)),
		( 4.5, (0.9, 0.0, 0.0)),
		( 6.5, (0.8, 0.8, 0.0)),
		( 7.5, (0.0, 0.9, 0.0)),
		( 8.5, (0.0, 0.8, 0.8)),
		(10.0, (0.7, 0.7, 0.7)),
	],
	"SPECIAL": (0.4, 0.0, 0.8),
	"BROWN": (139.0/255.0,69.0/255.0,19.0/255.0),
	"BARBIE": (244.0/255.0, 33.0/255.0, 138.0/255.0),
	"ROSE AND CAMELLIA": (0.8705882352941177, 0.6313725490196078, 0.5764705882352941),
}

"""
if DARK:
	Plot.style.use("dark_background")
"""

SCORES = {
	"K": 0, "T": 0,
	"ZZZZ": 2.9,
	"ZZZ-": 3.0,
	"ZZZ": 3.1, "ZZ": 3.15, "Z-": 3.2, "Z": 3.25, "DNF": 3.25,
	"L": 3.25, "N": 3.25,
	"FFF": 3.5, "FF": 3.7, 
	"F-": 3.7, "F": 4.0, "F+": 4.3, "E": 4.5,
	"D-": 4.7, "D": 5.0, "D+": 5.3,
	"C-": 5.7, "C": 6.0, "C+": 6.3,
	"B-": 6.7, "B": 7.0, "B+": 7.3,
	"A-": 7.7, "A": 8.0, "A+": 8.3,
	"S-": 8.7, "S": 9.0, "P": 9.0, "S+": 9.3,
	"SS": 9.3, "SSS": 9.5, "W": 9.7,
	"SSS+": 9.8,
	"SSSS": 9.9,
	"HH": 10 
}
TOTAL_SCORES = { name: 0 for name in SCORES.keys() }

if SORT_BY is not None:
	COLOR_BY = SORT_BY

# runtime globals
THREADS = {}

MARKERS = {}
REVERSE_MARKERS = {}

def add_marker( name, tag, color=COLORS["TEXT"], reverse=None, zoomer=None ):
	MARKERS[name] = {
		"tag": tag,
		"color": color,
		"count": 0,
		"reverse": reverse,
		"zoomer": zoomer,
	}

add_marker("girl", tag="!", reverse="femcel", zoomer="gyatt")
add_marker("foid", tag="...", reverse="sex worker", zoomer="skibidi")
add_marker("tranny", tag="*", reverse="real woman", zoomer="fr")
add_marker("ladyboy", tag="¿", reverse="tomboy")
add_marker("black", tag="&", reverse="white", zoomer="kang")
add_marker("biohazard", tag="#") # fatales
add_marker("male", tag="♂") # fatales 
add_marker("female", tag="♀") # fatales
add_marker("BOOBS", tag="(  Y  )") # memetic
add_marker("vt", tag="^") # rarely used
add_marker("amogus", tag=" sus") # rarely used
add_marker("race", tag="@") # unused now
add_marker("savestated", tag="\\") # rarely used, should be invalid now
#add_marker("trainwreck/cringekino", tag="%", reverse="flawless", zoomer="fanum tax")
add_marker("trainwreck", tag="%", reverse="flawless", zoomer="fanum tax")
add_marker("cringekino", tag="%", reverse="kinocringe", zoomer="fanum tax")
add_marker("DNF/invalid", tag="$", reverse="WR", zoomer="ohio")
add_marker("overestimate", tag=">", reverse="underestimate", zoomer="💀")
add_marker("ad", tag="✡", reverse="organic", zoomer="sigma")
add_marker("nonrun", tag="@", reverse="letsplay", zoomer="content")
add_marker("neporun", tag="\\", reverse="indie", zoomer="collab")
add_marker("it", tag="※", reverse="human", zoomer="it")
#add_marker("ad/nonrun", tag="✡", reverse="organic", zoomer="sigma")

def sample_image(image_path, px, py_local, min_x, max_x, min_y, max_y):
	if image_path not in IMAGE_CACHE:
		try:
			IMAGE_CACHE[image_path] = Image.open(image_path).convert('RGB')
		except Exception as e:
			print(f"Could not load {image_path}: {e}")
			IMAGE_CACHE[image_path] = None

	img = IMAGE_CACHE[image_path]
	if img is None:
		return None

	range_x = max_x - min_x if max_x > min_x else 1.0
	range_y = max_y - min_y if max_y > min_y else 1.0

	u = clamp((px - min_x) / range_x)
	v = clamp((py_local - min_y) / range_y)

	ix = int(u * (img.width - 1))
	iy = int((1.0 - v) * (img.height - 1))

	r, g, b = img.getpixel((ix, iy))
	return (r / 255.0, g / 255.0, b / 255.0)

# yuck
class MyHTMLParser(HTMLParser):
	def __init__(self):
		self.str = ""
		super().__init__()

	def handle_starttag(self, tag, attrs):
		if tag == "br":
			self.str += "\n"

	def handle_data(self, data):
		self.str += data

def curl(url):
	if url in THREADS:
		return THREADS[url]
	try:
		conn = urllib.request.urlopen(url)
		data = conn.read()
		data = data.decode()
		data = json.loads(data)
		conn.close()
		THREADS[url] = data
		return data
	except:
		return None

# ick
def parse_rating(comment):
	matches = RE_RATING_NEWLINE.findall(comment)
	binned = re.sub(r'>>\d+\n', "", comment)

	if len(matches) == 1:
		match = matches[0]
		if match[0] in "SABCDEFNTZKsabcdefntzk": 
			return match, None

	matches = RE_RATING.findall(comment)
	if len(matches) <= 0:
		return None, binned
	if len(matches) >= 2:
		return None, binned

	match = matches[0]
	if match[0] not in "SABCDEFNTZKsabcdefntzk": 
		return None, binned

	return match, None

def fetch_ratings( queue=[] ):
	try:
		ratings = json.load(open(IN_RATINGS_FILE, "r", encoding='utf-8'))
	except Exception as e:
		ratings = {}
		raise e
	
	if not queue:
		try:
			queue = json.load(open(IN_QUEUE_FILE, "r", encoding='utf-8'))
		except Exception as e:
			print(str(e))
			pass	

	for entry in queue:
		name = entry["name"]
		post = entry["post"]

		if post == "" or post is None:
			continue

		url, _, post_no = post.partition("#p")
		prefix, __, thread = url.partition("/thread/")

		# remove training slash
		if thread[-1] == "/":
			thread = thread[:-1]

		# pick API
		"""
		if "p.ecker.tech" in prefix:
			url = f'https://p.ecker.tech/chan/browse/v/thread/{thread}/?update'
		else:
			url = f'https://a.4cdn.org/v/thread/{thread}.json'
		"""
		url = f'https://a.4cdn.org/v/thread/{thread}.json'

		post_no = int(post_no)

		if name not in ratings:
			ratings[name] = {}

		# coerse marker => markers
		if "marker" in entry and "markers" not in entry:
			entry["markers"] = [ entry["marker"] ]
			del entry["marker"]
		
		# in case an entry is already defined without these
		if "markers" in entry:
			ratings[name]["markers"] = entry["markers"]

		if "posts" not in ratings[name]:
			ratings[name]["posts"] = []
		
		if post not in ratings[name]["posts"]:
			ratings[name]["posts"].append(post)

		if "ratings" not in ratings[name]:
			ratings[name]["ratings"] = {}
		
		if "binned" not in ratings[name]:
			ratings[name]["binned"] = {}
		
		if "times" not in ratings[name]:
			ratings[name]["times"] = {}

		print(f"Fetching {name}: {post}")

		data = curl(url)
		if data is None:
			print(f"404: {name}: {post}")
			continue

		re_link = re.compile(f">&gt;&gt;{post_no}<")

		# coerce to array
		if isinstance(data["posts"], dict):
			data["posts"] = data["posts"].values()

		for post in data["posts"]:
			if "com" not in post: 
				continue

			if "no" not in post:
				continue

			com = post["com"]
			no = str(post["no"])
			time = post["time"]

			match = re_link.search(com)
			
			if match is None and f'{no}' != f'{post_no}':
				continue
			
			if no not in ratings[name]["times"]:
				ratings[name]["times"][no] = time

			if no in ratings[name]["ratings"]:
				continue
			
			if no in ratings[name]["binned"]:
				continue

			parser = MyHTMLParser()
			parser.feed(com)
			com = parser.str

			rating, binned = parse_rating(com)
			if rating is not None:
				ratings[name]["ratings"][no] = rating
			else:
				ratings[name]["binned"][no] = binned

		for no in ratings[name]["ratings"]:
			if no not in ratings[name]["times"]:
				print("MISSING:", name, no)

		json.dump(ratings, open(IN_RATINGS_FILE, "w"), indent='\t')
	
	json.dump(ratings, open(IN_RATINGS_FILE, "w"), indent='\t')

def stat_to_str(stat):
	l = [f"{stat['name']:25} ::"]
	l.append(f"n = {stat['count']:2},")
	l.append(f"mean = {stat['mean']:.2f},")
	l.append(f"ms = {stat['mean_smart']:.2f},")
	l.append(f"sd = {stat['stdev']:.2f},")
	l.append(f"med = {stat['median']:.1f}")
	return " ".join(l)

# Plot related
def rating_to_point(rating, count, RAND_PT_DX=1/5, RAND_PT_DY=1/10):
	dx = random.gauss(0, RAND_PT_DX)
	dy = random.gauss(0, RAND_PT_DY)

	if count < 6:
		dx = dx * (math.sqrt(count) / 6)

	px = abs(rating + dx)
	if rating >= 3.5:
		if px < 3.5 or px > 9.5: 
			px = abs(rating + dx / 2)
	else:
		px = abs(rating - dx / 2)
	
	# clamp
	if px <= 3.1:
		px = 3.1 + abs(dx) / 2
		dy = dy * 1.125

	if px > 9.9:
		px = 9.9 - abs(dx) / 2
		dy = dy * 1.125

	py = dy
	return px, py

def clamp(x, lo=0, hi=1):
	if x < lo:
		return lo
	if x > hi:
		return hi
	return x

def li(v0, v1, a): 
	return v0 * (1- clamp(a)) + v1 * a

def lic(c0, c1, a):
	return tuple(li(v0,v1,a) for v0,v1 in zip(c0,c1))

def lerp( a, b, t ):
	return a + (b - a) * clamp(t)

# interpolate between the closest defined points
def rating_from_color_table( rating, table, distance=0 ):
	for i in range(len(table)-1):
		t0, t1 = table[i], table[i+1]
		if t0[0] <= rating <= t1[0]:
			alpha = (rating-t0[0]) / (t1[0]-t0[0])
			c = lic(t0[1], t1[1], alpha)
			r = c[0] / (1 + distance)
			g = c[1] / (1 + distance)
			b = c[2] / (1 + distance)
			return (r, g, b)

	return (1.0, 0.0, 1.0)

def rating_to_color(rating, stat):
	target = stat[COLOR_BY]
	mean = stat['mean']
	stdev = stat['stdev']
	markers = stat['markers']
	
	table = COLORS["RATINGS"]
	
	if "kino" in markers:
		return COLORS["SPECIAL"]
	
	if "shart" in markers:
		return COLORS["BROWN"]

	if "french" in markers:
		RED = (1.0, 0.0, 0.0)
		WHITE = (1.0, 1.0, 1.0)
		BLUE = (0.0, 0.0, 1.0)

		distance = rating - mean / stdev

		if distance < -1.0:
			return BLUE
		if distance > 0.5:
			return RED
		return WHITE

	if "trans rights" in markers:
		#F5A9B8
		BLUE = (0.3569, 0.8078, 0.9804)
		PINK = (0.9608, 0.6627, 0.7216)
		WHITE = (1.0, 1.0, 1.0)

		distance = rating # - mean / stdev

		if distance < 4.0:
			return BLUE
		if distance < 4.5:
			return PINK
		if distance > 7.0:
			return BLUE
		if distance > 6.0:
			return PINK

		return WHITE

	distance = 0
	if stdev > 0:
		distance = (abs(rating - mean) / stdev)
	if distance < 2:
		distance = 0
	distance = distance * 0.75

	return rating_from_color_table( target, table, distance )

def title_format(s):
	if USE_LATEX:
		return s.replace('&', r'\&').replace("^", r'\^')
	return s

def to_plotly_color(color_tuple):
	if isinstance(color_tuple, str): return color_tuple # if it's already a hex string
	return f"rgb({int(color_tuple[0]*255)}, {int(color_tuple[1]*255)}, {int(color_tuple[2]*255)})"

def plot_sub_scatter(fig, stats, row, col):
	if DROP_Z_S:
		xticks = [("Bussin", 9), ("Mid Sheesh", 6), ("L", 4)] if ZOOMER else [("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4)]
		lo, hi = xticks[-1][1] - 1, xticks[0][1] + 1
	else:
		xticks = [("Bussin", 10), ("Mid Sheesh", 6), ("L", 3)] if ZOOMER else [("SSS", 10), ("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4), ("Z", 3)]
		lo, hi = xticks[-1][1], xticks[0][1]

	fig.update_xaxes(
		side="top",
		range=[lo, hi], tickvals=[t[1] for t in xticks], ticktext=[f"{t[0]} " for t in xticks],
		showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]),
		mirror="allticks", ticks="inside", ticklen=5, tickwidth=1.5, tickcolor=to_plotly_color(COLORS["LINE"]),
		showgrid=False, zeroline=False, row=row, col=col
	)
	for t_label, x_val in xticks:
		fig.add_annotation(
			x=x_val, y=0,
			text=f"{t_label} ",
			showarrow=False,
			yanchor="top",
			yshift=-10,
			font=dict(color=to_plotly_color(COLORS["TEXT"]), size=11),
			row=row, col=col
		)
	fig.update_yaxes(
		range=[0, len(stats)], showticklabels=False,
		showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]),
		mirror=True, showgrid=False, zeroline=False, row=row, col=col
	)

	# draw horizontal bands
	for y in range(len(stats)):
		col_band = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"]
		y_inv = len(stats) - y - 1 # flip
		fig.add_hrect(y0=y_inv, y1=y_inv+1, fillcolor=to_plotly_color(col_band), opacity=1, layer="below", line_width=0, row=row, col=col)

	# draw vertical lines
	for _, x in xticks:
		fig.add_vline(x=x, line_width=1.5, line_color=to_plotly_color(COLORS["LINE"]), layer="below", row=row, col=col)

	stats.reverse()

	# draw stat points
	all_xs, all_ys, all_colors, all_sizes, all_texts, all_customdata = [], [], [], [], [], []

	line_traces = {
		"mean": {"x": [], "y": [], "color": to_plotly_color(COLORS["MEAN"])},
		"mean_smart": {"x": [], "y": [], "color": to_plotly_color(COLORS["MEAN"])},
		"median": {"x": [], "y": [], "color": to_plotly_color(COLORS["MEDIAN"])},
		"stdev": {"x": [], "y": [], "color": to_plotly_color(COLORS["STDEV"])}
	}

	for y, stat in enumerate(stats):
		if stat is None: continue

		name = stat['name']
		for marker in stat["markers"]:
			if marker in MARKERS:
				name += MARKERS[marker]["tag"]

		xs = [ p[0] for p in stat['points'] ]
		ys = [ p[1] + y + 0.35 for p in stat['points'] ]

		if stat['points']:
			stdev_mult = 1.5

			min_x = stat['mean'] - (stat['stdev'] * stdev_mult)
			max_x = stat['mean'] + (stat['stdev'] * stdev_mult)

			pys = [p[1] for p in stat['points']]
			mean_y = statistics.mean(pys)
			stdev_y = statistics.pstdev(pys)

			if stdev_y == 0: stdev_y = 0.1

			min_y = mean_y - (stdev_y * stdev_mult)
			max_y = mean_y + (stdev_y * stdev_mult)
		else:
			min_x, max_x, min_y, max_y = 3.1, 9.9, -0.35, 0.35

		color_points = []
		for rating, (px, py) in zip(stat['ratings'], stat['points']):
			c = None

			# to-do: data-driven loop
			if "french" in stat["markers"]:
				c = sample_image("./assets/french.png", px, py, min_x, max_x, min_y, max_y)
			elif "ToT" in stat["markers"]:
				c = sample_image("./assets/crying.png", px, py, min_x, max_x, min_y, max_y)
			elif "trans rights" in stat["markers"]:
				c = sample_image("./assets/trans.png", px, py, min_x, max_x, min_y, max_y)
			elif "pajeet" in stat["markers"]:
				c = sample_image("./assets/india.png", px, py, min_x, max_x, min_y, max_y)

			if c is None:
				c = rating_to_color(rating, stat)

			color_points.append(to_plotly_color(c))

		all_xs.extend(xs)
		all_ys.extend(ys)
		all_colors.extend(color_points)
		all_sizes.extend([lerp(10, 4, stat['count']/60.0) * UI_SCALE] * len(xs))
		if "scores" in stat:
			all_texts.extend([f">>{no}<br>Rating: {r}<br>Click to view" for r, no in stat['scores']])
		all_customdata.extend(stat.get('urls', []))

		fig.add_annotation(x=3.1, y=y+0.8, text=name, showarrow=False, xanchor="left", yanchor="middle",font=dict(color=to_plotly_color(COLORS["TEXT"]), size=8 * UI_SCALE), row=row, col=col)
		fig.add_annotation(x=9.9, y=y+0.8, text=str(stat['count']), showarrow=False, xanchor="right", yanchor="middle",font=dict(color=to_plotly_color(COLORS["STATS"]), size=8 * UI_SCALE), row=row, col=col)

		if "mean" in LINES:
			line_traces["mean"]["x"].extend([stat['mean'], stat['mean'], None])
			line_traces["mean"]["y"].extend([y+0.1, y+0.9, None])
		if "mean_smart" in LINES:
			line_traces["mean_smart"]["x"].extend([stat['mean_smart'], stat['mean_smart'], None])
			line_traces["mean_smart"]["y"].extend([y+0.1, y+0.9, None])
		if "median" in LINES:
			line_traces["median"]["x"].extend([stat['median'], stat['median'], None])
			line_traces["median"]["y"].extend([y+0.1, y+0.9, None])
		if "stdev" in LINES:
			line_traces["stdev"]["x"].extend([stat["mean"] - stat["stdev"], stat["mean"] - stat["stdev"], None, stat["mean"] + stat["stdev"], stat["mean"] + stat["stdev"], None])
			line_traces["stdev"]["y"].extend([y+0.2, y+0.8, None, y+0.2, y+0.8, None])

	for key, data in line_traces.items():
		if data["x"]:
			fig.add_trace(go.Scatter(
				x=data["x"], y=data["y"], mode="lines",
				line=dict(color=data["color"], width=1.5),
				hoverinfo="skip", showlegend=False
			), row=row, col=col)

	fig.add_trace(
		go.Scatter(
			x=all_xs, y=all_ys, mode='markers',
			marker=dict(size=all_sizes, color=all_colors, line=dict(width=0)),
			hovertext=all_texts, hoverinfo="text",
			customdata=all_customdata,
			showlegend=False
		), row=row, col=col
	)

def plot_sub_boxplot(fig, stats, row, col):
	if DROP_Z_S:
		xticks = [("Bussin", 9), ("Mid Sheesh", 6), ("L", 4)] if ZOOMER else [("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4)]
		lo, hi = xticks[-1][1] - 1, xticks[0][1] + 1
	else:
		xticks = [("Bussin", 10), ("Mid Sheesh", 6), ("L", 3)] if ZOOMER else [("SSS", 10), ("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4), ("Z", 3)]
		lo, hi = xticks[-1][1], xticks[0][1]

	fig.update_xaxes(
		range=[lo, hi], tickvals=[t[1] for t in xticks], ticktext=[f"{t[0]} " for t in xticks],
		showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]),
		mirror="allticks", ticks="inside", ticklen=5, tickwidth=1.5, tickcolor=to_plotly_color(COLORS["LINE"]),
		showgrid=False, zeroline=False, row=row, col=col
	)
	fig.update_yaxes(
		range=[0, len(stats)], showticklabels=False,
		showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]),
		mirror=True, showgrid=False, zeroline=False, row=row, col=col
	)

	for y in range(len(stats)):
		col_band = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"]
		y_inv = len(stats) - y - 1
		fig.add_hrect(y0=y_inv, y1=y_inv+1, fillcolor=to_plotly_color(col_band), opacity=1, layer="below", line_width=0, row=row, col=col)

	for _, x in xticks:
		fig.add_vline(x=x, line_width=1.5, line_color=to_plotly_color(COLORS["LINE"]), layer="below", row=row, col=col)

	stats.reverse()

	for y, stat in enumerate(stats):
		if stat is None: continue

		name = stat['name']
		for marker in stat["markers"]:
			if marker in MARKERS and (not CULL_SINGLETON_MARKERS or MARKERS[marker]["count"] > 1):
				name += MARKERS[marker]["tag"]

		color_box = to_plotly_color(rating_from_color_table(stat['mean_smart'], COLORS["RATINGS"]))

		fig.add_trace(
			go.Box(
				x=stat['ratings'], y=[y + 0.4] * len(stat['ratings']),
				name=name, fillcolor=color_box, line=dict(color=to_plotly_color(COLORS["MEAN"])),
				boxmean=True, orientation='h', showlegend=False, hoverinfo="x+name"
			), row=row, col=col
		)

		fig.add_annotation(x=3.1, y=y+0.8, text=name, showarrow=False, xanchor="left", yanchor="middle", font=dict(color=to_plotly_color(COLORS["TEXT"]), size=8), row=row, col=col)
		fig.add_annotation(x=9.9, y=y+0.8, text=str(stat['count']), showarrow=False, xanchor="right", yanchor="middle", font=dict(color=to_plotly_color(COLORS["STATS"]), size=8), row=row, col=col)

def plot_sub_bars(fig, stats, row, col):
	fig.update_xaxes(
		showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]),
		mirror="allticks", ticks="inside", ticklen=5, tickwidth=1.5, tickcolor=to_plotly_color(COLORS["LINE"]),
		showgrid=False, zeroline=False, row=row, col=col
	)
	fig.update_yaxes(
		range=[0, len(stats)], showticklabels=False,
		showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]),
		mirror=True, showgrid=False, zeroline=False, row=row, col=col
	)

	for y in range(len(stats)):
		col_band = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"]
		y_inv = len(stats) - y - 1
		fig.add_hrect(y0=y_inv, y1=y_inv+1, fillcolor=to_plotly_color(col_band), opacity=1, layer="below", line_width=0, row=row, col=col)

	stats.reverse()
	ys = [0.5 + y for y in range(len(stats))]

	buckets = [(4, "F", "#73172d"), (5, "D", "#df3e23"), (6, "C", "#f9a31b"), (7, "B", "#fffc40"), (8, "A", "#9cdb43"), (9, "S", "#20d6c7")]

	for i, label, color in buckets:
		counts = [(0 if s is None else s['buckets'][i]) for s in stats]
		fig.add_trace(
			go.Bar(y=ys, x=counts, orientation='h', name=label, marker=dict(color=color), showlegend=(row==1 and col==1)),
			row=row, col=col
		)

	for y, stat in enumerate(stats):
		if stat is None: continue
		name = stat['name']
		for marker in stat["markers"]:
			if marker in MARKERS and (not CULL_SINGLETON_MARKERS or MARKERS[marker]["count"] > 1):
				name += MARKERS[marker]["tag"]

		fig.add_annotation(x=0.0, y=y+0.85, text=name, showarrow=False, xanchor="left", yanchor="middle", font=dict(color=to_plotly_color(COLORS["TEXT"]), size=8), row=row, col=col)

def plot_sub_pie(fig, stats, row, col):
	total_buckets = [0] * 10
	for stat in stats:
		if stat is None: continue
		for i in range(len(stat['buckets'])):
			total_buckets[i] += stat['buckets'][i]

	bucket_config = [
		(4, "F", "#73172d"), (5, "D", "#df3e23"), (6, "C", "#f9a31b"),
		(7, "B", "#fffc40"), (8, "A", "#9cdb43"), (9, "S", "#20d6c7"),
	]

	labels, sizes, colors = [], [], []
	for i, label, color in bucket_config:
		if total_buckets[i] > 0:
			labels.append(label)
			sizes.append(total_buckets[i])
			colors.append(color)

	if not sizes:
		fig.add_annotation(x=0.5, y=0.5, text="No Data", showarrow=False, row=row, col=col)
		return

	fig.add_trace(
		go.Pie(
			labels=labels,
			values=sizes,
			marker=dict(colors=colors),
			textinfo='percent',
			textfont=dict(color='black', weight='bold'),
			showlegend=True
		),
		row=row, col=col
	)

def create_plot( stats ):
	columns_data = []
	column_titles = []

	if SORT_BY is not None:
		stats = sorted(stats, key=lambda s: s[SORT_BY])

	if AUX_MODE == "day":
		days_dict = {}
		for stat in stats:
			st = stat.get("start_time", 0)
			if st > 0:
				day_str = datetime.fromtimestamp(st).strftime('%Y-%m-%d')
			else:
				day_str = "Unknown"

			if day_str not in days_dict:
				days_dict[day_str] = []
			days_dict[day_str].append(stat)

		sorted_days = sorted(days_dict.keys())
		cols = len(sorted_days) if sorted_days else 1
		items_per_col = max([len(days_dict[d]) for d in sorted_days]) if sorted_days else 1

		for day in sorted_days:
			sub_stats = days_dict[day]
			if SORT_BY is not None:
				sub_stats = sorted(sub_stats, key=lambda s: s[SORT_BY])

			sub_stats += [None] * (items_per_col - len(sub_stats))

			column_titles.append(day)
			columns_data.append(sub_stats)

	else:
		if SORT_BY is not None:
			stats = sorted(stats, key=lambda s: s[SORT_BY])

		cols = max(MIN_COLUMNS, min(5, round(math.sqrt(len(stats) / 6))))
		if cols == 0: cols = 1
		items_per_col = math.ceil(len(stats) / cols)

		stats += [None] * (cols * items_per_col - len(stats))

		for i in range(cols):
			column_titles.append(" ")
			columns_data.append(stats[i * items_per_col : (i + 1) * items_per_col])


	specs = [[{"type": "domain" if MODE == "pie" else "xy"} for _ in range(cols)]]
	fig = make_subplots(rows=1, cols=cols,specs=specs,horizontal_spacing=0.02)

	if USE_LEGEND:
		handles = []
		if "mean" in LINES or "mean_smart" in LINES:
			handles.append(('Mean', COLORS["MEAN"], True))
		if "median" in LINES:
			handles.append(('Median', COLORS["MEDIAN"], True))
		if "stdev" in LINES:
			handles.append(('Std. Dev.', COLORS["STDEV"], True))

		for marker, entry in MARKERS.items():
			tag = entry["tag"]
			count = entry["count"]
			if count <= (1 if CULL_SINGLETON_MARKERS else 0) or tag == "": continue

			m_text = entry.get("reverse", marker) if REVERSE and entry.get("reverse") else marker
			m_text = entry.get("zoomer", m_text) if ZOOMER and entry.get("zoomer") else m_text

			title = f'({count}) {m_text}{tag}'
			handles.append((title, COLORS["TEXT"], False))

		if handles:
			cols_count = 3 if len(handles) > 4 else (2 if len(handles) > 1 else 1)

			rows_html = []
			for r in range(math.ceil(len(handles) / cols_count)):
				row_items = []
				for c in range(cols_count):
					idx = r * cols_count + c
					if idx < len(handles):
						name, color, is_line = handles[idx]
						c_str = to_plotly_color(color)

						symbol = f"<span style='color:{c_str};'>■</span>" if is_line else " "

						name_padded = name.ljust(22).replace(" ", " ")
						row_items.append(f"{symbol} {name_padded}")

				rows_html.append("   ".join(row_items))

			legend_html = "<br>".join(rows_html)

			fig.add_annotation(
				text=legend_html,
				xref="paper", yref="paper",
				x=1.0, y=1.0,
				yshift=40 * UI_SCALE,
				xanchor="right", yanchor="bottom",
				align="left",
				font=dict(family="monospace", size=11 * UI_SCALE, color=to_plotly_color(COLORS["TEXT"])),
				bgcolor=to_plotly_color(COLORS["BAND0"]),
				bordercolor=to_plotly_color(COLORS["LINE"]),
				borderwidth=1.5,
				borderpad=8,
				showarrow=False
			)

	for i, stats_sub in enumerate(columns_data):
		col_idx = i + 1
		row_idx = 1

		title_text = column_titles[i]
		if title_text and title_text.strip():
			x_ref_str = "x domain" if col_idx == 1 else f"x{col_idx} domain"

			fig.add_annotation(
				text=title_text,
				xref=x_ref_str,
				yref="paper",
				x=0.5, y=1.0,
				yshift=20 * UI_SCALE,
				xanchor="center", yanchor="bottom",
				showarrow=False,
				font=dict(color=to_plotly_color(COLORS["TEXT"]), size=14 * UI_SCALE)
			)

		if MODE == "scatter":
			plot_sub_scatter(fig, stats_sub, row_idx, col_idx)
		elif MODE == "bars":
			plot_sub_bars(fig, stats_sub, row_idx, col_idx)
		elif MODE == "boxplot":
			plot_sub_boxplot(fig, stats_sub, row_idx, col_idx)
		elif MODE == "pie":
			plot_sub_pie(fig, stats_sub, row_idx, col_idx)
		else:
			raise Exception(f'invalid mode: {MODE}')

	plot_height = (items_per_col * 50 + 200) * UI_SCALE
	plot_width = (cols * 400 + 100) * UI_SCALE
	fig.update_layout(
		title=dict(
			text=f"{TITLE}<br><span style='font-size: 14px; color: {to_plotly_color(COLORS['STATS'])};'>{SUBTITLE}</span>" if 'SUBTITLE' in globals() else TITLE,
			x=0.5, xanchor='center', yanchor='top',
			font=dict(size=24 * UI_SCALE)
		),
		font=dict(size=12 * UI_SCALE),
		annotationdefaults=dict(font=dict(color=to_plotly_color(COLORS["TEXT"]), size=14 * UI_SCALE)),
		template="plotly_dark" if DARK else "plotly_white",
		plot_bgcolor=to_plotly_color(COLORS["BAND0"]),
		paper_bgcolor=to_plotly_color(COLORS["BAND0"]),
		height=plot_height,
		width=plot_width,
		margin=dict(t=160 * UI_SCALE, b=60 * UI_SCALE, l=20 * UI_SCALE, r=20 * UI_SCALE),
		barmode='stack'
	)

	"""
	fig.add_annotation(
		#text=f"https://git.ecker.tech/chart/{TITLE.replace(' ', '')}",
		text=f"https://chartfag.neocities.org/charts/",
		xref="paper", yref="paper",
		x=0.0,
		y=0.0,
		yshift=-40,
		xanchor="left", yanchor="top",
		showarrow=False,
		font=dict(color=to_plotly_color(COLORS["STATS"]), size=10)
	)
	"""

	return fig

def compute_mean_smart(ratings):
	ratings = sorted( ratings )
	
	cutoff = len(ratings) // 8
	if cutoff > 0:
		ratings = ratings[cutoff:-cutoff]
	
	return statistics.mean(ratings)

def stat_new(name, ratings, entry={}):
	buckets = [0] * 10
	for r in ratings:
		r = min(9, max(4, round(r)))
		buckets[r] += 1

	stat = {}
	stat['name'] = name
	stat['ratings'] = ratings
	stat['points'] = [rating_to_point(r, len(ratings)) for r in ratings]
	stat['buckets'] = buckets
	stat['count'] = len(ratings)
	stat['mean'] = statistics.mean(ratings) if ratings else 0.0
	stat['median'] = statistics.median(ratings) if ratings else 0.0
	stat['mode'] = statistics.mode(ratings) if ratings else 0.0
	stat['median_grouped'] = statistics.median_grouped(round(r) for r in ratings) if ratings else 0.0
	stat['stdev'] = statistics.pstdev(ratings) if ratings else 0.0
	stat['mean_smart'] = compute_mean_smart(ratings) if ratings else 0.0

	# attach extra data
	stat["markers"] = []
	if "marker" in entry:
		stat["markers"] = [ entry["marker"] ]
	elif "markers" in entry:
		stat["markers"] = entry["markers"]

	# fix up previously split markers
	for i, marker in enumerate(stat["markers"]):
		if marker in ["DNF", "invalid"]:
			stat["markers"][i] = "DNF/invalid"
		"""
		elif marker in ["trainwreck", "cringekino"]:
			stat["markers"][i] = "trainwreck/cringekino"
		elif marker in ["ad", "nonrun"]:
			stat["markers"][i] = "ad/nonrun"
		"""

	if "event" in entry:
		stat["event"] = entry["event"]

	if "urls" in entry:
		stat["urls"] = entry["urls"]
	
	if "scores" in entry:
		stat["scores"] = entry["scores"]

	if PRINT_STATS:
		print(stat_to_str(stat))

	return stat

# read from json
def read_stats(filename):
	stats = []
	aux = {
		"total": [],
		"markers": {},
	}

	data = json.load(open(filename, "r", encoding='utf-8'))

	for name, entry in data.items():
		ratings = []

		if "ignore" in entry and entry["ignore"]:
			continue

		op = entry["posts"][0].split("#p")[-1] if "posts" in entry and entry["posts"] else None
		start_time = entry["times"][op] if "times" in entry and op in entry["times"] else 0
		thread_url = entry["posts"][0].split("#")[0] if "posts" in entry and entry["posts"] else ""

		# convert lists to dicts (for old data)
		if isinstance( entry["ratings"], list ):
			entry["ratings"] = { f'{i}': r for i, r in enumerate(entry["ratings"]) }

		entry["urls"] = []
		entry["scores"] = []

		# parse ratings in each run
		for no, score in entry["ratings"].items():
			#cur_time = entry["times"][no] if "times" in entry and no in entry["times"] else 0
			if start_time > 0 and CUTOFF_SECONDS > 0:
				if no not in entry["times"]:
					continue

				cur_time = entry["times"][no]
				if cur_time - start_time > CUTOFF_SECONDS:
					print("Culled", name, no, score, cur_time - start_time)
					continue

			score = score.upper()
			
			# trim extraneous characters (for example: ZZZZZZZ) if not already a valid score
			if score not in SCORES:
				# check if SSSS / ZZZZ
				if score[:4] in SCORES:
					score = score[:4]
				# check if first letter would match instead (per-run ratings that just share the first letter but keep it as a whole word)
				elif score[:1] in SCORES:
					score = score[:1]
				else:
					score = score[:3]

			# cull anything not A-F
			if DROP_Z_S:
				if SCORES[score] <= SCORES["Z"]  or SCORES["S"] < SCORES[score]:
					continue

			# invalid score
			if score not in SCORES:
				continue

			# increment totals (I don't remember what I originally used this for)
			TOTAL_SCORES[score] += 1

			# score modifiers
			rating = SCORES[score]
			
			# randomize downward
			#if "randomize" in entry and CUTOFF_SECONDS == 0 and False:
			if "randomize" in entry:
				lo = SCORES[list(SCORES.keys())[0]]
				hi = SCORES[list(SCORES.keys())[-1]]
				
				if True: #if random.random() < entry["randomize"] and rating > hi:
					rating = random.uniform(lo, hi)
			
			if ("reverse" in entry and random.random() < entry["reverse"]) or REVERSE:
				rating = 10 - rating + 2.75

			ratings.append( rating )
			entry["scores"].append( ( score, no ) )
			entry["urls"].append( f"{thread_url}#p{no}" )
			aux["total"].append( rating )

		stat = stat_new(name, ratings, entry)
		stat["start_time"] = start_time - DATE_OFFSET

		if FILTER_BY and FILTER_BY not in {*stat["markers"]}:
			continue

		stats.append(stat)

		# flatten
		for marker in stat["markers"]:
			if marker not in MARKERS:
				continue
			MARKERS[marker]["count"] += 1

		# increment marker totals
		for marker in stat["markers"]:
			for rating in ratings:
				if marker not in aux["markers"]:
					aux["markers"][marker] = []
				aux["markers"][marker].append( rating )

		if not stat["markers"]:
			marker = "none"
			for rating in ratings:
				if marker not in aux["markers"]:
					aux["markers"][marker] = []
				aux["markers"][marker].append( rating )

	# show aggregate total
	if AUX_MODE == "total":
		stats = [ stat_new("total", aux["total"]) ]
	# aggregate by markers
	elif AUX_MODE == "markers":
		stats = [ stat_new(name, ratings) for name, ratings in aux["markers"].items() ]

	return stats

def main():
	# yuck
	global REVERSE
	global ZOOMER
	global DARK
	global USE_LEGEND
	global MIN_COLUMNS
	global CUTOFF_SECONDS
	global SORT_BY
	global FILTER_BY
	global MODE
	global AUX_MODE
	global OUT_FILE
	global PRINT_STATS

	parser = argparse.ArgumentParser("Charter")
	# actions
	parser.add_argument("--fetch", action="store_true", help="Fetch ratings from queue")
	parser.add_argument("--plot", action="store_true", help="Plot ratings from file")
	parser.add_argument("--dump", action="store_true", help="Dumps rating as metrics")
	parser.add_argument("--upload", action="store_true", help="Uploads to neocities")
	# queue-less args
	parser.add_argument("--url", type=str, default=None, help="Post link to fetch ratings from its replies")
	parser.add_argument("--name", type=str, default=None, help="Name of run for rating")
	parser.add_argument("--markers", type=str, default=None, help="Markers to attach to run rating")
	# modifiers
	parser.add_argument("--sort-by", type=str, default=None, help="Sort plotted ratings by this value")
	parser.add_argument("--filter-by", type=str, default=None, help="Filters for runs with this value")
	parser.add_argument("--mode", type=str, default=None, help="Additional modes (scatter | bars | boxplot)")
	parser.add_argument("--aux-mode", type=str, default=None, help="Additional modes (total | markers)")
	parser.add_argument("--cutoff-seconds", type=int, default=None, help="Ignore ratings made X seconds after the rating post")
	parser.add_argument("--cutoff-minutes", type=int, default=None, help="Ignore ratings made X minutes after the rating post")
	# lesser used modifiers
	parser.add_argument("--reverse", action="store_true", help="Reverses the ratings")
	parser.add_argument("--zoomer", action="store_true", help="Zoomer friendly scales")
	parser.add_argument("--light", action="store_true", help="Light mode")
	parser.add_argument("--no-legend", action="store_true", help="Hides the legend")
	parser.add_argument("--no-html", action="store_true", help="Skip saving an HTML copy")
	parser.add_argument("--copy", action="store_true", help="Saves a copy by the timestamp")

	args = parser.parse_args()
	
	if args.sort_by:
		SORT_BY = args.sort_by
	
	if args.filter_by:
		FILTER_BY = args.filter_by

	if args.aux_mode:
		AUX_MODE = args.aux_mode
		if AUX_MODE == "total":
			args.no_legend = True
			MIN_COLUMNS = 1

	modifiers = [ f'[{SORT_BY or AUX_MODE or "chronological"}]' ]
	
	if args.mode:
		MODE = args.mode
		modifiers.append(f'[type={MODE}]')
	
	if args.filter_by:
		modifiers.append(f'[filter={FILTER_BY}]')

	if args.cutoff_seconds:
		CUTOFF_SECONDS = args.cutoff_seconds
	elif args.cutoff_minutes:
		CUTOFF_SECONDS = args.cutoff_minutes * 60	
	if CUTOFF_SECONDS > 0:
		modifiers.append(f'[cutoff={CUTOFF_SECONDS//60}]')

	OUT_FILE = f'./images/ratings{"".join(modifiers)}.png'

	REVERSE = args.reverse
	ZOOMER = args.zoomer
	DARK = not args.light
	USE_LEGEND = not args.no_legend
	PRINT_STATS = args.dump

	if args.fetch:
		queue = []
		
		# inject to queue
		if args.url and args.name:
			queue = [{ "post": args.url, "name": args.name, "markers": args.markers.split(",") if args.markers else None }]

		fetch_ratings(queue=queue)

	if args.plot:
		stats = read_stats(IN_RATINGS_FILE)
		fig = create_plot(stats)

		if not args.no_html:
			click_js = """
			var myPlot = document.getElementsByClassName('plotly-graph-div')[0];

			myPlot.on('plotly_click', function(data){
				if (data.points && data.points.length > 0) {
					var url = data.points[0].customdata;
					if(url && typeof url === 'string') {
						window.open(url, '_blank');
					}
				}
			});

			myPlot.on('plotly_hover', function(data){
				if (data.points && data.points.length > 0 && data.points[0].customdata) {
					myPlot.style.cursor = 'pointer';
				}
			});

			myPlot.on('plotly_unhover', function(data){
				myPlot.style.cursor = 'default';
			});
			"""

			html_file = OUT_FILE.replace('.png', '.html')
			html_str = fig.to_html(full_html=True, post_script=click_js)

			css_injection = f"<style>body {{ background-color: {COLORS['BACKGROUND']}; margin: 0; padding: 20px; text-align: center; }}</style>"
			html_str = html_str.replace("<head>", f"<head>\n\t{css_injection}")

			with open(html_file, "w", encoding="utf-8") as f:
				f.write(html_str)

			"""
			if args.upload:
				import neocities
				nc = neocities.NeoCities(api_key=YOUR_API_KEY)
				nc.upload((html_file, 'charts/index.html'))
			"""

		if args.copy:
			print(f'Saving chart copy: {OUT_FILE_TIMESTAMP}')
			fig.write_image(OUT_FILE_TIMESTAMP, scale=1.0)

		print(f'Saving image chart: {OUT_FILE}')
		fig.write_image(OUT_FILE, scale=2.0)

	if not args.fetch and not args.plot:
		raise Exception("Specify --fetch or --plot")


if __name__ == "__main__":
	main()