From 6c23eb409f9ebfeebbdc1ab1550dc2effe1d479e Mon Sep 17 00:00:00 2001 From: mrq Date: Sun, 12 Jul 2026 07:38:18 +0000 Subject: [PATCH] updated script (for HTML support) --- README.md | 2 +- chart.py | 1267 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1268 insertions(+), 1 deletion(-) create mode 100755 chart.py diff --git a/README.md b/README.md index 3a6a2da..0846f4e 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -When linking, please use `https://chartfag.neocities.org/` (this domain gives an automatic 90-day ban when posted). +When linking, please use [`https://chartfag.neocities.org/`](https://chartfag.neocities.org/), as this domain (`ecker.tech`) gives an automatic 90-day ban when posted). ## Requirements diff --git a/chart.py b/chart.py new file mode 100755 index 0000000..1d73004 --- /dev/null +++ b/chart.py @@ -0,0 +1,1267 @@ +import math +import random +import statistics +import time +import json +import os.path +import argparse +import sys +import urllib.request +import re + +from datetime import datetime +from html.parser import HTMLParser + +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from PIL import Image + +# consts +TITLE = "SGDQ 2026" +SUBTITLE = "as rated by /v/" +USE_LEGEND = False # display the legend with markers +TITLE_OFFSET = 0 # 2.5 # to-do: dynamically set this to how many columns are set with legend +UI_SCALE = 1.0 +DATE_OFFSET = (60 * 60 * 8) +# les constant consts +AUX_MODE = None # total | markers | None +SORT_BY = None # sort runs by the values +CUTOFF_SECONDS = 0 # 60 * 5 +MODE = "scatter" +FILTER_BY = None # filters by markers +PRINT_STATS = False + +# more constant consts +TIMESTAMP = str(int(time.time_ns()/1000/1000)) + +IN_QUEUE_FILE = "./data/queue.json" +IN_RATINGS_FILE = "./data/ratings.json" +OUT_FILE_TIMESTAMP = f'./images/{TIMESTAMP}.png' +OUT_FILE = f'./images/ratings[{SORT_BY or AUX_MODE or "chronological"}].png' + +MIN_COLUMNS = 2 # looks better if there's more than one column with the legend +CULL_SINGLETON_MARKERS = False # remove any marker that only has 1 entry +COLOR_BY = "mean" # color by this stat's value +FADE_BY_STDEV = True # fade outliers +LINES = ["mean_smart", "median"] # show mean and median lines (or stdev too) +USE_LATEX = False # never got this to work +DROP_Z_S = False + +RE_RATING = re.compile(r"(\b[A-HJ-Zz]+(?:[+-]|\b))") +RE_RATING_NEWLINE = re.compile(r"(\b[A-HJ-Zz]+(?:[+-]|\b))\n") # just the above, but for newlines + +IMAGE_CACHE = {} + +COL_WIDTH = 4.0 +ROW_HEIGHT = 0.5 + +# gimmicks +REVERSE = False # classic reverse tier lists +ZOOMER = False # sets scale to zoomer-friendly ratings + +# theme colors +DARK = True +COLORS = { + "BACKGROUND": "#0c0c0c", + "BAND0": "#101010", + "BAND1": "#080808", + "LINE": "#181818", + "MEAN": "#404040", + "STDEV": "#404040", + "TEXT": "#e0e0e0", + "STATS": "#606060", + "MEDIAN": "#AA00AA", + "RATINGS": [ + ( 0.0, (0.2, 0.2, 0.2)), + ( 3.5, (0.6, 0.2, 0.2)), + ( 4.5, (1.0, 0.2, 0.2)), + ( 6.5, (1.0, 1.0, 0.2)), + ( 7.5, (0.2, 1.0, 0.2)), + ( 8.5, (0.2, 1.0, 1.0)), + (10.0, (1.0, 1.0, 1.0)), + ], + "SPECIAL": (0.6, 0.2, 1.0), + "BROWN": (139.0/255.0,69.0/255.0,19.0/255.0), + "BARBIE": (244.0/255.0, 33.0/255.0, 138.0/255.0), + "ROSE AND CAMELLIA": (0.8705882352941177, 0.6313725490196078, 0.5764705882352941), +} if DARK else { + "BACKGROUND": "#ffffff", + "BAND0": "#f0f0f0", + "BAND1": "#f8f8f8", + "LINE": "#e8e8e8", + "MEAN": "#404040", + "STDEV": "#404040", + "TEXT": "#404040", + "STATS": "#a0a0a0", + "MEDIAN": "#AA00AA", + "RATINGS": [ + ( 0.0, (0.0, 0.0, 0.0)), + ( 3.5, (0.4, 0.0, 0.0)), + ( 4.5, (0.9, 0.0, 0.0)), + ( 6.5, (0.8, 0.8, 0.0)), + ( 7.5, (0.0, 0.9, 0.0)), + ( 8.5, (0.0, 0.8, 0.8)), + (10.0, (0.7, 0.7, 0.7)), + ], + "SPECIAL": (0.4, 0.0, 0.8), + "BROWN": (139.0/255.0,69.0/255.0,19.0/255.0), + "BARBIE": (244.0/255.0, 33.0/255.0, 138.0/255.0), + "ROSE AND CAMELLIA": (0.8705882352941177, 0.6313725490196078, 0.5764705882352941), +} + +""" +if DARK: + Plot.style.use("dark_background") +""" + +SCORES = { + "K": 0, "T": 0, + "ZZZZ": 2.9, + "ZZZ-": 3.0, + "ZZZ": 3.1, "ZZ": 3.15, "Z-": 3.2, "Z": 3.25, "DNF": 3.25, + "L": 3.25, "N": 3.25, + "FFF": 3.5, "FF": 3.7, + "F-": 3.7, "F": 4.0, "F+": 4.3, "E": 4.5, + "D-": 4.7, "D": 5.0, "D+": 5.3, + "C-": 5.7, "C": 6.0, "C+": 6.3, + "B-": 6.7, "B": 7.0, "B+": 7.3, + "A-": 7.7, "A": 8.0, "A+": 8.3, + "S-": 8.7, "S": 9.0, "P": 9.0, "S+": 9.3, + "SS": 9.3, "SSS": 9.5, "W": 9.7, + "SSS+": 9.8, + "SSSS": 9.9, + "HH": 10 +} +TOTAL_SCORES = { name: 0 for name in SCORES.keys() } + +if SORT_BY is not None: + COLOR_BY = SORT_BY + +# runtime globals +THREADS = {} + +MARKERS = {} +REVERSE_MARKERS = {} + +def add_marker( name, tag, color=COLORS["TEXT"], reverse=None, zoomer=None ): + MARKERS[name] = { + "tag": tag, + "color": color, + "count": 0, + "reverse": reverse, + "zoomer": zoomer, + } + +add_marker("girl", tag="!", reverse="femcel", zoomer="gyatt") +add_marker("foid", tag="...", reverse="sex worker", zoomer="skibidi") +add_marker("tranny", tag="*", reverse="real woman", zoomer="fr") +add_marker("ladyboy", tag="¿", reverse="tomboy") +add_marker("black", tag="&", reverse="white", zoomer="kang") +add_marker("biohazard", tag="#") # fatales +add_marker("male", tag="♂") # fatales +add_marker("female", tag="♀") # fatales +add_marker("BOOBS", tag="( Y )") # memetic +add_marker("vt", tag="^") # rarely used +add_marker("amogus", tag=" sus") # rarely used +add_marker("race", tag="@") # unused now +add_marker("savestated", tag="\\") # rarely used, should be invalid now +#add_marker("trainwreck/cringekino", tag="%", reverse="flawless", zoomer="fanum tax") +add_marker("trainwreck", tag="%", reverse="flawless", zoomer="fanum tax") +add_marker("cringekino", tag="%", reverse="kinocringe", zoomer="fanum tax") +add_marker("DNF/invalid", tag="$", reverse="WR", zoomer="ohio") +add_marker("overestimate", tag=">", reverse="underestimate", zoomer="💀") +add_marker("ad", tag="✡", reverse="organic", zoomer="sigma") +add_marker("nonrun", tag="@", reverse="letsplay", zoomer="content") +add_marker("neporun", tag="\\", reverse="indie", zoomer="collab") +add_marker("it", tag="※", reverse="human", zoomer="it") +#add_marker("ad/nonrun", tag="✡", reverse="organic", zoomer="sigma") + +def sample_image(image_path, px, py_local, min_x, max_x, min_y, max_y): + if image_path not in IMAGE_CACHE: + try: + IMAGE_CACHE[image_path] = Image.open(image_path).convert('RGB') + except Exception as e: + print(f"Could not load {image_path}: {e}") + IMAGE_CACHE[image_path] = None + + img = IMAGE_CACHE[image_path] + if img is None: + return None + + range_x = max_x - min_x if max_x > min_x else 1.0 + range_y = max_y - min_y if max_y > min_y else 1.0 + + u = clamp((px - min_x) / range_x) + v = clamp((py_local - min_y) / range_y) + + ix = int(u * (img.width - 1)) + iy = int((1.0 - v) * (img.height - 1)) + + r, g, b = img.getpixel((ix, iy)) + return (r / 255.0, g / 255.0, b / 255.0) + +# yuck +class MyHTMLParser(HTMLParser): + def __init__(self): + self.str = "" + super().__init__() + + def handle_starttag(self, tag, attrs): + if tag == "br": + self.str += "\n" + + def handle_data(self, data): + self.str += data + +def curl(url): + if url in THREADS: + return THREADS[url] + try: + conn = urllib.request.urlopen(url) + data = conn.read() + data = data.decode() + data = json.loads(data) + conn.close() + THREADS[url] = data + return data + except: + return None + +# ick +def parse_rating(comment): + matches = RE_RATING_NEWLINE.findall(comment) + binned = re.sub(r'>>\d+\n', "", comment) + + if len(matches) == 1: + match = matches[0] + if match[0] in "SABCDEFNTZKsabcdefntzk": + return match, None + + matches = RE_RATING.findall(comment) + if len(matches) <= 0: + return None, binned + if len(matches) >= 2: + return None, binned + + match = matches[0] + if match[0] not in "SABCDEFNTZKsabcdefntzk": + return None, binned + + return match, None + +def fetch_ratings( queue=[] ): + try: + ratings = json.load(open(IN_RATINGS_FILE, "r", encoding='utf-8')) + except Exception as e: + ratings = {} + raise e + + if not queue: + try: + queue = json.load(open(IN_QUEUE_FILE, "r", encoding='utf-8')) + except Exception as e: + print(str(e)) + pass + + for entry in queue: + name = entry["name"] + post = entry["post"] + + if post == "" or post is None: + continue + + url, _, post_no = post.partition("#p") + prefix, __, thread = url.partition("/thread/") + + # remove training slash + if thread[-1] == "/": + thread = thread[:-1] + + # pick API + """ + if "p.ecker.tech" in prefix: + url = f'https://p.ecker.tech/chan/browse/v/thread/{thread}/?update' + else: + url = f'https://a.4cdn.org/v/thread/{thread}.json' + """ + url = f'https://a.4cdn.org/v/thread/{thread}.json' + + post_no = int(post_no) + + if name not in ratings: + ratings[name] = {} + + # coerse marker => markers + if "marker" in entry and "markers" not in entry: + entry["markers"] = [ entry["marker"] ] + del entry["marker"] + + # in case an entry is already defined without these + if "markers" in entry: + ratings[name]["markers"] = entry["markers"] + + if "posts" not in ratings[name]: + ratings[name]["posts"] = [] + + if post not in ratings[name]["posts"]: + ratings[name]["posts"].append(post) + + if "ratings" not in ratings[name]: + ratings[name]["ratings"] = {} + + if "binned" not in ratings[name]: + ratings[name]["binned"] = {} + + if "times" not in ratings[name]: + ratings[name]["times"] = {} + + print(f"Fetching {name}: {post}") + + data = curl(url) + if data is None: + print(f"404: {name}: {post}") + continue + + re_link = re.compile(f">>>{post_no}<") + + # coerce to array + if isinstance(data["posts"], dict): + data["posts"] = data["posts"].values() + + for post in data["posts"]: + if "com" not in post: + continue + + if "no" not in post: + continue + + com = post["com"] + no = str(post["no"]) + time = post["time"] + + match = re_link.search(com) + + if match is None and f'{no}' != f'{post_no}': + continue + + if no not in ratings[name]["times"]: + ratings[name]["times"][no] = time + + if no in ratings[name]["ratings"]: + continue + + if no in ratings[name]["binned"]: + continue + + parser = MyHTMLParser() + parser.feed(com) + com = parser.str + + rating, binned = parse_rating(com) + if rating is not None: + ratings[name]["ratings"][no] = rating + else: + ratings[name]["binned"][no] = binned + + for no in ratings[name]["ratings"]: + if no not in ratings[name]["times"]: + print("MISSING:", name, no) + + json.dump(ratings, open(IN_RATINGS_FILE, "w"), indent='\t') + + json.dump(ratings, open(IN_RATINGS_FILE, "w"), indent='\t') + +def stat_to_str(stat): + l = [f"{stat['name']:25} ::"] + l.append(f"n = {stat['count']:2},") + l.append(f"mean = {stat['mean']:.2f},") + l.append(f"ms = {stat['mean_smart']:.2f},") + l.append(f"sd = {stat['stdev']:.2f},") + l.append(f"med = {stat['median']:.1f}") + return " ".join(l) + +# Plot related +def rating_to_point(rating, count, RAND_PT_DX=1/5, RAND_PT_DY=1/10): + dx = random.gauss(0, RAND_PT_DX) + dy = random.gauss(0, RAND_PT_DY) + + if count < 6: + dx = dx * (math.sqrt(count) / 6) + + px = abs(rating + dx) + if rating >= 3.5: + if px < 3.5 or px > 9.5: + px = abs(rating + dx / 2) + else: + px = abs(rating - dx / 2) + + # clamp + if px <= 3.1: + px = 3.1 + abs(dx) / 2 + dy = dy * 1.125 + + if px > 9.9: + px = 9.9 - abs(dx) / 2 + dy = dy * 1.125 + + py = dy + return px, py + +def clamp(x, lo=0, hi=1): + if x < lo: + return lo + if x > hi: + return hi + return x + +def li(v0, v1, a): + return v0 * (1- clamp(a)) + v1 * a + +def lic(c0, c1, a): + return tuple(li(v0,v1,a) for v0,v1 in zip(c0,c1)) + +def lerp( a, b, t ): + return a + (b - a) * clamp(t) + +# interpolate between the closest defined points +def rating_from_color_table( rating, table, distance=0 ): + for i in range(len(table)-1): + t0, t1 = table[i], table[i+1] + if t0[0] <= rating <= t1[0]: + alpha = (rating-t0[0]) / (t1[0]-t0[0]) + c = lic(t0[1], t1[1], alpha) + r = c[0] / (1 + distance) + g = c[1] / (1 + distance) + b = c[2] / (1 + distance) + return (r, g, b) + + return (1.0, 0.0, 1.0) + +def rating_to_color(rating, stat): + target = stat[COLOR_BY] + mean = stat['mean'] + stdev = stat['stdev'] + markers = stat['markers'] + + table = COLORS["RATINGS"] + + if "kino" in markers: + return COLORS["SPECIAL"] + + if "shart" in markers: + return COLORS["BROWN"] + + if "french" in markers: + RED = (1.0, 0.0, 0.0) + WHITE = (1.0, 1.0, 1.0) + BLUE = (0.0, 0.0, 1.0) + + distance = rating - mean / stdev + + if distance < -1.0: + return BLUE + if distance > 0.5: + return RED + return WHITE + + if "trans rights" in markers: + #F5A9B8 + BLUE = (0.3569, 0.8078, 0.9804) + PINK = (0.9608, 0.6627, 0.7216) + WHITE = (1.0, 1.0, 1.0) + + distance = rating # - mean / stdev + + if distance < 4.0: + return BLUE + if distance < 4.5: + return PINK + if distance > 7.0: + return BLUE + if distance > 6.0: + return PINK + + return WHITE + + distance = 0 + if stdev > 0: + distance = (abs(rating - mean) / stdev) + if distance < 2: + distance = 0 + distance = distance * 0.75 + + return rating_from_color_table( target, table, distance ) + +def title_format(s): + if USE_LATEX: + return s.replace('&', r'\&').replace("^", r'\^') + return s + +def to_plotly_color(color_tuple): + if isinstance(color_tuple, str): return color_tuple # if it's already a hex string + return f"rgb({int(color_tuple[0]*255)}, {int(color_tuple[1]*255)}, {int(color_tuple[2]*255)})" + +def plot_sub_scatter(fig, stats, row, col): + if DROP_Z_S: + xticks = [("Bussin", 9), ("Mid Sheesh", 6), ("L", 4)] if ZOOMER else [("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4)] + lo, hi = xticks[-1][1] - 1, xticks[0][1] + 1 + else: + xticks = [("Bussin", 10), ("Mid Sheesh", 6), ("L", 3)] if ZOOMER else [("SSS", 10), ("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4), ("Z", 3)] + lo, hi = xticks[-1][1], xticks[0][1] + + fig.update_xaxes( + side="top", + range=[lo, hi], tickvals=[t[1] for t in xticks], ticktext=[f"{t[0]} " for t in xticks], + showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]), + mirror="allticks", ticks="inside", ticklen=5, tickwidth=1.5, tickcolor=to_plotly_color(COLORS["LINE"]), + showgrid=False, zeroline=False, row=row, col=col + ) + for t_label, x_val in xticks: + fig.add_annotation( + x=x_val, y=0, + text=f"{t_label} ", + showarrow=False, + yanchor="top", + yshift=-10, + font=dict(color=to_plotly_color(COLORS["TEXT"]), size=11), + row=row, col=col + ) + fig.update_yaxes( + range=[0, len(stats)], showticklabels=False, + showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]), + mirror=True, showgrid=False, zeroline=False, row=row, col=col + ) + + # draw horizontal bands + for y in range(len(stats)): + col_band = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"] + y_inv = len(stats) - y - 1 # flip + fig.add_hrect(y0=y_inv, y1=y_inv+1, fillcolor=to_plotly_color(col_band), opacity=1, layer="below", line_width=0, row=row, col=col) + + # draw vertical lines + for _, x in xticks: + fig.add_vline(x=x, line_width=1.5, line_color=to_plotly_color(COLORS["LINE"]), layer="below", row=row, col=col) + + stats.reverse() + + # draw stat points + all_xs, all_ys, all_colors, all_sizes, all_texts, all_customdata = [], [], [], [], [], [] + + line_traces = { + "mean": {"x": [], "y": [], "color": to_plotly_color(COLORS["MEAN"])}, + "mean_smart": {"x": [], "y": [], "color": to_plotly_color(COLORS["MEAN"])}, + "median": {"x": [], "y": [], "color": to_plotly_color(COLORS["MEDIAN"])}, + "stdev": {"x": [], "y": [], "color": to_plotly_color(COLORS["STDEV"])} + } + + for y, stat in enumerate(stats): + if stat is None: continue + + name = stat['name'] + for marker in stat["markers"]: + if marker in MARKERS: + name += MARKERS[marker]["tag"] + + xs = [ p[0] for p in stat['points'] ] + ys = [ p[1] + y + 0.35 for p in stat['points'] ] + + if stat['points']: + stdev_mult = 1.5 + + min_x = stat['mean'] - (stat['stdev'] * stdev_mult) + max_x = stat['mean'] + (stat['stdev'] * stdev_mult) + + pys = [p[1] for p in stat['points']] + mean_y = statistics.mean(pys) + stdev_y = statistics.pstdev(pys) + + if stdev_y == 0: stdev_y = 0.1 + + min_y = mean_y - (stdev_y * stdev_mult) + max_y = mean_y + (stdev_y * stdev_mult) + else: + min_x, max_x, min_y, max_y = 3.1, 9.9, -0.35, 0.35 + + color_points = [] + for rating, (px, py) in zip(stat['ratings'], stat['points']): + c = None + + # to-do: data-driven loop + if "french" in stat["markers"]: + c = sample_image("./assets/french.png", px, py, min_x, max_x, min_y, max_y) + elif "ToT" in stat["markers"]: + c = sample_image("./assets/crying.png", px, py, min_x, max_x, min_y, max_y) + elif "trans rights" in stat["markers"]: + c = sample_image("./assets/trans.png", px, py, min_x, max_x, min_y, max_y) + elif "pajeet" in stat["markers"]: + c = sample_image("./assets/india.png", px, py, min_x, max_x, min_y, max_y) + + if c is None: + c = rating_to_color(rating, stat) + + color_points.append(to_plotly_color(c)) + + all_xs.extend(xs) + all_ys.extend(ys) + all_colors.extend(color_points) + all_sizes.extend([lerp(10, 4, stat['count']/60.0) * UI_SCALE] * len(xs)) + if "scores" in stat: + all_texts.extend([f">>{no}
Rating: {r}
Click to view" for r, no in stat['scores']]) + all_customdata.extend(stat.get('urls', [])) + + fig.add_annotation(x=3.1, y=y+0.8, text=name, showarrow=False, xanchor="left", yanchor="middle",font=dict(color=to_plotly_color(COLORS["TEXT"]), size=8 * UI_SCALE), row=row, col=col) + fig.add_annotation(x=9.9, y=y+0.8, text=str(stat['count']), showarrow=False, xanchor="right", yanchor="middle",font=dict(color=to_plotly_color(COLORS["STATS"]), size=8 * UI_SCALE), row=row, col=col) + + if "mean" in LINES: + line_traces["mean"]["x"].extend([stat['mean'], stat['mean'], None]) + line_traces["mean"]["y"].extend([y+0.1, y+0.9, None]) + if "mean_smart" in LINES: + line_traces["mean_smart"]["x"].extend([stat['mean_smart'], stat['mean_smart'], None]) + line_traces["mean_smart"]["y"].extend([y+0.1, y+0.9, None]) + if "median" in LINES: + line_traces["median"]["x"].extend([stat['median'], stat['median'], None]) + line_traces["median"]["y"].extend([y+0.1, y+0.9, None]) + if "stdev" in LINES: + line_traces["stdev"]["x"].extend([stat["mean"] - stat["stdev"], stat["mean"] - stat["stdev"], None, stat["mean"] + stat["stdev"], stat["mean"] + stat["stdev"], None]) + line_traces["stdev"]["y"].extend([y+0.2, y+0.8, None, y+0.2, y+0.8, None]) + + for key, data in line_traces.items(): + if data["x"]: + fig.add_trace(go.Scatter( + x=data["x"], y=data["y"], mode="lines", + line=dict(color=data["color"], width=1.5), + hoverinfo="skip", showlegend=False + ), row=row, col=col) + + fig.add_trace( + go.Scatter( + x=all_xs, y=all_ys, mode='markers', + marker=dict(size=all_sizes, color=all_colors, line=dict(width=0)), + hovertext=all_texts, hoverinfo="text", + customdata=all_customdata, + showlegend=False + ), row=row, col=col + ) + +def plot_sub_boxplot(fig, stats, row, col): + if DROP_Z_S: + xticks = [("Bussin", 9), ("Mid Sheesh", 6), ("L", 4)] if ZOOMER else [("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4)] + lo, hi = xticks[-1][1] - 1, xticks[0][1] + 1 + else: + xticks = [("Bussin", 10), ("Mid Sheesh", 6), ("L", 3)] if ZOOMER else [("SSS", 10), ("S", 9), ("A", 8), ("B", 7), ("C", 6), ("D", 5), ("F", 4), ("Z", 3)] + lo, hi = xticks[-1][1], xticks[0][1] + + fig.update_xaxes( + range=[lo, hi], tickvals=[t[1] for t in xticks], ticktext=[f"{t[0]} " for t in xticks], + showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]), + mirror="allticks", ticks="inside", ticklen=5, tickwidth=1.5, tickcolor=to_plotly_color(COLORS["LINE"]), + showgrid=False, zeroline=False, row=row, col=col + ) + fig.update_yaxes( + range=[0, len(stats)], showticklabels=False, + showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]), + mirror=True, showgrid=False, zeroline=False, row=row, col=col + ) + + for y in range(len(stats)): + col_band = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"] + y_inv = len(stats) - y - 1 + fig.add_hrect(y0=y_inv, y1=y_inv+1, fillcolor=to_plotly_color(col_band), opacity=1, layer="below", line_width=0, row=row, col=col) + + for _, x in xticks: + fig.add_vline(x=x, line_width=1.5, line_color=to_plotly_color(COLORS["LINE"]), layer="below", row=row, col=col) + + stats.reverse() + + for y, stat in enumerate(stats): + if stat is None: continue + + name = stat['name'] + for marker in stat["markers"]: + if marker in MARKERS and (not CULL_SINGLETON_MARKERS or MARKERS[marker]["count"] > 1): + name += MARKERS[marker]["tag"] + + color_box = to_plotly_color(rating_from_color_table(stat['mean_smart'], COLORS["RATINGS"])) + + fig.add_trace( + go.Box( + x=stat['ratings'], y=[y + 0.4] * len(stat['ratings']), + name=name, fillcolor=color_box, line=dict(color=to_plotly_color(COLORS["MEAN"])), + boxmean=True, orientation='h', showlegend=False, hoverinfo="x+name" + ), row=row, col=col + ) + + fig.add_annotation(x=3.1, y=y+0.8, text=name, showarrow=False, xanchor="left", yanchor="middle", font=dict(color=to_plotly_color(COLORS["TEXT"]), size=8), row=row, col=col) + fig.add_annotation(x=9.9, y=y+0.8, text=str(stat['count']), showarrow=False, xanchor="right", yanchor="middle", font=dict(color=to_plotly_color(COLORS["STATS"]), size=8), row=row, col=col) + +def plot_sub_bars(fig, stats, row, col): + fig.update_xaxes( + showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]), + mirror="allticks", ticks="inside", ticklen=5, tickwidth=1.5, tickcolor=to_plotly_color(COLORS["LINE"]), + showgrid=False, zeroline=False, row=row, col=col + ) + fig.update_yaxes( + range=[0, len(stats)], showticklabels=False, + showline=True, linewidth=1.5, linecolor=to_plotly_color(COLORS["LINE"]), + mirror=True, showgrid=False, zeroline=False, row=row, col=col + ) + + for y in range(len(stats)): + col_band = COLORS["BAND0"] if y % 2 == 0 else COLORS["BAND1"] + y_inv = len(stats) - y - 1 + fig.add_hrect(y0=y_inv, y1=y_inv+1, fillcolor=to_plotly_color(col_band), opacity=1, layer="below", line_width=0, row=row, col=col) + + stats.reverse() + ys = [0.5 + y for y in range(len(stats))] + + buckets = [(4, "F", "#73172d"), (5, "D", "#df3e23"), (6, "C", "#f9a31b"), (7, "B", "#fffc40"), (8, "A", "#9cdb43"), (9, "S", "#20d6c7")] + + for i, label, color in buckets: + counts = [(0 if s is None else s['buckets'][i]) for s in stats] + fig.add_trace( + go.Bar(y=ys, x=counts, orientation='h', name=label, marker=dict(color=color), showlegend=(row==1 and col==1)), + row=row, col=col + ) + + for y, stat in enumerate(stats): + if stat is None: continue + name = stat['name'] + for marker in stat["markers"]: + if marker in MARKERS and (not CULL_SINGLETON_MARKERS or MARKERS[marker]["count"] > 1): + name += MARKERS[marker]["tag"] + + fig.add_annotation(x=0.0, y=y+0.85, text=name, showarrow=False, xanchor="left", yanchor="middle", font=dict(color=to_plotly_color(COLORS["TEXT"]), size=8), row=row, col=col) + +def plot_sub_pie(fig, stats, row, col): + total_buckets = [0] * 10 + for stat in stats: + if stat is None: continue + for i in range(len(stat['buckets'])): + total_buckets[i] += stat['buckets'][i] + + bucket_config = [ + (4, "F", "#73172d"), (5, "D", "#df3e23"), (6, "C", "#f9a31b"), + (7, "B", "#fffc40"), (8, "A", "#9cdb43"), (9, "S", "#20d6c7"), + ] + + labels, sizes, colors = [], [], [] + for i, label, color in bucket_config: + if total_buckets[i] > 0: + labels.append(label) + sizes.append(total_buckets[i]) + colors.append(color) + + if not sizes: + fig.add_annotation(x=0.5, y=0.5, text="No Data", showarrow=False, row=row, col=col) + return + + fig.add_trace( + go.Pie( + labels=labels, + values=sizes, + marker=dict(colors=colors), + textinfo='percent', + textfont=dict(color='black', weight='bold'), + showlegend=True + ), + row=row, col=col + ) + +def create_plot( stats ): + columns_data = [] + column_titles = [] + + if SORT_BY is not None: + stats = sorted(stats, key=lambda s: s[SORT_BY]) + + if AUX_MODE == "day": + days_dict = {} + for stat in stats: + st = stat.get("start_time", 0) + if st > 0: + day_str = datetime.fromtimestamp(st).strftime('%Y-%m-%d') + else: + day_str = "Unknown" + + if day_str not in days_dict: + days_dict[day_str] = [] + days_dict[day_str].append(stat) + + sorted_days = sorted(days_dict.keys()) + cols = len(sorted_days) if sorted_days else 1 + items_per_col = max([len(days_dict[d]) for d in sorted_days]) if sorted_days else 1 + + for day in sorted_days: + sub_stats = days_dict[day] + if SORT_BY is not None: + sub_stats = sorted(sub_stats, key=lambda s: s[SORT_BY]) + + sub_stats += [None] * (items_per_col - len(sub_stats)) + + column_titles.append(day) + columns_data.append(sub_stats) + + else: + if SORT_BY is not None: + stats = sorted(stats, key=lambda s: s[SORT_BY]) + + cols = max(MIN_COLUMNS, min(5, round(math.sqrt(len(stats) / 6)))) + if cols == 0: cols = 1 + items_per_col = math.ceil(len(stats) / cols) + + stats += [None] * (cols * items_per_col - len(stats)) + + for i in range(cols): + column_titles.append(" ") + columns_data.append(stats[i * items_per_col : (i + 1) * items_per_col]) + + + specs = [[{"type": "domain" if MODE == "pie" else "xy"} for _ in range(cols)]] + fig = make_subplots(rows=1, cols=cols,specs=specs,horizontal_spacing=0.02) + + if USE_LEGEND: + handles = [] + if "mean" in LINES or "mean_smart" in LINES: + handles.append(('Mean', COLORS["MEAN"], True)) + if "median" in LINES: + handles.append(('Median', COLORS["MEDIAN"], True)) + if "stdev" in LINES: + handles.append(('Std. Dev.', COLORS["STDEV"], True)) + + for marker, entry in MARKERS.items(): + tag = entry["tag"] + count = entry["count"] + if count <= (1 if CULL_SINGLETON_MARKERS else 0) or tag == "": continue + + m_text = entry.get("reverse", marker) if REVERSE and entry.get("reverse") else marker + m_text = entry.get("zoomer", m_text) if ZOOMER and entry.get("zoomer") else m_text + + title = f'({count}) {m_text}{tag}' + handles.append((title, COLORS["TEXT"], False)) + + if handles: + cols_count = 3 if len(handles) > 4 else (2 if len(handles) > 1 else 1) + + rows_html = [] + for r in range(math.ceil(len(handles) / cols_count)): + row_items = [] + for c in range(cols_count): + idx = r * cols_count + c + if idx < len(handles): + name, color, is_line = handles[idx] + c_str = to_plotly_color(color) + + symbol = f"■" if is_line else " " + + name_padded = name.ljust(22).replace(" ", " ") + row_items.append(f"{symbol} {name_padded}") + + rows_html.append(" ".join(row_items)) + + legend_html = "
".join(rows_html) + + fig.add_annotation( + text=legend_html, + xref="paper", yref="paper", + x=1.0, y=1.0, + yshift=40 * UI_SCALE, + xanchor="right", yanchor="bottom", + align="left", + font=dict(family="monospace", size=11 * UI_SCALE, color=to_plotly_color(COLORS["TEXT"])), + bgcolor=to_plotly_color(COLORS["BAND0"]), + bordercolor=to_plotly_color(COLORS["LINE"]), + borderwidth=1.5, + borderpad=8, + showarrow=False + ) + + for i, stats_sub in enumerate(columns_data): + col_idx = i + 1 + row_idx = 1 + + title_text = column_titles[i] + if title_text and title_text.strip(): + x_ref_str = "x domain" if col_idx == 1 else f"x{col_idx} domain" + + fig.add_annotation( + text=title_text, + xref=x_ref_str, + yref="paper", + x=0.5, y=1.0, + yshift=20 * UI_SCALE, + xanchor="center", yanchor="bottom", + showarrow=False, + font=dict(color=to_plotly_color(COLORS["TEXT"]), size=14 * UI_SCALE) + ) + + if MODE == "scatter": + plot_sub_scatter(fig, stats_sub, row_idx, col_idx) + elif MODE == "bars": + plot_sub_bars(fig, stats_sub, row_idx, col_idx) + elif MODE == "boxplot": + plot_sub_boxplot(fig, stats_sub, row_idx, col_idx) + elif MODE == "pie": + plot_sub_pie(fig, stats_sub, row_idx, col_idx) + else: + raise Exception(f'invalid mode: {MODE}') + + plot_height = (items_per_col * 50 + 200) * UI_SCALE + plot_width = (cols * 400 + 100) * UI_SCALE + fig.update_layout( + title=dict( + text=f"{TITLE}
{SUBTITLE}" if 'SUBTITLE' in globals() else TITLE, + x=0.5, xanchor='center', yanchor='top', + font=dict(size=24 * UI_SCALE) + ), + font=dict(size=12 * UI_SCALE), + annotationdefaults=dict(font=dict(color=to_plotly_color(COLORS["TEXT"]), size=14 * UI_SCALE)), + template="plotly_dark" if DARK else "plotly_white", + plot_bgcolor=to_plotly_color(COLORS["BAND0"]), + paper_bgcolor=to_plotly_color(COLORS["BAND0"]), + height=plot_height, + width=plot_width, + margin=dict(t=160 * UI_SCALE, b=60 * UI_SCALE, l=20 * UI_SCALE, r=20 * UI_SCALE), + barmode='stack' + ) + + """ + fig.add_annotation( + #text=f"https://git.ecker.tech/chart/{TITLE.replace(' ', '')}", + text=f"https://chartfag.neocities.org/charts/", + xref="paper", yref="paper", + x=0.0, + y=0.0, + yshift=-40, + xanchor="left", yanchor="top", + showarrow=False, + font=dict(color=to_plotly_color(COLORS["STATS"]), size=10) + ) + """ + + return fig + +def compute_mean_smart(ratings): + ratings = sorted( ratings ) + + cutoff = len(ratings) // 8 + if cutoff > 0: + ratings = ratings[cutoff:-cutoff] + + return statistics.mean(ratings) + +def stat_new(name, ratings, entry={}): + buckets = [0] * 10 + for r in ratings: + r = min(9, max(4, round(r))) + buckets[r] += 1 + + stat = {} + stat['name'] = name + stat['ratings'] = ratings + stat['points'] = [rating_to_point(r, len(ratings)) for r in ratings] + stat['buckets'] = buckets + stat['count'] = len(ratings) + stat['mean'] = statistics.mean(ratings) if ratings else 0.0 + stat['median'] = statistics.median(ratings) if ratings else 0.0 + stat['mode'] = statistics.mode(ratings) if ratings else 0.0 + stat['median_grouped'] = statistics.median_grouped(round(r) for r in ratings) if ratings else 0.0 + stat['stdev'] = statistics.pstdev(ratings) if ratings else 0.0 + stat['mean_smart'] = compute_mean_smart(ratings) if ratings else 0.0 + + # attach extra data + stat["markers"] = [] + if "marker" in entry: + stat["markers"] = [ entry["marker"] ] + elif "markers" in entry: + stat["markers"] = entry["markers"] + + # fix up previously split markers + for i, marker in enumerate(stat["markers"]): + if marker in ["DNF", "invalid"]: + stat["markers"][i] = "DNF/invalid" + """ + elif marker in ["trainwreck", "cringekino"]: + stat["markers"][i] = "trainwreck/cringekino" + elif marker in ["ad", "nonrun"]: + stat["markers"][i] = "ad/nonrun" + """ + + if "event" in entry: + stat["event"] = entry["event"] + + if "urls" in entry: + stat["urls"] = entry["urls"] + + if "scores" in entry: + stat["scores"] = entry["scores"] + + if PRINT_STATS: + print(stat_to_str(stat)) + + return stat + +# read from json +def read_stats(filename): + stats = [] + aux = { + "total": [], + "markers": {}, + } + + data = json.load(open(filename, "r", encoding='utf-8')) + + for name, entry in data.items(): + ratings = [] + + if "ignore" in entry and entry["ignore"]: + continue + + op = entry["posts"][0].split("#p")[-1] if "posts" in entry and entry["posts"] else None + start_time = entry["times"][op] if "times" in entry and op in entry["times"] else 0 + thread_url = entry["posts"][0].split("#")[0] if "posts" in entry and entry["posts"] else "" + + # convert lists to dicts (for old data) + if isinstance( entry["ratings"], list ): + entry["ratings"] = { f'{i}': r for i, r in enumerate(entry["ratings"]) } + + entry["urls"] = [] + entry["scores"] = [] + + # parse ratings in each run + for no, score in entry["ratings"].items(): + #cur_time = entry["times"][no] if "times" in entry and no in entry["times"] else 0 + if start_time > 0 and CUTOFF_SECONDS > 0: + if no not in entry["times"]: + continue + + cur_time = entry["times"][no] + if cur_time - start_time > CUTOFF_SECONDS: + print("Culled", name, no, score, cur_time - start_time) + continue + + score = score.upper() + + # trim extraneous characters (for example: ZZZZZZZ) if not already a valid score + if score not in SCORES: + # check if SSSS / ZZZZ + if score[:4] in SCORES: + score = score[:4] + # check if first letter would match instead (per-run ratings that just share the first letter but keep it as a whole word) + elif score[:1] in SCORES: + score = score[:1] + else: + score = score[:3] + + # cull anything not A-F + if DROP_Z_S: + if SCORES[score] <= SCORES["Z"] or SCORES["S"] < SCORES[score]: + continue + + # invalid score + if score not in SCORES: + continue + + # increment totals (I don't remember what I originally used this for) + TOTAL_SCORES[score] += 1 + + # score modifiers + rating = SCORES[score] + + # randomize downward + #if "randomize" in entry and CUTOFF_SECONDS == 0 and False: + if "randomize" in entry: + lo = SCORES[list(SCORES.keys())[0]] + hi = SCORES[list(SCORES.keys())[-1]] + + if True: #if random.random() < entry["randomize"] and rating > hi: + rating = random.uniform(lo, hi) + + if ("reverse" in entry and random.random() < entry["reverse"]) or REVERSE: + rating = 10 - rating + 2.75 + + ratings.append( rating ) + entry["scores"].append( ( score, no ) ) + entry["urls"].append( f"{thread_url}#p{no}" ) + aux["total"].append( rating ) + + stat = stat_new(name, ratings, entry) + stat["start_time"] = start_time - DATE_OFFSET + + if FILTER_BY and FILTER_BY not in {*stat["markers"]}: + continue + + stats.append(stat) + + # flatten + for marker in stat["markers"]: + if marker not in MARKERS: + continue + MARKERS[marker]["count"] += 1 + + # increment marker totals + for marker in stat["markers"]: + for rating in ratings: + if marker not in aux["markers"]: + aux["markers"][marker] = [] + aux["markers"][marker].append( rating ) + + if not stat["markers"]: + marker = "none" + for rating in ratings: + if marker not in aux["markers"]: + aux["markers"][marker] = [] + aux["markers"][marker].append( rating ) + + # show aggregate total + if AUX_MODE == "total": + stats = [ stat_new("total", aux["total"]) ] + # aggregate by markers + elif AUX_MODE == "markers": + stats = [ stat_new(name, ratings) for name, ratings in aux["markers"].items() ] + + return stats + +def main(): + # yuck + global REVERSE + global ZOOMER + global DARK + global USE_LEGEND + global MIN_COLUMNS + global CUTOFF_SECONDS + global SORT_BY + global FILTER_BY + global MODE + global AUX_MODE + global OUT_FILE + global PRINT_STATS + + parser = argparse.ArgumentParser("Charter") + # actions + parser.add_argument("--fetch", action="store_true", help="Fetch ratings from queue") + parser.add_argument("--plot", action="store_true", help="Plot ratings from file") + parser.add_argument("--dump", action="store_true", help="Dumps rating as metrics") + parser.add_argument("--upload", action="store_true", help="Uploads to neocities") + # queue-less args + parser.add_argument("--url", type=str, default=None, help="Post link to fetch ratings from its replies") + parser.add_argument("--name", type=str, default=None, help="Name of run for rating") + parser.add_argument("--markers", type=str, default=None, help="Markers to attach to run rating") + # modifiers + parser.add_argument("--sort-by", type=str, default=None, help="Sort plotted ratings by this value") + parser.add_argument("--filter-by", type=str, default=None, help="Filters for runs with this value") + parser.add_argument("--mode", type=str, default=None, help="Additional modes (scatter | bars | boxplot)") + parser.add_argument("--aux-mode", type=str, default=None, help="Additional modes (total | markers)") + parser.add_argument("--cutoff-seconds", type=int, default=None, help="Ignore ratings made X seconds after the rating post") + parser.add_argument("--cutoff-minutes", type=int, default=None, help="Ignore ratings made X minutes after the rating post") + # lesser used modifiers + parser.add_argument("--reverse", action="store_true", help="Reverses the ratings") + parser.add_argument("--zoomer", action="store_true", help="Zoomer friendly scales") + parser.add_argument("--light", action="store_true", help="Light mode") + parser.add_argument("--no-legend", action="store_true", help="Hides the legend") + parser.add_argument("--no-html", action="store_true", help="Skip saving an HTML copy") + parser.add_argument("--copy", action="store_true", help="Saves a copy by the timestamp") + + args = parser.parse_args() + + if args.sort_by: + SORT_BY = args.sort_by + + if args.filter_by: + FILTER_BY = args.filter_by + + if args.aux_mode: + AUX_MODE = args.aux_mode + if AUX_MODE == "total": + args.no_legend = True + MIN_COLUMNS = 1 + + modifiers = [ f'[{SORT_BY or AUX_MODE or "chronological"}]' ] + + if args.mode: + MODE = args.mode + modifiers.append(f'[type={MODE}]') + + if args.filter_by: + modifiers.append(f'[filter={FILTER_BY}]') + + if args.cutoff_seconds: + CUTOFF_SECONDS = args.cutoff_seconds + elif args.cutoff_minutes: + CUTOFF_SECONDS = args.cutoff_minutes * 60 + if CUTOFF_SECONDS > 0: + modifiers.append(f'[cutoff={CUTOFF_SECONDS//60}]') + + OUT_FILE = f'./images/ratings{"".join(modifiers)}.png' + + REVERSE = args.reverse + ZOOMER = args.zoomer + DARK = not args.light + USE_LEGEND = not args.no_legend + PRINT_STATS = args.dump + + if args.fetch: + queue = [] + + # inject to queue + if args.url and args.name: + queue = [{ "post": args.url, "name": args.name, "markers": args.markers.split(",") if args.markers else None }] + + fetch_ratings(queue=queue) + + if args.plot: + stats = read_stats(IN_RATINGS_FILE) + fig = create_plot(stats) + + if not args.no_html: + click_js = """ + var myPlot = document.getElementsByClassName('plotly-graph-div')[0]; + + myPlot.on('plotly_click', function(data){ + if (data.points && data.points.length > 0) { + var url = data.points[0].customdata; + if(url && typeof url === 'string') { + window.open(url, '_blank'); + } + } + }); + + myPlot.on('plotly_hover', function(data){ + if (data.points && data.points.length > 0 && data.points[0].customdata) { + myPlot.style.cursor = 'pointer'; + } + }); + + myPlot.on('plotly_unhover', function(data){ + myPlot.style.cursor = 'default'; + }); + """ + + html_file = OUT_FILE.replace('.png', '.html') + html_str = fig.to_html(full_html=True, post_script=click_js) + + css_injection = f"" + html_str = html_str.replace("", f"\n\t{css_injection}") + + with open(html_file, "w", encoding="utf-8") as f: + f.write(html_str) + + """ + if args.upload: + import neocities + nc = neocities.NeoCities(api_key=YOUR_API_KEY) + nc.upload((html_file, 'charts/index.html')) + """ + + if args.copy: + print(f'Saving chart copy: {OUT_FILE_TIMESTAMP}') + fig.write_image(OUT_FILE_TIMESTAMP, scale=1.0) + + print(f'Saving image chart: {OUT_FILE}') + fig.write_image(OUT_FILE, scale=2.0) + + if not args.fetch and not args.plot: + raise Exception("Specify --fetch or --plot") + + +if __name__ == "__main__": + main() \ No newline at end of file