Per-user scheduling + global balancing: add Annotator ID, per-user dedup, count-based prioritization, CSV adds annotator_id; update UI bindings accordingly

app.py

@@ -121,11 +121,36 @@ def _build_image_pairs(rows: List[Dict[str, str]], task_name: str) -> List[Dict[
     return pairs
 
 
-def _read_existing_eval_keys(task_name: str) -> set:
-    """Read already-evaluated pair keys from persistent CSV, return a set of keys.
-    Key is (test_id, frozenset({model1_name, model2_name}), org_img) to ignore A/B order.
+def _read_eval_counts(task_name: str) -> Dict[Tuple[str, frozenset, str], int]:
+    """Global counts per pair key across all annotators."""
+    counts: Dict[Tuple[str, frozenset, str], int] = {}
+    csv_path = _persist_csv_path_for_task(task_name)
+    if not os.path.exists(csv_path):
+        return counts
+    try:
+        with open(csv_path, newline="", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            for r in reader:
+                tid = str(r.get("test_id", "")).strip()
+                m1 = str(r.get("model1_name", "")).strip()
+                m2 = str(r.get("model2_name", "")).strip()
+                org = str(r.get("org_img", "")).strip()
+                if not (tid and m1 and m2 and org):
+                    continue
+                key = (tid, frozenset({m1, m2}), org)
+                counts[key] = counts.get(key, 0) + 1
+    except Exception:
+        pass
+    return counts
+
+
+def _read_user_done_keys(task_name: str, annotator_id: str) -> set:
+    """Keys already evaluated by the given annotator.
+    If CSV has no annotator_id column (legacy rows), those rows are ignored for per-user filtering.
     """
     keys = set()
+    if not annotator_id:
+        return keys
     csv_path = _persist_csv_path_for_task(task_name)
     if not os.path.exists(csv_path):
         return keys
@@ -133,6 +158,8 @@ def _read_existing_eval_keys(task_name: str) -> set:
         with open(csv_path, newline="", encoding="utf-8") as f:
             reader = csv.DictReader(f)
             for r in reader:
+                if str(r.get("annotator_id", "")).strip() != str(annotator_id).strip():
+                    continue
                 tid = str(r.get("test_id", "")).strip()
                 m1 = str(r.get("model1_name", "")).strip()
                 m2 = str(r.get("model2_name", "")).strip()
@@ -166,23 +193,31 @@ def _schedule_round_robin_by_test_id(pairs: List[Dict[str, str]], seed: Optional
     return ordered
 
 
-def load_task(task_name: str):
+def load_task(task_name: str, annotator_id: str = ""):
     if not task_name:
         raise gr.Error("Please select a task first.")
 
     rows = _load_task_rows(task_name)
     pairs_all = _build_image_pairs(rows, task_name)
-    # Filter out already evaluated pairs from persistent CSV
-    done_keys = _read_existing_eval_keys(task_name)
+    # Per-user filtering and global balancing
     def key_of(p: Dict[str, str]):
         return (p["test_id"], frozenset({p["model1_name"], p["model2_name"]}), p["org_img"])
-    pairs = [p for p in pairs_all if key_of(p) not in done_keys]
-    done_len = len([p for p in pairs_all if key_of(p) in done_keys])
+    user_done_keys = _read_user_done_keys(task_name, annotator_id)
+    global_counts = _read_eval_counts(task_name)
+    pairs = [p for p in pairs_all if key_of(p) not in user_done_keys]
 
-    # Balanced schedule across test_ids with a stable randomization
+    # Balanced schedule: prioritize low-count pairs, and within same count do round-robin by test_id
     seed_env = os.environ.get("SCHEDULE_SEED")
     seed = int(seed_env) if seed_env and seed_env.isdigit() else None
-    pairs = _schedule_round_robin_by_test_id(pairs, seed=seed)
+    def count_of(p: Dict[str, str]):
+        return global_counts.get(key_of(p), 0)
+    buckets: Dict[int, List[Dict[str, str]]] = {}
+    for p in sorted(pairs, key=count_of):
+        buckets.setdefault(count_of(p), []).append(p)
+    ordered: List[Dict[str, str]] = []
+    for c in sorted(buckets.keys()):
+        ordered.extend(_schedule_round_robin_by_test_id(buckets[c], seed=seed))
+    pairs = ordered
 
     # Assign A/B order to counteract position bias: alternate after scheduling
     for idx, p in enumerate(pairs):
@@ -192,7 +227,7 @@ def load_task(task_name: str):
         try:
             print("[VisArena] No pending pairs.")
             print("[VisArena] total_pairs=", len(pairs_all))
-            print("[VisArena] already_done=", done_len)
+            print("[VisArena] already_done_by_user=", len(user_done_keys))
             print("[VisArena] persist_csv=", _persist_csv_path_for_task(task_name))
         except Exception:
             pass
@@ -234,6 +269,7 @@ def _append_local_persist_csv(task_name: str, row: Dict[str, object]) -> bool:
     csv_exists = os.path.exists(csv_path)
     fieldnames = [
         "eval_date",
+        "annotator_id",
         "test_id",
         "model1_name",
         "model2_name",
@@ -301,8 +337,8 @@ def _upload_eval_record_to_dataset(task_name: str, row: Dict[str, object]) -> Tu
         return False, f"Exception: {type(e).__name__}: {e}"
 
 
-def on_task_change(task_name: str, _state_pairs: List[Dict[str, str]]):
-    pairs = load_task(task_name)
+def on_task_change(task_name: str, annotator_id: str, _state_pairs: List[Dict[str, str]]):
+    pairs = load_task(task_name, annotator_id)
     # Defaults for A and B (8 sliders total)
     default_scores = [3, 3, 3, 3, 3, 3, 3, 3]
     if not pairs:
@@ -366,6 +402,7 @@ def on_pair_navigate(index: int, pairs: List[Dict[str, str]]):
 
 def on_submit(
     task_name: str,
+    annotator_id: str,
     index: int,
     pairs: List[Dict[str, str]],
     a_physical_score: int,
@@ -431,9 +468,10 @@ def on_submit(
         }
     # Build record
     row = _build_eval_row(pair, score_map)
+    row["annotator_id"] = annotator_id
 
     # Idempotency: check if this pair already evaluated; if so, skip writing
-    done_keys = _read_existing_eval_keys(task_name)
+    done_keys = _read_user_done_keys(task_name, annotator_id)
     eval_key = (pair["test_id"], frozenset({pair["model1_name"], pair["model2_name"]}), pair["org_img"])
     if eval_key in done_keys:
         ok_local = False
@@ -502,6 +540,7 @@ with gr.Blocks(title="VisArena Human Evaluation") as demo:
             interactive=True,
             value="Scene Composition & Object Insertion",
         )
+        annotator_id_input = gr.Textbox(label="Annotator ID", placeholder="请输入你的唯一标识 (如昵称/学号)")
         index_slider = gr.Slider(
             label="Pair Index",
             value=0,
@@ -541,7 +580,7 @@ with gr.Blocks(title="VisArena Human Evaluation") as demo:
         # Event bindings
         task_selector.change(
             fn=on_task_change,
-            inputs=[task_selector, pair_state],
+            inputs=[task_selector, annotator_id_input, pair_state],
             outputs=[
                 pair_state,
                 index_slider,
@@ -585,6 +624,7 @@ with gr.Blocks(title="VisArena Human Evaluation") as demo:
             fn=on_submit,
             inputs=[
                 task_selector,
+                annotator_id_input,
                 index_slider,
                 pair_state,
                 a_physical_input,
@@ -618,7 +658,7 @@ with gr.Blocks(title="VisArena Human Evaluation") as demo:
         # Auto-load default task on startup
         demo.load(
             fn=on_task_change,
-            inputs=[task_selector, pair_state],
+            inputs=[task_selector, annotator_id_input, pair_state],
             outputs=[
                 pair_state,
                 index_slider,