% Human-evaluation section for the canine vocalization demo.
% Self-contained: compiles on its own, or drop the section into the paper.
% After running score_human_eval.py, replace every \RED{XX} with the printed
% number and replace the red takeaway note with one sentence.
\documentclass{article}
\usepackage[margin=1in]{article}
\usepackage{booktabs}
\usepackage{xcolor}
\newcommand{\RED}[1]{\textcolor{red}{#1}}
\begin{document}

\section{Human Evaluation}
\label{sec:human-eval}

\paragraph{Setup.}
We assess whether the system's semantic output matches human judgment on video.
We sample $N=30$ dog clips spread across breed, duration, and source video, and
run each through the full pipeline (denoise, sound-event detection, HuBERT phone
transcription, and semantic inference). For every clip the system emits four
predictions: \emph{environment}, \emph{activity}, \emph{intention}, and a
one-sentence \emph{interpretation}. \textbf{Two annotators} independently score
each clip against its video, which serves as ground truth. The three attributes
are scored $0$/$1$/$2$ (Incorrect / Plausible / Correct) and the interpretation
$1$--$5$ (nonsensical to accurate \& informative).

\paragraph{Metrics.}
We report \emph{Accuracy} (fraction of ratings equal to Correct, averaged over
annotators), \emph{Acceptability} (fraction $\geq$ Plausible), exact
inter-annotator \emph{Agreement}, and Fleiss' $\kappa$. We use accuracy averaged
over annotators rather than majority vote, which is degenerate with two raters,
and report \% Agreement alongside $\kappa$ because the latter can be depressed by
skewed label distributions.

\begin{table}[t]
\centering
\caption{Human evaluation of the canine demo ($N=30$ clips, two annotators).}
\label{tab:human-eval}
\begin{tabular}{lcccc}
\toprule
Attribute & Accuracy & Acceptability & Agreement & Fleiss' $\kappa$ \\
\midrule
Environment   & \RED{XX}\% & \RED{XX}\% & \RED{XX}\% & \RED{XX} \\
Activity      & \RED{XX}\% & \RED{XX}\% & \RED{XX}\% & \RED{XX} \\
Intention     & \RED{XX}\% & \RED{XX}\% & \RED{XX}\% & \RED{XX} \\
\midrule
\textbf{Overall} & \RED{XX}\% & \RED{XX}\% & \RED{XX}\% & \RED{XX} \\
\bottomrule
\end{tabular}
\end{table}

Interpretation quality averaged $\RED{XX}\pm\RED{XX}$ on the $1$--$5$ scale.

\noindent\RED{\textbf{Takeaway (replace):} one sentence summarizing what the
numbers show.}

\end{document}
