"""Coverage, reuse terms, and evaluation-content checks.""" import re import ahocorasick from .schema import normalized, state_hash, unpack def category(record: dict, mappings: dict): m = record["metadata"] source, repo = m["upstream_source"], m["repo"] if repo == "bigcode/commitpackft": return "engineering", "objective_commit_message_pair" if "tasksource-jev" in repo: for name, prefixes in mappings["tasksource"].items(): for prefix in prefixes: if source.startswith(prefix): return name, "source:" + prefix return None, "unmapped_source" if "procedural" in repo: return mappings["procedural"].get(source), "procedural:" + source if "LocalLLaMA" in repo: return mappings["localllama"].get(source), "workflow:" + source # Synthetic domain tags can contradict application text. Use the task's # application and question skill rather than blindly trusting those tags. application = (m.get("application") or "").casefold() skill = (m.get("skill") or "").casefold() item = (m.get("item_source") or "").casefold() rules = [ ("trust_safety", skill in {"toxicity", "fraud_likelihood", "safety", "policy_violation"}), ( "legal", any( s in application for s in ("legal compliance", "legal aid", "legal case", "contract management") ), ), ("documents_meetings", "meeting" in item or "handover" in item or "shift notes" in item), ( "engineering", "devops" in application or "on-call engineer" in application or "network operations" in application, ), ( "finance", any( s in application for s in ("financial", "fraud operations", "insurance claim", "payment card") ), ), ( "product_management", "feedback" in application and any(s in application for s in ("analytics", "voice-of-customer", "improve")), ), ( "sales_commerce", "sales pipeline" in application or "sales team" in application or "procurement" in application, ), ( "customer_support", any( s in application for s in ( "support", "customer service", "helpdesk", "guest-relations", "customer relations", ) ), ), ( "trust_safety", any( s in application for s in ("moderation", "threat intelligence", "security operations", "phishing") ), ), ] for name, matched in rules: if matched: return name, "synthetic_application_or_skill:" + name return None, "unmapped_workflow" def reusable(record: dict) -> bool: m = record["metadata"] license_name = m.get("license", "").casefold() return ( m.get("license_use") == "commercial" and license_name not in ("", "unspecified", "unknown", "other") and "non-commercial" not in license_name and "by-nc" not in license_name and "custom" not in license_name ) def leaf_text(value): value = unpack(value) if isinstance(value, str): yield value elif isinstance(value, dict): for v in value.values(): yield from leaf_text(v) elif isinstance(value, list): for v in value: yield from leaf_text(v) def match_text(value: str) -> str: return " ".join(re.findall(r"\w+", normalized(value))) class BenchmarkIndex: """Exact states and literal 32-word fragments; this is not semantic dedup.""" def __init__(self, rows): self.exact = set() self.automaton = ahocorasick.Automaton() self.rows = 0 self.anchors = 0 for row in rows: self.rows += 1 state = row["state"] self.exact.add(state_hash(state)) for leaf in leaf_text(state): words = match_text(leaf).split() for start in range(0, len(words) - 31, 24): anchor = " ".join(words[start : start + 32]) if len(anchor) >= 100: self.automaton.add_word(anchor, str(row.get("id", self.rows))) self.anchors += 1 if len(self.automaton): self.automaton.make_automaton() def match(self, state: str): if state_hash(state) in self.exact: return "exact_state" if len(self.automaton): for leaf in leaf_text(state): value = match_text(leaf) for _, row_id in self.automaton.iter(value): return "literal_fragment:" + row_id return None