Files
breeze/scripts/narration_segments.py
T
waozi 939593a1e6 Checkpoint media compression, narration, and app updates
Preserve current AVIF compression, local voice pipelines, practice workflows, UI API migration, locale updates, and packaging work. Native integration validation remains pending.
2026-11-10 22:20:31 -03:00

117 lines
4.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Lossless natural text spans for independently conditioned speech requests.
Offsets always address the original Python string. No normalization, inserted
punctuation, or stripping is allowed here: concatenating the spans recovers the
source, including its whitespace. A clause without a safe boundary stays whole
so the caller can reject its token budget instead of cutting a word.
"""
import re
VERSION = 1
DEFAULT_MAX_UNITS = 24
SENTENCE_ENDS = frozenset('.!?。!?')
CLAUSE_ENDS = frozenset(',;:,;:、')
CLOSERS = frozenset('"\'”’»」』)》】')
CJK = r'\u3400-\u9fff\u3040-\u30ff\uac00-\ud7af'
WORD = r'[^\W_' + CJK + r']+'
UNITS = re.compile(r'[' + CJK + r']|' + WORD + r'(?:[\'’.-]' + WORD + r')*')
def semantic_units(text):
return len(UNITS.findall(text))
def _punctuation_cuts(text, punctuation):
cuts = []
for index, char in enumerate(text):
if char not in punctuation:
continue
before = text[index - 1] if index else ''
after = text[index + 1] if index + 1 < len(text) else ''
# Decimal points, thousands separators and numeric colon notation are
# indivisible, as are ASCII punctuation characters inside a word.
if before.isdecimal() and after.isdecimal():
continue
if char in '.!,;:?' and before.isalnum() and after.isalnum():
continue
end = index + 1
while end < len(text) and (text[end] in CLOSERS or text[end] in punctuation):
end += 1
while end < len(text) and text[end].isspace():
end += 1
if end == len(text) or any(c.isalnum() for c in text[end:]):
cuts.append(end)
return sorted(set(cuts))
def _pieces(text, cuts):
start = 0
for end in sorted(set(cuts) | {len(text)}):
if end > start:
yield text[start:end]
start = end
def _pack(pieces, maximum):
current = ''
for piece in pieces:
if current and semantic_units(current + piece) > maximum:
yield current
current = ''
current += piece
if current:
yield current
def _shorten(sentence, maximum):
if semantic_units(sentence) <= maximum:
return [sentence]
clauses = list(_pieces(sentence, _punctuation_cuts(sentence, CLAUSE_ENDS)))
pieces = []
for clause in clauses:
if semantic_units(clause) <= maximum:
pieces.append(clause)
continue
# A whitespace boundary never separates digits in a grouped number.
cuts = [match.end() for match in re.finditer(r'\s+', clause)
if match.start() > 0 and match.end() < len(clause)
and not (clause[match.start() - 1].isdecimal()
and clause[match.end()].isdecimal())]
pieces.extend(_pack(_pieces(clause, cuts), maximum))
return list(_pack(pieces, maximum))
def plan(text, maximum=DEFAULT_MAX_UNITS):
"""Return exact ordered source spans and a deterministic assembly contract."""
if not isinstance(text, str) or not any(c.isalnum() for c in text):
raise ValueError('Narration requires text with spoken content')
if type(maximum) is not int or maximum < 1:
raise ValueError('Narration segment limit must be a positive integer')
pieces = [text]
if semantic_units(text) > maximum:
pieces = []
for sentence in _pieces(text, _punctuation_cuts(text, SENTENCE_ENDS)):
pieces.extend(_shorten(sentence, maximum))
# Punctuation/whitespace without spoken content belongs to its neighbor.
merged = []
pending = ''
for piece in pieces:
if not any(c.isalnum() for c in piece):
if merged:
merged[-1] += piece
else:
pending += piece
else:
merged.append(pending + piece)
pending = ''
spans = []
offset = 0
for piece in merged:
spans.append(dict(start=offset, end=offset + len(piece), text=piece))
offset += len(piece)
if ''.join(span['text'] for span in spans) != text:
raise ValueError('Narration segmentation changed its source')
return dict(version=VERSION, maximum_units=maximum, pause_ms=0,
assembly='concatenate', spans=spans)