Files
breeze/scripts/local_narration_context.py
T
waozi 939593a1e6 Checkpoint media compression, narration, and app updates
Preserve current AVIF compression, local voice pipelines, practice workflows, UI API migration, locale updates, and packaging work. Native integration validation remains pending.
2026-11-10 22:20:31 -03:00

774 lines
43 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Explicit short-cue context, with contiguous audio and separate speaker evidence.
This wrapper leaves the underlying synthesis profile and ordinary requests intact.
Public proof WAVs allow offline receipt checks without models or private caches.
Prepare the base engines normally, then run this file with --configure explicitly.
"""
import argparse
import array
import difflib
import importlib.util
import json
import math
from pathlib import Path, PurePosixPath
import re
import subprocess
import sys
import unicodedata
import wave
from practice_media import narration_text_path
from f5_russian_narration import file_hash, json_hash, text_hash as local_text_hash
import ogg_opus
ROOT = Path(__file__).resolve().parents[1]
VERSION = 'contiguous-prefix-v1'
ASR_MODEL = 'openai/whisper-large-v3-turbo'
ASR_REVISION = '41f01f3fe87f28c78e2fbf8b568835947dd65ed9'
POLICY = ROOT / 'design/narration/short-context.json'
RETAIN_LEADING_PCM0 = 'retain-leading-pcm0-v1'
CZECH_COUNT_COMPARISON = 'czech-count-1-2-3-v1'
CZECH_COUNT_PARENTS = {
'3.': 'Tři, dva, jedna, připrav se na další cvik.',
'2.': 'Dva, jedna, připrav se na další cvik.',
'1.': 'Jedna, připrav se na další cvik a klidně dýchej.',
}
PORTUGUESE_COUNT_COMPARISON = 'portuguese-count-1-2-3-v1'
PORTUGUESE_COUNT_PARENTS = {
'Três.': 'Três. Prepare o corpo para o próximo exercício.',
'Dois.': 'Dois. Prepare o corpo para o próximo exercício.',
'Um.': 'Um. Prepare o corpo para o próximo exercício.',
}
class ContextRejected(RuntimeError):
"""One synthesized candidate failed quality checks; its recipe stays valid."""
def __init__(self, reason, diagnostics=None):
super().__init__(reason)
self.diagnostics = dict(diagnostics or {})
def words(text):
normalized = unicodedata.normalize('NFKC', text).casefold().replace('氣', '气')
normalized = re.sub(r'([\u3400-\u9fff])', r' \1 ', normalized)
return re.findall(r"\w+(?:['’\-]\w+)*", normalized)
def rule_comparison(rule):
"""Only explicitly registered workout count rules may equate counts."""
comparison = rule.get('comparison')
if comparison is None:
return None
if comparison == {'version': PORTUGUESE_COUNT_COMPARISON}:
if (rule.get('locale') != 'pt' or rule.get('voice_directory') != 'workout/voice' or
rule.get('text') not in PORTUGUESE_COUNT_PARENTS or
rule.get('parent_text') != PORTUGUESE_COUNT_PARENTS[rule['text']] or
any(field in rule for field in ('prefix_alignment', 'crop_asr', 'parent_source'))):
raise ValueError('Unsupported Portuguese count comparison rule')
return PORTUGUESE_COUNT_COMPARISON
if (comparison != {'version': CZECH_COUNT_COMPARISON} or
rule.get('locale') != 'cs' or rule.get('voice_directory') != 'workout/voice' or
rule.get('text') not in CZECH_COUNT_PARENTS or
rule.get('parent_text') != CZECH_COUNT_PARENTS[rule['text']] or
any(field in rule for field in ('prefix_alignment', 'crop_asr', 'parent_source'))):
raise ValueError('Unsupported Czech count comparison rule')
return CZECH_COUNT_COMPARISON
def comparison_words(text, comparison=None):
if comparison not in (None, CZECH_COUNT_COMPARISON, PORTUGUESE_COUNT_COMPARISON):
raise ValueError('Unsupported context comparison version')
tokens = words(text)
if comparison == PORTUGUESE_COUNT_COMPARISON:
# Validate raw ASCII boundaries before words() can discard a sign or
# normalize a different numeric spelling. Default/Czech views stay intact.
numeric = lambda token: any(char.isnumeric() for char in token)
if any(numeric(token) for token in tokens):
first = re.match(r'^\s*([123])(?:[.!?,:;])?(?=\s|$)', text)
if (first and tokens[0] == first[1] and
not any(numeric(token) for token in tokens[1:])):
tokens[0] = {'1': 'um', '2': 'dois', '3': 'três'}[first[1]]
return tokens
if comparison is not None:
spellings = {'1': 'jedna', '2': 'dva', '3': 'tři'}
tokens = [spellings.get(token, token) for token in tokens]
return tokens
def comparison_views(rule, text, transcript):
comparison = rule_comparison(rule)
if comparison is None:
return text, transcript
if text not in (rule['text'], rule['parent_text']):
raise ValueError('Count comparison requires its exact registered text')
return (' '.join(comparison_words(text, comparison)),
' '.join(comparison_words(transcript, comparison)))
def parent_match(match, config, code, rule, transcript, length):
expected, heard = comparison_views(rule, rule['parent_text'], transcript)
if rule_comparison(rule) is not None and expected != heard:
return 0.0
return match(config, code, expected, heard, length, True)
def comparison_rule_from_proof(code, text, proof):
"""Resolve a sealed rule; never infer equivalence from locale or digits."""
if (not isinstance(proof, dict) or proof.get('scope') != 'context_parent' or
proof.get('version') != VERSION or proof.get('locale') != code or
proof.get('text_sha256') != pcm_hash(text.encode()) or
proof.get('proof_sha256') != json_hash({k: v for k, v in proof.items() if k != 'proof_sha256'})):
raise ValueError('Invalid context comparison proof')
rule = recipe(ROOT / proof['policy_path'], code, text, proof['voice_directory'])
if rule is None or rule['sha256'] != proof['recipe_sha256']:
raise ValueError('Changed context comparison recipe')
return rule if rule_comparison(rule) is not None else None
def load_policy(path):
path = Path(path).resolve()
if not path.is_relative_to(ROOT / 'design/narration'):
raise ValueError('Context policy must be a public narration descriptor')
policy = json.loads(path.read_text())
if (policy.get('version') != VERSION or policy.get('helper_sha256') != file_hash(__file__) or
policy.get('verifier_sha256') != file_hash(ROOT / 'scripts/generate-lumi-guide-narration.py') or
policy.get('asr_model') != ASR_MODEL or policy.get('asr_revision') != ASR_REVISION):
raise ValueError('Changed or unsupported context verification recipe')
return policy
def recipe(policy_path, code, text, voice_directory):
policy = load_policy(policy_path)
if voice_directory != policy['voice_directory']:
return None
candidates = [rule for rule in policy['rules'] if rule['locale'] == code and rule['text'] == text]
if not candidates:
return None
if len(candidates) != 1:
raise ValueError('Duplicate context rule')
rule = dict(candidates[0])
if 'prefix_alignment' in rule and (rule['prefix_alignment'] != RETAIN_LEADING_PCM0 or
(code, text) not in (('cs', 'Vydechněte.'), ('ru', 'Вдохните.')) or
voice_directory != 'sunsalutation/voice' or 'parent_source' not in rule):
raise ValueError('Unsupported retained-leading-PCM context rule')
if 'crop_asr' in rule and (code != 'zh' or text not in ('吸气。', '呼气。') or
voice_directory != 'sunsalutation/voice' or
'parent_source' not in rule or
rule['crop_asr'] != dict(model=ASR_MODEL, revision=ASR_REVISION, artifacts=policy['asr_artifacts'])):
raise ValueError('Unsupported short-context final transcription route')
if 'parent_source' in rule:
source = rule['parent_source']
config = json.loads((ROOT / 'design/sun-salutation/narration/narration.json').read_text())
source_path = narration_text_path(config, code, ROOT / 'design/sun-salutation/narration/text')
entries = dict(re.findall(r'\[([^\]]+)\]\n(.*?)(?=\n---|\Z)', source_path.read_text(), re.S))
if (source['file'] != str(source_path.relative_to(ROOT)) or source['sha256'] != file_hash(source_path) or
entries[source['key']].strip() != rule['parent_text']):
raise ValueError('Changed registered context parent source')
comparison = rule_comparison(dict(rule, voice_directory=voice_directory))
prefix = comparison_words(text, comparison)
if (not 1 <= len(prefix) <= 4 or len(text) > 80 or
comparison_words(rule['parent_text'], comparison)[:len(prefix)] != prefix or
len(comparison_words(rule['parent_text'], comparison)) <= len(prefix) or
not 0.5 <= rule['maximum_parent_seconds'] <= 5.8 or
rule['minimum_parent_match'] != 0.92):
raise ValueError('Context must begin with the exact requested short phrase')
rule.update(policy_path=str(Path(policy_path).resolve().relative_to(ROOT)),
policy_sha256=file_hash(policy_path), voice_directory=voice_directory)
rule['sha256'] = json_hash(rule)
return rule
def packet_hash(path):
"""Bind Opus audio, header gain and playback length; ignore Ogg serial/tags."""
data = Path(path).read_bytes()
if len(data) > 2 * 1024 * 1024:
raise ValueError('Oversized context clip')
packets, pending = [], bytearray()
for offset, length, _, _ in ogg_opus.pages(data):
page = bytearray(data[offset:offset + length])
checksum = int.from_bytes(page[22:26], 'little')
page[22:26] = b'\0' * 4
if ogg_opus.ogg_crc(page) != checksum:
raise ValueError('Changed context Ogg page')
segments = data[offset + 26]
at = offset + 27 + segments
for size in data[offset + 27:offset + 27 + segments]:
pending.extend(data[at:at + size])
at += size
if size < 255:
packets.append(bytes(pending))
pending.clear()
if pending or len(packets) < 3 or not packets[0].startswith(b'OpusHead'):
raise ValueError('Incomplete context Opus stream')
import hashlib
digest = hashlib.sha256()
digest.update(str(ogg_opus.opus_frames(data)).encode())
for packet in packets:
if not packet.startswith(b'OpusTags'):
digest.update(len(packet).to_bytes(4, 'little'))
digest.update(packet)
return digest.hexdigest()
def pcm_hash(pcm):
import hashlib
return hashlib.sha256(pcm).hexdigest()
def read_wav(path):
if not 44 <= Path(path).stat().st_size <= 2 * 1024 * 1024:
raise ValueError('Invalid context WAV size')
with wave.open(str(path), 'rb') as audio:
if audio.getnchannels() != 1 or audio.getsampwidth() != 2 or audio.getframerate() != 24000:
raise ValueError('Context proof requires mono 24 kHz PCM16')
pcm = audio.readframes(audio.getnframes())
if len(pcm) != audio.getnframes() * 2:
raise ValueError('Truncated context PCM')
return pcm
def write_wav(path, pcm):
with wave.open(str(path), 'wb') as audio:
audio.setnchannels(1)
audio.setsampwidth(2)
audio.setframerate(24000)
audio.writeframes(pcm)
def prefix_range(text, alignment, frames, *, retain_leading_pcm0=False, comparison=None):
if type(retain_leading_pcm0) is not bool:
raise ValueError('Explicit leading-PCM extraction mode required')
if comparison is not None and retain_leading_pcm0:
raise ValueError('Count alignment retains the ordinary onset ceiling')
target = comparison_words(text, comparison)
chunks = alignment['words']
if not isinstance(chunks, list) or not 2 <= len(chunks) <= 256:
raise ValueError('Invalid context alignment size')
alignment_text = ' '.join(chunk['text'] for chunk in chunks)
# Whisper may split an actual internal connector: penchez / -vous, l' / air.
# Reconstruct only that spacing for coherence; retain the original chunks.
alignment_text = re.sub(r"(?<=[^\W\d_])\s*([-’'])\s*(?=[^\W\d_])", r'\1', alignment_text)
if 'text' in alignment and comparison_words(alignment_text, comparison) != comparison_words(alignment['text'], comparison):
raise ValueError('Context timestamps do not describe the actual full transcription')
previous = 0.0
for chunk in chunks:
start, end = chunk['start'], chunk['end']
if (not all(isinstance(value, (int, float)) and math.isfinite(value) for value in (start, end)) or
not previous <= start <= end <= frames / 24000 + 0.04):
raise ValueError('Nonmonotonic context alignment')
previous = end
consumed = []
boundary = None
for index, chunk in enumerate(chunks):
consumed.extend(comparison_words(chunk['text'], comparison))
if consumed == target:
boundary = index
break
if target[:len(consumed)] != consumed:
break
if boundary is None or boundary + 1 >= len(chunks):
raise ValueError('Alignment does not start with the exact short phrase')
end = chunks[boundary]['end']
following = chunks[boundary + 1]['start']
if (not all(isinstance(value, (int, float)) and math.isfinite(value)
for value in (end, following, chunks[0]['start'])) or
not 0 <= chunks[0]['start'] <= end or
(not retain_leading_pcm0 and chunks[0]['start'] > 0.3) or
not 0 < end <= following <= frames / 24000 + 0.04):
raise ValueError('Invalid or overlapping context word boundary')
end_sample = round(min(end + 0.025, following) * 24000)
if not 12000 <= end_sample < frames:
raise ValueError('Context crop would be empty, too short or include the parent ending')
# ASR onset can be late within the first word. Always retain sample zero;
# the explicit mode omits only the onset ceiling, never preceding audio.
return 0, end_sample
def verifier():
spec = importlib.util.spec_from_file_location('context_parent_verifier', ROOT / 'scripts/generate-lumi-guide-narration.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.complete_match
def _parent_source(config, proof, actual_clip):
relative = PurePosixPath(proof['parent']['file'])
directory = PurePosixPath(proof['voice_directory'])
prefix = directory.parent / (directory.name + '-context') / proof['locale']
if relative != prefix / (proof['parent']['sha256'] + '.wav'):
raise ValueError('Invalid public context artifact path')
for artifact in config.get('_context_artifacts', []):
if artifact['destination_relative'] == str(relative) and artifact['sha256'] == proof['parent']['sha256']:
path = Path(artifact['source']).resolve()
if not path.is_relative_to(ROOT / 'backup/narration'):
raise ValueError('Context runtime artifact escaped ignored narration storage')
return path
actual = Path(actual_clip).resolve()
media = actual.parents[len(directory.parts) + 1]
if actual.relative_to(media).parts[:-1] != (*directory.parts, proof['locale']):
raise ValueError('Context receipt is not beside its practice voice directory')
return media / str(relative)
def source_identity(profile):
if profile.get('model') == 'inclusionAI/Ming-3.0-tts-0.5B':
from ming_narration import provenance
return provenance({key: value for key, value in profile.items() if key != 'verification'})
from f5_russian_narration import provenance
return provenance(profile)
def _crop_rule(profile, code, proof):
if (profile.get('model') != 'inclusionAI/Ming-3.0-tts-0.5B' or code != 'zh' or
not isinstance(proof, dict) or proof.get('scope') != 'context_parent' or proof.get('locale') != code or
proof.get('version') != VERSION or
proof.get('voice_directory') != 'sunsalutation/voice'):
return None
policy_path = ROOT / proof['policy_path']
policy = load_policy(policy_path)
candidates = [row for row in policy['rules'] if row['locale'] == code and
pcm_hash(row['text'].encode()) == proof['text_sha256']]
if len(candidates) != 1:
return None
rule = recipe(policy_path, code, candidates[0]['text'], proof['voice_directory'])
if (not rule.get('crop_asr') or rule['sha256'] != proof['recipe_sha256'] or
proof['proof_sha256'] != json_hash({key: value for key, value in proof.items() if key != 'proof_sha256'}) or
proof['source_identity'] != source_identity(profile)):
return None
return rule
def crop_verification_identity(profile, code, proof):
"""Explicit Chinese short-crop QA only; ordinary synthesis identity is intact."""
try:
rule = _crop_rule(profile, code, proof)
if rule is None:
return None
route = rule['crop_asr']
return dict(profile_sha256=rule['policy_sha256'], model=route['model'], revision=route['revision'],
artifacts_sha256=json_hash(route['artifacts']), adapter_sha256=file_hash(__file__),
context_recipe_sha256=rule['sha256'])
except (KeyError, TypeError, ValueError, OSError):
return None
def _verified_crop_metadata(profile, code, proof, row, identity):
try:
expected = crop_verification_identity(profile, code, proof)
if expected is None or identity != expected:
return False
samples = row['input_samples']
return (row['model'] == expected['model'] and row['revision'] == expected['revision'] and
row['artifacts_sha256'] == expected['artifacts_sha256'] and
row['adapter_sha256'] == expected['adapter_sha256'] and row['language'] == code and
row['audio_sha256'] == proof['clip_sha256'] and row['truncation'] is False and
row['long_form'] is True and type(samples) is int and 0 < samples <= 16000 * 5.8 and
type(row['seconds']) in (int, float) and math.isfinite(row['seconds']) and
abs(samples / 16000 - row['seconds']) <= 1 / 16000)
except (KeyError, TypeError, ValueError, OSError):
return False
def verified_crop_transcription(profile, code, proof, row, identity):
if not _verified_crop_metadata(profile, code, proof, row, identity):
return False
try:
rule = _crop_rule(profile, code, proof)
return words(row['text']) == words(rule['text'])
except (KeyError, TypeError, ValueError, OSError):
return False
def verify_speaker(config, profile, code, text, row, actual_clip_path):
"""Offline structural verification; parent and actual short metrics stay distinct."""
try:
proof = row['speaker_verification']
body = {key: value for key, value in proof.items() if key != 'proof_sha256'}
if (proof['scope'] != 'context_parent' or proof['version'] != VERSION or
proof['proof_sha256'] != json_hash(body) or proof['locale'] != code or
proof['text_sha256'] != pcm_hash(text.encode()) or
proof['actual_clip_cosine'] != row['speaker_cosine'] or
not math.isfinite(proof['actual_clip_cosine']) or
not -1.00001 <= proof['actual_clip_cosine'] <= 1.00001 or
not profile['minimum_speaker_cosine'] <= proof['parent_cosine'] <= 1.00001):
return False
rule = recipe(ROOT / proof['policy_path'], code, text, proof['voice_directory'])
if (not rule or rule['sha256'] != proof['recipe_sha256'] or
(config.get('_context_voice_directory') is not None and
config['_context_voice_directory'] != proof['voice_directory'])):
return False
if (file_hash(actual_clip_path) != proof['clip_sha256'] or row['sha256'] != proof['clip_sha256'] or
packet_hash(actual_clip_path) != proof['crop']['encoded_packets_sha256'] or
proof['source_identity'] != row['synthesis'] or
proof['source_identity'] != source_identity(profile) or
proof['source_identity']['model'] != profile['model'] or
proof['source_identity']['model_revision'] != profile['model_revision'] or
proof['source_identity']['reference_sha256'] != profile['reference_sha256']):
return False
parent = proof['parent']
path = _parent_source(config, proof, actual_clip_path)
pcm = read_wav(path)
start, end = prefix_range(text, parent['alignment'], len(pcm) // 2,
retain_leading_pcm0=rule.get('prefix_alignment') == RETAIN_LEADING_PCM0,
comparison=rule_comparison(rule))
if (file_hash(path) != parent['sha256'] or pcm_hash(pcm) != parent['pcm_sha256'] or
parent['text'] != rule['parent_text'] or parent['text_sha256'] != pcm_hash(parent['text'].encode()) or
not 0.5 <= len(pcm) / 48000 <= rule['maximum_parent_seconds'] or
proof['crop']['start_sample'] != start or proof['crop']['end_sample'] != end or
proof['crop']['pcm_sha256'] != pcm_hash(pcm[start * 2:end * 2]) or
not 0.5 <= row['seconds'] <= 5.8 or not 0 <= proof['crop']['float_peak'] < 1):
return False
alignment = parent['alignment']
if (alignment['model'] != ASR_MODEL or alignment['revision'] != ASR_REVISION or
alignment['language'] != code or alignment['audio_sha256'] != parent['sha256'] or
alignment['helper_sha256'] != file_hash(__file__) or
alignment['truncation'] is not False or alignment['long_form'] is not True or
abs(alignment['input_samples'] - len(pcm) / 3) > 2 or
abs(alignment['seconds'] - len(pcm) / 48000) > 2 / 16000 or
parent_match(verifier(), config, code, rule, alignment['text'], len(pcm) / 48000) < rule['minimum_parent_match']):
return False
if 'transcription' in row:
transcript = row['transcription']
crop_identity = crop_verification_identity(profile, code, proof)
if rule.get('crop_asr'):
if not verified_crop_transcription(profile, code, proof, transcript, row.get('verification')):
return False
asr_model, asr_revision = crop_identity['model'], crop_identity['revision']
elif profile.get('model') == 'inclusionAI/Ming-3.0-tts-0.5B':
route = profile['verification']['routes'][code]
asr_model, asr_revision = route['model'], route['revision']
else:
asr_model, asr_revision = profile['transcriber'], profile['asr_revision']
duration = ogg_opus.opus_seconds(Path(actual_clip_path).read_bytes())
if (transcript['audio_sha256'] != proof['clip_sha256'] or
comparison_words(transcript['text'], rule_comparison(rule)) != comparison_words(text, rule_comparison(rule)) or transcript['truncation'] is not False or
transcript['long_form'] is not True or transcript['model'] != asr_model or
transcript['revision'] != asr_revision or transcript['language'] != code or
abs(transcript['input_samples'] / 16000 - duration) > 2 / 16000 or
abs(transcript['seconds'] - duration) > 2 / 16000 or
abs(row['seconds'] - duration) > 0.001):
return False
return True
except (KeyError, IndexError, TypeError, ValueError, OSError, RuntimeError):
return False
class ContextEngine:
def __init__(self, engine, config, policy_path, local_path, *, encode, complete_match, identity, voice_directory):
self.engine = engine
self.profile = engine.profile
self.config = config
self.policy_path = Path(policy_path).resolve()
load_policy(self.policy_path)
self.local = json.loads(Path(local_path).read_text())
self.encode = encode
self.complete_match = complete_match
self.identity = identity
self.voice_directory = str(voice_directory)
self.context = None
self.context_artifacts = []
self.last_speaker_verification = None
self.context_diagnostics = {}
self.last_context_verification = None
self.last_transcription = None
def __getattr__(self, name):
return getattr(self.engine, name)
def base_speak(self, config, text, code, work, attempt):
if config.get('_cached_only') and self.profile['model'] != 'inclusionAI/Ming-3.0-tts-0.5B':
request = local_text_hash(config, text, self.profile, code)
identity = json_hash(dict(request=request, attempt=attempt))
cache = Path(self.engine.local['cache'])
pcm, receipt = cache / (identity + '.pcm'), cache / (identity + '.json')
if not pcm.is_file() or not receipt.is_file():
raise RuntimeError('Cached-only context QA requires the original local PCM: ' + identity)
saved = json.loads(receipt.read_text())
if saved.get('request') != identity or saved.get('sha256') != file_hash(pcm):
raise RuntimeError('Cached-only context QA found changed local PCM or receipt: ' + identity)
return self.engine.speak(config, text, code, work, attempt)
def speak(self, config, text, code, work, attempt):
self.context = None
self.context_artifacts = []
self.last_speaker_verification = None
self.context_diagnostics = {}
self.last_context_verification = None
self.last_transcription = None
rule = recipe(self.policy_path, code, text, self.voice_directory)
if not rule:
return self.base_speak(config, text, code, work, attempt)
if config['pcm_sample_rate'] != 24000:
raise ValueError('Context extraction requires explicit 24 kHz PCM')
cache = Path(self.local['cache']).resolve()
subprocess.run(['git', '-C', str(ROOT), 'check-ignore', '-q', str(cache / 'probe')], check=True)
cache.mkdir(parents=True, exist_ok=True)
parent_work = Path(work) / 'context-parent'
parent_work.mkdir(exist_ok=True)
pcm, identity = self.base_speak(config, rule['parent_text'], code, parent_work, attempt)
self.context_diagnostics = dict(locale=code, text=text, parent_text=rule['parent_text'],
attempt=attempt, source_identity=identity, parent_pcm_sha256=pcm_hash(pcm),
parent_seconds=len(pcm) / 48000, maximum_parent_seconds=rule['maximum_parent_seconds'])
if identity != self.identity:
raise RuntimeError('Context parent source identity failed')
if not 0.5 <= len(pcm) / 48000 <= rule['maximum_parent_seconds']:
raise ContextRejected('Context parent timing failed', self.context_diagnostics)
wav = cache / (pcm_hash(pcm) + '.wav')
write_wav(wav, pcm)
command = [self.local['python'], str(Path(__file__).resolve()), '--align-audio', str(wav),
'--language', code, '--model', self.local['model'], '--policy', str(self.policy_path)]
aligned_path = wav.with_suffix('.alignment.json')
if aligned_path.exists():
alignment = json.loads(aligned_path.read_text())
else:
alignment = None
if not alignment or alignment.get('helper_sha256') != file_hash(__file__):
alignment = json.loads(subprocess.run(command, capture_output=True, text=True, check=True).stdout)
aligned_path.write_text(json.dumps(alignment, ensure_ascii=False, indent=2) + '\n')
if (alignment['audio_sha256'] != file_hash(wav) or alignment['model'] != ASR_MODEL or
alignment['revision'] != ASR_REVISION or alignment['language'] != code or
alignment['truncation'] is not False or alignment['long_form'] is not True or
abs(alignment['input_samples'] / 16000 - len(pcm) / 48000) > 2 / 16000 or
abs(alignment['seconds'] - len(pcm) / 48000) > 2 / 16000):
raise RuntimeError('Context parent transcription integrity failed')
parent_score = parent_match(self.complete_match, config, code, rule, alignment['text'], len(pcm) / 48000)
self.context_diagnostics.update(parent_audio_sha256=file_hash(wav), parent_transcription=alignment,
parent_match=parent_score, minimum_parent_match=rule['minimum_parent_match'])
if parent_score < rule['minimum_parent_match']:
raise ContextRejected('Complete context parent transcription failed', self.context_diagnostics)
cosine = self.engine.speaker_match(wav)
self.context_diagnostics.update(parent_cosine=cosine,
minimum_parent_cosine=self.profile['minimum_speaker_cosine'])
if not self.profile['minimum_speaker_cosine'] <= cosine <= 1.00001:
raise ContextRejected('Context parent female-reference consistency failed', self.context_diagnostics)
try:
start, end = prefix_range(text, alignment, len(pcm) // 2,
retain_leading_pcm0=rule.get('prefix_alignment') == RETAIN_LEADING_PCM0,
comparison=rule_comparison(rule))
except ValueError as error:
raise ContextRejected(str(error), self.context_diagnostics) from error
crop = pcm[start * 2:end * 2]
self.context_diagnostics.update(crop_start_sample=start, crop_end_sample=end,
crop_pcm_sha256=pcm_hash(crop))
parent_sha = file_hash(wav)
directory = PurePosixPath(self.voice_directory)
relative = directory.parent / (directory.name + '-context') / code / (parent_sha + '.wav')
self.context_artifacts = [dict(source=str(wav), destination_relative=str(relative), sha256=parent_sha)]
self.context = dict(rule=rule, text=text, code=code, pcm=crop, cosine=cosine,
parent=dict(file=str(relative), sha256=parent_sha, pcm_sha256=pcm_hash(pcm),
text=rule['parent_text'], text_sha256=pcm_hash(rule['parent_text'].encode()),
alignment=alignment), start=start, end=end, work=Path(work))
return crop, identity
def speaker_match(self, clip):
actual_cosine = self.engine.speaker_match(clip)
if self.context is None:
self.last_speaker_verification = None
return actual_cosine
state = self.context
self.context_diagnostics.update(actual_clip_cosine=actual_cosine,
actual_clip_sha256=file_hash(clip))
expected_work = state['work'] / 'context-expected-crop'
expected_work.mkdir(exist_ok=True)
expected = self.encode(self.config, state['pcm'], expected_work)
if packet_hash(clip) != packet_hash(expected):
raise RuntimeError('Encoded audio is not the exact expected parent crop')
raw = subprocess.check_output(['ffmpeg', '-nostdin', '-v', 'error', '-threads', '1',
'-i', str(clip), '-ar', '16000', '-ac', '1', '-f', 'f32le', '-'])
samples = array.array('f', raw)
if not samples or not all(math.isfinite(value) for value in samples):
raise RuntimeError('Nonfinite context crop audio')
peak = max(abs(value) for value in samples)
self.context_diagnostics['crop_float_peak'] = peak
if peak >= 1:
raise ContextRejected('Context crop exceeded full scale', self.context_diagnostics)
proof = dict(scope='context_parent', version=VERSION, policy_path=state['rule']['policy_path'],
recipe_sha256=state['rule']['sha256'], locale=state['code'], text_sha256=pcm_hash(state['text'].encode()),
voice_directory=self.voice_directory, source_identity=self.identity,
actual_clip_cosine=actual_cosine, parent_cosine=state['cosine'], gender_verified=False,
clip_sha256=file_hash(clip), parent=state['parent'],
crop=dict(start_sample=state['start'], end_sample=state['end'], pcm_sha256=pcm_hash(state['pcm']),
encoded_packets_sha256=packet_hash(expected), float_peak=peak))
proof['proof_sha256'] = json_hash(proof)
self.last_speaker_verification = proof
return actual_cosine
def transcribe(self, clip, code):
proof = self.last_speaker_verification
if self.context is not None and self.context['rule'].get('crop_asr'):
identity = crop_verification_identity(self.profile, code, proof)
if identity is None:
raise ValueError('Unsupported final context transcription identity')
# The routed Ming verifier retains its recognizer. Release it before
# running the separately pinned short-crop recognizer subprocess.
if hasattr(self.engine, 'asr'):
import gc
self.engine.asr = None
self.engine.asr_model = None
gc.collect()
command = [self.local['python'], str(Path(__file__).resolve()), '--transcribe-audio', str(clip),
'--language', code, '--model', self.local['model'], '--policy', str(self.policy_path)]
row = json.loads(subprocess.run(command, capture_output=True, text=True, check=True).stdout)
if not _verified_crop_metadata(self.profile, code, proof, row, identity):
raise RuntimeError('Final context transcription integrity failed')
if words(row['text']) != words(self.context['text']):
self.context_diagnostics['crop_transcript'] = row['text']
self.context_diagnostics['crop_transcription'] = row
raise ContextRejected('Context crop contains missing, changed or adjacent words', self.context_diagnostics)
self.last_context_verification = identity
self.last_transcription = row
text = row['text']
else:
text = self.engine.transcribe(clip, code)
self.last_transcription = getattr(self.engine, 'last_transcription', None)
if (self.context is not None and
comparison_words(text, rule_comparison(self.context['rule'])) !=
comparison_words(self.context['text'], rule_comparison(self.context['rule']))):
self.context_diagnostics['crop_transcript'] = text
raise ContextRejected('Context crop contains missing, changed or adjacent words', self.context_diagnostics)
return text
def align_audio(audio, code, model_path, policy_path):
"""Blind official Whisper token timestamps: only audio and language are inputs."""
policy = load_policy(policy_path)
helper_sha256 = file_hash(__file__)
model_path = Path(model_path)
for name, digest in policy['asr_artifacts'].items():
if file_hash(model_path / name) != digest:
raise ValueError('Changed context alignment model artifact')
import os
os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', OMP_NUM_THREADS='2', MKL_NUM_THREADS='2')
import numpy as np
import torch
from transformers import WhisperProcessor, WhisperForConditionalGeneration
torch.set_num_threads(2)
raw = subprocess.check_output(['ffmpeg', '-nostdin', '-v', 'error', '-threads', '1', '-i', str(audio),
'-ac', '1', '-ar', '16000', '-f', 'f32le', '-'])
waveform = np.frombuffer(raw, dtype='<f4').copy()
if not 0 < len(waveform) <= 16000 * 5.8 or not np.isfinite(waveform).all():
raise ValueError('Context parent exceeds its complete alignment budget')
processor = WhisperProcessor.from_pretrained(model_path, local_files_only=True)
model = WhisperForConditionalGeneration.from_pretrained(model_path, local_files_only=True,
low_cpu_mem_usage=True, attn_implementation='eager').eval()
features = processor(waveform, sampling_rate=16000, return_tensors='pt', return_attention_mask=True)
with torch.inference_mode():
output = model.generate(input_features=features.input_features, attention_mask=features.attention_mask,
language=code, task='transcribe', return_timestamps=True, return_token_timestamps=True,
return_dict_in_generate=True, max_new_tokens=256, do_sample=False,
num_frames=len(waveform) // processor.feature_extractor.hop_length)
text, timing = processor.tokenizer._decode_asr(
[dict(tokens=output.sequences.cpu().numpy(), token_timestamps=output.token_timestamps.cpu().numpy())],
return_timestamps='word', return_language=False, time_precision=0.02)
return dict(text=text.strip(), language=code, model=ASR_MODEL, revision=ASR_REVISION,
audio_sha256=file_hash(audio), input_samples=len(waveform), seconds=len(waveform) / 16000,
truncation=False, long_form=True, helper_sha256=helper_sha256,
words=[dict(text=row['text'].strip(), start=row['timestamp'][0], end=row['timestamp'][1])
for row in timing['chunks']])
def transcribe_audio(audio, code, model_path, policy_path):
"""Blind complete-crop QA; the only inputs are actual audio and language."""
policy = load_policy(policy_path)
if code != 'zh':
raise ValueError('The explicit final-crop recognizer is limited to Chinese')
model_path = Path(model_path)
for name, digest in policy['asr_artifacts'].items():
if file_hash(model_path / name) != digest:
raise ValueError('Changed context crop recognition model artifact')
import os
os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', OMP_NUM_THREADS='2', MKL_NUM_THREADS='2')
import numpy as np
import torch
from transformers import WhisperProcessor, WhisperForConditionalGeneration
torch.set_num_threads(2)
raw = subprocess.check_output(['ffmpeg', '-nostdin', '-v', 'error', '-threads', '1', '-i', str(audio),
'-ac', '1', '-ar', '16000', '-f', 'f32le', '-'])
waveform = np.frombuffer(raw, dtype='<f4').copy()
if not 0 < len(waveform) <= 16000 * 5.8 or not np.isfinite(waveform).all():
raise ValueError('Invalid complete context crop audio')
processor = WhisperProcessor.from_pretrained(model_path, local_files_only=True)
model = WhisperForConditionalGeneration.from_pretrained(model_path, local_files_only=True,
low_cpu_mem_usage=True).eval()
features = processor(waveform, sampling_rate=16000, return_tensors='pt', truncation=False,
padding='longest', return_attention_mask=True)
with torch.inference_mode():
tokens = model.generate(input_features=features.input_features, attention_mask=features.attention_mask,
language=code, task='transcribe', return_timestamps=True, max_new_tokens=440, do_sample=False)
return dict(text=processor.batch_decode(tokens, skip_special_tokens=True)[0].strip(), language=code,
model=ASR_MODEL, revision=ASR_REVISION, artifacts_sha256=json_hash(policy['asr_artifacts']),
adapter_sha256=file_hash(__file__), audio_sha256=file_hash(audio),
input_samples=len(waveform), seconds=len(waveform) / 16000, truncation=False, long_form=True)
def configure():
text_root = ROOT / 'design/sun-salutation/narration/text'
config = json.loads((ROOT / 'design/sun-salutation/narration/narration.json').read_text())
rules = []
for code in ('pt', 'cs', 'ru', 'es', 'fr', 'zh'):
source_path = narration_text_path(config, code, text_root)
raw = source_path.read_text()
entries = dict(re.findall(r'\[([^\]]+)\]\n(.*?)(?=\n---|\Z)', raw, re.S))
for short_key, parent_key in [('sun_salutation_voice_breathe_in', 'sun_salutation_voice_lunge_right'),
('sun_salutation_voice_breathe_out', 'sun_salutation_voice_forward_fold')]:
if code == 'zh' and short_key == 'sun_salutation_voice_breathe_out':
parent_key = 'sun_salutation_voice_plank'
if code == 'pt' and short_key == 'sun_salutation_voice_breathe_out':
parent_key = 'sun_salutation_voice_downward_dog'
text = entries[short_key].strip()
parent = entries.get(parent_key, '').strip()
if not parent or words(parent)[:len(words(text))] != words(text):
alternatives = [(key, value.strip()) for key, value in entries.items()
if key != short_key and words(value)[:len(words(text))] == words(text)
and len(words(value)) > len(words(text))]
if not alternatives:
continue
parent_key, parent = min(alternatives, key=lambda item: len(item[1]))
rules.append(dict(locale=code, text=text, parent_text=parent,
parent_source=dict(file=str(source_path.relative_to(ROOT)),
key=parent_key, sha256=file_hash(source_path)),
maximum_parent_seconds=5.8, minimum_parent_match=0.92))
if (code, short_key) in (('cs', 'sun_salutation_voice_breathe_out'),
('ru', 'sun_salutation_voice_breathe_in')):
rules[-1]['prefix_alignment'] = RETAIN_LEADING_PCM0
profile = json.loads((ROOT / 'design/narration/chatterbox-czech.json').read_text())
for rule in rules:
if rule['locale'] == 'zh':
rule['crop_asr'] = dict(model=ASR_MODEL, revision=ASR_REVISION, artifacts=profile['asr_artifacts'])
policy = dict(version=VERSION, helper_sha256=file_hash(__file__),
verifier_sha256=file_hash(ROOT / 'scripts/generate-lumi-guide-narration.py'),
voice_directory='sunsalutation/voice', asr_model=ASR_MODEL, asr_revision=ASR_REVISION,
asr_artifacts=profile['asr_artifacts'], rules=rules)
POLICY.write_text(json.dumps(policy, ensure_ascii=False, indent=2) + '\n')
local_path = ROOT / 'backup/narration/context-local.json'
subprocess.run(['git', '-C', str(ROOT), 'check-ignore', '-q', str(local_path)], check=True)
local = dict(python='/mnt/storage/Projects/waozixyz/learning/jungle/local/speech-development/ming-evaluation/venv/bin/python',
model='/mnt/storage/Projects/harness/models/weights/asr/whisper-large-v3-turbo',
cache=str(ROOT / 'backup/narration/local-models/context-cache'))
local_path.write_text(json.dumps(local, indent=2) + '\n')
for rule in rules:
recipe(POLICY, rule['locale'], rule['text'], policy['voice_directory'])
print('Explicit context policy and ignored local paths configured; no synthesis performed.')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--configure', action='store_true')
parser.add_argument('--align-audio', type=Path)
parser.add_argument('--transcribe-audio', type=Path)
parser.add_argument('--language')
parser.add_argument('--model', type=Path)
parser.add_argument('--policy', type=Path, default=POLICY)
args = parser.parse_args()
if args.configure:
configure()
elif args.transcribe_audio and args.language and args.model:
print(json.dumps(transcribe_audio(args.transcribe_audio, args.language, args.model, args.policy), ensure_ascii=False))
elif args.align_audio and args.language and args.model:
print(json.dumps(align_audio(args.align_audio, args.language, args.model, args.policy), ensure_ascii=False))
else:
parser.error('Select --configure or --align-audio with --language and --model')