Style-Bert-VITS2 で手軽に Unity の正確な口パク情報を出力する方法

'phones': [0, 0, 0, 65, 0, 37, 0, 8, 0, 99, 0, 65, 0, 65, 0, 35, 0, 65, 0, 100, 0, 8, 0, 40, 0, 60, 0, 8, 0, 76, 0, 82, 0, 107, 0, 0, 0]

dur_frame : [15, 3, 3, 3, 3, 4, 6, 3, 5, 2, 2, 4, 4, 3, 2, 4, 3, 3, 3, 3, 4, 6, 3, 3, 6, 2, 4, 3, 3, 4, 3, 4, 5, 4, 5, 3, 8]

phonemes : ['_', '_', '_', 'o', '_', 'h', '_', 'a', '_', 'y', '_', 'o', '_', 'o', '_', 'g', '_', 'o', '_', 'z', '_', 'a', '_', 'i', '_', 'm', '_', 'a', '_', 's', '_', 'u', '_', '.', '_', '_', '_']

[
  {"t0": 0.2438, "t1": 0.4296, "v": "O", "w": 95}, 
  {"t0": 0.4296, "t1": 0.5689, "v": "A", "w": 95}, 
  {"t0": 0.5689, "t1": 0.952, "v": "O", "w": 95}, 
  {"t0": 0.952, "t1": 1.0565, "v": "A", "w": 95}, 
  {"t0": 1.0565, "t1": 1.2307, "v": "I", "w": 95}, 
  {"t0": 1.2307, "t1": 1.3816, "v": "A", "w": 95}, 
  {"t0": 1.3816, "t1": 1.4861, "v": "U", "w": 95}, 
  {"t0": 1.4861, "t1": 1.5325, "v": "REST", "w": 60}
]

{
  "utterance": "おはようございます。",
  "sample_rate": 44100,
  "blend": "{\"events\": [{\"t0\": 0.2438, \"t1\": 0.4296, \"v\": \"O\", \"w\": 95}, {\"t0\": 0.4296, \"t1\": 0.5805, \"v\": \"A\", \"w\": 95}, {\"t0\": 0.5805, \"t1\": 0.9868, \"v\": \"O\", \"w\": 95}, {\"t0\": 0.9868, \"t1\": 1.0565, \"v\": \"A\", \"w\": 95}, {\"t0\": 1.0565, \"t1\": 1.2307, \"v\": \"I\", \"w\": 95}, {\"t0\": 1.2307, \"t1\": 1.3816, \"v\": \"A\", \"w\": 95}, {\"t0\": 1.3816, \"t1\": 1.4745, \"v\": \"U\", \"w\": 95}, {\"t0\": 1.4745, \"t1\": 1.5209, \"v\": \"REST\", \"w\": 60}]}",
  "audio_wave": [
    -0.000213623046875,
    0.00006103515625,
    -0.000152587890625,
    ...
    0.000274658203125,
    -0.0001220703125,
    ]
}

[Serializable]
public class SBV2LipSync
{
    public float t0;
    public float t1;
    public string v;
    public float w;
}

[Serializable]
public class LipSyncTimeline
{
    public SBV2LipSync[] events;
}

[Serializable]
public class TextToSpeech
{
    public string utterance;
    public string blend;
    public float[] audio_wave;
}

    public IEnumerator Upload(string text)
    {
        # 公開 API に呼び出し
        using (UnityWebRequest www = new UnityWebRequest("https://lycoris52.net/ttsapi/tts", "POST"))
        {
            string dialogue = $"{{\"text\":\"{text}\"}}";
            
            byte[] bodyRaw = Encoding.UTF8.GetBytes(dialogue);
            www.uploadHandler = (UploadHandler)new UploadHandlerRaw(bodyRaw);
            www.downloadHandler = (DownloadHandler)new DownloadHandlerBuffer();
            www.SetRequestHeader("Content-Type", "application/json");
            yield return www.SendWebRequest();
            if (www.isNetworkError || www.isHttpError)
            {
                Debug.Log(www.error);
            }
            else
            {
                # API の結果を Deserialize してオブジェクト化
                string result = www.downloadHandler.text;
                TextToSpeech tts = JsonConvert.DeserializeObject<TextToSpeech>(result);
                aiueoTimelinePlayer.Play(tts.blend);

                # 同時にもらった音声データも生成する
                AudioClip audioClip = AudioClip.Create("voice", tts.audio_wave.Length, 1, 44100, false);
                audioClip.SetData(tts.audio_wave, 0);
                mAudioClipList.Add(audioClip);

                Debug.Log("API execute Successfully.");
            }
        }
    }

using System;
using UnityEngine;

public sealed class AiueoTimelinePlayer : MonoBehaviour
{
    [Header("References")]
    [SerializeField] private AudioSource audioSource;
    [SerializeField] private AiueoLipSync lipSync;
    private string timelineJson;

    [Header("Timing")]
    [Tooltip("Start mouth slightly earlier for more natural sync.")]
    [SerializeField] private float leadTime = 0.04f;

    private LipSyncTimeline _timeline;
    private int _currentIndex = -1;
    private bool _isLoaded;
    
    private void Update()
    {
        if (!_isLoaded || lipSync == null)
            return;

        float t = Mathf.Max(0f, audioSource.time + leadTime);

        int eventIndex = FindActiveEventIndex(t);
        if (eventIndex == _currentIndex)
            return;

        _currentIndex = eventIndex;

        if (_currentIndex < 0)
        {
            lipSync.SetVowel(null);
            return;
        }

        ApplyEvent(_timeline.events[_currentIndex]);
    }

    private void LoadTimeline()
    {
        if (String.IsNullOrEmpty(timelineJson))
        {
            Debug.LogWarning("Timeline JSON is not assigned.");
            return;
        }

        _timeline = JsonUtility.FromJson<LipSyncTimeline>(timelineJson);

        if (_timeline == null || _timeline.events == null || _timeline.events.Length == 0)
        {
            Debug.LogWarning("Failed to parse lip sync timeline JSON.");
            return;
        }

        _isLoaded = true;
    }

    private int FindActiveEventIndex(float time)
    {
        if (_timeline == null || _timeline.events == null)
            return -1;

        // Simple linear scan is fine for short dialogue.
        for (int i = 0; i < _timeline.events.Length; i++)
        {
            var e = _timeline.events[i];
            if (time >= e.t0 && time < e.t1)
                return i;
        }

        return -1;
    }

    private void ApplyEvent(SBV2LipSync e)
    {
        switch (e.v)
        {
            case "A":
                lipSync.SetVowel(JpVowel.A, e.w);
                break;
            case "I":
                lipSync.SetVowel(JpVowel.I, e.w);
                break;
            case "U":
                lipSync.SetVowel(JpVowel.U, e.w);
                break;
            case "E":
                lipSync.SetVowel(JpVowel.E, e.w);
                break;
            case "O":
                lipSync.SetVowel(JpVowel.O, e.w);
                break;
            case "REST":
            default:
                lipSync.SetVowel(null, 0f, e.w);
                break;
        }
    }

    public void Play(string json)
    {
        timelineJson = json;
        _currentIndex = -1;
        _isLoaded = false;

        LoadTimeline();
    }
}

using UnityEngine;

public enum JpVowel { A,I,U,E,O }

public sealed class AiueoLipSync : MonoBehaviour
{
    [Header("Target")]
    [SerializeField] private SkinnedMeshRenderer faceRenderer;

    [Header("Blendshape Indices")]
    [SerializeField] private int blendA = -1;
    [SerializeField] private int blendI = -1;
    [SerializeField] private int blendU = -1;
    [SerializeField] private int blendE = -1;
    [SerializeField] private int blendO = -1;
    [SerializeField] private int blendRest = -1;

    [Header("Smoothing")]
    [Range(1f, 40f)]
    [SerializeField] private float weightSpeed = 12f;

    [SerializeField] private bool normalizeTo100 = true;
    
    [Range(0.1f, 1f)]
    [SerializeField] private float blendWeight = 1.0f;

    private const float MaxIntensity = 100.0f;
    private const float MaxRestIntensity = 40.0f;
    private const int BlendShapeCount = 5;
    

    private readonly float[] _current = new float[5];
    private readonly float[] _target = new float[5];
    private float _restCurrent;
    private float _restTarget;

    public SkinnedMeshRenderer FaceRenderer => faceRenderer;

    public int BlendA => blendA;
    public int BlendI => blendI;
    public int BlendU => blendU;
    public int BlendE => blendE;
    public int BlendO => blendO;
    public int BlendRest => blendRest;

    public void SetBlendShapeIndex(JpVowel vowel, int index)
    {
        switch (vowel)
        {
            case JpVowel.A: blendA = index; break;
            case JpVowel.I: blendI = index; break;
            case JpVowel.U: blendU = index; break;
            case JpVowel.E: blendE = index; break;
            case JpVowel.O: blendO = index; break;
        }
    }

    public void SetRestBlendShapeIndex(int index)
    {
        blendRest = index;
    }

    public void SetVowel(JpVowel? vowel, float intensity = MaxIntensity, float restWeight = 0f)
    {
        intensity = Mathf.Clamp(intensity, 0f, MaxIntensity);
        restWeight = Mathf.Clamp(restWeight, 0f, MaxIntensity);

        System.Array.Clear(_target, 0, _target.Length);

        if (vowel.HasValue)
        {
            _target[(int)vowel.Value] = intensity;
            _restTarget = 0f;
        }
        else
        {
            _restTarget = blendRest >= 0 ? Mathf.Max(restWeight, MaxRestIntensity) : 0f;
        }

        if (normalizeTo100)
            NormalizeTargets();
    }

    private void Update()
    {
        if (faceRenderer == null || faceRenderer.sharedMesh == null)
            return;

        float k = 1f - Mathf.Exp(-weightSpeed * Time.deltaTime);

        for (int i = 0; i < BlendShapeCount; i++)
            _current[i] = Mathf.Lerp(_current[i], _target[i], k);

        _restCurrent = Mathf.Lerp(_restCurrent, _restTarget, k);

        ApplyBlendshapes();
    }

    private void NormalizeTargets()
    {
        float sum = 0f;
        for (int i = 0; i < BlendShapeCount; i++)
            sum += _target[i];

        if (sum > MaxIntensity)
        {
            float scale = MaxIntensity / sum;
            for (int i = 0; i < BlendShapeCount; i++)
                _target[i] *= scale;
        }

        for (int i = 0; i < BlendShapeCount; i++)
            _target[i] *= blendWeight;
    }

    private void ApplyBlendshapes()
    {
        if (blendA >= 0) faceRenderer.SetBlendShapeWeight(blendA, _current[(int)JpVowel.A]);
        if (blendI >= 0) faceRenderer.SetBlendShapeWeight(blendI, _current[(int)JpVowel.I]);
        if (blendU >= 0) faceRenderer.SetBlendShapeWeight(blendU, _current[(int)JpVowel.U]);
        if (blendE >= 0) faceRenderer.SetBlendShapeWeight(blendE, _current[(int)JpVowel.E]);
        if (blendO >= 0) faceRenderer.SetBlendShapeWeight(blendO, _current[(int)JpVowel.O]);

        if (blendRest >= 0)
            faceRenderer.SetBlendShapeWeight(blendRest, _restCurrent);
    }
}

using System.Collections.Generic;
using UnityEditor;
using UnityEngine;

[CustomEditor(typeof(AiueoLipSync))]
public sealed class AiueoLipSyncEditor : Editor
{
    private SerializedProperty _faceRendererProp;
    private SerializedProperty _blendAProp;
    private SerializedProperty _blendIProp;
    private SerializedProperty _blendUProp;
    private SerializedProperty _blendEProp;
    private SerializedProperty _blendOProp;
    private SerializedProperty _blendRestProp;
    private SerializedProperty _weightSpeedProp;
    private SerializedProperty _normalizeTo100Prop;
    private SerializedProperty _blendWeightProp;

    private void OnEnable()
    {
        _faceRendererProp = serializedObject.FindProperty("faceRenderer");
        _blendAProp = serializedObject.FindProperty("blendA");
        _blendIProp = serializedObject.FindProperty("blendI");
        _blendUProp = serializedObject.FindProperty("blendU");
        _blendEProp = serializedObject.FindProperty("blendE");
        _blendOProp = serializedObject.FindProperty("blendO");
        _blendRestProp = serializedObject.FindProperty("blendRest");
        _weightSpeedProp = serializedObject.FindProperty("weightSpeed");
        _normalizeTo100Prop = serializedObject.FindProperty("normalizeTo100");
        _blendWeightProp = serializedObject.FindProperty("blendWeight");
    }

    public override void OnInspectorGUI()
    {
        serializedObject.Update();

        EditorGUILayout.PropertyField(_faceRendererProp);
        EditorGUILayout.Space();

        var smr = _faceRendererProp.objectReferenceValue as SkinnedMeshRenderer;
        var names = GetBlendShapeNames(smr);

        if (smr == null || smr.sharedMesh == null || names.Count == 0)
        {
            EditorGUILayout.HelpBox("Assign a SkinnedMeshRenderer with blendshapes.", MessageType.Info);

            EditorGUILayout.PropertyField(_blendAProp);
            EditorGUILayout.PropertyField(_blendIProp);
            EditorGUILayout.PropertyField(_blendUProp);
            EditorGUILayout.PropertyField(_blendEProp);
            EditorGUILayout.PropertyField(_blendOProp);
            EditorGUILayout.PropertyField(_blendRestProp);
        }
        else
        {
            DrawBlendShapePopup("A", _blendAProp, names);
            DrawBlendShapePopup("I", _blendIProp, names);
            DrawBlendShapePopup("U", _blendUProp, names);
            DrawBlendShapePopup("E", _blendEProp, names);
            DrawBlendShapePopup("O", _blendOProp, names);
            DrawBlendShapePopup("Rest", _blendRestProp, names, true);

            EditorGUILayout.Space();

            if (GUILayout.Button("Auto Assign From Name"))
            {
                AutoAssign(_blendAProp, names, "a", "mouth_a", "vrc.aa", "aa");
                AutoAssign(_blendIProp, names, "i", "mouth_i", "vrc.ii", "ii");
                AutoAssign(_blendUProp, names, "u", "mouth_u", "vrc.uu", "uu");
                AutoAssign(_blendEProp, names, "e", "mouth_e", "vrc.ee", "ee");
                AutoAssign(_blendOProp, names, "o", "mouth_o", "vrc.oo", "oo");
                AutoAssign(_blendRestProp, names, "rest", "neutral", "close", "mouth_rest", "mouth_close");
            }
        }

        EditorGUILayout.Space();
        EditorGUILayout.PropertyField(_weightSpeedProp);
        EditorGUILayout.PropertyField(_normalizeTo100Prop);
        EditorGUILayout.PropertyField(_normalizeTo100Prop);
        EditorGUILayout.PropertyField(_blendWeightProp);

        serializedObject.ApplyModifiedProperties();
    }

    private static List<string> GetBlendShapeNames(SkinnedMeshRenderer smr)
    {
        var result = new List<string>();

        if (smr == null || smr.sharedMesh == null)
            return result;

        Mesh mesh = smr.sharedMesh;
        for (int i = 0; i < mesh.blendShapeCount; i++)
            result.Add(mesh.GetBlendShapeName(i));

        return result;
    }

    private static void DrawBlendShapePopup(string label, SerializedProperty prop, List<string> names, bool allowNone = false)
    {
        var options = new List<string>();

        if (allowNone)
            options.Add("None (-1)");
        else
            options.Add("None");

        options.AddRange(names);

        int currentIndex = prop.intValue;
        int popupIndex = currentIndex >= 0 ? currentIndex + 1 : 0;

        popupIndex = EditorGUILayout.Popup(label, popupIndex, options.ToArray());
        prop.intValue = popupIndex - 1;
    }

    private static void AutoAssign(SerializedProperty prop, List<string> names, params string[] keywords)
    {
        for (int i = 0; i < names.Count; i++)
        {
            string lower = names[i].ToLowerInvariant();
            foreach (string keyword in keywords)
            {
                if (lower.Contains(keyword))
                {
                    prop.intValue = i;
                    return;
                }
            }
        }
    }
}