Sound Scheduling Improvements (#491)

* port conductor adjustments

* scheduled sounds prebake

* allow aiff files to be imported

add vbr mp3 warning to readme

* improve wording

.gitignore

@@ -83,4 +83,6 @@ crashlytics-build.properties
 # Built AssetBundles
 /[Aa]ssets/[Ss]treamingAssets/*/*
 /[Aa]ssets/[Ss]treamingAssets/*.manifest
-/[Aa]ssets/[Ss]treamingAssets/*.meta
+/[Aa]ssets/[Ss]treamingAssets/*.meta
+/[Aa]ssets/[Ss]treamingAssets/[Ss]treamingAssets
+Assets/TextMesh Pro/Resources/Fonts & Materials/LiberationSans SDF - Fallback.asset

Assets/Scripts/Conductor.cs

@@ -88,27 +88,33 @@ namespace HeavenStudio
             minigamePitch = pitch;
             musicSource.pitch = SongPitch;
         }
-        
 
         void Awake()
         {
             instance = this;
         }
 
+        void Start()
+        {
+            musicSource.priority = 0;
+        }
+
         public void SetBeat(double beat)
         {
-            double secFromBeat = GetSongPosFromBeat(beat);
+            var chart = GameManager.instance.Beatmap;
+            double offset = chart.data.offset;
+            double startPos = GetSongPosFromBeat(beat);
 
-            if (musicSource.clip != null)
+            double dspTime = AudioSettings.dspTime;
-            {
-                if (secFromBeat < musicSource.clip.length)
-                    musicSource.time = (float) secFromBeat;
-                else
-                    musicSource.time = 0;
-            }
 
-            GameManager.instance.SetCurrentEventToClosest((float) beat);
+            time = startPos;
-            songPosBeat = beat;
+            firstBeatOffset = offset;
+
+            SeekMusicToTime(startPos);
+
+            songPosBeat = GetBeatFromSongPos(time);
+            
+            GameManager.instance.SetCurrentEventToClosest(beat);
         }
 
         public void Play(double beat)
@@ -116,35 +122,35 @@ namespace HeavenStudio
             if (isPlaying) return;
             var chart = GameManager.instance.Beatmap;
             double offset = chart.data.offset;
-            bool negativeOffset = offset < 0;
             double dspTime = AudioSettings.dspTime;
             GameManager.instance.SortEventsList();
 
             double startPos = GetSongPosFromBeat(beat);
+            firstBeatOffset = offset;
             time = startPos;
 
             if (musicSource.clip != null && startPos < musicSource.clip.length - offset)
             {
-                // https://www.desmos.com/calculator/81ywfok6xk
+                SeekMusicToTime(startPos);
                 double musicStartDelay = -offset - startPos;
                 if (musicStartDelay > 0)
                 {
-                    musicSource.time = 0;
-                    // this can break if the user changes pitch before the audio starts playing
                     musicScheduledTime = dspTime + musicStartDelay / SongPitch;
                     musicScheduledPitch = SongPitch;
                     musicSource.PlayScheduled(musicScheduledTime);
                 }
                 else
                 {
-                    musicSource.time = (float)-musicStartDelay;
+                    musicScheduledTime = dspTime;
-                    musicSource.PlayScheduled(dspTime);
+                    musicScheduledPitch = SongPitch;
+
+                    musicSource.Play();
                 }
             }
 
             songPosBeat = GetBeatFromSongPos(time);
             startTime = DateTime.Now;
-            lastAbsTime = (DateTime.Now - startTime).TotalSeconds;
+            lastAbsTime = 0;
             lastDspTime = AudioSettings.dspTime;
             dspStart = dspTime;
             startBeat = songPosBeat;
@@ -175,6 +181,26 @@ namespace HeavenStudio
             musicSource.Stop();
         }
 
+        void SeekMusicToTime(double startPos)
+        {
+            double offset = GameManager.instance.Beatmap.data.offset;
+            if (musicSource.clip != null && startPos < musicSource.clip.length - offset)
+            {
+                // https://www.desmos.com/calculator/81ywfok6xk
+                double musicStartDelay = -offset - startPos;
+                if (musicStartDelay > 0)
+                {
+                    musicSource.timeSamples = 0;
+                }
+                else
+                {
+                    int freq = musicSource.clip.frequency;
+                    int samples = (int)(freq * (startPos + offset));
+
+                    musicSource.timeSamples = samples;
+                }
+            }
+        }
         double deltaTimeReal { get { 
             double ret = absTime - lastAbsTime; 
             lastAbsTime = absTime;
@@ -217,7 +243,7 @@ namespace HeavenStudio
                 time += dt * SongPitch;
 
                 songPos = time;
-                songPosBeat = GetBeatFromSongPos(songPos - firstBeatOffset);
+                songPosBeat = GetBeatFromSongPos(songPos);
             }
         }
 
@@ -349,7 +375,7 @@ namespace HeavenStudio
             public double GetBeatFromSongPos(double seconds)
             {
                 double lastTempoChangeBeat = 0f;
-                double counterSeconds = -firstBeatOffset;
+                double counterSeconds = 0;
                 float lastBpm = 120f;
                 
                 foreach (RiqEntity t in GameManager.instance.Beatmap.TempoChanges)

Assets/Scripts/LevelEditor/Editor.cs

@@ -250,7 +250,7 @@ namespace HeavenStudio.Editor
         {
             var extensions = new[]
             {
-                new ExtensionFilter("Music Files", "mp3", "ogg", "wav")
+                new ExtensionFilter("Music Files", "mp3", "ogg", "wav", "aiff", "aif", "aifc")
             };
 
             #if UNITY_STANDALONE_WINDOWS

Assets/Scripts/Util/Sound.cs

@@ -33,7 +33,9 @@ namespace HeavenStudio.Util
 
         bool playInstant = false;
         bool played = false;
+        bool queued = false;
 
+        const double PREBAKE_TIME = 0.5;
         private void Start()
         {
             audioSource = GetComponent<AudioSource>();
@@ -56,18 +58,29 @@ namespace HeavenStudio.Util
                 playInstant = false;
                 scheduledPitch = cnd.SongPitch;
                 startTime = (AudioSettings.dspTime + (cnd.GetSongPosFromBeat(beat) - cnd.songPositionAsDouble)/(double)scheduledPitch) - offset;
-                audioSource.PlayScheduled(startTime);
+
+                if (scheduledPitch != 0 && AudioSettings.dspTime >= startTime)
+                {
+                    audioSource.PlayScheduled(startTime);
+                    queued = true;
+                }
             }
         }
 
         private void Update()
         {
             Conductor cnd = Conductor.instance;
+            double dspTime = AudioSettings.dspTime;
             if (!played)
             {
                 if (scheduled)
                 {
-                    if (scheduledPitch != 0 && AudioSettings.dspTime > scheduledTime)
+                    if (!queued && dspTime > scheduledTime - PREBAKE_TIME)
+                    {
+                        audioSource.PlayScheduled(scheduledTime);
+                        queued = true;
+                    }
+                    if (scheduledPitch != 0 && dspTime > scheduledTime)
                     {
                         StartCoroutine(NotRelyOnBeatSound());
                         played = true;
@@ -75,7 +88,13 @@ namespace HeavenStudio.Util
                 }
                 else if (!playInstant)
                 {
-                    if (scheduledPitch != 0 && AudioSettings.dspTime > startTime)
+                    if (!queued && dspTime > startTime - PREBAKE_TIME)
+                    {
+                        startTime = (dspTime + (cnd.GetSongPosFromBeat(beat) - cnd.songPositionAsDouble)/(double)scheduledPitch) - offset;
+                        audioSource.PlayScheduled(startTime);
+                        queued = true;
+                    }
+                    if (scheduledPitch != 0 && dspTime > startTime)
                     {
                         played = true;
                         StartCoroutine(NotRelyOnBeatSound());
@@ -87,7 +106,8 @@ namespace HeavenStudio.Util
                             if (cnd.SongPitch == 0)
                             {
                                 scheduledPitch = cnd.SongPitch;
-                                audioSource.Pause();
+                                if (queued)
+                                    audioSource.Pause();
                             }
                             else
                             {
@@ -96,8 +116,9 @@ namespace HeavenStudio.Util
                                     audioSource.UnPause();
                                 }
                                 scheduledPitch = cnd.SongPitch;
-                                startTime = (AudioSettings.dspTime + (cnd.GetSongPosFromBeat(beat) - cnd.songPositionAsDouble)/(double)scheduledPitch);
+                                startTime = (dspTime + (cnd.GetSongPosFromBeat(beat) - cnd.songPositionAsDouble)/(double)scheduledPitch);
-                                audioSource.SetScheduledStartTime(startTime);
+                                if (queued)
+                                    audioSource.SetScheduledStartTime(startTime);
                             }
                         }
                     }
@@ -121,7 +142,7 @@ namespace HeavenStudio.Util
         {
             if (!looping) // Looping sounds are destroyed manually.
             {
-                yield return new WaitForSeconds(clip.length / pitch);
+                yield return new WaitUntil(() => !audioSource.isPlaying);
                 Delete();
             }
         }

Assets/Scripts/Util/SoundByte.cs

@@ -235,7 +235,6 @@ namespace HeavenStudio.Util
             
             snd.scheduled = true;
             snd.scheduledTime = targetTime;
-            audioSource.PlayScheduled(targetTime);
             
             GameManager.instance.SoundObjects.Add(oneShot);
 

Packages/packages-lock.json

@@ -8,7 +8,7 @@
         "com.unity.nuget.newtonsoft-json": "3.2.1",
         "jillejr.newtonsoft.json-for-unity.converters": "1.5.1"
       },
-      "hash": "2fb235a6da261fb1ae60112f9e9ba0d924e96bb8"
+      "hash": "ac56f4d95417af5dbedf20f3b8d79b9f67c72659"
     },
     "com.unity.2d.sprite": {
       "version": "1.0.0",

README.md

@@ -39,6 +39,7 @@ These builds include experimental new features that will not be included in Rele
 
 
 #### Important Notes:
+- MP3 audio with variable bitrate encoding may [desync when seeking in the editor](https://github.com/RHeavenStudio/HeavenStudio/issues/490). Either use MP3s with constant bitrate encoding or use one of our other supported formats (OGG Vorbis, WAV...)
 - On MacOS and Linux builds you may [experience bugs with audio-related tasks](https://github.com/RHeavenStudio/HeavenStudio/issues/72), but in most cases Heaven Studio works perfectly.
 - On MacOS you'll need to have Discord open in the background for now, there's a bug that causes the DiscordSDK library to crash when the rich presence is updated while Discord is not open in the background.