Bump version to 0.1.3

Batching multiple updates to the leaky bucket at a fixed interval
improves the accuracy of the rate limiter. Previously the rate would
drop substantially over the course of the sync operation.

example_config.yml

@@ -16,4 +16,4 @@ spotify:
 
 # increasing these parameters should increase the search speed, while decreasing reduces likelihood of 429 errors
 max_concurrency: 10 # max concurrent connections at any given time
-rate_limit:      12 # max sustained connections per second
+rate_limit:      10 # max sustained connections per second

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotify_to_tidal"
-version = "0.1.2"
+version = "0.1.3"
 requires-python = ">= 3.10"
 
 dependencies = [

readme.md

@@ -1,4 +1,4 @@
-A command line tool for importing your Spotify playlists into Tidal
+A command line tool for importing your Spotify playlists into Tidal. Due to various performance optimisations, it is particularly suited for periodic synchronisation of very large collections.
 
 Installation
 -----------

src/spotify_to_tidal/sync.py

@@ -2,6 +2,7 @@
 
 import asyncio
 from .cache import failure_cache, track_match_cache
+import datetime
 from functools import partial
 from typing import List, Sequence, Set, Mapping
 import math
@@ -224,10 +225,16 @@ def get_tracks_for_new_tidal_playlist(spotify_tracks: Sequence[t_spotify.Spotify
 
 async def sync_playlist(spotify_session: spotipy.Spotify, tidal_session: tidalapi.Session, spotify_playlist, tidal_playlist: tidalapi.Playlist | None, config):
     async def _run_rate_limiter(semaphore):
-        ''' Leaky bucket algorithm for rate limiting. Periodically releases an item from semaphore at rate_limit'''
+        ''' Leaky bucket algorithm for rate limiting. Periodically releases items from semaphore at rate_limit'''
+        _sleep_time = config.get('max_concurrency', 10)/config.get('rate_limit', 10)/4 # aim to sleep approx time to drain 1/4 of 'bucket'
+        t0 = datetime.datetime.now()
         while True:
-            await asyncio.sleep(1/config.get('rate_limit', 12)) # sleep for min time between new function executions
+            await asyncio.sleep(_sleep_time)
-            semaphore.release() # leak one item from the 'bucket'
+            t = datetime.datetime.now()
+            dt = (t - t0).total_seconds()
+            new_items = round(config.get('rate_limit', 10)*dt)
+            t0 = t
+            [semaphore.release() for i in range(new_items)] # leak new_items from the 'bucket'
 
     # Create a new Tidal playlist if required
     if not tidal_playlist:
@@ -296,6 +303,7 @@ async def get_playlists_from_spotify(spotify_session: spotipy.Spotify, config):
     print("Loading Spotify playlists")
     results = spotify_session.user_playlists(config['spotify']['username'])
     exclude_list = set([x.split(':')[-1] for x in config.get('excluded_playlists', [])])
+    playlists.extend([p for p in results['items'] if p['owner']['id'] == config['spotify']['username'] and not p['id'] in exclude_list])
       
     # get all the remaining playlists in parallel
     if results['next']: