Bugfix/remove limit on popular words (#113)

* Replace in-memory caching with Redis caching.

* Solve cycling problem for the next 100+ years

Api/Program.cs

@@ -1,5 +1,4 @@
 using Api.ExceptionHandler;
-using Application.Cache;
 using Application.Domain;
 using Application.Events;
 using Application.Migrator;
@@ -126,7 +125,6 @@
 services.AddSingleton<ITwitterService, TwitterService>();
 services.AddTwitterClient(configuration);
 
-builder.Services.AddMemoryCache();
 builder.Services.SetupHangfire(Guard.Against.NullOrEmpty(configuration.GetRequiredSection("MongoDB:ConnectionString").Value));
 builder.Services.SetupRedis(configuration);
 

Core/Cache/IRecentIndexesCache.cs

@@ -1,6 +1,6 @@
 namespace Core.Cache
 {
-    public interface IRecentIndexesCache : ICache<string>
+    public interface IRecentIndexesCache : ISetBasedCache<string>
     {
     }
 }

Core/Cache/IRecentSearchesCache.cs

@@ -1,6 +1,6 @@
 namespace Core.Cache
 {
-    public interface IRecentSearchesCache : ICache<string>
+    public interface IRecentSearchesCache : ISetBasedCache<string>
     {
         Task<IEnumerable<string>> GetMostPopular();
     }

Core/Cache/ICache.cs → Core/Cache/ISetBasedCache.cs

@@ -6,7 +6,7 @@
 
 namespace Core.Cache
 {
-    public interface ICache<T>
+    public interface ISetBasedCache<T>
     {
         Task<IEnumerable<T>> Get();
         Task Stack(T item);

Core/Cache/ISimpleCache.cs

@@ -0,0 +1,8 @@
+namespace Core.Cache
+{
+    public interface ISimpleCache
+    {
+        Task SetAsync<T>(string key, T value, TimeSpan? expiry = null);
+        Task<T?> GetAsync<T>(string key);
+    }
+}

Infrastructure/Redis/DependencyInjection.cs

@@ -1,4 +1,5 @@
 using Ardalis.GuardClauses;
+using Core.Cache;
 using Infrastructure.Configuration;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -15,6 +16,7 @@ public static IServiceCollection SetupRedis(this IServiceCollection services, IC
             services.Configure<RedisConfig>(configuration.GetRequiredSection(SectionName));
             var redisConnectionString = Guard.Against.NullOrEmpty(configuration.GetConnectionString(SectionName));
             services.AddSingleton<IConnectionMultiplexer>(ConnectionMultiplexer.Connect(redisConnectionString));
+            services.AddSingleton<ISimpleCache, SimpleRedisCache>();
             return services;
         }
     }

Infrastructure/Redis/RedisRecentSearchesCache.cs

@@ -11,9 +11,14 @@ public class RedisRecentSearchesCache(
     {
         private const string RecentSearchesKey = "recent_searches";
         private const string PopularSearchesKey = "popular_searches";
+        private static readonly DateTime StartDate;
         private const int MaxItemsToReturn = 5;
         private const int MaxRecentSearches = 10;
-        private const int MaxPopularSearches = 1000; // Use a large number to ensure that items have time to get promoted.
+
+        static RedisRecentSearchesCache()
+        {
+            StartDate = new(2024, 9, 17); // Do not change
+        }
 
         public async Task<IEnumerable<string>> Get()
         {
@@ -46,8 +51,7 @@ public async Task Stack(string item)
 
             // TODO: Do a periodic caching, like daily where the most popular items from the previous period are brought forward into the next day
             var currentScore = (await _cache.SortedSetScoreAsync(PopularSearchesKey, item)) ?? 0;
-            _ = transaction.SortedSetAddAsync(PopularSearchesKey, item, (int)currentScore + 1 + GetNormalizedTimestamp());
-            _ = transaction.SortedSetRemoveRangeByRankAsync(PopularSearchesKey, 0, -(MaxPopularSearches + 1));
+            _ = transaction.SortedSetAddAsync(PopularSearchesKey, item, (int)++currentScore + GetNormalizedTimestamp());
 
             // Execute the transaction
             bool committed = await transaction.ExecuteAsync();
@@ -59,9 +63,8 @@ public async Task Stack(string item)
 
         static double GetNormalizedTimestamp()
         {
-            // This can be improved by addressing the time-cycle reset problem.
-            long unixTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
-            return (unixTimestamp % 1000000) / 1000000.0;
+            TimeSpan timeSinceStartDate = DateTime.Now - StartDate;
+            return timeSinceStartDate.TotalSeconds / 10_000_000_000; // It will take over 100 years for this value to grow to 1.
         }
     }
 }

Infrastructure/Redis/SimpleRedisCache.cs

@@ -0,0 +1,32 @@
+using Ardalis.GuardClauses;
+using Core.Cache;
+using Infrastructure.Configuration;
+using Microsoft.Extensions.Options;
+using StackExchange.Redis;
+namespace Infrastructure.Redis
+{
+    public class SimpleRedisCache(IConnectionMultiplexer connectionMultiplexer, IOptions<RedisConfig> redisConfig) :
+        RedisCache(connectionMultiplexer, redisConfig), ISimpleCache
+    {
+        public async Task<T?> GetAsync<T>(string key)
+        {
+            RedisValue theValue = await _cache.StringGetAsync(key);
+
+            return theValue.IsNullOrEmpty ? default : ConvertToType<T>(theValue);
+        }
+
+        private static T ConvertToType<T>(RedisValue value)
+        {
+            if(typeof(T) == typeof(DateTimeOffset))
+            {
+                return (T)(object)DateTimeOffset.Parse(value.ToString());
+            }
+            return (T)Convert.ChangeType(value.ToString(), typeof(T));
+        }
+
+        public async Task SetAsync<T>(string key, T value, TimeSpan? expiry = null)
+        {
+            await _cache.StringSetAsync(key, Guard.Against.Null(value)!.ToString(), expiry);
+        }
+    }
+}

Infrastructure/Twitter/TwitterService.cs

@@ -2,23 +2,23 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using System.Diagnostics;
-using Microsoft.Extensions.Caching.Memory;
 using Hangfire;
 using Infrastructure.Configuration;
+using Core.Cache;
 
 namespace Infrastructure.Twitter
 {
     public class TwitterService(
         ITwitterClientV2 twitterApiClient,
         ILogger<TwitterService> logger,
         IOptions<TwitterConfig> twitterConfig,
-        IMemoryCache cache,
+        ISimpleCache cache,
         IBackgroundJobClientV2 backgroundJobClient) : ITwitterService
     {
         private readonly ITwitterClientV2 _twitterApiClient = twitterApiClient;
         private readonly ILogger<TwitterService> _logger = logger;
         private readonly TwitterConfig _twitterConfig = twitterConfig.Value;
-        private readonly IMemoryCache _memoryCache = cache;
+        private readonly ISimpleCache _simpleCache = cache;
         private readonly IBackgroundJobClientV2 _backgroundJobClient = backgroundJobClient;
         private static readonly SemaphoreSlim _semaphore;
         private const string LastTweetPublishedKey = "LastTweetPublished";
@@ -48,10 +48,10 @@ private async Task PostTweetAsync(string tweetText, CancellationToken cancellati
             await _semaphore.WaitAsync(cancellationToken); // We want to be scheduling only one tweet at a time.
             try
             {
-                var foundLastPublished = _memoryCache.TryGetValue(LastTweetPublishedKey, out DateTimeOffset lastTweetPublished);
+                var lastTweetPublished = await _simpleCache.GetAsync<DateTimeOffset>(LastTweetPublishedKey);
                 var nextTweetTime = lastTweetPublished.AddSeconds(_twitterConfig.TweetIntervalSeconds);
 
-                if (foundLastPublished && nextTweetTime > DateTimeOffset.Now)
+                if (lastTweetPublished != default && nextTweetTime > DateTimeOffset.Now)
                 {
                     _backgroundJobClient.Schedule(() => SendTweetAsync(tweetText), nextTweetTime);
                 }
@@ -61,7 +61,7 @@ private async Task PostTweetAsync(string tweetText, CancellationToken cancellati
                     _backgroundJobClient.Enqueue(() => SendTweetAsync(tweetText));
                 }
 
-                _memoryCache.Set(LastTweetPublishedKey, nextTweetTime);
+                await _simpleCache.SetAsync(LastTweetPublishedKey, nextTweetTime);
             }
             finally
             {