diff --git a/Api/Api.csproj b/Api/Api.csproj index 672f7ae..7a53800 100644 --- a/Api/Api.csproj +++ b/Api/Api.csproj @@ -8,6 +8,7 @@ ..\docker-compose.dcproj + diff --git a/Api/Controllers/SearchController.cs b/Api/Controllers/SearchController.cs index 9d2250b..90714fa 100644 --- a/Api/Controllers/SearchController.cs +++ b/Api/Controllers/SearchController.cs @@ -55,9 +55,9 @@ public async Task Search([FromQuery(Name = "q"), Required] string var matches = await _searchService.Search(searchTerm); // TODO: Check if the comparison here removes takes diacrits into consideration - if (matches.Count() == 1 && matches.First().Name.ToLower() == searchTerm.ToLower()) + if (matches.Count() == 1 && matches.First().Name.Equals(searchTerm, StringComparison.CurrentCultureIgnoreCase)) { - await _eventPubService.PublishEvent(new ExactNameSearched(searchTerm)); + await _eventPubService.PublishEvent(new ExactNameSearched(matches.First().Name)); } return Ok(matches.MapToDtoCollection()); @@ -96,7 +96,7 @@ public async Task SearchOne(string searchTerm) if(nameEntry != null) { - await _eventPubService.PublishEvent(new ExactNameSearched(searchTerm)); + await _eventPubService.PublishEvent(new ExactNameSearched(nameEntry.Name)); } return Ok(nameEntry); diff --git a/Api/Program.cs b/Api/Program.cs index 1128f4a..a3d1baf 100644 --- a/Api/Program.cs +++ b/Api/Program.cs @@ -18,6 +18,8 @@ using System.Text.Json.Serialization; using Hangfire; using Infrastructure.Hangfire; +using Infrastructure.Redis; +using Ardalis.GuardClauses; var builder = WebApplication.CreateBuilder(args); var configuration = builder.Configuration; @@ -44,8 +46,15 @@ services.AddAuthorization(options => { options.AddPolicy("AdminOnly", policy => policy.RequireRole(Role.ADMIN.ToString())); - options.AddPolicy("AdminAndLexicographers", policy => policy.RequireRole(Role.ADMIN.ToString(), Role.PRO_LEXICOGRAPHER.ToString(), Role.BASIC_LEXICOGRAPHER.ToString())); - options.AddPolicy("AdminAndProLexicographers", policy => policy.RequireRole(Role.ADMIN.ToString(), Role.PRO_LEXICOGRAPHER.ToString())); + options.AddPolicy("AdminAndLexicographers", policy => policy.RequireRole( + Role.ADMIN.ToString(), + Role.PRO_LEXICOGRAPHER.ToString(), + Role.BASIC_LEXICOGRAPHER.ToString() + )); + options.AddPolicy("AdminAndProLexicographers", policy => policy.RequireRole( + Role.ADMIN.ToString(), + Role.PRO_LEXICOGRAPHER.ToString() + )); }); services.AddControllers().AddJsonOptions(options => @@ -87,10 +96,12 @@ }); }); var mongoDbSettings = configuration.GetRequiredSection("MongoDB"); -services.InitializeDatabase(mongoDbSettings.GetValue("ConnectionString"), mongoDbSettings.GetValue("DatabaseName")); +services.InitializeDatabase( + Guard.Against.NullOrEmpty(mongoDbSettings.GetValue("ConnectionString")), + Guard.Against.NullOrEmpty(mongoDbSettings.GetValue("DatabaseName"))); builder.Services.AddTransient(x => - new MySqlConnection(builder.Configuration.GetSection("MySQL:ConnectionString").Value)); + new MySqlConnection(Guard.Against.NullOrEmpty(configuration.GetSection("MySQL:ConnectionString").Value))); services.AddSingleton(); services.AddSingleton(); @@ -103,8 +114,8 @@ services.AddScoped(); services.AddScoped(); services.AddScoped(); -services.AddSingleton(); -services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); //Validation services.AddValidatorsFromAssemblyContaining(); @@ -116,7 +127,8 @@ services.AddTwitterClient(configuration); builder.Services.AddMemoryCache(); -builder.Services.SetupHangfire(configuration.GetRequiredSection("MongoDB:ConnectionString").Value!); +builder.Services.SetupHangfire(Guard.Against.NullOrEmpty(configuration.GetRequiredSection("MongoDB:ConnectionString").Value)); +builder.Services.SetupRedis(configuration); var app = builder.Build(); @@ -137,6 +149,9 @@ app.MapControllers(); -app.UseHangfireDashboard("/backJobMonitor"); +if (app.Environment.IsDevelopment()) +{ + app.UseHangfireDashboard("/backJobMonitor"); +} app.Run(); diff --git a/Api/appsettings.Development.json b/Api/appsettings.Development.json index 63e58ce..4dea487 100644 --- a/Api/appsettings.Development.json +++ b/Api/appsettings.Development.json @@ -15,5 +15,8 @@ "Twitter": { "TweetTemplate": "{name}: \"{meaning}\" {link}", "TweetIntervalSeconds": 60 + }, + "Redis": { + "DatabaseIndex": 1 } } diff --git a/Api/appsettings.json b/Api/appsettings.json index 07c914b..7b3f023 100644 --- a/Api/appsettings.json +++ b/Api/appsettings.json @@ -7,6 +7,9 @@ } }, "AllowedHosts": "*", + "ConnectionStrings": { + "Redis": "redis:6379" + }, "Twitter": { "AccessToken": "your-access-token", "AccessTokenSecret": "your-access-token-secret", @@ -15,5 +18,8 @@ "NameUrlPrefix": "https://www.yorubaname.com/entries", "TweetTemplate": "New name entry: {name}, {meaning}. More here: {link}", "TweetIntervalSeconds": 180 + }, + "Redis": { + "DatabaseIndex": 0 } } diff --git a/Application/Cache/ICacheService.cs b/Application/Cache/ICacheService.cs new file mode 100644 index 0000000..8cc97c4 --- /dev/null +++ b/Application/Cache/ICacheService.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Application.Cache +{ + public interface ICacheService + { + /// + /// Retrieves a collection of items from the cache. + /// + /// The key under which the items are cached. + /// A task that represents the asynchronous operation. The task result contains a collection of items. + Task> GetAsync(string key); + + /// + /// Adds an item to the cache and updates its recency. + /// + /// The key under which the item is cached. + /// The item to be added to the cache. + /// A task that represents the asynchronous operation. + Task StackAsync(string key, T item); + + /// + /// Removes an item from the cache. + /// + /// The key of the item to be removed. + /// A task that represents the asynchronous operation. The task result contains a boolean indicating whether the removal was successful. + Task RemoveAsync(string key, string searchTerm); + + /// + /// Retrieves the most popular items based on their frequency. + /// + /// A task that represents the asynchronous operation. The task result contains a collection of the most popular items. + Task> GetMostPopularAsync(string key); + } +} diff --git a/Application/Cache/InMemoryCache.cs b/Application/Cache/InMemoryCache.cs deleted file mode 100644 index 456bb32..0000000 --- a/Application/Cache/InMemoryCache.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Core.Cache; - -namespace Application.Cache -{ - public abstract class InMemoryCache : ICache - { - protected readonly List _itemCache; - protected readonly Dictionary _itemFrequency; - - private int recencyLimit = 5; - - public InMemoryCache() - { - _itemCache = new List(); - _itemFrequency = new Dictionary(); - } - - public async Task> Get() - { - return await Task.FromResult(_itemCache.ToArray()); - } - - public async Task Stack(string name) - { - await Insert(name); - int count = _itemCache.Count; - if (count > recencyLimit) - { - _itemCache.RemoveAt(count - 1); - } - } - - private async Task Insert(string name) - { - if (_itemCache.Contains(name)) - { - _itemCache.Remove(name); - } - _itemCache.Insert(0, name); - await UpdateFrequency(name); - } - - public async Task Remove(string name) - { - if (_itemCache.Contains(name)) - { - _itemCache.Remove(name); - _itemFrequency.Remove(name); - return true; - } - return false; - } - - private async Task UpdateFrequency(string name) - { - if (!_itemFrequency.ContainsKey(name)) - { - _itemFrequency.Add(name, 0); - } - _itemFrequency[name]++; - } - } -} diff --git a/Application/Cache/RecentIndexesCache.cs b/Application/Cache/RecentIndexesCache.cs deleted file mode 100644 index 1f999b6..0000000 --- a/Application/Cache/RecentIndexesCache.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Core.Cache; - -namespace Application.Cache -{ - public class RecentIndexesCache : InMemoryCache, IRecentIndexesCache - { - } -} diff --git a/Application/Cache/RecentSearchesCache.cs b/Application/Cache/RecentSearchesCache.cs deleted file mode 100644 index 5fb4abd..0000000 --- a/Application/Cache/RecentSearchesCache.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Core.Cache; - -namespace Application.Cache -{ - public class RecentSearchesCache : InMemoryCache, IRecentSearchesCache - { - private int popularListLimit = 5; - - public RecentSearchesCache() : base() - { - } - - - public async Task> GetMostPopular() - { - var frequency = GetNameWithSearchFrequency(); - return frequency.Select(item => item.Key).Take(popularListLimit); - } - - - private Dictionary GetNameWithSearchFrequency() - { - return _itemFrequency - .OrderByDescending(item => item.Value) - .ToDictionary(item => item.Key, item => item.Value); - } - } -} diff --git a/Infrastructure/Configuration/RedisConfig.cs b/Infrastructure/Configuration/RedisConfig.cs new file mode 100644 index 0000000..973cd05 --- /dev/null +++ b/Infrastructure/Configuration/RedisConfig.cs @@ -0,0 +1,7 @@ +namespace Infrastructure.Configuration +{ + public record RedisConfig + { + public int DatabaseIndex { get; set; } + } +} diff --git a/Infrastructure/Hangfire/DependencyInjection.cs b/Infrastructure/Hangfire/DependencyInjection.cs index d9e7642..499775c 100644 --- a/Infrastructure/Hangfire/DependencyInjection.cs +++ b/Infrastructure/Hangfire/DependencyInjection.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.AspNetCore.Builder; using Api.Utilities; +using Ardalis.GuardClauses; namespace Infrastructure.Hangfire { @@ -20,7 +21,7 @@ public static IServiceCollection SetupHangfire(this IServiceCollection services, .SetDataCompatibilityLevel(CompatibilityLevel.Version_180) .UseSimpleAssemblyNameTypeSerializer() .UseRecommendedSerializerSettings() - .UseMongoStorage(mongoClient, mongoUrlBuilder.DatabaseName, new MongoStorageOptions + .UseMongoStorage(mongoClient, Guard.Against.NullOrEmpty(mongoUrlBuilder.DatabaseName), new MongoStorageOptions { MigrationOptions = new MongoMigrationOptions { diff --git a/Infrastructure/Infrastructure.csproj b/Infrastructure/Infrastructure.csproj index 7b37e79..6b0bce6 100644 --- a/Infrastructure/Infrastructure.csproj +++ b/Infrastructure/Infrastructure.csproj @@ -7,11 +7,13 @@ + + diff --git a/Infrastructure/Redis/DependencyInjection.cs b/Infrastructure/Redis/DependencyInjection.cs new file mode 100644 index 0000000..9dc6fd9 --- /dev/null +++ b/Infrastructure/Redis/DependencyInjection.cs @@ -0,0 +1,21 @@ +using Ardalis.GuardClauses; +using Infrastructure.Configuration; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StackExchange.Redis; + +namespace Infrastructure.Redis +{ + public static class DependencyInjection + { + private const string SectionName = "Redis"; + + public static IServiceCollection SetupRedis(this IServiceCollection services, IConfiguration configuration) + { + services.Configure(configuration.GetRequiredSection(SectionName)); + var redisConnectionString = Guard.Against.NullOrEmpty(configuration.GetConnectionString(SectionName)); + services.AddSingleton(ConnectionMultiplexer.Connect(redisConnectionString)); + return services; + } + } +} diff --git a/Infrastructure/Redis/RedisCache.cs b/Infrastructure/Redis/RedisCache.cs new file mode 100644 index 0000000..71dd1a0 --- /dev/null +++ b/Infrastructure/Redis/RedisCache.cs @@ -0,0 +1,13 @@ +using Infrastructure.Configuration; +using Microsoft.Extensions.Options; +using StackExchange.Redis; + +namespace Infrastructure.Redis +{ + public abstract class RedisCache( + IConnectionMultiplexer connectionMultiplexer, + IOptions redisConfig) + { + protected readonly IDatabase _cache = connectionMultiplexer.GetDatabase(redisConfig.Value.DatabaseIndex); + } +} diff --git a/Infrastructure/Redis/RedisRecentIndexesCache.cs b/Infrastructure/Redis/RedisRecentIndexesCache.cs new file mode 100644 index 0000000..755fee1 --- /dev/null +++ b/Infrastructure/Redis/RedisRecentIndexesCache.cs @@ -0,0 +1,46 @@ +using Core.Cache; +using Infrastructure.Configuration; +using Microsoft.Extensions.Options; +using StackExchange.Redis; + +namespace Infrastructure.Redis +{ + public class RedisRecentIndexesCache( + IConnectionMultiplexer connectionMultiplexer, + IOptions redisConfig) : RedisCache(connectionMultiplexer, redisConfig), IRecentIndexesCache + { + private const string Key = "recent_indexes"; + private const int MaxItemsToReturn = 5; + private const int MaxItemsToStore = 10; + + public async Task> Get() + { + var results = await _cache.SortedSetRangeByRankAsync(Key, 0, MaxItemsToReturn - 1, Order.Descending); + return results.Select(r => r.ToString()); + } + + public async Task Remove(string item) + { + var tran = _cache.CreateTransaction(); + _ = tran.SortedSetRemoveAsync(Key, item); + return await tran.ExecuteAsync(); + } + + public async Task Stack(string item) + { + // Use a Redis transaction to ensure atomicity of both operations + var transaction = _cache.CreateTransaction(); + + // Add the search term to the front of the Redis list + _ = transaction.SortedSetAddAsync(Key, item, DateTime.UtcNow.Ticks); + _ = transaction.SortedSetRemoveRangeByRankAsync(Key, 0, -(MaxItemsToStore + 1)); + + // Execute the transaction + bool committed = await transaction.ExecuteAsync(); + if (!committed) + { + throw new Exception("Redis Transaction failed"); + } + } + } +} diff --git a/Infrastructure/Redis/RedisRecentSearchesCache.cs b/Infrastructure/Redis/RedisRecentSearchesCache.cs new file mode 100644 index 0000000..e08f764 --- /dev/null +++ b/Infrastructure/Redis/RedisRecentSearchesCache.cs @@ -0,0 +1,67 @@ +using Core.Cache; +using Infrastructure.Configuration; +using Microsoft.Extensions.Options; +using StackExchange.Redis; + +namespace Infrastructure.Redis +{ + public class RedisRecentSearchesCache( + IConnectionMultiplexer connectionMultiplexer, + IOptions redisConfig) : RedisCache(connectionMultiplexer, redisConfig), IRecentSearchesCache + { + private const string RecentSearchesKey = "recent_searches"; + private const string PopularSearchesKey = "popular_searches"; + private const int MaxItemsToReturn = 5; + private const int MaxRecentSearches = 10; + private const int MaxPopularSearches = 1000; // Use a large number to ensure that items have time to get promoted. + + public async Task> Get() + { + var results = await _cache.SortedSetRangeByRankAsync(RecentSearchesKey, 0, MaxItemsToReturn -1, Order.Descending); + return results.Select(r => r.ToString()); + } + + public async Task> GetMostPopular() + { + var results = await _cache.SortedSetRangeByRankAsync(PopularSearchesKey, 0, MaxItemsToReturn -1, Order.Descending); + return results.Select(r => r.ToString()); + } + + public async Task Remove(string item) + { + var tran = _cache.CreateTransaction(); + _ = tran.SortedSetRemoveAsync(RecentSearchesKey, item); + _ = tran.SortedSetRemoveAsync(PopularSearchesKey, item); + return await tran.ExecuteAsync(); + } + + public async Task Stack(string item) + { + // Use a Redis transaction to ensure atomicity of both operations + var transaction = _cache.CreateTransaction(); + + // Add the search term to the front of the Redis list + _ = transaction.SortedSetAddAsync(RecentSearchesKey, item, DateTime.UtcNow.Ticks); + _ = transaction.SortedSetRemoveRangeByRankAsync(RecentSearchesKey, 0, -(MaxRecentSearches + 1)); + + // TODO: Do a periodic caching, like daily where the most popular items from the previous period are brought forward into the next day + var currentScore = (await _cache.SortedSetScoreAsync(PopularSearchesKey, item)) ?? 0; + _ = transaction.SortedSetAddAsync(PopularSearchesKey, item, (int)currentScore + 1 + GetNormalizedTimestamp()); + _ = transaction.SortedSetRemoveRangeByRankAsync(PopularSearchesKey, 0, -(MaxPopularSearches + 1)); + + // Execute the transaction + bool committed = await transaction.ExecuteAsync(); + if (!committed) + { + throw new Exception("Redis Transaction failed"); + } + } + + static double GetNormalizedTimestamp() + { + // This can be improved by addressing the time-cycle reset problem. + long unixTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + return (unixTimestamp % 1000000) / 1000000.0; + } + } +} diff --git a/Infrastructure/Twitter/TwitterService.cs b/Infrastructure/Twitter/TwitterService.cs index 0b6eb53..5dc8057 100644 --- a/Infrastructure/Twitter/TwitterService.cs +++ b/Infrastructure/Twitter/TwitterService.cs @@ -69,6 +69,7 @@ private async Task PostTweetAsync(string tweetText, CancellationToken cancellati } } + [AutomaticRetry(Attempts = 3)] public async Task SendTweetAsync(string tweetText) { if (!Debugger.IsAttached) // To prevent tweets from getting posted while testing. Could be better, but... diff --git a/docker-compose.yml b/docker-compose.yml index 1890559..4e69291 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,10 +14,21 @@ services: dockerfile: Api/Dockerfile depends_on: - mongodb + - redis mongodb: image: mongo:latest container_name: ynd-mongodb ports: - "27020:27017" + redis: + image: redis:latest + container_name: redis + ports: + - "6379:6379" + volumes: + - redis-data:/data + +volumes: + redis-data: