Merge pull request #27 from ahmedayman4a/release-1.2

Release v1.2

LLCD.CourseExtractor.Tests/ExtractorTest.cs

@@ -12,46 +12,62 @@ namespace LLCD.CourseExtractor.Tests
     [TestClass]
     public class ExtractorTest
     {
-        private const string VALID_TOKEN = "";
+        private const string VALID_CHROME_TOKEN = "";
+        private const string VALID_FIREFOX_TOKEN = "";
         private const string EXPIRED_TOKEN = "";
-        private const string INVALID_TOKEN = "QEGAGQBAAAAAAVW6BkPg4R_VgAAR3VybjpsaTplbnRlcnByaXNlUHJvZmlsZToodXJuOmxpOmVudGVycHJpc2VBY2NvdW50OjEwNDk0MjIxMCwxMjY5MzU4NjgpvFufSjdKltNIvqBCEfUtk_v1dKyDW1v4v4T-ULf5HfsBuTtkjYwXKhAq4tzlv77b0TAKjaEB9KG88zz46-O34O-ymauMqZ_C8mWvdKTctBXPEPM0";
+        private const string INVALID_CHROME_TOKEN = "QEGAGQBAAAAAAVW6BkPg4R_VgAAR3VybjpsaTplbnRlcnByaXNlUHJvZmlsZToodXJuOmxpOmVudGVycHJpc2VBY2NvdW50OjEwNDk0MjIxMCwxMjY5MzU4NjgpvFufSjdKltNIvqBCEfUtk_v1dKyDW1v4v4T-ULf5HfsBuTtkjYwXKhAq4tzlv77b0TAKjaEB9KG88zz46-O34O-ymauMqZ_C8mWvdKTctBXPEPM0";
+        private const string INVALID_FIREFOX_TOKEN = "QEGAGQBAAAAAAVW6BkPg4R_VgAAR3VybjpsaTplbnRlcnByaXNlUHJvZmlsZToodXJuOmxpOmVudGVycHJpc2VBY2NvdW50OjEwNDk0MjIxMCwxMjY5MzU4NjgpvFufSjdKltNIvqBCEfUtk_v1dKyDW1v4v4T-ULf5HfsBuTtkjYwXKhAq4tzlv77b0TAKjaEB9KG88zz46-O34O-ymauMqZ_C8mWvdKTctBXPEPM0";
         private const string VALID_EnterpriseProfileHash = "";
 
         [TestMethod]
         [TestCategory("Cookie Extraction")]
-        public void ExtractToken_ValidCookieExtraction_ReturnsEqualTokenValue()
+        public void ExtractToken_ValidFirefoxCookieExtraction_ReturnsEqualTokenValue()
         {
-            Assert.AreEqual(VALID_TOKEN, Extractor.ExtractToken(Browser.Firefox));
+            Assert.AreEqual(VALID_FIREFOX_TOKEN, Extractor.ExtractToken(Browser.Firefox));
         }
 
         [TestMethod]
         [TestCategory("Cookie Extraction")]
-        public void ExtractToken_InvalidCookieExtraction_ReturnsNonEqualTokenValue()
+        public void ExtractToken_InvalidFirefoxCookieExtraction_ReturnsNonEqualTokenValue()
         {
-            Assert.AreNotEqual(INVALID_TOKEN, Extractor.ExtractToken(Browser.Firefox));
+            Assert.AreNotEqual(INVALID_FIREFOX_TOKEN, Extractor.ExtractToken(Browser.Firefox));
+        }
+
+        [TestMethod]
+        [TestCategory("Cookie Extraction")]
+        public void ExtractToken_ValidChromeCookieExtraction_ReturnsEqualTokenValue()
+        {
+            Assert.AreEqual(VALID_CHROME_TOKEN, Extractor.ExtractToken(Browser.Chrome));
+        }
+
+        [TestMethod]
+        [TestCategory("Cookie Extraction")]
+        public void ExtractToken_InvalidChromeCookieExtraction_ReturnsNonEqualTokenValue()
+        {
+            Assert.AreNotEqual(INVALID_CHROME_TOKEN, Extractor.ExtractToken(Browser.Chrome));
         }
 
         [TestMethod]
         [TestCategory("Token Validity")]
         public async Task ExtractToken_ValidToken_ReturnsTrue()
         {
-            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_TOKEN);
+            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_FIREFOX_TOKEN);
             Assert.IsTrue(await extractor.HasValidToken());
         }
 
         [TestMethod]
         [TestCategory("Token Validity")]
         public async Task ExtractToken_InvalidToken_ReturnsFalse()
         {
-            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, INVALID_TOKEN);
+            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_FIREFOX_TOKEN);
             Assert.IsFalse(await extractor.HasValidToken());
         }
 
         [TestMethod]
         [TestCategory("Course Extraction")]
         public async Task GetCourse_ValidCourse_ReturnsEqualCourseData()
         {
-            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_TOKEN);
+            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_FIREFOX_TOKEN);
             var progress = new Progress<float>(progressPercent => ConsoleOutput.Instance.WriteLine((progressPercent * 100).ToString(), OutputLevel.Information));
             var course = await extractor.GetCourse(progress);
             CompareLogic compareLogic = new CompareLogic();
@@ -93,7 +109,7 @@ public async Task GetCourse_CoursesExtractionStressTest_ReturnsNumberOfExtractio
             foreach (var link in links)
             {
                 i++;
-                var extractor = new Extractor(link, Quality.Low, VALID_TOKEN);
+                var extractor = new Extractor(link, Quality.Low, VALID_FIREFOX_TOKEN);
                 var course = await extractor.GetCourse();
             }
             ConsoleOutput.Instance.WriteLine($"Extracted {i} courses", OutputLevel.Information);
@@ -103,7 +119,7 @@ public async Task GetCourse_CoursesExtractionStressTest_ReturnsNumberOfExtractio
         [TestCategory("Course Extraction")]
         public async Task GetCourse_InValidCourse_ReturnsNonEqualCourseData()
         {
-            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_TOKEN);
+            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_FIREFOX_TOKEN);
             var course = await extractor.GetCourse();
             Assert.AreNotEqual(CourseObjects.INVALIDCOURSE, course);
         }
@@ -112,15 +128,15 @@ public async Task GetCourse_InValidCourse_ReturnsNonEqualCourseData()
         [TestCategory("Link Validity")]
         public void HasValidUrl_ValidUrl_ReturnsTrue()
         {
-            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive/welcome?u=104942210", Quality.Low, VALID_TOKEN);
+            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive/welcome?u=104942210", Quality.Low, VALID_FIREFOX_TOKEN);
             Assert.IsTrue(extractor.HasValidUrl());
         }
 
         [TestMethod]
         [TestCategory("EnterpriseProfileHash Validity")]
         public async Task GetEnterpriseProfileHash_ValidEnterpriseProfileHash_ReturnsSameValue()
         {
-            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_TOKEN);
+            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_FIREFOX_TOKEN);
             await extractor.GetCourse();
             Assert.AreEqual(VALID_EnterpriseProfileHash, extractor.EnterpriseProfileHash);
         }
@@ -129,7 +145,7 @@ public async Task GetEnterpriseProfileHash_ValidEnterpriseProfileHash_ReturnsSam
         [TestCategory("EnterpriseProfileHash Validity")]
         public async Task GetEnterpriseProfileHash_InvalidToken_ReturnsNull()
         {
-            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, INVALID_TOKEN);
+            var extractor = new Extractor("https://www.linkedin.com/learning/learning-to-be-assertive?autoplay=true&u=104942210", Quality.Low, VALID_FIREFOX_TOKEN);
             await extractor.GetCourse();
             Assert.IsNull(extractor.EnterpriseProfileHash);
         }

LLCD.CourseExtractor/CookiesExtractor.cs

@@ -10,6 +10,7 @@
 using Org.BouncyCastle.Crypto.Modes;
 using Org.BouncyCastle.Crypto.Parameters;
 using System.IO;
+using Serilog;
 
 namespace LLCD.CourseExtractor
 {
@@ -58,15 +59,32 @@ internal List<DBCookie> ReadFirefoxCookies()
         }
         private List<DBCookie> ReadChromiumCookies(string profilePath)
         {
-            var cookies = new List<DBCookie>();
-
-
-            // Big thanks to https://stackoverflow.com/a/60611673/6481581 for answering how Chrome 80 and up changed the way cookies are encrypted.
-            string dbPath = Path.Combine(profilePath, @"Default\Cookies");
             string encKey = File.ReadAllText(Path.Combine(profilePath, "Local State"));
             encKey = JObject.Parse(encKey)["os_crypt"]["encrypted_key"].ToString();
             var decodedKey = ProtectedData.Unprotect(Convert.FromBase64String(encKey).Skip(5).ToArray(), null, DataProtectionScope.LocalMachine);
 
+            // Big thanks to https://stackoverflow.com/a/60611673/6481581 for answering how Chrome 80 and up changed the way cookies are encrypted.
+
+            List<DBCookie> cookies;
+            try
+            {
+                string dbPath = Path.Combine(profilePath, @"Default\Network\Cookies");
+                cookies = GetChromeCookiesFromDB(dbPath, decodedKey);
+            }
+            catch (SQLiteException ex)
+            {
+                Log.Error(ex, @"Cookies not found at ""Default\Network\Cookies"". Trying ""Default\Cookies""");
+                string dbPath = Path.Combine(profilePath, @"Default\Cookies");
+                cookies = GetChromeCookiesFromDB(dbPath, decodedKey);
+            }
+
+
+            return cookies;
+        }
+
+        private List<DBCookie> GetChromeCookiesFromDB(string dbPath, byte[] decodedKey)
+        {
+            var cookies = new List<DBCookie>();
             var connectionString = "Data Source=" + dbPath + ";pooling=false";
 
             using (var conn = new SQLiteConnection(connectionString))
@@ -94,7 +112,6 @@ private List<DBCookie> ReadChromiumCookies(string profilePath)
             return cookies;
         }
 
-
         private string DecryptWithKey(byte[] message, byte[] key, int nonSecretPayloadLength)
         {
             const int KEY_BIT_SIZE = 256;

LLCD.CourseExtractor/Extractor.cs

@@ -12,25 +12,29 @@
 using LLCD.CourseContent;
 using Microsoft.CSharp;
 using Newtonsoft.Json;
+using Serilog;
 
 namespace LLCD.CourseExtractor
 {
     public class Extractor
     {
         public delegate void LinksExtractionEventHandler();
         private readonly Quality _quality;
+        private readonly int _delay;
         private string _courseUrl;
         private string _courseSlug;
         private HttpClient _client;
         private CookieContainer _cookieContainer;
         private string _linkedinHomeRaw;
         private bool _isTokenChecked = false;
+
         public string EnterpriseProfileHash { get; set; }
 
-        public Extractor(string courseUrl, Quality quality, string token)
+        public Extractor(string courseUrl, Quality quality, string token, int delay = 0)
         {
             _courseUrl = courseUrl;
             _quality = quality;
+            _delay = delay;
             _cookieContainer = new CookieContainer();
             _cookieContainer.Add(new Cookie("li_at", token, "/", ".www.linkedin.com"));
             var clienthandler = new HttpClientHandler { UseCookies = true, CookieContainer = _cookieContainer };
@@ -80,7 +84,25 @@ public async Task<Course> GetCourse(IProgress<float> progress = null)
             }
             var courseResponse = await _client.GetAsync($"https://www.linkedin.com/learning-api/detailedCourses?courseSlug={_courseSlug}&fields=chapters,title,exerciseFiles&addParagraphsToTranscript=true&q=slugs");
             var courseResponseText = await courseResponse.Content.ReadAsStringAsync();
-            var course = Course.FromJson(courseResponseText);
+
+            Course course;
+            try
+            {
+                course = Course.FromJson(courseResponseText);
+            }
+            catch (Exception ex)
+            {
+                if (courseResponseText.Contains("CSRF check failed"))
+                {
+                    throw new ArgumentException("Token is expired. Please use a new one.", ex);
+                }
+                else
+                {
+                    Log.Error("Course Deserialization failed. \nResponse text : " + courseResponseText);
+                    throw;
+                }
+            }
+
             course.Slug = _courseSlug;
             float j = 1;
             float totalCount = course.Chapters.SelectMany(c => c.Videos).Count();
@@ -92,7 +114,15 @@ public async Task<Course> GetCourse(IProgress<float> progress = null)
                     string slug = video.Slug;
                     var videoResponse = await _client.GetAsync($"https://www.linkedin.com/learning-api/detailedCourses?courseSlug={_courseSlug}&resolution=_{_quality.ToHeight()}&q=slugs&fields=selectedVideo&videoSlug={video.Slug}");
                     var videoResponseText = await videoResponse.Content.ReadAsStringAsync();
-                    video = Video.FromJson(videoResponseText);
+                    try
+                    {
+                        video = Video.FromJson(videoResponseText);
+                    }
+                    catch (Exception)
+                    {
+                        Log.Error("Video Deserialization failed. \nResponse text : " + videoResponseText);
+                        throw;
+                    }
                     video.Slug = slug;
                     if (String.IsNullOrWhiteSpace(video.DownloadUrl))
                     {
@@ -101,6 +131,7 @@ public async Task<Course> GetCourse(IProgress<float> progress = null)
                     }
                     chapter.Videos[i] = video;
                     progress?.Report(j / totalCount);
+                    await Task.Delay(_delay * 1000);
                 }
             }
             return course;

LLCD.CourseExtractor/LLCD.CourseExtractor.csproj

@@ -7,10 +7,10 @@
 
   <ItemGroup>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.8.10" />
-    <PackageReference Include="Serilog.Sinks.File" Version="4.1.0" />
-    <PackageReference Include="System.Data.SQLite" Version="1.0.113.7" />
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="5.0.0" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
+    <PackageReference Include="System.Data.SQLite.Core" Version="1.0.115.5" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="6.0.0" />
   </ItemGroup>
 
   <ItemGroup>

LLCD.DownloaderGUI/App.config

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <configSections>
-    <!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
+
     <section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
   </configSections>
   <startup>
@@ -13,10 +13,5 @@
       <provider invariantName="System.Data.SQLite.EF6" type="System.Data.SQLite.EF6.SQLiteProviderServices, System.Data.SQLite.EF6" />
     </providers>
   </entityFramework>
-  <system.data>
-    <DbProviderFactories>
-      <remove invariant="System.Data.SQLite.EF6" />
-      <add name="SQLite Data Provider (Entity Framework 6)" invariant="System.Data.SQLite.EF6" description=".NET Framework Data Provider for SQLite (Entity Framework 6)" type="System.Data.SQLite.EF6.SQLiteProviderFactory, System.Data.SQLite.EF6" />
-    <remove invariant="System.Data.SQLite" /><add name="SQLite Data Provider" invariant="System.Data.SQLite" description=".NET Framework Data Provider for SQLite" type="System.Data.SQLite.SQLiteFactory, System.Data.SQLite" /></DbProviderFactories>
-  </system.data>
+
 </configuration>

LLCD.DownloaderGUI/CourseStatusUserControl.cs

@@ -29,21 +29,25 @@ public CourseStatus Status
                 switch (value)
                 {
                     case CourseStatus.NotRunning:
-                        lblCourseStatus.Text = "Not Running";
-                        lblCourseStatus.BackColor = Color.Maroon;
+                        lblCourseStatus.Text = "Not running";
+                        lblCourseStatus.BackColor = Color.FromArgb(249, 222, 220);
+                        lblCourseStatus.ForeColor = Color.FromArgb(65, 14, 11);
                         break;
                     case CourseStatus.Starting:
                     case CourseStatus.Running:
                         lblCourseStatus.Text = value.ToString();
-                        lblCourseStatus.BackColor = Color.DarkBlue;
+                        lblCourseStatus.BackColor = Color.FromArgb(3, 218, 198);
+                        lblCourseStatus.ForeColor = Color.Black;
                         break;
                     case CourseStatus.Finished:
                         lblCourseStatus.Text = value.ToString();
-                        lblCourseStatus.BackColor = Color.DarkGreen;
+                        lblCourseStatus.BackColor = Color.FromArgb(183, 243, 151);
+                        lblCourseStatus.ForeColor = Color.FromArgb(4, 33, 0);
                         break;
                     case CourseStatus.Failed:
                         lblCourseStatus.Text = value.ToString();
-                        lblCourseStatus.BackColor = Color.DarkRed;
+                        lblCourseStatus.BackColor = Color.FromArgb(207, 102, 121);
+                        lblCourseStatus.ForeColor = Color.Black;
                         break;
                 }
                 _status = value;