Merge pull request #65 from mbaraa/dev

Dev to Main

.env.example

@@ -2,7 +2,6 @@ PORT=3000
 HOSTNAME="http://localhost:3000"
 JWT_SECRET="spoikoninochi"
 
-YOUTUBE_SCAPER_URL="http://localhost:1234"
 YOUTUBE_DOWNLOADER_URL="http://localhost:4321"
 YOUTUBE_MUSIC_DOWNLOAD_PATH="./_serve/"
 

README.md

@@ -88,7 +88,6 @@ docker compose up -f docker-compose-dev.yml
 - **This project is not affiliated with YouTube or Google, or anyone to that matter in any sort of ways.**
 - Frank’s original image was taken from [dingusland.biz](https://dingusland.biz)
 - Colorscheme is inspired from [Dankpods](https://www.youtube.com/@DankPods)
-- youtube-scrape was used to search videos without using the actual YouTube API (small quota): MIT licenses by [Herman Fassett](https://github.com/HermanFassett)
 
 ---
 

app/config/env.go

@@ -15,11 +15,9 @@ func initEnvVars() {
 		Hostname:  getEnv("HOSTNAME"),
 		JwtSecret: getEnv("JWT_SECRET"),
 		YouTube: struct {
-			ScraperUrl    string
 			DownloaderUrl string
 			MusicDir      string
 		}{
-			ScraperUrl:    getEnv("YOUTUBE_SCAPER_URL"),
 			DownloaderUrl: getEnv("YOUTUBE_DOWNLOADER_URL"),
 			MusicDir:      getEnv("YOUTUBE_MUSIC_DOWNLOAD_PATH"),
 		},
@@ -62,7 +60,6 @@ type config struct {
 	Hostname  string
 	JwtSecret string
 	YouTube   struct {
-		ScraperUrl    string
 		DownloaderUrl string
 		MusicDir      string
 	}

app/services/youtube/search/search_scraper.go

@@ -1,53 +1,153 @@
 package search
 
 import (
-	"dankmuzikk/config"
 	"dankmuzikk/entities"
 	"encoding/json"
-	"fmt"
+	"io"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"
 )
 
-type scrapSearchResult struct {
-	Results []struct {
-		Video struct {
-			Id           string `json:"id"`
-			Title        string `json:"title"`
-			Url          string `json:"url"`
-			Duration     string `json:"duration"`
-			ThumbnailUrl string `json:"thumbnail_src"`
-		} `json:"video"`
-		Uploader struct {
-			Username string `json:"username"`
-			Url      string `json:"url"`
-		} `json:"uploader"`
-	} `json:"results"`
+var (
+	pat0 = regexp.MustCompile(`"innertubeApiKey":"([^"]*)`)
+	pat  = regexp.MustCompile(`ytInitialData[^{]*(.*?);\s*<\/script>`)
+	pat2 = regexp.MustCompile(`ytInitialData"[^{]*(.*);\s*window\["ytInitialPlayerResponse"\]`)
+)
+
+type videoResult struct {
+	Id           string `json:"id"`
+	Title        string `json:"title"`
+	Url          string `json:"url"`
+	Duration     string `json:"duration"`
+	ThumbnailUrl string `json:"thumbnail_src"`
+	Uploader     string `json:"username"`
 }
 
-// ScraperSearch is a scrapper enabled YouTube search, using the search service under ~/ytscraper
-type ScraperSearch struct{}
+type req struct {
+	Version          string `json:"version"`
+	Parser           string `json:"parser"`
+	Key              string `json:"key"`
+	EstimatedResults string `json:"estimatedResults"`
+}
 
-func (y *ScraperSearch) Search(query string) (results []entities.Song, err error) {
-	// TODO: write a proper scraper instead of this hacky node js api
-	resp, err := http.Get(fmt.Sprintf("%s/api/search?q=%s", config.Env().YouTube.ScraperUrl, url.QueryEscape(query)))
+type videoRenderer struct {
+	VideoId string `json:"videoId"`
+	Title   struct {
+		Runs []struct {
+			Text string `json:"text"`
+		} `json:"runs"`
+	} `json:"title"`
+	LengthText struct {
+		SimpleText string `json:"simpleText"`
+	} `json:"lengthText"`
+	Thumbnail struct {
+		Thumbnails []struct {
+			URL string `json:"url"`
+		} `json:"thumbnails"`
+	} `json:"thumbnail"`
+	OwnerText struct {
+		Runs []struct {
+			Text string `json:"text"`
+		} `json:"runs"`
+	} `json:"ownerText"`
+}
+
+type ytSearchData struct {
+	EstimatedResults string `json:"estimatedResults"`
+	Contents         struct {
+		TwoColumnSearchResultsRenderer struct {
+			PrimaryContents struct {
+				SectionListRenderer struct {
+					Contents []struct { // sectionList
+						ItemSectionRenderer struct {
+							Contents []struct {
+								ChannelRenderer any `json:"channelRenderer"`
+
+								VideoRenderer videoRenderer `json:"videoRenderer"`
+
+								RadioRenderer    any `json:"radioRenderer"`
+								PlaylistRenderer any `json:"playlistRenderer"`
+							} `json:"contents"`
+						} `json:"itemSectionRenderer"`
+					} `json:"contents"`
+				} `json:"sectionListRenderer"`
+			} `json:"primaryContents"`
+		} `json:"twoColumnSearchResultsRenderer"`
+	} `json:"contents"`
+}
+
+func search(q string) ([]videoResult, error) {
+	// get ze results
+	url := "https://www.youtube.com/results?q=" + url.QueryEscape(q)
+	resp, err := http.Get(url)
 	if err != nil {
-		return
+		return nil, err
 	}
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	jojo := req{
+		Version: "0.1.5",
+		Parser:  "json_format",
+		Key:     "",
+	}
+	key := pat0.FindSubmatch(respBody)
+	jojo.Key = string(key[1])
 
-	var apiResults scrapSearchResult
-	err = json.NewDecoder(resp.Body).Decode(&apiResults)
+	matches := pat.FindSubmatch(respBody)
+	if len(matches) > 1 {
+		jojo.Parser += ".object_var"
+	} else {
+		jojo.Parser += ".original"
+		matches = pat2.FindSubmatch(respBody)
+	}
+	data := ytSearchData{}
+	err = json.Unmarshal(matches[1], &data)
+	if err != nil {
+		return nil, err
+	}
+	jojo.EstimatedResults = data.EstimatedResults
+
+	// parse JSON data
+
+	resSuka := make([]videoResult, 0)
+	for _, sectionList := range data.Contents.TwoColumnSearchResultsRenderer.PrimaryContents.SectionListRenderer.Contents {
+		for _, content := range sectionList.ItemSectionRenderer.Contents {
+			_ = content
+			if content.VideoRenderer.VideoId == "" {
+				continue
+			}
+			resSuka = append(resSuka, videoResult{
+				Id:           content.VideoRenderer.VideoId,
+				Title:        content.VideoRenderer.Title.Runs[0].Text,
+				Duration:     content.VideoRenderer.LengthText.SimpleText,
+				ThumbnailUrl: content.VideoRenderer.Thumbnail.Thumbnails[len(content.VideoRenderer.Thumbnail.Thumbnails)-1].URL,
+				Uploader:     content.VideoRenderer.OwnerText.Runs[0].Text,
+			})
+		}
+	}
+
+	return resSuka, nil
+}
+
+// ScraperSearch is a scrapper enabled YouTube search, using the search service under ~/ytscraper
+type ScraperSearch struct{}
+
+func (y *ScraperSearch) Search(query string) (results []entities.Song, err error) {
+	res, err := search(query)
 	if err != nil {
-		return
+		return nil, err
 	}
 
-	for _, res := range apiResults.Results {
-		if res.Video.Id == "" || res.Video.Title == "" || res.Video.ThumbnailUrl == "" || res.Uploader.Username == "" {
+	for _, res := range res {
+		if res.Id == "" || res.Title == "" || res.ThumbnailUrl == "" || res.Uploader == "" {
 			continue
 		}
-		duration := strings.Split(res.Video.Duration, ":")
+		duration := strings.Split(res.Duration, ":")
 		if len(duration[0]) == 1 {
 			duration[0] = "0" + duration[0]
 		}
@@ -69,10 +169,10 @@ func (y *ScraperSearch) Search(query string) (results []entities.Song, err error
 		}
 
 		results = append(results, entities.Song{
-			YtId:         res.Video.Id,
-			Title:        res.Video.Title,
-			Artist:       res.Uploader.Username,
-			ThumbnailUrl: res.Video.ThumbnailUrl,
+			YtId:         res.Id,
+			Title:        res.Title,
+			Artist:       res.Uploader,
+			ThumbnailUrl: res.ThumbnailUrl,
 			Duration:     strings.Join(duration, ":"),
 		})
 	}

app/views/pages/about.templ

@@ -70,7 +70,6 @@ templ aboutContent() {
 				</li>
 				<li>Frank’s original image was taken from <a class={ "nav-link" } href="https://dingusland.biz" target="_blank">dingusland.biz</a></li>
 				<li>Colorscheme is inspired from  <a class={ "nav-link" } href="https://youtube.com/@Dankpods" target="_blank">&commat;Dankpods</a> </li>
-				<li>youtube-scrape was used to search videos without using the actual YouTube API (small quota): MIT licenses by <a class={ "nav-link" } href="https://github.com/HermanFassett" target="_blank">Herman Fassett</a>.</li>
 			</ul>
 		</section>
 	</div>

docker-compose-dev.yml

@@ -19,7 +19,6 @@ services:
       - danknetwork
     depends_on:
       - dank-db
-      - yt-scraper
       - yt-dl
 
   dank-db:
@@ -37,17 +36,6 @@ services:
     networks:
       - danknetwork
 
-  yt-scraper:
-    container_name: "yt-scraper"
-    build:
-      context: ./ytscraper
-    image: "yt-scraper"
-    restart: "always"
-    ports:
-      - 8080
-    networks:
-      - danknetwork
-
   yt-dl:
     container_name: "ytdl"
     build:

docker-compose-services.yml

@@ -14,15 +14,6 @@ services:
       - db-config:/etc/mysql
       - db-data:/var/lib/mysql
 
-  yt-scraper:
-    container_name: "yt-scraper"
-    build:
-      context: ./ytscraper
-    image: "yt-scraper"
-    restart: "always"
-    ports:
-      - "1234:8080"
-
   yt-dl:
     container_name: "yt-dl"
     build:

docker-compose.yml

@@ -18,20 +18,8 @@ services:
     networks:
       - danknetwork
     depends_on:
-      - yt-scraper
       - yt-dl
 
-  yt-scraper:
-    container_name: "yt-scraper"
-    build:
-      context: ./ytscraper
-    image: "yt-scraper"
-    restart: "always"
-    ports:
-      - 8080
-    networks:
-      - danknetwork
-
   yt-dl:
     container_name: "ytdl"
     build: