Merge branch 'snrt.ma' of https://github.com/fraudiay79/epg into pr/2534

iptv-org · Dec 22, 2024 · 73a80fb · 73a80fb
2 parents d0acdfe + d70147c
commit 73a80fb
Show file tree

Hide file tree

Showing 24 changed files with 3,829 additions and 635 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -30,6 +30,7 @@
     "@alex_neo/jest-expect-message": "^1.0.5",
     "@freearhey/core": "^0.3.1",
     "@freearhey/search-js": "^0.1.1",
+    "@ntlab/sfetch": "^1.0.0",
     "@octokit/core": "^4.1.0",
     "@types/cli-progress": "^3.11.3",
     "@types/fs-extra": "^11.0.2",

diff --git a/sites/firstmedia.com/firstmedia.com.config.js b/sites/firstmedia.com/firstmedia.com.config.js
@@ -43,7 +43,6 @@ module.exports = {
   },
   async channels() {
     const axios = require('axios')
-    const cheerio = require('cheerio')
     const result = await axios
       .get(
         `https://api.firstmedia.com/api/content/tv-guide/list?date=${dayjs().format(

diff --git a/sites/i24news.tv/i24news.tv.channels.xml b/sites/i24news.tv/i24news.tv.channels.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <channels>
-  <channel site="i24news.tv" lang="ar" xmltv_id="i24NEWSArabic.il" site_id="ar#world">I24NEWS عربى</channel>
-  <channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishUSA.il" site_id="en#usa">I24NEWS English (USA)</channel>
-  <channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishWorld.il" site_id="en#world">I24NEWS English (World)</channel>
-  <channel site="i24news.tv" lang="fr" xmltv_id="i24NEWSFrench.il" site_id="fr#world">I24NEWS Français</channel>
+  <channel site="i24news.tv" lang="ar" xmltv_id="i24NEWSArabic.il" site_id="ar">I24NEWS عربى</channel>
+  <channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishUSA.il" site_id="en">I24NEWS English (USA)</channel>
+  <channel site="i24news.tv" lang="fr" xmltv_id="i24NEWSFrench.il" site_id="fr">I24NEWS Français</channel>
+  <channel site="i24news.tv" lang="he" xmltv_id="i24NEWSHebrew.il" site_id="he">I24NEWS עברית</channel>
 </channels>
diff --git a/sites/i24news.tv/i24news.tv.config.js b/sites/i24news.tv/i24news.tv.config.js
@@ -11,9 +11,7 @@ module.exports = {
   site: 'i24news.tv',
   days: 2,
   url: function ({ channel }) {
-    const [lang, region] = channel.site_id.split('#')
-
-    return `https://api.i24news.tv/v2/${lang}/schedules/${region}`
+    return `https://api.i24news.tv/v2/${channel.site_id}/schedules`
   },
   parser: function ({ content, date }) {
     let programs = []

diff --git a/sites/i24news.tv/i24news.tv.test.js b/sites/i24news.tv/i24news.tv.test.js
@@ -7,12 +7,12 @@ dayjs.extend(utc)
 
 const date = dayjs.utc('2022-03-06', 'YYYY-MM-DD').startOf('d')
 const channel = {
-  site_id: 'ar#',
+  site_id: 'ar',
   xmltv_id: 'I24NewsArabic.il'
 }
 
 it('can generate valid url', () => {
-  expect(url({ channel })).toBe('https://api.i24news.tv/v2/ar/schedules/world')
+  expect(url({ channel })).toBe('https://api.i24news.tv/v2/ar/schedules')
 })
 
 it('can parse response', () => {

diff --git a/sites/iltalehti.fi/iltalehti.fi.test.js b/sites/iltalehti.fi/iltalehti.fi.test.js
@@ -9,7 +9,7 @@ dayjs.extend(utc)
 
 const date = dayjs.utc('2022-10-29', 'YYYY-MM-DD').startOf('d')
 const channel = {
-  site_id: 'default_builtin_channelgroup1#yle-tv1',
+  site_id: '1#yle-tv1',
   xmltv_id: 'YleTV1.fi'
 }
 

diff --git a/sites/mncvision.id/mncvision.id.config.js b/sites/mncvision.id/mncvision.id.config.js
@@ -1,19 +1,23 @@
-const _ = require('lodash')
 const axios = require('axios')
 const cheerio = require('cheerio')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
+const doFetch = require('@ntlab/sfetch')
+const debug = require('debug')('site:mncvision.id')
 
 dayjs.extend(utc)
 dayjs.extend(timezone)
 dayjs.extend(customParseFormat)
 
+doFetch
+  .setCheckResult(false)
+  .setDebugger(debug)
+
 const languages = { en: 'english', id: 'indonesia' }
 const cookies = {}
 const timeout = 30000
-const nworker = 25
 
 module.exports = {
   site: 'mncvision.id',
@@ -55,8 +59,6 @@ module.exports = {
     return await parseItems(content, date, cookies[channel.lang])
   },
   async channels({ lang = 'id' }) {
-    const axios = require('axios')
-    const cheerio = require('cheerio')
     const result = await axios
       .get('https://www.mncvision.id/schedule')
       .then(response => response.data)
@@ -117,42 +119,31 @@ async function parseItems(content, date, cookies) {
   const $ = cheerio.load(content)
   const items = $('tr[valign="top"]').toArray()
   if (items.length) {
-    const workers = []
-    const n = Math.min(nworker, items.length)
-    while (workers.length < n) {
-      const worker = () => {
-        if (items.length) {
-          const $item = $(items.shift())
-          const done = (description = null) => {
-            const start = parseStart($item, date)
-            const duration = parseDuration($item)
-            const stop = start.add(duration, 'm')
-            programs.push({
-              title: parseTitle($item),
-              season: parseSeason($item),
-              episode: parseEpisode($item),
-              description,
-              start,
-              stop
-            })
-            worker()
-          }
-          loadDescription($item, cookies)
-            .then(description => done(description))
-        } else {
-          workers.splice(workers.indexOf(worker), 1)
-        }
+    const queues = []
+    for (const item of items) {
+      const $item = $(item)
+      const url = $item.find('a').attr('href')
+      const headers = {
+        'X-Requested-With': 'XMLHttpRequest',
+        Cookie: cookies,
       }
-      workers.push(worker)
-      worker()
+      queues.push({ i: $item, url, params: { headers, timeout } })
     }
-    await new Promise(resolve => {
-      const interval = setInterval(() => {
-        if (workers.length === 0) {
-          clearInterval(interval)
-          resolve()
-        }
-      }, 500)
+    await doFetch(queues, (queue, res) => {
+      const $item = queue.i
+      const $page = cheerio.load(res)
+      const description = $page('.synopsis').text().trim()
+      const start = parseStart($item, date)
+      const duration = parseDuration($item)
+      const stop = start.add(duration, 'm')
+      programs.push({
+        title: parseTitle($item),
+        season: parseSeason($item),
+        episode: parseEpisode($item),
+        description: description && description !== '-' ? description : null,
+        start,
+        stop
+      })
     })
   }
 
@@ -168,24 +159,6 @@ function loadLangCookies(channel) {
     .catch(error => console.error(error.message))
 }
 
-async function loadDescription($item, cookies) {
-  const url = $item.find('a').attr('href')
-  if (!url) return null
-  const content = await axios
-    .get(url, {
-      headers: { 'X-Requested-With': 'XMLHttpRequest', Cookie: cookies },
-      timeout
-    })
-    .then(r => r.data)
-    .catch(error => console.error(error.message))
-  if (!content) return null
-
-  const $page = cheerio.load(content)
-  const description = $page('.synopsis').text().trim()
-
-  return description !== '-' ? description : null
-}
-
 function parseCookies(headers) {
   const cookies = []
   if (Array.isArray(headers['set-cookie'])) {

diff --git a/sites/mytelly.co.uk/mytelly.co.uk.config.js b/sites/mytelly.co.uk/mytelly.co.uk.config.js
@@ -3,15 +3,17 @@ const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:mytelly.co.uk')
 
 dayjs.extend(utc)
 dayjs.extend(timezone)
 dayjs.extend(customParseFormat)
 
+doFetch.setDebugger(debug)
+
 const detailedGuide = true
 const tz = 'Europe/London'
-const nworker = 25
 
 module.exports = {
   site: 'mytelly.co.uk',
@@ -108,8 +110,7 @@ module.exports = {
   },
   async channels() {
     const channels = {}
-    const axios = require('axios')
-    const queues = [{ t: 'p', m: 'post', u: 'https://www.mytelly.co.uk/getform' }]
+    const queues = [{ t: 'p', method: 'post', url: 'https://www.mytelly.co.uk/getform' }]
     await doFetch(queues, (queue, res) => {
       // process form -> provider
       if (queue.t === 'p') {
@@ -118,7 +119,7 @@ module.exports = {
           .forEach(el => {
             const opt = $(el)
             const provider = opt.attr('value')
-            queues.push({ t: 'r', m: 'post', u: 'https://www.mytelly.co.uk/getregions', params: { provider } })
+            queues.push({ t: 'r', method: 'post', url: 'https://www.mytelly.co.uk/getregions', params: { provider } })
           })
       }
       // process provider -> region
@@ -134,7 +135,7 @@ module.exports = {
             u_time: now.format('HHmm'),
             is_mobile: 1
           }
-          queues.push({ t: 's', m: 'post', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
+          queues.push({ t: 's', method: 'post', url: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
         }
       }
       // process schedule -> channels
@@ -191,67 +192,3 @@ function parseText($item) {
 
   return text
 }
-
-async function doFetch(queues, cb) {
-  const axios = require('axios')
-
-  let n = Math.min(nworker, queues.length)
-  const workers = []
-  const adjustWorker = () => {
-    if (queues.length > workers.length && workers.length < nworker) {
-      let nw = Math.min(nworker, queues.length)
-      if (n < nw) {
-        n = nw
-        createWorker()
-      }
-    }
-  }
-  const createWorker = () => {
-    while (workers.length < n) {
-      startWorker()
-    }
-  }
-  const startWorker = () => {
-    const worker = () => {
-      if (queues.length) {
-        const queue = queues.shift()
-        const done = res => {
-          if (res) {
-            cb(queue, res)
-            adjustWorker()
-          }
-          worker()
-        }
-        const url = typeof queue === 'string' ? queue : queue.u
-        const params = typeof queue === 'object' && queue.params ? queue.params : {}
-        const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
-        debug(`fetch %s with %s`, url, JSON.stringify(params))
-        if (method === 'post') {
-          axios
-            .post(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        } else {
-          axios
-            .get(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        }
-      } else {
-        workers.splice(workers.indexOf(worker), 1)
-      }
-    }
-    workers.push(worker)
-    worker()
-  }
-  createWorker()
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (workers.length === 0) {
-        clearInterval(interval)
-        resolve()
-      }
-    }, 500)
-  })
-}
-