Merge pull request #658 from Guovin/dev

Feat

Pipfile

@@ -25,7 +25,6 @@ flask = "*"
 opencc-python-reimplemented = "*"
 fake-useragent = "*"
 pillow = "*"
-yt-dlp = "*"
 m3u8 = "*"
 
 [packages]
@@ -39,7 +38,6 @@ opencc-python-reimplemented = "*"
 fake-useragent = "*"
 gunicorn = "*"
 pillow = "*"
-yt-dlp = "*"
 m3u8 = "*"
 
 [requires]

config/config.ini

@@ -13,6 +13,8 @@ open_keep_all = False
 open_sort = True
 sort_timeout = 10
 open_ffmpeg = True
+open_filter_speed = True
+min_speed = 0.5
 open_filter_resolution = True
 min_resolution = 1920x1080
 delay_weight = 0.25
@@ -26,7 +28,7 @@ ipv6_num = 5
 open_m3u_result = True
 url_keywords_blacklist = epg.pw,skype.serv00.net,iptv.yjxfz.com,live-hls-web-ajb.getaj.net,live.goodiptv.club,hc73k3dhwo5gfkt.wcetv.com,stream1.freetv.fun,zw9999.cnstream.top,zsntlqj.xicp.net
 open_subscribe = True
-subscribe_urls = https://iptv.b2og.com/txt/fmml_ipv6.txt,https://ghp.ci/raw.githubusercontent.com/suxuang/myIPTV/main/ipv6.m3u,https://live.zbds.top/tv/iptv6.txt,https://live.zbds.top/tv/iptv4.txt,https://live.fanmingming.com/tv/m3u/ipv6.m3u,https://ghp.ci/https://raw.githubusercontent.com/joevess/IPTV/main/home.m3u8,https://aktv.top/live.txt,http://175.178.251.183:6689/live.txt,https://ghp.ci/https://raw.githubusercontent.com/kimwang1978/collect-tv-txt/main/merged_output.txt,https://m3u.ibert.me/txt/fmml_dv6.txt,https://m3u.ibert.me/txt/o_cn.txt,https://m3u.ibert.me/txt/j_iptv.txt,https://ghp.ci/https://raw.githubusercontent.com/xzw832/cmys/main/S_CCTV.txt,https://ghp.ci/https://raw.githubusercontent.com/xzw832/cmys/main/S_weishi.txt,http://itv.22m.top/ITVBox/tv/tvonline.txt,https://ghp.ci//https://raw.githubusercontent.com/asdjkl6/tv/tv/.m3u/整套直播源/测试/整套直播源/l.txt,https://ghp.ci//https://raw.githubusercontent.com/asdjkl6/tv/tv/.m3u/整套直播源/测试/整套直播源/kk.txt
+subscribe_urls = https://iptv.b2og.com/txt/fmml_ipv6.txt,https://ghp.ci/raw.githubusercontent.com/suxuang/myIPTV/main/ipv6.m3u,https://live.zbds.top/tv/iptv6.txt,https://live.zbds.top/tv/iptv4.txt,https://live.fanmingming.com/tv/m3u/ipv6.m3u,https://ghp.ci/https://raw.githubusercontent.com/joevess/IPTV/main/home.m3u8,https://aktv.top/live.txt,http://175.178.251.183:6689/live.txt,https://ghp.ci/https://raw.githubusercontent.com/kimwang1978/collect-tv-txt/main/merged_output.txt,https://m3u.ibert.me/txt/fmml_dv6.txt,https://m3u.ibert.me/txt/o_cn.txt,https://m3u.ibert.me/txt/j_iptv.txt,https://ghp.ci/https://raw.githubusercontent.com/xzw832/cmys/main/S_CCTV.txt,https://ghp.ci/https://raw.githubusercontent.com/xzw832/cmys/main/S_weishi.txt,https://ghp.ci//https://raw.githubusercontent.com/asdjkl6/tv/tv/.m3u/整套直播源/测试/整套直播源/l.txt,https://ghp.ci//https://raw.githubusercontent.com/asdjkl6/tv/tv/.m3u/整套直播源/测试/整套直播源/kk.txt
 open_multicast = True
 open_multicast_foodie = True
 open_multicast_fofa = True

docs/config.md

@@ -17,6 +17,8 @@
 | sort_timeout           | 5                                       | 单个接口测速超时时长，单位秒(s)；数值越大测速所属时间越长，能提高获取接口数量，但质量会有所下降；数值越小测速所需时间越短，能获取低延时的接口，质量较好；调整此值能优化更新时间 |
 | open_ffmpeg            | True                                    | 开启使用 FFmpeg 进行测速，获取更准确的速度与分辨率信息，需要提前手动安装                                                  |
 | open_m3u_result        | True                                    | 开启转换生成 m3u 文件类型结果链接，支持显示频道图标                                                              |
+| open_filter_speed      | True                                    | 开启速率过滤，低于最小速率（min_speed）的接口将会被过滤                                                          |
+| min_speed              | 0.5                                     | 接口最小速率（单位M/s），需要开启 open_filter_speed 才能生效                                                 |
 | open_filter_resolution | True                                    | 开启分辨率过滤，低于最小分辨率（min_resolution）的接口将会被过滤                                                   |
 | min_resolution         | 1920x1080                               | 接口最小分辨率，需要开启 open_filter_resolution 才能生效                                                  |
 | speed_weight           | 0.5                                     | 速率权重值（所有权重值总和应为 1）                                                                        |

docs/config_en.md

@@ -17,6 +17,8 @@
 | sort_timeout           | 5                                          | The timeout duration for speed testing of a single interface, in seconds (s). A larger value means a longer testing period, which can increase the number of interfaces obtained but may decrease their quality. A smaller value means a shorter testing time, which can obtain low-latency interfaces with better quality. Adjusting this value can optimize the update time. |
 | open_ffmpeg            | True                                       | Enable speed testing using FFmpeg to obtain more accurate speed and resolution information. Manual installation is required in advance.                                                                                                                                                                                                                                        |
 | open_m3u_result        | True                                       | Enable the conversion to generate m3u file type result links, supporting the display of channel icons                                                                                                                                                                                                                                                                          |
+| open_filter_speed      | True                                       | Enable speed filtering, interfaces with speed lower than the minimum speed (min_speed) will be filtered                                                                                                                                                                                                                                                                        |
+| min_speed              | 0.5                                        | Minimum interface speed (M/s), requires enabling open_filter_speed to take effect                                                                                                                                                                                                                                                                                              |
 | open_filter_resolution | True                                       | Enable resolution filtering, interfaces with resolution lower than the minimum resolution (min_resolution) will be filtered                                                                                                                                                                                                                                                    |
 | min_resolution         | 1920x1080                                  | Minimum interface resolution, requires enabling open_filter_resolution to take effect                                                                                                                                                                                                                                                                                          |
 | speed_weight           | 0.5                                        | Speed weight value (the sum of all weight values should be 1)                                                                                                                                                                                                                                                                                                                  |

tkinter_ui/tkinter_ui.spec

@@ -8,7 +8,9 @@ a = Analysis(
         ('../config/config.ini', 'config'),
         ('../config/demo.txt', 'config'),
         ('../config/rtp', 'config/rtp'),
+        ('../updates/hotel/cache.pkl', 'updates/hotel'),
         ('../updates/multicast/multicast_map.json', 'updates/multicast'),
+        ('../updates/multicast/cache.pkl', 'updates/multicast'),
         ('../updates/fofa/fofa_hotel_region_result.pkl', 'updates/fofa'),
         ('../updates/fofa/fofa_multicast_region_result.pkl', 'updates/fofa'),
         ('../static/images/favicon.ico', 'static/images'),

updates/hotel/request.py

@@ -23,7 +23,7 @@
     retry_func,
     find_clickable_element_with_retry,
 )
-from utils.tools import get_pbar_remaining, get_soup, merge_objects
+from utils.tools import get_pbar_remaining, get_soup, merge_objects, resource_path
 
 if config.open_driver:
     try:
@@ -34,17 +34,16 @@
 
 async def get_channels_by_hotel(callback=None):
     """
-    Get the channels by multicase
+    Get the channels by hotel
     """
     channels = {}
     if config.open_use_cache:
         try:
             with open(
-                    constants.config_cache_path,
+                    resource_path("updates/hotel/cache.pkl"),
                     "rb",
             ) as file:
-                cache = pickle.load(file) or {}
-                channels = cache.get("hotel", {})
+                channels = pickle.load(file) or {}
         except:
             pass
     if config.open_request:

updates/multicast/request.py

@@ -26,7 +26,7 @@
     retry_func,
     find_clickable_element_with_retry,
 )
-from utils.tools import get_pbar_remaining, get_soup, merge_objects
+from utils.tools import get_pbar_remaining, get_soup, merge_objects, resource_path
 from .update_tmp import get_multicast_region_result_by_rtp_txt
 
 if config.open_driver:
@@ -44,11 +44,11 @@ async def get_channels_by_multicast(names, callback=None):
     if config.open_use_cache:
         try:
             with open(
-                    constants.config_cache_path,
+                    resource_path("updates/multicast/cache.pkl"),
                     "rb",
             ) as file:
                 cache = pickle.load(file) or {}
-                channels = {key: value for key, value in cache.get("multicast", {}).items() if key in names}
+                channels = {key: value for key, value in cache.items() if key in names}
         except:
             pass
     if config.open_request:

utils/channel.py

@@ -6,7 +6,6 @@
 import pickle
 import re
 from collections import defaultdict
-from logging import INFO
 
 from bs4 import NavigableString
 from opencc import OpenCC
@@ -15,17 +14,16 @@
 from utils.config import config
 from utils.speed import (
     get_speed,
-    sort_urls_by_speed_and_resolution,
+    sort_urls,
 )
 from utils.tools import (
     check_url_by_patterns,
-    get_total_urls_from_info_list,
+    get_total_urls,
     process_nested_dict,
     add_url_info,
     remove_cache_info,
     resource_path,
     write_content_into_txt,
-    get_logger,
 )
 
 
@@ -574,10 +572,9 @@ async def limited_get_speed(info, ipv6_proxy, callback):
         for info in info_list
     ]
     await asyncio.gather(*tasks)
-    logger = get_logger(constants.sort_log_path, level=INFO, init=True)
     for cate, obj in data.items():
         for name, info_list in obj.items():
-            info_list = sort_urls_by_speed_and_resolution(name, info_list, logger)
+            info_list = sort_urls(name, info_list)
             append_data_to_info_data(
                 result,
                 cate,
@@ -602,14 +599,18 @@ def write_channel_to_file(data, ipv6=False, callback=None):
         write_content_into_txt(f"{update_time},url", path)
     no_result_name = []
     open_empty_category = config.open_empty_category
+    ipv_type_prefer = list(config.ipv_type_prefer)
+    if any(pref in ipv_type_prefer for pref in ["自动", "auto"]) or not ipv_type_prefer:
+        ipv_type_prefer = ["ipv6", "ipv4"] if (ipv6 or (not os.environ.get("GITHUB_ACTIONS"))) else ["ipv4", "ipv6"]
+    origin_type_prefer = config.origin_type_prefer
     for cate, channel_obj in data.items():
         print(f"\n{cate}:", end=" ")
         write_content_into_txt(f"{cate},#genre#", path)
         channel_obj_keys = channel_obj.keys()
         names_len = len(list(channel_obj_keys))
         for i, name in enumerate(channel_obj_keys):
             info_list = data.get(cate, {}).get(name, [])
-            channel_urls = get_total_urls_from_info_list(info_list, ipv6=ipv6)
+            channel_urls = get_total_urls(info_list, ipv_type_prefer, origin_type_prefer)
             end_char = ", " if i < names_len - 1 else ""
             print(f"{name}:", len(channel_urls), end=end_char)
             if not channel_urls:

utils/config.py

@@ -58,6 +58,12 @@ def open_use_cache(self):
     def open_request(self):
         return self.config.getboolean("Settings", "open_request", fallback=False)
 
+    @property
+    def open_filter_speed(self):
+        return self.config.getboolean(
+            "Settings", "open_filter_speed", fallback=True
+        )
+
     @property
     def open_filter_resolution(self):
         return self.config.getboolean(
@@ -79,7 +85,7 @@ def ipv_type_prefer(self):
         return [
             type.strip().lower()
             for type in self.config.get(
-                "Settings", "ipv_type_prefer", fallback="ipv4"
+                "Settings", "ipv_type_prefer", fallback="auto"
             ).split(",")
         ]
 
@@ -135,6 +141,10 @@ def source_limits(self):
             "online_search": self.online_search_num,
         }
 
+    @property
+    def min_speed(self):
+        return self.config.getfloat("Settings", "min_speed", fallback=0.5)
+
     @property
     def min_resolution(self):
         return self.config.get("Settings", "min_resolution", fallback="1920x1080")

utils/constants.py

@@ -4,8 +4,6 @@
 
 output_path = "output"
 
-config_cache_path = os.path.join(config_path, "cache.pkl")
-
 result_path = os.path.join(output_path, "result_new.txt")
 
 cache_path = os.path.join(output_path, "cache.pkl")

utils/speed.py

@@ -1,19 +1,17 @@
 import asyncio
 import re
 import subprocess
+from logging import INFO
 from time import time
 from urllib.parse import quote
 
 import m3u8
-import yt_dlp
 from aiohttp import ClientSession, TCPConnector
 
 import utils.constants as constants
 from utils.config import config
 from utils.tools import is_ipv6, remove_cache_info, get_resolution_value, get_logger
 
-logger = get_logger(constants.log_path)
-
 
 async def get_speed_with_download(url: str, timeout: int = config.sort_timeout) -> dict[str, float | None]:
     """
@@ -68,42 +66,6 @@ async def get_speed_m3u8(url: str, timeout: int = config.sort_timeout) -> dict[s
         return info
 
 
-def get_info_yt_dlp(url, timeout=config.sort_timeout):
-    """
-    Get the url info by yt_dlp
-    """
-    ydl_opts = {
-        "socket_timeout": timeout,
-        "skip_download": True,
-        "quiet": True,
-        "no_warnings": True,
-        "format": "best",
-        "logger": logger,
-    }
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        return ydl.sanitize_info(ydl.extract_info(url, download=False))
-
-
-async def get_delay_yt_dlp(url, timeout=config.sort_timeout):
-    """
-    Get the delay of the url by yt_dlp
-    """
-    try:
-        start_time = time()
-        info = await asyncio.wait_for(
-            asyncio.to_thread(get_info_yt_dlp, url, timeout), timeout=timeout
-        )
-        fps = int(round((time() - start_time) * 1000)) if len(info) else float("inf")
-        resolution = (
-            f"{info['width']}x{info['height']}"
-            if "width" in info and "height" in info
-            else None
-        )
-        return fps, resolution
-    except:
-        return float("inf"), None
-
-
 async def get_delay_requests(url, timeout=config.sort_timeout, proxy=None):
     """
     Get the delay of the url by requests
@@ -242,11 +204,13 @@ async def get_speed(url, ipv6_proxy=None, callback=None):
             callback()
 
 
-def sort_urls_by_speed_and_resolution(name, data, logger=None):
+def sort_urls(name, data, logger=None):
     """
-    Sort by speed and resolution
+    Sort the urls with info
     """
     filter_data = []
+    if logger is None:
+        logger = get_logger(constants.sort_log_path, level=INFO, init=True)
     for url, date, resolution, origin in data:
         result = {
             "url": remove_cache_info(url),
@@ -274,10 +238,14 @@ def sort_urls_by_speed_and_resolution(name, data, logger=None):
                             )
                     except Exception as e:
                         print(e)
+                    if config.open_filter_speed and speed < config.min_speed:
+                        continue
                     result["delay"] = delay
                     result["speed"] = speed
                     result["resolution"] = resolution
                     filter_data.append(result)
+    if logger:
+        logger.handlers.clear()
 
     def combined_key(item):
         speed, delay, resolution, origin = item["speed"], item["delay"], item["resolution"], item["origin"]

utils/tools.py

@@ -141,20 +141,15 @@ def get_resolution_value(resolution_str):
         return 0
 
 
-def get_total_urls_from_info_list(infoList, ipv6=False):
+def get_total_urls(info_list, ipv_type_prefer, origin_type_prefer):
     """
     Get the total urls from info list
     """
-    ipv_type_prefer = list(config.ipv_type_prefer)
-    if any(pref in ipv_type_prefer for pref in ["自动", "auto"]) or not ipv_type_prefer:
-        ipv_type_prefer = ["ipv6", "ipv4"] if (ipv6 or os.environ.get("GITHUB_ACTIONS")) else ["ipv4", "ipv6"]
-    origin_type_prefer = config.origin_type_prefer
     categorized_urls = {
         origin: {"ipv4": [], "ipv6": []} for origin in origin_type_prefer
     }
-
     total_urls = []
-    for url, _, resolution, origin in infoList:
+    for url, _, resolution, origin in info_list:
         if not origin:
             continue
 
@@ -205,13 +200,16 @@ def get_total_urls_from_info_list(infoList, ipv6=False):
             if len(total_urls) >= urls_limit:
                 break
             if ipv_num[ipv_type] < config.ipv_limit[ipv_type]:
+                urls = categorized_urls[origin][ipv_type]
+                if not urls:
+                    break
                 limit = min(
                     max(config.source_limits[origin] - ipv_num[ipv_type], 0),
                     max(config.ipv_limit[ipv_type] - ipv_num[ipv_type], 0),
                 )
-                urls = categorized_urls[origin][ipv_type][:limit]
-                total_urls.extend(urls)
-                ipv_num[ipv_type] += len(urls)
+                limit_urls = urls[:limit]
+                total_urls.extend(limit_urls)
+                ipv_num[ipv_type] += len(limit_urls)
             else:
                 continue
 
@@ -274,7 +272,7 @@ def check_ipv6_support():
             return True
     except Exception:
         pass
-    print("Your network does not support IPv6")
+    print("Your network does not support IPv6, don't worry, these results will be saved")
     return False