🎉 V6.2.0 - Various Fixes (#1089)

* GitHub actions failing bc TikTok seems to be detecting the IPs of the GitHub actions 💀 (or ms_tokens are now tied to a specific ip) tests pass locally 
* changed docs to address bug reports (#1052)

* Update video_example.py class name (#1056)

I was looking at the examples and noticed that the name of the function in video_example.py is the same as in user_example.py, which led me to believe it was a mistake. I just changed it for better understanding.

* Update tiktok.py (#1070)

proxy support fix

* feat: add comments replies retrieval (#1072)

* Correctly filling in emptyresponseexception constructor parameters (#1073)

* Update tiktok.py (#1080)

Resolves #1074

* Correct file name (#1082)

* Update id and keys for video info scraping (Issue fix #1064) (#1086)

* Update id and keys for video info scraping

* Added error handling for the case such as no video

* V6.2.0

* bump version

---------

Co-authored-by: Pranav Dronavalli <[email protected]>
Co-authored-by: Edmundo Meyer <[email protected]>
Co-authored-by: Vijay Raghuwanshi <[email protected]>
Co-authored-by: Giancarlo Rocha <[email protected]>
Co-authored-by: Ben Steel <[email protected]>
Co-authored-by: Glenn Jocher <[email protected]>
Co-authored-by: Marco <[email protected]>
Co-authored-by: kenki931128 <[email protected]>

.sphinx/conf.py

@@ -16,7 +16,7 @@
 project = "TikTokAPI"
 copyright = "2023, David Teather"
 author = "David Teather"
-release = "v6.1.1"
+release = "v6.2.0"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/main/usage/configuration.html#general-configuration

CITATION.cff

@@ -5,5 +5,5 @@ authors:
  orcid: "https://orcid.org/0000-0002-9467-4676"
 title: "TikTokAPI"
 url: "https://github.com/davidteather/tiktok-api"
-version: 6.1.1
-date-released: 2023-8-20
+version: 6.2.0
+date-released: 2023-11-27

README.md

@@ -85,6 +85,8 @@ Please don't open an issue if you're experiencing one of these just comment if t
 
 * **Browser Has no Attribute** - make sure you ran `python3 -m playwright install`, if your error persists try the [playwright-python](https://github.com/microsoft/playwright-python) quickstart guide and diagnose issues from there.
 
+* **API methods returning Coroutine** - many of the API's methods are async so make sure your program awaits them for proper functionality
+
 ## Quick Start Guide
 
 Here's a quick bit of code to get the most recent trending videos on TikTok. There's more examples in the [examples](https://github.com/davidteather/TikTok-Api/tree/main/examples) directory.
@@ -111,7 +113,7 @@ if __name__ == "__main__":
 
 To directly run the example scripts from the repository root, use the `-m` option on python.
 ```sh
-python -m examples.get_trending
+python -m examples.trending_example
 ```
 
 You can access the full data dictionary the object was created from with `.as_dict`. On a video this may look like

TikTokApi/api/comment.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
-from typing import ClassVar, Optional
+from typing import ClassVar, Iterator, Optional
 from typing import TYPE_CHECKING, ClassVar, Optional
 
+from TikTokApi.exceptions import InvalidResponseException
+
 if TYPE_CHECKING:
  from ..tiktok import TikTokApi
  from .user import User
@@ -49,6 +51,38 @@ def __extract_from_data(self):
  )
  self.likes_count = self.as_dict["digg_count"]
 
+ async def replies(self, count=20, cursor=0, **kwargs) -> Iterator[Comment]:
+ found = 0
+
+ while found < count:
+ params = {
+ "count": 20,
+ "cursor": cursor,
+ "item_id": self.author.user_id,
+ "comment_id": self.id,
+ }
+
+ resp = await self.parent.make_request(
+ url="https://www.tiktok.com/api/comment/list/reply/",
+ params=params,
+ headers=kwargs.get("headers"),
+ session_index=kwargs.get("session_index"),
+ )
+
+ if resp is None:
+ raise InvalidResponseException(
+ resp, "TikTok returned an invalid response."
+ )
+
+ for comment in resp.get("comments", []):
+ yield self.parent.comment(data=comment)
+ found += 1
+
+ if not resp.get("has_more", False):
+ return
+
+ cursor = resp.get("cursor")
+
  def __repr__(self):
  return self.__str__()
 

TikTokApi/api/video.py

@@ -102,25 +102,55 @@ async def info(self, **kwargs) -> dict:
  r.text, "TikTok returned an invalid response.", error_code=r.status_code
  )
 
+ # Try SIGI_STATE first
  # extract tag <script id="SIGI_STATE" type="application/json">{..}</script>
  # extract json in the middle
 
  start = r.text.find('<script id="SIGI_STATE" type="application/json">')
- if start == -1:
- raise InvalidResponseException(
- r.text, "TikTok returned an invalid response.", error_code=r.status_code
- )
+ if start != -1:
+ start += len('<script id="SIGI_STATE" type="application/json">')
+ end = r.text.find("</script>", start)
 
- start += len('<script id="SIGI_STATE" type="application/json">')
- end = r.text.find("</script>", start)
+ if end == -1:
+ raise InvalidResponseException(
+ r.text, "TikTok returned an invalid response.", error_code=r.status_code
+ )
 
- if end == -1:
- raise InvalidResponseException(
-  r.text, "TikTok returned an invalid response.", error_code=r.status_code
- )
+  data = json.loads(r.text[start:end])
+ video_info = data["ItemModule"][self.id]
+ else:
+ # Try __UNIVERSAL_DATA_FOR_REHYDRATION__ next
 
- data = json.loads(r.text[start:end])
- video_info = data["ItemModule"][self.id]
+ # extract tag <script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">{..}</script>
+ # extract json in the middle
+
+ start = r.text.find('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')
+ if start == -1:
+ raise InvalidResponseException(
+ r.text, "TikTok returned an invalid response.", error_code=r.status_code
+ )
+
+ start += len('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')
+ end = r.text.find("</script>", start)
+
+ if end == -1:
+ raise InvalidResponseException(
+ r.text, "TikTok returned an invalid response.", error_code=r.status_code
+ )
+
+ data = json.loads(r.text[start:end])
+ default_scope = data.get("__DEFAULT_SCOPE__", {})
+ video_detail = default_scope.get("webapp.video-detail", {})
+ if video_detail.get("statusCode", 0) != 0: # assume 0 if not present
+ raise InvalidResponseException(
+ r.text, "TikTok returned an invalid response structure.", error_code=r.status_code
+ )
+ video_info = video_detail.get("itemInfo", {}).get("itemStruct")
+ if video_info is None:
+ raise InvalidResponseException(
+ r.text, "TikTok returned an invalid response structure.", error_code=r.status_code
+ )
+
  self.as_dict = video_info
  self.__extract_from_data()
  return video_info

TikTokApi/tiktok.py

@@ -243,7 +243,7 @@ async def create_sessions(
  override_browser_args = ["--headless=new"]
  headless = False # managed by the arg
  self.browser = await self.playwright.chromium.launch(
- headless=headless, args=override_browser_args
+ headless=headless, args=override_browser_args, proxy=random_choice(proxies)
  )
 
  await asyncio.gather(
@@ -338,6 +338,7 @@ async def run_fetch_script(self, url: str, headers: dict, **kwargs):
  async def generate_x_bogus(self, url: str, **kwargs):
  """Generate the X-Bogus header for a url"""
  _, session = self._get_session(**kwargs)
+ await session.page.wait_for_function("window.byted_acrawler !== undefined")
  result = await session.page.evaluate(
  f'() => {{ return window.byted_acrawler.frontierSign("{url}") }}'
  )
@@ -426,7 +427,7 @@ async def make_request(
  raise Exception("TikTokApi.run_fetch_script returned None")
 
  if result == "":
- raise EmptyResponseException()
+ raise EmptyResponseException(result, "TikTok returned an empty response")
 
  try:
  data = json.loads(result)

examples/video_example.py

@@ -7,7 +7,7 @@
 ) # set your own ms_token, think it might need to have visited a profile
 
 
-async def user_example():
+async def get_video_example():
  async with TikTokApi() as api:
  await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
  video = api.video(
@@ -23,4 +23,4 @@ async def user_example():
 
 
 if __name__ == "__main__":
- asyncio.run(user_example())
+ asyncio.run(get_video_example())

setup.py

@@ -8,7 +8,7 @@
 setuptools.setup(
  name="TikTokApi",
  packages=setuptools.find_packages(),
- version="6.1.1",
+ version="6.2.0",
  license="MIT",
  description="The Unofficial TikTok API Wrapper in Python 3.",
  author="David Teather",

tests/test_search.py

@@ -4,9 +4,20 @@
 
 ms_token = os.environ.get("ms_token", None)
 
-
 @pytest.mark.asyncio
-async def test_users():
+async def test_users_single_page():
+ api = TikTokApi()
+ async with api:
+ await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)
+ count = 0
+ async for user in api.search.users("therock", count=10):
+ count += 1
+
+ assert count >= 10
+
+#@pytest.mark.asyncio
+@pytest.mark.skip(reason="Known issue, see #1088 (https://github.com/davidteather/TikTok-Api/issues/1088)")
+async def test_users_multi_page():
  api = TikTokApi()
  async with api:
  await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3)