From 79d34d9fd3162ceec7204965f5178c5d578757e7 Mon Sep 17 00:00:00 2001 From: gram Date: Wed, 2 Aug 2023 09:05:56 +0200 Subject: [PATCH 1/3] add command to add post ids --- assign_ids.py | 18 ------- sdk/commands/__init__.py | 2 + sdk/commands/_add_ids.py | 108 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 18 deletions(-) delete mode 100644 assign_ids.py create mode 100644 sdk/commands/_add_ids.py diff --git a/assign_ids.py b/assign_ids.py deleted file mode 100644 index 15a0d4f..0000000 --- a/assign_ids.py +++ /dev/null @@ -1,18 +0,0 @@ -import json -from pathlib import Path - - -meta = json.loads(Path('result.json').read_text('utf8')) -dates = {} -for msg in meta['messages']: - if msg['type'] == 'message': - d = msg['date'].split('T')[0] - dates[d] = msg['id'] - -for pp in Path('posts').iterdir(): - if pp.is_dir(): - continue - t = pp.read_text('utf8') - for d, id in dates.items(): - t = t.replace(f'published: {d}', f'published: {d}\nid: {id}') - pp.write_text(t, 'utf8') diff --git a/sdk/commands/__init__.py b/sdk/commands/__init__.py index 5b67861..0819565 100644 --- a/sdk/commands/__init__.py +++ b/sdk/commands/__init__.py @@ -1,3 +1,4 @@ +from ._add_ids import AddIDsCommand from ._check_all import CheckAllCommand from ._command import Command from ._html import HTMLCommand @@ -10,6 +11,7 @@ __all__ = ['Command', 'COMMANDS'] COMMANDS = ( + AddIDsCommand, HTMLCommand, RunCodeCommand, ScheduleCommand, diff --git a/sdk/commands/_add_ids.py b/sdk/commands/_add_ids.py new file mode 100644 index 0000000..3e2c73b --- /dev/null +++ b/sdk/commands/_add_ids.py @@ -0,0 +1,108 @@ + +from __future__ import annotations + +import asyncio +import os +from functools import cached_property +from pathlib import Path + +from telethon import TelegramClient + +from ..post import Post +from ._command import Command +import re + +CHANNEL = 'pythonetc' +REX_URL = re.compile(r'https?\:[a-zA-Z0-9\-\/\.\(]+') +REX_PYTHON = re.compile(r'3\.[0-9]{1,2}') +REX_QNAME = re.compile(r'[a-z]{2,}\.[a-zA-Z0-9]{2,}') +URLs = tuple[str, ...] + + +class AddIDsCommand(Command): + """Add Telegram post IDs to all posts that don't have it. + """ + name = 'add-ids' + + def run(self) -> int: + if 'API_ID' not in os.environ: + self.warn( + 'API_ID and API_HASH env vars required, ' + 'you can get them at https://my.telegram.org/apps', + ) + return 1 + asyncio.run(self._run()) + return 0 + + async def _run(self) -> None: + self.print('reading posts...') + paths = self._get_paths() + self.print('fetching IDs...') + async with self._client: + ids = await self._get_ids() + self.print('setting IDs...') + for keyword, id in ids.items(): + path = paths.get(keyword) + if path is None: + continue + content = path.read_text(encoding='utf-8') + lines = content.splitlines() + lines.insert(2, f'id: {id}') + new_content = '\n'.join(lines) + new_content = new_content.rstrip() + '\n' + assert new_content != content + path.write_text(new_content, encoding='utf-8') + self.print(f'added ID for {path.name}') + + def _get_paths(self) -> dict[URLs, Path]: + paths: dict[URLs, Path] = {} + for path in sorted(Path('posts').iterdir()): + if path.suffix != '.md': + continue + post = Post.from_path(path) + if post.id is not None: + continue + keywords = self._get_keywords(post.md_content) + if not keywords: + continue + if keywords in paths: + name1 = path.name + name2 = paths[keywords].name + msg = f'duplicate set of keywords: {name1} and {name2}' + raise RuntimeError(msg) + paths[keywords] = path + return paths + + async def _get_ids(self) -> dict[URLs, int]: + ids = {} + async for message in self._client.iter_messages(CHANNEL): + if message.text is None: + continue + keywords = self._get_keywords(message.text) + if not keywords: + continue + if keywords in ids: + continue + ids[keywords] = message.id + return ids + + def _get_keywords(self, text: str) -> URLs: + """ + Get some key components from the text (URLs, qualnames, Python versions) + that can be used to uniquely identify a text. + + It allows us to match the same text ignoring changes in formatting + or corrected typos. + """ + result = REX_URL.findall(text) + result.extend(REX_PYTHON.findall(text)) + result.extend(REX_QNAME.findall(text)) + return tuple(result) + + @cached_property + def _client(self) -> TelegramClient: + return TelegramClient( + 'bot', + api_id=os.environ['API_ID'], + api_hash=os.environ['API_HASH'], + ) From b3b3a412af9a70db305f0b7bb240fec2db3b00d4 Mon Sep 17 00:00:00 2001 From: gram Date: Wed, 2 Aug 2023 09:06:24 +0200 Subject: [PATCH 2/3] add missed ids to posts --- posts/add-note.md | 1 + posts/assert-type.md | 1 + posts/asyncio-create-task.md | 1 + posts/asyncio-gather.md | 1 + posts/asyncio-run.md | 1 + posts/asyncio-sleep.md | 1 + posts/build-system.md | 1 + posts/chain-is.md | 1 + posts/contextlib-chdir.md | 1 + posts/dataclass-transform.md | 1 + posts/datetime-fold.md | 1 + posts/except-star.md | 1 + posts/exception-group.md | 1 + posts/generator-scope.md | 1 + posts/isinstance.md | 1 + posts/literal-string.md | 1 + posts/math-fsum.md | 1 + posts/os-curdir.md | 1 + posts/pyproject-toml.md | 1 + posts/python-3-11.md | 1 + posts/reveal-type.md | 1 + posts/string-template.md | 1 + posts/tomllib.md | 1 + posts/traceback-col.md | 1 + posts/typing-self.md | 1 + posts/unittest-mock-seal.md | 1 + posts/warnings.md | 1 + posts/wheel.md | 1 + posts/zoneinfo.md | 1 + 29 files changed, 29 insertions(+) diff --git a/posts/add-note.md b/posts/add-note.md index ff56d05..c139b64 100644 --- a/posts/add-note.md +++ b/posts/add-note.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 704 published: 2022-11-01 traces: - [type: BaseException, method: add_note] diff --git a/posts/assert-type.md b/posts/assert-type.md index c7c4742..922d022 100644 --- a/posts/assert-type.md +++ b/posts/assert-type.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 709 published: 2022-11-29 traces: - [module: typing, function: assert_type] diff --git a/posts/asyncio-create-task.md b/posts/asyncio-create-task.md index 18db936..6f73003 100644 --- a/posts/asyncio-create-task.md +++ b/posts/asyncio-create-task.md @@ -1,5 +1,6 @@ --- published: 2023-04-18 +id: 724 author: orsinium traces: - [module: asyncio, function: create_task] diff --git a/posts/asyncio-gather.md b/posts/asyncio-gather.md index 89b4658..0305c4e 100644 --- a/posts/asyncio-gather.md +++ b/posts/asyncio-gather.md @@ -1,5 +1,6 @@ --- published: 2023-04-11 +id: 723 author: orsinium traces: - [module: asyncio, function: gather] diff --git a/posts/asyncio-run.md b/posts/asyncio-run.md index c4e6ea4..a100578 100644 --- a/posts/asyncio-run.md +++ b/posts/asyncio-run.md @@ -1,5 +1,6 @@ --- published: 2023-03-28 +id: 721 author: orsinium traces: - [module: asyncio, function: run] diff --git a/posts/asyncio-sleep.md b/posts/asyncio-sleep.md index 849e93a..0599f22 100644 --- a/posts/asyncio-sleep.md +++ b/posts/asyncio-sleep.md @@ -1,5 +1,6 @@ --- published: 2023-04-04 +id: 722 author: orsinium traces: - [module: asyncio, function: sleep] diff --git a/posts/build-system.md b/posts/build-system.md index 9e2568b..b7eeb06 100644 --- a/posts/build-system.md +++ b/posts/build-system.md @@ -1,5 +1,6 @@ --- published: 2023-02-28 +id: 717 author: orsinium topics: - packaging diff --git a/posts/chain-is.md b/posts/chain-is.md index f081a52..235be4f 100644 --- a/posts/chain-is.md +++ b/posts/chain-is.md @@ -1,5 +1,6 @@ --- published: 2022-09-27 +id: 699 author: orsinium traces: - [keyword: is] diff --git a/posts/contextlib-chdir.md b/posts/contextlib-chdir.md index 96361a8..6712483 100644 --- a/posts/contextlib-chdir.md +++ b/posts/contextlib-chdir.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 708 published: 2022-11-22 traces: - [module: contextlib, function: chdir] diff --git a/posts/dataclass-transform.md b/posts/dataclass-transform.md index b6a21d7..c0fa346 100644 --- a/posts/dataclass-transform.md +++ b/posts/dataclass-transform.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 710 published: 2022-12-06 pep: 681 python: "3.11" diff --git a/posts/datetime-fold.md b/posts/datetime-fold.md index 139103b..27b693d 100644 --- a/posts/datetime-fold.md +++ b/posts/datetime-fold.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 727 published: 2023-07-25 pep: 495 python: "3.6" diff --git a/posts/except-star.md b/posts/except-star.md index 3d12b0f..d3b08a8 100644 --- a/posts/except-star.md +++ b/posts/except-star.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 706 published: 2022-11-15 traces: - [{exception: ExceptionGroup}] diff --git a/posts/exception-group.md b/posts/exception-group.md index ee0141a..97b0eb7 100644 --- a/posts/exception-group.md +++ b/posts/exception-group.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 705 published: 2022-11-08 traces: - [{exception: ExceptionGroup}] diff --git a/posts/generator-scope.md b/posts/generator-scope.md index 34ea835..335fba3 100644 --- a/posts/generator-scope.md +++ b/posts/generator-scope.md @@ -1,5 +1,6 @@ --- author: ypankovych +id: 700 published: 2022-10-11 --- diff --git a/posts/isinstance.md b/posts/isinstance.md index f155720..89dc7ce 100644 --- a/posts/isinstance.md +++ b/posts/isinstance.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 714 published: 2023-01-03 traces: - [{function: isinstance}] diff --git a/posts/literal-string.md b/posts/literal-string.md index 322c87e..c3b77ae 100644 --- a/posts/literal-string.md +++ b/posts/literal-string.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 713 published: 2022-12-27 pep: 675 python: "3.11" diff --git a/posts/math-fsum.md b/posts/math-fsum.md index a52421e..cb5d66c 100644 --- a/posts/math-fsum.md +++ b/posts/math-fsum.md @@ -1,5 +1,6 @@ --- published: 2023-03-14 +id: 719 author: orsinium traces: - [module: math, function: fsum] diff --git a/posts/os-curdir.md b/posts/os-curdir.md index c9fde35..93eb790 100644 --- a/posts/os-curdir.md +++ b/posts/os-curdir.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 701 published: 2022-10-18 traces: - [module: os, function: curdir] diff --git a/posts/pyproject-toml.md b/posts/pyproject-toml.md index 8cec789..77ecb57 100644 --- a/posts/pyproject-toml.md +++ b/posts/pyproject-toml.md @@ -1,5 +1,6 @@ --- published: 2023-02-21 +id: 716 author: orsinium topics: - packaging diff --git a/posts/python-3-11.md b/posts/python-3-11.md index c8a9fb7..b996f6e 100644 --- a/posts/python-3-11.md +++ b/posts/python-3-11.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 702 published: 2022-10-24 topics: - news diff --git a/posts/reveal-type.md b/posts/reveal-type.md index 61cc453..0fa6d44 100644 --- a/posts/reveal-type.md +++ b/posts/reveal-type.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 712 published: 2022-12-20 topics: - typing diff --git a/posts/string-template.md b/posts/string-template.md index 2fc95a6..f3f765c 100644 --- a/posts/string-template.md +++ b/posts/string-template.md @@ -1,5 +1,6 @@ --- published: 2022-09-20 +id: 698 author: orsinium traces: - [module: string, type: Template] diff --git a/posts/tomllib.md b/posts/tomllib.md index a8b05dd..2c425c5 100644 --- a/posts/tomllib.md +++ b/posts/tomllib.md @@ -1,5 +1,6 @@ --- published: 2023-03-07 +id: 718 author: orsinium traces: - [module: tomllib] diff --git a/posts/traceback-col.md b/posts/traceback-col.md index 95ae8b5..d5159cd 100644 --- a/posts/traceback-col.md +++ b/posts/traceback-col.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 703 published: 2022-10-25 pep: 657 python: "3.11" diff --git a/posts/typing-self.md b/posts/typing-self.md index e82f9b6..f7aae2a 100644 --- a/posts/typing-self.md +++ b/posts/typing-self.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 711 published: 2022-12-03 traces: - [module: typing, type: Self] diff --git a/posts/unittest-mock-seal.md b/posts/unittest-mock-seal.md index 6c7786f..40d2bdb 100644 --- a/posts/unittest-mock-seal.md +++ b/posts/unittest-mock-seal.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 728 published: 2023-08-01 traces: - [module: unittest.mock, function: seal] diff --git a/posts/warnings.md b/posts/warnings.md index 85a2453..1f4e731 100644 --- a/posts/warnings.md +++ b/posts/warnings.md @@ -1,5 +1,6 @@ --- published: 2022-09-13 +id: 697 author: orsinium traces: - [module: warnings] diff --git a/posts/wheel.md b/posts/wheel.md index 6b5f904..781ef2a 100644 --- a/posts/wheel.md +++ b/posts/wheel.md @@ -1,5 +1,6 @@ --- published: 2023-02-14 +id: 715 author: orsinium topics: - packaging diff --git a/posts/zoneinfo.md b/posts/zoneinfo.md index 7552deb..9a2af7b 100644 --- a/posts/zoneinfo.md +++ b/posts/zoneinfo.md @@ -1,5 +1,6 @@ --- author: orsinium +id: 726 published: 2023-07-18 pep: 615 python: "3.6" From a91f5c039cb59efca713ad93276c98b902514139 Mon Sep 17 00:00:00 2001 From: gram Date: Wed, 2 Aug 2023 09:09:50 +0200 Subject: [PATCH 3/3] sort imports --- sdk/commands/_add_ids.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/commands/_add_ids.py b/sdk/commands/_add_ids.py index 3e2c73b..8d5e102 100644 --- a/sdk/commands/_add_ids.py +++ b/sdk/commands/_add_ids.py @@ -3,6 +3,7 @@ import asyncio import os +import re from functools import cached_property from pathlib import Path @@ -10,7 +11,7 @@ from ..post import Post from ._command import Command -import re + CHANNEL = 'pythonetc' REX_URL = re.compile(r'https?\:[a-zA-Z0-9\-\/\.\(]+')