Entry icon url (#91)

* add support for favicon from html

* add Entry.icon_url

* parse favicon for standalone entries

* fix favicon parsing from html

* use entry favicon as fallback

* more lenient parsing of article title

Author: facundoolano

feedi/models.py

@@ -518,6 +518,9 @@ class Entry(db.Model):
     header = sa.Column(
         sa.String, doc="an html line to put above the title, such as 'user reblogged'.")
 
+    icon_url = sa.Column(
+        sa.String, doc="To be used for standalone entry avatars or as a fallback when the feed has no icon.")
+
     __table_args__ = (sa.UniqueConstraint("feed_id", "remote_id"),
                       sa.Index("entry_sort_ts", sort_date.desc()))
 

feedi/parsers/html.py

@@ -23,7 +23,7 @@ def fetch(url):
     soup = BeautifulSoup(response.content, 'lxml')
     metadata = scraping.all_meta(soup)
 
-    title = metadata.get('og:title', metadata.get('twitter:title'))
+    title = metadata.get('og:title', metadata.get('twitter:title', getattr(soup.title, 'text')))
 
     if not title or (metadata.get('og:type') and metadata['og:type'] != 'article'):
         raise ValueError(f"{url} is missing article metadata")
@@ -35,6 +35,8 @@ def fetch(url):
 
     username = metadata.get('author', '').split(',')[0]
 
+    icon_url = scraping.get_favicon(url, html=response.content)
+
     entry = {
         'remote_id': url,
         'title': title,
@@ -45,7 +47,7 @@ def fetch(url):
         'media_url': metadata.get('og:image', metadata.get('twitter:image')),
         'target_url': url,
         'content_url': url,
-        'raw_data': json.dumps(metadata)
-    }
+        'raw_data': json.dumps(metadata),
+        'icon_url': icon_url}
 
     return entry

feedi/scraping.py

@@ -5,21 +5,26 @@
 import urllib
 import zipfile
 
-import favicon
+# use internal module to access unexported .tags function
+import favicon.favicon as favicon
 from bs4 import BeautifulSoup
 
-from feedi.requests import requests
+from feedi.requests import USER_AGENT, requests
 
 logger = logging.getLogger(__name__)
 
 
-def get_favicon(url):
+def get_favicon(url, html=None):
     "Return the best favicon from the given url, or None."
     url_parts = urllib.parse.urlparse(url)
     url = f'{url_parts.scheme}://{url_parts.netloc}'
 
     try:
-        favicons = favicon.get(url)
+        if not html:
+            favicons = favicon.get(url, headers={'User-Agent': USER_AGENT}, timeout=2)
+        else:
+            favicons = sorted(favicon.tags(url, html),
+                              key=lambda i: i.width + i.height, reverse=True)
     except Exception:
         logger.exception("error fetching favicon: %s", url)
         return

feedi/templates/entry_avatar.html

@@ -8,5 +8,5 @@
     </a>
     {% else %}
     {% set domain = entry.target_url | url_domain %}
-    <img class="is-rounded feed-avatar" alt="{{ domain.0.upper() }}" title="{{ domain }}">
+    <img class="is-rounded feed-avatar" {% if entry.icon_url %}src="{{ entry.icon_url }}"{% endif %} alt="{{ domain.0.upper() }}" title="{{ domain }}">
     {% endif %}

migrations/versions/29fa12e6895d_entry_icon_url.py

@@ -0,0 +1,34 @@
+"""entry icon_url
+
+Revision ID: 29fa12e6895d
+Revises: aed00250a7ad
+Create Date: 2024-01-08 15:15:50.478699
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '29fa12e6895d'
+down_revision: Union[str, None] = 'aed00250a7ad'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('entries', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('icon_url', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('entries', schema=None) as batch_op:
+        batch_op.drop_column('icon_url')
+
+    # ### end Alembic commands ###