Some sites just refuses bot connection, so no Twitterbot agent from now

Author: polyrabbit

.github/workflows/static.yml

@@ -43,7 +43,7 @@ jobs:
           echo -e "df -h:\n`df -h`"
           echo "pwd: `pwd`"
           echo "nproc: `nproc`"
-          echo "curl -s ifconfig.me/all:\n`curl -s ifconfig.me/all`"
+          echo -e "curl -s ifconfig.me/all:\n`curl -s ifconfig.me/all`"
 
       - name: Export vars to env
         env:

page_content_extractor/http.py

@@ -51,7 +51,7 @@ def init_poolmanager(self, *args, **kwargs):
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # Some sites just refuses bot connection
-ua = UserAgent(browsers=['chrome'], fallback='Twitterbot/1.0')
+ua = UserAgent(browsers=['chrome'], min_percentage=10.0)
 ua_str = ua.random
 logger.info(f'Use user-agent {ua_str}')
 

templates/base.html

@@ -280,8 +280,7 @@ <h3 id="{{ news.slug() }}">
                 <li class="last-updated"
                     data-submitted="{{ last_updated.strftime("%Y-%m-%dT%H:%M:%SZ") }}"
                     data-toggle="tooltip"
-                    title="{{ last_updated.replace(microsecond=0) }} UTC">Last
-                    updated: <span>{{ last_updated.replace(microsecond=0) }} UTC</span>
+                    title="{{ last_updated.replace(microsecond=0) }} UTC">Last updated: <span>{{ last_updated.replace(microsecond=0) }} UTC</span>
                 </li>
             {% endif %}
         </ul>