From d5082fab739cdb687ebf9cf142113086defd5f51 Mon Sep 17 00:00:00 2001
From: Erik Johnson <palehose@gmail.com>
Date: Thu, 27 Aug 2020 20:49:26 -0500
Subject: [PATCH] Fix both broken weather modules (#789)

* Fix both broken weather modules

The Weather.com changed its API again, this fixes that as well as fixing
a bug that caused the weather conditions to always show as "None".

For Weather Underground, the module had been broken for some time due to
the discontinuation of their API. The module has been rewritten to use
the same API calls that the website itself uses.

* Fix double-click browser launch in wunderground module

* Add example of longer location_code for weather.com
---
 i3pystatus/weather/__init__.py     |  74 +++----
 i3pystatus/weather/weathercom.py   |  22 +-
 i3pystatus/weather/wunderground.py | 323 ++++++++++++-----------------
 3 files changed, 174 insertions(+), 245 deletions(-)

diff --git a/i3pystatus/weather/__init__.py b/i3pystatus/weather/__init__.py
index 6c244282..405e4c5b 100644
--- a/i3pystatus/weather/__init__.py
+++ b/i3pystatus/weather/__init__.py
@@ -2,7 +2,7 @@
 import re
 import threading
 import time
-from urllib.request import urlopen
+from urllib.request import Request, urlopen
 
 from i3pystatus import SettingsBase, IntervalModule, formatp
 from i3pystatus.core.util import user_open, internet, require
@@ -12,32 +12,37 @@ class WeatherBackend(SettingsBase):
     settings = ()
 
     @require(internet)
-    def api_request(self, url):
+    def http_request(self, url, headers=None):
+        req = Request(url, headers=headers or {})
+        with urlopen(req) as content:
+            try:
+                content_type = dict(content.getheaders())['Content-Type']
+                charset = re.search(r'charset=(.*)', content_type).group(1)
+            except AttributeError:
+                charset = 'utf-8'
+            return content.read().decode(charset)
+
+    @require(internet)
+    def api_request(self, url, headers=None):
         self.logger.debug('Making API request to %s', url)
         try:
-            with urlopen(url) as content:
-                try:
-                    content_type = dict(content.getheaders())['Content-Type']
-                    charset = re.search(r'charset=(.*)', content_type).group(1)
-                except AttributeError:
-                    charset = 'utf-8'
-                response_json = content.read().decode(charset).strip()
-                if not response_json:
-                    self.logger.debug('JSON response from %s was blank', url)
-                    return {}
-                try:
-                    response = json.loads(response_json)
-                except json.decoder.JSONDecodeError as exc:
-                    self.logger.error('Error loading JSON: %s', exc)
-                    self.logger.debug('JSON text that failed to load: %s',
-                                      response_json)
-                    return {}
-                self.logger.log(5, 'API response: %s', response)
-                error = self.check_response(response)
-                if error:
-                    self.logger.error('Error in JSON response: %s', error)
-                    return {}
-                return response
+            response_json = self.http_request(url, headers=headers).strip()
+            if not response_json:
+                self.logger.debug('JSON response from %s was blank', url)
+                return {}
+            try:
+                response = json.loads(response_json)
+            except json.decoder.JSONDecodeError as exc:
+                self.logger.error('Error loading JSON: %s', exc)
+                self.logger.debug('JSON text that failed to load: %s',
+                                  response_json)
+                return {}
+            self.logger.log(5, 'API response: %s', response)
+            error = self.check_response(response)
+            if error:
+                self.logger.error('Error in JSON response: %s', error)
+                return {}
+            return response
         except Exception as exc:
             self.logger.error(
                 'Failed to make API request to %s. Exception follows:', url,
@@ -45,8 +50,8 @@ def api_request(self, url):
             )
             return {}
 
-    def check_response(response):
-        raise NotImplementedError
+    def check_response(self, response):
+        return False
 
 
 class Weather(IntervalModule):
@@ -99,21 +104,6 @@ class Weather(IntervalModule):
     syntax to conditionally show the value of the **update_error** config value
     when the backend encounters an error during an update.
 
-    The extended string format syntax also comes in handy for the
-    :py:mod:`weathercom <.weather.weathercom>` backend, which at a certain
-    point in the afternoon will have a blank ``{high_temp}`` value. Using the
-    following snippet in your format string will only display the high
-    temperature information if it is not blank:
-
-    ::
-
-        {current_temp}{temp_unit}[ Hi: {high_temp}] Lo: {low_temp}[ {update_error}]
-
-    Brackets are evaluated from the outside-in, so the fact that the only
-    formatter in the outer block (``{high_temp}``) is empty would keep the
-    inner block from being evaluated at all, and entire block would not be
-    displayed.
-
     See the following links for usage examples for the available weather
     backends:
 
diff --git a/i3pystatus/weather/weathercom.py b/i3pystatus/weather/weathercom.py
index 1e33abfb..566983e3 100644
--- a/i3pystatus/weather/weathercom.py
+++ b/i3pystatus/weather/weathercom.py
@@ -7,8 +7,6 @@
 from i3pystatus.core.util import internet, require
 from i3pystatus.weather import WeatherBackend
 
-USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0'
-
 
 class WeathercomHTMLParser(HTMLParser):
     '''
@@ -16,6 +14,7 @@ class WeathercomHTMLParser(HTMLParser):
     through some other source at runtime and added as <script> elements to the
     page source.
     '''
+    user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0'
 
     def __init__(self, logger):
         self.logger = logger
@@ -24,7 +23,7 @@ def __init__(self, logger):
     def get_weather_data(self, url):
         self.logger.debug('Making request to %s to retrieve weather data', url)
         self.weather_data = None
-        req = Request(url, headers={'User-Agent': USER_AGENT})
+        req = Request(url, headers={'User-Agent': self.user_agent})
         with urlopen(req) as content:
             try:
                 content_type = dict(content.getheaders())['Content-Type']
@@ -98,7 +97,8 @@ class Weathercom(WeatherBackend):
     parameter should be set to the location code from weather.com. To obtain
     this code, search for your location on weather.com, and when you go to the
     forecast page, the code you need will be everything after the last slash in
-    the URL (e.g. ``94107:4:US``).
+    the URL (e.g. ``94107:4:US``, or
+    ``8b7867695971473a260df2c5d49ff92dc9079dcb673c545f5f107f5c4ab30732``).
 
     .. _weather-usage-weathercom:
 
@@ -154,16 +154,11 @@ def init(self):
 
         # Setting the locale to en-AU returns units in metric. Leaving it blank
         # causes weather.com to return the default, which is imperial.
-        self.locale = 'en-AU' if self.units == 'metric' else ''
+        self.locale = 'en-CA' if self.units == 'metric' else ''
 
         self.forecast_url = self.url_template.format(**vars(self))
         self.parser = WeathercomHTMLParser(self.logger)
 
-    def check_response(self, response):
-        # Errors for weather.com API manifest in HTTP error codes, not in the
-        # JSON response.
-        return False
-
     @require(internet)
     def check_weather(self):
         '''
@@ -211,7 +206,7 @@ def check_weather(self):
                 return
 
             try:
-                forecast = self.parser.weather_data['getSunV3DailyForecastUrlConfig']
+                forecast = self.parser.weather_data['getSunV3CurrentObservationsUrlConfig']
                 # Same as above, use next(iter(forecast)) to drill down to the
                 # correct nested dict level.
                 forecast = forecast[next(iter(forecast))]['data']
@@ -255,14 +250,13 @@ def check_weather(self):
             else:
                 pressure_trend = ''
 
-            self.logger.critical('forecast = %s', forecast)
             try:
-                high_temp = forecast.get('temperatureMax', [])[0] or ''
+                high_temp = forecast.get('temperatureMax24Hour', '')
             except (AttributeError, IndexError):
                 high_temp = ''
 
             try:
-                low_temp = forecast.get('temperatureMin', [])[0]
+                low_temp = forecast.get('temperatureMin24Hour', '')
             except (AttributeError, IndexError):
                 low_temp = ''
 
diff --git a/i3pystatus/weather/wunderground.py b/i3pystatus/weather/wunderground.py
index 7a26c169..0250f585 100644
--- a/i3pystatus/weather/wunderground.py
+++ b/i3pystatus/weather/wunderground.py
@@ -1,55 +1,36 @@
-from i3pystatus.core.util import internet, require
-from i3pystatus.weather import WeatherBackend
-
+import re
 from datetime import datetime
-from urllib.request import urlopen
+from urllib.request import Request, urlopen
 
-GEOLOOKUP_URL = 'http://api.wunderground.com/api/%s/geolookup%s/q/%s.json'
-STATION_QUERY_URL = 'http://api.wunderground.com/api/%s/%s/q/%s.json'
+from i3pystatus.core.util import internet, require
+from i3pystatus.weather import WeatherBackend
 
 
 class Wunderground(WeatherBackend):
     '''
-    This module retrieves weather data using the Weather Underground API.
+    This module retrieves weather data from Weather Underground.
 
     .. note::
-        A Weather Underground API key is required to use this module, you can
-        sign up for a developer API key free at
-        https://www.wunderground.com/weather/api/
-
-        Valid values for ``location_code`` include:
-
-        * **State/City_Name** - CA/San_Francisco
-        * **Country/City** - France/Paris
-        * **Geolocation by IP** - autoip
-        * **Zip or Postal Code** - 60616
-        * **ICAO Airport Code** - icao:LAX
-        * **Latitude/Longitude** - 41.8301943,-87.6342619
-        * **Personal Weather Station (PWS)** - pws:KILCHICA30
+        Previous versions of this module required an API key to work. Weather
+        Underground has since discontinued their API, and this module has been
+        rewritten to reflect that.
 
-        When not using a ``pws`` or ``icao`` station ID, the location will be
-        queried (this uses an API query), and the closest station will be used.
-        For a list of PWS station IDs, visit the following URL:
+    .. rubric:: Finding your weather station
 
-        http://www.wunderground.com/weatherstation/ListStations.asp
+    To use this module, you must provide a weather station code (as the
+    ``location_code`` option). To find your weather station, first search for
+    your city and click to view the current conditions. Below the city name you
+    will see the station name, and to the right of that a ``CHANGE`` link.
+    Clicking that link will display a map, where you can find the station
+    closest to you. Clicking on that station will take you back to the current
+    conditions page. The weather station code will now be the last part of the
+    URL. For example:
 
-        .. rubric:: API usage
+    .. code-block:: text
 
-        An API key is allowed 500 queries per day, and no more than 10 in a
-        given minute. Therefore, it is recommended to be conservative when
-        setting the update interval (the default is 1800 seconds, or 30
-        minutes), and one should be careful how often one restarts i3pystatus
-        and how often a refresh is forced by left-clicking the module.
+        https://www.wunderground.com/weather/us/ma/cambridge/KMACAMBR4
 
-        As noted above, when not using a ``pws`` or ``icao`` station ID, an API
-        query will be used to determine the station ID to use. This will be
-        done once when i3pystatus is started, and not repeated until the next
-        time i3pystatus is started.
-
-        When updating weather data, one API query will be used to obtain the
-        current conditions. The high/low temperature forecast requires an
-        additonal API query, and is optional (disabled by default). To enable
-        forecast checking, set ``forecast=True``.
+    In this case, the weather station code would be ``KMACAMBR4``.
 
     .. _weather-usage-wunderground:
 
@@ -68,10 +49,8 @@ class Wunderground(WeatherBackend):
             colorize=True,
             hints={'markup': 'pango'},
             backend=wunderground.Wunderground(
-                api_key='api_key_goes_here',
-                location_code='pws:MAT645',
+                location_code='KMACAMBR4',
                 units='imperial',
-                forecast=True,
                 update_error='<span color="#ff0000">!</span>',
             ),
         )
@@ -82,204 +61,170 @@ class Wunderground(WeatherBackend):
     used.
     '''
     settings = (
-        ('api_key', 'Weather Underground API key'),
         ('location_code', 'Location code from wunderground.com'),
         ('units', '\'metric\' or \'imperial\''),
-        ('use_pws', 'Set to False to use only airport stations'),
-        ('forecast', 'Set to ``True`` to check forecast (generates one '
-                     'additional API request per weather update). If set to '
-                     '``False``, then the ``low_temp`` and ``high_temp`` '
-                     'formatters will be set to empty strings.'),
         ('update_error', 'Value for the ``{update_error}`` formatter when an '
                          'error is encountered while checking weather data'),
     )
 
-    required = ('api_key', 'location_code')
+    required = ('location_code',)
 
-    api_key = None
     location_code = None
     units = 'metric'
-    use_pws = True
-    forecast = False
     update_error = '!'
 
-    # These will be set once weather data has been checked
-    station_id = None
+    url_template = 'https://www.wunderground.com/dashboard/pws/{location_code}'
+
+    # This will be set in the init based on the passed location code
     forecast_url = None
 
-    def init(self):
-        '''
-        Use the location_code to perform a geolookup and find the closest
-        station. If the location is a pws or icao station ID, no lookup will be
-        peformed.
-        '''
-        try:
-            for no_lookup in ('pws', 'icao'):
-                sid = self.location_code.partition(no_lookup + ':')[-1]
-                if sid:
-                    self.station_id = self.location_code
-                    return
-        except AttributeError:
-            # Numeric or some other type, either way we'll just stringify
-            # it below and perform a lookup.
-            pass
-
-        self.get_station_id()
+    summary_url = 'https://api.weather.com/v2/pws/dailysummary/1day?apiKey={api_key}&stationId={location_code}&format=json&units={units_type}'
+    observation_url = 'https://api.weather.com/v2/pws/observations/current?apiKey={api_key}&stationId={location_code}&format=json&units={units_type}'
+    overview_url = 'https://api.weather.com/v3/aggcommon/v3alertsHeadlines;v3-wx-observations-current;v3-location-point?apiKey={api_key}&geocodes={lat:.2f}%2C{lon:.2f}&language=en-US&units=e&format=json'
 
-    @require(internet)
-    def get_forecast(self):
-        '''
-        If configured to do so, make an API request to retrieve the forecast
-        data for the configured/queried weather station, and return the low and
-        high temperatures. Otherwise, return two empty strings.
-        '''
-        no_data = ('', '')
-        if self.forecast:
-            query_url = STATION_QUERY_URL % (self.api_key,
-                                             'forecast',
-                                             self.station_id)
-            try:
-                response = self.api_request(query_url)['forecast']
-                response = response['simpleforecast']['forecastday'][0]
-            except (KeyError, IndexError, TypeError):
-                self.logger.error(
-                    'No forecast data found for %s', self.station_id)
-                self.data['update_error'] = self.update_error
-                return no_data
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0',
+        'Referer': 'https://www.wunderground.com/dashboard/pws/{location_code}',
+        'Accept': 'application/json, text/plain, */*',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Connection': 'keep-alive',
+    }
 
-            unit = 'celsius' if self.units == 'metric' else 'fahrenheit'
-            low_temp = response.get('low', {}).get(unit, '')
-            high_temp = response.get('high', {}).get(unit, '')
-            return low_temp, high_temp
-        else:
-            return no_data
+    def init(self):
+        self.units_type = 'm' if self.units == 'metric' else 'e'
+        self.forecast_url = self.url_template.format(**vars(self))
 
     @require(internet)
-    def get_station_id(self):
+    def get_api_key(self):
         '''
-        Use geolocation to get the station ID
+        Grab the API key out of the page source from the home page
         '''
-        extra_opts = '/pws:0' if not self.use_pws else ''
-        api_url = GEOLOOKUP_URL % (self.api_key,
-                                   extra_opts,
-                                   self.location_code)
-        response = self.api_request(api_url)
-        station_type = 'pws' if self.use_pws else 'airport'
+        url = 'https://www.wunderground.com'
         try:
-            stations = response['location']['nearby_weather_stations']
-            nearest = stations[station_type]['station'][0]
-        except (KeyError, IndexError):
-            raise Exception(
-                'No locations matched location_code %s' % self.location_code)
-
-        self.logger.error('nearest = %s', nearest)
-        if self.use_pws:
-            nearest_pws = nearest.get('id', '')
-            if not nearest_pws:
-                raise Exception('No id entry for nearest PWS')
-            self.station_id = 'pws:%s' % nearest_pws
+            page_source = self.http_request(
+                url,
+                headers={
+                    'User-Agent': self.headers['User-Agent'],
+                    'Accept-Language': self.headers['Accept-Language'],
+                    'Conncetion': self.headers['Connection'],
+                },
+            )
+        except Exception as exc:
+            self.logger.exception('Failed to load %s', url)
         else:
-            nearest_airport = nearest.get('icao', '')
-            if not nearest_airport:
-                raise Exception('No icao entry for nearest airport')
-            self.station_id = 'icao:%s' % nearest_airport
+            try:
+                return re.search(r'apiKey=([0-9a-f]+)', page_source).group(1)
+            except AttributeError:
+                self.logger.error('Failed to find API key in mainpage source')
 
-    def check_response(self, response):
-        try:
-            return response['response']['error']['description']
-        except KeyError:
-            # No error in response
-            return False
+    @require(internet)
+    def api_request(self, url, headers=None):
+        if headers is None:
+            headers = {}
+        return super(Wunderground, self).api_request(
+            url,
+            headers=dict([(k, v.format(**vars(self))) for k, v in headers.items()]))
 
     @require(internet)
     def check_weather(self):
         '''
-        Query the configured/queried station and return the weather data
+        Query the desired station and return the weather data
         '''
-        if self.station_id is None:
-            # Failed to get the nearest station ID when first launched, so
-            # retry it.
-            self.get_station_id()
+        # Get the API key from the page source
+        self.api_key = self.get_api_key()
+        if self.api_key is None:
+            self.data['update_error'] = self.update_error
+            return
 
         self.data['update_error'] = ''
         try:
-            query_url = STATION_QUERY_URL % (self.api_key,
-                                             'conditions',
-                                             self.station_id)
             try:
-                response = self.api_request(query_url)['current_observation']
-                self.forecast_url = response.pop('ob_url', None)
-            except KeyError:
-                self.logger.error('No weather data found for %s', self.station_id)
+                summary = self.api_request(self.summary_url.format(**vars(self)))['summaries'][0]
+            except (IndexError, KeyError):
+                self.logger.error(
+                    'Failed to retrieve summary data from API response. '
+                    'Run module with debug logging to get more information.'
+                )
                 self.data['update_error'] = self.update_error
                 return
 
-            if self.forecast:
-                query_url = STATION_QUERY_URL % (self.api_key,
-                                                 'forecast',
-                                                 self.station_id)
-                try:
-                    forecast = self.api_request(query_url)['forecast']
-                    forecast = forecast['simpleforecast']['forecastday'][0]
-                except (KeyError, IndexError, TypeError):
-                    self.logger.error(
-                        'No forecast data found for %s', self.station_id)
-                    # This is a non-fatal error, so don't return but do set the
-                    # error flag.
-                    self.data['update_error'] = self.update_error
-
-                unit = 'celsius' if self.units == 'metric' else 'fahrenheit'
-                low_temp = forecast.get('low', {}).get(unit, '')
-                high_temp = forecast.get('high', {}).get(unit, '')
-            else:
-                low_temp = high_temp = ''
+            try:
+                observation = self.api_request(self.observation_url.format(**vars(self)))['observations'][0]
+            except (IndexError, KeyError):
+                self.logger.error(
+                    'Failed to retrieve observation data from API response. '
+                    'Run module with debug logging to get more information.'
+                )
+                self.data['update_error'] = self.update_error
+                return
+
+            self.lat = observation['lat']
+            self.lon = observation['lon']
+
+            try:
+                overview = self.api_request(self.overview_url.format(**vars(self)))[0]
+            except IndexError:
+                self.logger.error(
+                    'Failed to retrieve overview data from API response. '
+                    'Run module with debug logging to get more information.'
+                )
+                self.data['update_error'] = self.update_error
+                return
 
             if self.units == 'metric':
-                temp_unit = 'c'
+                temp_unit = '°C'
                 speed_unit = 'kph'
                 distance_unit = 'km'
                 pressure_unit = 'mb'
             else:
-                temp_unit = 'f'
+                temp_unit = '°F'
                 speed_unit = 'mph'
                 distance_unit = 'mi'
                 pressure_unit = 'in'
 
-            def _find(key, data=None, default=''):
-                if data is None:
-                    data = response
-                return str(data.get(key, default))
-
             try:
-                observation_epoch = _find('observation_epoch') or _find('local_epoch')
-                observation_time = datetime.fromtimestamp(int(observation_epoch))
-            except (TypeError, ValueError):
-                log.debug(
-                    'Observation time \'%s\' is not a UNIX timestamp',
-                    observation_epoch
-                )
+                observation_time_str = observation.get('obsTimeLocal', '')
+                observation_time = datetime.strptime(observation_time_str,
+                                                     '%Y-%m-%d %H:%M:%S')
+            except (ValueError, AttributeError):
                 observation_time = datetime.fromtimestamp(0)
 
-            self.data['city'] = _find('city', response['observation_location'])
-            self.data['condition'] = _find('weather')
+            def _find(path, data, default=''):
+                ptr = data
+                try:
+                    for item in path.split(':'):
+                        if item == 'units':
+                            item = self.units
+                        ptr = ptr[item]
+                except (KeyError, IndexError, TypeError):
+                    return default
+                return str(ptr)
+
+            pressure_tendency = _find(
+                'v3-wx-observations-current:pressureTendencyTrend',
+                overview).lower()
+            pressure_trend = '+' if pressure_tendency == 'rising' else '-'
+
+            self.data['city'] = _find('v3-location-point:location:city', overview)
+            self.data['condition'] = _find('v3-wx-observations-current:wxPhraseShort', overview)
             self.data['observation_time'] = observation_time
-            self.data['current_temp'] = _find('temp_' + temp_unit).split('.')[0]
-            self.data['low_temp'] = low_temp
-            self.data['high_temp'] = high_temp
-            self.data['temp_unit'] = '°' + temp_unit.upper()
-            self.data['feelslike'] = _find('feelslike_' + temp_unit)
-            self.data['dewpoint'] = _find('dewpoint_' + temp_unit)
-            self.data['wind_speed'] = _find('wind_' + speed_unit)
+            self.data['current_temp'] = _find('units:temp', observation, '0')
+            self.data['low_temp'] = _find('units:tempLow', summary)
+            self.data['high_temp'] = _find('units:tempHigh', summary)
+            self.data['temp_unit'] = temp_unit
+            self.data['feelslike'] = _find('units:heatIndex', observation)
+            self.data['dewpoint'] = _find('units:dewpt', observation)
+            self.data['wind_speed'] = _find('units:windSpeed', observation)
             self.data['wind_unit'] = speed_unit
-            self.data['wind_direction'] = _find('wind_dir')
-            self.data['wind_gust'] = _find('wind_gust_' + speed_unit)
-            self.data['pressure'] = _find('pressure_' + pressure_unit)
+            self.data['wind_direction'] = _find('v3-wx-observations-current:windDirectionCardinal', overview)
+            self.data['wind_gust'] = _find('units:windGust', observation)
+            self.data['pressure'] = _find('units:pressure', observation)
             self.data['pressure_unit'] = pressure_unit
-            self.data['pressure_trend'] = _find('pressure_trend')
-            self.data['visibility'] = _find('visibility_' + distance_unit)
+            self.data['pressure_trend'] = pressure_trend
+            self.data['visibility'] = _find('v3-wx-observations-current:visibility', overview)
             self.data['visibility_unit'] = distance_unit
-            self.data['humidity'] = _find('relative_humidity').rstrip('%')
-            self.data['uv_index'] = _find('UV')
+            self.data['humidity'] = _find('humidity', observation)
+            self.data['uv_index'] = _find('uv', observation)
         except Exception:
             # Don't let an uncaught exception kill the update thread
             self.logger.error(