diff --git a/.testenv b/.testenv index f60567fe7..b79554636 100644 --- a/.testenv +++ b/.testenv @@ -28,7 +28,7 @@ FLASK_RECAPTCHA_PRIVATE_KEY=6LeIxAcTAAAAAGG-vFI1TnRWxMZNFuojJ4WifJWe FLASK_RECAPTCHA_OPTIONS="" # Use hostaliases on supported platforms HOSTALIASES=${PWD}/HOSTALIASES -# These settings should be customisable from a .env file (TODO) +# These settings can be customised in .env.testing FLASK_SECRET_KEYS='["testkey"]' FLASK_LASTUSER_SECRET_KEYS='["testkey"]' FLASK_LASTUSER_COOKIE_DOMAIN='.funnel.test:3002' @@ -45,6 +45,11 @@ FLASK_IMGEE_HOST='http://imgee.test:4500' FLASK_IMAGE_URL_DOMAINS='["images.example.com"]' FLASK_IMAGE_URL_SCHEMES='["https"]' FLASK_SES_NOTIFICATION_TOPICS=null +# WhatsApp config (These null entries disable production entries in .env) +FLASK_WHATSAPP_PHONE_ID_META=null +FLASK_WHATSAPP_TOKEN_META=null +FLASK_WHATSAPP_PHONE_ID_HOSTED=null +FLASK_WHATSAPP_TOKEN_HOSTED=null # Per app config APP_FUNNEL_SITE_ID=hasgeek-test APP_FUNNEL_SERVER_NAME=funnel.test:3002 diff --git a/funnel/models/account.py b/funnel/models/account.py index 4c7e40466..6ef2a3728 100644 --- a/funnel/models/account.py +++ b/funnel/models/account.py @@ -1515,7 +1515,7 @@ class AccountEmail(EmailAddressMixin, BaseMixin, Model): 'related': {'email', 'private', 'type'}, } - def __init__(self, account: Account, **kwargs) -> None: + def __init__(self, *, account: Account, **kwargs) -> None: email = kwargs.pop('email', None) if email: kwargs['email_address'] = EmailAddress.add_for(account, email) @@ -1701,13 +1701,11 @@ class AccountEmailClaim(EmailAddressMixin, BaseMixin, Model): 'related': {'email', 'private', 'type'}, } - def __init__(self, account: Account, **kwargs) -> None: - email = kwargs.pop('email', None) - if email: - kwargs['email_address'] = EmailAddress.add_for(account, email) + def __init__(self, account: Account, email: str, **kwargs) -> None: + kwargs['email_address'] = EmailAddress.add_for(account, email) super().__init__(account=account, **kwargs) - self.blake2b = hashlib.blake2b( - self.email.lower().encode(), digest_size=16 + self.blake2b = hashlib.blake2b( # self.email is not optional, so this ignore: + self.email.lower().encode(), digest_size=16 # type: ignore[union-attr] ).digest() def __repr__(self) -> str: @@ -1887,7 +1885,7 @@ class AccountPhone(PhoneNumberMixin, BaseMixin, Model): 'related': {'phone', 'private', 'type'}, } - def __init__(self, account, **kwargs) -> None: + def __init__(self, *, account: Account, **kwargs) -> None: phone = kwargs.pop('phone', None) if phone: kwargs['phone_number'] = PhoneNumber.add_for(account, phone) @@ -1902,7 +1900,7 @@ def __str__(self) -> str: return self.phone or '' @cached_property - def parsed(self) -> phonenumbers.PhoneNumber: + def parsed(self) -> phonenumbers.PhoneNumber | None: """Return parsed phone number using libphonenumbers.""" return self.phone_number.parsed diff --git a/funnel/models/notification.py b/funnel/models/notification.py index 629c99e04..62936cb85 100644 --- a/funnel/models/notification.py +++ b/funnel/models/notification.py @@ -1427,7 +1427,7 @@ def main_notification_preferences(self) -> NotificationPreferences: by_sms=True, by_webpush=False, by_telegram=False, - by_whatsapp=False, + by_whatsapp=True, ) db.session.add(main) return main diff --git a/funnel/models/phone_number.py b/funnel/models/phone_number.py index 72d002a86..02b8135a4 100644 --- a/funnel/models/phone_number.py +++ b/funnel/models/phone_number.py @@ -267,7 +267,7 @@ class PhoneNumber(BaseMixin, Model): #: BLAKE2b 160-bit hash of :attr:`phone`. Kept permanently even if phone is #: removed. SQLAlchemy type LargeBinary maps to PostgreSQL BYTEA. Despite the name, #: we're only storing 20 bytes - blake2b160 = immutable( + blake2b160: Mapped[bytes] = immutable( sa.orm.mapped_column( sa.LargeBinary, sa.CheckConstraint( diff --git a/funnel/transports/__init__.py b/funnel/transports/__init__.py index 7fbcf41f9..540311cc1 100644 --- a/funnel/transports/__init__.py +++ b/funnel/transports/__init__.py @@ -1,7 +1,5 @@ """Transport layer for communication with users (email, SMS, others).""" -from __future__ import annotations - from . import email, sms, telegram, webpush, whatsapp from .base import init, platform_transports from .exc import ( diff --git a/funnel/transports/base.py b/funnel/transports/base.py index 22eb85d2a..8ab23aee8 100644 --- a/funnel/transports/base.py +++ b/funnel/transports/base.py @@ -17,10 +17,12 @@ } -def init(): +def init() -> None: if app.config.get('MAIL_SERVER'): platform_transports['email'] = True if sms_init(): platform_transports['sms'] = True + if app.config.get('WHATSAPP_TOKEN') and app.config.get('WHATSAPP_PHONE_ID'): + platform_transports['whatsapp'] = True # Other transports are not supported yet diff --git a/funnel/transports/exc.py b/funnel/transports/exc.py index 007687790..2ab878b6f 100644 --- a/funnel/transports/exc.py +++ b/funnel/transports/exc.py @@ -1,9 +1,6 @@ """Transport exceptions.""" -from __future__ import annotations - - class TransportError(Exception): """Base class for transport exceptions.""" diff --git a/funnel/transports/sms/__init__.py b/funnel/transports/sms/__init__.py index 069fa32f8..b6850bd3f 100644 --- a/funnel/transports/sms/__init__.py +++ b/funnel/transports/sms/__init__.py @@ -1,7 +1,5 @@ """SMS transport support.""" # flake8: noqa -from __future__ import annotations - from .send import * from .template import * diff --git a/funnel/transports/whatsapp.py b/funnel/transports/whatsapp.py deleted file mode 100644 index 03f963f21..000000000 --- a/funnel/transports/whatsapp.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Support functions for sending a WhatsApp message. Forthcoming.""" - -from __future__ import annotations diff --git a/funnel/transports/whatsapp/__init__.py b/funnel/transports/whatsapp/__init__.py new file mode 100644 index 000000000..489b6257e --- /dev/null +++ b/funnel/transports/whatsapp/__init__.py @@ -0,0 +1,5 @@ +"""WhatsApp transport support.""" +# flake8: noqa + +from .send import * +from .template import * diff --git a/funnel/transports/whatsapp/send.py b/funnel/transports/whatsapp/send.py new file mode 100644 index 000000000..64a4f524e --- /dev/null +++ b/funnel/transports/whatsapp/send.py @@ -0,0 +1,161 @@ +"""Support functions for sending an WhatsApp messages.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from dataclasses import dataclass +from typing import cast + +import phonenumbers +import requests + +from baseframe import _ + +from ... import app +from ...models import PhoneNumber, PhoneNumberBlockedError, sa +from ..exc import ( + TransportConnectionError, + TransportRecipientError, + TransportTransactionError, +) +from .template import WhatsappTemplate + + +@dataclass +class WhatsappSender: + """A WhatsApp sender.""" + + requires_config: set[str] + func: Callable[[str, WhatsappTemplate], str] + init: Callable | None = None + + +def get_phone_number( + phone: str | phonenumbers.PhoneNumber | PhoneNumber, +) -> PhoneNumber: + if isinstance(phone, PhoneNumber): + if not phone.number: + raise TransportRecipientError(_("This phone number is not available")) + # TODO: Confirm this phone number is available on WhatsApp + return phone + try: + phone_number = PhoneNumber.add(phone) + except PhoneNumberBlockedError as exc: + raise TransportRecipientError(_("This phone number has been blocked")) from exc + if not phone_number.number: + # This should never happen as :meth:`PhoneNumber.add` will restore the number + raise TransportRecipientError(_("This phone number is not available")) + # TODO: Confirm this phone number is available on WhatsApp + return phone_number + + +def send_via_meta(phone: str, message: WhatsappTemplate) -> str: + """ + Send the WhatsApp message using Meta Cloud API. + + :param phone: Phone number + :param message: Message to deliver to phone number + :return: Transaction id + """ + phone_number = get_phone_number(phone) + sid = app.config['WHATSAPP_PHONE_ID'] + token = app.config['WHATSAPP_TOKEN'] + headers = {'Authorization': f'Bearer {token}', 'Content-Type': 'application/json'} + payload = { + 'messaging_product': 'whatsapp', + 'recipient_type': 'individual', + 'to': cast(str, phone_number.number).lstrip('+'), + 'type': 'template', + 'template': json.dumps(message.template), + } + try: + r = requests.post( + f'https://graph.facebook.com/v18.0/{sid}/messages', + timeout=30, + headers=headers, + data=payload, + ) + if r.status_code == 200: + jsonresponse = r.json() + transactionid = jsonresponse['messages'][0].get('id') + phone_number.msg_wa_sent_at = sa.func.utcnow() + return transactionid + raise TransportTransactionError(_("WhatsApp API error"), r.status_code, r.text) + except requests.ConnectionError as exc: + raise TransportConnectionError(_("WhatsApp not reachable")) from exc + + +def send_via_hosted(phone: str, message: WhatsappTemplate) -> str: + """ + Send the WhatsApp message using On-Premise API. + + :param phone: Phone number + :param message: Message to deliver to phone number + :return: Transaction id + """ + phone_number = get_phone_number(phone) + sid = app.config['WHATSAPP_PHONE_ID'] + token = app.config['WHATSAPP_TOKEN'] + payload = { + 'messaging_product': 'whatsapp', + 'recipient_type': 'individual', + 'to': cast(str, phone_number.number).lstrip('+'), + 'type': 'template', + 'body': str(message), + } + try: + r = requests.post( + f'https://graph.facebook.com/v18.0/{sid}/messages', + timeout=30, + auth=(token), # FIXME: This is not a valid auth parameter + data=payload, + ) + if r.status_code == 200: + jsonresponse = r.json() + transactionid = jsonresponse['messages'].get('id') + phone_number.msg_wa_sent_at = sa.func.utcnow() + + return transactionid + raise TransportTransactionError(_("WhatsApp API error"), r.status_code, r.text) + except requests.ConnectionError as exc: + raise TransportConnectionError(_("WhatsApp not reachable")) from exc + + +#: Supported senders (ordered by priority) +sender_registry = [ + WhatsappSender( + {'WHATSAPP_PHONE_ID_HOSTED', 'WHATSAPP_TOKEN_HOSTED'}, send_via_hosted + ), + WhatsappSender({'WHATSAPP_PHONE_ID_META', 'WHATSAPP_TOKEN_META'}, send_via_meta), +] + +senders: list[Callable[[str, WhatsappTemplate], str]] = [] + + +def init() -> bool: + """Process available senders.""" + for provider in sender_registry: + if all(app.config.get(var) for var in provider.requires_config): + senders.append(provider.func) + if provider.init: + provider.init() + return bool(senders) + + +def send_whatsapp( + phone: str | phonenumbers.PhoneNumber | PhoneNumber, + message: WhatsappTemplate, +) -> str: + """ + Send a WhatsApp message to a given phone number and return a transaction id. + + :param phone_number: Phone number + :param message: Message to deliver to phone number + :return: Transaction id + """ + phone_number = get_phone_number(phone) + phone = cast(str, phone_number.number) + for sender in senders: + return sender(phone, message) + raise TransportRecipientError(_("No service provider available for this recipient")) diff --git a/funnel/transports/whatsapp/template.py b/funnel/transports/whatsapp/template.py new file mode 100644 index 000000000..b7806d833 --- /dev/null +++ b/funnel/transports/whatsapp/template.py @@ -0,0 +1,47 @@ +"""WhatsApp template validator.""" + +from __future__ import annotations + + +class WhatsappTemplate: + """WhatsApp template formatter.""" + + registered_template_name: str + registered_template_language_code: str + registered_template: str + template: str + + +class OTPTemplate(WhatsappTemplate): + """OTP template formatter.""" + + registered_template_name = "otp2" + registered_template_language_code = "en" + # Registered template for reference + registered_template = """ + + OTP is *{{1}}* for Hasgeek. + + If you did not request this, report misuse at https://has.gy/not-my-otp +""" + template = { + 'name': registered_template_name, + 'language': { + 'code': registered_template_language_code, + }, + 'components': [ + { + 'type': 'body', + "parameters": [ + { + "type": "text", + }, + ], + } + ], + } + otp: str + + def __init__(self, otp: str = ''): + self.otp = otp + self.template['components'][0]['parameters'][0]['text'] = otp diff --git a/funnel/views/api/__init__.py b/funnel/views/api/__init__.py index ebadd6087..60df96480 100644 --- a/funnel/views/api/__init__.py +++ b/funnel/views/api/__init__.py @@ -12,4 +12,5 @@ shortlink, sms_events, support, + whatsapp_events, ) diff --git a/funnel/views/api/whatsapp_events.py b/funnel/views/api/whatsapp_events.py new file mode 100644 index 000000000..a0ccb2abd --- /dev/null +++ b/funnel/views/api/whatsapp_events.py @@ -0,0 +1,85 @@ +"""Callback handlers from WhatsApp.""" + +from __future__ import annotations + +from flask import abort, current_app, request + +from baseframe import statsd + +from ... import app +from ...models import PhoneNumber, PhoneNumberInvalidError, db, sa +from ...typing import ReturnView +from ...utils import abort_null + + +@app.route('/api/1/whatsapp/meta_event', methods=['GET']) +def process_whatsapp_webhook_verification() -> ReturnView: + """Meta verifies the webhook URL by sending a GET request with a token.""" + # FIXME: This looks half baked. What is the "challenge" being returned? + verify_token = app.config['WHATSAPP_WEBHOOK_SECRET'] + mode = request.args['hub.mode'] + token = request.args['hub.verify_token'] + challenge = request.args['hub.challenge'] + + if mode and token: + if mode == 'subscribe' and token == verify_token: + return challenge, 200 + return 'Forbidden', 403 + return 'Success', 200 + + +@app.route('/api/1/whatsapp/meta_event', methods=['POST']) +def process_whatsapp_event() -> ReturnView: + """Process WhatsApp callback event from Meta Cloud API.""" + # Register the fact that we got a WhatsApp event. + # If there are too many rejects, then most likely a hack attempt. + + # FIXME: Where is the call verification? + statsd.incr( + 'phone_number.event', tags={'engine': 'whatsapp-meta', 'stage': 'received'} + ) + if not request.json: + abort(400) + + # FIXME: Handle multiple events in a single call, as it clearly implied by `changes` + # and `statuses` being a list in this call + statuses = request.json['entry'][0]['changes'][0]['value']['statuses'][0] + whatsapp_to = abort_null(statuses['recepient_id']) + if not whatsapp_to: + return {'status': 'eror', 'error': 'invalid_phone'}, 422 + + try: + phone_number = PhoneNumber.add(phone=f'+{whatsapp_to}') + except PhoneNumberInvalidError: + current_app.logger.info( + "WhatsApp Meta Cloud API event for phone: %s invalid number for Whatsapp", + whatsapp_to, + ) + return {'status': 'error', 'error': 'invalid_phone'}, 422 + + msg_status = statuses['status'] + + if msg_status == 'sent': + phone_number.msg_wa_sent_at = sa.func.utcnow() + elif msg_status == 'delivered': + phone_number.msg_wa_delivered_at = sa.func.utcnow() + phone_number.mark_has_wa(True) + elif msg_status == 'failed': + phone_number.msg_wa_failed_at = sa.func.utcnow() + db.session.commit() + + current_app.logger.info( + "WhatsApp Meta Cloud API event for phone: %s %s", + whatsapp_to, + msg_status, + ) + + statsd.incr( + 'phone_number.event', + tags={ + 'engine': 'whatsapp-meta', + 'stage': 'processed', + 'event': msg_status, + }, + ) + return {'status': 'ok'}, 200 diff --git a/funnel/views/notification.py b/funnel/views/notification.py index 48dc4a651..91e9a6901 100644 --- a/funnel/views/notification.py +++ b/funnel/views/notification.py @@ -30,8 +30,7 @@ db, ) from ..serializers import token_serializer -from ..transports import TransportError, email, platform_transports, sms -from ..transports.sms import SmsTemplate +from ..transports import TransportError, email, platform_transports, sms, whatsapp from .helpers import make_cached_token from .jobs import rqjob @@ -385,7 +384,7 @@ def email_from(self) -> str: return f"{self.notification.preference_context.title} (via Hasgeek)" return "Hasgeek" - def sms(self) -> SmsTemplate: + def sms(self) -> sms.SmsTemplate: """ Render a short text message. Templates must use a single line with a link. @@ -397,7 +396,7 @@ def text(self) -> str: """Render a short plain text notification using the SMS template.""" return self.sms().text - def sms_with_unsubscribe(self) -> SmsTemplate: + def sms_with_unsubscribe(self) -> sms.SmsTemplate: """Add an unsubscribe link to the SMS message.""" msg = self.sms() msg.unsubscribe_url = self.unsubscribe_short_url('sms') @@ -419,13 +418,9 @@ def telegram(self) -> str: """ return self.text() - def whatsapp(self) -> str: - """ - Render a WhatsApp-formatted text message. - - Default implementation uses :meth:`text`. - """ - return self.text() + def whatsapp(self) -> whatsapp.WhatsappTemplate: + """Render a WhatsApp-formatted text message.""" + raise NotImplementedError("Subclasses must implement `whatsapp`") # --- Dispatch functions --------------------------------------------------------------- @@ -557,10 +552,10 @@ def dispatch_transport_email( # pylint: enable=consider-using-f-string ) ), - 'List-Help': f'<{url_for("notification_preferences")}>', + 'List-Help': f'<{url_for("notification_preferences", _external=True)}>', 'List-Unsubscribe': f'<{view.unsubscribe_url_email}>', 'List-Unsubscribe-Post': 'One-Click', - 'List-Archive': f'<{url_for("notifications")}>', + 'List-Archive': f'<{url_for("notifications", _external=True)}>', }, base_url=view.email_base_url, ) @@ -586,8 +581,13 @@ def dispatch_transport_sms( # the worker may be delayed and the user may have changed their preference. notification_recipient.messageid_sms = 'cancelled' return + try: + message = view.sms_with_unsubscribe() + except NotImplementedError: + notification_recipient.messageid_sms = 'not-implemented' + return notification_recipient.messageid_sms = sms.send_sms( - str(view.transport_for('sms')), view.sms_with_unsubscribe() + str(view.transport_for('sms')), message ) statsd.incr( 'notification.transport', @@ -598,8 +598,41 @@ def dispatch_transport_sms( ) +@rqjob() +@transport_worker_wrapper +def dispatch_transport_whatsapp( + notification_recipient: NotificationRecipient, view: RenderNotification +): + if not notification_recipient.user.main_notification_preferences.by_transport( + 'whatsapp' + ): + # Cancel delivery if user's main switch is off. This was already checked, but + # the worker may be delayed and the user may have changed their preference. + notification_recipient.messageid_whatsapp = 'cancelled' + return + try: + message = view.whatsapp() + except NotImplementedError: + notification_recipient.messageid_whatsapp = 'not-implemented' + return + notification_recipient.messageid_whatsapp = whatsapp.send_whatsapp( + str(view.transport_for('whatsapp')), message + ) + statsd.incr( + 'notification.transport', + tags={ + 'notification_type': notification_recipient.notification_type, + 'transport': 'whatsapp', + }, + ) + + # Add transport workers here as their worker methods are written -transport_workers = {'email': dispatch_transport_email, 'sms': dispatch_transport_sms} +transport_workers = { + 'email': dispatch_transport_email, + 'sms': dispatch_transport_sms, + 'whatsapp': dispatch_transport_whatsapp, +} # --- Notification background workers -------------------------------------------------- @@ -615,9 +648,7 @@ def dispatch_notification_job(eventid: UUID, notification_ids: Sequence[UUID]) - for notification in notifications: if notification is not None: generator = notification.dispatch() - # TODO: Use walrus operator := after we move off Python 3.7 - batch = tuple(islice(generator, DISPATCH_BATCH_SIZE)) - while batch: + while batch := tuple(islice(generator, DISPATCH_BATCH_SIZE)): db.session.commit() notification_recipient_ids = [ notification_recipient.identity for notification_recipient in batch @@ -629,7 +660,6 @@ def dispatch_notification_job(eventid: UUID, notification_ids: Sequence[UUID]) - tags={'notification_type': notification.type}, ) # Continue to the next batch - batch = tuple(islice(generator, DISPATCH_BATCH_SIZE)) @rqjob() @@ -637,6 +667,7 @@ def dispatch_notification_recipients_job( notification_recipient_ids: Sequence[tuple[int, UUID]] ) -> None: """Process notifications for users and enqueue transport delivery.""" + # TODO: Can this be a single query instead of a loop of queries? queue = [ NotificationRecipient.query.get(identity) for identity in notification_recipient_ids diff --git a/funnel/views/otp.py b/funnel/views/otp.py index cc9c7b711..cce89493a 100644 --- a/funnel/views/otp.py +++ b/funnel/views/otp.py @@ -34,6 +34,7 @@ TransportRecipientError, TransportTransactionError, sms, + whatsapp, ) from ..transports.email import jsonld_view_action, send_email from ..utils import blake2b160_hex, mask_email, mask_phone @@ -272,6 +273,30 @@ def send_sms( return msg return None + # Send whatsapp message + def send_whatsapp( + self, flash_success: bool = True, flash_failure: bool = True + ) -> whatsapp.WhatsappTemplate | None: + """Send an OTP via WhatsApp to a phone number.""" + if not self.phone: + return None + message = whatsapp.OTPTemplate(self.otp) + try: + whatsapp.send_whatsapp(self.phone, message) + except TransportRecipientError as exc: + if flash_failure: + flash(str(exc), 'error') + else: + raise + if flash_success: + flash( + _("An OTP has been sent to your phone number {number}").format( + number=self.display_phone + ), + 'success', + ) + return message + def send_email( self, flash_success: bool = True, flash_failure: bool = True ) -> str | None: diff --git a/sample.env b/sample.env index 159905cbc..c80fc1ccd 100644 --- a/sample.env +++ b/sample.env @@ -243,3 +243,11 @@ FLASK_SMS_DLT_TEMPLATE_IDS__update_template=null FLASK_SMS_DLT_TEMPLATE_IDS__comment_project_template=null FLASK_SMS_DLT_TEMPLATE_IDS__comment_proposal_template=null FLASK_SMS_DLT_TEMPLATE_IDS__comment_reply_template=null + +# --- WhatsApp integrations +# Meta Cloud API +FLASK_WHATSAPP_PHONE_ID_META=null +FLASK_WHATSAPP_TOKEN_META=null +# On-premise hosted +FLASK_WHATSAPP_PHONE_ID_HOSTED=null +FLASK_WHATSAPP_TOKEN_HOSTED=null