From 3e7a07cc848995ccc66c15b84f49f75a61fa8648 Mon Sep 17 00:00:00 2001 From: sergioisidoro Date: Thu, 2 Jan 2020 21:01:36 +0200 Subject: [PATCH 1/3] Refactor adaptors into their own module --- examples/get_ble_data.py | 2 +- ruuvitag_sensor/__init__.py | 2 +- ruuvitag_sensor/adaptors/__init__.py | 16 ++++ ruuvitag_sensor/adaptors/dummy.py | 18 +++++ .../nix_hci.py} | 73 ++++++++----------- ruuvitag_sensor/ruuvi.py | 4 +- tests/test_data_formats.py | 1 - tests/test_decoder.py | 1 - tests/test_ruuvitag_sensor.py | 22 +++--- 9 files changed, 78 insertions(+), 61 deletions(-) create mode 100644 ruuvitag_sensor/adaptors/__init__.py create mode 100644 ruuvitag_sensor/adaptors/dummy.py rename ruuvitag_sensor/{ble_communication.py => adaptors/nix_hci.py} (63%) diff --git a/examples/get_ble_data.py b/examples/get_ble_data.py index 24ade25..aafb1eb 100644 --- a/examples/get_ble_data.py +++ b/examples/get_ble_data.py @@ -2,7 +2,7 @@ Get all BLE device broadcasts """ -from ruuvitag_sensor.ble_communication import BleCommunicationNix +from ruuvitag_sensor.adaptors.nix_hci import BleCommunicationNix import ruuvitag_sensor.log ruuvitag_sensor.log.enable_console() diff --git a/ruuvitag_sensor/__init__.py b/ruuvitag_sensor/__init__.py index 2d7893e..24a20a5 100644 --- a/ruuvitag_sensor/__init__.py +++ b/ruuvitag_sensor/__init__.py @@ -1 +1 @@ -__version__ = '0.13.0' +__version__ = '0.13.0' \ No newline at end of file diff --git a/ruuvitag_sensor/adaptors/__init__.py b/ruuvitag_sensor/adaptors/__init__.py new file mode 100644 index 0000000..f122e92 --- /dev/null +++ b/ruuvitag_sensor/adaptors/__init__.py @@ -0,0 +1,16 @@ + +import abc + +class BleCommunication(object): + """Bluetooth LE communication""" + __metaclass__ = abc.ABCMeta + + @staticmethod + @abc.abstractmethod + def get_data(mac, bt_device=''): + pass + + @staticmethod + @abc.abstractmethod + def get_datas(blacklist=[], bt_device=''): + pass \ No newline at end of file diff --git a/ruuvitag_sensor/adaptors/dummy.py b/ruuvitag_sensor/adaptors/dummy.py new file mode 100644 index 0000000..c0cc473 --- /dev/null +++ b/ruuvitag_sensor/adaptors/dummy.py @@ -0,0 +1,18 @@ +from ruuvitag_sensor.adaptors import BleCommunication + +class BleCommunicationDummy(BleCommunication): + """TODO: Find some working BLE implementation for Windows and OSX""" + + @staticmethod + def get_data(mac, bt_device=''): + return '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD' + + @staticmethod + def get_datas(blacklist=[], bt_device=''): + datas = [ + ('DU:MM:YD:AT:A9:3D', '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD'), + ('NO:TS:UP:PO:RT:ED', '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD') + ] + + for data in datas: + yield data diff --git a/ruuvitag_sensor/ble_communication.py b/ruuvitag_sensor/adaptors/nix_hci.py similarity index 63% rename from ruuvitag_sensor/ble_communication.py rename to ruuvitag_sensor/adaptors/nix_hci.py index bd5f9be..28aa1db 100644 --- a/ruuvitag_sensor/ble_communication.py +++ b/ruuvitag_sensor/adaptors/nix_hci.py @@ -1,45 +1,12 @@ -import abc import logging import os import subprocess import sys import time -log = logging.getLogger(__name__) - - -class BleCommunication(object): - """Bluetooth LE communication""" - __metaclass__ = abc.ABCMeta - - @staticmethod - @abc.abstractmethod - def get_data(mac, bt_device=''): - pass - - @staticmethod - @abc.abstractmethod - def get_datas(blacklist=[], bt_device=''): - pass - - -class BleCommunicationDummy(BleCommunication): - """TODO: Find some working BLE implementation for Windows and OSX""" - - @staticmethod - def get_data(mac, bt_device=''): - return '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD' - - @staticmethod - def get_datas(blacklist=[], bt_device=''): - datas = [ - ('DU:MM:YD:AT:A9:3D', '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD'), - ('NO:TS:UP:PO:RT:ED', '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD') - ] - - for data in datas: - yield data +from ruuvitag_sensor.adaptors import BleCommunication +log = logging.getLogger(__name__) class BleCommunicationNix(BleCommunication): """Bluetooth LE communication for Linux""" @@ -50,33 +17,53 @@ def start(bt_device=''): Attributes: device (string): BLE device (default hci0) """ - # import ptyprocess here so as long as all implementations are in the same file, all will work + # import ptyprocess here so as long as all implementations are in + # the same file, all will work import ptyprocess if not bt_device: bt_device = 'hci0' log.info('Start receiving broadcasts (device %s)', bt_device) - DEVNULL = subprocess.DEVNULL if sys.version_info >= (3, 3) else open(os.devnull, 'wb') + if sys.version_info >= (3, 3): + DEVNULL = subprocess.DEVNULL + else: + open(os.devnull, 'wb') def reset_ble_adapter(): - return subprocess.call('sudo hciconfig %s reset' % bt_device, shell=True, stdout=DEVNULL) + log.info("FYI: Calling a process with sudo!") + return subprocess.call( + 'sudo hciconfig %s reset' % bt_device, + shell=True, + stdout=DEVNULL + ) def start_with_retry(func, try_count, interval, msg): retcode = func() if retcode != 0 and try_count > 0: log.info(msg) time.sleep(interval) - return start_with_retry(func, try_count - 1, interval + interval, msg) + return start_with_retry( + func, try_count - 1, interval + interval, msg) return retcode - retcode = start_with_retry(reset_ble_adapter, 3, 1, 'Problem with hciconfig reset. Retry reset.') + retcode = start_with_retry( + reset_ble_adapter, + 3, 1, + 'Problem with hciconfig reset. Retry reset.' + ) + if retcode != 0: log.info('Problem with hciconfig reset. Exit.') exit(1) - hcitool = ptyprocess.PtyProcess.spawn(['sudo', '-n', 'hcitool', '-i', bt_device, 'lescan2', '--duplicates']) - hcidump = ptyprocess.PtyProcess.spawn(['sudo', '-n', 'hcidump', '-i', bt_device, '--raw']) + log.info("FYI: Spawning 2 processes with sudo!") + hcitool = ptyprocess.PtyProcess.spawn( + ['sudo', '-n', 'hcitool', '-i', bt_device, 'lescan2', '--duplicates'] + ) + hcidump = ptyprocess.PtyProcess.spawn( + ['sudo', '-n', 'hcidump', '-i', bt_device, '--raw'] + ) return (hcitool, hcidump) @staticmethod @@ -99,7 +86,7 @@ def get_lines(hcidump): else: if data: data += line.strip().replace(' ', '') - except KeyboardInterrupt as ex: + except KeyboardInterrupt: return except Exception as ex: log.info(ex) diff --git a/ruuvitag_sensor/ruuvi.py b/ruuvitag_sensor/ruuvi.py index 94cd98e..f217006 100644 --- a/ruuvitag_sensor/ruuvi.py +++ b/ruuvitag_sensor/ruuvi.py @@ -11,10 +11,10 @@ if not sys.platform.startswith('linux') or os.environ.get('CI') == 'True': # Use BleCommunicationDummy also for CI as it can't use bluez - from ruuvitag_sensor.ble_communication import BleCommunicationDummy + from ruuvitag_sensor.adaptors.dummy import BleCommunicationDummy ble = BleCommunicationDummy() else: - from ruuvitag_sensor.ble_communication import BleCommunicationNix + from ruuvitag_sensor.adaptors.nix_hci import BleCommunicationNix ble = BleCommunicationNix() diff --git a/tests/test_data_formats.py b/tests/test_data_formats.py index 4a5a7c3..7ca7fbb 100644 --- a/tests/test_data_formats.py +++ b/tests/test_data_formats.py @@ -2,7 +2,6 @@ from ruuvitag_sensor.data_formats import DataFormats - # pylint: disable=W0613 class TestDataFormats(TestCase): diff --git a/tests/test_decoder.py b/tests/test_decoder.py index 3827800..0b68497 100644 --- a/tests/test_decoder.py +++ b/tests/test_decoder.py @@ -2,7 +2,6 @@ from ruuvitag_sensor.decoder import get_decoder, UrlDecoder, Df3Decoder, Df5Decoder - class TestDecoder(TestCase): def test_getcorrectdecoder(self): diff --git a/tests/test_ruuvitag_sensor.py b/tests/test_ruuvitag_sensor.py index 4164f29..3788d73 100644 --- a/tests/test_ruuvitag_sensor.py +++ b/tests/test_ruuvitag_sensor.py @@ -13,10 +13,8 @@ def get_data(self, mac, bt_device): data = '043E2A0201030157168974A5F41E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD' return data[26:] - @patch('ruuvitag_sensor.ble_communication.BleCommunicationNix.get_data', - get_data) - @patch('ruuvitag_sensor.ble_communication.BleCommunicationDummy.get_data', - get_data) + @patch('ruuvitag_sensor.adaptors.nix_hci.BleCommunicationNix.get_data', get_data) + @patch('ruuvitag_sensor.adaptors.dummy.BleCommunicationDummy.get_data', get_data) def test_tag_update_is_valid(self): tag = RuuviTag('48:2C:6A:1E:59:3D') @@ -56,17 +54,17 @@ def get_datas(self, blacklist=[], bt_device=''): for data in datas: yield data - @patch('ruuvitag_sensor.ble_communication.BleCommunicationDummy.get_datas', + @patch('ruuvitag_sensor.adaptors.dummy.BleCommunicationDummy.get_datas', get_datas) - @patch('ruuvitag_sensor.ble_communication.BleCommunicationNix.get_datas', + @patch('ruuvitag_sensor.adaptors.nix_hci.BleCommunicationNix.get_datas', get_datas) def test_find_tags(self): tags = RuuviTagSensor.find_ruuvitags() self.assertEqual(7, len(tags)) - @patch('ruuvitag_sensor.ble_communication.BleCommunicationDummy.get_datas', + @patch('ruuvitag_sensor.adaptors.dummy.BleCommunicationDummy.get_datas', get_datas) - @patch('ruuvitag_sensor.ble_communication.BleCommunicationNix.get_datas', + @patch('ruuvitag_sensor.adaptors.nix_hci.BleCommunicationNix.get_datas', get_datas) def test_get_data_for_sensors(self): macs = ['CC:2C:6A:1E:59:3D', 'DD:2C:6A:1E:59:3D', 'EE:2C:6A:1E:59:3D'] @@ -78,18 +76,18 @@ def test_get_data_for_sensors(self): self.assertTrue(data['EE:2C:6A:1E:59:3D']['temperature'] == 25.12) self.assertTrue(data['EE:2C:6A:1E:59:3D']['identifier'] == '0') - @patch('ruuvitag_sensor.ble_communication.BleCommunicationDummy.get_datas', + @patch('ruuvitag_sensor.adaptors.dummy.BleCommunicationDummy.get_datas', get_datas) - @patch('ruuvitag_sensor.ble_communication.BleCommunicationNix.get_datas', + @patch('ruuvitag_sensor.adaptors.nix_hci.BleCommunicationNix.get_datas', get_datas) def test_get_datas(self): datas = [] RuuviTagSensor.get_datas(datas.append) self.assertEqual(7, len(datas)) - @patch('ruuvitag_sensor.ble_communication.BleCommunicationDummy.get_datas', + @patch('ruuvitag_sensor.adaptors.dummy.BleCommunicationDummy.get_datas', get_datas) - @patch('ruuvitag_sensor.ble_communication.BleCommunicationNix.get_datas', + @patch('ruuvitag_sensor.adaptors.nix_hci.BleCommunicationNix.get_datas', get_datas) def test_get_datas_with_macs(self): datas = [] From 5bed03a357521c54658bbd96e3cdd51d36406445 Mon Sep 17 00:00:00 2001 From: sergioisidoro Date: Thu, 2 Jan 2020 22:51:59 +0200 Subject: [PATCH 2/3] Port work from @ttu for bleson, fix tests, and work for mac os --- ruuvitag_sensor/adaptors/bleson.py | 120 +++++++++++++++++++++++++++++ ruuvitag_sensor/adaptors/dummy.py | 4 +- ruuvitag_sensor/data_formats.py | 15 +++- ruuvitag_sensor/ruuvi.py | 16 ++-- ruuvitag_sensor/ruuvi_rx.py | 9 ++- setup.py | 4 +- tests/test_data_formats.py | 3 +- 7 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 ruuvitag_sensor/adaptors/bleson.py diff --git a/ruuvitag_sensor/adaptors/bleson.py b/ruuvitag_sensor/adaptors/bleson.py new file mode 100644 index 0000000..2f0140d --- /dev/null +++ b/ruuvitag_sensor/adaptors/bleson.py @@ -0,0 +1,120 @@ +import time +import logging +from queue import Queue +from bleson import get_provider, Observer +from multiprocessing import Manager, Process +from ruuvitag_sensor.adaptors import BleCommunication + +log = logging.getLogger(__name__) + +class BleCommunicationBleson(BleCommunication): + '''Bluetooth LE communication with Bleson''' + + @staticmethod + def _run_get_data_background(queue, shared_data, bt_device): + (observer, q) = BleCommunicationBleson.start(bt_device) + + for line in BleCommunicationBleson.get_lines(q): + if shared_data['stop']: + break + try: + mac = line.address.address + if mac in shared_data['blacklist']: + continue + data = line.service_data or line.mfg_data + if data is None: + continue + queue.put((mac, data)) + except GeneratorExit: + break + except: + continue + + BleCommunicationBleson.stop(observer) + + @staticmethod + def start(bt_device=''): + ''' + Attributes: + device (string): BLE device (default 0) + ''' + + if not bt_device: + bt_device = 0 + else: + # Old communication used hci0 etc. + bt_device = bt_device.replace('hci', '') + + log.info('Start receiving broadcasts (device %s)', bt_device) + + q = Queue() + + adapter = get_provider().get_adapter(int(bt_device)) + observer = Observer(adapter) + observer.on_advertising_data = q.put + observer.start() + + return (observer, q) + + @staticmethod + def stop(observer): + observer.stop() + + @staticmethod + def get_lines(queue): + try: + while True: + next_item = queue.get(True, None) + yield next_item + except KeyboardInterrupt: + return + except Exception as ex: + log.info(ex) + return + + @staticmethod + def get_datas(blacklist=[], bt_device=''): + m = Manager() + q = m.Queue() + + # Use Manager dict to share data between processes + shared_data = m.dict() + shared_data['blacklist'] = blacklist + shared_data['stop'] = False + + # Start background process + proc = Process( + target=BleCommunicationBleson._run_get_data_background, + args=[q, shared_data, bt_device] + ) + proc.start() + + try: + while True: + while not q.empty(): + data = q.get() + yield data + log.info("Sleep") + time.sleep(1) + except GeneratorExit: + pass + except KeyboardInterrupt: + pass + + shared_data['stop'] = True + proc.join() + return + + @staticmethod + def get_data(mac, bt_device=''): + data = None + data_iter = BleCommunicationBleson.get_datas(bt_device) + + for data in data_iter: + if mac == data[0]: + log.info('Data found') + data_iter.send(StopIteration) + data = data[1] + break + + return data diff --git a/ruuvitag_sensor/adaptors/dummy.py b/ruuvitag_sensor/adaptors/dummy.py index c0cc473..1752854 100644 --- a/ruuvitag_sensor/adaptors/dummy.py +++ b/ruuvitag_sensor/adaptors/dummy.py @@ -10,8 +10,8 @@ def get_data(mac, bt_device=''): @staticmethod def get_datas(blacklist=[], bt_device=''): datas = [ - ('DU:MM:YD:AT:A9:3D', '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD'), - ('NO:TS:UP:PO:RT:ED', '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD') + ('75:B6:68:32:18:49', '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD'), + ('36:9B:7E:16:18:9B', '1E0201060303AAFE1616AAFE10EE037275752E76692F23416A7759414D4663CD') ] for data in datas: diff --git a/ruuvitag_sensor/data_formats.py b/ruuvitag_sensor/data_formats.py index 9f14dd2..c88023c 100644 --- a/ruuvitag_sensor/data_formats.py +++ b/ruuvitag_sensor/data_formats.py @@ -11,12 +11,22 @@ def convert_data(raw): Returns: tuple (int, string): Data Format type and Sensor data """ - data = DataFormats._get_data_format_3(raw) + + #Python duck typing + try: + string_data = raw.hex() + if string_data[0:2] != 'FF': + string_data = 'FF' + string_data + + except (AttributeError): + string_data = raw + + data = DataFormats._get_data_format_3(string_data) if data is not None: return (3, data) - data = DataFormats._get_data_format_5(raw) + data = DataFormats._get_data_format_5(string_data) if data is not None: return (5, data) @@ -49,7 +59,6 @@ def _get_data_format_2and4(raw): index = data.find('ruu.vi/#') if index > -1: return data[(index + 8):] - return None except: return None diff --git a/ruuvitag_sensor/ruuvi.py b/ruuvitag_sensor/ruuvi.py index f217006..21d2f73 100644 --- a/ruuvitag_sensor/ruuvi.py +++ b/ruuvitag_sensor/ruuvi.py @@ -2,21 +2,23 @@ import os import time import logging +from multiprocessing import Manager from ruuvitag_sensor.data_formats import DataFormats from ruuvitag_sensor.decoder import get_decoder log = logging.getLogger(__name__) - -if not sys.platform.startswith('linux') or os.environ.get('CI') == 'True': +if os.environ.get('CI') == 'True': # Use BleCommunicationDummy also for CI as it can't use bluez from ruuvitag_sensor.adaptors.dummy import BleCommunicationDummy ble = BleCommunicationDummy() else: - from ruuvitag_sensor.adaptors.nix_hci import BleCommunicationNix - ble = BleCommunicationNix() + from ruuvitag_sensor.adaptors.bleson import BleCommunicationBleson + ble = BleCommunicationBleson() +# from ruuvitag_sensor.adaptors.bleson import BleCommunicationBleson +# ble = BleCommunicationBleson() class RunFlag(object): """ @@ -127,18 +129,18 @@ def _get_ruuvitag_datas(macs=[], search_duratio_sec=None, run_flag=RunFlag(), bt tuple: MAC and State of RuuviTag sensor data """ - mac_blacklist = [] + mac_blacklist = Manager().list() start_time = time.time() data_iter = ble.get_datas(mac_blacklist, bt_device) for ble_data in data_iter: # Check duration if search_duratio_sec and time.time() - start_time > search_duratio_sec: - data_iter.send(StopIteration) + data_iter.send(StopIteration("Timeout")) break # Check running flag if not run_flag.running: - data_iter.send(StopIteration) + data_iter.send(StopIteration("Not running")) break # Check MAC whitelist if macs and not ble_data[0] in macs: diff --git a/ruuvitag_sensor/ruuvi_rx.py b/ruuvitag_sensor/ruuvi_rx.py index 08646b0..763ded1 100644 --- a/ruuvitag_sensor/ruuvi_rx.py +++ b/ruuvitag_sensor/ruuvi_rx.py @@ -1,10 +1,11 @@ +import rx +import time + +from threading import Thread from datetime import datetime +from rx.subject import Subject from multiprocessing import Manager -from threading import Thread -import time from concurrent.futures import ProcessPoolExecutor -from rx.subjects import Subject - from ruuvitag_sensor.ruuvi import RuuviTagSensor, RunFlag diff --git a/setup.py b/setup.py index 02de727..8d76e2d 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,8 @@ keywords='RuuviTag BLE', install_requires=[ 'rx', - 'futures;python_version<"3.3"', - 'ptyprocess;platform_system=="Linux"' + 'ptyprocess', + 'bleson @ https://github.com/TheCellule/python-bleson/tarball/master/#egg=bleson', ], license='MIT', packages=['ruuvitag_sensor'], diff --git a/tests/test_data_formats.py b/tests/test_data_formats.py index 7ca7fbb..c1ba794 100644 --- a/tests/test_data_formats.py +++ b/tests/test_data_formats.py @@ -18,7 +18,8 @@ def test_convert_data_valid_data(self): self.assertEqual(result, encoded[1]) def test_convert_data_not_valid_binary(self): - data = b'\x99\x04\x03P\x15]\xceh\xfd\x88\x03\x05\x00\x1b\x0c\x13\x00\x00\x00\x00' + # To make this data pseudo-vaid, it shoud start with \99 instead of \98 + data = b'\x98\x04\x03P\x15]\xceh\xfd\x88\x03\x05\x00\x1b\x0c\x13\x00\x00\x00\x00' encoded = DataFormats.convert_data(data) self.assertIsNone(encoded[0]) self.assertIsNone(encoded[1]) From 1c50c88a92ed7d29a0a759fd040ca7a49b723787 Mon Sep 17 00:00:00 2001 From: sergioisidoro Date: Thu, 2 Jan 2020 22:52:20 +0200 Subject: [PATCH 3/3] Draft for tox and travis ci --- .travis.yml | 9 +++++++++ pytest.ini | 2 ++ tox.ini | 15 +++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 .travis.yml create mode 100644 pytest.ini create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ffa3f20 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +sudo: false +language: python +python: + - "3.5" + - "3.6" + - "3.7" + - "3.8" +install: pip install tox-travis +script: tox \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..ad16166 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +testpaths = tests/ \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..7657c24 --- /dev/null +++ b/tox.ini @@ -0,0 +1,15 @@ +# content of: tox.ini , put in same dir as setup.py +[tox] +envlist = py35,py36,py37,py38 + +[testenv] +setenv = + PYTHONPATH = {toxinidir} + CI = True +# install pytest in the virtualenv where commands will be executed +deps = pytest + mock + +commands = + # NOTE: you can run any command line tool here - not just tests + python setup.py install && pytest \ No newline at end of file