diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..9640858 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,141 @@ + +name: 'Tests' + +on: + push: + + pull_request: + types: [opened, synchronize] #labeled, assigned] + +jobs: + build-and-test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.8] # 2.7, + + steps: + # --- Install steps + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + # --- Versioning + - name: Versioning + id: versioning + run: | + git fetch --unshallow + export VERSION_TAG="" + export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` + echo "GIT DESCRIBE: `git describe`" + echo "GITHUB_REF: $GITHUB_REF" + echo "VERSION_NAME $VERSION_NAME" + if [[ $GITHUB_REF == *"tags"* ]]; then export VERSION_TAG=${GITHUB_REF/refs\/tags\//} ; fi + echo "VERSION_TAG $VERSION_TAG" + if [[ "$VERSION_TAG" == "" ]]; then export VERSION_TAG="vdev" ; fi + if [[ "$VERSION_TAG" != "vdev" ]]; then export VERSION_NAME=$VERSION_TAG ; fi + echo "VERSION_NAME: $VERSION_NAME" + echo "VERSION_TAG : $VERSION_TAG" + if [[ "$VERSION_TAG" == "vdev" ]]; then export VERSION_NAME=$VERSION_NAME"-dev" ; fi + if [[ "$VERSION_TAG" != "vdev" ]]; then export FULL_VERSION_NAME="version $VERSION_NAME" ; fi + if [[ "$VERSION_TAG" == "vdev" ]]; then export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" ; fi + echo "VERSION_NAME: $VERSION_NAME" + echo "FULL_VERSION_NAME: $FULL_VERSION_NAME" + echo "::set-output name=FULL_VERSION_NAME::$FULL_VERSION_NAME" + echo "::set-output name=VERSION_NAME::$VERSION_NAME" + echo "::set-output name=VERSION_TAG::$VERSION_TAG" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install -r _tools/travis_requirements.txt + pip install -r weio/requirements.txt + + - name: System info + run: | + echo "Actor : $GITHUB_ACTOR" + echo "Branch: $GITHUB_REF" + pip list + ls + + # --- Run Tests + - name: Tests + run: | + make test + + + # --- Check if Deployment if needed + - name: Check if deployment is needed + id: check_deploy + env: + PY_VERSION: ${{matrix.python-version}} + GH_EVENT : ${{github.event_name}} + run: | + echo "GH_EVENT : $GH_EVENT" + echo "PY_VERSION : $PY_VERSION" + export OK=0 + if [[ $PY_VERSION == "3.6" ]]; then export OK=1 ; fi + if [[ "$OK" == "1" && $GH_EVENT == "push" ]]; then export OK=1 ; fi + echo "DEPLOY : $OK" + echo "::set-output name=GO::$OK" + + # --- Run Deployments + - name: Install system dependencies + if: ${{ steps.check_deploy.outputs.GO == '1'}} + run: sudo apt-get install nsis + + + - name: Before deploy + if: ${{ steps.check_deploy.outputs.GO == '1'}} + id: before_deploy + env: + FULL_VERSION_NAME: ${{steps.versioning.outputs.FULL_VERSION_NAME}} + VERSION_NAME: ${{steps.versioning.outputs.VERSION_NAME}} + VERSION_TAG: ${{steps.versioning.outputs.VERSION_TAG}} + run: | + echo "FULL_VERSION_NAME: $FULL_VERSION_NAME" + echo "VERSION_NAME : $VERSION_NAME" + echo "VERSION_TAG : $VERSION_TAG" + pip install pynsist + pip install distlib + git clone https://github.com/takluyver/pynsist + mv pynsist/nsist nsist + sh _tools/setVersion.sh $VERSION_NAME + make installer + mv build/nsis/pyDatView_setup.exe "pyDatView_"$VERSION_NAME"_setup.exe" + mv _tools/pyDatView.cmd build/nsis/ + mv _tools/pyDatView_Test.bat build/nsis/ + mv _tools/pyDatView.exe build/nsis/ + mv build/nsis build/pyDatView_$VERSION_NAME + cd build && zip -r "../pyDatView_"$VERSION_NAME"_portable.zip" pyDatView_$VERSION_NAME + cd .. + if [[ "$VERSION_TAG" == "vdev" ]]; then cp "pyDatView_"$VERSION_NAME"_setup.exe" "pyDatView_LatestVersion_setup.exe" ;fi + if [[ "$VERSION_TAG" == "vdev" ]]; then cp "pyDatView_"$VERSION_NAME"_portable.zip" "pyDatView_LatestVersion_portable.zip" ;fi + ls + + - name: Deploy + if: ${{ steps.check_deploy.outputs.GO == '1'}} + env: + FULL_VERSION_NAME: ${{steps.versioning.outputs.FULL_VERSION_NAME}} + VERSION_NAME: ${{steps.versioning.outputs.VERSION_NAME}} + VERSION_TAG: ${{steps.versioning.outputs.VERSION_TAG}} + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: pyDatView_*.* + release_name: ${{steps.versioning.outputs.FULL_VERSION_NAME}} + tag: ${{steps.versioning.outputs.VERSION_TAG}} + overwrite: true + file_glob: true + body: | + Different development versions are found in the "Assets" below. + + Select the one with the highest number to get the latest development version. + + Use a file labelled "setup" for a windows installer. No admin right is required for this installation, but the application is not signed. You may use a file labelled "portable" for a self contained zip files. diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 962d72d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Travis-CI file for pyDatView -language: python - -python: - - "2.7" - - "3.6" - - "3.7" - - "3.8" -os: - - linux -sudo: true - -install: -# - sudo apt-get install python-wxgtk3.0 - - sudo apt-get install nsis - - pip install -r _tools/travis_requirements.txt - - pip install -r weio/requirements.txt - -script: - - make test - - if [[ "$TRAVIS_TAG" == "" ]]; then export TRAVIS_TAG="vdev" ; fi - -before_deploy: - - git fetch --unshallow - - export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` - - export FULL_VERSION_NAME="version $VERSION_NAME" - - if [[ "$TRAVIS_TAG" == "vdev" ]]; then export VERSION_NAME=$VERSION_NAME"-dev" ; fi - - if [[ "$TRAVIS_TAG" == "vdev" ]]; then export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" ; fi - - echo $VERSION_NAME - - pip install pynsist - - pip install distlib - - git clone https://github.com/takluyver/pynsist - - mv pynsist/nsist nsist - - make installer - - mv build/nsis/pyDatView.exe "pyDatView_"$VERSION_NAME"_setup.exe" - - mv _tools/pyDatView.cmd build/nsis/ - - mv _tools/pyDatView.exe build/nsis/ - - mv build/nsis build/pyDatView_$VERSION_NAME - - cd build && zip -r "../pyDatView_"$VERSION_NAME"_portable.zip" pyDatView_$VERSION_NAME - - cd .. - - ls - -deploy: - provider: releases - api_key: $GITHUB_TOKEN - file_glob: true - overwrite: true - skip_cleanup: true - file: - - pyDatView*.exe - - pyDatView*.zip - name: $FULL_VERSION_NAME - target_commitish: $TRAVIS_COMMIT - tag_name: $TRAVIS_TAG - on: - tags: true - branch: master - condition: $TRAVIS_PYTHON_VERSION = 3.6 diff --git a/Makefile b/Makefile index 491d533..f79965f 100644 --- a/Makefile +++ b/Makefile @@ -1,85 +1,87 @@ -# --- Detecting OS -ifeq '$(findstring ;,$(PATH))' ';' - detected_OS := Windows -else - detected_OS := $(shell uname 2>/dev/null || echo Unknown) - detected_OS := $(patsubst CYGWIN%,Cygwin,$(detected_OS)) - detected_OS := $(patsubst MSYS%,MSYS,$(detected_OS)) - detected_OS := $(patsubst MINGW%,MSYS,$(detected_OS)) -endif - -testfile= weio/weio/tests/example_files/FASTIn_arf_coords.txt -all: -ifeq ($(detected_OS),Darwin) # Mac OS X - ./pythonmac pyDatView.py $(testfile) -else - python pyDatView.py $(testfile) -endif - - - - -deb: - python DEBUG.py - -install: - python setup.py install - -dep: - python -m pip install -r requirements.txt - -pull: - git pull --recurse-submodules -update:pull - - -help: - @echo "Available rules:" - @echo " all run the standalone program" - @echo " install install the python package in the system" - @echo " dep download the dependencies " - @echo " pull download the latest version " - @echo " test run the unit tests " - -test: -ifeq ($(detected_OS),Darwin) # Mac OS X - ./pythonmac -m unittest discover -v tests -else - python -m unittest discover -v tests -endif - -prof: - python -m cProfile -o tests/prof_all.prof tests/prof_all.py - python -m pyprof2calltree -i tests/prof_all.prof -o tests/callgrind.prof_all.prof - snakeviz tests/prof_all.prof - - -exe: - python -m nuitka --follow-imports --include-plugin-directory --include-plugin-files --show-progress --show-modules --output-dir=build-nuitka pyDatView.py - -exestd: - python -m nuitka --python-flag=no_site --assume-yes-for-downloads --standalone --follow-imports --include-plugin-directory --include-plugin-files --show-progress --show-modules --output-dir=build-nuitka-std pyDatView.py - -clean: - rm -rf __pycache__ - rm -rf *.egg-info - rm -rf *.spec - rm -rf build* - rm -rf dist - - -pyexe: - pyinstaller --onedir pyDatView.py - -version: -ifeq ($(OS),Windows_NT) - @echo "Doing nothing" -else - @sh _tools/setVersion.sh -endif - -installer: version - python -m nsist installer.cfg - - - +# --- Detecting OS +ifeq '$(findstring ;,$(PATH))' ';' + detected_OS := Windows +else + detected_OS := $(shell uname 2>/dev/null || echo Unknown) + detected_OS := $(patsubst CYGWIN%,Cygwin,$(detected_OS)) + detected_OS := $(patsubst MSYS%,MSYS,$(detected_OS)) + detected_OS := $(patsubst MINGW%,MSYS,$(detected_OS)) +endif + +testfile= weio/weio/tests/example_files/FASTIn_arf_coords.txt +all: +ifeq ($(detected_OS),Darwin) # Mac OS X + ./pythonmac pyDatView.py $(testfile) +else + python pyDatView.py $(testfile) +endif + + + + +deb: + python DEBUG.py + +install: + python setup.py install + +dep: + python -m pip install -r requirements.txt + +pull: + git pull --recurse-submodules +update:pull + + +help: + @echo "Available rules:" + @echo " all run the standalone program" + @echo " install install the python package in the system" + @echo " dep download the dependencies " + @echo " pull download the latest version " + @echo " test run the unit tests " + +test: +ifeq ($(detected_OS),Darwin) # Mac OS X + ./pythonmac -m unittest discover -v tests + ./pythonmac -m unittest discover -v pydatview/plugins/tests +else + python -m unittest discover -v tests + python -m unittest discover -v pydatview/plugins/tests +endif + +prof: + python -m cProfile -o tests/prof_all.prof tests/prof_all.py + python -m pyprof2calltree -i tests/prof_all.prof -o tests/callgrind.prof_all.prof + snakeviz tests/prof_all.prof + + +exe: + python -m nuitka --follow-imports --include-plugin-directory --include-plugin-files --show-progress --show-modules --output-dir=build-nuitka pyDatView.py + +exestd: + python -m nuitka --python-flag=no_site --assume-yes-for-downloads --standalone --follow-imports --include-plugin-directory --include-plugin-files --show-progress --show-modules --output-dir=build-nuitka-std pyDatView.py + +clean: + rm -rf __pycache__ + rm -rf *.egg-info + rm -rf *.spec + rm -rf build* + rm -rf dist + + +pyexe: + pyinstaller --onedir pyDatView.py + +version: +ifeq ($(OS),Windows_NT) + @echo "Doing nothing" +else + @sh _tools/setVersion.sh +endif + +installer: + python -m nsist installer.cfg + + + diff --git a/README.md b/README.md index 4e5c053..637ab40 100644 --- a/README.md +++ b/README.md @@ -1,205 +1,247 @@ -[![Build Status](https://travis-ci.com/ebranlard/pyDatView.svg?branch=master)](https://travis-ci.com/ebranlard/pyDatView) -Donate just a small amount, buy me a coffee! - - - -# pyDatView - -A crossplatform GUI to display tabulated data from files or python pandas dataframes. It's compatible Windows, Linux and MacOS, python 2 and python 3. Some of its features are: multiples plots, FFT plots, probability plots, export of figures... -The file formats supported, are: CSV files and other formats present in the [weio](http://github.com/ebranlard/weio/) library. -Additional file formats can easily be added. - -![Scatter](/../screenshots/screenshots/PlotScatter.png) - -## QuickStart -For **Windows** users, an installer executable is available [here](https://github.com/ebranlard/pyDatView/releases) (look for the latest pyDatView\*.exe) - -**Linux** and **MacOS** users can use the command lines below. **Linux** users may need to install the package python-wxgtk\* (e.g. `python-gtk3.0`) from their distribution. **MacOS** users can use a `brew`, `anaconda` or `virtualenv` version of python and pip, but the final version of python that calls the script needs to have access to the screen (see [details for MacOS](#macos-installation)). The main commands for **Linux** and **MacOS** users are: -```bash -git clone --recurse-submodules https://github.com/ebranlard/pyDatView -cd pyDatView -python -m pip install --user -r requirements.txt -make # executes: 'python pyDatView.py' (on linux) or './pythonmac pyDatView.py' (on Mac) -``` -More information about the download, requirements and installation is provided [further down this page](#installation) - - -## Usage -### Launching the GUI -Windows users that used a `setup.exe` file should be able to look for `pyDatView` in the Windows menu, then launch it, and pin the program to the taskbar for easier access. - -If you cloned this repository, the main script at the root (`pyDatView.py`) is executable and will open the GUI directly. A command line interface is provided, e.g.: -```bash -pyDatView file.csv -``` -The python package can also be used directly from python/jupyter to display a dataframe or show the data in a file -```python -import pydatview -pydatview.show(dataframe=df) -# OR -pydatview.show(filenames=['file.csv']) -# OR -pydatview.show('file.csv') -``` -### Workflow -Documentation is scarce for now, but here are some tips for using the program: - - You can drag and drop files to the GUI directly to open them. Hold the Ctrl key to add. - - You can open several files at once, with same or different filetypes. Upon opening multiple files, a new table appears with the list of open files. - - To add multiple channels or data from multiple files to a plot, use `ctrl + click` or shift-click to make selections. - - Look for the menus indicated by the "sandwich" symbol (3 horizontal bars ੆). These menus are also accessible with right clicks. - - The menus will allow you to edit tables (rename, delete them), add or remove columns (for instance to convert a signal from one unit to another unit), or change the values displayed in the information table at the bottom. - - Few options are also available in the menus `data` and `tools` located at the top of the program. - - The modes and file format drop down menus at the top can usually be kept on `auto`. If a file cannot be read, pay attention to the file extension used, and possibly select a specific file format in the dropdown menu instead of `auto`. - - - -## Features -Main features: -- Plot of tabular data within a file -- Automatic detection of fileformat (based on [weio](http://github.com/ebranlard/weio/) but possibility to add more formats) -- Reload of data (e.g. on file change) -- Display of statistics -- Export figure as pdf, png, eps, svg - -Different kind of plots: -- Scatter plots or line plots -- Multiple plots using sub-figures or a different colors -- Probability density function (PDF) plot -- Fast Fourier Transform (FFT) plot - -Plot options: -- Logarithmic scales on x and y axis -- Scaling of data between 0 and 1 using min and max -- Synchronization of the x-axis of the sub-figures while zooming - -Data manipulation options: - - Remove columns in a table, add columns using a given formula, and export the table to csv - - Mask part of the data (for instance selecting times above a certain value to remove the transient). Apply the mask temporarily, or create a new table from it - - Estimate logarithmic decrement from a signal tthat is decaying - - Extract radial data from OpenFAST input files - - - -## Screenshots - -Scatter plot (by selecting `Scatter`) and several plots on the same figure: - -![Scatter](/../screenshots/screenshots/PlotScatter.png) - - - -Fast Fourier Transform of the signals (by selecting `FFT`) and displaying several plots using subfigures (by selecting `Subplot`). - -![SubPlotFFT](/../screenshots/screenshots/SubPlotFFT.png) - -Probability density function: - -![PlotPDF](/../screenshots/screenshots/PlotPDF.png) - -Scaling all plots between 0 and 1 (by selecting `MinMax`) -![PlotMinMax](/../screenshots/screenshots/PlotMinMax.png) - - - -## Installation - -### Windows installation -For Windows users, installer executables are available [here](https://github.com/ebranlard/pyDatView/releases) (look for the latest pyDatView\*.exe) - -### Linux installation -The script is compatible python 2.7 and python 3 and relies on the following python packages: `numpy` `matplotlib`, `pandas`, `wxpython`. -To download the code and install the dependencies (with pip) run the following: -```bash -git clone --recurse-submodules https://github.com/ebranlard/pyDatView -cd pyDatView -python -m pip install --user -r requirements.txt -``` -If the installation of `wxpython` fails, you may need to install the package python-wxgtk\* (e.g. `python-gtk3.0`) from your distribution. For Debian/Ubuntu systems, try: -`sudo apt-get install python-wxgtk3.0`. -For further troubleshooting you can check the [wxPython wiki page](https://wiki.wxpython.org/). - -If the requirements are successfully installed you can run pyDatView by typing: -```bash -python pyDatView.py -``` -To easily access it later, you can add an alias to your `.bashrc` or install the pydatview module: -```bash -echo "alias pydat='python `pwd`/pyDatview.py'" >> ~/.bashrc -# or -python setup.py install -``` - - -## MacOS installation -The installation works with python2 and python3, with `brew` (with or without a `virtualenv`) or `anaconda`. -First, download the source code: -```bash -git clone --recurse-submodules https://github.com/ebranlard/pyDatView -cd pyDatView -``` -Before installing the requirements, you need to be aware of the two following issues with MacOS: -- If you are using the native version of python, there is an incompatibility between the native version of `matplotlib` on MacOS and the version of `wxpython`. The solution is to use `virtualenv`, `brew` or `anaconda`. -- To use a GUI app, you need a python program that has access to the screen. These special python programs are in different locations. For the system-python, it's usually in `/System`, the `brew` versions are usually in `/usr/local/Cellar`, and the `anaconda` versions are usually called `python.app`. -The script `pythonmac` provided in this repository attempt to find the correct python program depending if you are in a virtual environment, in a conda environment, a system-python or a python from brew or conda. - -Different solutions are provided below depending on your preferred way of working. - -### Brew-python version (outside of a virtualenv) -If you have `brew` installed, and you installed python with `brew install python`, then the easiest is to use your `python3` version: -``` -python3 -m pip install --user -r requirements.txt -python3 pyDatView.py -``` - -### Brew-python version (inside a virtualenv) -If you are inside a virtualenv, with python 2 or 3, use: -``` -pip install -r requirements.txt -./pythonmac pyDatView.py -``` -If the `pythonmac` commands fails, contact the developer, and in the meantime try to replace it with something like: -``` -$(brew --prefix)/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX -``` -where the result from `brew --prefix` is usually `/usr/loca/` and the `XXX` above corresponds to the version of python you are using in your virtual environment. - - -### Anaconda-python version (outside a virtualenv) -The installation of anaconda sometimes replaces the system python with the anaconda version of python. You can see that by typing `which python`. Use the following: -``` -python -m pip install --user -r requirements.txt -./pythonmac pyDatView.py -``` -If the `pythonmac` commands fails, contact the developer, and in the meantime try to replace it with a path similar to -```bash -/anaconda3/bin/python.app -``` -where `/anaconda3/bin/` is the path that would be returned by the command `which conda`. Note the `.app` at the end. If you don't have `python.app`, try installing it with `conda install -c anaconda python.app` - - -### Easy access -To easily access the program later, you can add an alias to your `.bashrc` or install the pydatview module: -```bash -echo "alias pydat='python `pwd`/pyDatview.py'" >> ~/.bashrc -# or -python setup.py install -``` - - - - - -## Adding more file formats -File formats can be added by implementing a subclass of `weio/File.py`, for instance `weio/VTKFile.py`. Existing examples are found in the folder `weio`. -Once implemented the fileformat needs to be registered in `weio/__init__.py` by adding an import line at the beginning of this script and adding a line in the function `fileFormats()` of the form `formats.append(FileFormat(VTKFile))` - - - -## Contributing -Any contributions to this project are welcome! If you find this project useful, you can also buy me a coffee (donate a small amount) with the link below: - - -Donate just a small amount, buy me a coffee! - - - +[![Build status](https://github.com/ebranlard/pyDatView/workflows/Tests/badge.svg)](https://github.com/ebranlard/pyDatView/actions?query=workflow%3A%22Tests%22) +Donate just a small amount, buy me a coffee! + + + +# pyDatView + +A crossplatform GUI to display tabulated data from files or python pandas dataframes. It's compatible Windows, Linux and MacOS, python 2 and python 3. Some of its features are: multiples plots, FFT plots, probability plots, export of figures... +The file formats supported, are: CSV files and other formats present in the [weio](http://github.com/ebranlard/weio/) library. +Additional file formats can easily be added. + +![Scatter](/../screenshots/screenshots/PlotScatter.png) + +## QuickStart +For **Windows** users, an installer executable is available [here](https://github.com/ebranlard/pyDatView/releases) (look for the latest pyDatView\*.exe) + +**Linux** and **MacOS** users can use the command lines below. **Linux** users may need to install the package python-wxgtk\* (e.g. `python-gtk3.0`) from their distribution. **MacOS** users can use a `brew`, `anaconda` or `virtualenv` version of python and pip, but the final version of python that calls the script needs to have access to the screen (see [details for MacOS](#macos-installation)). The main commands for **Linux** and **MacOS** users are: +```bash +git clone --recurse-submodules https://github.com/ebranlard/pyDatView +cd pyDatView +python -m pip install --user -r requirements.txt +make # executes: 'python pyDatView.py' (on linux) or './pythonmac pyDatView.py' (on Mac) +echo "alias pydat='make -C `pwd`'" >> ~/.bashrc +``` +More information about the download, requirements and installation is provided [further down this page](#installation) + + +## Usage +### Launching the GUI +Windows users that used a `setup.exe` file should be able to look for `pyDatView` in the Windows menu, then launch it, and pin the program to the taskbar for easier access. + +If you cloned this repository, the main script at the root (`pyDatView.py`) is executable and will open the GUI directly. A command line interface is provided, e.g.: +```bash +pyDatView file.csv +``` +The python package can also be used directly from python/jupyter to display a dataframe or show the data in a file +```python +import pydatview +pydatview.show(dataframe=df) +# OR +pydatview.show(filenames=['file.csv']) +# OR +pydatview.show('file.csv') +``` + +### Quicklaunch/Shortcut +**Windows** + - If you used the `setup.exe`, you will find the `pyDatView` App in the windows menu, you can launch it from there, pin it to start, pin it to the startbar, open the file location to + - If you used the portable version, you'll find `pyDatView.exe` at the root of the directory. You can launch it and pin it to your taskbar. You can also right click, and create a short cut to add to your desktop or start menu. + - If you clone the repository, you can create a shortcut at the root of the repository. In explorer, right click on an empty space, select New , Shortcut. Set the shortcut as follows: +``` + "C:\PYTHON_DIR\PythonXX\pythonw.exe" "C:\INSTALL_DIR\pyDatView\pyDatView.launch.pyw" +``` + +**Linux** + +You can add an alias to your bashrc as follows. Navigate to the root of the pyDatView repository, and type: +``` + echo "alias pydat='python `pwd`/pyDatView.py'" >> ~/.bashrc +``` +Next time you open a terminal, you can type `pydat` to launch pyDatView. +Adapt to another terminal like `.shrc` + +**MacOS** + +The procedure is the same as for linux, the small issue is that you need to find the "proper" python to call. When you run `./pythonmac` from the root of the directory, the script tries to find the right version for you and finishes by showing a line of the form: `[INFO] Using: /PATH/TO/PYTHON `. This line gives you the path to python. Add pydat as an alias by running the line below (after adapting the `PATH/TO/PYTHON`): +``` + echo "alias pydat='PATH/TO/PYTHON `pwd`/pyDatView.py'" >> ~/.zshrc +``` +Next time you open a terminal, you can type `pydat` to launch pyDatView. + + + +### File association +**Windows** + +To associate a given file type with pyDatView, follow the following steps: + +1. Locate `pyDatView.exe`. If you installed using `setup.exe` or the portable `zip`, you'll find `pyDatView.exe` at the root of the installation folder (default is `C:\Users\%USERNAME%\AppData\Local\pyDatView\`). If you cannot find the exe, download it from [the repository](/_tools/pyDatView.exe). If you cloned the repository, you'll find the executable in the subfolder `_tools\` of the repository. + +2. Verify that the exe works. Double click on the executable to verify that it lauches pyDatView. If it doesnt, run it from a terminal and look at the outputs. + +3. Add the file association. Right click on a file you want to associate pyDatView with. Select "Open With" > "More Apps" > scroll to "Look for another App on my PC" > Navigate to the location of `pyDatView.exe` mentioned above. If this works, repeat the operation and check the box "Always use this App for his filetype". + + +### Workflow +Documentation is scarce for now, but here are some tips for using the program: + - You can drag and drop files to the GUI directly to open them. Hold the Ctrl key to add. + - You can open several files at once, with same or different filetypes. Upon opening multiple files, a new table appears with the list of open files. + - To add multiple channels or data from multiple files to a plot, use `ctrl + click` or shift-click to make selections. + - Look for the menus indicated by the "sandwich" symbol (3 horizontal bars ੆). These menus are also accessible with right clicks. + - The menus will allow you to edit tables (rename, delete them), add or remove columns (for instance to convert a signal from one unit to another unit), or change the values displayed in the information table at the bottom. + - Few options are also available in the menus `data` and `tools` located at the top of the program. + - The modes and file format drop down menus at the top can usually be kept on `auto`. If a file cannot be read, pay attention to the file extension used, and possibly select a specific file format in the dropdown menu instead of `auto`. + + + +## Features +Main features: +- Plot of tabular data within a file +- Automatic detection of fileformat (based on [weio](http://github.com/ebranlard/weio/) but possibility to add more formats) +- Reload of data (e.g. on file change) +- Display of statistics +- Export figure as pdf, png, eps, svg + +Different kind of plots: +- Scatter plots or line plots +- Multiple plots using sub-figures or a different colors +- Probability density function (PDF) plot +- Fast Fourier Transform (FFT) plot + +Plot options: +- Logarithmic scales on x and y axis +- Scaling of data between 0 and 1 using min and max +- Synchronization of the x-axis of the sub-figures while zooming + +Data manipulation options: + - Remove columns in a table, add columns using a given formula, and export the table to csv + - Mask part of the data (for instance selecting times above a certain value to remove the transient). Apply the mask temporarily, or create a new table from it + - Estimate logarithmic decrement from a signal tthat is decaying + - Extract radial data from OpenFAST input files + + + +## Screenshots + +Scatter plot (by selecting `Scatter`) and several plots on the same figure: + +![Scatter](/../screenshots/screenshots/PlotScatter.png) + + + +Fast Fourier Transform of the signals (by selecting `FFT`) and displaying several plots using subfigures (by selecting `Subplot`). + +![SubPlotFFT](/../screenshots/screenshots/SubPlotFFT.png) + +Probability density function: + +![PlotPDF](/../screenshots/screenshots/PlotPDF.png) + +Scaling all plots between 0 and 1 (by selecting `MinMax`) +![PlotMinMax](/../screenshots/screenshots/PlotMinMax.png) + + + +## Installation + +### Windows installation +For Windows users, installer executables are available [here](https://github.com/ebranlard/pyDatView/releases) (look for the latest pyDatView\*.exe) + +### Linux installation +The script is compatible python 2.7 and python 3 and relies on the following python packages: `numpy` `matplotlib`, `pandas`, `wxpython`. +To download the code and install the dependencies (with pip) run the following: +```bash +git clone --recurse-submodules https://github.com/ebranlard/pyDatView +cd pyDatView +python -m pip install --user -r requirements.txt +``` +If the installation of `wxpython` fails, you may need to install the package python-wxgtk\* (e.g. `python-gtk3.0`) from your distribution. For Debian/Ubuntu systems, try: +`sudo apt-get install python-wxgtk3.0`. +For further troubleshooting you can check the [wxPython wiki page](https://wiki.wxpython.org/). + +If the requirements are successfully installed you can run pyDatView by typing: +```bash +python pyDatView.py +``` +To easily access it later, you can add an alias to your `.bashrc` or install the pydatview module: +```bash +echo "alias pydat='python `pwd`/pyDatview.py'" >> ~/.bashrc +# or +python setup.py install +``` + + +## MacOS installation +The installation works with python2 and python3, with `brew` (with or without a `virtualenv`) or `anaconda`. +First, download the source code: +```bash +git clone --recurse-submodules https://github.com/ebranlard/pyDatView +cd pyDatView +``` +Before installing the requirements, you need to be aware of the two following issues with MacOS: +- If you are using the native version of python, there is an incompatibility between the native version of `matplotlib` on MacOS and the version of `wxpython`. The solution is to use `virtualenv`, `brew` or `anaconda`. +- To use a GUI app, you need a python program that has access to the screen. These special python programs are in different locations. For the system-python, it's usually in `/System`, the `brew` versions are usually in `/usr/local/Cellar`, and the `anaconda` versions are usually called `python.app`. +The script `pythonmac` provided in this repository attempt to find the correct python program depending if you are in a virtual environment, in a conda environment, a system-python or a python from brew or conda. + +Different solutions are provided below depending on your preferred way of working. + +### Brew-python version (outside of a virtualenv) +If you have `brew` installed, and you installed python with `brew install python`, then the easiest is to use your `python3` version: +``` +python3 -m pip install --user -r requirements.txt +python3 pyDatView.py +``` + +### Brew-python version (inside a virtualenv) +If you are inside a virtualenv, with python 2 or 3, use: +``` +pip install -r requirements.txt +./pythonmac pyDatView.py +``` +If the `pythonmac` commands fails, contact the developer, and in the meantime try to replace it with something like: +``` +$(brew --prefix)/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX +``` +where the result from `brew --prefix` is usually `/usr/loca/` and the `XXX` above corresponds to the version of python you are using in your virtual environment. + + +### Anaconda-python version (outside a virtualenv) +The installation of anaconda sometimes replaces the system python with the anaconda version of python. You can see that by typing `which python`. Use the following: +``` +python -m pip install --user -r requirements.txt +./pythonmac pyDatView.py +``` +If the `pythonmac` commands fails, contact the developer, and in the meantime try to replace it with a path similar to +```bash +/anaconda3/bin/python.app +``` +where `/anaconda3/bin/` is the path that would be returned by the command `which conda`. Note the `.app` at the end. If you don't have `python.app`, try installing it with `conda install -c anaconda python.app` + + +### Easy access +To easily access the program later, you can add an alias to your `.bashrc` or install the pydatview module: +```bash +echo "alias pydat='python `pwd`/pyDatview.py'" >> ~/.bashrc +# or +python setup.py install +``` + + + + + +## Adding more file formats +File formats can be added by implementing a subclass of `weio/File.py`, for instance `weio/VTKFile.py`. Existing examples are found in the folder `weio`. +Once implemented the fileformat needs to be registered in `weio/__init__.py` by adding an import line at the beginning of this script and adding a line in the function `fileFormats()` of the form `formats.append(FileFormat(VTKFile))` + + + +## Contributing +Any contributions to this project are welcome! If you find this project useful, you can also buy me a coffee (donate a small amount) with the link below: + + +Donate just a small amount, buy me a coffee! + + + diff --git a/_tools/Makefile b/_tools/Makefile index 601adc2..b813bfe 100644 --- a/_tools/Makefile +++ b/_tools/Makefile @@ -1,8 +1,17 @@ -all: compile test - -compile: - cl pyDatView.c - -test: - @echo -------------------------------------------------- - ./pyDatView.exe AA BB CC +all: compile test + +compile: + @echo -------------------------------------------------------------------------- + @cl /nologo -c pyDatView.c + @echo -------------------------------------------------------------------------- + @rc /nologo pyDatView.rc + @echo -------------------------------------------------------------------------- + @link /nologo pyDatView.obj pyDatView.res /out:pyDatView.exe + @rm *.obj *.res + +test: + @echo -------------------------------------------------------------------------- + ./pyDatView.exe C:\Bin\test.outb BB CC + +clean: + rm *.obj *.res pyDatView.exe diff --git a/_tools/NewRelease.md b/_tools/NewRelease.md new file mode 100644 index 0000000..4f14fc9 --- /dev/null +++ b/_tools/NewRelease.md @@ -0,0 +1,9 @@ + + +Steps: +- Change PROG\_VERSION in pydateview/main.py +- Change version in installler.cfg +- Commit changes and push to pull request +- Merge pull request from old to new version +- Tag new version v0.x `git tag -a v0.x` +- Push tags: `git push --tags` diff --git a/_tools/pyDatView.c b/_tools/pyDatView.c index 15f9cf9..c66b963 100644 --- a/_tools/pyDatView.c +++ b/_tools/pyDatView.c @@ -1,85 +1,244 @@ -#define _WIN32_WINNT 0x0500 -#define MAX 1024 - -#include -#include -#include - -#include -#pragma comment(lib,"User32.lib") // for static linking and using ShowWindow - -char* concat(const char *s1, const char *s2) -{ - char *result = malloc(strlen(s1) + strlen(s2) + 1); // +1 for the null-terminator - strcpy(result, s1); - strcat(result, s2); - return result; -} - -void concatenate(char p[], char q[]) { - int c, d; - c = 0; - while (p[c] != '\0') { - c++; - } - d = 0; - while (q[d] != '\0') { - p[c] = q[d]; - d++; - c++; - } - p[c] = '\0'; -} - - -int main (int argc, char** argv) { - char mainCommand[] = "\\Python\\pythonw.exe -c \"import pydatview; pydatview.show();\""; - char wd[MAX]; - char args[MAX]=""; - char fullCommand[MAX]=""; - char path [MAX_PATH]; - char* pfullCommand ; - - // --- Hidding window - HWND hWnd = GetConsoleWindow(); - ShowWindow( hWnd, SW_MINIMIZE ); //won't hide the window without SW_MINIMIZE - //ShowWindow( hWnd, SW_HIDE ); - - // --- List of argumenst to python list of string - int i; - concatenate(args, "["); - for(i = 1; i <= argc-1; i++) { - concatenate(args, "'"); - concatenate(args, argv[i]); - concatenate(args, "'"); - if (i +#include +#include +#include // stat +#include // bool type + + +#include +#pragma comment(lib,"User32.lib") // for static linking and using ShowWindow + + +/* True if file exists */ +bool file_exists (char *filename) { + struct stat buffer; + return (stat (filename, &buffer) == 0); +} + +/* Returns path of current executable */ +char* getpath() +{ + TCHAR path[MAX]; + DWORD length; + length = GetModuleFileName(NULL, path, 1000); + char *result = malloc(length+ 1); // +1 for the null-terminator + int c =0 ; + while (path[c] != '\0') { + result[c] = path[c] ; + c++; + } + result[c] = '\0'; + //char *result = path; + return result; +} + + +/* Concatenate two strings */ +char* concat(const char *s1, const char *s2) +{ + char *result = malloc(strlen(s1) + strlen(s2) + 1); // +1 for the null-terminator + strcpy(result, s1); + strcat(result, s2); + return result; +} + +/* Concatenate string2 to string1 */ +void concatenate(char p[], char q[]) { + int c, d; + c = 0; + while (p[c] != '\0') { + c++; + } + d = 0; + while (q[d] != '\0') { + p[c] = q[d]; + d++; + c++; + } + p[c] = '\0'; +} + + +/* Launch pydatview using local pythonw and command line arguments*/ +int main (int argc, char** argv) { + char wd[MAX]; + char args[MAX]=""; + char fullCommand[MAX]=""; + char path [MAX_PATH]=""; + char pydatpy[MAX_PATH]=""; + char pythonwpath0[MAX_PATH]=""; + char pythonwpath1[MAX_PATH]=""; + char pythonwpath2[MAX_PATH]=""; + char pythonwpath3[MAX_PATH]=""; + char pythonwpath4[MAX_PATH]=""; + char* pfullCommand ; + bool useImport = true; + int index=0; + + // --- Hidding window + HWND hWnd = GetConsoleWindow(); + //ShowWindow( hWnd, SW_MINIMIZE ); //won't hide the window without SW_MINIMIZE + ShowWindow( hWnd, SW_HIDE ); + + // --- Get user name (for AppData path) + char* user = getenv("USERNAME"); + printf("Username : %s\n", user); + + // --- Executable path + char * exename = getpath(); + printf("Exe name : %s\n", exename); + char *exedir = exename; + exedir[strlen(exedir) - 13] = '\0'; // remove pyDatView.exe from path + char parentdir[7]; + strncpy(parentdir, &exedir[strlen(exedir)-7],6); + printf("Exe dir : %s\n", exedir); + printf("Exe dir-7 : %s\n", parentdir); + + // --- Current directory + wd[MAX-1] = '\0'; + if(getcwd(wd, MAX-1) == NULL) { + printf ("[WARN] Can not get current working directory\n"); + wd[0] = '.'; + } + printf("Current Dir: %s\n", wd); + + // --- Get PYDATPATH if defined as env variable + char* pydatpath = getenv("PYDATPATH"); + if (pydatpath) { + printf("PYDATPATH : %s\n", pydatpath); + }else{ + printf("PYDATPATH : (environmental variable not set)\n"); + } + + // --- Pythonw path (assuming it's in system path) + concatenate(pythonwpath1, "pythonw "); + printf("Pythonw1 : %s\n", pythonwpath1); + + + // --- Pythonw path (assuming close to current executable) + concatenate(pythonwpath2, exedir); + concatenate(pythonwpath2,"Python\\pythonw.exe"); + printf("Pythonw2 : %s\n", pythonwpath2); + + // --- Pythonw path (assuming user installed in AppData) + concatenate(pythonwpath3,"C:\\Users\\"); + concatenate(pythonwpath3,user); + concatenate(pythonwpath3,"\\AppData\\Local\\pyDatView\\Python\\pythonw.exe"); + printf("Pythonw3 : %s\n", pythonwpath3); + + // --- Pythonw path (using PYDATPATH env variable) + if (pydatpath) { + concatenate(pythonwpath4, pydatpath); + concatenate(pythonwpath4,"\\Python\\pythonw.exe"); + printf("Pythonw4 : %s\n", pythonwpath4); + } + + + + // --- Selecting pythonw path that exist + if (strcmp(parentdir,"_tools")==0) { + exedir[strlen(exedir) - 7] = '\0'; // remove pyDatView.exe from path + printf("Repo dir : %s\n", exedir); + concatenate(pythonwpath0, pythonwpath1); + useImport =false; + printf(">>> Using Pythonw1\n"); + + } else if (file_exists(pythonwpath2)) { + concatenate(pythonwpath0, pythonwpath2); + printf(">>> Using Pythonw2\n"); + + } else if (file_exists(pythonwpath3)) { + concatenate(pythonwpath0, pythonwpath3); + printf(">>> Using Pythonw3\n"); + + } else if (file_exists(pythonwpath4)) { + concatenate(pythonwpath0, pythonwpath4); + printf(">>> Using Pythonw4\n"); + + } else { + ShowWindow( hWnd, SW_RESTORE); + printf("\n"); + printf("[ERROR] Cannot find pythonw.exe. Try the following options: \n"); + printf(" - place the program at the root of the installation directory\n"); + printf(" - rename the program 'pyDatView.exe' \n"); + printf(" - define the environmental variable PYDATPATH to the root install dir. \n"); + printf(" If none of these options work. Contact the developper with the above outputs'\n"); + printf("\n"); + printf("Press any key to close this window\n"); + getchar(); + return -1; + } + + // --- Convert List of argumenst to python list of string or space separated string + int i; + if (useImport) { + concatenate(args, "["); + for(i = 1; i <= argc-1; i++) { + concatenate(args, "'"); + // replacing slashes + index=0; + while(argv[i][index]) + { + if(argv[i][index] == '\\') + argv[i][index] = '/'; + else + index++; + } + concatenate(args, argv[i]); + concatenate(args, "'"); + if (i $INSTDIR + _tools/pyDatView_Test.bat > $INSTDIR + LICENSE.TXT > $INSTDIR + +pypi_wheels = + numpy==1.19.3 + wxPython==4.0.3 + matplotlib==3.0.0 + pyparsing==2.2.2 + cycler==0.10.0 + six==1.11.0 + python-dateutil==2.7.3 + kiwisolver==1.0.1 + pandas==0.23.4 + pytz==2018.5 + chardet==3.0.4 + scipy==1.1.0 + pyarrow==4.0.1 + +# numpy==1.20.3 +# wxPython==4.0.7 +# matplotlib==3.4.2 +# pyparsing==2.4.7 +# cycler==0.10.0 +# six==1.11.0 +# python-dateutil==2.7.3 +# kiwisolver==1.0.1 +# pandas==1.1.5 +# pytz==2018.5 +# chardet==3.0.4 +# scipy==1.5.4 + +# PyYAML==5.1.2 + +packages=weio + future + +exclude=weio/.git* + weio/tests + pkgs/weio/examples + pkgs/weio/weio/tests + pkgs/numpy/core/include + pkgs/numpy/doc + pkgs/numpy/f2py + pkgs/numpy/core/lib + pkgs/numpy/tests + pkgs/numpy/*/tests + pkgs/pyarrow/include + pkgs/pyarrow/includes + pkgs/pyarrow/tensorflow + pkgs/pyarrow/tests + pkgs/pandas/tests + pkgs/matplotlib/sphinxext + pkgs/matplotlib/testing + pkgs/matplotlib/mpl-data/sample_data + pkgs/matplotlib/mpl-data/fonts + pkgs/matplotlib/mpl-data/images/*.pdf + pkgs/matplotlib/mpl-data/images/*.svg + pkgs/matplotlib/mpl-data/images/*.ppm + pkgs/matplotlib/mpl-data/stylelib/seaborn*.mplstyle + pkgs/matplotlib/backends/qt_editor + pkgs/matplotlib/backends/web_backend + pkgs/wx/locale + pkgs/wx/py + pkgs/wx/lib/agw + pkgs/wx/lib/analogclock + pkgs/wx/lib/art + pkgs/wx/lib/colourchooser + pkgs/wx/lib/editor + pkgs/wx/lib/floatcanvas + pkgs/wx/lib/gizmos + pkgs/wx/lib/masked + pkgs/wx/lib/ogl + pkgs/wx/lib/pdfviewer + pkgs/wx/lib/plot + pkgs/wx/lib/pubsub + pkgs/wx/lib/wxcairo + pkgs/scipy/cluster + pkgs/scipy/constants + pkgs/scipy/io/tests + pkgs/scipy/io/arff + pkgs/scipy/io/matlab/tests + pkgs/scipy/io/harwell_boieng/tests + pkgs/scipy/ndimage + pkgs/scipy/odr + pkgs/scipy/extra-dll/libbanded* + pkgs/scipy/extra-dll/libd_odr* + pkgs/scipy/extra-dll/libdcosqb* + pkgs/scipy/extra-dll/libdfft_sub* + pkgs/scipy/*/tests + pkgs/scipy/fftpack + pkgs/scipy/signal + +# pkgs\matplotlib\mpl-data +##Click==7.0 + +[Build] +#directory= +installer_name=pyDatView_setup.exe diff --git a/pydatview/GUICommon.py b/pydatview/GUICommon.py index 2ab907f..59470be 100644 --- a/pydatview/GUICommon.py +++ b/pydatview/GUICommon.py @@ -3,55 +3,58 @@ import os import platform +_MONOFONTSIZE=9 +_FONTSIZE=9 + # --------------------------------------------------------------------------------} -# --- +# --- FONT # --------------------------------------------------------------------------------{ -def getMonoFontAbs(): - #return wx.Font(9, wx.MODERN, wx.NORMAL, wx.NORMAL, False, u'Monospace') - if os.name=='nt': - font=wx.Font(9, wx.TELETYPE, wx.NORMAL, wx.NORMAL, False) - elif os.name=='posix': - font=wx.Font(10, wx.TELETYPE, wx.NORMAL, wx.NORMAL, False) - else: - font=wx.Font(8, wx.TELETYPE, wx.NORMAL, wx.NORMAL, False) +def setMonoFontSize(fs): + global _MONOFONTSIZE + _MONOFONTSIZE=int(fs) + +def getMonoFontSize(): + global _MONOFONTSIZE + return _MONOFONTSIZE + +def setFontSize(fs): + global _FONTSIZE + _FONTSIZE=int(fs) + +def getFontSize(): + global _FONTSIZE + return _FONTSIZE + + +def getFont(widget): + global _FONTSIZE + font = widget.GetFont() + #font.SetFamily(wx.TELETYPE) + font.SetPointSize(_FONTSIZE) + #font=wx.Font(_FONTSIZE-1, wx.TELETYPE, wx.NORMAL, wx.NORMAL, False) return font def getMonoFont(widget): + global _MONOFONTSIZE font = widget.GetFont() font.SetFamily(wx.TELETYPE) - if platform.system()=='Windows': - pass - elif platform.system()=='Linux': - pass - elif platform.system()=='Darwin': - font.SetPointSize(font.GetPointSize()-1) - else: - pass + font.SetPointSize(_MONOFONTSIZE) return font - - -# def getColumn(df,i): -# if i == wx.NOT_FOUND or i == 0: -# x = np.array(range(df.shape[0])) -# c = None -# isString = False -# isDate = False -# else: -# c = df.iloc[:, i-1] -# x = df.iloc[:, i-1].values -# isString = c.dtype == np.object and isinstance(c.values[0], str) -# if isString: -# x=x.astype(str) -# isDate = np.issubdtype(c.dtype, np.datetime64) -# if isDate: -# x=x.astype('datetime64[s]') -# -# return x,isString,isDate,c -# # --------------------------------------------------------------------------------} # --- Helper functions # --------------------------------------------------------------------------------{ +def About(parent, message): + class MessageBox(wx.Dialog): + def __init__(self, parent, title, message): + wx.Dialog.__init__(self, parent, title=title, style=wx.CAPTION|wx.CLOSE_BOX) + text = wx.TextCtrl(self, style=wx.TE_READONLY|wx.BORDER_NONE|wx.TE_MULTILINE|wx.TE_AUTO_URL) + text.SetValue(message) + text.SetBackgroundColour(wx.SystemSettings.GetColour(4)) + self.ShowModal() + self.Destroy() + MessageBox(parent, 'About', message) + def YesNo(parent, question, caption = 'Yes or no?'): dlg = wx.MessageDialog(parent, question, caption, wx.YES_NO | wx.ICON_QUESTION) result = dlg.ShowModal() == wx.ID_YES diff --git a/pydatview/GUIInfoPanel.py b/pydatview/GUIInfoPanel.py index 5d80862..e198a78 100644 --- a/pydatview/GUIInfoPanel.py +++ b/pydatview/GUIInfoPanel.py @@ -59,11 +59,16 @@ def onSelChange(self,event): self.parent._showStats(erase=True) +def selectColumns(columnList, selectedNames): + """ set Columns as selected based on a list of column names """ + for col in columnList: + col['s']=col['name'] in selectedNames + class InfoPanel(wx.Panel): """ Display the list of the columns for the user to select """ #---------------------------------------------------------------------- - def __init__(self, parent): + def __init__(self, parent, data=None): wx.Panel.__init__(self, parent, -1) # --- # List of dictionaries for available "statistical" signals. Dictionary keys: @@ -71,130 +76,144 @@ def __init__(self, parent): # al : alignement (L,R,C for left,right or center) # f : function used to evaluate value # s : selected or not + if data is None: + print('>>> Using default settings for info panel') + from .appdata import defaultInfoPanelData + data = defaultInfoPanelData() + + + # TODO TODO Consider using an OrderedDict instead, with 'name' as key, and maybe use function instead of string self.ColsReg=[] - self.ColsReg.append({'name':'Directory' , 'al':'L' , 'm':'baseDir' , 's':False}) - self.ColsReg.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's':False}) - self.ColsReg.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's':False}) - self.ColsReg.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's':True}) - self.ColsReg.append({'name':'Median' , 'al':'R' , 'm':'yMedian' , 's' :False}) - self.ColsReg.append({'name':'Mean' , 'al':'R' , 'm':'y0Mean' , 's' :True}) - self.ColsReg.append({'name':'Std' , 'al':'R' , 'm':'y0Std' , 's' :True}) - self.ColsReg.append({'name':'Var' , 'al':'R' , 'm':'y0Var' , 's' :False}) - self.ColsReg.append({'name':'Std/Mean (TI)', 'al':'R' , 'm':'y0TI' , 's' :False}) - self.ColsReg.append({'name':'Min' , 'al':'R' , 'm':'y0Min' , 's' :True}) - self.ColsReg.append({'name':'Max' , 'al':'R' , 'm':'y0Max' , 's' :True}) - self.ColsReg.append({'name':'x@Min' , 'al':'R' , 'm':'xAtYMin' , 's' :False}) - self.ColsReg.append({'name':'x@Max' , 'al':'R' , 'm':'xAtYMax' , 's' :False}) - self.ColsReg.append({'name':'Abs. Max' , 'al':'R' , 'm':'yAbsMax', 's' :False}) - self.ColsReg.append({'name':'Range' , 'al':'R' , 'm':'yRange', 's' :True}) - self.ColsReg.append({'name':'dx' , 'al':'R' , 'm':'dx' , 's' :True}) - self.ColsReg.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) - self.ColsReg.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) - self.ColsReg.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) - self.ColsReg.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) - self.ColsReg.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) - self.ColsReg.append({'name':'x@Min (Meas)' , 'al':'R' , 'm':'xAtYMinMeas' , 's' :False}) - self.ColsReg.append({'name':'x@Max (Meas)' , 'al':'R' , 'm':'xAtYMaxMeas' , 's' :False}) - self.ColsReg.append({'name':'xMin' , 'al':'R' , 'm':'xMin' , 's' :False}) - self.ColsReg.append({'name':'xMax' , 'al':'R' , 'm':'xMax' , 's' :False}) - self.ColsReg.append({'name':'xRange' , 'al':'R' , 'm':'xRange', 's' :False}) - self.ColsReg.append({'name':u'\u222By' , 'al':'R' , 'm':'inty' , 's' :False}) - self.ColsReg.append({'name':u'\u222By/\u222Bdx', 'al':'R' , 'm':'intyintdx' , 's' :False}) - self.ColsReg.append({'name':u'\u222By.x ' , 'al':'R' , 'm':'intyx1' , 's' :False}) - self.ColsReg.append({'name':u'\u222By.x/\u222By','al':'R' , 'm':'intyx1_scaled' , 's' :False}) - self.ColsReg.append({'name':u'\u222By.x^2' , 'al':'R' , 'm':'intyx2' , 's' :False}) - self.ColsReg.append({'name':'L_eq(m=3)' , 'al':'R' , 'f':lambda x:x.leq(m=3) , 's' :False}) - self.ColsReg.append({'name':'L_eq(m=4)' , 'al':'R' , 'f':lambda x:x.leq(m=4) , 's' :False}) - self.ColsReg.append({'name':'L_eq(m=5)' , 'al':'R' , 'f':lambda x:x.leq(m=5) , 's' :False}) - self.ColsReg.append({'name':'L_eq(m=7)' , 'al':'R' , 'f':lambda x:x.leq(m=7) , 's' :False}) - self.ColsReg.append({'name':'L_eq(m=8)' , 'al':'R' , 'f':lambda x:x.leq(m=8) , 's' :False}) - self.ColsReg.append({'name':'L_eq(m=9)' , 'al':'R' , 'f':lambda x:x.leq(m=9) , 's' :False}) - self.ColsReg.append({'name':'L_eq(m=10)' , 'al':'R' , 'f':lambda x:x.leq(m=10), 's' :False}) - self.ColsReg.append({'name':'L_eq(m=12)' , 'al':'R' , 'f':lambda x:x.leq(m=12), 's' :False}) - self.ColsReg.append({'name':'n' , 'al':'R' , 'm':'ylen' , 's' :True}) + self.ColsReg.append({'name':'Directory' , 'al':'L' , 'm':'baseDir' , 's' :False}) + self.ColsReg.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's' :False}) + self.ColsReg.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's' :False}) + self.ColsReg.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's' :True}) + self.ColsReg.append({'name':'Median' , 'al':'R' , 'm':'yMedian' , 's' :False}) + self.ColsReg.append({'name':'Mean' , 'al':'R' , 'm':'y0Mean' , 's' :True}) + self.ColsReg.append({'name':'Std' , 'al':'R' , 'm':'y0Std' , 's' :True}) + self.ColsReg.append({'name':'Var' , 'al':'R' , 'm':'y0Var' , 's' :False}) + self.ColsReg.append({'name':'Std/Mean (TI)', 'al':'R' , 'm':'y0TI' , 's' :False}) + self.ColsReg.append({'name':'Min' , 'al':'R' , 'm':'y0Min' , 's' :True}) + self.ColsReg.append({'name':'Max' , 'al':'R' , 'm':'y0Max' , 's' :True}) + self.ColsReg.append({'name':'x@Min' , 'al':'R' , 'm':'xAtYMin' , 's' :False}) + self.ColsReg.append({'name':'x@Max' , 'al':'R' , 'm':'xAtYMax' , 's' :False}) + self.ColsReg.append({'name':'Abs. Max' , 'al':'R' , 'm':'yAbsMax', 's' :False}) + self.ColsReg.append({'name':'Range' , 'al':'R' , 'm':'yRange', 's' :True}) + self.ColsReg.append({'name':'dx' , 'al':'R' , 'm':'dx' , 's' :True}) + self.ColsReg.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) + self.ColsReg.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) + self.ColsReg.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) + self.ColsReg.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) + self.ColsReg.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) + self.ColsReg.append({'name':'x@Min (Meas)' , 'al':'R' , 'm':'xAtYMinMeas' , 's' :False}) + self.ColsReg.append({'name':'x@Max (Meas)' , 'al':'R' , 'm':'xAtYMaxMeas' , 's' :False}) + self.ColsReg.append({'name':'xMin' , 'al':'R' , 'm':'xMin' , 's' :False}) + self.ColsReg.append({'name':'xMax' , 'al':'R' , 'm':'xMax' , 's' :False}) + self.ColsReg.append({'name':'xRange' , 'al':'R' , 'm':'xRange', 's' :False}) + self.ColsReg.append({'name':u'\u222By' , 'al':'R' , 'm':'inty' , 's' :False}) + self.ColsReg.append({'name':u'\u222By/\u222Bdx', 'al':'R' , 'm':'intyintdx' , 's' :False}) + self.ColsReg.append({'name':u'\u222By.x ' , 'al':'R' , 'm':'intyx1' , 's' :False}) + self.ColsReg.append({'name':u'\u222By.x/\u222By','al':'R' , 'm':'intyx1_scaled' , 's' :False}) + self.ColsReg.append({'name':u'\u222By.x^2' , 'al':'R' , 'm':'intyx2' , 's' :False}) + self.ColsReg.append({'name':'L_eq(m=3)' , 'al':'R' , 'f':lambda x:x.leq(m=3) , 's' :False}) + self.ColsReg.append({'name':'L_eq(m=4)' , 'al':'R' , 'f':lambda x:x.leq(m=4) , 's' :False}) + self.ColsReg.append({'name':'L_eq(m=5)' , 'al':'R' , 'f':lambda x:x.leq(m=5) , 's' :False}) + self.ColsReg.append({'name':'L_eq(m=7)' , 'al':'R' , 'f':lambda x:x.leq(m=7) , 's' :False}) + self.ColsReg.append({'name':'L_eq(m=8)' , 'al':'R' , 'f':lambda x:x.leq(m=8) , 's' :False}) + self.ColsReg.append({'name':'L_eq(m=9)' , 'al':'R' , 'f':lambda x:x.leq(m=9) , 's' :False}) + self.ColsReg.append({'name':'L_eq(m=10)' , 'al':'R' , 'f':lambda x:x.leq(m=10), 's' :False}) + self.ColsReg.append({'name':'L_eq(m=12)' , 'al':'R' , 'f':lambda x:x.leq(m=12), 's' :False}) + self.ColsReg.append({'name':'n' , 'al':'R' , 'm':'ylen' , 's' :True}) self.ColsFFT=[] - self.ColsFFT.append({'name':'Directory' , 'al':'L' , 'm':'baseDir' , 's':False}) - self.ColsFFT.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's':False}) - self.ColsFFT.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's':False}) - self.ColsFFT.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's':True}) - self.ColsFFT.append({'name':'Mean' , 'al':'R' , 'm':'y0Mean' , 's' :True}) - self.ColsFFT.append({'name':'Std' , 'al':'R' , 'm':'y0Std' , 's' :True}) - self.ColsFFT.append({'name':'Min' , 'al':'R' , 'm':'y0Min' , 's' :True}) - self.ColsFFT.append({'name':'Max' , 'al':'R' , 'm':'y0Max' , 's' :True}) - self.ColsFFT.append({'name':'x@Min' , 'al':'R' , 'm':'xAtYMin' , 's' :False}) - self.ColsFFT.append({'name':'x@Max' , 'al':'R' , 'm':'xAtYMax' , 's' :False}) - self.ColsFFT.append({'name':'Mean(FFT)' , 'al':'R' , 'm':'yMean' , 's' :False}) - self.ColsFFT.append({'name':'Std(FFT)' , 'al':'R' , 'm':'yStd' , 's' :False}) - self.ColsFFT.append({'name':'Min(FFT)' , 'al':'R' , 'm':'yMin' , 's' :True}) - self.ColsFFT.append({'name':'Max(FFT)' , 'al':'R' , 'm':'yMax' , 's' :True}) - self.ColsFFT.append({'name':'Var' , 'al':'R' , 'm':'y0Var' , 's' :False}) - self.ColsFFT.append({'name':u'\u222By(FFT)' , 'al':'R' , 'm':'inty' , 's' :True}) - self.ColsFFT.append({'name':'dx(FFT)' , 'al':'R' , 'm':'dx' , 's' :True}) - self.ColsFFT.append({'name':'xMax(FFT)' , 'al':'R' , 'm':'xMax' , 's' :True}) - self.ColsFFT.append({'name':'nOvlp(FFT)' , 'al':'R' , 'f':lambda x:x.Info('LOvlp') , 's' :False}) - self.ColsFFT.append({'name':'nSeg(FFT)' , 'al':'R' , 'f':lambda x:x.Info('LSeg') , 's' :False}) - self.ColsFFT.append({'name':'nWin(FFT)' , 'al':'R' , 'f':lambda x:x.Info('LWin') , 's' :False}) - self.ColsFFT.append({'name':'nFFT(FFT)' , 'al':'R' , 'f':lambda x:x.Info('nFFT') , 's' :False}) - self.ColsFFT.append({'name':'n(FFT)' , 'al':'R' , 'm':'ylen' , 's' :True}) - self.ColsFFT.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) - self.ColsFFT.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) - self.ColsFFT.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) - self.ColsFFT.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) - self.ColsFFT.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) - self.ColsFFT.append({'name':'x@Min (Meas)' , 'al':'R' , 'm':'xAtYMinMeas' , 's' :False}) - self.ColsFFT.append({'name':'x@Max (Meas)' , 'al':'R' , 'm':'xAtYMaxMeas' , 's' :False}) - self.ColsFFT.append({'name':'n ' , 'al':'R' , 'm':'n0' , 's' :True}) + self.ColsFFT.append({'name':'Directory' , 'al':'L' , 'm':'baseDir' , 's' :False}) + self.ColsFFT.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's' :False}) + self.ColsFFT.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's' :False}) + self.ColsFFT.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's' :True}) + self.ColsFFT.append({'name':'Mean' , 'al':'R' , 'm':'y0Mean' , 's' :True}) + self.ColsFFT.append({'name':'Std' , 'al':'R' , 'm':'y0Std' , 's' :True}) + self.ColsFFT.append({'name':'Min' , 'al':'R' , 'm':'y0Min' , 's' :True}) + self.ColsFFT.append({'name':'Max' , 'al':'R' , 'm':'y0Max' , 's' :True}) + self.ColsFFT.append({'name':'x@Min' , 'al':'R' , 'm':'xAtYMin' , 's' :False}) + self.ColsFFT.append({'name':'x@Max' , 'al':'R' , 'm':'xAtYMax' , 's' :False}) + self.ColsFFT.append({'name':'Mean(FFT)' , 'al':'R' , 'm':'yMean' , 's' :False}) + self.ColsFFT.append({'name':'Std(FFT)' , 'al':'R' , 'm':'yStd' , 's' :False}) + self.ColsFFT.append({'name':'Min(FFT)' , 'al':'R' , 'm':'yMin' , 's' :True}) + self.ColsFFT.append({'name':'Max(FFT)' , 'al':'R' , 'm':'yMax' , 's' :True}) + self.ColsFFT.append({'name':'Var' , 'al':'R' , 'm':'y0Var' , 's' :False}) + self.ColsFFT.append({'name':u'\u222By(FFT)' , 'al':'R' , 'm':'inty' , 's' :True}) + self.ColsFFT.append({'name':'dx(FFT)' , 'al':'R' , 'm':'dx' , 's' :True}) + self.ColsFFT.append({'name':'xMax(FFT)' , 'al':'R' , 'm':'xMax' , 's' :True}) + self.ColsFFT.append({'name':'nOvlp(FFT)' , 'al':'R' , 'f':lambda x:x.Info('LOvlp') , 's' :False}) + self.ColsFFT.append({'name':'nSeg(FFT)' , 'al':'R' , 'f':lambda x:x.Info('LSeg') , 's' :False}) + self.ColsFFT.append({'name':'nWin(FFT)' , 'al':'R' , 'f':lambda x:x.Info('LWin') , 's' :False}) + self.ColsFFT.append({'name':'nFFT(FFT)' , 'al':'R' , 'f':lambda x:x.Info('nFFT') , 's' :False}) + self.ColsFFT.append({'name':'n(FFT)' , 'al':'R' , 'm':'ylen' , 's' :True}) + self.ColsFFT.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) + self.ColsFFT.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) + self.ColsFFT.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) + self.ColsFFT.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) + self.ColsFFT.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) + self.ColsFFT.append({'name':'x@Min (Meas)' , 'al':'R' , 'm':'xAtYMinMeas' , 's' :False}) + self.ColsFFT.append({'name':'x@Max (Meas)' , 'al':'R' , 'm':'xAtYMaxMeas' , 's' :False}) + self.ColsFFT.append({'name':'n' , 'al':'R' , 'm':'n0' , 's' :True}) self.ColsMinMax=[] - self.ColsMinMax.append({'name':'Directory' , 'al':'L' , 'm':'baseDir', 's':False}) - self.ColsMinMax.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's':False}) - self.ColsMinMax.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's':False}) - self.ColsMinMax.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's':True}) - self.ColsMinMax.append({'name':'Mean' , 'al':'R' , 'm':'y0Mean' , 's' :True}) - self.ColsMinMax.append({'name':'Std' , 'al':'R' , 'm':'y0Std' , 's' :True}) - self.ColsMinMax.append({'name':'Min' , 'al':'R' , 'm':'y0Min' , 's' :True}) - self.ColsMinMax.append({'name':'Max' , 'al':'R' , 'm':'y0Max' , 's' :True}) - self.ColsMinMax.append({'name':'Mean(MinMax)' , 'al':'R' , 'm':'yMean' , 's' :True}) - self.ColsMinMax.append({'name':'Std(MinMax)' , 'al':'R' , 'm':'yStd' , 's' :True}) - self.ColsMinMax.append({'name':u'\u222By(MinMax)' , 'al':'R' , 'm':'inty' , 's' :True}) - self.ColsMinMax.append({'name':u'\u222By.x(MinMax) ' , 'al':'R' , 'm':'intyx1' , 's' :False}) + self.ColsMinMax.append({'name':'Directory' , 'al':'L' , 'm':'baseDir', 's' :False}) + self.ColsMinMax.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's' :False}) + self.ColsMinMax.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's' :False}) + self.ColsMinMax.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's' :True}) + self.ColsMinMax.append({'name':'Mean' , 'al':'R' , 'm':'y0Mean' , 's' :True}) + self.ColsMinMax.append({'name':'Std' , 'al':'R' , 'm':'y0Std' , 's' :True}) + self.ColsMinMax.append({'name':'Min' , 'al':'R' , 'm':'y0Min' , 's' :True}) + self.ColsMinMax.append({'name':'Max' , 'al':'R' , 'm':'y0Max' , 's' :True}) + self.ColsMinMax.append({'name':'Mean(MinMax)' , 'al':'R' , 'm':'yMean' , 's' :True}) + self.ColsMinMax.append({'name':'Std(MinMax)' , 'al':'R' , 'm':'yStd' , 's' :True}) + self.ColsMinMax.append({'name':u'\u222By(MinMax)' , 'al':'R' , 'm':'inty' , 's' :True}) + self.ColsMinMax.append({'name':u'\u222By.x(MinMax) ' , 'al':'R' , 'm':'intyx1' , 's' :False}) self.ColsMinMax.append({'name':u'\u222By.x/\u222By(MinMax)' , 'al':'R' , 'm':'intyx1_scaled' , 's' :False}) - self.ColsMinMax.append({'name':u'\u222By.x^2(MinMax)' , 'al':'R' , 'm':'intyx2' , 's' :False}) - self.ColsMinMax.append({'name':'dx(MinMax)' , 'al':'R' , 'm':'dx' , 's' :False}) - self.ColsMinMax.append({'name':'n' , 'al':'R' , 'm':'ylen' , 's' :True}) + self.ColsMinMax.append({'name':u'\u222By.x^2(MinMax)' , 'al':'R' , 'm':'intyx2' , 's' :False}) + self.ColsMinMax.append({'name':'dx(MinMax)' , 'al':'R' , 'm':'dx' , 's' :False}) + self.ColsMinMax.append({'name':'n' , 'al':'R' , 'm':'ylen' , 's' :True}) self.ColsPDF=[] - self.ColsPDF.append({'name':'Directory' , 'al':'L' , 'm':'baseDir' , 's':False}) - self.ColsPDF.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's':False}) - self.ColsPDF.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's':False}) - self.ColsPDF.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's':True}) - self.ColsPDF.append({'name':'Mean' , 'al':'R' , 'm':'y0Mean' , 's' :True}) - self.ColsPDF.append({'name':'Std' , 'al':'R' , 'm':'y0Std' , 's' :True}) - self.ColsPDF.append({'name':'Min' , 'al':'R' , 'm':'y0Min' , 's' :True}) - self.ColsPDF.append({'name':'Max' , 'al':'R' , 'm':'y0Max' , 's' :True}) - self.ColsPDF.append({'name':'x@Min' , 'al':'R' , 'm':'xAtYMin' , 's' :False}) - self.ColsPDF.append({'name':'x@Max' , 'al':'R' , 'm':'xAtYMax' , 's' :False}) - self.ColsPDF.append({'name':'Mean(PDF)' , 'al':'R' , 'm':'yMean' , 's' :False}) - self.ColsPDF.append({'name':'Std(PDF)' , 'al':'R' , 'm':'yStd' , 's' :False}) - self.ColsPDF.append({'name':'Min(PDF)' , 'al':'R' , 'm':'yMin' , 's' :True}) - self.ColsPDF.append({'name':'Max(PDF)' , 'al':'R' , 'm':'yMax' , 's' :True}) - self.ColsPDF.append({'name':u'\u222By(PDF)' , 'al':'R' , 'm':'inty' , 's' :True}) - self.ColsPDF.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) - self.ColsPDF.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) - self.ColsPDF.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) - self.ColsPDF.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) - self.ColsPDF.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) - self.ColsPDF.append({'name':'x@Min (Meas)' , 'al':'R' , 'm':'xAtYMinMeas' , 's' :False}) - self.ColsPDF.append({'name':'x@Max (Meas)' , 'al':'R' , 'm':'xAtYMaxMeas' , 's' :False}) - self.ColsPDF.append({'name':'n(PDF)' , 'al':'R' , 'm':'ylen' , 's' :True}) + self.ColsPDF.append({'name':'Directory' , 'al':'L' , 'm':'baseDir' , 's' :False}) + self.ColsPDF.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's' :False}) + self.ColsPDF.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's' :False}) + self.ColsPDF.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's' :True}) + self.ColsPDF.append({'name':'Mean' , 'al':'R' , 'm':'y0Mean' , 's' :True}) + self.ColsPDF.append({'name':'Std' , 'al':'R' , 'm':'y0Std' , 's' :True}) + self.ColsPDF.append({'name':'Min' , 'al':'R' , 'm':'y0Min' , 's' :True}) + self.ColsPDF.append({'name':'Max' , 'al':'R' , 'm':'y0Max' , 's' :True}) + self.ColsPDF.append({'name':'x@Min' , 'al':'R' , 'm':'xAtYMin' , 's' :False}) + self.ColsPDF.append({'name':'x@Max' , 'al':'R' , 'm':'xAtYMax' , 's' :False}) + self.ColsPDF.append({'name':'Mean(PDF)' , 'al':'R' , 'm':'yMean' , 's' :False}) + self.ColsPDF.append({'name':'Std(PDF)' , 'al':'R' , 'm':'yStd' , 's' :False}) + self.ColsPDF.append({'name':'Min(PDF)' , 'al':'R' , 'm':'yMin' , 's' :True}) + self.ColsPDF.append({'name':'Max(PDF)' , 'al':'R' , 'm':'yMax' , 's' :True}) + self.ColsPDF.append({'name':u'\u222By(PDF)' , 'al':'R' , 'm':'inty' , 's' :True}) + self.ColsPDF.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) + self.ColsPDF.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) + self.ColsPDF.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) + self.ColsPDF.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) + self.ColsPDF.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) + self.ColsPDF.append({'name':'x@Min (Meas)' , 'al':'R' , 'm':'xAtYMinMeas' , 's' :False}) + self.ColsPDF.append({'name':'x@Max (Meas)' , 'al':'R' , 'm':'xAtYMaxMeas' , 's' :False}) + self.ColsPDF.append({'name':'n(PDF)' , 'al':'R' , 'm':'ylen' , 's' :True}) self.ColsCmp=[] - self.ColsCmp.append({'name':'Directory' , 'al':'L' , 'm':'baseDir' , 's':False}) - self.ColsCmp.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's':False}) - self.ColsCmp.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's':False}) - self.ColsCmp.append({'name':'Column' , 'al':'L' , 'm':'yName' ,'s':True}) - self.ColsCmp.append({'name':'Mean(Cmp)' , 'al':'R' , 'm':'yMean' , 's' :True}) - self.ColsCmp.append({'name':'Std(Cmp)' , 'al':'R' , 'm':'yStd' , 's' :True}) - self.ColsCmp.append({'name':'Min(Cmp)' , 'al':'R' , 'm':'yMin' , 's' :True}) - self.ColsCmp.append({'name':'Max(Cmp)' , 'al':'R' , 'm':'yMax' , 's' :True}) - self.ColsCmp.append({'name':'n(Cmp)' , 'al':'R' , 'm':'ylen' , 's' :True}) + self.ColsCmp.append({'name':'Directory' , 'al':'L' , 'm':'baseDir' , 's' :False}) + self.ColsCmp.append({'name':'Filename' , 'al':'L' , 'm':'fileName', 's' :False}) + self.ColsCmp.append({'name':'Table' , 'al':'L' , 'm':'tabName' , 's' :False}) + self.ColsCmp.append({'name':'Column' , 'al':'L' , 'm':'yName' , 's' :True}) + self.ColsCmp.append({'name':'Mean(Cmp)' , 'al':'R' , 'm':'yMean' , 's' :True}) + self.ColsCmp.append({'name':'Std(Cmp)' , 'al':'R' , 'm':'yStd' , 's' :True}) + self.ColsCmp.append({'name':'Min(Cmp)' , 'al':'R' , 'm':'yMin' , 's' :True}) + self.ColsCmp.append({'name':'Max(Cmp)' , 'al':'R' , 'm':'yMax' , 's' :True}) + self.ColsCmp.append({'name':'n(Cmp)' , 'al':'R' , 'm':'ylen' , 's' :True}) + + # Select columns based on data + selectColumns(self.ColsReg , data['ColumnsRegular']) + selectColumns(self.ColsFFT , data['ColumnsFFT']) + selectColumns(self.ColsMinMax, data['ColumnsMinMax']) + selectColumns(self.ColsPDF , data['ColumnsPDF']) + selectColumns(self.ColsCmp , data['ColumnsCmp']) self.menuReg=ColCheckMenu(self) self.menuReg.setColumns(self.ColsReg) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 885cba4..d80e08c 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -1,1397 +1,1468 @@ -import os -import numpy as np -import wx -import wx.lib.buttons as buttons -import dateutil # required by matplotlib -#from matplotlib import pyplot as plt -import matplotlib -matplotlib.use('wxAgg') # Important for Windows version of installer. NOTE: changed from Agg to wxAgg -from matplotlib import rc as matplotlib_rc -try: - from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas -except Exception as e: - print('') - print('Error: problem importing `matplotlib.backends.backend_wx`.') - import platform - if platform.system()=='Darwin': - print('') - print('pyDatView help:') - print(' This is a typical issue on MacOS, most likely you are') - print(' using the native MacOS python with the native matplolib') - print(' library, which is incompatible with `wxPython`.') - print('') - print(' You can solve this by either:') - print(' - using python3, and pip3 e.g. installing it with brew') - print(' - using a virtual environment with python 2 or 3') - print(' - using anaconda with python 2 or 3'); - print('') - import sys - sys.exit(1) - else: - raise e -from matplotlib.figure import Figure -from matplotlib.pyplot import rcParams as pyplot_rc -from matplotlib import font_manager -from pandas.plotting import register_matplotlib_converters - -import gc - -from .common import * # unique, CHAR -from .plotdata import PlotData, compareMultiplePD -from .GUICommon import * -from .GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool -from .GUIMeasure import GUIMeasure -from . import icons - -font = {'size' : 8} -matplotlib_rc('font', **font) -pyplot_rc['agg.path.chunksize'] = 20000 - - -class PDFCtrlPanel(wx.Panel): - def __init__(self, parent): - super(PDFCtrlPanel,self).__init__(parent) - self.parent = parent - lb = wx.StaticText( self, -1, 'Number of bins:') - self.scBins = wx.SpinCtrl(self, value='50',size=wx.Size(70,-1)) - self.scBins.SetRange(3, 10000) - self.cbSmooth = wx.CheckBox(self, -1, 'Smooth',(10,10)) - self.cbSmooth.SetValue(False) - dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer.Add(lb ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.scBins ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.cbSmooth ,0, flag = wx.CENTER|wx.LEFT,border = 6) - self.SetSizer(dummy_sizer) - self.Bind(wx.EVT_TEXT , self.onPDFOptionChange, self.scBins) - self.Bind(wx.EVT_CHECKBOX, self.onPDFOptionChange) - self.Hide() - - def onPDFOptionChange(self,event=None): - self.parent.load_and_draw(); # DATA HAS CHANGED - -class MinMaxPanel(wx.Panel): - def __init__(self, parent): - super(MinMaxPanel,self).__init__(parent) - self.parent = parent - self.cbxMinMax = wx.CheckBox(self, -1, 'xMinMax',(10,10)) - self.cbyMinMax = wx.CheckBox(self, -1, 'yMinMax',(10,10)) - self.cbxMinMax.SetValue(False) - self.cbyMinMax.SetValue(True) - dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer.Add(self.cbxMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) - dummy_sizer.Add(self.cbyMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) - self.SetSizer(dummy_sizer) - self.Bind(wx.EVT_CHECKBOX, self.onMinMaxChange) - self.Hide() - - def onMinMaxChange(self,event=None): - self.parent.load_and_draw(); # DATA HAS CHANGED - -class CompCtrlPanel(wx.Panel): - def __init__(self, parent): - super(CompCtrlPanel,self).__init__(parent) - self.parent = parent - lblList = ['Relative', '|Relative|','Ratio','Absolute','Y-Y'] - self.rbType = wx.RadioBox(self, label = 'Type', choices = lblList, - majorDimension = 1, style = wx.RA_SPECIFY_ROWS) - dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer.Add(self.rbType ,0, flag = wx.CENTER|wx.LEFT,border = 1) - self.SetSizer(dummy_sizer) - self.rbType.Bind(wx.EVT_RADIOBOX,self.onTypeChange) - self.Hide() - - def onTypeChange(self,e): - self.parent.load_and_draw(); # DATA HAS CHANGED - - -class SpectralCtrlPanel(wx.Panel): - def __init__(self, parent): - super(SpectralCtrlPanel,self).__init__(parent) - self.parent = parent - # --- GUI widgets - lb = wx.StaticText( self, -1, 'Type:') - self.cbType = wx.ComboBox(self, choices=['PSD','f x PSD','Amplitude'] , style=wx.CB_READONLY) - self.cbType.SetSelection(0) - lbAveraging = wx.StaticText( self, -1, 'Avg.:') - self.cbAveraging = wx.ComboBox(self, choices=['None','Welch'] , style=wx.CB_READONLY) - self.cbAveraging.SetSelection(1) - self.lbAveragingMethod = wx.StaticText( self, -1, 'Window:') - self.cbAveragingMethod = wx.ComboBox(self, choices=['Hamming','Hann','Rectangular'] , style=wx.CB_READONLY) - self.cbAveragingMethod.SetSelection(0) - self.lbP2 = wx.StaticText( self, -1, '2^n:') - self.scP2 = wx.SpinCtrl(self, value='11',size=wx.Size(40,-1)) - self.lbWinLength = wx.StaticText( self, -1, '(2048) ') - self.scP2.SetRange(3, 19) - lbMaxFreq = wx.StaticText( self, -1, 'Xlim:') - self.tMaxFreq = wx.TextCtrl(self,size = (30,-1),style=wx.TE_PROCESS_ENTER) - self.tMaxFreq.SetValue("-1") - self.cbDetrend = wx.CheckBox(self, -1, 'Detrend',(10,10)) - lbX = wx.StaticText( self, -1, 'x:') - self.cbTypeX = wx.ComboBox(self, choices=['1/x','2pi/x','x'] , style=wx.CB_READONLY) - self.cbTypeX.SetSelection(0) - # Layout - dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer.Add(lb ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.cbType ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbAveraging ,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.cbAveraging ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.lbAveragingMethod,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.cbAveragingMethod,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.lbP2 ,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.scP2 ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.lbWinLength ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbMaxFreq ,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.tMaxFreq ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbX ,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.cbTypeX ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.cbDetrend ,0, flag = wx.CENTER|wx.LEFT,border = 7) - self.SetSizer(dummy_sizer) - self.Bind(wx.EVT_COMBOBOX ,self.onSpecCtrlChange) - self.Bind(wx.EVT_TEXT ,self.onP2ChangeText ,self.scP2 ) - self.Bind(wx.EVT_TEXT_ENTER,self.onXlimChange ,self.tMaxFreq ) - self.Bind(wx.EVT_CHECKBOX ,self.onDetrendChange ,self.cbDetrend) - self.Hide() - - def onXlimChange(self,event=None): - self.parent.redraw_same_data(); - def onSpecCtrlChange(self,event=None): - self.parent.load_and_draw() # Data changes - def onDetrendChange(self,event=None): - self.parent.load_and_draw() # Data changes - - def onP2ChangeText(self,event=None): - nExp=self.scP2.GetValue() - self.updateP2(nExp) - self.parent.load_and_draw() # Data changes - - def updateP2(self,P2): - self.lbWinLength.SetLabel("({})".format(2**P2)) - - - - -class PlotTypePanel(wx.Panel): - def __init__(self, parent): - # Superclass constructor - super(PlotTypePanel,self).__init__(parent) - #self.SetBackgroundColour('yellow') - # data - self.parent = parent - # --- Ctrl Panel - self.cbRegular = wx.RadioButton(self, -1, 'Regular',style=wx.RB_GROUP) - self.cbPDF = wx.RadioButton(self, -1, 'PDF' , ) - self.cbFFT = wx.RadioButton(self, -1, 'FFT' , ) - self.cbMinMax = wx.RadioButton(self, -1, 'MinMax' , ) - self.cbCompare = wx.RadioButton(self, -1, 'Compare', ) - self.cbRegular.SetValue(True) - self.Bind(wx.EVT_RADIOBUTTON, self.pdf_select , self.cbPDF ) - self.Bind(wx.EVT_RADIOBUTTON, self.fft_select , self.cbFFT ) - self.Bind(wx.EVT_RADIOBUTTON, self.minmax_select , self.cbMinMax ) - self.Bind(wx.EVT_RADIOBUTTON, self.compare_select, self.cbCompare) - self.Bind(wx.EVT_RADIOBUTTON, self.regular_select, self.cbRegular) - # LAYOUT - cb_sizer = wx.FlexGridSizer(rows=5, cols=1, hgap=0, vgap=0) - cb_sizer.Add(self.cbRegular , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbPDF , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbFFT , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbMinMax , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbCompare , 0, flag=wx.ALL, border=1) - self.SetSizer(cb_sizer) - - def plotType(self): - plotType='Regular' - if self.cbMinMax.GetValue(): - plotType='MinMax' - elif self.cbPDF.GetValue(): - plotType='PDF' - elif self.cbFFT.GetValue(): - plotType='FFT' - elif self.cbCompare.GetValue(): - plotType='Compare' - return plotType - - def regular_select(self, event=None): - self.clear_measures() - self.parent.cbLogY.SetValue(False) - # - self.parent.spcPanel.Hide(); - self.parent.pdfPanel.Hide(); - self.parent.cmpPanel.Hide(); - self.parent.mmxPanel.Hide(); - self.parent.slEsth.Hide(); - self.parent.plotsizer.Layout() - # - self.parent.load_and_draw() # Data changes - - def compare_select(self, event=None): - self.clear_measures() - self.parent.cbLogY.SetValue(False) - self.parent.show_hide(self.parent.cmpPanel, self.cbCompare.GetValue()) - self.parent.spcPanel.Hide(); - self.parent.pdfPanel.Hide(); - self.parent.mmxPanel.Hide(); - self.parent.plotsizer.Layout() - self.parent.load_and_draw() # Data changes - - def fft_select(self, event=None): - self.clear_measures() - self.parent.show_hide(self.parent.spcPanel, self.cbFFT.GetValue()) - self.parent.cbLogY.SetValue(self.cbFFT.GetValue()) - self.parent.pdfPanel.Hide(); - self.parent.mmxPanel.Hide(); - self.parent.plotsizer.Layout() - self.parent.load_and_draw() # Data changes - - def pdf_select(self, event=None): - self.clear_measures() - self.parent.cbLogX.SetValue(False) - self.parent.cbLogY.SetValue(False) - self.parent.show_hide(self.parent.pdfPanel, self.cbPDF.GetValue()) - self.parent.spcPanel.Hide(); - self.parent.cmpPanel.Hide(); - self.parent.mmxPanel.Hide(); - self.parent.plotsizer.Layout() - self.parent.load_and_draw() # Data changes - - def minmax_select(self, event): - self.clear_measures() - self.parent.cbLogY.SetValue(False) - self.parent.show_hide(self.parent.mmxPanel, self.cbMinMax.GetValue()) - self.parent.spcPanel.Hide(); - self.parent.pdfPanel.Hide(); - self.parent.cmpPanel.Hide(); - self.parent.plotsizer.Layout() - self.parent.load_and_draw() # Data changes - - def clear_measures(self): - self.parent.rightMeasure.clear() - self.parent.leftMeasure.clear() - self.parent.lbDeltaX.SetLabel('') - self.parent.lbDeltaY.SetLabel('') - -class EstheticsPanel(wx.Panel): - def __init__(self, parent): - wx.Panel.__init__(self, parent) - self.parent=parent - #self.SetBackgroundColour('red') - - lbFont = wx.StaticText( self, -1, 'Font:') - self.cbFont = wx.ComboBox(self, choices=['6','7','8','9','10','11','12','13','14','15','16','17','18'] , style=wx.CB_READONLY) - self.cbFont.SetSelection(2) - # NOTE: we don't offer "best" since best is slow - lbLegend = wx.StaticText( self, -1, 'Legend:') - self.cbLegend = wx.ComboBox(self, choices=['None','Upper right','Upper left','Lower left','Lower right','Right','Center left','Center right','Lower center','Upper center','Center'] , style=wx.CB_READONLY) - self.cbLegend.SetSelection(1) - lbLgdFont = wx.StaticText( self, -1, 'Legend font:') - self.cbLgdFont = wx.ComboBox(self, choices=['6','7','8','9','10','11','12','13','14','15','16','17','18'] , style=wx.CB_READONLY) - self.cbLgdFont.SetSelection(2) - lbLW = wx.StaticText( self, -1, 'Line width:') - self.cbLW = wx.ComboBox(self, choices=['0.5','1.0','1.5','2.0','2.5','3.0'] , style=wx.CB_READONLY) - self.cbLW.SetSelection(2) - lbMS = wx.StaticText( self, -1, 'Marker size:') - self.cbMS= wx.ComboBox(self, choices=['0.5','1','2','3','4','5','6','7','8'] , style=wx.CB_READONLY) - self.cbMS.SetSelection(2) - - # Layout - #dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer = wx.WrapSizer(orient=wx.HORIZONTAL) - dummy_sizer.Add(lbFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.cbFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbLW ,0, flag = wx.CENTER|wx.LEFT,border = 5) - dummy_sizer.Add(self.cbLW ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbMS ,0, flag = wx.CENTER|wx.LEFT,border = 5) - dummy_sizer.Add(self.cbMS ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbLegend ,0, flag = wx.CENTER|wx.LEFT,border = 5) - dummy_sizer.Add(self.cbLegend ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbLgdFont ,0, flag = wx.CENTER|wx.LEFT,border = 5) - dummy_sizer.Add(self.cbLgdFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) - self.SetSizer(dummy_sizer) - self.Hide() - # Callbacks - self.Bind(wx.EVT_COMBOBOX ,self.onAnyEsthOptionChange) - self.cbFont.Bind(wx.EVT_COMBOBOX ,self.onFontOptionChange) - - def onAnyEsthOptionChange(self,event=None): - self.parent.redraw_same_data() - - def onFontOptionChange(self,event=None): - matplotlib_rc('font', **{'size':int(self.cbFont.Value) }) # affect all (including ticks) - self.onAnyEsthOptionChange() - - -class PlotPanel(wx.Panel): - def __init__(self, parent, selPanel,infoPanel=None, mainframe=None): - - # Superclass constructor - super(PlotPanel,self).__init__(parent) - - # Font handling - font = parent.GetFont() - font.SetPointSize(font.GetPointSize()-1) - self.SetFont(font) - # Preparing a special font manager for chinese characters - self.specialFont=None - try: - pyplot_path = matplotlib.get_data_path() - except: - pyplot_path = pyplot_rc['datapath'] - CH_F_PATHS = [ - os.path.join(pyplot_path, 'fonts/ttf/SimHei.ttf'), - os.path.join(os.path.dirname(__file__),'../SimHei.ttf')] - for fpath in CH_F_PATHS: - if os.path.exists(fpath): - fontP = font_manager.FontProperties(fname=fpath) - fontP.set_size(font.GetPointSize()) - self.specialFont=fontP - break - # data - self.selPanel = selPanel # <<< dependency with selPanel should be minimum - self.selMode = '' - self.infoPanel=infoPanel - self.infoPanel.setPlotMatrixCallbacks(self._onPlotMatrixLeftClick, self._onPlotMatrixRightClick) - self.parent = parent - self.mainframe= mainframe - self.plotData = [] - self.plotDataOptions=dict() - if self.selPanel is not None: - bg=self.selPanel.BackgroundColour - self.SetBackgroundColour(bg) # sowhow, our parent has a wrong color - #self.SetBackgroundColour('red') - self.leftMeasure = GUIMeasure(1, 'firebrick') - self.rightMeasure = GUIMeasure(2, 'darkgreen') - self.xlim_prev = [[0, 1]] - self.ylim_prev = [[0, 1]] - # GUI - self.fig = Figure(facecolor="white", figsize=(1, 1)) - register_matplotlib_converters() - self.canvas = FigureCanvas(self, -1, self.fig) - self.canvas.mpl_connect('motion_notify_event', self.onMouseMove) - self.canvas.mpl_connect('button_press_event', self.onMouseClick) - self.canvas.mpl_connect('button_release_event', self.onMouseRelease) - self.canvas.mpl_connect('draw_event', self.onDraw) - self.clickLocation = (None, 0, 0) - - self.navTBTop = MyNavigationToolbar2Wx(self.canvas, ['Home', 'Pan']) - self.navTBBottom = MyNavigationToolbar2Wx(self.canvas, ['Subplots', 'Save']) - TBAddCheckTool(self.navTBBottom,'', icons.chart.GetBitmap(), self.onEsthToggle) - self.esthToggle=False - - self.navTBBottom.Realize() - - #self.navTB = wx.ToolBar(self, style=wx.TB_HORIZONTAL|wx.TB_HORZ_LAYOUT|wx.TB_NODIVIDER|wx.TB_FLAT) - #self.navTB.SetMargins(0,0) - #self.navTB.SetToolPacking(0) - #self.navTB.AddCheckTool(-1, label='', bitmap1=icons.chart.GetBitmap()) - #self.navTB.Realize() - - self.toolbar_sizer = wx.BoxSizer(wx.VERTICAL) - self.toolbar_sizer.Add(self.navTBTop) - self.toolbar_sizer.Add(self.navTBBottom) - - - # --- Tool Panel - self.toolSizer= wx.BoxSizer(wx.VERTICAL) - # --- PlotType Panel - self.pltTypePanel= PlotTypePanel(self); - # --- Plot type specific options - self.spcPanel = SpectralCtrlPanel(self) - self.pdfPanel = PDFCtrlPanel(self) - self.cmpPanel = CompCtrlPanel(self) - self.mmxPanel = MinMaxPanel(self) - # --- Esthetics panel - self.esthPanel = EstheticsPanel(self) - - - # --- Ctrl Panel - self.ctrlPanel= wx.Panel(self) - #self.ctrlPanel.SetBackgroundColour('blue') - # Check Boxes - self.cbCurveType = wx.ComboBox(self.ctrlPanel, choices=['Plain','LS','Markers','Mix'] , style=wx.CB_READONLY) - self.cbCurveType.SetSelection(1) - self.cbSub = wx.CheckBox(self.ctrlPanel, -1, 'Subplot',(10,10)) - self.cbLogX = wx.CheckBox(self.ctrlPanel, -1, 'Log-x',(10,10)) - self.cbLogY = wx.CheckBox(self.ctrlPanel, -1, 'Log-y',(10,10)) - self.cbSync = wx.CheckBox(self.ctrlPanel, -1, 'Sync-x',(10,10)) - self.cbXHair = wx.CheckBox(self.ctrlPanel, -1, 'CrossHair',(10,10)) - self.cbPlotMatrix = wx.CheckBox(self.ctrlPanel, -1, 'Matrix',(10,10)) - self.cbAutoScale = wx.CheckBox(self.ctrlPanel, -1, 'AutoScale',(10,10)) - self.cbGrid = wx.CheckBox(self.ctrlPanel, -1, 'Grid',(10,10)) - self.cbStepPlot = wx.CheckBox(self.ctrlPanel, -1, 'StepPlot',(10,10)) - self.cbMeasure = wx.CheckBox(self.ctrlPanel, -1, 'Measure',(10,10)) - #self.cbSub.SetValue(True) # DEFAULT TO SUB? - self.cbSync.SetValue(True) - self.cbXHair.SetValue(True) # Have cross hair by default - self.cbAutoScale.SetValue(True) - # Callbacks - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSub ) - self.Bind(wx.EVT_COMBOBOX, self.redraw_event , self.cbCurveType) - self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogX ) - self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogY ) - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSync ) - self.Bind(wx.EVT_CHECKBOX, self.crosshair_event , self.cbXHair ) - self.Bind(wx.EVT_CHECKBOX, self.plot_matrix_select, self.cbPlotMatrix ) - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbAutoScale ) - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbGrid ) - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbStepPlot ) - self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) - self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) - # LAYOUT - cb_sizer = wx.FlexGridSizer(rows=4, cols=3, hgap=0, vgap=0) - cb_sizer.Add(self.cbCurveType , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbSub , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbAutoScale , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbLogX , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbLogY , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbStepPlot , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbXHair , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbGrid , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbSync , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbPlotMatrix, 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbMeasure , 0, flag=wx.ALL, border=1) - - self.ctrlPanel.SetSizer(cb_sizer) - - # --- Crosshair Panel - crossHairPanel= wx.Panel(self) - self.lbCrossHairX = wx.StaticText(crossHairPanel, -1, 'x = ... ') - self.lbCrossHairY = wx.StaticText(crossHairPanel, -1, 'y = ... ') - self.lbDeltaX = wx.StaticText(crossHairPanel, -1, ' ') - self.lbDeltaY = wx.StaticText(crossHairPanel, -1, ' ') - self.lbCrossHairX.SetFont(getMonoFont(self)) - self.lbCrossHairY.SetFont(getMonoFont(self)) - self.lbDeltaX.SetFont(getMonoFont(self)) - self.lbDeltaY.SetFont(getMonoFont(self)) - cbCH = wx.FlexGridSizer(rows=4, cols=1, hgap=0, vgap=0) - cbCH.Add(self.lbCrossHairX , 0, flag=wx.ALL, border=1) - cbCH.Add(self.lbCrossHairY , 0, flag=wx.ALL, border=1) - cbCH.Add(self.lbDeltaX , 0, flag=wx.ALL, border=1) - cbCH.Add(self.lbDeltaY , 0, flag=wx.ALL, border=1) - crossHairPanel.SetSizer(cbCH) - - # --- layout of panels - row_sizer = wx.BoxSizer(wx.HORIZONTAL) - sl2 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) - sl3 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) - sl4 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) - row_sizer.Add(self.pltTypePanel , 0 , flag=wx.LEFT|wx.RIGHT|wx.CENTER , border=1) - row_sizer.Add(sl2 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) - row_sizer.Add(self.toolbar_sizer, 0 , flag=wx.LEFT|wx.RIGHT|wx.CENTER , border=1) - row_sizer.Add(sl3 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) - row_sizer.Add(self.ctrlPanel , 1 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) - row_sizer.Add(sl4 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) - row_sizer.Add(crossHairPanel , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=1) - - plotsizer = wx.BoxSizer(wx.VERTICAL) - self.slCtrl = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) - self.slCtrl.Hide() - self.slEsth = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) - self.slEsth.Hide() - sl1 = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) - plotsizer.Add(self.toolSizer,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.canvas ,1,flag = wx.EXPAND,border = 5 ) - plotsizer.Add(sl1 ,0,flag = wx.EXPAND,border = 0) - plotsizer.Add(self.spcPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.pdfPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.cmpPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.mmxPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.slEsth ,0,flag = wx.EXPAND,border = 0) - plotsizer.Add(self.esthPanel,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.slCtrl ,0,flag = wx.EXPAND,border = 0) - plotsizer.Add(row_sizer ,0,flag = wx.EXPAND|wx.NORTH ,border = 2) - - self.show_hide(self.spcPanel, self.pltTypePanel.cbFFT.GetValue()) - self.show_hide(self.cmpPanel, self.pltTypePanel.cbCompare.GetValue()) - self.show_hide(self.pdfPanel, self.pltTypePanel.cbPDF.GetValue()) - self.show_hide(self.mmxPanel, self.pltTypePanel.cbMinMax.GetValue()) - - self.SetSizer(plotsizer) - self.plotsizer=plotsizer; - self.set_subplot_spacing(init=True) - - def onEsthToggle(self,event): - self.esthToggle=not self.esthToggle - if self.esthToggle: - self.slCtrl.Show() - self.esthPanel.Show() - else: - self.slCtrl.Hide() - self.esthPanel.Hide() - self.plotsizer.Layout() - event.Skip() - - def set_subplot_spacing(self, init=False): - """ - Handle default subplot spacing - - NOTE: - - Tight fails when the ylabel is too long, especially for fft with multiplt signals - - might need to change depending on window size/resizing - - need to change if right axis needed - - this will override the user settings - """ - #self.fig.set_tight_layout(True) # NOTE: works almost fine, but problem with FFT multiple - # TODO this is definitely not generic, but tight fails.. - if init: - # NOTE: at init size is (20,20) because sizer is not initialized yet - bottom = 0.12 - left = 0.12 - else: - if self.Size[1]<300: - bottom=0.20 - elif self.Size[1]<350: - bottom=0.18 - elif self.Size[1]<430: - bottom=0.16 - elif self.Size[1]<600: - bottom=0.13 - elif self.Size[1]<800: - bottom=0.09 - else: - bottom=0.07 - if self.Size[0]<300: - left=0.22 - elif self.Size[0]<450: - left=0.20 - elif self.Size[0]<950: - left=0.12 - else: - left=0.06 - #print(self.Size,'bottom', bottom, 'left',left) - if self.cbPlotMatrix.GetValue(): # TODO detect it - self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98-left) - else: - self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98) - - def plot_matrix_select(self, event): - self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) - self.redraw_same_data() - - def measure_select(self, event): - if self.cbMeasure.IsChecked(): - self.cbAutoScale.SetValue(False) - self.redraw_same_data() - - def redraw_event(self, event): - self.redraw_same_data() - - def log_select(self, event): - if self.pltTypePanel.cbPDF.GetValue(): - self.cbLogX.SetValue(False) - self.cbLogY.SetValue(False) - else: - self.redraw_same_data() - - def crosshair_event(self, event): - try: - self.multiCursors.vertOn =self.cbXHair.GetValue() - self.multiCursors.horizOn=self.cbXHair.GetValue() - self.multiCursors._update() - except: - pass - - def show_hide(self,panel,bShow): - if bShow: - panel.Show() - self.slEsth.Show() - else: - self.slEsth.Hide() - panel.Hide() - - @property - def sharex(self): - return self.cbSync.IsChecked() and (not self.pltTypePanel.cbPDF.GetValue()) - - def set_subplots(self,nPlots): - self.set_subplot_spacing() - # Creating subplots - for ax in self.fig.axes: - self.fig.delaxes(ax) - sharex=None - for i in range(nPlots): - # Vertical stack - if i==0: - ax=self.fig.add_subplot(nPlots,1,i+1) - if self.sharex: - sharex=ax - else: - ax=self.fig.add_subplot(nPlots,1,i+1,sharex=sharex) - # Horizontal stack - #self.fig.add_subplot(1,nPlots,i+1) - - def onMouseMove(self, event): - if event.inaxes: - x, y = event.xdata, event.ydata - self.lbCrossHairX.SetLabel('x =' + self.formatLabelValue(x)) - self.lbCrossHairY.SetLabel('y =' + self.formatLabelValue(y)) - - def onMouseClick(self, event): - self.clickLocation = (event.inaxes, event.xdata, event.ydata) - - def onMouseRelease(self, event): - if self.cbMeasure.GetValue(): - for ax, ax_idx in zip(self.fig.axes, range(len(self.fig.axes))): - if event.inaxes == ax: - x, y = event.xdata, event.ydata - if self.clickLocation != (ax, x, y): - # Ignore measurements for zoom-actions. Possibly add small tolerance. - # Zoom-actions disable autoscale - self.cbAutoScale.SetValue(False) - return - if event.button == 1: - self.infoPanel.setMeasurements((x, y), None) - self.leftMeasure.set(ax_idx, x, y) - self.leftMeasure.plot(ax, ax_idx) - elif event.button == 3: - self.infoPanel.setMeasurements(None, (x, y)) - self.rightMeasure.set(ax_idx, x, y) - self.rightMeasure.plot(ax, ax_idx) - else: - return - if self.cbAutoScale.IsChecked() is False: - self._restore_limits() - - if self.leftMeasure.axis_idx == self.rightMeasure.axis_idx and self.leftMeasure.axis_idx != -1: - self.lbDeltaX.SetLabel('dx=' + self.formatLabelValue(self.rightMeasure.x - self.leftMeasure.x)) - self.lbDeltaY.SetLabel('dy=' + self.formatLabelValue(self.rightMeasure.y - self.leftMeasure.y)) - else: - self.lbDeltaX.SetLabel('') - self.lbDeltaY.SetLabel('') - return - - def onDraw(self, event): - self._store_limits() - - def formatLabelValue(self, value): - try: - if abs(value)<1000 and abs(value)>1e-4: - s = '{:10.5f}'.format(value) - else: - s = '{:10.3e}'.format(value) - except TypeError: - s = ' ' - return s - - def removeTools(self,event=None,Layout=True): - try: - self.toolPanel.destroy() # call the "destroy" function which might clean up data - except: - pass - try: - # Python3 - self.toolSizer.Clear(delete_windows=True) # Delete Windows - except: - # Python2 - if hasattr(self,'toolPanel'): - self.toolSizer.Remove(self.toolPanel) - self.toolPanel.Destroy() - del self.toolPanel - self.toolSizer.Clear() # Delete Windows - if Layout: - self.plotsizer.Layout() - - def showTool(self,toolName=''): - from .GUITools import TOOLS - self.Freeze() - self.removeTools(Layout=False) - if toolName in TOOLS.keys(): - self.toolPanel=TOOLS[toolName](self) # calling the panel constructor - else: - raise Exception('Unknown tool {}'.format(toolName)) - self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) - self.plotsizer.Layout() - self.Thaw() - - def setPD_PDF(self,PD,c): - """ Convert plot data to PDF data based on GUI options""" - # ---PDF - nBins = self.pdfPanel.scBins.GetValue() - bSmooth = self.pdfPanel.cbSmooth.GetValue() - nBins_out= PD.toPDF(nBins,bSmooth) - if nBins_out!=nBins: - self.pdfPanel.scBins.SetValue(nBins) - - def setPD_MinMax(self,PD): - """ Convert plot data to MinMax data based on GUI options""" - yScale=self.mmxPanel.cbyMinMax.IsChecked() - xScale=self.mmxPanel.cbxMinMax.IsChecked() - try: - PD.toMinMax(xScale,yScale) - except Exception as e: - self.mmxPanel.cbxMinMax.SetValue(False) - raise e # Used to be Warn - - def setPD_FFT(self,pd): - """ Convert plot data to FFT data based on GUI options""" - yType = self.spcPanel.cbType.GetStringSelection() - xType = self.spcPanel.cbTypeX.GetStringSelection() - avgMethod = self.spcPanel.cbAveraging.GetStringSelection() - avgWindow = self.spcPanel.cbAveragingMethod.GetStringSelection() - bDetrend = self.spcPanel.cbDetrend.IsChecked() - nExp = self.spcPanel.scP2.GetValue() - # Convert plotdata to FFT data - try: - Info = pd.toFFT(yType=yType, xType=xType, avgMethod=avgMethod, avgWindow=avgWindow, bDetrend=bDetrend, nExp=nExp) - # Trigger - if hasattr(Info,'nExp') and Info.nExp!=nExp: - self.spcPanel.scP2.SetValue(Info.nExp) - self.spcPanel.updateP2(Info.nExp) - except Exception as e: - self.spcPanel.Hide(); - self.plotsizer.Layout() - raise e - - - def transformPlotData(self,PD): - """" - Apply MinMax, PDF or FFT transform to plot based on GUI data - """ - plotType=self.pltTypePanel.plotType() - if plotType=='MinMax': - self.setPD_MinMax(PD) - elif plotType=='PDF': - self.setPD_PDF(PD,PD.c) - elif plotType=='FFT': - self.setPD_FFT(PD) - - def getPlotData(self,plotType): - ID,SameCol,selMode=self.selPanel.getPlotDataSelection() - self.selMode=selMode # we store the selection mode - del self.plotData - self.plotData=[] - tabs=self.selPanel.tabList.getTabs() # TODO, selPanel should just return the PlotData... - try: - for i,idx in enumerate(ID): - # Initialize each plotdata based on selected table and selected id channels - pd=PlotData(); - pd.fromIDs(tabs,i,idx,SameCol, self.plotDataOptions) - # Possible change of data - if plotType=='MinMax': - self.setPD_MinMax(pd) - elif plotType=='PDF': - self.setPD_PDF(pd,pd.c) - elif plotType=='FFT': - self.setPD_FFT(pd) - self.plotData.append(pd) - except Exception as e: - self.plotData=[] - raise e - - def PD_Compare(self,mode): - """ Perform comparison of the selected PlotData, returns new plotData with the comparison. """ - sComp = self.cmpPanel.rbType.GetStringSelection() - try: - self.plotData = compareMultiplePD(self.plotData,mode, sComp) - except Exception as e: - self.pltTypePanel.cbRegular.SetValue(True) - raise e - - def _onPlotMatrixLeftClick(self, event): - """Toggle plot-states from None, to left-axis, to right-axis. - Left-click goes forwards, right-click goes backwards. - IndexError to avoid "holes" in matrix with outer adjacent populated entries - """ - btn = event.GetEventObject() - label = btn.GetLabelText() - if label == '-': - btn.SetLabel('1') - try: - self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) - except IndexError: - btn.SetLabel('-') - elif label == '1': - btn.SetLabel('2') - else: - btn.SetLabel('-') - try: - self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) - except IndexError: - btn.SetLabel('1') - self.redraw_same_data() - - def _onPlotMatrixRightClick(self, event): - btn = event.GetEventObject() - label = btn.GetLabelText() - if label == '-': - btn.SetLabel('2') - try: - self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) - except IndexError: - btn.SetLabel('-') - elif label == '1': - btn.SetLabel('-') - try: - self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) - except IndexError: - btn.SetLabel('2') - else: - btn.SetLabel('1') - self.redraw_same_data() - - def set_axes_lim(self, PDs, axis): - """ - It's usually faster to set the axis limits first (before plotting) - and disable autoscaling. This way the limits are not recomputed when plot data are added. - Also, we already have computed the min and max, so we leverage that. - NOTE: - doesnt not work with strings - doesnt not work for FFT and compare - - INPUTS: - PDs: list of plot data - """ - # TODO option for tight axes - tight=False - - plotType=self.pltTypePanel.plotType() - if plotType in ['FFT','Compare']: - axis.autoscale(True, axis='both', tight=tight) - return - vXString=[PDs[i].xIsString for i in axis.iPD] - vYString=[PDs[i].yIsString for i in axis.iPD] - if not any(vXString) and not self.cbLogX.IsChecked(): - try: - xMin=np.min([PDs[i]._xMin[0] for i in axis.iPD]) - xMax=np.max([PDs[i]._xMax[0] for i in axis.iPD]) - if np.isclose(xMin,xMax): - delta=1 if np.isclose(xMax,0) else 0.1*xMax - else: - if tight: - delta=0 - else: - delta = (xMax-xMin)*pyplot_rc['axes.xmargin'] - axis.set_xlim(xMin-delta,xMax+delta) - axis.autoscale(False, axis='x', tight=False) - except: - pass - if not any(vYString) and not self.cbLogY.IsChecked(): - try: - yMin=np.min([PDs[i]._yMin[0] for i in axis.iPD]) - yMax=np.max([PDs[i]._yMax[0] for i in axis.iPD]) - delta = (yMax-yMin)*pyplot_rc['axes.ymargin'] - if np.isclose(yMin,yMax): - delta=1 if np.isclose(yMax,0) else 0.1*yMax - else: - if tight: - delta=0 - else: - delta = (yMax-yMin)*pyplot_rc['axes.xmargin'] - axis.set_ylim(yMin-delta,yMax+delta) - axis.autoscale(False, axis='y', tight=False) - except: - pass - - def plot_all(self, keep_limits=True): - self.multiCursors=[] - - if self.cbMeasure.GetValue() is False: - for measure in [self.leftMeasure, self.rightMeasure]: - measure.clear() - self.infoPanel.setMeasurements(None, None) - self.lbDeltaX.SetLabel('') - self.lbDeltaY.SetLabel('') - - axes=self.fig.axes - PD=self.plotData - - - # --- Plot options - bStep = self.cbStepPlot.IsChecked() - plot_options = dict() - plot_options['lw']=float(self.esthPanel.cbLW.Value) - plot_options['ms']=float(self.esthPanel.cbMS.Value) - if self.cbCurveType.Value=='Plain': - plot_options['LineStyles'] = ['-'] - plot_options['Markers'] = [''] - elif self.cbCurveType.Value=='LS': - plot_options['LineStyles'] = ['-','--','-.',':'] - plot_options['Markers'] = [''] - elif self.cbCurveType.Value=='Markers': - plot_options['LineStyles'] = [''] - plot_options['Markers'] = ['o','d','v','^','s'] - elif self.cbCurveType.Value=='Mix': # NOTE, can be improved - plot_options['LineStyles'] = ['-','--', '-','-','-'] - plot_options['Markers'] = ['' ,'' ,'o','^','s'] - else: - # Combination of linestyles markers, colors, etc. - # But at that stage, if the user really want this, then we can implement an option to set styles per plot. Not high priority. - raise Exception('Not implemented') - - - - # --- Font options - font_options = dict() - font_options_legd = dict() - font_options['size'] = int(self.esthPanel.cbFont.Value) # affect labels - font_options_legd['fontsize'] = int(self.esthPanel.cbLgdFont.Value) - needChineseFont = any([pd.needChineseFont for pd in PD]) - if needChineseFont and self.specialFont is not None: - font_options['fontproperties']= self.specialFont - font_options_legd['prop'] = self.specialFont - - # --- Loop on axes. Either use ax.iPD to chose the plot data, or rely on plotmatrix - for axis_idx, ax_left in enumerate(axes): - ax_right = None - # Checks - vDate=[PD[i].yIsDate for i in ax_left.iPD] - if any(vDate) and len(vDate)>1: - Error(self,'Cannot plot date and other value on the same axis') - return - - # Set limit before plot when possible, for optimization - self.set_axes_lim(PD, ax_left) - - # Actually plot - pm = self.infoPanel.getPlotMatrix(PD, self.cbSub.IsChecked()) - __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) - ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) - - self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) - for measure in [self.leftMeasure, self.rightMeasure]: - measure.plot(ax_left, axis_idx) - - # Log Axes - if self.cbLogX.IsChecked(): - try: - ax_left.set_xscale("log", nonpositive='clip') # latest - except: - ax_left.set_xscale("log", nonposx='clip') # legacy - - if self.cbLogY.IsChecked(): - if bAllNegLeft is False: - try: - ax_left.set_yscale("log", nonpositive='clip') # latest - except: - ax_left.set_yscale("log", nonposy='clip') - if bAllNegRight is False and ax_right is not None: - try: - ax_right.set_yscale("log", nonpositive='clip') # latest - except: - ax_left.set_yscale("log", nonposy='clip') # legacy - - # XLIM - TODO FFT ONLY NASTY - if self.pltTypePanel.cbFFT.GetValue(): - try: - xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) - if xlim>0: - ax_left.set_xlim([0,xlim]) - pd=PD[ax_left.iPD[0]] - I=pd.x 0 and len(yleft_labels) <= 3: - ax_left.set_ylabel(' and '.join(yleft_labels), **font_options) - elif ax_left is not None: - ax_left.set_ylabel('') - if len(yright_labels) > 0 and len(yright_labels) <= 3: - ax_right.set_ylabel(' and '.join(yright_labels), **font_options) - elif ax_right is not None: - ax_right.set_ylabel('') - - # Legends - lgdLoc = self.esthPanel.cbLegend.Value.lower() - if (self.pltTypePanel.cbCompare.GetValue() or - ((len(yleft_legends) + len(yright_legends)) > 1)): - if lgdLoc !='none': - if len(yleft_legends) > 0: - ax_left.legend(fancybox=False, loc=lgdLoc, **font_options_legd) - if ax_right is not None and len(yright_legends) > 0: - ax_right.legend(fancybox=False, loc=4, **font_options_legd) - elif len(axes)>1 and len(axes)==len(PD): - # TODO: can this be removed? If there is only one unique signal - # per subplot, normally only ylabel is displayed and no legend. - # Special case when we have subplots and all plots have the same label - if lgdLoc !='none': - usy = unique([pd.sy for pd in PD]) - if len(usy)==1: - for ax in axes: - ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) - - axes[-1].set_xlabel(PD[axes[-1].iPD[0]].sx, **font_options) - - #print('sy :',[pd.sy for pd in PD]) - #print('syl:',[pd.syl for pd in PD]) - - # --- Cursors for each individual plot - # NOTE: cursors needs to be stored in the object! - #for ax_left in self.fig.axes: - # self.cursors.append(MyCursor(ax_left,horizOn=True, vertOn=False, useblit=True, color='gray', linewidth=0.5, linestyle=':')) - # Vertical cusor for all, commonly - bXHair = self.cbXHair.GetValue() - self.multiCursors = MyMultiCursor(self.canvas, tuple(self.fig.axes), useblit=True, horizOn=bXHair, vertOn=bXHair, color='gray', linewidth=0.5, linestyle=':') - - def plotSignals(self, ax, axis_idx, PD, pm, left_right, is_step, opts): - axis = None - bAllNeg = True - if pm is None: - loop_range = ax.iPD - else: - loop_range = range(len(PD)) - - iPlot=-1 - for signal_idx in loop_range: - do_plot = False - if left_right == 1 and (pm is None or pm[signal_idx][axis_idx] == left_right): - do_plot = True - axis = ax - elif left_right == 2 and pm is not None and pm[signal_idx][axis_idx] == left_right: - do_plot = True - if axis is None: - axis = ax.twinx() - ax.set_zorder(axis.get_zorder()+1) - ax.patch.set_visible(False) - axis._get_lines.prop_cycler = ax._get_lines.prop_cycler - pd=PD[signal_idx] - if do_plot: - iPlot+=1 - # --- styling per plot - if len(pd.x)==1: - marker='o'; ls='' - else: - # TODO allow PlotData to override for "per plot" options in the future - marker = opts['Markers'][np.mod(iPlot,len(opts['Markers']))] - ls = opts['LineStyles'][np.mod(iPlot,len(opts['LineStyles']))] - if is_step: - plot = axis.step - else: - plot = axis.plot - plot(pd.x,pd.y,label=pd.syl,ms=opts['ms'], lw=opts['lw'], marker=marker, ls=ls) - try: - bAllNeg = bAllNeg and all(pd.y<=0) - except: - pass # Dates or strings - return axis, bAllNeg - - def findPlotMode(self,PD): - uTabs = unique([pd.it for pd in PD]) - usy = unique([pd.sy for pd in PD]) - uiy = unique([pd.iy for pd in PD]) - if len(uTabs)<=0: - raise Exception('No Table. Contact developer') - if len(uTabs)==1: - mode='1Tab_nCols' - else: - if PD[0].SameCol: - mode='nTabs_SameCols' - else: - # Now that we allow multiple selections detecting "simColumns" is more difficult - if len(uTabs) == len(PD): - mode='nTabs_1Col' - elif self.selMode=='simColumnsMode': - mode='nTabs_SimCols' - else: - mode='nTabs_mCols' - return mode - - def findSubPlots(self,PD,mode): - uTabs = unique([pd.it for pd in PD]) - usy = unique([pd.sy for pd in PD]) - bSubPlots = self.cbSub.IsChecked() - bCompare = self.pltTypePanel.cbCompare.GetValue() # NOTE bCompare somehow always 1Tab_nCols - nSubPlots=1 - spreadBy='none' - self.infoPanel.setTabMode(mode) - if mode=='1Tab_nCols': - if bSubPlots: - if bCompare or len(uTabs)==1: - nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) - else: - nSubPlots=len(usy) - spreadBy='iy' - elif mode=='nTabs_SameCols': - if bSubPlots: - if bCompare: - print('>>>TODO ',mode,len(usy),len(uTabs)) - else: - if len(usy)==1: - # Temporary hack until we have an option for spread by tabs or col - nSubPlots=len(uTabs) - spreadBy='it' - else: - nSubPlots=len(usy) - spreadBy='iy' - elif mode=='nTabs_SimCols': - if bSubPlots: - if bCompare: - print('>>>TODO ',mode,len(usy),len(uTabs)) - else: - nSubPlots=int(len(PD)/len(uTabs)) - spreadBy='mod-ip' - elif mode=='nTabs_mCols': - if bSubPlots: - if bCompare: - print('>>>TODO ',mode,len(usy),len(uTabs)) - else: - if bCompare or len(uTabs)==1: - nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) - else: - nSubPlots=len(PD) - spreadBy='mod-ip' - elif mode=='nTabs_1Col': - if bSubPlots: - if bCompare: - print('>>> TODO',mode,len(uTabs)) - else: - nSubPlots=len(uTabs) - spreadBy='it' - else: - raise Exception('Unknown mode, contact developer.') - return nSubPlots,spreadBy - - def distributePlots(self,mode,nSubPlots,spreadBy): - """ Assigns plot data to axes and axes to plot data """ - axes=self.fig.axes - - # Link plot data to axes - if nSubPlots==1 or spreadBy=='none': - axes[0].iPD=[i for i in range(len(self.plotData))] - else: - for ax in axes: - ax.iPD=[] - PD=self.plotData - uTabs=unique([pd.it for pd in PD]) - uiy=unique([pd.iy for pd in PD]) - if spreadBy=='iy': - for ipd,pd in enumerate(PD): - i=uiy.index(pd.iy) - if i < len(axes): - axes[i].iPD.append(ipd) - elif spreadBy=='it': - for ipd,pd in enumerate(PD): - i=uTabs.index(pd.it) - axes[i].iPD.append(ipd) - elif spreadBy=='mod-ip': - for ipd,pd in enumerate(PD): - i=np.mod(ipd, nSubPlots) - axes[i].iPD.append(ipd) - else: - raise Exception('Wrong spreadby value') - - def setLegendLabels(self,mode): - """ Set labels for legend """ - if mode=='1Tab_nCols': - for pd in self.plotData: - if self.pltTypePanel.cbMinMax.GetValue(): - pd.syl = no_unit(pd.sy) - else: - pd.syl = pd.sy - - elif mode=='nTabs_SameCols': - for pd in self.plotData: - pd.syl=pd.st - - elif mode=='nTabs_1Col': - usy=unique([pd.sy for pd in self.plotData]) - if len(usy)==1: - for pd in self.plotData: - pd.syl=pd.st - else: - for pd in self.plotData: - if self.pltTypePanel.cbMinMax.GetValue(): - pd.syl=no_unit(pd.sy) - else: - pd.syl=pd.sy #pd.syl=pd.st + ' - '+pd.sy - elif mode=='nTabs_SimCols': - bSubPlots = self.cbSub.IsChecked() - if bSubPlots: # spread by table name - for pd in self.plotData: - pd.syl=pd.st - else: - for pd in self.plotData: - pd.syl=pd.st + ' - '+pd.sy - elif mode=='nTabs_mCols': - usy=unique([pd.sy for pd in self.plotData]) - bSubPlots = self.cbSub.IsChecked() - if bSubPlots and len(usy)==1: # spread by table name - for pd in self.plotData: - pd.syl=pd.st - else: - for pd in self.plotData: - pd.syl=pd.st + ' - '+pd.sy - else: - raise Exception('Unknown mode {}'.format(mode)) - - - def empty(self): - self.cleanPlot() - - def clean_memory(self): - if hasattr(self,'plotData'): - del self.plotData - self.plotData=[] - for ax in self.fig.axes: - ax.iPD=[] - self.fig.delaxes(ax) - gc.collect() - - def clean_memory_plot(self): - pass - - def cleanPlot(self): - for ax in self.fig.axes: - if hasattr(ax,'iPD'): - del ax.iPD - self.fig.delaxes(ax) - gc.collect() - self.fig.add_subplot(111) - ax = self.fig.axes[0] - ax.set_axis_off() - #ax.plot(1,1) - self.canvas.draw() - gc.collect() - - def load_and_draw(self): - """ Full draw event: - - Get plot data based on selection - - Plot them - - Trigger changes to infoPanel - - """ - self.clean_memory() - self.getPlotData(self.pltTypePanel.plotType()) - if len(self.plotData)==0: - self.cleanPlot(); - return - mode=self.findPlotMode(self.plotData) - if self.pltTypePanel.cbCompare.GetValue(): - self.PD_Compare(mode) - if len(self.plotData)==0: - self.cleanPlot(); - return - self.redraw_same_data() - if self.infoPanel is not None: - self.infoPanel.showStats(self.plotData,self.pltTypePanel.plotType()) - - def redraw_same_data(self, keep_limits=True): - if len(self.plotData)==0: - self.cleanPlot(); - return - elif len(self.plotData) == 1: - if self.plotData[0].xIsString or self.plotData[0].yIsString or self.plotData[0].xIsDate or self.plotData[0].yIsDate: - self.cbAutoScale.SetValue(True) - else: - if len(self.xlim_prev)==0: # Might occur if some date didn't plot before (e.g. strings) - self.cbAutoScale.SetValue(True) - elif rectangleOverlap(self.plotData[0]._xMin[0], self.plotData[0]._yMin[0], - self.plotData[0]._xMax[0], self.plotData[0]._yMax[0], - self.xlim_prev[0][0], self.ylim_prev[0][0], - self.xlim_prev[0][1], self.ylim_prev[0][1]): - pass - else: - self.cbAutoScale.SetValue(True) - - mode=self.findPlotMode(self.plotData) - nPlots,spreadBy=self.findSubPlots(self.plotData,mode) - - self.clean_memory_plot() - self.set_subplots(nPlots) - self.distributePlots(mode,nPlots,spreadBy) - - if not self.pltTypePanel.cbCompare.GetValue(): - self.setLegendLabels(mode) - - self.plot_all(keep_limits) - self.canvas.draw() - - - def _store_limits(self): - self.xlim_prev = [] - self.ylim_prev = [] - for ax in self.fig.axes: - self.xlim_prev.append(ax.get_xlim()) - self.ylim_prev.append(ax.get_ylim()) - - def _restore_limits(self): - for ax, xlim, ylim in zip(self.fig.axes, self.xlim_prev, self.ylim_prev): - ax.set_xlim(xlim) - ax.set_ylim(ylim) - - -if __name__ == '__main__': - import pandas as pd; - from Tables import Table,TableList - - app = wx.App(False) - self=wx.Frame(None,-1,"Title") - self.SetSize((800, 600)) - #self.SetBackgroundColour('red') - class FakeSelPanel(wx.Panel): - def __init__(self, parent): - super(FakeSelPanel,self).__init__(parent) - d ={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0,'ColC':np.random.normal(0,1,100)+1} - df = pd.DataFrame(data=d) - self.tabList=TableList([Table(data=df)]) - - def getPlotDataSelection(self): - ID=[] - ID.append([0,0,2,'x','ColB','tab']) - ID.append([0,0,3,'x','ColC','tab']) - return ID,True - - selpanel=FakeSelPanel(self) - # selpanel.SetBackgroundColour('blue') - p1=PlotPanel(self,selpanel) - p1.load_and_draw() - #p1=SpectralCtrlPanel(self) - sizer = wx.BoxSizer(wx.VERTICAL) - sizer.Add(selpanel,0, flag = wx.EXPAND|wx.ALL,border = 10) - sizer.Add(p1,1, flag = wx.EXPAND|wx.ALL,border = 10) - self.SetSizer(sizer) - - self.Center() - self.Layout() - self.SetSize((800, 600)) - self.Show() - self.SendSizeEvent() - - #p1.showStats(None,[tab],[0],[0,1],tab.columns,0,erase=False) - - app.MainLoop() - - +import os +import numpy as np +import wx +import wx.lib.buttons as buttons +import dateutil # required by matplotlib +#from matplotlib import pyplot as plt +import matplotlib +matplotlib.use('wxAgg') # Important for Windows version of installer. NOTE: changed from Agg to wxAgg +from matplotlib import rc as matplotlib_rc +try: + from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas +except Exception as e: + print('') + print('Error: problem importing `matplotlib.backends.backend_wx`.') + import platform + if platform.system()=='Darwin': + print('') + print('pyDatView help:') + print(' This is a typical issue on MacOS, most likely you are') + print(' using the native MacOS python with the native matplolib') + print(' library, which is incompatible with `wxPython`.') + print('') + print(' You can solve this by either:') + print(' - using python3, and pip3 e.g. installing it with brew') + print(' - using a virtual environment with python 2 or 3') + print(' - using anaconda with python 2 or 3'); + print('') + import sys + sys.exit(1) + else: + raise e +from matplotlib.figure import Figure +from matplotlib.pyplot import rcParams as pyplot_rc +from matplotlib import font_manager +from pandas.plotting import register_matplotlib_converters + +import gc + +from .common import * # unique, CHAR +from .plotdata import PlotData, compareMultiplePD +from .GUICommon import * +from .GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool +from .GUIMeasure import GUIMeasure +from . import icons + +font = {'size' : 8} +matplotlib_rc('font', **font) +pyplot_rc['agg.path.chunksize'] = 20000 + + +class PDFCtrlPanel(wx.Panel): + def __init__(self, parent): + super(PDFCtrlPanel,self).__init__(parent) + self.parent = parent + lb = wx.StaticText( self, -1, 'Number of bins:') + self.scBins = wx.SpinCtrl(self, value='51',size=wx.Size(70,-1), style=wx.TE_RIGHT) + self.scBins.SetRange(3, 10000) + self.cbSmooth = wx.CheckBox(self, -1, 'Smooth',(10,10)) + self.cbSmooth.SetValue(False) + dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer.Add(lb ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.scBins ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.cbSmooth ,0, flag = wx.CENTER|wx.LEFT,border = 6) + self.SetSizer(dummy_sizer) + self.Bind(wx.EVT_TEXT , self.onPDFOptionChange, self.scBins) + self.Bind(wx.EVT_CHECKBOX, self.onPDFOptionChange) + self.Hide() + + def onPDFOptionChange(self,event=None): + self.parent.load_and_draw(); # DATA HAS CHANGED + +class MinMaxPanel(wx.Panel): + def __init__(self, parent): + super(MinMaxPanel,self).__init__(parent) + self.parent = parent + self.cbxMinMax = wx.CheckBox(self, -1, 'xMinMax',(10,10)) + self.cbyMinMax = wx.CheckBox(self, -1, 'yMinMax',(10,10)) + self.cbxMinMax.SetValue(False) + self.cbyMinMax.SetValue(True) + dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer.Add(self.cbxMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) + dummy_sizer.Add(self.cbyMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) + self.SetSizer(dummy_sizer) + self.Bind(wx.EVT_CHECKBOX, self.onMinMaxChange) + self.Hide() + + def onMinMaxChange(self,event=None): + self.parent.load_and_draw(); # DATA HAS CHANGED + +class CompCtrlPanel(wx.Panel): + def __init__(self, parent): + super(CompCtrlPanel,self).__init__(parent) + self.parent = parent + lblList = ['Relative', '|Relative|','Ratio','Absolute','Y-Y'] + self.rbType = wx.RadioBox(self, label = 'Type', choices = lblList, + majorDimension = 1, style = wx.RA_SPECIFY_ROWS) + dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer.Add(self.rbType ,0, flag = wx.CENTER|wx.LEFT,border = 1) + self.SetSizer(dummy_sizer) + self.rbType.Bind(wx.EVT_RADIOBOX,self.onTypeChange) + self.Hide() + + def onTypeChange(self,e): + self.parent.load_and_draw(); # DATA HAS CHANGED + + +class SpectralCtrlPanel(wx.Panel): + def __init__(self, parent): + super(SpectralCtrlPanel,self).__init__(parent) + self.parent = parent + # --- GUI widgets + lb = wx.StaticText( self, -1, 'Type:') + self.cbType = wx.ComboBox(self, choices=['PSD','f x PSD','Amplitude'] , style=wx.CB_READONLY) + self.cbType.SetSelection(0) + lbAveraging = wx.StaticText( self, -1, 'Avg.:') + self.cbAveraging = wx.ComboBox(self, choices=['None','Welch','Binning'] , style=wx.CB_READONLY) + self.cbAveraging.SetSelection(1) + self.lbAveragingMethod = wx.StaticText( self, -1, 'Window:') + self.cbAveragingMethod = wx.ComboBox(self, choices=['Hamming','Hann','Rectangular'] , style=wx.CB_READONLY) + self.cbAveragingMethod.SetSelection(0) + self.lbP2 = wx.StaticText( self, -1, '2^n:') + self.scP2 = wx.SpinCtrl(self, value='11',size=wx.Size(40,-1)) + self.lbWinLength = wx.StaticText( self, -1, '(2048) ') + self.scP2.SetRange(3, 50) + self.previousNExp = 8 + self.previousNDec = 20 + lbMaxFreq = wx.StaticText( self, -1, 'Xlim:') + self.tMaxFreq = wx.TextCtrl(self,size = (30,-1),style=wx.TE_PROCESS_ENTER) + self.tMaxFreq.SetValue("-1") + self.cbDetrend = wx.CheckBox(self, -1, 'Detrend',(10,10)) + lbX = wx.StaticText( self, -1, 'x:') + self.cbTypeX = wx.ComboBox(self, choices=['1/x','2pi/x','x'] , style=wx.CB_READONLY) + self.cbTypeX.SetSelection(0) + # Layout + dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer.Add(lb ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.cbType ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbAveraging ,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.cbAveraging ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.lbAveragingMethod,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.cbAveragingMethod,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.lbP2 ,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.scP2 ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.lbWinLength ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbMaxFreq ,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.tMaxFreq ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbX ,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.cbTypeX ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.cbDetrend ,0, flag = wx.CENTER|wx.LEFT,border = 7) + self.SetSizer(dummy_sizer) + self.Bind(wx.EVT_COMBOBOX ,self.onSpecCtrlChange) + self.Bind(wx.EVT_TEXT ,self.onP2ChangeText ,self.scP2 ) + self.Bind(wx.EVT_TEXT_ENTER,self.onXlimChange ,self.tMaxFreq ) + self.Bind(wx.EVT_CHECKBOX ,self.onDetrendChange ,self.cbDetrend) + self.Hide() + + def onXlimChange(self,event=None): + self.parent.redraw_same_data(); + def onSpecCtrlChange(self,event=None): + if self.cbAveraging.GetStringSelection()=='None': + self.scP2.Enable(False) + self.cbAveragingMethod.Enable(False) + self.lbP2.SetLabel('') + self.lbWinLength.SetLabel('') + elif self.cbAveraging.GetStringSelection()=='Binning': + self.previousNExp= self.scP2.GetValue() + self.scP2.SetValue(self.previousNDec) + self.scP2.Enable(True) + self.cbAveragingMethod.Enable(False) + self.lbP2.SetLabel('n:') + self.lbWinLength.SetLabel('') + else: + self.previousDec= self.scP2.GetValue() + self.scP2.SetValue(self.previousNExp) + self.lbP2.SetLabel('2^n:') + self.scP2.Enable(True) + self.cbAveragingMethod.Enable(True) + self.onP2ChangeText(event=None) + self.parent.load_and_draw() # Data changes + + def onDetrendChange(self,event=None): + self.parent.load_and_draw() # Data changes + + def onP2ChangeText(self,event=None): + if self.cbAveraging.GetStringSelection()=='Binning': + pass + else: + nExp=self.scP2.GetValue() + self.updateP2(nExp) + self.parent.load_and_draw() # Data changes + + def updateP2(self,P2): + self.lbWinLength.SetLabel("({})".format(2**P2)) + + + + +class PlotTypePanel(wx.Panel): + def __init__(self, parent): + # Superclass constructor + super(PlotTypePanel,self).__init__(parent) + #self.SetBackgroundColour('yellow') + # data + self.parent = parent + # --- Ctrl Panel + self.cbRegular = wx.RadioButton(self, -1, 'Regular',style=wx.RB_GROUP) + self.cbPDF = wx.RadioButton(self, -1, 'PDF' , ) + self.cbFFT = wx.RadioButton(self, -1, 'FFT' , ) + self.cbMinMax = wx.RadioButton(self, -1, 'MinMax' , ) + self.cbCompare = wx.RadioButton(self, -1, 'Compare', ) + self.cbRegular.SetValue(True) + self.Bind(wx.EVT_RADIOBUTTON, self.pdf_select , self.cbPDF ) + self.Bind(wx.EVT_RADIOBUTTON, self.fft_select , self.cbFFT ) + self.Bind(wx.EVT_RADIOBUTTON, self.minmax_select , self.cbMinMax ) + self.Bind(wx.EVT_RADIOBUTTON, self.compare_select, self.cbCompare) + self.Bind(wx.EVT_RADIOBUTTON, self.regular_select, self.cbRegular) + # LAYOUT + cb_sizer = wx.FlexGridSizer(rows=5, cols=1, hgap=0, vgap=0) + cb_sizer.Add(self.cbRegular , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbPDF , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbFFT , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbMinMax , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbCompare , 0, flag=wx.ALL, border=1) + self.SetSizer(cb_sizer) + + def plotType(self): + plotType='Regular' + if self.cbMinMax.GetValue(): + plotType='MinMax' + elif self.cbPDF.GetValue(): + plotType='PDF' + elif self.cbFFT.GetValue(): + plotType='FFT' + elif self.cbCompare.GetValue(): + plotType='Compare' + return plotType + + def regular_select(self, event=None): + self.clear_measures() + self.parent.cbLogY.SetValue(False) + # + self.parent.spcPanel.Hide(); + self.parent.pdfPanel.Hide(); + self.parent.cmpPanel.Hide(); + self.parent.mmxPanel.Hide(); + self.parent.slEsth.Hide(); + self.parent.plotsizer.Layout() + # + self.parent.load_and_draw() # Data changes + + def compare_select(self, event=None): + self.clear_measures() + self.parent.cbLogY.SetValue(False) + self.parent.show_hide(self.parent.cmpPanel, self.cbCompare.GetValue()) + self.parent.spcPanel.Hide(); + self.parent.pdfPanel.Hide(); + self.parent.mmxPanel.Hide(); + self.parent.plotsizer.Layout() + self.parent.load_and_draw() # Data changes + + def fft_select(self, event=None): + self.clear_measures() + self.parent.show_hide(self.parent.spcPanel, self.cbFFT.GetValue()) + self.parent.cbLogY.SetValue(self.cbFFT.GetValue()) + self.parent.pdfPanel.Hide(); + self.parent.mmxPanel.Hide(); + self.parent.plotsizer.Layout() + self.parent.load_and_draw() # Data changes + + def pdf_select(self, event=None): + self.clear_measures() + self.parent.cbLogX.SetValue(False) + self.parent.cbLogY.SetValue(False) + self.parent.show_hide(self.parent.pdfPanel, self.cbPDF.GetValue()) + self.parent.spcPanel.Hide(); + self.parent.cmpPanel.Hide(); + self.parent.mmxPanel.Hide(); + self.parent.plotsizer.Layout() + self.parent.load_and_draw() # Data changes + + def minmax_select(self, event): + self.clear_measures() + self.parent.cbLogY.SetValue(False) + self.parent.show_hide(self.parent.mmxPanel, self.cbMinMax.GetValue()) + self.parent.spcPanel.Hide(); + self.parent.pdfPanel.Hide(); + self.parent.cmpPanel.Hide(); + self.parent.plotsizer.Layout() + self.parent.load_and_draw() # Data changes + + def clear_measures(self): + self.parent.rightMeasure.clear() + self.parent.leftMeasure.clear() + self.parent.lbDeltaX.SetLabel('') + self.parent.lbDeltaY.SetLabel('') + +class EstheticsPanel(wx.Panel): + def __init__(self, parent, data): + wx.Panel.__init__(self, parent) + self.parent=parent + #self.SetBackgroundColour('red') + + # Font + lbFont = wx.StaticText( self, -1, 'Font:') + fontChoices = ['6','7','8','9','10','11','12','13','14','15','16','17','18'] + self.cbFont = wx.ComboBox(self, choices=fontChoices , style=wx.CB_READONLY) + try: + i = fontChoices.index(str(data['Font'])) + except ValueError: + i = 2 + self.cbFont.SetSelection(i) + # Legend + # NOTE: we don't offer "best" since best is slow + lbLegend = wx.StaticText( self, -1, 'Legend:') + lbChoices = ['None','Upper right','Upper left','Lower left','Lower right','Right','Center left','Center right','Lower center','Upper center','Center'] + self.cbLegend = wx.ComboBox(self, choices=lbChoices, style=wx.CB_READONLY) + try: + i = lbChoices.index(str(data['LegendPosition'])) + except ValueError: + i=1 + self.cbLegend.SetSelection(i) + # Legend Font + lbLgdFont = wx.StaticText( self, -1, 'Legend font:') + self.cbLgdFont = wx.ComboBox(self, choices=fontChoices, style=wx.CB_READONLY) + try: + i = fontChoices.index(str(data['LegendFont'])) + except ValueError: + i = 2 + self.cbLgdFont.SetSelection(i) + # Line Width Font + lbLW = wx.StaticText( self, -1, 'Line width:') + LWChoices = ['0.5','1.0','1.25','1.5','2.0','2.5','3.0'] + self.cbLW = wx.ComboBox(self, choices=LWChoices , style=wx.CB_READONLY) + try: + i = LWChoices.index(str(data['LineWidth'])) + except ValueError: + i = 3 + self.cbLW.SetSelection(i) + # Marker Size + lbMS = wx.StaticText( self, -1, 'Marker size:') + MSChoices = ['0.5','1','2','3','4','5','6','7','8'] + self.cbMS= wx.ComboBox(self, choices=MSChoices, style=wx.CB_READONLY) + try: + i = MSChoices.index(str(data['MarkerSize'])) + except ValueError: + i = 2 + self.cbMS.SetSelection(i) + + # Layout + #dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer = wx.WrapSizer(orient=wx.HORIZONTAL) + dummy_sizer.Add(lbFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.cbFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbLW ,0, flag = wx.CENTER|wx.LEFT,border = 5) + dummy_sizer.Add(self.cbLW ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbMS ,0, flag = wx.CENTER|wx.LEFT,border = 5) + dummy_sizer.Add(self.cbMS ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbLegend ,0, flag = wx.CENTER|wx.LEFT,border = 5) + dummy_sizer.Add(self.cbLegend ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbLgdFont ,0, flag = wx.CENTER|wx.LEFT,border = 5) + dummy_sizer.Add(self.cbLgdFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) + self.SetSizer(dummy_sizer) + self.Hide() + # Callbacks + self.Bind(wx.EVT_COMBOBOX ,self.onAnyEsthOptionChange) + self.cbFont.Bind(wx.EVT_COMBOBOX ,self.onFontOptionChange) + + def onAnyEsthOptionChange(self,event=None): + self.parent.redraw_same_data() + + def onFontOptionChange(self,event=None): + matplotlib_rc('font', **{'size':int(self.cbFont.Value) }) # affect all (including ticks) + self.onAnyEsthOptionChange() + + +class PlotPanel(wx.Panel): + def __init__(self, parent, selPanel,infoPanel=None, mainframe=None): + + # Superclass constructor + super(PlotPanel,self).__init__(parent) + + # Font handling + font = parent.GetFont() + font.SetPointSize(font.GetPointSize()-1) + self.SetFont(font) + # Preparing a special font manager for chinese characters + self.specialFont=None + try: + pyplot_path = matplotlib.get_data_path() + except: + pyplot_path = pyplot_rc['datapath'] + CH_F_PATHS = [ + os.path.join(pyplot_path, 'fonts/ttf/SimHei.ttf'), + os.path.join(os.path.dirname(__file__),'../SimHei.ttf')] + for fpath in CH_F_PATHS: + if os.path.exists(fpath): + fontP = font_manager.FontProperties(fname=fpath) + fontP.set_size(font.GetPointSize()) + self.specialFont=fontP + break + # data + self.selPanel = selPanel # <<< dependency with selPanel should be minimum + self.selMode = '' + self.infoPanel=infoPanel + self.infoPanel.setPlotMatrixCallbacks(self._onPlotMatrixLeftClick, self._onPlotMatrixRightClick) + self.parent = parent + self.mainframe= mainframe + self.plotData = [] + self.plotDataOptions=dict() + try: + self.data = mainframe.data['plotPanel'] + except: + print('>>> Using default settings for plot panel') + from .appdata import defaultPlotPanelData + self.data = defaultPlotPanelData() + if self.selPanel is not None: + bg=self.selPanel.BackgroundColour + self.SetBackgroundColour(bg) # sowhow, our parent has a wrong color + #self.SetBackgroundColour('red') + self.leftMeasure = GUIMeasure(1, 'firebrick') + self.rightMeasure = GUIMeasure(2, 'darkgreen') + self.xlim_prev = [[0, 1]] + self.ylim_prev = [[0, 1]] + # GUI + self.fig = Figure(facecolor="white", figsize=(1, 1)) + register_matplotlib_converters() + self.canvas = FigureCanvas(self, -1, self.fig) + self.canvas.mpl_connect('motion_notify_event', self.onMouseMove) + self.canvas.mpl_connect('button_press_event', self.onMouseClick) + self.canvas.mpl_connect('button_release_event', self.onMouseRelease) + self.canvas.mpl_connect('draw_event', self.onDraw) + self.clickLocation = (None, 0, 0) + + self.navTBTop = MyNavigationToolbar2Wx(self.canvas, ['Home', 'Pan']) + self.navTBBottom = MyNavigationToolbar2Wx(self.canvas, ['Subplots', 'Save']) + TBAddCheckTool(self.navTBBottom,'', icons.chart.GetBitmap(), self.onEsthToggle) + self.esthToggle=False + + self.navTBBottom.Realize() + + #self.navTB = wx.ToolBar(self, style=wx.TB_HORIZONTAL|wx.TB_HORZ_LAYOUT|wx.TB_NODIVIDER|wx.TB_FLAT) + #self.navTB.SetMargins(0,0) + #self.navTB.SetToolPacking(0) + #self.navTB.AddCheckTool(-1, label='', bitmap1=icons.chart.GetBitmap()) + #self.navTB.Realize() + + self.toolbar_sizer = wx.BoxSizer(wx.VERTICAL) + self.toolbar_sizer.Add(self.navTBTop) + self.toolbar_sizer.Add(self.navTBBottom) + + + # --- Tool Panel + self.toolSizer= wx.BoxSizer(wx.VERTICAL) + # --- PlotType Panel + self.pltTypePanel= PlotTypePanel(self); + # --- Plot type specific options + self.spcPanel = SpectralCtrlPanel(self) + self.pdfPanel = PDFCtrlPanel(self) + self.cmpPanel = CompCtrlPanel(self) + self.mmxPanel = MinMaxPanel(self) + # --- Esthetics panel + self.esthPanel = EstheticsPanel(self, data=self.data['plotStyle']) + + + # --- Ctrl Panel + self.ctrlPanel= wx.Panel(self) + #self.ctrlPanel.SetBackgroundColour('blue') + # Check Boxes + self.cbCurveType = wx.ComboBox(self.ctrlPanel, choices=['Plain','LS','Markers','Mix'] , style=wx.CB_READONLY) + self.cbCurveType.SetSelection(1) + self.cbSub = wx.CheckBox(self.ctrlPanel, -1, 'Subplot',(10,10)) + self.cbLogX = wx.CheckBox(self.ctrlPanel, -1, 'Log-x',(10,10)) + self.cbLogY = wx.CheckBox(self.ctrlPanel, -1, 'Log-y',(10,10)) + self.cbSync = wx.CheckBox(self.ctrlPanel, -1, 'Sync-x',(10,10)) + self.cbXHair = wx.CheckBox(self.ctrlPanel, -1, 'CrossHair',(10,10)) + self.cbPlotMatrix = wx.CheckBox(self.ctrlPanel, -1, 'Matrix',(10,10)) + self.cbAutoScale = wx.CheckBox(self.ctrlPanel, -1, 'AutoScale',(10,10)) + self.cbGrid = wx.CheckBox(self.ctrlPanel, -1, 'Grid',(10,10)) + self.cbStepPlot = wx.CheckBox(self.ctrlPanel, -1, 'StepPlot',(10,10)) + self.cbMeasure = wx.CheckBox(self.ctrlPanel, -1, 'Measure',(10,10)) + #self.cbSub.SetValue(True) # DEFAULT TO SUB? + self.cbSync.SetValue(True) + self.cbXHair.SetValue(self.data['CrossHair']) # Have cross hair by default + self.cbAutoScale.SetValue(True) + self.cbGrid.SetValue(self.data['Grid']) + # Callbacks + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSub ) + self.Bind(wx.EVT_COMBOBOX, self.redraw_event , self.cbCurveType) + self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogX ) + self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogY ) + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSync ) + self.Bind(wx.EVT_CHECKBOX, self.crosshair_event , self.cbXHair ) + self.Bind(wx.EVT_CHECKBOX, self.plot_matrix_select, self.cbPlotMatrix ) + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbAutoScale ) + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbGrid ) + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbStepPlot ) + self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) + self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) + # LAYOUT + cb_sizer = wx.FlexGridSizer(rows=4, cols=3, hgap=0, vgap=0) + cb_sizer.Add(self.cbCurveType , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbSub , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbAutoScale , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbLogX , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbLogY , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbStepPlot , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbXHair , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbGrid , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbSync , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbPlotMatrix, 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbMeasure , 0, flag=wx.ALL, border=1) + + self.ctrlPanel.SetSizer(cb_sizer) + + # --- Crosshair Panel + crossHairPanel= wx.Panel(self) + self.lbCrossHairX = wx.StaticText(crossHairPanel, -1, 'x = ... ') + self.lbCrossHairY = wx.StaticText(crossHairPanel, -1, 'y = ... ') + self.lbDeltaX = wx.StaticText(crossHairPanel, -1, ' ') + self.lbDeltaY = wx.StaticText(crossHairPanel, -1, ' ') + self.lbCrossHairX.SetFont(getMonoFont(self)) + self.lbCrossHairY.SetFont(getMonoFont(self)) + self.lbDeltaX.SetFont(getMonoFont(self)) + self.lbDeltaY.SetFont(getMonoFont(self)) + cbCH = wx.FlexGridSizer(rows=4, cols=1, hgap=0, vgap=0) + cbCH.Add(self.lbCrossHairX , 0, flag=wx.ALL, border=1) + cbCH.Add(self.lbCrossHairY , 0, flag=wx.ALL, border=1) + cbCH.Add(self.lbDeltaX , 0, flag=wx.ALL, border=1) + cbCH.Add(self.lbDeltaY , 0, flag=wx.ALL, border=1) + crossHairPanel.SetSizer(cbCH) + + # --- layout of panels + row_sizer = wx.BoxSizer(wx.HORIZONTAL) + sl2 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) + sl3 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) + sl4 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) + row_sizer.Add(self.pltTypePanel , 0 , flag=wx.LEFT|wx.RIGHT|wx.CENTER , border=1) + row_sizer.Add(sl2 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) + row_sizer.Add(self.toolbar_sizer, 0 , flag=wx.LEFT|wx.RIGHT|wx.CENTER , border=1) + row_sizer.Add(sl3 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) + row_sizer.Add(self.ctrlPanel , 1 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) + row_sizer.Add(sl4 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) + row_sizer.Add(crossHairPanel , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=1) + + plotsizer = wx.BoxSizer(wx.VERTICAL) + self.slCtrl = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) + self.slCtrl.Hide() + self.slEsth = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) + self.slEsth.Hide() + sl1 = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) + plotsizer.Add(self.toolSizer,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.canvas ,1,flag = wx.EXPAND,border = 5 ) + plotsizer.Add(sl1 ,0,flag = wx.EXPAND,border = 0) + plotsizer.Add(self.spcPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.pdfPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.cmpPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.mmxPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.slEsth ,0,flag = wx.EXPAND,border = 0) + plotsizer.Add(self.esthPanel,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.slCtrl ,0,flag = wx.EXPAND,border = 0) + plotsizer.Add(row_sizer ,0,flag = wx.EXPAND|wx.NORTH ,border = 2) + + self.show_hide(self.spcPanel, self.pltTypePanel.cbFFT.GetValue()) + self.show_hide(self.cmpPanel, self.pltTypePanel.cbCompare.GetValue()) + self.show_hide(self.pdfPanel, self.pltTypePanel.cbPDF.GetValue()) + self.show_hide(self.mmxPanel, self.pltTypePanel.cbMinMax.GetValue()) + + self.SetSizer(plotsizer) + self.plotsizer=plotsizer; + self.set_subplot_spacing(init=True) + + def onEsthToggle(self,event): + self.esthToggle=not self.esthToggle + if self.esthToggle: + self.slCtrl.Show() + self.esthPanel.Show() + else: + self.slCtrl.Hide() + self.esthPanel.Hide() + self.plotsizer.Layout() + event.Skip() + + def set_subplot_spacing(self, init=False): + """ + Handle default subplot spacing + + NOTE: + - Tight fails when the ylabel is too long, especially for fft with multiplt signals + - might need to change depending on window size/resizing + - need to change if right axis needed + - this will override the user settings + """ + #self.fig.set_tight_layout(True) # NOTE: works almost fine, but problem with FFT multiple + # TODO this is definitely not generic, but tight fails.. + if init: + # NOTE: at init size is (20,20) because sizer is not initialized yet + bottom = 0.12 + left = 0.12 + else: + if self.Size[1]<300: + bottom=0.20 + elif self.Size[1]<350: + bottom=0.18 + elif self.Size[1]<430: + bottom=0.16 + elif self.Size[1]<600: + bottom=0.13 + elif self.Size[1]<800: + bottom=0.09 + else: + bottom=0.07 + if self.Size[0]<300: + left=0.22 + elif self.Size[0]<450: + left=0.20 + elif self.Size[0]<950: + left=0.12 + else: + left=0.06 + #print(self.Size,'bottom', bottom, 'left',left) + if self.cbPlotMatrix.GetValue(): # TODO detect it + self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98-left) + else: + self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98) + + def plot_matrix_select(self, event): + self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) + self.redraw_same_data() + + def measure_select(self, event): + if self.cbMeasure.IsChecked(): + self.cbAutoScale.SetValue(False) + self.redraw_same_data() + + def redraw_event(self, event): + self.redraw_same_data() + + def log_select(self, event): + if self.pltTypePanel.cbPDF.GetValue(): + self.cbLogX.SetValue(False) + self.cbLogY.SetValue(False) + else: + self.redraw_same_data() + + def crosshair_event(self, event): + try: + self.multiCursors.vertOn =self.cbXHair.GetValue() + self.multiCursors.horizOn=self.cbXHair.GetValue() + self.multiCursors._update() + except: + pass + + def show_hide(self,panel,bShow): + if bShow: + panel.Show() + self.slEsth.Show() + else: + self.slEsth.Hide() + panel.Hide() + + @property + def sharex(self): + return self.cbSync.IsChecked() and (not self.pltTypePanel.cbPDF.GetValue()) + + def set_subplots(self,nPlots): + self.set_subplot_spacing() + # Creating subplots + for ax in self.fig.axes: + self.fig.delaxes(ax) + sharex=None + for i in range(nPlots): + # Vertical stack + if i==0: + ax=self.fig.add_subplot(nPlots,1,i+1) + if self.sharex: + sharex=ax + else: + ax=self.fig.add_subplot(nPlots,1,i+1,sharex=sharex) + # Horizontal stack + #self.fig.add_subplot(1,nPlots,i+1) + + def onMouseMove(self, event): + if event.inaxes: + x, y = event.xdata, event.ydata + self.lbCrossHairX.SetLabel('x =' + self.formatLabelValue(x)) + self.lbCrossHairY.SetLabel('y =' + self.formatLabelValue(y)) + + def onMouseClick(self, event): + self.clickLocation = (event.inaxes, event.xdata, event.ydata) + + def onMouseRelease(self, event): + if self.cbMeasure.GetValue(): + for ax, ax_idx in zip(self.fig.axes, range(len(self.fig.axes))): + if event.inaxes == ax: + x, y = event.xdata, event.ydata + if self.clickLocation != (ax, x, y): + # Ignore measurements for zoom-actions. Possibly add small tolerance. + # Zoom-actions disable autoscale + self.cbAutoScale.SetValue(False) + return + if event.button == 1: + self.infoPanel.setMeasurements((x, y), None) + self.leftMeasure.set(ax_idx, x, y) + self.leftMeasure.plot(ax, ax_idx) + elif event.button == 3: + self.infoPanel.setMeasurements(None, (x, y)) + self.rightMeasure.set(ax_idx, x, y) + self.rightMeasure.plot(ax, ax_idx) + else: + return + if not self.cbAutoScale.IsChecked(): + self._restore_limits() + + if self.leftMeasure.axis_idx == self.rightMeasure.axis_idx and self.leftMeasure.axis_idx != -1: + self.lbDeltaX.SetLabel('dx=' + self.formatLabelValue(self.rightMeasure.x - self.leftMeasure.x)) + self.lbDeltaY.SetLabel('dy=' + self.formatLabelValue(self.rightMeasure.y - self.leftMeasure.y)) + else: + self.lbDeltaX.SetLabel('') + self.lbDeltaY.SetLabel('') + return + + def onDraw(self, event): + self._store_limits() + + def formatLabelValue(self, value): + try: + if abs(value)<1000 and abs(value)>1e-4: + s = '{:10.5f}'.format(value) + else: + s = '{:10.3e}'.format(value) + except TypeError: + s = ' ' + return s + + def removeTools(self,event=None,Layout=True): + try: + self.toolPanel.destroy() # call the "destroy" function which might clean up data + except: + pass + try: + # Python3 + self.toolSizer.Clear(delete_windows=True) # Delete Windows + except: + # Python2 + if hasattr(self,'toolPanel'): + self.toolSizer.Remove(self.toolPanel) + self.toolPanel.Destroy() + del self.toolPanel + self.toolSizer.Clear() # Delete Windows + if Layout: + self.plotsizer.Layout() + + def showTool(self,toolName=''): + from .GUITools import TOOLS + if toolName in TOOLS.keys(): + self.showToolPanel(TOOLS[toolName]) + else: + raise Exception('Unknown tool {}'.format(toolName)) + + def showToolPanel(self, panelClass): + """ Show a tool panel based on a panel class (should inherit from GUIToolPanel)""" + from .GUITools import TOOLS + self.Freeze() + self.removeTools(Layout=False) + self.toolPanel=panelClass(parent=self) # calling the panel constructor + self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) + self.plotsizer.Layout() + self.Thaw() + + + def setPD_PDF(self,PD,c): + """ Convert plot data to PDF data based on GUI options""" + # ---PDF + nBins = self.pdfPanel.scBins.GetValue() + bSmooth = self.pdfPanel.cbSmooth.GetValue() + nBins_out= PD.toPDF(nBins,bSmooth) + if nBins_out!=nBins: + self.pdfPanel.scBins.SetValue(nBins) + + def setPD_MinMax(self,PD): + """ Convert plot data to MinMax data based on GUI options""" + yScale=self.mmxPanel.cbyMinMax.IsChecked() + xScale=self.mmxPanel.cbxMinMax.IsChecked() + try: + PD.toMinMax(xScale,yScale) + except Exception as e: + self.mmxPanel.cbxMinMax.SetValue(False) + raise e # Used to be Warn + + def setPD_FFT(self,pd): + """ Convert plot data to FFT data based on GUI options""" + yType = self.spcPanel.cbType.GetStringSelection() + xType = self.spcPanel.cbTypeX.GetStringSelection() + avgMethod = self.spcPanel.cbAveraging.GetStringSelection() + avgWindow = self.spcPanel.cbAveragingMethod.GetStringSelection() + bDetrend = self.spcPanel.cbDetrend.IsChecked() + nExp = self.spcPanel.scP2.GetValue() + nPerDecade = self.spcPanel.scP2.GetValue() + # Convert plotdata to FFT data + try: + Info = pd.toFFT(yType=yType, xType=xType, avgMethod=avgMethod, avgWindow=avgWindow, bDetrend=bDetrend, nExp=nExp, nPerDecade=nPerDecade) + # Trigger + if hasattr(Info,'nExp') and Info.nExp!=nExp: + self.spcPanel.scP2.SetValue(Info.nExp) + self.spcPanel.updateP2(Info.nExp) + except Exception as e: + self.spcPanel.Hide(); + self.plotsizer.Layout() + raise e + + + def transformPlotData(self,PD): + """" + Apply MinMax, PDF or FFT transform to plot based on GUI data + """ + plotType=self.pltTypePanel.plotType() + if plotType=='MinMax': + self.setPD_MinMax(PD) + elif plotType=='PDF': + self.setPD_PDF(PD,PD.c) + elif plotType=='FFT': + self.setPD_FFT(PD) + + def getPlotData(self,plotType): + ID,SameCol,selMode=self.selPanel.getPlotDataSelection() + self.selMode=selMode # we store the selection mode + del self.plotData + self.plotData=[] + tabs=self.selPanel.tabList.getTabs() # TODO, selPanel should just return the PlotData... + try: + for i,idx in enumerate(ID): + # Initialize each plotdata based on selected table and selected id channels + pd=PlotData(); + pd.fromIDs(tabs,i,idx,SameCol, self.plotDataOptions) + # Possible change of data + if plotType=='MinMax': + self.setPD_MinMax(pd) + elif plotType=='PDF': + self.setPD_PDF(pd,pd.c) + elif plotType=='FFT': + self.setPD_FFT(pd) + self.plotData.append(pd) + except Exception as e: + self.plotData=[] + raise e + + def PD_Compare(self,mode): + """ Perform comparison of the selected PlotData, returns new plotData with the comparison. """ + sComp = self.cmpPanel.rbType.GetStringSelection() + try: + self.plotData = compareMultiplePD(self.plotData,mode, sComp) + except Exception as e: + self.pltTypePanel.cbRegular.SetValue(True) + raise e + + def _onPlotMatrixLeftClick(self, event): + """Toggle plot-states from None, to left-axis, to right-axis. + Left-click goes forwards, right-click goes backwards. + IndexError to avoid "holes" in matrix with outer adjacent populated entries + """ + btn = event.GetEventObject() + label = btn.GetLabelText() + if label == '-': + btn.SetLabel('1') + try: + self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) + except IndexError: + btn.SetLabel('-') + elif label == '1': + btn.SetLabel('2') + else: + btn.SetLabel('-') + try: + self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) + except IndexError: + btn.SetLabel('1') + self.redraw_same_data() + + def _onPlotMatrixRightClick(self, event): + btn = event.GetEventObject() + label = btn.GetLabelText() + if label == '-': + btn.SetLabel('2') + try: + self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) + except IndexError: + btn.SetLabel('-') + elif label == '1': + btn.SetLabel('-') + try: + self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) + except IndexError: + btn.SetLabel('2') + else: + btn.SetLabel('1') + self.redraw_same_data() + + def set_axes_lim(self, PDs, axis): + """ + It's usually faster to set the axis limits first (before plotting) + and disable autoscaling. This way the limits are not recomputed when plot data are added. + Also, we already have computed the min and max, so we leverage that. + NOTE: + doesnt not work with strings + doesnt not work for FFT and compare + + INPUTS: + PDs: list of plot data + """ + # TODO option for tight axes + tight=False + + plotType=self.pltTypePanel.plotType() + if plotType in ['FFT','Compare']: + axis.autoscale(True, axis='both', tight=tight) + return + vXString=[PDs[i].xIsString for i in axis.iPD] + vYString=[PDs[i].yIsString for i in axis.iPD] + if not any(vXString) and not self.cbLogX.IsChecked(): + try: + xMin=np.min([PDs[i]._xMin[0] for i in axis.iPD]) + xMax=np.max([PDs[i]._xMax[0] for i in axis.iPD]) + if np.isclose(xMin,xMax): + delta=1 if np.isclose(xMax,0) else 0.1*xMax + else: + if tight: + delta=0 + else: + delta = (xMax-xMin)*pyplot_rc['axes.xmargin'] + axis.set_xlim(xMin-delta,xMax+delta) + axis.autoscale(False, axis='x', tight=False) + except: + pass + if not any(vYString) and not self.cbLogY.IsChecked(): + try: + yMin=np.min([PDs[i]._yMin[0] for i in axis.iPD]) + yMax=np.max([PDs[i]._yMax[0] for i in axis.iPD]) + delta = (yMax-yMin)*pyplot_rc['axes.ymargin'] + if np.isclose(yMin,yMax): + delta=1 if np.isclose(yMax,0) else 0.1*yMax + else: + if tight: + delta=0 + else: + delta = (yMax-yMin)*pyplot_rc['axes.xmargin'] + axis.set_ylim(yMin-delta,yMax+delta) + axis.autoscale(False, axis='y', tight=False) + except: + pass + + def plot_all(self, keep_limits=True): + self.multiCursors=[] + + if self.cbMeasure.GetValue() is False: + for measure in [self.leftMeasure, self.rightMeasure]: + measure.clear() + self.infoPanel.setMeasurements(None, None) + self.lbDeltaX.SetLabel('') + self.lbDeltaY.SetLabel('') + + axes=self.fig.axes + PD=self.plotData + + + # --- Plot options + bStep = self.cbStepPlot.IsChecked() + plot_options = dict() + plot_options['lw']=float(self.esthPanel.cbLW.Value) + plot_options['ms']=float(self.esthPanel.cbMS.Value) + if self.cbCurveType.Value=='Plain': + plot_options['LineStyles'] = ['-'] + plot_options['Markers'] = [''] + elif self.cbCurveType.Value=='LS': + plot_options['LineStyles'] = ['-','--','-.',':'] + plot_options['Markers'] = [''] + elif self.cbCurveType.Value=='Markers': + plot_options['LineStyles'] = [''] + plot_options['Markers'] = ['o','d','v','^','s'] + elif self.cbCurveType.Value=='Mix': # NOTE, can be improved + plot_options['LineStyles'] = ['-','--', '-','-','-'] + plot_options['Markers'] = ['' ,'' ,'o','^','s'] + else: + # Combination of linestyles markers, colors, etc. + # But at that stage, if the user really want this, then we can implement an option to set styles per plot. Not high priority. + raise Exception('Not implemented') + + + + # --- Font options + font_options = dict() + font_options_legd = dict() + font_options['size'] = int(self.esthPanel.cbFont.Value) # affect labels + font_options_legd['fontsize'] = int(self.esthPanel.cbLgdFont.Value) + needChineseFont = any([pd.needChineseFont for pd in PD]) + if needChineseFont and self.specialFont is not None: + font_options['fontproperties']= self.specialFont + font_options_legd['prop'] = self.specialFont + + # --- Loop on axes. Either use ax.iPD to chose the plot data, or rely on plotmatrix + for axis_idx, ax_left in enumerate(axes): + ax_right = None + # Checks + vDate=[PD[i].yIsDate for i in ax_left.iPD] + if any(vDate) and len(vDate)>1: + Error(self,'Cannot plot date and other value on the same axis') + return + + # Set limit before plot when possible, for optimization + self.set_axes_lim(PD, ax_left) + + # Actually plot + pm = self.infoPanel.getPlotMatrix(PD, self.cbSub.IsChecked()) + __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) + ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) + + self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) + for measure in [self.leftMeasure, self.rightMeasure]: + measure.plot(ax_left, axis_idx) + + # Log Axes + if self.cbLogX.IsChecked(): + try: + ax_left.set_xscale("log", nonpositive='clip') # latest + except: + ax_left.set_xscale("log", nonposx='clip') # legacy + + if self.cbLogY.IsChecked(): + if bAllNegLeft is False: + try: + ax_left.set_yscale("log", nonpositive='clip') # latest + except: + ax_left.set_yscale("log", nonposy='clip') + if bAllNegRight is False and ax_right is not None: + try: + ax_right.set_yscale("log", nonpositive='clip') # latest + except: + ax_left.set_yscale("log", nonposy='clip') # legacy + + # XLIM - TODO FFT ONLY NASTY + if self.pltTypePanel.cbFFT.GetValue(): + try: + if self.cbAutoScale.IsChecked(): + xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) + if xlim>0: + ax_left.set_xlim([0,xlim]) + pd=PD[ax_left.iPD[0]] + I=pd.x 0 and len(yleft_labels) <= 3: + ax_left.set_ylabel(' and '.join(yleft_labels), **font_options) + elif ax_left is not None: + ax_left.set_ylabel('') + if len(yright_labels) > 0 and len(yright_labels) <= 3: + ax_right.set_ylabel(' and '.join(yright_labels), **font_options) + elif ax_right is not None: + ax_right.set_ylabel('') + + # Legends + lgdLoc = self.esthPanel.cbLegend.Value.lower() + if (self.pltTypePanel.cbCompare.GetValue() or + ((len(yleft_legends) + len(yright_legends)) > 1)): + if lgdLoc !='none': + if len(yleft_legends) > 0: + ax_left.legend(fancybox=False, loc=lgdLoc, **font_options_legd) + if ax_right is not None and len(yright_legends) > 0: + ax_right.legend(fancybox=False, loc=4, **font_options_legd) + elif len(axes)>1 and len(axes)==len(PD): + # TODO: can this be removed? If there is only one unique signal + # per subplot, normally only ylabel is displayed and no legend. + # Special case when we have subplots and all plots have the same label + if lgdLoc !='none': + usy = unique([pd.sy for pd in PD]) + if len(usy)==1: + for ax in axes: + ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) + + axes[-1].set_xlabel(PD[axes[-1].iPD[0]].sx, **font_options) + + #print('sy :',[pd.sy for pd in PD]) + #print('syl:',[pd.syl for pd in PD]) + + # --- Cursors for each individual plot + # NOTE: cursors needs to be stored in the object! + #for ax_left in self.fig.axes: + # self.cursors.append(MyCursor(ax_left,horizOn=True, vertOn=False, useblit=True, color='gray', linewidth=0.5, linestyle=':')) + # Vertical cusor for all, commonly + bXHair = self.cbXHair.GetValue() + self.multiCursors = MyMultiCursor(self.canvas, tuple(self.fig.axes), useblit=True, horizOn=bXHair, vertOn=bXHair, color='gray', linewidth=0.5, linestyle=':') + + def plotSignals(self, ax, axis_idx, PD, pm, left_right, is_step, opts): + axis = None + bAllNeg = True + if pm is None: + loop_range = ax.iPD + else: + loop_range = range(len(PD)) + + iPlot=-1 + for signal_idx in loop_range: + do_plot = False + if left_right == 1 and (pm is None or pm[signal_idx][axis_idx] == left_right): + do_plot = True + axis = ax + elif left_right == 2 and pm is not None and pm[signal_idx][axis_idx] == left_right: + do_plot = True + if axis is None: + axis = ax.twinx() + ax.set_zorder(axis.get_zorder()+1) + ax.patch.set_visible(False) + axis._get_lines.prop_cycler = ax._get_lines.prop_cycler + pd=PD[signal_idx] + if do_plot: + iPlot+=1 + # --- styling per plot + if len(pd.x)==1: + marker='o'; ls='' + else: + # TODO allow PlotData to override for "per plot" options in the future + marker = opts['Markers'][np.mod(iPlot,len(opts['Markers']))] + ls = opts['LineStyles'][np.mod(iPlot,len(opts['LineStyles']))] + if is_step: + plot = axis.step + else: + plot = axis.plot + plot(pd.x,pd.y,label=pd.syl,ms=opts['ms'], lw=opts['lw'], marker=marker, ls=ls) + try: + bAllNeg = bAllNeg and all(pd.y<=0) + except: + pass # Dates or strings + return axis, bAllNeg + + def findPlotMode(self,PD): + uTabs = unique([pd.it for pd in PD]) + usy = unique([pd.sy for pd in PD]) + uiy = unique([pd.iy for pd in PD]) + if len(uTabs)<=0: + raise Exception('No Table. Contact developer') + if len(uTabs)==1: + mode='1Tab_nCols' + else: + if PD[0].SameCol: + mode='nTabs_SameCols' + else: + # Now that we allow multiple selections detecting "simColumns" is more difficult + if len(uTabs) == len(PD): + mode='nTabs_1Col' + elif self.selMode=='simColumnsMode': + mode='nTabs_SimCols' + else: + mode='nTabs_mCols' + return mode + + def findSubPlots(self,PD,mode): + uTabs = unique([pd.it for pd in PD]) + usy = unique([pd.sy for pd in PD]) + bSubPlots = self.cbSub.IsChecked() + bCompare = self.pltTypePanel.cbCompare.GetValue() # NOTE bCompare somehow always 1Tab_nCols + nSubPlots=1 + spreadBy='none' + self.infoPanel.setTabMode(mode) + if mode=='1Tab_nCols': + if bSubPlots: + if bCompare or len(uTabs)==1: + nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) + else: + nSubPlots=len(usy) + spreadBy='iy' + elif mode=='nTabs_SameCols': + if bSubPlots: + if bCompare: + print('>>>TODO ',mode,len(usy),len(uTabs)) + else: + if len(usy)==1: + # Temporary hack until we have an option for spread by tabs or col + nSubPlots=len(uTabs) + spreadBy='it' + else: + nSubPlots=len(usy) + spreadBy='iy' + elif mode=='nTabs_SimCols': + if bSubPlots: + if bCompare: + print('>>>TODO ',mode,len(usy),len(uTabs)) + else: + nSubPlots=int(len(PD)/len(uTabs)) + spreadBy='mod-ip' + elif mode=='nTabs_mCols': + if bSubPlots: + if bCompare: + print('>>>TODO ',mode,len(usy),len(uTabs)) + else: + if bCompare or len(uTabs)==1: + nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) + else: + nSubPlots=len(PD) + spreadBy='mod-ip' + elif mode=='nTabs_1Col': + if bSubPlots: + if bCompare: + print('>>> TODO',mode,len(uTabs)) + else: + nSubPlots=len(uTabs) + spreadBy='it' + else: + raise Exception('Unknown mode, contact developer.') + return nSubPlots,spreadBy + + def distributePlots(self,mode,nSubPlots,spreadBy): + """ Assigns plot data to axes and axes to plot data """ + axes=self.fig.axes + + # Link plot data to axes + if nSubPlots==1 or spreadBy=='none': + axes[0].iPD=[i for i in range(len(self.plotData))] + else: + for ax in axes: + ax.iPD=[] + PD=self.plotData + uTabs=unique([pd.it for pd in PD]) + uiy=unique([pd.iy for pd in PD]) + if spreadBy=='iy': + for ipd,pd in enumerate(PD): + i=uiy.index(pd.iy) + if i < len(axes): + axes[i].iPD.append(ipd) + elif spreadBy=='it': + for ipd,pd in enumerate(PD): + i=uTabs.index(pd.it) + axes[i].iPD.append(ipd) + elif spreadBy=='mod-ip': + for ipd,pd in enumerate(PD): + i=np.mod(ipd, nSubPlots) + axes[i].iPD.append(ipd) + else: + raise Exception('Wrong spreadby value') + + def setLegendLabels(self,mode): + """ Set labels for legend """ + if mode=='1Tab_nCols': + for pd in self.plotData: + if self.pltTypePanel.cbMinMax.GetValue(): + pd.syl = no_unit(pd.sy) + else: + pd.syl = pd.sy + + elif mode=='nTabs_SameCols': + for pd in self.plotData: + pd.syl=pd.st + + elif mode=='nTabs_1Col': + usy=unique([pd.sy for pd in self.plotData]) + if len(usy)==1: + for pd in self.plotData: + pd.syl=pd.st + else: + for pd in self.plotData: + if self.pltTypePanel.cbMinMax.GetValue(): + pd.syl=no_unit(pd.sy) + else: + pd.syl=pd.sy #pd.syl=pd.st + ' - '+pd.sy + elif mode=='nTabs_SimCols': + bSubPlots = self.cbSub.IsChecked() + if bSubPlots: # spread by table name + for pd in self.plotData: + pd.syl=pd.st + else: + for pd in self.plotData: + pd.syl=pd.st + ' - '+pd.sy + elif mode=='nTabs_mCols': + usy=unique([pd.sy for pd in self.plotData]) + bSubPlots = self.cbSub.IsChecked() + if bSubPlots and len(usy)==1: # spread by table name + for pd in self.plotData: + pd.syl=pd.st + else: + for pd in self.plotData: + pd.syl=pd.st + ' - '+pd.sy + else: + raise Exception('Unknown mode {}'.format(mode)) + + + def empty(self): + self.cleanPlot() + + def clean_memory(self): + if hasattr(self,'plotData'): + del self.plotData + self.plotData=[] + for ax in self.fig.axes: + ax.iPD=[] + self.fig.delaxes(ax) + gc.collect() + + def clean_memory_plot(self): + pass + + def cleanPlot(self): + for ax in self.fig.axes: + if hasattr(ax,'iPD'): + del ax.iPD + self.fig.delaxes(ax) + gc.collect() + self.fig.add_subplot(111) + ax = self.fig.axes[0] + ax.set_axis_off() + #ax.plot(1,1) + self.canvas.draw() + gc.collect() + + def load_and_draw(self): + """ Full draw event: + - Get plot data based on selection + - Plot them + - Trigger changes to infoPanel + + """ + self.clean_memory() + self.getPlotData(self.pltTypePanel.plotType()) + if len(self.plotData)==0: + self.cleanPlot(); + return + mode=self.findPlotMode(self.plotData) + if self.pltTypePanel.cbCompare.GetValue(): + self.PD_Compare(mode) + if len(self.plotData)==0: + self.cleanPlot(); + return + self.redraw_same_data() + if self.infoPanel is not None: + self.infoPanel.showStats(self.plotData,self.pltTypePanel.plotType()) + + def redraw_same_data(self, keep_limits=True): + if len(self.plotData)==0: + self.cleanPlot(); + return + elif len(self.plotData) == 1: + if self.plotData[0].xIsString or self.plotData[0].yIsString or self.plotData[0].xIsDate or self.plotData[0].yIsDate: + self.cbAutoScale.SetValue(True) + else: + if len(self.xlim_prev)==0: # Might occur if some date didn't plot before (e.g. strings) + self.cbAutoScale.SetValue(True) + elif rectangleOverlap(self.plotData[0]._xMin[0], self.plotData[0]._yMin[0], + self.plotData[0]._xMax[0], self.plotData[0]._yMax[0], + self.xlim_prev[0][0], self.ylim_prev[0][0], + self.xlim_prev[0][1], self.ylim_prev[0][1]): + pass + else: + self.cbAutoScale.SetValue(True) + + mode=self.findPlotMode(self.plotData) + nPlots,spreadBy=self.findSubPlots(self.plotData,mode) + + self.clean_memory_plot() + self.set_subplots(nPlots) + self.distributePlots(mode,nPlots,spreadBy) + + if not self.pltTypePanel.cbCompare.GetValue(): + self.setLegendLabels(mode) + + self.plot_all(keep_limits) + self.canvas.draw() + + + def _store_limits(self): + self.xlim_prev = [] + self.ylim_prev = [] + for ax in self.fig.axes: + self.xlim_prev.append(ax.get_xlim()) + self.ylim_prev.append(ax.get_ylim()) + + def _restore_limits(self): + for ax, xlim, ylim in zip(self.fig.axes, self.xlim_prev, self.ylim_prev): + ax.set_xlim(xlim) + ax.set_ylim(ylim) + + +if __name__ == '__main__': + import pandas as pd; + from Tables import Table,TableList + + app = wx.App(False) + self=wx.Frame(None,-1,"Title") + self.SetSize((800, 600)) + #self.SetBackgroundColour('red') + class FakeSelPanel(wx.Panel): + def __init__(self, parent): + super(FakeSelPanel,self).__init__(parent) + d ={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0,'ColC':np.random.normal(0,1,100)+1} + df = pd.DataFrame(data=d) + self.tabList=TableList([Table(data=df)]) + + def getPlotDataSelection(self): + ID=[] + ID.append([0,0,2,'x','ColB','tab']) + ID.append([0,0,3,'x','ColC','tab']) + return ID,True + + selpanel=FakeSelPanel(self) + # selpanel.SetBackgroundColour('blue') + p1=PlotPanel(self,selpanel) + p1.load_and_draw() + #p1=SpectralCtrlPanel(self) + sizer = wx.BoxSizer(wx.VERTICAL) + sizer.Add(selpanel,0, flag = wx.EXPAND|wx.ALL,border = 10) + sizer.Add(p1,1, flag = wx.EXPAND|wx.ALL,border = 10) + self.SetSizer(sizer) + + self.Center() + self.Layout() + self.SetSize((800, 600)) + self.Show() + self.SendSizeEvent() + + #p1.showStats(None,[tab],[0],[0,1],tab.columns,0,erase=False) + + app.MainLoop() + + diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index d7177aa..a623bee 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -1,1324 +1,1324 @@ -import wx -import platform -try: - from .common import * - from .GUICommon import * - from .GUIMultiSplit import MultiSplit - from .GUIToolBox import GetKeyString -except: - raise -# from common import * -# from GUICommon import * -# from GUIMultiSplit import MultiSplit - - -__all__ = ['ColumnPanel', 'TablePanel', 'SelectionPanel','SEL_MODES','SEL_MODES_ID','TablePopup','ColumnPopup'] - -SEL_MODES = ['auto','Same tables' ,'Sim. tables' ,'2 tables','3 tables (exp.)' ] -SEL_MODES_ID = ['auto','sameColumnsMode','simColumnsMode','twoColumnsMode' ,'threeColumnsMode' ] - -def ireplace(text, old, new): - """ Replace case insensitive """ - try: - index_l = text.lower().index(old.lower()) - return text[:index_l] + new + text[index_l + len(old):] - except: - return text - - -# --------------------------------------------------------------------------------} -# --- Formula diagog -# --------------------------------------------------------------------------------{ -class FormulaDialog(wx.Dialog): - def __init__(self, title='', name='', formula='',columns=[],unit='',xcol='',xunit=''): - wx.Dialog.__init__(self, None, title=title) - # --- Data - self.unit=unit.strip().replace(' ','') - self.columns=['{'+c+'}' for c in columns] - self.xcol='{'+xcol+'}' - self.xunit=xunit.strip().replace(' ','') - if len(formula)==0: - formula=' + '.join(self.columns) - if len(name)==0: - name=self.getDefaultName() - self.formula_in=formula - - - quick_lbl = wx.StaticText(self, label="Predefined: " ) - self.cbQuick = wx.ComboBox(self, choices=['None','x 1000','/ 1000','deg2rad','rad2deg','rpm2radps','radps2rpm','norm','squared','d/dx'], style=wx.CB_READONLY) - self.cbQuick.SetSelection(0) - self.cbQuick.Bind(wx.EVT_COMBOBOX ,self.onQuickFormula) - - # Formula info - formula_lbl = wx.StaticText(self, label="Formula: ") - self.formula = wx.TextCtrl(self) - #self.formula.SetFont(getMonoFont(self)) - - self.formula.SetValue(formula) - formula_sizer = wx.BoxSizer(wx.HORIZONTAL) - formula_sizer.Add(formula_lbl ,0,wx.ALL|wx.RIGHT|wx.CENTER,5) - formula_sizer.Add(self.formula,1,wx.ALL|wx.EXPAND|wx.CENTER,5) - formula_sizer.Add(quick_lbl ,0,wx.ALL|wx.CENTER,5) - formula_sizer.Add(self.cbQuick,0,wx.ALL|wx.CENTER,5) - - - # name info - name_lbl = wx.StaticText(self, label="New name: " ) - self.name = wx.TextCtrl(self, size=wx.Size(200,-1)) - self.name.SetValue(name) - #self.name.SetFont(getMonoFont(self)) - name_sizer = wx.BoxSizer(wx.HORIZONTAL) - name_sizer.Add(name_lbl ,0,wx.ALL|wx.RIGHT|wx.CENTER,5) - name_sizer.Add(self.name,0,wx.ALL|wx.CENTER,5) - - info ='The formula needs to have a valid python syntax for an array manipulation. The available arrays are \n' - info+='the columns of the current table. The column names (without units) are surrounded by curly brackets.\n' - info+='You have access to numpy using `np`.\n\n' - info+='For instance, if you have two columns called `ColA [m]` and `ColB [m]` you can use:\n' - info+=' - ` {ColA} + {ColB} `\n' - info+=' - ` np.sqrt( {ColA}**2/1000 + 1/{ColB}**2 ) `\n' - info+=' - ` np.sin ( {ColA}*2*np.pi + {ColB} ) `\n' - help_lbl = wx.StaticText(self, label='Help: ') - info_lbl = wx.StaticText(self, label=info) - help_sizer = wx.BoxSizer(wx.HORIZONTAL) - help_sizer.Add(help_lbl ,0,wx.ALL|wx.RIGHT|wx.TOP,5) - help_sizer.Add(info_lbl ,0,wx.ALL|wx.TOP,5) - - - - self.btOK = wx.Button(self, wx.ID_OK)#, label = "OK" ) - btCL = wx.Button(self,label = "Cancel") - bt_sizer = wx.BoxSizer(wx.HORIZONTAL) - bt_sizer.Add(self.btOK, 0 ,wx.ALL,5) - bt_sizer.Add(btCL, 0 ,wx.ALL,5) - #btOK.Bind(wx.EVT_BUTTON,self.onOK ) - btCL.Bind(wx.EVT_BUTTON,self.onCancel) - - - main_sizer = wx.BoxSizer(wx.VERTICAL) - #main_sizer.Add(quick_sizer ,0,wx.ALL|wx.EXPAND,5) - main_sizer.Add(formula_sizer,0,wx.ALL|wx.EXPAND,5) - main_sizer.Add(name_sizer ,0,wx.ALL|wx.EXPAND,5) - main_sizer.Add(help_sizer ,0 ,wx.ALL|wx.CENTER, 5) - main_sizer.Add(bt_sizer ,0, wx.ALL|wx.CENTER, 5) - self.SetSizer(main_sizer) - self.Fit() - - def stripBrackets(self,s): - return s.replace('{','').replace('}','') - - def getOneColName(self): - if len(self.columns)>0: - return self.columns[-1] - else: - return '' - - def get_unit(self): - if len(self.unit)>0: - return ' ['+self.unit+']' - else: - return '' - def get_squared_unit(self): - if len(self.unit)>0: - if self.unit[0].lower()=='-': - return ' [-]' - else: - return ' [('+self.unit+')^2]' - else: - return '' - def get_kilo_unit(self): - if len(self.unit)>0: - if len(self.unit)>=1: - if self.unit[0].lower()=='-': - return ' [-]' - elif self.unit[0].lower()=='G': - r='T' - elif self.unit[0].lower()=='M': - r='G' - elif self.unit[0]=='k': - r='M' - elif self.unit[0]=='m': - if len(self.unit)==1: - r='km' - elif self.unit[1]=='/': - r='km' - else: - r='' - else: - r='k'+self.unit[0] - return ' ['+r+self.unit[1:]+']' - else: - return ' [k'+self.unit+']' - else: - return '' - def get_milli_unit(self): - if len(self.unit)>=1: - if self.unit[0].lower()=='-': - return ' [-]' - elif self.unit[0].lower()=='T': - r='G' - elif self.unit[0]=='G': - r='M' - elif self.unit[0]=='M': - r='k' - elif self.unit[0].lower()=='k': - r='' - elif self.unit[0]=='m': - if len(self.unit)==1: - r='mm' - elif self.unit[1]=='/': - r='mm' - else: - r='mu' - else: - r='m'+self.unit[0] - - return ' ['+r+self.unit[1:]+']' - else: - return '' - def get_deriv_unit(self): - if self.unit==self.xunit: - return ' [-]' - else: - return ' ['+self.unit+'/'+self.xunit+']' - - def getDefaultName(self): - if len(self.columns)>0: - return self.stripBrackets(self.getOneColName())+' New'+self.get_unit() - else: - return '' - - def onQuickFormula(self, event): - i = self.cbQuick.GetSelection() - s = self.cbQuick.GetStringSelection() - if s=='None': - self.formula.SetValue(self.formula_in) - return - - #self.formula_in=self.formula.GetValue() - c1 = self.getOneColName() - n1 = self.stripBrackets(c1) - - if s=='x 1000': - self.formula.SetValue(c1+' * 1000') - self.name.SetValue(n1+'_x1000'+ self.get_milli_unit()) - elif s=='/ 1000': - self.formula.SetValue(c1+' / 1000') - self.name.SetValue(n1+'_/1000'+self.get_kilo_unit()) - elif s=='deg2rad': - self.formula.SetValue(c1+' *np.pi/180') - self.name.SetValue(n1+'_rad [rad]') - elif s=='rad2deg': - self.formula.SetValue(c1+' *180/np.pi') - self.name.SetValue(n1+'_deg [deg]') - elif s=='rpm2radps': - self.formula.SetValue(c1+' *2*np.pi/60') - self.name.SetValue(n1+'_radps [rad/s]') - elif s=='radps2rpm': - self.formula.SetValue(c1+' *60/(2*np.pi)') - self.name.SetValue(n1+'_rpm [rpm]') - elif s=='norm': - self.formula.SetValue('np.sqrt( '+'**2 + '.join(self.columns)+'**2 )') - self.name.SetValue(n1+'_norm'+self.get_unit()) - elif s=='squared': - self.formula.SetValue('**2 + '.join(self.columns)+'**2 ') - self.name.SetValue(n1+'^2'+self.get_squared_unit()) - elif s=='d/dx': - self.formula.SetValue('np.gradient( '+'+'.join(self.columns)+ ', '+self.xcol+' )') - nx = self.stripBrackets(self.xcol) - bDoNewName=True - if self.xunit=='s': - if n1.lower().find('speed')>=0: - n1=ireplace(n1,'speed','Acceleration') - bDoNewName=False - elif n1.lower().find('velocity')>=0: - n1=ireplace(n1,'velocity','Acceleration') - bDoNewName=False - elif n1.lower().find('vel')>=0: - n1=ireplace(n1,'vel','Acc') - bDoNewName=False - elif n1.lower().find('position')>=0: - n1=ireplace(n1,'position','speed') - bDoNewName=False - elif n1.lower().find('pos')>=0: - n1=ireplace(n1,'pos','Vel') - bDoNewName=False - else: - n1='d('+n1+')/dt' - else: - n1='d('+n1+')/d('+nx+')' - self.name.SetValue(n1+self.get_deriv_unit()) - else: - raise Exception('Unknown quick formula {}'.s) - - def onCancel(self, event): - self.Destroy() -# --------------------------------------------------------------------------------} -# --- Popup menus -# --------------------------------------------------------------------------------{ -class TablePopup(wx.Menu): - def __init__(self, mainframe, parent, fullmenu=False): - wx.Menu.__init__(self) - self.parent = parent # parent is listbox - self.mainframe = mainframe - self.ISel = self.parent.GetSelections() - - if fullmenu: - self.itNameFile = wx.MenuItem(self, -1, "Naming: by file names", kind=wx.ITEM_CHECK) - self.MyAppend(self.itNameFile) - self.Bind(wx.EVT_MENU, self.OnNaming, self.itNameFile) - self.Check(self.itNameFile.GetId(), self.parent.GetParent().tabList.Naming=='FileNames') # Checking the menu box - - item = wx.MenuItem(self, -1, "Sort by name") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.OnSort, item) - - item = wx.MenuItem(self, -1, "Add") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.mainframe.onAdd, item) - - if len(self.ISel)>0: - item = wx.MenuItem(self, -1, "Delete") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.OnDeleteTabs, item) - - if len(self.ISel)==1: - tabPanel=self.parent.GetParent() - if tabPanel.tabList.Naming!='FileNames': - item = wx.MenuItem(self, -1, "Rename") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.OnRenameTab, item) - - if len(self.ISel)==1: - item = wx.MenuItem(self, -1, "Export") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.OnExportTab, item) - - def MyAppend(self, item): - try: - self.Append(item) # python3 - except: - self.AppendItem(item) # python2 - - def OnNaming(self, event=None): - tabPanel=self.parent.GetParent() - if self.itNameFile.IsChecked(): - tabPanel.tabList.setNaming('FileNames') - else: - tabPanel.tabList.setNaming('Ellude') - - tabPanel.updateTabNames() - - def OnDeleteTabs(self, event): - self.mainframe.deleteTabs(self.ISel) - - def OnRenameTab(self, event): - oldName = self.parent.GetString(self.ISel[0]) - dlg = wx.TextEntryDialog(self.parent, 'New table name:', 'Rename table',oldName,wx.OK|wx.CANCEL) - dlg.CentreOnParent() - if dlg.ShowModal() == wx.ID_OK: - newName=dlg.GetValue() - self.mainframe.renameTable(self.ISel[0],newName) - - def OnExportTab(self, event): - self.mainframe.exportTab(self.ISel[0]); - - def OnSort(self, event): - self.mainframe.sortTabs() - -class ColumnPopup(wx.Menu): - def __init__(self, parent, fullmenu=False): - wx.Menu.__init__(self) - self.parent = parent - self.ISel = self.parent.lbColumns.GetSelections() - - self.itShowID = wx.MenuItem(self, -1, "Show ID", kind=wx.ITEM_CHECK) - self.MyAppend(self.itShowID) - self.Bind(wx.EVT_MENU, self.OnShowID, self.itShowID) - self.Check(self.itShowID.GetId(), self.parent.bShowID) - - if self.parent.tab is not None: # TODO otherwise - item = wx.MenuItem(self, -1, "Add") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.OnAddColumn, item) - - if len(self.ISel)==1 and self.ISel[0]>=0: - item = wx.MenuItem(self, -1, "Rename") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.OnRenameColumn, item) - if len(self.ISel) == 1 and any( - f['pos'] == self.ISel[0] for f in self.parent.tab.formulas): - item = wx.MenuItem(self, -1, "Edit") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.OnEditColumn, item) - if len(self.ISel)>=1 and self.ISel[0]>=0: - item = wx.MenuItem(self, -1, "Delete") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.OnDeleteColumn, item) - - def MyAppend(self, item): - try: - self.Append(item) # python3 - except: - self.AppendItem(item) # python2 - - def OnShowID(self, event=None): - self.parent.bShowID=self.itShowID.IsChecked() - xSel,ySel,_,_ = self.parent.getColumnSelection() - self.parent.setGUIColumns(xSel=xSel, ySel=ySel) - - def OnRenameColumn(self, event=None): - iFilt = self.ISel[0] - if self.parent.bShowID: - oldName = self.parent.lbColumns.GetString(iFilt)[4:] - else: - oldName = self.parent.lbColumns.GetString(iFilt) - dlg = wx.TextEntryDialog(self.parent, 'New column name:', 'Rename column',oldName,wx.OK|wx.CANCEL) - dlg.CentreOnParent() - if dlg.ShowModal() == wx.ID_OK: - newName=dlg.GetValue() - main=self.parent.mainframe - ITab,STab=main.selPanel.getSelectedTables() - # TODO adapt me for Sim. tables mode - iFull = self.parent.Filt2Full[iFilt] - if iFull>0: # Important since -1 would rename last column of table - if main.tabList.haveSameColumns(ITab): - for iTab,sTab in zip(ITab,STab): - main.tabList.get(iTab).renameColumn(iFull-1,newName) - else: - self.parent.tab.renameColumn(iFull-1,newName) - self.parent.updateColumn(iFilt,newName) #faster - self.parent.selPanel.updateLayout() - # a trigger for the plot is required but skipped for now - - def OnEditColumn(self, event): - main=self.parent.mainframe - if len(self.ISel) != 1: - raise ValueError('Only one signal can be edited!') - ITab, STab = main.selPanel.getSelectedTables() - for iTab,sTab in zip(ITab,STab): - if sTab == self.parent.tab.active_name: - for f in main.tabList.get(iTab).formulas: - if f['pos'] == self.ISel[0]: - sName = f['name'] - sFormula = f['formula'] - break - else: - raise ValueError('No formula found at {0} for table {1}!'.format(self.ISel[0], sTab)) - self.showFormulaDialog('Edit column', sName, sFormula) - - def OnDeleteColumn(self, event): - main=self.parent.mainframe - iX = self.parent.comboX.GetSelection() - ITab,STab=main.selPanel.getSelectedTables() - # TODO adapt me for Sim. tables mode - IFull = [self.parent.Filt2Full[iFilt]-1 for iFilt in self.ISel] - IFull = [iFull for iFull in IFull if iFull>=0] - if main.tabList.haveSameColumns(ITab): - for iTab,sTab in zip(ITab,STab): - main.tabList.get(iTab).deleteColumns(IFull) - else: - self.parent.tab.deleteColumns(IFull) - self.parent.setColumns() - self.parent.setGUIColumns(xSel=iX) - main.redraw() - - def OnAddColumn(self, event): - main=self.parent.mainframe - self.showFormulaDialog('Add a new column') - - def showFormulaDialog(self, title, name='', formula=''): - bValid=False - bCancelled=False - main=self.parent.mainframe - sName=name - sFormula=formula - - if self.parent.bShowID: - columns=[no_unit(self.parent.lbColumns.GetString(i)[4:]) for i in self.ISel] - else: - columns=[no_unit(self.parent.lbColumns.GetString(i)) for i in self.ISel] - if len(self.ISel)>0: - main_unit=unit(self.parent.lbColumns.GetString(self.ISel[-1])) - else: - main_unit='' - xcol = self.parent.comboX.GetStringSelection() - xunit = unit(xcol) - xcol = no_unit(xcol) - - while (not bValid) and (not bCancelled): - dlg = FormulaDialog(title=title,columns=columns,xcol=xcol,xunit=xunit,unit=main_unit,name=sName,formula=sFormula) - dlg.CentreOnParent() - if dlg.ShowModal()==wx.ID_OK: - sName = dlg.name.GetValue() - sFormula = dlg.formula.GetValue() - dlg.Destroy() - if len(self.ISel)>0: - iFilt=self.ISel[-1] - iFull=self.parent.Filt2Full[iFilt] - else: - iFull = -1 - - ITab,STab=main.selPanel.getSelectedTables() - #if main.tabList.haveSameColumns(ITab): - sError='' - nError=0 - haveSameColumns=main.tabList.haveSameColumns(ITab) - for iTab,sTab in zip(ITab,STab): - if haveSameColumns or self.parent.tab.active_name == sTab: - # apply formula to all tables with same columns, otherwise only to active table - if title.startswith('Edit'): - bValid=main.tabList.get(iTab).setColumnByFormula(sName,sFormula,iFull) - else: - bValid=main.tabList.get(iTab).addColumnByFormula(sName,sFormula,iFull) - if not bValid: - sError+='The formula didn''t eval for table {}\n'.format(sTab) - nError+=1 - if len(sError)>0: - Error(self.parent,sError) - if nErrorlen(self.columns): - print('',self.Filt2Full) - print('',self.columns) - raise Exception('Error in Filt2Full') - return self.columns[self.Filt2Full] - - def setReadOnly(self, tabLabel='', cols=[]): - """ Set this list of columns as readonly and non selectable """ - self.tab=None - self.bReadOnly=True - self.lb.SetLabel(tabLabel) - self.setColumns(columnNames=cols) - self.setGUIColumns() - self.lbColumns.Enable(True) - self.comboX.Enable(False) - self.lbColumns.SetSelection(-1) - self.bt.Enable(False) - self.bShowID=False - self.btClear.Enable(False) - self.btFilter.Enable(False) - self.tFilter.Enable(False) - self.tFilter.SetValue('') - - def setTab(self,tab=None,xSel=-1,ySel=[],colNames=None, tabLabel=''): - """ Set the table used for the columns, update the GUI """ - self.tab=tab; - self.lbColumns.Enable(True) - self.comboX.Enable(True) - self.bReadOnly=False - self.btClear.Enable(True) - self.btFilter.Enable(True) - self.tFilter.Enable(True) - self.bt.Enable(True) - if tab is not None: - self.Filt2Full=None # TODO - if tab.active_name!='default': - self.lb.SetLabel(' '+tab.active_name) - self.setColumns() - self.setGUIColumns(xSel=xSel, ySel=ySel) - else: - self.Filt2Full=None # TODO Decide whether filter should be applied... - self.lb.SetLabel(tabLabel) - self.setColumns(columnNames=colNames) - self.setGUIColumns(xSel=xSel, ySel=ySel) - - def updateColumn(self,i,newName): - """ Update of one column name - i: index in GUI - """ - iFull = self.Filt2Full[i] - if self.bShowID: - newName='{:03d} '.format(iFull)+newName - self.lbColumns.SetString(i,newName) - self.comboX.SetString (i,newName) - self.columns[iFull] = newName - - def Full2Filt(self,iFull): - try: - return self.Filt2Full.index(iFull) - except: - return -1 - - def setColumns(self, columnNames=None): - # Get columns from user inputs, or table, or stored. - if columnNames is not None: - # Populating based on user inputs.. - columns=columnNames - elif self.tab is None: - columns=self.columns - else: - # Populating based on table (safest if table was updated) - columns=['Index']+self.tab.columns - # Storing columns, considered as "Full" - self.columns=np.array(columns) - - def setGUIColumns(self, xSel=-1, ySel=[]): - """ Set GUI columns based on self.columns and potential filter """ - # Filtering columns if neeed - sFilt = self.tFilter.GetLineText(0).strip() - if len(sFilt)>0: - Lf, If = filter_list(self.columns, sFilt) - self.Filt2Full = If - else: - self.Filt2Full = list(np.arange(len(self.columns))) - columns=self.columns[self.Filt2Full] - - # GUI update - self.Freeze() - if self.bShowID: - columnsY= ['{:03d} '.format(i)+c for i,c in enumerate(columns)] - columnsX= ['{:03d} '.format(i)+c for i,c in enumerate(self.columns)] - else: - columnsY= columns - columnsX= self.columns - self.lbColumns.Set(columnsY) # potentially filterd - # Slow line for many columns - # NOTE: limiting to 300 for now.. I'm not sure anywant would want to scroll more than that - # Consider adding a "more button" - # see e.g. https://comp.soft-sys.wxwindows.narkive.com/gDfA1Ds5/long-load-time-in-wxpython - self.comboX.Set(columnsX[:300]) # non filtered - - # Set selection for y, if any, and considering filtering - for iFull in ySel: - if iFull=0: - iFilt = self.Full2Filt(iFull) - if iFilt>0: - self.lbColumns.SetSelection(iFilt) - self.lbColumns.EnsureVisible(iFilt) - if len(self.lbColumns.GetSelections())<=0: - self.lbColumns.SetSelection(self.getDefaultColumnY(self.tab,len(columnsY)-1)) - - # Set selection for x, if any, NOTE x is not filtered! - if (xSel<0) or xSel>len(columnsX): - self.comboX.SetSelection(self.getDefaultColumnX(self.tab,len(columnsX)-1)) - else: - self.comboX.SetSelection(xSel) - self.Thaw() - - def forceOneSelection(self): - ISel=self.lbColumns.GetSelections() - self.lbColumns.SetSelection(-1) - if len(ISel)>0: - self.lbColumns.SetSelection(ISel[0]) - - def forceZeroSelection(self): - self.lbColumns.SetSelection(-1) - - def empty(self): - self.lbColumns.Clear() - self.comboX.Clear() - self.lb.SetLabel('') - self.bReadOnly=False - self.lbColumns.Enable(False) - self.comboX.Enable(False) - self.bt.Enable(False) - self.tab=None - self.columns=[] - self.Filt2Full=None - self.btClear.Enable(False) - self.btFilter.Enable(False) - self.tFilter.Enable(False) - self.tFilter.SetValue('') - - def getColumnSelection(self): - iX = self.comboX.GetSelection() - if self.bShowID: - sX = self.comboX.GetStringSelection()[4:] - else: - sX = self.comboX.GetStringSelection() - IY = self.lbColumns.GetSelections() - if self.bShowID: - SY = [self.lbColumns.GetString(i)[4:] for i in IY] - else: - SY = [self.lbColumns.GetString(i) for i in IY] - iXFull = iX # NOTE: x is always in full - IYFull = [self.Filt2Full[iY] for iY in IY] - return iXFull,IYFull,sX,SY - - def onClearFilter(self, event=None): - self.tFilter.SetValue('') - self.onFilterChange() - - def onFilterChange(self, event=None): - xSel,ySel,_,_ = self.getColumnSelection() # (indices in full) - self.setGUIColumns(xSel=xSel, ySel=ySel) # <<< Filtering done here - self.triggerPlot() # Trigger a col selection event - - def onFilterKey(self, event=None): - s=GetKeyString(event) - if s=='ESCAPE' or s=='Ctrl+C': - self.onClearFilter() - event.Skip() - - def triggerPlot(self): - event=wx.PyCommandEvent(wx.EVT_LISTBOX.typeId, self.lbColumns.GetId()) - wx.PostEvent(self.GetEventHandler(), event) - - -# --------------------------------------------------------------------------------} -# --- Selection Panel -# --------------------------------------------------------------------------------{ -class SelectionPanel(wx.Panel): - """ Display options for the user to select data """ - def __init__(self, parent, tabList, mode='auto',mainframe=None): - # Superclass constructor - super(SelectionPanel,self).__init__(parent) - # DATA - self.mainframe = mainframe - self.tabList = None - self.itabForCol = None - self.parent = parent - self.tabSelections = {} - self.tabSelected = [] # NOTE only used to remember a selection after a reload - self.modeRequested = mode - self.currentMode = None - self.nSplits = -1 - - # GUI DATA - self.splitter = MultiSplit(self, style=wx.SP_LIVE_UPDATE) - self.splitter.SetMinimumPaneSize(70) - self.tabPanel = TablePanel (self.splitter,mainframe, tabList) - self.colPanel1 = ColumnPanel(self.splitter, self, mainframe); - self.colPanel2 = ColumnPanel(self.splitter, self, mainframe); - self.colPanel3 = ColumnPanel(self.splitter, self, mainframe); - self.tabPanel.Hide() - self.colPanel1.Hide() - self.colPanel2.Hide() - self.colPanel3.Hide() - - # Layout - self.updateLayout() - VertSizer = wx.BoxSizer(wx.VERTICAL) - VertSizer.Add(self.splitter, 2, flag=wx.EXPAND, border=0) - self.SetSizer(VertSizer) - - # TRIGGERS - self.setTables(tabList) - - def updateLayout(self,mode=None): - self.Freeze() - if mode is None: - mode=self.modeRequested - else: - self.modeRequested = mode - if mode=='auto': - self.autoMode() - elif mode=='sameColumnsMode': - self.sameColumnsMode() - elif mode=='simColumnsMode': - self.simColumnsMode() - elif mode=='twoColumnsMode': - self.twoColumnsMode() - elif mode=='threeColumnsMode': - self.threeColumnsMode() - else: - self.Thaw() - raise Exception('Wrong mode for selection layout: {}'.format(mode)) - self.Thaw() - - - def autoMode(self): - ISel=self.tabPanel.lbTab.GetSelections() - if self.tabList is not None: - if self.tabList.len()<=0: - self.nSplits=-1 - self.splitter.removeAll() - elif self.tabList.haveSameColumns(): - self.sameColumnsMode() - elif self.tabList.haveSameColumns(ISel): - # We don't do same column because we know at least one table is different - # to avoid "jumping" too much - self.twoColumnsMode() - else: - # See if tables are quite similar - IKeepPerTab, IMissPerTab, IDuplPerTab= getTabCommonColIndices([self.tabList.get(i) for i in ISel]) - if np.all(np.array([len(I) for I in IMissPerTab])<30) and np.all(np.array([len(I) for I in IKeepPerTab])>=2): - self.simColumnsMode() - elif len(ISel)==2: - self.twoColumnsMode() - elif len(ISel)==3: - self.threeColumnsMode() - else: - #self.simColumnsMode(self) - raise Exception('Too many panels selected with significant columns differences.') - - def sameColumnsMode(self): - self.currentMode = 'sameColumnsMode' - if self.nSplits==1: - return - if self.nSplits==0 and self.tabList.len()<=1: - return - self.splitter.removeAll() - if self.tabList is not None: - if self.tabList.len()>1: - self.splitter.AppendWindow(self.tabPanel) - self.splitter.AppendWindow(self.colPanel1) - if self.mainframe is not None: - self.mainframe.mainFrameUpdateLayout() - if self.tabList is not None: - if self.tabList.len()<=1: - self.nSplits=0 - else: - self.nSplits=1 - else: - self.nSplits=0 - - def simColumnsMode(self): - self.currentMode = 'simColumnsMode' - self.splitter.removeAll() - self.splitter.AppendWindow(self.tabPanel) - self.splitter.AppendWindow(self.colPanel2) - self.splitter.AppendWindow(self.colPanel1) - self.splitter.setEquiSash() - if self.nSplits<2 and self.mainframe is not None: - self.mainframe.mainFrameUpdateLayout() - self.nSplits=2 - - def twoColumnsMode(self): - self.currentMode = 'twoColumnsMode' - if self.nSplits==2: - return - self.splitter.removeAll() - self.splitter.AppendWindow(self.tabPanel) - self.splitter.AppendWindow(self.colPanel2) - self.splitter.AppendWindow(self.colPanel1) - self.splitter.setEquiSash() - if self.nSplits<2 and self.mainframe is not None: - self.mainframe.mainFrameUpdateLayout() - self.nSplits=2 - - def threeColumnsMode(self): - self.currentMode = 'threeColumnsMode' - if self.nSplits==3: - return - self.splitter.removeAll() - self.splitter.AppendWindow(self.tabPanel) - self.splitter.AppendWindow(self.colPanel3) - self.splitter.AppendWindow(self.colPanel2) - self.splitter.AppendWindow(self.colPanel1) - self.splitter.setEquiSash() - if self.mainframe is not None: - self.mainframe.mainFrameUpdateLayout() - self.nSplits=3 - - def setTables(self,tabList,update=False): - """ Set the list of tables. Keeping the selection if it's an update """ - # TODO PUT ME IN TABLE PANEL - # Find a better way to remember selection - #print('UPDATING TABLES') - # Emptying GUI - TODO only if needed - self.colPanel1.empty() - self.colPanel2.empty() - self.colPanel3.empty() - # Adding - self.tabList = tabList - self.tabPanel.tabList = self.tabList - tabnames = self.tabList.tabNames - self.tabPanel.updateTabNames() - for tn in tabnames: - if tn not in self.tabSelections.keys(): - self.tabSelections[tn]={'xSel':-1,'ySel':[]} - else: - pass # do nothing - - # Reselecting - if len(self.tabSelected)>0: - # Removed line below since two column mode implemented - #if not haveSameColumns(tabs,ISel): - # ISel=[ISel[0]] - for i in self.tabSelected: - if i0: - # Trigger - updating columns and layout - ISel=self.tabPanel.lbTab.GetSelections() - self.tabSelected=ISel - if self.currentMode=='simColumnsMode': - self.setColForSimTab(ISel) - else: - if len(ISel)==1: - self.setTabForCol(ISel[0],1) - elif len(ISel)==2: - self.setTabForCol(ISel[0],1) - self.setTabForCol(ISel[1],2) - elif len(ISel)==3: - self.setTabForCol(ISel[0],1) - self.setTabForCol(ISel[1],2) - self.setTabForCol(ISel[2],3) - else: # Likely all tables have the same columns - self.setTabForCol(ISel[0],1) - self.updateLayout(self.modeRequested) - - def setTabForCol(self,iTabSel,iPanel): - t = self.tabList.get(iTabSel) - ts = self.tabSelections[t.name] - if iPanel==1: - self.colPanel1.setTab(t,ts['xSel'],ts['ySel']) - elif iPanel==2: - self.colPanel2.setTab(t,ts['xSel'],ts['ySel']) - elif iPanel==3: - self.colPanel3.setTab(t,ts['xSel'],ts['ySel']) - else: - raise Exception('Wrong ipanel') - - def setColForSimTab(self,ISel): - """ Set column panels for similar tables """ - tabs = [self.tabList.get(i) for i in ISel] - IKeepPerTab, IMissPerTab, IDuplPerTab = getTabCommonColIndices(tabs) - LenMiss = np.array([len(I) for I in IMissPerTab]) - LenKeep = np.array([len(I) for I in IKeepPerTab]) - LenDupl = np.array([len(I) for I in IDuplPerTab]) - - ColInfo = ['Sim. table mode '] - ColInfo += [''] - if self.tabList.haveSameColumns(ISel): - if len(ISel)>1: - ColInfo += ['Columns identical',''] - else: - if (np.all(np.array(LenMiss)==0)): - ColInfo += ['Columns identical'] - ColInfo += ['Order different!'] - - ColInfo += ['','First difference:'] - ColInfo.append('----------------------------------') - bFirst=True - for it,t in enumerate(tabs): - print('IKeep',IKeepPerTab[it]) - if it==0: - continue - INotOrdered=[ii for i,ii in enumerate(IKeepPerTab[it]) if ii!=IKeepPerTab[0][i]] - print('INot',INotOrdered) - if len(INotOrdered)>0: - im=INotOrdered[0] - if bFirst: - ColInfo.append('{}:'.format(tabs[0].active_name)) - ColInfo.append('{:03d} {:s}'.format(im, tabs[0].columns[im])) - bFirst=False - ColInfo.append('{}:'.format(t.active_name)) - ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) - ColInfo.append('----------------------------------') - - else: - ColInfo += ['Columns different!'] - ColInfo += ['(similar: {})'.format(LenKeep[0])] - ColInfo += ['','Missing columns:'] - ColInfo.append('----------------------------------') - for it,t in enumerate(tabs): - ColInfo.append('{}:'.format(t.active_name)) - if len(IMissPerTab[it])==0: - ColInfo.append(' (None) ') - for im in IMissPerTab[it]: - ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) - ColInfo.append('----------------------------------') - - if (np.any(np.array(LenDupl)>0)): - if len(ISel)>1: - ColInfo += ['','Common duplicates:'] - else: - ColInfo += ['','Duplicates:'] - ColInfo.append('----------------------------------') - for it,t in enumerate(tabs): - ColInfo.append('{}:'.format(t.active_name)) - if len(IDuplPerTab[it])==0: - ColInfo.append(' (None) ') - for im in IDuplPerTab[it]: - ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) - ColInfo.append('----------------------------------') - - - colNames = ['Index'] + [tabs[0].columns[i] for i in IKeepPerTab[0]] - self.colPanel1.setTab(tab=None, colNames=colNames, tabLabel=' Tab. Intersection') - self.colPanel2.setReadOnly(' Tab. Difference', ColInfo) - self.IKeepPerTab=IKeepPerTab - - - - def selectDefaultTable(self): - # Selecting the first table - if self.tabPanel.lbTab.GetCount()>0: - self.tabPanel.lbTab.SetSelection(0) - self.tabSelected=[0] - else: - self.tabSelected=[] - - def tabSelectionChanged(self): - # TODO This can be cleaned-up and merged with updateLayout - #print('Tab selection change') - # Storing the previous selection - #self.printSelection() - self.saveSelection() # - #self.printSelection() - ISel=self.tabPanel.lbTab.GetSelections() - if len(ISel)>0: - if self.modeRequested=='auto': - self.autoMode() - if self.currentMode=='simColumnsMode':# and len(ISel)>1: - self.setColForSimTab(ISel) - self.tabSelected=self.tabPanel.lbTab.GetSelections() - return - - if self.tabList.haveSameColumns(ISel): - # Setting tab - self.setTabForCol(ISel[0],1) - self.colPanel2.empty() - self.colPanel3.empty() - else: - if self.nSplits==2: - if len(ISel)>2: - Error(self,'In this mode, only two tables can be selected. To compare three tables, uses the "3 different tables" mode. Otherwise the tables need to have the same columns.') - ISel=ISel[0:2] - self.tabPanel.lbTab.SetSelection(wx.NOT_FOUND) - for isel in ISel: - self.tabPanel.lbTab.SetSelection(isel) - self.colPanel3.empty() - elif self.nSplits==3: - if len(ISel)>3: - Error(self,'In this mode, only three tables can be selected. To compare more than three tables, the tables need to have the same columns.') - ISel=ISel[0:3] - self.tabPanel.lbTab.SetSelection(wx.NOT_FOUND) - for isel in ISel: - self.tabPanel.lbTab.SetSelection(isel) - else: - Error(self,'The tables selected have different columns.\n\nThis is not compatible with the "Same tables" mode. To compare them, chose one of the following mode: "2 tables", "3 tables" or "Sim. tables".') - self.colPanel2.empty() - self.colPanel3.empty() - # unselect all and select only the first one - ISel=[ISel[0]] - self.tabPanel.lbTab.SetSelection(wx.NOT_FOUND) - self.tabPanel.lbTab.SetSelection(ISel[0]) - for iPanel,iTab in enumerate(ISel): - self.setTabForCol(iTab,iPanel+1) - #print('>>>Updating tabSelected, from',self.tabSelected,'to',self.tabPanel.lbTab.GetSelections()) - self.tabSelected=self.tabPanel.lbTab.GetSelections() - - def colSelectionChanged(self): - """ Simple triggers when column selection is changed, NOTE: does not redraw """ - if self.currentMode=='simColumnsMode': - self.colPanel2.forceZeroSelection() - else: - if self.nSplits in [2,3]: - ISel=self.tabPanel.lbTab.GetSelections() - if self.tabList.haveSameColumns(ISel): - pass # TODO: this test is identical to onTabSelectionChange. Unification. - # elif len(ISel)==2: - # self.colPanel1.forceOneSelection() - # self.colPanel2.forceOneSelection() - # elif len(ISel)==3: - # self.colPanel1.forceOneSelection() - # self.colPanel2.forceOneSelection() - # self.colPanel3.forceOneSelection() - - def update_tabs(self, tabList): - self.setTables(tabList, update=True) - - def renameTable(self,iTab, oldName, newName): - #self.printSelection() - self.tabSelections[newName] = self.tabSelections.pop(oldName) - self.tabPanel.updateTabNames() - #self.printSelection() - - def saveSelection(self): - #self.ISel=self.tabPanel.lbTab.GetSelections() - ISel=self.tabSelected # - if self.tabList.haveSameColumns(ISel): - for ii in ISel: - t=self.tabList.get(ii) - self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() - self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() - else: - if len(ISel)>=1: - t=self.tabList.get(ISel[0]) - self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() - self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() - if len(ISel)>=2: - t=self.tabList.get(ISel[1]) - self.tabSelections[t.name]['xSel'] = self.colPanel2.comboX.GetSelection() - self.tabSelections[t.name]['ySel'] = self.colPanel2.lbColumns.GetSelections() - if len(ISel)>=3: - t=self.tabList.get(ISel[2]) - self.tabSelections[t.name]['xSel'] = self.colPanel3.comboX.GetSelection() - self.tabSelections[t.name]['ySel'] = self.colPanel3.lbColumns.GetSelections() - self.tabSelected = self.tabPanel.lbTab.GetSelections(); - - def printSelection(self): - print('Number of tabSelections stored:',len(self.tabSelections)) - TS=self.tabSelections - for i,tn in enumerate(self.tabList.tabNames): - if tn not in TS.keys(): - print('Tab',i,'>>> Name {} not found in selection'.format(tn)) - else: - print('Tab',i,'xSel:',TS[tn]['xSel'],'ySel:',TS[tn]['ySel'],'Name:',tn) - - def getPlotDataSelection(self): - ID = [] - SameCol=False - if self.tabList is not None and self.tabList.len()>0: - ITab,STab = self.getSelectedTables() - if self.currentMode=='simColumnsMode' and len(ITab)>1: - iiX1,IY1,ssX1,SY1 = self.colPanel1.getColumnSelection() - SameCol=False - for i,(itab,stab) in enumerate(zip(ITab,STab)): - IKeep=self.IKeepPerTab[i] - for j,(iiy,ssy) in enumerate(zip(IY1,SY1)): - if iiy==0: - iy = 0 - sy = ssy - else: - iy = IKeep[iiy-1]+1 - sy = self.tabList.get(itab).columns[IKeep[iiy-1]] - if iiX1==0: - iX1 = 0 - sX1 = ssX1 - else: - iX1 = IKeep[iiX1-1]+1 - sX1 = self.tabList.get(itab).columns[IKeep[iiX1-1]] - ID.append([itab,iX1,iy,sX1,sy,stab]) - else: - iX1,IY1,sX1,SY1 = self.colPanel1.getColumnSelection() - SameCol=self.tabList.haveSameColumns(ITab) - if self.nSplits in [0,1] or SameCol: - for i,(itab,stab) in enumerate(zip(ITab,STab)): - for j,(iy,sy) in enumerate(zip(IY1,SY1)): - ID.append([itab,iX1,iy,sX1,sy,stab]) - elif self.nSplits in [2,3]: - if len(ITab)>=1: - for j,(iy,sy) in enumerate(zip(IY1,SY1)): - ID.append([ITab[0],iX1,iy,sX1,sy,STab[0]]) - if len(ITab)>=2: - iX2,IY2,sX2,SY2 = self.colPanel2.getColumnSelection() - for j,(iy,sy) in enumerate(zip(IY2,SY2)): - ID.append([ITab[1],iX2,iy,sX2,sy,STab[1]]) - if len(ITab)>=3: - iX2,IY2,sX2,SY2 = self.colPanel3.getColumnSelection() - for j,(iy,sy) in enumerate(zip(IY2,SY2)): - ID.append([ITab[2],iX2,iy,sX2,sy,STab[2]]) - else: - raise Exception('Wrong number of splits {}'.format(self.nSplits)) - return ID,SameCol,self.currentMode - - def getSelectedTables(self): - I=self.tabPanel.lbTab.GetSelections() - S=[self.tabPanel.lbTab.GetString(i) for i in I] - return I,S - - def getAllTables(self): - I=range(self.tabPanel.lbTab.GetCount()) - S=[self.tabPanel.lbTab.GetString(i) for i in I] - return I,S - - def clean_memory(self): - self.colPanel1.empty() - self.colPanel2.empty() - self.colPanel3.empty() - self.tabPanel.empty() - del self.tabList - self.tabList=None - - @property - def xCol(self): - iX, _, sX, _ = self.colPanel1.getColumnSelection() - return iX,sX - - -if __name__ == '__main__': - import pandas as pd; - from Tables import Table - import numpy as np - - def OnTabPopup(event): - self.PopupMenu(TablePopup(self,selPanel.tabPanel.lbTab), event.GetPosition()) - - app = wx.App(False) - self=wx.Frame(None,-1,"Title") - tab=Table(data=pd.DataFrame(data={'ColA': np.random.normal(0,1,100)+1,'ColB':np.random.normal(0,1,100)+2})) - selPanel=SelectionPanel(self,[tab],mode='twoColumnsMode') - self.SetSize((800, 600)) - self.Center() - self.Show() - selPanel.tabPanel.lbTab.Bind(wx.EVT_RIGHT_DOWN, OnTabPopup) - - - app.MainLoop() - +import wx +import platform +try: + from .common import * + from .GUICommon import * + from .GUIMultiSplit import MultiSplit + from .GUIToolBox import GetKeyString +except: + raise +# from common import * +# from GUICommon import * +# from GUIMultiSplit import MultiSplit + + +__all__ = ['ColumnPanel', 'TablePanel', 'SelectionPanel','SEL_MODES','SEL_MODES_ID','TablePopup','ColumnPopup'] + +SEL_MODES = ['auto','Same tables' ,'Sim. tables' ,'2 tables','3 tables (exp.)' ] +SEL_MODES_ID = ['auto','sameColumnsMode','simColumnsMode','twoColumnsMode' ,'threeColumnsMode' ] + +def ireplace(text, old, new): + """ Replace case insensitive """ + try: + index_l = text.lower().index(old.lower()) + return text[:index_l] + new + text[index_l + len(old):] + except: + return text + + +# --------------------------------------------------------------------------------} +# --- Formula diagog +# --------------------------------------------------------------------------------{ +class FormulaDialog(wx.Dialog): + def __init__(self, title='', name='', formula='',columns=[],unit='',xcol='',xunit=''): + wx.Dialog.__init__(self, None, title=title) + # --- Data + self.unit=unit.strip().replace(' ','') + self.columns=['{'+c+'}' for c in columns] + self.xcol='{'+xcol+'}' + self.xunit=xunit.strip().replace(' ','') + if len(formula)==0: + formula=' + '.join(self.columns) + if len(name)==0: + name=self.getDefaultName() + self.formula_in=formula + + + quick_lbl = wx.StaticText(self, label="Predefined: " ) + self.cbQuick = wx.ComboBox(self, choices=['None','x 1000','/ 1000','deg2rad','rad2deg','rpm2radps','radps2rpm','norm','squared','d/dx'], style=wx.CB_READONLY) + self.cbQuick.SetSelection(0) + self.cbQuick.Bind(wx.EVT_COMBOBOX ,self.onQuickFormula) + + # Formula info + formula_lbl = wx.StaticText(self, label="Formula: ") + self.formula = wx.TextCtrl(self) + #self.formula.SetFont(getMonoFont(self)) + + self.formula.SetValue(formula) + formula_sizer = wx.BoxSizer(wx.HORIZONTAL) + formula_sizer.Add(formula_lbl ,0,wx.ALL|wx.RIGHT|wx.CENTER,5) + formula_sizer.Add(self.formula,1,wx.ALL|wx.EXPAND|wx.CENTER,5) + formula_sizer.Add(quick_lbl ,0,wx.ALL|wx.CENTER,5) + formula_sizer.Add(self.cbQuick,0,wx.ALL|wx.CENTER,5) + + + # name info + name_lbl = wx.StaticText(self, label="New name: " ) + self.name = wx.TextCtrl(self, size=wx.Size(200,-1)) + self.name.SetValue(name) + #self.name.SetFont(getMonoFont(self)) + name_sizer = wx.BoxSizer(wx.HORIZONTAL) + name_sizer.Add(name_lbl ,0,wx.ALL|wx.RIGHT|wx.CENTER,5) + name_sizer.Add(self.name,0,wx.ALL|wx.CENTER,5) + + info ='The formula needs to have a valid python syntax for an array manipulation. The available arrays are \n' + info+='the columns of the current table. The column names (without units) are surrounded by curly brackets.\n' + info+='You have access to numpy using `np`.\n\n' + info+='For instance, if you have two columns called `ColA [m]` and `ColB [m]` you can use:\n' + info+=' - ` {ColA} + {ColB} `\n' + info+=' - ` np.sqrt( {ColA}**2/1000 + 1/{ColB}**2 ) `\n' + info+=' - ` np.sin ( {ColA}*2*np.pi + {ColB} ) `\n' + help_lbl = wx.StaticText(self, label='Help: ') + info_lbl = wx.StaticText(self, label=info) + help_sizer = wx.BoxSizer(wx.HORIZONTAL) + help_sizer.Add(help_lbl ,0,wx.ALL|wx.RIGHT|wx.TOP,5) + help_sizer.Add(info_lbl ,0,wx.ALL|wx.TOP,5) + + + + self.btOK = wx.Button(self, wx.ID_OK)#, label = "OK" ) + btCL = wx.Button(self,label = "Cancel") + bt_sizer = wx.BoxSizer(wx.HORIZONTAL) + bt_sizer.Add(self.btOK, 0 ,wx.ALL,5) + bt_sizer.Add(btCL, 0 ,wx.ALL,5) + #btOK.Bind(wx.EVT_BUTTON,self.onOK ) + btCL.Bind(wx.EVT_BUTTON,self.onCancel) + + + main_sizer = wx.BoxSizer(wx.VERTICAL) + #main_sizer.Add(quick_sizer ,0,wx.ALL|wx.EXPAND,5) + main_sizer.Add(formula_sizer,0,wx.ALL|wx.EXPAND,5) + main_sizer.Add(name_sizer ,0,wx.ALL|wx.EXPAND,5) + main_sizer.Add(help_sizer ,0 ,wx.ALL|wx.CENTER, 5) + main_sizer.Add(bt_sizer ,0, wx.ALL|wx.CENTER, 5) + self.SetSizer(main_sizer) + self.Fit() + + def stripBrackets(self,s): + return s.replace('{','').replace('}','') + + def getOneColName(self): + if len(self.columns)>0: + return self.columns[-1] + else: + return '' + + def get_unit(self): + if len(self.unit)>0: + return ' ['+self.unit+']' + else: + return '' + def get_squared_unit(self): + if len(self.unit)>0: + if self.unit[0].lower()=='-': + return ' [-]' + else: + return ' [('+self.unit+')^2]' + else: + return '' + def get_kilo_unit(self): + if len(self.unit)>0: + if len(self.unit)>=1: + if self.unit[0].lower()=='-': + return ' [-]' + elif self.unit[0].lower()=='G': + r='T' + elif self.unit[0].lower()=='M': + r='G' + elif self.unit[0]=='k': + r='M' + elif self.unit[0]=='m': + if len(self.unit)==1: + r='km' + elif self.unit[1]=='/': + r='km' + else: + r='' + else: + r='k'+self.unit[0] + return ' ['+r+self.unit[1:]+']' + else: + return ' [k'+self.unit+']' + else: + return '' + def get_milli_unit(self): + if len(self.unit)>=1: + if self.unit[0].lower()=='-': + return ' [-]' + elif self.unit[0].lower()=='T': + r='G' + elif self.unit[0]=='G': + r='M' + elif self.unit[0]=='M': + r='k' + elif self.unit[0].lower()=='k': + r='' + elif self.unit[0]=='m': + if len(self.unit)==1: + r='mm' + elif self.unit[1]=='/': + r='mm' + else: + r='mu' + else: + r='m'+self.unit[0] + + return ' ['+r+self.unit[1:]+']' + else: + return '' + def get_deriv_unit(self): + if self.unit==self.xunit: + return ' [-]' + else: + return ' ['+self.unit+'/'+self.xunit+']' + + def getDefaultName(self): + if len(self.columns)>0: + return self.stripBrackets(self.getOneColName())+' New'+self.get_unit() + else: + return '' + + def onQuickFormula(self, event): + i = self.cbQuick.GetSelection() + s = self.cbQuick.GetStringSelection() + if s=='None': + self.formula.SetValue(self.formula_in) + return + + #self.formula_in=self.formula.GetValue() + c1 = self.getOneColName() + n1 = self.stripBrackets(c1) + + if s=='x 1000': + self.formula.SetValue(c1+' * 1000') + self.name.SetValue(n1+'_x1000'+ self.get_milli_unit()) + elif s=='/ 1000': + self.formula.SetValue(c1+' / 1000') + self.name.SetValue(n1+'_/1000'+self.get_kilo_unit()) + elif s=='deg2rad': + self.formula.SetValue(c1+' *np.pi/180') + self.name.SetValue(n1+'_rad [rad]') + elif s=='rad2deg': + self.formula.SetValue(c1+' *180/np.pi') + self.name.SetValue(n1+'_deg [deg]') + elif s=='rpm2radps': + self.formula.SetValue(c1+' *2*np.pi/60') + self.name.SetValue(n1+'_radps [rad/s]') + elif s=='radps2rpm': + self.formula.SetValue(c1+' *60/(2*np.pi)') + self.name.SetValue(n1+'_rpm [rpm]') + elif s=='norm': + self.formula.SetValue('np.sqrt( '+'**2 + '.join(self.columns)+'**2 )') + self.name.SetValue(n1+'_norm'+self.get_unit()) + elif s=='squared': + self.formula.SetValue('**2 + '.join(self.columns)+'**2 ') + self.name.SetValue(n1+'^2'+self.get_squared_unit()) + elif s=='d/dx': + self.formula.SetValue('np.gradient( '+'+'.join(self.columns)+ ', '+self.xcol+' )') + nx = self.stripBrackets(self.xcol) + bDoNewName=True + if self.xunit=='s': + if n1.lower().find('speed')>=0: + n1=ireplace(n1,'speed','Acceleration') + bDoNewName=False + elif n1.lower().find('velocity')>=0: + n1=ireplace(n1,'velocity','Acceleration') + bDoNewName=False + elif n1.lower().find('vel')>=0: + n1=ireplace(n1,'vel','Acc') + bDoNewName=False + elif n1.lower().find('position')>=0: + n1=ireplace(n1,'position','speed') + bDoNewName=False + elif n1.lower().find('pos')>=0: + n1=ireplace(n1,'pos','Vel') + bDoNewName=False + else: + n1='d('+n1+')/dt' + else: + n1='d('+n1+')/d('+nx+')' + self.name.SetValue(n1+self.get_deriv_unit()) + else: + raise Exception('Unknown quick formula {}'.s) + + def onCancel(self, event): + self.Destroy() +# --------------------------------------------------------------------------------} +# --- Popup menus +# --------------------------------------------------------------------------------{ +class TablePopup(wx.Menu): + def __init__(self, mainframe, parent, fullmenu=False): + wx.Menu.__init__(self) + self.parent = parent # parent is listbox + self.mainframe = mainframe + self.ISel = self.parent.GetSelections() + + if fullmenu: + self.itNameFile = wx.MenuItem(self, -1, "Naming: by file names", kind=wx.ITEM_CHECK) + self.MyAppend(self.itNameFile) + self.Bind(wx.EVT_MENU, self.OnNaming, self.itNameFile) + self.Check(self.itNameFile.GetId(), self.parent.GetParent().tabList.Naming=='FileNames') # Checking the menu box + + item = wx.MenuItem(self, -1, "Sort by name") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.OnSort, item) + + item = wx.MenuItem(self, -1, "Add") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.mainframe.onAdd, item) + + if len(self.ISel)>0: + item = wx.MenuItem(self, -1, "Delete") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.OnDeleteTabs, item) + + if len(self.ISel)==1: + tabPanel=self.parent.GetParent() + if tabPanel.tabList.Naming!='FileNames': + item = wx.MenuItem(self, -1, "Rename") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.OnRenameTab, item) + + if len(self.ISel)==1: + item = wx.MenuItem(self, -1, "Export") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.OnExportTab, item) + + def MyAppend(self, item): + try: + self.Append(item) # python3 + except: + self.AppendItem(item) # python2 + + def OnNaming(self, event=None): + tabPanel=self.parent.GetParent() + if self.itNameFile.IsChecked(): + tabPanel.tabList.setNaming('FileNames') + else: + tabPanel.tabList.setNaming('Ellude') + + tabPanel.updateTabNames() + + def OnDeleteTabs(self, event): + self.mainframe.deleteTabs(self.ISel) + + def OnRenameTab(self, event): + oldName = self.parent.GetString(self.ISel[0]) + dlg = wx.TextEntryDialog(self.parent, 'New table name:', 'Rename table',oldName,wx.OK|wx.CANCEL) + dlg.CentreOnParent() + if dlg.ShowModal() == wx.ID_OK: + newName=dlg.GetValue() + self.mainframe.renameTable(self.ISel[0],newName) + + def OnExportTab(self, event): + self.mainframe.exportTab(self.ISel[0]); + + def OnSort(self, event): + self.mainframe.sortTabs() + +class ColumnPopup(wx.Menu): + def __init__(self, parent, fullmenu=False): + wx.Menu.__init__(self) + self.parent = parent + self.ISel = self.parent.lbColumns.GetSelections() + + self.itShowID = wx.MenuItem(self, -1, "Show ID", kind=wx.ITEM_CHECK) + self.MyAppend(self.itShowID) + self.Bind(wx.EVT_MENU, self.OnShowID, self.itShowID) + self.Check(self.itShowID.GetId(), self.parent.bShowID) + + if self.parent.tab is not None: # TODO otherwise + item = wx.MenuItem(self, -1, "Add") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.OnAddColumn, item) + + if len(self.ISel)==1 and self.ISel[0]>=0: + item = wx.MenuItem(self, -1, "Rename") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.OnRenameColumn, item) + if len(self.ISel) == 1 and any( + f['pos'] == self.ISel[0] for f in self.parent.tab.formulas): + item = wx.MenuItem(self, -1, "Edit") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.OnEditColumn, item) + if len(self.ISel)>=1 and self.ISel[0]>=0: + item = wx.MenuItem(self, -1, "Delete") + self.MyAppend(item) + self.Bind(wx.EVT_MENU, self.OnDeleteColumn, item) + + def MyAppend(self, item): + try: + self.Append(item) # python3 + except: + self.AppendItem(item) # python2 + + def OnShowID(self, event=None): + self.parent.bShowID=self.itShowID.IsChecked() + xSel,ySel,_,_ = self.parent.getColumnSelection() + self.parent.setGUIColumns(xSel=xSel, ySel=ySel) + + def OnRenameColumn(self, event=None): + iFilt = self.ISel[0] + if self.parent.bShowID: + oldName = self.parent.lbColumns.GetString(iFilt)[4:] + else: + oldName = self.parent.lbColumns.GetString(iFilt) + dlg = wx.TextEntryDialog(self.parent, 'New column name:', 'Rename column',oldName,wx.OK|wx.CANCEL) + dlg.CentreOnParent() + if dlg.ShowModal() == wx.ID_OK: + newName=dlg.GetValue() + main=self.parent.mainframe + ITab,STab=main.selPanel.getSelectedTables() + # TODO adapt me for Sim. tables mode + iFull = self.parent.Filt2Full[iFilt] + if iFull>0: # Important since -1 would rename last column of table + if main.tabList.haveSameColumns(ITab): + for iTab,sTab in zip(ITab,STab): + main.tabList.get(iTab).renameColumn(iFull-1,newName) + else: + self.parent.tab.renameColumn(iFull-1,newName) + self.parent.updateColumn(iFilt,newName) #faster + self.parent.selPanel.updateLayout() + # a trigger for the plot is required but skipped for now + + def OnEditColumn(self, event): + main=self.parent.mainframe + if len(self.ISel) != 1: + raise ValueError('Only one signal can be edited!') + ITab, STab = main.selPanel.getSelectedTables() + for iTab,sTab in zip(ITab,STab): + if sTab == self.parent.tab.active_name: + for f in main.tabList.get(iTab).formulas: + if f['pos'] == self.ISel[0]: + sName = f['name'] + sFormula = f['formula'] + break + else: + raise ValueError('No formula found at {0} for table {1}!'.format(self.ISel[0], sTab)) + self.showFormulaDialog('Edit column', sName, sFormula) + + def OnDeleteColumn(self, event): + main=self.parent.mainframe + iX = self.parent.comboX.GetSelection() + ITab,STab=main.selPanel.getSelectedTables() + # TODO adapt me for Sim. tables mode + IFull = [self.parent.Filt2Full[iFilt]-1 for iFilt in self.ISel] + IFull = [iFull for iFull in IFull if iFull>=0] + if main.tabList.haveSameColumns(ITab): + for iTab,sTab in zip(ITab,STab): + main.tabList.get(iTab).deleteColumns(IFull) + else: + self.parent.tab.deleteColumns(IFull) + self.parent.setColumns() + self.parent.setGUIColumns(xSel=iX) + main.redraw() + + def OnAddColumn(self, event): + main=self.parent.mainframe + self.showFormulaDialog('Add a new column') + + def showFormulaDialog(self, title, name='', formula=''): + bValid=False + bCancelled=False + main=self.parent.mainframe + sName=name + sFormula=formula + + if self.parent.bShowID: + columns=[no_unit(self.parent.lbColumns.GetString(i)[4:]) for i in self.ISel] + else: + columns=[no_unit(self.parent.lbColumns.GetString(i)) for i in self.ISel] + if len(self.ISel)>0: + main_unit=unit(self.parent.lbColumns.GetString(self.ISel[-1])) + else: + main_unit='' + xcol = self.parent.comboX.GetStringSelection() + xunit = unit(xcol) + xcol = no_unit(xcol) + + while (not bValid) and (not bCancelled): + dlg = FormulaDialog(title=title,columns=columns,xcol=xcol,xunit=xunit,unit=main_unit,name=sName,formula=sFormula) + dlg.CentreOnParent() + if dlg.ShowModal()==wx.ID_OK: + sName = dlg.name.GetValue() + sFormula = dlg.formula.GetValue() + dlg.Destroy() + if len(self.ISel)>0: + iFilt=self.ISel[-1] + iFull=self.parent.Filt2Full[iFilt] + else: + iFull = -1 + + ITab,STab=main.selPanel.getSelectedTables() + #if main.tabList.haveSameColumns(ITab): + sError='' + nError=0 + haveSameColumns=main.tabList.haveSameColumns(ITab) + for iTab,sTab in zip(ITab,STab): + if haveSameColumns or self.parent.tab.active_name == sTab: + # apply formula to all tables with same columns, otherwise only to active table + if title.startswith('Edit'): + bValid=main.tabList.get(iTab).setColumnByFormula(sName,sFormula,iFull) + else: + bValid=main.tabList.get(iTab).addColumnByFormula(sName,sFormula,iFull) + if not bValid: + sError+='The formula didn''t eval for table {}\n'.format(sTab) + nError+=1 + if len(sError)>0: + Error(self.parent,sError) + if nErrorlen(self.columns): + print('',self.Filt2Full) + print('',self.columns) + raise Exception('Error in Filt2Full') + return self.columns[self.Filt2Full] + + def setReadOnly(self, tabLabel='', cols=[]): + """ Set this list of columns as readonly and non selectable """ + self.tab=None + self.bReadOnly=True + self.lb.SetLabel(tabLabel) + self.setColumns(columnNames=cols) + self.setGUIColumns() + self.lbColumns.Enable(True) + self.comboX.Enable(False) + self.lbColumns.SetSelection(-1) + self.bt.Enable(False) + self.bShowID=False + self.btClear.Enable(False) + self.btFilter.Enable(False) + self.tFilter.Enable(False) + self.tFilter.SetValue('') + + def setTab(self,tab=None,xSel=-1,ySel=[],colNames=None, tabLabel=''): + """ Set the table used for the columns, update the GUI """ + self.tab=tab; + self.lbColumns.Enable(True) + self.comboX.Enable(True) + self.bReadOnly=False + self.btClear.Enable(True) + self.btFilter.Enable(True) + self.tFilter.Enable(True) + self.bt.Enable(True) + if tab is not None: + self.Filt2Full=None # TODO + if tab.active_name!='default': + self.lb.SetLabel(' '+tab.active_name) + self.setColumns() + self.setGUIColumns(xSel=xSel, ySel=ySel) + else: + self.Filt2Full=None # TODO Decide whether filter should be applied... + self.lb.SetLabel(tabLabel) + self.setColumns(columnNames=colNames) + self.setGUIColumns(xSel=xSel, ySel=ySel) + + def updateColumn(self,i,newName): + """ Update of one column name + i: index in GUI + """ + iFull = self.Filt2Full[i] + if self.bShowID: + newName='{:03d} '.format(iFull)+newName + self.lbColumns.SetString(i,newName) + self.comboX.SetString (i,newName) + self.columns[iFull] = newName + + def Full2Filt(self,iFull): + try: + return self.Filt2Full.index(iFull) + except: + return -1 + + def setColumns(self, columnNames=None): + # Get columns from user inputs, or table, or stored. + if columnNames is not None: + # Populating based on user inputs.. + columns=columnNames + elif self.tab is None: + columns=self.columns + else: + # Populating based on table (safest if table was updated) + columns=['Index']+self.tab.columns + # Storing columns, considered as "Full" + self.columns=np.array(columns) + + def setGUIColumns(self, xSel=-1, ySel=[]): + """ Set GUI columns based on self.columns and potential filter """ + # Filtering columns if neeed + sFilt = self.tFilter.GetLineText(0).strip() + if len(sFilt)>0: + Lf, If = filter_list(self.columns, sFilt) + self.Filt2Full = If + else: + self.Filt2Full = list(np.arange(len(self.columns))) + columns=self.columns[self.Filt2Full] + + # GUI update + self.Freeze() + if self.bShowID: + columnsY= ['{:03d} '.format(i)+c for i,c in enumerate(columns)] + columnsX= ['{:03d} '.format(i)+c for i,c in enumerate(self.columns)] + else: + columnsY= columns + columnsX= self.columns + self.lbColumns.Set(columnsY) # potentially filterd + # Slow line for many columns + # NOTE: limiting to 300 for now.. I'm not sure anywant would want to scroll more than that + # Consider adding a "more button" + # see e.g. https://comp.soft-sys.wxwindows.narkive.com/gDfA1Ds5/long-load-time-in-wxpython + self.comboX.Set(columnsX[:300]) # non filtered + + # Set selection for y, if any, and considering filtering + for iFull in ySel: + if iFull=0: + iFilt = self.Full2Filt(iFull) + if iFilt>0: + self.lbColumns.SetSelection(iFilt) + self.lbColumns.EnsureVisible(iFilt) + if len(self.lbColumns.GetSelections())<=0: + self.lbColumns.SetSelection(self.getDefaultColumnY(self.tab,len(columnsY)-1)) + + # Set selection for x, if any, NOTE x is not filtered! + if (xSel<0) or xSel>len(columnsX): + self.comboX.SetSelection(self.getDefaultColumnX(self.tab,len(columnsX)-1)) + else: + self.comboX.SetSelection(xSel) + self.Thaw() + + def forceOneSelection(self): + ISel=self.lbColumns.GetSelections() + self.lbColumns.SetSelection(-1) + if len(ISel)>0: + self.lbColumns.SetSelection(ISel[0]) + + def forceZeroSelection(self): + self.lbColumns.SetSelection(-1) + + def empty(self): + self.lbColumns.Clear() + self.comboX.Clear() + self.lb.SetLabel('') + self.bReadOnly=False + self.lbColumns.Enable(False) + self.comboX.Enable(False) + self.bt.Enable(False) + self.tab=None + self.columns=[] + self.Filt2Full=None + self.btClear.Enable(False) + self.btFilter.Enable(False) + self.tFilter.Enable(False) + self.tFilter.SetValue('') + + def getColumnSelection(self): + iX = self.comboX.GetSelection() + if self.bShowID: + sX = self.comboX.GetStringSelection()[4:] + else: + sX = self.comboX.GetStringSelection() + IY = self.lbColumns.GetSelections() + if self.bShowID: + SY = [self.lbColumns.GetString(i)[4:] for i in IY] + else: + SY = [self.lbColumns.GetString(i) for i in IY] + iXFull = iX # NOTE: x is always in full + IYFull = [self.Filt2Full[iY] for iY in IY] + return iXFull,IYFull,sX,SY + + def onClearFilter(self, event=None): + self.tFilter.SetValue('') + self.onFilterChange() + + def onFilterChange(self, event=None): + xSel,ySel,_,_ = self.getColumnSelection() # (indices in full) + self.setGUIColumns(xSel=xSel, ySel=ySel) # <<< Filtering done here + self.triggerPlot() # Trigger a col selection event + + def onFilterKey(self, event=None): + s=GetKeyString(event) + if s=='ESCAPE' or s=='Ctrl+C': + self.onClearFilter() + event.Skip() + + def triggerPlot(self): + event=wx.PyCommandEvent(wx.EVT_LISTBOX.typeId, self.lbColumns.GetId()) + wx.PostEvent(self.GetEventHandler(), event) + + +# --------------------------------------------------------------------------------} +# --- Selection Panel +# --------------------------------------------------------------------------------{ +class SelectionPanel(wx.Panel): + """ Display options for the user to select data """ + def __init__(self, parent, tabList, mode='auto',mainframe=None): + # Superclass constructor + super(SelectionPanel,self).__init__(parent) + # DATA + self.mainframe = mainframe + self.tabList = None + self.itabForCol = None + self.parent = parent + self.tabSelections = {} + self.tabSelected = [] # NOTE only used to remember a selection after a reload + self.modeRequested = mode + self.currentMode = None + self.nSplits = -1 + + # GUI DATA + self.splitter = MultiSplit(self, style=wx.SP_LIVE_UPDATE) + self.splitter.SetMinimumPaneSize(70) + self.tabPanel = TablePanel (self.splitter,mainframe, tabList) + self.colPanel1 = ColumnPanel(self.splitter, self, mainframe); + self.colPanel2 = ColumnPanel(self.splitter, self, mainframe); + self.colPanel3 = ColumnPanel(self.splitter, self, mainframe); + self.tabPanel.Hide() + self.colPanel1.Hide() + self.colPanel2.Hide() + self.colPanel3.Hide() + + # Layout + self.updateLayout() + VertSizer = wx.BoxSizer(wx.VERTICAL) + VertSizer.Add(self.splitter, 2, flag=wx.EXPAND, border=0) + self.SetSizer(VertSizer) + + # TRIGGERS + self.setTables(tabList) + + def updateLayout(self,mode=None): + self.Freeze() + if mode is None: + mode=self.modeRequested + else: + self.modeRequested = mode + if mode=='auto': + self.autoMode() + elif mode=='sameColumnsMode': + self.sameColumnsMode() + elif mode=='simColumnsMode': + self.simColumnsMode() + elif mode=='twoColumnsMode': + self.twoColumnsMode() + elif mode=='threeColumnsMode': + self.threeColumnsMode() + else: + self.Thaw() + raise Exception('Wrong mode for selection layout: {}'.format(mode)) + self.Thaw() + + + def autoMode(self): + ISel=self.tabPanel.lbTab.GetSelections() + if self.tabList is not None: + if self.tabList.len()<=0: + self.nSplits=-1 + self.splitter.removeAll() + elif self.tabList.haveSameColumns(): + self.sameColumnsMode() + elif self.tabList.haveSameColumns(ISel): + # We don't do same column because we know at least one table is different + # to avoid "jumping" too much + self.twoColumnsMode() + else: + # See if tables are quite similar + IKeepPerTab, IMissPerTab, IDuplPerTab, nCols = getTabCommonColIndices([self.tabList.get(i) for i in ISel]) + if np.all(np.array([len(I) for I in IMissPerTab]))=2): + self.simColumnsMode() + elif len(ISel)==2: + self.twoColumnsMode() + elif len(ISel)==3: + self.threeColumnsMode() + else: + #self.simColumnsMode(self) + raise Exception('Too many panels selected with significant columns differences.') + + def sameColumnsMode(self): + self.currentMode = 'sameColumnsMode' + if self.nSplits==1: + return + if self.nSplits==0 and self.tabList.len()<=1: + return + self.splitter.removeAll() + if self.tabList is not None: + if self.tabList.len()>1: + self.splitter.AppendWindow(self.tabPanel) + self.splitter.AppendWindow(self.colPanel1) + if self.mainframe is not None: + self.mainframe.mainFrameUpdateLayout() + if self.tabList is not None: + if self.tabList.len()<=1: + self.nSplits=0 + else: + self.nSplits=1 + else: + self.nSplits=0 + + def simColumnsMode(self): + self.currentMode = 'simColumnsMode' + self.splitter.removeAll() + self.splitter.AppendWindow(self.tabPanel) + self.splitter.AppendWindow(self.colPanel2) + self.splitter.AppendWindow(self.colPanel1) + self.splitter.setEquiSash() + if self.nSplits<2 and self.mainframe is not None: + self.mainframe.mainFrameUpdateLayout() + self.nSplits=2 + + def twoColumnsMode(self): + self.currentMode = 'twoColumnsMode' + if self.nSplits==2: + return + self.splitter.removeAll() + self.splitter.AppendWindow(self.tabPanel) + self.splitter.AppendWindow(self.colPanel2) + self.splitter.AppendWindow(self.colPanel1) + self.splitter.setEquiSash() + if self.nSplits<2 and self.mainframe is not None: + self.mainframe.mainFrameUpdateLayout() + self.nSplits=2 + + def threeColumnsMode(self): + self.currentMode = 'threeColumnsMode' + if self.nSplits==3: + return + self.splitter.removeAll() + self.splitter.AppendWindow(self.tabPanel) + self.splitter.AppendWindow(self.colPanel3) + self.splitter.AppendWindow(self.colPanel2) + self.splitter.AppendWindow(self.colPanel1) + self.splitter.setEquiSash() + if self.mainframe is not None: + self.mainframe.mainFrameUpdateLayout() + self.nSplits=3 + + def setTables(self,tabList,update=False): + """ Set the list of tables. Keeping the selection if it's an update """ + # TODO PUT ME IN TABLE PANEL + # Find a better way to remember selection + #print('UPDATING TABLES') + # Emptying GUI - TODO only if needed + self.colPanel1.empty() + self.colPanel2.empty() + self.colPanel3.empty() + # Adding + self.tabList = tabList + self.tabPanel.tabList = self.tabList + tabnames = self.tabList.tabNames + self.tabPanel.updateTabNames() + for tn in tabnames: + if tn not in self.tabSelections.keys(): + self.tabSelections[tn]={'xSel':-1,'ySel':[]} + else: + pass # do nothing + + # Reselecting + if len(self.tabSelected)>0: + # Removed line below since two column mode implemented + #if not haveSameColumns(tabs,ISel): + # ISel=[ISel[0]] + for i in self.tabSelected: + if i0: + # Trigger - updating columns and layout + ISel=self.tabPanel.lbTab.GetSelections() + self.tabSelected=ISel + if self.currentMode=='simColumnsMode': + self.setColForSimTab(ISel) + else: + if len(ISel)==1: + self.setTabForCol(ISel[0],1) + elif len(ISel)==2: + self.setTabForCol(ISel[0],1) + self.setTabForCol(ISel[1],2) + elif len(ISel)==3: + self.setTabForCol(ISel[0],1) + self.setTabForCol(ISel[1],2) + self.setTabForCol(ISel[2],3) + else: # Likely all tables have the same columns + self.setTabForCol(ISel[0],1) + self.updateLayout(self.modeRequested) + + def setTabForCol(self,iTabSel,iPanel): + t = self.tabList.get(iTabSel) + ts = self.tabSelections[t.name] + if iPanel==1: + self.colPanel1.setTab(t,ts['xSel'],ts['ySel']) + elif iPanel==2: + self.colPanel2.setTab(t,ts['xSel'],ts['ySel']) + elif iPanel==3: + self.colPanel3.setTab(t,ts['xSel'],ts['ySel']) + else: + raise Exception('Wrong ipanel') + + def setColForSimTab(self,ISel): + """ Set column panels for similar tables """ + tabs = [self.tabList.get(i) for i in ISel] + IKeepPerTab, IMissPerTab, IDuplPerTab, _ = getTabCommonColIndices(tabs) + LenMiss = np.array([len(I) for I in IMissPerTab]) + LenKeep = np.array([len(I) for I in IKeepPerTab]) + LenDupl = np.array([len(I) for I in IDuplPerTab]) + + ColInfo = ['Sim. table mode '] + ColInfo += [''] + if self.tabList.haveSameColumns(ISel): + if len(ISel)>1: + ColInfo += ['Columns identical',''] + else: + if (np.all(np.array(LenMiss)==0)): + ColInfo += ['Columns identical'] + ColInfo += ['Order different!'] + + ColInfo += ['','First difference:'] + ColInfo.append('----------------------------------') + bFirst=True + for it,t in enumerate(tabs): + print('IKeep',IKeepPerTab[it]) + if it==0: + continue + INotOrdered=[ii for i,ii in enumerate(IKeepPerTab[it]) if ii!=IKeepPerTab[0][i]] + print('INot',INotOrdered) + if len(INotOrdered)>0: + im=INotOrdered[0] + if bFirst: + ColInfo.append('{}:'.format(tabs[0].active_name)) + ColInfo.append('{:03d} {:s}'.format(im, tabs[0].columns[im])) + bFirst=False + ColInfo.append('{}:'.format(t.active_name)) + ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) + ColInfo.append('----------------------------------') + + else: + ColInfo += ['Columns different!'] + ColInfo += ['(similar: {})'.format(LenKeep[0])] + ColInfo += ['','Missing columns:'] + ColInfo.append('----------------------------------') + for it,t in enumerate(tabs): + ColInfo.append('{}:'.format(t.active_name)) + if len(IMissPerTab[it])==0: + ColInfo.append(' (None) ') + for im in IMissPerTab[it]: + ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) + ColInfo.append('----------------------------------') + + if (np.any(np.array(LenDupl)>0)): + if len(ISel)>1: + ColInfo += ['','Common duplicates:'] + else: + ColInfo += ['','Duplicates:'] + ColInfo.append('----------------------------------') + for it,t in enumerate(tabs): + ColInfo.append('{}:'.format(t.active_name)) + if len(IDuplPerTab[it])==0: + ColInfo.append(' (None) ') + for im in IDuplPerTab[it]: + ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) + ColInfo.append('----------------------------------') + + + colNames = ['Index'] + [tabs[0].columns[i] for i in IKeepPerTab[0]] + self.colPanel1.setTab(tab=None, colNames=colNames, tabLabel=' Tab. Intersection') + self.colPanel2.setReadOnly(' Tab. Difference', ColInfo) + self.IKeepPerTab=IKeepPerTab + + + + def selectDefaultTable(self): + # Selecting the first table + if self.tabPanel.lbTab.GetCount()>0: + self.tabPanel.lbTab.SetSelection(0) + self.tabSelected=[0] + else: + self.tabSelected=[] + + def tabSelectionChanged(self): + # TODO This can be cleaned-up and merged with updateLayout + #print('Tab selection change') + # Storing the previous selection + #self.printSelection() + self.saveSelection() # + #self.printSelection() + ISel=self.tabPanel.lbTab.GetSelections() + if len(ISel)>0: + if self.modeRequested=='auto': + self.autoMode() + if self.currentMode=='simColumnsMode':# and len(ISel)>1: + self.setColForSimTab(ISel) + self.tabSelected=self.tabPanel.lbTab.GetSelections() + return + + if self.tabList.haveSameColumns(ISel): + # Setting tab + self.setTabForCol(ISel[0],1) + self.colPanel2.empty() + self.colPanel3.empty() + else: + if self.nSplits==2: + if len(ISel)>2: + Error(self,'In this mode, only two tables can be selected. To compare three tables, uses the "3 different tables" mode. Otherwise the tables need to have the same columns.') + ISel=ISel[0:2] + self.tabPanel.lbTab.SetSelection(wx.NOT_FOUND) + for isel in ISel: + self.tabPanel.lbTab.SetSelection(isel) + self.colPanel3.empty() + elif self.nSplits==3: + if len(ISel)>3: + Error(self,'In this mode, only three tables can be selected. To compare more than three tables, the tables need to have the same columns.') + ISel=ISel[0:3] + self.tabPanel.lbTab.SetSelection(wx.NOT_FOUND) + for isel in ISel: + self.tabPanel.lbTab.SetSelection(isel) + else: + Error(self,'The tables selected have different columns.\n\nThis is not compatible with the "Same tables" mode. To compare them, chose one of the following mode: "2 tables", "3 tables" or "Sim. tables".') + self.colPanel2.empty() + self.colPanel3.empty() + # unselect all and select only the first one + ISel=[ISel[0]] + self.tabPanel.lbTab.SetSelection(wx.NOT_FOUND) + self.tabPanel.lbTab.SetSelection(ISel[0]) + for iPanel,iTab in enumerate(ISel): + self.setTabForCol(iTab,iPanel+1) + #print('>>>Updating tabSelected, from',self.tabSelected,'to',self.tabPanel.lbTab.GetSelections()) + self.tabSelected=self.tabPanel.lbTab.GetSelections() + + def colSelectionChanged(self): + """ Simple triggers when column selection is changed, NOTE: does not redraw """ + if self.currentMode=='simColumnsMode': + self.colPanel2.forceZeroSelection() + else: + if self.nSplits in [2,3]: + ISel=self.tabPanel.lbTab.GetSelections() + if self.tabList.haveSameColumns(ISel): + pass # TODO: this test is identical to onTabSelectionChange. Unification. + # elif len(ISel)==2: + # self.colPanel1.forceOneSelection() + # self.colPanel2.forceOneSelection() + # elif len(ISel)==3: + # self.colPanel1.forceOneSelection() + # self.colPanel2.forceOneSelection() + # self.colPanel3.forceOneSelection() + + def update_tabs(self, tabList): + self.setTables(tabList, update=True) + + def renameTable(self,iTab, oldName, newName): + #self.printSelection() + self.tabSelections[newName] = self.tabSelections.pop(oldName) + self.tabPanel.updateTabNames() + #self.printSelection() + + def saveSelection(self): + #self.ISel=self.tabPanel.lbTab.GetSelections() + ISel=self.tabSelected # + if self.tabList.haveSameColumns(ISel): + for ii in ISel: + t=self.tabList.get(ii) + self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() + self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() + else: + if len(ISel)>=1: + t=self.tabList.get(ISel[0]) + self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() + self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() + if len(ISel)>=2: + t=self.tabList.get(ISel[1]) + self.tabSelections[t.name]['xSel'] = self.colPanel2.comboX.GetSelection() + self.tabSelections[t.name]['ySel'] = self.colPanel2.lbColumns.GetSelections() + if len(ISel)>=3: + t=self.tabList.get(ISel[2]) + self.tabSelections[t.name]['xSel'] = self.colPanel3.comboX.GetSelection() + self.tabSelections[t.name]['ySel'] = self.colPanel3.lbColumns.GetSelections() + self.tabSelected = self.tabPanel.lbTab.GetSelections(); + + def printSelection(self): + print('Number of tabSelections stored:',len(self.tabSelections)) + TS=self.tabSelections + for i,tn in enumerate(self.tabList.tabNames): + if tn not in TS.keys(): + print('Tab',i,'>>> Name {} not found in selection'.format(tn)) + else: + print('Tab',i,'xSel:',TS[tn]['xSel'],'ySel:',TS[tn]['ySel'],'Name:',tn) + + def getPlotDataSelection(self): + ID = [] + SameCol=False + if self.tabList is not None and self.tabList.len()>0: + ITab,STab = self.getSelectedTables() + if self.currentMode=='simColumnsMode' and len(ITab)>1: + iiX1,IY1,ssX1,SY1 = self.colPanel1.getColumnSelection() + SameCol=False + for i,(itab,stab) in enumerate(zip(ITab,STab)): + IKeep=self.IKeepPerTab[i] + for j,(iiy,ssy) in enumerate(zip(IY1,SY1)): + if iiy==0: + iy = 0 + sy = ssy + else: + iy = IKeep[iiy-1]+1 + sy = self.tabList.get(itab).columns[IKeep[iiy-1]] + if iiX1==0: + iX1 = 0 + sX1 = ssX1 + else: + iX1 = IKeep[iiX1-1]+1 + sX1 = self.tabList.get(itab).columns[IKeep[iiX1-1]] + ID.append([itab,iX1,iy,sX1,sy,stab]) + else: + iX1,IY1,sX1,SY1 = self.colPanel1.getColumnSelection() + SameCol=self.tabList.haveSameColumns(ITab) + if self.nSplits in [0,1] or SameCol: + for i,(itab,stab) in enumerate(zip(ITab,STab)): + for j,(iy,sy) in enumerate(zip(IY1,SY1)): + ID.append([itab,iX1,iy,sX1,sy,stab]) + elif self.nSplits in [2,3]: + if len(ITab)>=1: + for j,(iy,sy) in enumerate(zip(IY1,SY1)): + ID.append([ITab[0],iX1,iy,sX1,sy,STab[0]]) + if len(ITab)>=2: + iX2,IY2,sX2,SY2 = self.colPanel2.getColumnSelection() + for j,(iy,sy) in enumerate(zip(IY2,SY2)): + ID.append([ITab[1],iX2,iy,sX2,sy,STab[1]]) + if len(ITab)>=3: + iX2,IY2,sX2,SY2 = self.colPanel3.getColumnSelection() + for j,(iy,sy) in enumerate(zip(IY2,SY2)): + ID.append([ITab[2],iX2,iy,sX2,sy,STab[2]]) + else: + raise Exception('Wrong number of splits {}'.format(self.nSplits)) + return ID,SameCol,self.currentMode + + def getSelectedTables(self): + I=self.tabPanel.lbTab.GetSelections() + S=[self.tabPanel.lbTab.GetString(i) for i in I] + return I,S + + def getAllTables(self): + I=range(self.tabPanel.lbTab.GetCount()) + S=[self.tabPanel.lbTab.GetString(i) for i in I] + return I,S + + def clean_memory(self): + self.colPanel1.empty() + self.colPanel2.empty() + self.colPanel3.empty() + self.tabPanel.empty() + del self.tabList + self.tabList=None + + @property + def xCol(self): + iX, _, sX, _ = self.colPanel1.getColumnSelection() + return iX,sX + + +if __name__ == '__main__': + import pandas as pd; + from Tables import Table + import numpy as np + + def OnTabPopup(event): + self.PopupMenu(TablePopup(self,selPanel.tabPanel.lbTab), event.GetPosition()) + + app = wx.App(False) + self=wx.Frame(None,-1,"Title") + tab=Table(data=pd.DataFrame(data={'ColA': np.random.normal(0,1,100)+1,'ColB':np.random.normal(0,1,100)+2})) + selPanel=SelectionPanel(self,[tab],mode='twoColumnsMode') + self.SetSize((800, 600)) + self.Center() + self.Show() + selPanel.tabPanel.lbTab.Bind(wx.EVT_RIGHT_DOWN, OnTabPopup) + + + app.MainLoop() + diff --git a/pydatview/GUIToolBox.py b/pydatview/GUIToolBox.py index 4606d36..a32141e 100644 --- a/pydatview/GUIToolBox.py +++ b/pydatview/GUIToolBox.py @@ -195,6 +195,11 @@ def _update(self,currentaxes=None): class MyNavigationToolbar2Wx(NavigationToolbar2Wx): + """ + Wrapped version of the Navigation toolbar from WX with the following features: + - Tools can be removed, if not in `keep_tools` + - Zoom is set by default, and the toggling between zoom and pan is handled internally + """ def __init__(self, canvas, keep_tools): # Taken from matplotlib/backend_wx.py but added style: self.VERSION = matplotlib.__version__ @@ -215,6 +220,8 @@ def __init__(self, canvas, keep_tools): else: NavigationToolbar2Wx.__init__(self, canvas) + self.pan_on=False + # Make sure we start in zoom mode if 'Pan' in keep_tools: self.zoom() # NOTE: #22 BREAK cursors #12! @@ -225,32 +232,33 @@ def __init__(self, canvas, keep_tools): if t.GetLabel() not in keep_tools: self.DeleteToolByPos(i) - def press_zoom(self, event): - NavigationToolbar2Wx.press_zoom(self,event) - #self.SetToolBitmapSize((22,22)) - - def press_pan(self, event): - NavigationToolbar2Wx.press_pan(self,event) - def zoom(self, *args): - NavigationToolbar2Wx.zoom(self,*args) + # NEW - MPL>=3.0.0 + if self.pan_on: + pass + else: + NavigationToolbar2.zoom(self,*args) # We skip wx and use the parent + # BEFORE + #NavigationToolbar2Wx.zoom(self,*args) def pan(self, *args): - try: - #if self.VERSION[0]=='2' or self.VERSION[0]=='1': - isPan = self._active=='PAN' - except: - try: - from matplotlib.backend_bases import _Mode - isPan = self.mode == _Mode.PAN - except: - raise Exception('Pan not found, report a pyDatView bug, with matplotlib version.') - if isPan: - NavigationToolbar2Wx.pan(self,*args) + self.pan_on=not self.pan_on + # NEW - MPL >= 3.0.0 + NavigationToolbar2.pan(self, *args) # We skip wx and use to parent + if not self.pan_on: self.zoom() - else: - NavigationToolbar2Wx.pan(self,*args) - + # BEFORE + #try: + # isPan = self._active=='PAN' + #except: + # try: + # from matplotlib.backend_bases import _Mode + # isPan = self.mode == _Mode.PAN + # except: + # raise Exception('Pan not found, report a pyDatView bug, with matplotlib version.') + #NavigationToolbar2Wx.pan(self,*args) + #if isPan: + # self.zoom() def home(self, *args): """Restore the original view.""" diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 86d2df9..3ff9d27 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -1,1071 +1,1145 @@ -from __future__ import absolute_import -import wx -import numpy as np -import pandas as pd -import copy -import platform -from collections import OrderedDict - -# For log dec tool -from .common import CHAR, Error, pretty_num_short, Info -from .plotdata import PlotData -from pydatview.tools.damping import logDecFromDecay -from pydatview.tools.curve_fitting import model_fit, extract_key_miscnum, extract_key_num, MODELS, FITTERS, set_common_keys - - -TOOL_BORDER=15 - -# --------------------------------------------------------------------------------} -# --- Default class for tools -# --------------------------------------------------------------------------------{ -class GUIToolPanel(wx.Panel): - def __init__(self, parent): - super(GUIToolPanel,self).__init__(parent) - self.parent = parent - - def destroy(self,event=None): - self.parent.removeTools() - - def getBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): - if Type is not None: - label=CHAR[Type]+' '+label - - bt=wx.Button(par,wx.ID_ANY, label, style=wx.BU_EXACTFIT) - #try: - # if bitmap is not None: - # bt.SetBitmapLabel(wx.ArtProvider.GetBitmap(bitmap)) #,size=(12,12))) - # else: - #except: - # pass - if callback is not None: - par.Bind(wx.EVT_BUTTON, callback, bt) - return bt - - def getToggleBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): - if Type is not None: - label=CHAR[Type]+' '+label - bt=wx.ToggleButton(par,wx.ID_ANY, label, style=wx.BU_EXACTFIT) - if callback is not None: - par.Bind(wx.EVT_TOGGLEBUTTON, callback, bt) - return bt - - - -# --------------------------------------------------------------------------------} -# --- Log Dec -# --------------------------------------------------------------------------------{ -class LogDecToolPanel(GUIToolPanel): - def __init__(self, parent): - super(LogDecToolPanel,self).__init__(parent) - btClose = self.getBtBitmap(self,'Close' ,'close' ,self.destroy ) - btComp = self.getBtBitmap(self,'Compute','compute',self.onCompute) - self.lb = wx.StaticText( self, -1, ' ') - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btClose ,0, flag = wx.LEFT|wx.CENTER,border = 1) - self.sizer.Add(btComp ,0, flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(self.lb ,0, flag = wx.LEFT|wx.CENTER,border = 5) - self.SetSizer(self.sizer) - - def onCompute(self,event=None): - if len(self.parent.plotData)!=1: - Error(self,'Log Dec tool only works with a single plot.') - return - pd =self.parent.plotData[0] - try: - logdec,DampingRatio,T,fn,fd,IPos,INeg,epos,eneg=logDecFromDecay(pd.y,pd.x) - lab='LogDec.: {:.4f} - Damping ratio: {:.4f} - F_n: {:.4f} - F_d: {:.4f} - T:{:.3f}'.format(logdec,DampingRatio,fn,fd,T) - self.lb.SetLabel(lab) - self.sizer.Layout() - ax=self.parent.fig.axes[0] - ax.plot(pd.x[IPos],pd.y[IPos],'o') - ax.plot(pd.x[INeg],pd.y[INeg],'o') - ax.plot(pd.x ,epos,'k--') - ax.plot(pd.x ,eneg,'k--') - self.parent.canvas.draw() - except: - self.lb.SetLabel('Failed. The signal needs to look like the decay of a first order system.') - #self.parent.load_and_draw(); # DATA HAS CHANGED - -# --------------------------------------------------------------------------------} -# --- Outliers -# --------------------------------------------------------------------------------{ -class OutlierToolPanel(GUIToolPanel): - """ - A quick and dirty solution to manipulate plotData - I need to think of a better way to do that - """ - def __init__(self, parent): - super(OutlierToolPanel,self).__init__(parent) - self.parent = parent # parent is GUIPlotPanel - - # Setting default states to parent - if 'RemoveOutliers' not in self.parent.plotDataOptions.keys(): - self.parent.plotDataOptions['RemoveOutliers']=False - if 'OutliersMedianDeviation' not in self.parent.plotDataOptions.keys(): - self.parent.plotDataOptions['OutliersMedianDeviation']=5 - - btClose = self.getBtBitmap(self,'Close','close',self.destroy) - self.btComp = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleCompute) - - lb1 = wx.StaticText(self, -1, 'Median deviation:') -# self.tMD = wx.TextCtrl(self, wx.ID_ANY,, size = (30,-1), style=wx.TE_PROCESS_ENTER) - self.tMD = wx.SpinCtrlDouble(self, value='11', size=wx.Size(60,-1)) - self.tMD.SetValue(self.parent.plotDataOptions['OutliersMedianDeviation']) - self.tMD.SetRange(0.0, 1000) - self.tMD.SetIncrement(0.5) - - self.lb = wx.StaticText( self, -1, '') - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btClose ,0,flag = wx.LEFT|wx.CENTER,border = 1) - self.sizer.Add(self.btComp,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(lb1 ,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(self.tMD ,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(self.lb ,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.SetSizer(self.sizer) - - self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onMDChangeArrow, self.tMD) - self.Bind(wx.EVT_TEXT_ENTER, self.onMDChangeEnter, self.tMD) - - if platform.system()=='Windows': - # See issue https://github.com/wxWidgets/Phoenix/issues/1762 - self.spintxt = self.tMD.Children[0] - assert isinstance(self.spintxt, wx.TextCtrl) - self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onMDChangeChar) - - self.onToggleCompute(init=True) - - def destroy(self,event=None): - self.parent.plotDataOptions['RemoveOutliers']=False - super(OutlierToolPanel,self).destroy() - - def onToggleCompute(self,event=None, init=False): - self.parent.plotDataOptions['OutliersMedianDeviation'] = float(self.tMD.Value) - - if not init: - self.parent.plotDataOptions['RemoveOutliers']= not self.parent.plotDataOptions['RemoveOutliers'] - - if self.parent.plotDataOptions['RemoveOutliers']: - self.lb.SetLabel('Outliers are now removed on the fly. Click "Clear" to stop.') - self.btComp.SetLabel(CHAR['sun']+' Clear') - else: - self.lb.SetLabel('Click on "Apply" to remove outliers on the fly for all new plot.') - self.btComp.SetLabel(CHAR['cloud']+' Apply') - - if not init: - self.parent.load_and_draw() # Data will change - - def onMDChange(self, event=None): - #print(self.tMD.Value) - self.parent.plotDataOptions['OutliersMedianDeviation'] = float(self.tMD.Value) - if self.parent.plotDataOptions['RemoveOutliers']: - self.parent.load_and_draw() # Data will change - - def onMDChangeArrow(self, event): - self.onMDChange() - event.Skip() - - def onMDChangeEnter(self, event): - self.onMDChange() - event.Skip() - - def onMDChangeChar(self, event): - event.Skip() - code = event.GetKeyCode() - if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: - #print(self.spintxt.Value) - self.tMD.SetValue(self.spintxt.Value) - self.onMDChangeEnter(event) - - - -# --------------------------------------------------------------------------------} -# --- Moving Average -# --------------------------------------------------------------------------------{ -class FilterToolPanel(GUIToolPanel): - """ - Moving average/Filters - A quick and dirty solution to manipulate plotData - I need to think of a better way to do that - """ - def __init__(self, parent): - from pydatview.tools.signal import FILTERS - super(FilterToolPanel,self).__init__(parent) - self.parent = parent # parent is GUIPlotPanel - - self._DEFAULT_FILTERS=FILTERS - - # Setting default states to parent - if 'Filter' not in self.parent.plotDataOptions.keys(): - self.parent.plotDataOptions['Filter']=None - self._filterApplied = type(self.parent.plotDataOptions['Filter'])==dict - - - btClose = self.getBtBitmap(self,'Close','close',self.destroy) - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear) - self.btComp = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleCompute) - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) - - lb1 = wx.StaticText(self, -1, 'Filter:') - self.cbFilters = wx.ComboBox(self, choices=[filt['name'] for filt in self._DEFAULT_FILTERS], style=wx.CB_READONLY) - self.lbParamName = wx.StaticText(self, -1, ' :') - self.cbFilters.SetSelection(0) - #self.tParam = wx.TextCtrl(self, wx.ID_ANY,, size = (30,-1), style=wx.TE_PROCESS_ENTER) - self.tParam = wx.SpinCtrlDouble(self, value='11', size=wx.Size(60,-1)) - self.lbInfo = wx.StaticText( self, -1, '') - - - # --- Layout - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btClear ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btComp,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btPlot ,0,flag = wx.ALL|wx.EXPAND, border = 1) - #btSizer.Add(btHelp ,0,flag = wx.ALL|wx.EXPAND, border = 1) - - horzSizer = wx.BoxSizer(wx.HORIZONTAL) - horzSizer.Add(lb1 ,0,flag = wx.LEFT|wx.CENTER,border = 5) - horzSizer.Add(self.cbFilters ,0,flag = wx.LEFT|wx.CENTER,border = 1) - horzSizer.Add(self.lbParamName ,0,flag = wx.LEFT|wx.CENTER,border = 5) - horzSizer.Add(self.tParam ,0,flag = wx.LEFT|wx.CENTER,border = 1) - - vertSizer = wx.BoxSizer(wx.VERTICAL) - vertSizer.Add(self.lbInfo ,0, flag = wx.LEFT ,border = 5) - vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) - self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) - self.SetSizer(self.sizer) - - # --- Events - self.cbFilters.Bind(wx.EVT_COMBOBOX, self.onSelectFilt) - self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow, self.tParam) - self.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter, self.tParam) - if platform.system()=='Windows': - # See issue https://github.com/wxWidgets/Phoenix/issues/1762 - self.spintxt = self.tParam.Children[0] - assert isinstance(self.spintxt, wx.TextCtrl) - self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onParamChangeChar) - - self.onSelectFilt() - self.onToggleCompute(init=True) - - def destroy(self,event=None): - self.parent.plotDataOptions['Filter']=None - super(FilterToolPanel,self).destroy() - - - def onSelectFilt(self, event=None): - """ Select the filter, but does not applied it to the plotData - parentFilt is unchanged - But if the parent already has - """ - iFilt = self.cbFilters.GetSelection() - filt = self._DEFAULT_FILTERS[iFilt] - self.lbParamName.SetLabel(filt['paramName']+':') - self.tParam.SetRange(filt['paramRange'][0], filt['paramRange'][1]) - self.tParam.SetIncrement(filt['increment']) - - parentFilt=self.parent.plotDataOptions['Filter'] - # Value - if type(parentFilt)==dict and parentFilt['name']==filt['name']: - self.tParam.SetValue(parentFilt['param']) - else: - self.tParam.SetValue(filt['param']) - - def onToggleCompute(self, event=None, init=False): - """ - apply Filter based on GUI Data - """ - parentFilt=self.parent.plotDataOptions['Filter'] - if not init: - self._filterApplied = not self._filterApplied - - if self._filterApplied: - self.parent.plotDataOptions['Filter'] =self._GUI2Filt() - #print('Apply', self.parent.plotDataOptions['Filter']) - self.lbInfo.SetLabel( - 'Filter is now applied on the fly. Change parameter live. Click "Clear" to stop. ' - ) - self.btPlot.Enable(False) - self.btClear.Enable(False) - self.btComp.SetLabel(CHAR['sun']+' Clear') - else: - self.parent.plotDataOptions['Filter'] = None - self.lbInfo.SetLabel( - 'Click on "Apply" to set filter on the fly for all plots. '+ - 'Click on "Plot" to try a filter on the current plot.' - ) - self.btPlot.Enable(True) - self.btClear.Enable(True) - self.btComp.SetLabel(CHAR['cloud']+' Apply') - - if not init: - self.parent.load_and_draw() # Data will change - - pass - - def _GUI2Filt(self): - iFilt = self.cbFilters.GetSelection() - filt = self._DEFAULT_FILTERS[iFilt].copy() - filt['param']=np.float(self.spintxt.Value) - return filt - - def onPlot(self, event=None): - """ - Overlay on current axis the filter - """ - from pydatview.tools.signal import applyFilter - if len(self.parent.plotData)!=1: - Error(self,'Plotting only works for a single plot. Plot less data.') - return - filt=self._GUI2Filt() - - PD = self.parent.plotData[0] - y_filt = applyFilter(PD.x0, PD.y0, filt) - ax = self.parent.fig.axes[0] - - PD_new = PlotData() - PD_new.fromXY(PD.x0, y_filt) - self.parent.transformPlotData(PD_new) - ax.plot(PD_new.x, PD_new.y, '-') - self.parent.canvas.draw() - - def onClear(self, event): - self.parent.load_and_draw() # Data will change - - def onParamChange(self, event=None): - if self._filterApplied: - self.parent.plotDataOptions['Filter'] =self._GUI2Filt() - #print('OnParamChange', self.parent.plotDataOptions['Filter']) - self.parent.load_and_draw() # Data will change - - def onParamChangeArrow(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeEnter(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeChar(self, event): - event.Skip() - code = event.GetKeyCode() - if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: - #print(self.spintxt.Value) - self.tParam.SetValue(self.spintxt.Value) - self.onParamChangeEnter(event) - - -# --------------------------------------------------------------------------------} -# --- Resample -# --------------------------------------------------------------------------------{ -class ResampleToolPanel(GUIToolPanel): - def __init__(self, parent): - super(ResampleToolPanel,self).__init__(parent) - - # --- Data from other modules - from pydatview.tools.signal import SAMPLERS - self.parent = parent # parent is GUIPlotPanel - self._SAMPLERS=SAMPLERS - # Setting default states to parent - if 'Sampler' not in self.parent.plotDataOptions.keys(): - self.parent.plotDataOptions['Sampler']=None - self._applied = type(self.parent.plotDataOptions['Sampler'])==dict - - - # --- GUI elements - self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) - self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) - self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) - - #self.lb = wx.StaticText( self, -1, """ Click help """) - self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) - self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS], style=wx.CB_READONLY) - - self.lbNewX = wx.StaticText(self, -1, 'New x: ') - self.textNewX = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER) - self.textOldX = wx.TextCtrl(self, wx.ID_ANY|wx.TE_READONLY) - self.textOldX.Enable(False) - - # --- Layout - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) - - msizer = wx.FlexGridSizer(rows=2, cols=4, hgap=2, vgap=0) - msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(wx.StaticText(self, -1, 'Current x: '), 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.textOldX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) - msizer.Add(wx.StaticText(self, -1, 'Method:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.cbMethods , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.lbNewX , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.textNewX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) - msizer.AddGrowableCol(3,1) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) - self.sizer.Add(msizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) - self.SetSizer(self.sizer) - - # --- Events - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) - self.cbMethods.Bind(wx.EVT_COMBOBOX, self.onMethodChange) - self.textNewX.Bind(wx.EVT_TEXT_ENTER,self.onParamChange) - - # --- Init triggers - self.cbMethods.SetSelection(3) - self.onMethodChange(init=True) - self.onToggleApply(init=True) - self.updateTabList() - self.textNewX.SetValue('2') - - def setCurrentX(self, x=None): - if x is None: - x= self.parent.plotData[0].x - if len(x)<50: - s=np.array2string(x, separator=', ') - else: - s =np.array2string(x[[0,1,2,3]], separator=', ') - s+=', ..., ' - s+=np.array2string(x[[-3,-2,-1]], separator=', ') - s=s.replace('[','').replace(']','').replace(' ','').replace(',',', ') - - self.textOldX.SetValue(s) - - def onMethodChange(self, event=None, init=True): - """ Select the method, but does not applied it to the plotData - User data and option is unchanged - But if the user already has some options, they are used - """ - iOpt = self.cbMethods.GetSelection() - opt = self._SAMPLERS[iOpt] - self.lbNewX.SetLabel(opt['paramName']+':') - - parentOpt=self.parent.plotDataOptions['Sampler'] - # Value - if len(self.textNewX.Value)==0: - if type(parentOpt)==dict: - self.textNewX.SetValue(str(parentOpt['param'])[1:-1]) - else: - self.textNewX.SetValue(str(opt['param'])[2:-2]) - self.onParamChange() - - def onParamChange(self, event=None): - if self._applied: - self.parent.plotDataOptions['Sampler'] =self._GUI2Data() - self.parent.load_and_draw() # Data will change - self.setCurrentX() - - def _GUI2Data(self): - iOpt = self.cbMethods.GetSelection() - opt = self._SAMPLERS[iOpt].copy() - s= self.textNewX.Value.strip().replace('[','').replace(']','') - if len(s)>0: - if s.find(','): - opt['param']=np.array(s.split(',')).astype(float) - else: - opt['param']=np.array(s.split('')).astype(float) - return opt - - def onToggleApply(self, event=None, init=False): - """ - apply sampler based on GUI Data - """ - parentFilt=self.parent.plotDataOptions['Sampler'] - if not init: - self._applied = not self._applied - - if self._applied: - self.parent.plotDataOptions['Sampler'] =self._GUI2Data() - #print('Apply', self.parent.plotDataOptions['Sampler']) - #self.lbInfo.SetLabel( - # 'Sampler is now applied on the fly. Change parameter live. Click "Clear" to stop. ' - # ) - self.btPlot.Enable(False) - self.btClear.Enable(False) - self.btApply.SetLabel(CHAR['sun']+' Clear') - else: - self.parent.plotDataOptions['Sampler'] = None - #self.lbInfo.SetLabel( - # 'Click on "Apply" to set filter on the fly for all plots. '+ - # 'Click on "Plot" to try a filter on the current plot.' - # ) - self.btPlot.Enable(True) - self.btClear.Enable(True) - self.btApply.SetLabel(CHAR['cloud']+' Apply') - - if not init: - self.parent.load_and_draw() # Data will change - self.setCurrentX() - - - def onAdd(self,event=None): - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe - icol, colname = self.parent.selPanel.xCol - print(icol,colname) - opt = self._GUI2Data() - errors=[] - if iSel==0: - dfs, names, errors = tabList.applyResampling(icol, opt, bAdd=True) - mainframe.load_dfs(dfs,names,bAdd=True) - else: - df, name = tabList.get(iSel-1).applyResampling(icol, opt, bAdd=True) - mainframe.load_df(df,name,bAdd=True) - self.updateTabList() - - if len(errors)>0: - raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) - - def onPlot(self,event=None): - from pydatview.tools.signal import applySampler - if len(self.parent.plotData)!=1: - Error(self,'Plotting only works for a single plot. Plot less data.') - return - opts=self._GUI2Data() - PD = self.parent.plotData[0] - x_new, y_new = applySampler(PD.x0, PD.y0, opts) - ax = self.parent.fig.axes[0] - - PD_new = PlotData() - PD_new.fromXY(x_new, y_new) - self.parent.transformPlotData(PD_new) - ax.plot(PD_new.x, PD_new.y, '-') - self.setCurrentX(x_new) - - self.parent.canvas.draw() - - def onClear(self,event=None): - self.parent.load_and_draw() # Data will change - # Update Current X - self.setCurrentX() - # Update Table list - self.updateTabList() - - - def onTabChange(self,event=None): - #tabList = self.parent.selPanel.tabList - #iSel=self.cbTabs.GetSelection() - pass - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - - def onHelp(self,event=None): - Info(self,"""Resampling. - -The resampling operation changes the "x" values of a table/plot and -adapt the "y" values accordingly. - -To resample perform the following step: - -- Chose a resampling method: - - replace: specify all the new x-values - - insert : insert a list of x values to the existing ones - - delete : delete a list of x values from the existing ones - - every-n : use every n values - - time-based: downsample using sample averaging or upsample using - linear interpolation, x-axis must already be in seconds - - delta x : specify a delta for uniform spacing of x values - -- Specify the x values as a space or comma separated list - -- Click on one of the following buttons: - - Plot: will display the resampled data on the figure - - Apply: will perform the resampling on the fly for all new plots - - Add: will create new table(s) with resampled values for all - signals. This process might take some time. - Select a table or choose all (default) -""") - - - -# --------------------------------------------------------------------------------} -# --- Mask -# --------------------------------------------------------------------------------{ -class MaskToolPanel(GUIToolPanel): - def __init__(self, parent): - super(MaskToolPanel,self).__init__(parent) - - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - - allMask = tabList.commonMaskString - if len(allMask)==0: - allMask=self.guessMask(tabList) # no known mask, we guess one to help the user - self.applied=False - else: - self.applied=True - - btClose = self.getBtBitmap(self, 'Close','close', self.destroy) - btComp = self.getBtBitmap(self, u'Mask (add)','add' , self.onApply) - if self.applied: - self.btCompMask = self.getToggleBtBitmap(self, 'Clear','sun', self.onToggleApplyMask) - self.btCompMask.SetValue(True) - else: - self.btCompMask = self.getToggleBtBitmap(self, 'Mask','cloud', self.onToggleApplyMask) - - self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'")""") - self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) - self.cbTabs.SetSelection(0) - - self.textMask = wx.TextCtrl(self, wx.ID_ANY, allMask) - #self.textMask.SetValue('({Time}>100) & ({Time}<400)') - #self.textMask.SetValue("{Date} > '2018-10-01'") - - btSizer = wx.FlexGridSizer(rows=2, cols=2, hgap=2, vgap=0) - btSizer.Add(btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(wx.StaticText(self, -1, '') ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btComp ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btCompMask ,0,flag = wx.ALL|wx.EXPAND, border = 1) - - row_sizer = wx.BoxSizer(wx.HORIZONTAL) - row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) - row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) - row_sizer.Add(wx.StaticText(self, -1, 'Mask:'), 0, wx.CENTER|wx.LEFT, 5) - row_sizer.Add(self.textMask, 1, wx.CENTER|wx.LEFT|wx.EXPAND, 5) - - vert_sizer = wx.BoxSizer(wx.VERTICAL) - vert_sizer.Add(self.lb ,0, flag = wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) - vert_sizer.Add(row_sizer ,1, flag = wx.EXPAND|wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) - self.sizer.Add(vert_sizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) - self.SetSizer(self.sizer) - self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) - - def onTabChange(self,event=None): - tabList = self.parent.selPanel.tabList - iSel=self.cbTabs.GetSelection() - if iSel==0: - maskString = tabList.commonMaskString - else: - maskString= tabList.get(iSel-1).maskString - if len(maskString)>0: - self.textMask.SetValue(maskString) - #else: - # self.textMask.SetValue('') # no known mask - # self.textMask.SetValue(self.guessMask) # no known mask - - def guessMask(self,tabList): - cols=[c.lower() for c in tabList.get(0).columns_clean] - if 'time' in cols: - return '{Time} > 100' - elif 'date' in cols: - return "{Date} > '2017-01-01" - else: - return '' - - def onClear(self,event=None): - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe - if iSel==0: - tabList.clearCommonMask() - else: - tabList.get(iSel-1).clearMask() - - mainframe.redraw() - self.onTabChange() - - def onToggleApplyMask(self,event=None): - self.applied = not self.applied - if self.applied: - self.btCompMask.SetLabel(CHAR['sun']+' Clear') - else: - self.btCompMask.SetLabel(CHAR['cloud']+' Mask') - - if self.applied: - self.onApply(event,bAdd=False) - else: - self.onClear() - - def onApply(self,event=None,bAdd=True): - maskString = self.textMask.GetLineText(0) - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe - if iSel==0: - dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=bAdd) - if bAdd: - mainframe.load_dfs(dfs,names,bAdd=bAdd) - else: - mainframe.redraw() - if len(errors)>0: - raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) - else: - dfs, name = tabList.get(iSel-1).applyMaskString(maskString, bAdd=bAdd) - if bAdd: - mainframe.load_df(df,name,bAdd=bAdd) - else: - mainframe.redraw() - self.updateTabList() - - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - -# --------------------------------------------------------------------------------} -# --- Radial -# --------------------------------------------------------------------------------{ -sAVG_METHODS = ['Last `n` seconds','Last `n` periods'] -AVG_METHODS = ['constantwindow','periods'] - -class RadialToolPanel(GUIToolPanel): - def __init__(self, parent): - super(RadialToolPanel,self).__init__(parent) - - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - - btClose = self.getBtBitmap(self,'Close' ,'close' , self.destroy) - btComp = self.getBtBitmap(self,'Average','compute', self.onApply) # ART_PLUS - - self.lb = wx.StaticText( self, -1, """Select tables, averaging method and average parameter (`Period` methods uses the `azimuth` signal) """) - self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) - self.cbMethod = wx.ComboBox(self, choices=sAVG_METHODS, style=wx.CB_READONLY) - self.cbTabs.SetSelection(0) - self.cbMethod.SetSelection(0) - - self.textAverageParam = wx.TextCtrl(self, wx.ID_ANY, '2',size = (36,-1), style=wx.TE_PROCESS_ENTER) - - btSizer = wx.FlexGridSizer(rows=2, cols=1, hgap=0, vgap=0) - #btSizer = wx.BoxSizer(wx.VERTICAL) - btSizer.Add(btClose ,0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btComp ,0, flag = wx.ALL|wx.EXPAND, border = 1) - - row_sizer = wx.BoxSizer(wx.HORIZONTAL) - row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) - row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) - row_sizer.Add(wx.StaticText(self, -1, 'Method:'), 0, wx.CENTER|wx.LEFT, 5) - row_sizer.Add(self.cbMethod , 0, wx.CENTER|wx.LEFT, 2) - row_sizer.Add(wx.StaticText(self, -1, 'Param:') , 0, wx.CENTER|wx.LEFT, 5) - row_sizer.Add(self.textAverageParam , 0, wx.CENTER|wx.LEFT|wx.RIGHT| wx.EXPAND, 2) - - vert_sizer = wx.BoxSizer(wx.VERTICAL) - vert_sizer.Add(self.lb ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) - vert_sizer.Add(row_sizer ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) - self.sizer.Add(vert_sizer ,0, flag = wx.LEFT|wx.EXPAND,border = TOOL_BORDER) - self.SetSizer(self.sizer) - self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) - - def onTabChange(self,event=None): - tabList = self.parent.selPanel.tabList - - def onApply(self,event=None): - try: - avgParam = float(self.textAverageParam.GetLineText(0)) - except: - raise Exception('Error: the averaging parameter needs to be an integer or a float') - iSel = self.cbTabs.GetSelection() - avgMethod = AVG_METHODS[self.cbMethod.GetSelection()] - tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe - if iSel==0: - dfs, names, errors = tabList.radialAvg(avgMethod,avgParam) - mainframe.load_dfs(dfs,names,bAdd=True) - if len(errors)>0: - raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) - else: - dfs, names = tabList.get(iSel-1).radialAvg(avgMethod,avgParam) - mainframe.load_dfs(dfs,names,bAdd=True) - - self.updateTabList() - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - - -# --------------------------------------------------------------------------------} -# --- Curve Fitting -# --------------------------------------------------------------------------------{ -MODELS_EXAMPLE =[ - {'label':'User defined model', 'id':'eval:', - 'formula':'{a}*x**2 + {b}', - 'coeffs':None, - 'consts':None, - 'bounds':None }, - ] -MODELS_EXTRA =[ -# {'label':'Exponential decay', 'id':'eval:', -# 'formula':'{A}*exp(-{k}*x)+{B}', -# 'coeffs' :'k=1, A=1, B=0', -# 'consts' :None, -# 'bounds' :None}, -] - -class CurveFitToolPanel(GUIToolPanel): - def __init__(self, parent): - super(CurveFitToolPanel,self).__init__(parent) - - # Data - self.x = None - self.y_fit = None - - # GUI Objecst - btClose = self.getBtBitmap(self, 'Close','close', self.destroy) - btClear = self.getBtBitmap(self, 'Clear','sun', self.onClear) # DELETE - btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) - btCompFit = self.getBtBitmap(self, 'Fit','check', self.onCurveFit) - btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) - - boldFont = self.GetFont().Bold() - lbOutputs = wx.StaticText(self, -1, 'Outputs') - lbInputs = wx.StaticText(self, -1, 'Inputs ') - lbOutputs.SetFont(boldFont) - lbInputs.SetFont(boldFont) - - self.textFormula = wx.TextCtrl(self, wx.ID_ANY, '') - self.textGuess = wx.TextCtrl(self, wx.ID_ANY, '') - self.textBounds = wx.TextCtrl(self, wx.ID_ANY, '') - self.textConstants = wx.TextCtrl(self, wx.ID_ANY, '') - - self.textFormulaNum = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) - self.textCoeffs = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) - self.textInfo = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) - - - self.Models=copy.deepcopy(MODELS_EXAMPLE) + copy.deepcopy(FITTERS) + copy.deepcopy(MODELS) + copy.deepcopy(MODELS_EXTRA) - sModels=[d['label'] for d in self.Models] - - - self.cbModels = wx.ComboBox(self, choices=sModels, style=wx.CB_READONLY) - self.cbModels.SetSelection(0) - - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btClear ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btAdd ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btCompFit ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btHelp ,0,flag = wx.ALL|wx.EXPAND, border = 1) - - inputSizer = wx.FlexGridSizer(rows=5, cols=2, hgap=0, vgap=0) - inputSizer.Add(lbInputs ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.cbModels ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.Add(wx.StaticText(self, -1, 'Formula:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.textFormula ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.Add(wx.StaticText(self, -1, 'Guess:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.textGuess ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.Add(wx.StaticText(self, -1, 'Bounds:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.textBounds ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.Add(wx.StaticText(self, -1, 'Constants:'),0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.textConstants ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.AddGrowableCol(1,1) - - outputSizer = wx.FlexGridSizer(rows=5, cols=2, hgap=0, vgap=0) - outputSizer.Add(lbOutputs ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(wx.StaticText(self, -1, '') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(wx.StaticText(self, -1, 'Formula:'),0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(self.textFormulaNum ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - outputSizer.Add(wx.StaticText(self, -1, 'Parameters:') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(self.textCoeffs ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - outputSizer.Add(wx.StaticText(self, -1, 'Accuracy:') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(self.textInfo ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - outputSizer.AddGrowableCol(1,0.5) - - horzSizer = wx.BoxSizer(wx.HORIZONTAL) - horzSizer.Add(inputSizer ,1.0, flag = wx.LEFT|wx.EXPAND,border = 2) - horzSizer.Add(outputSizer ,1.0, flag = wx.LEFT|wx.EXPAND,border = 9) - - vertSizer = wx.BoxSizer(wx.VERTICAL) -# vertSizer.Add(self.lbHelp ,0, flag = wx.LEFT ,border = 1) - vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) -# self.sizer.Add(vertSizerCB ,0, flag = wx.LEFT ,border = 1) - self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) - self.SetSizer(self.sizer) - - self.Bind(wx.EVT_COMBOBOX, self.onModelChange, self.cbModels) - - self.onModelChange() - - def onModelChange(self,event=None): - iModel = self.cbModels.GetSelection() - d = self.Models[iModel] - self.textFormula.SetEditable(True) - - if d['id'].find('fitter:')==0 : - self.textGuess.Enable(False) - self.textGuess.SetValue('') - self.textFormula.Enable(False) - self.textFormula.SetValue(d['formula']) - self.textBounds.Enable(False) - self.textBounds.SetValue('') - self.textConstants.Enable(True) - # NOTE: conversion to string works with list, and tuples, not numpy array - val = ', '.join([k+'='+str(v) for k,v in d['consts'].items()]) - self.textConstants.SetValue(val) - else: - # Formula - if d['id'].find('eval:')==0 : - self.textFormula.Enable(True) - self.textFormula.SetEditable(True) - else: - #self.textFormula.Enable(False) - self.textFormula.Enable(True) - self.textFormula.SetEditable(False) - self.textFormula.SetValue(d['formula']) - - # Guess - if d['coeffs'] is None: - self.textGuess.SetValue('') - else: - self.textGuess.SetValue(d['coeffs']) - - # Constants - if d['consts'] is None or len(d['consts'].strip())==0: - self.textConstants.Enable(False) - self.textConstants.SetValue('') - else: - self.textConstants.Enable(True) - self.textConstants.SetValue(d['consts']) - - # Bounds - self.textBounds.Enable(True) - if d['bounds'] is None or len(d['bounds'].strip())==0: - self.textBounds.SetValue('all=(-np.inf, np.inf)') - else: - self.textBounds.SetValue(d['bounds']) - - # Outputs - self.textFormulaNum.SetValue('(Click on Fit)') - self.textCoeffs.SetValue('') - self.textInfo.SetValue('') - - def onCurveFit(self,event=None): - self.x = None - self.y_fit = None - if len(self.parent.plotData)!=1: - Error(self,'Curve fitting tool only works with a single curve. Plot less data.') - return - PD =self.parent.plotData[0] - - iModel = self.cbModels.GetSelection() - d = self.Models[iModel] - - if d['id'].find('fitter:')==0 : - sFunc=d['id'] - p0=None - bounds=None - fun_kwargs=extract_key_miscnum(self.textConstants.GetLineText(0).replace('np.inf','inf')) - else: - # Formula - sFunc=d['id'] - if sFunc=='eval:': - sFunc+=self.textFormula.GetLineText(0) - # Bounds - bounds=self.textBounds.GetLineText(0).replace('np.inf','inf') - # Guess - p0=self.textGuess.GetLineText(0).replace('np.inf','inf') - fun_kwargs=extract_key_num(self.textConstants.GetLineText(0).replace('np.inf','inf')) - #print('>>> Model fit sFunc :',sFunc ) - #print('>>> Model fit p0 :',p0 ) - #print('>>> Model fit bounds:',bounds ) - #print('>>> Model fit kwargs:',fun_kwargs) - # Performing fit - y_fit, pfit, fitter = model_fit(sFunc, PD.x, PD.y, p0=p0, bounds=bounds,**fun_kwargs) - - formatter = lambda x: pretty_num_short(x, digits=3) - formula_num = fitter.formula_num(fmt=formatter) - # Update info - self.textFormulaNum.SetValue(formula_num) - self.textCoeffs.SetValue(', '.join(['{}={:s}'.format(k,formatter(v)) for k,v in fitter.model['coeffs'].items()])) - self.textInfo.SetValue('R2 = {:.3f} '.format(fitter.model['R2'])) - - # Saving - d['formula'] = self.textFormula.GetLineText(0) - d['bounds'] = self.textBounds.GetLineText(0).replace('np.inf','inf') - d['coeffs'] = self.textGuess.GetLineText(0).replace('np.inf','inf') - if d['id'].find('fitter:')==0 : - d['consts'], _ = set_common_keys(d['consts'],fun_kwargs) - else: - d['consts']= self.textConstants.GetLineText(0).replace('np.inf','inf') - - - # Plot - ax=self.parent.fig.axes[0] - ax.plot(PD.x,y_fit,'o', ms=4) - self.parent.canvas.draw() - - self.x=PD.x - self.y_fit=y_fit - self.sx=PD.sx - self.sy=PD.sy - - def onClear(self,event=None): - self.parent.load_and_draw() # DATA HAS CHANGED - self.onModelChange() - - def onAdd(self,event=None): - name='model_fit' - if self.x is not None and self.y_fit is not None: - df=pd.DataFrame({self.sx:self.x, self.sy:self.y_fit}) - self.parent.mainframe.load_df(df,name,bAdd=True) - - def onHelp(self,event=None): - Info(self,"""Curve fitting is still in beta. - -To perform a curve fit, adjusts the "Inputs section on the left": -- Select a predefined equation to fit, using the scrolldown menu. -- Adjust the initial gues for the parameters (if wanted) -- (Only for few models: set constants values) -- Click on "Fit" - -If you select a user-defined model: -- Equation parameters are specified using curly brackets -- Numpy functions are available using "np" - -Buttons: -- Clear: remove the fit from the plot -- Add: add the fit data to the list of tables (can then be exported) - -""") - - - -TOOLS={ - 'LogDec': LogDecToolPanel, - 'Outlier': OutlierToolPanel, - 'Filter': FilterToolPanel, - 'Resample': ResampleToolPanel, - 'Mask': MaskToolPanel, - 'FASTRadialAverage': RadialToolPanel, - 'CurveFitting': CurveFitToolPanel, -} +from __future__ import absolute_import +import wx +import numpy as np +import pandas as pd +import copy +import platform +from collections import OrderedDict + +# For log dec tool +from .common import CHAR, Error, pretty_num_short, Info +from .plotdata import PlotData +from pydatview.tools.damping import logDecFromDecay +from pydatview.tools.curve_fitting import model_fit, extract_key_miscnum, extract_key_num, MODELS, FITTERS, set_common_keys + + +TOOL_BORDER=15 + +# --------------------------------------------------------------------------------} +# --- Default class for tools +# --------------------------------------------------------------------------------{ +class GUIToolPanel(wx.Panel): + def __init__(self, parent): + super(GUIToolPanel,self).__init__(parent) + self.parent = parent + + def destroy(self,event=None): + self.parent.removeTools() + + def getBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): + if Type is not None: + label=CHAR[Type]+' '+label + + bt=wx.Button(par,wx.ID_ANY, label, style=wx.BU_EXACTFIT) + #try: + # if bitmap is not None: + # bt.SetBitmapLabel(wx.ArtProvider.GetBitmap(bitmap)) #,size=(12,12))) + # else: + #except: + # pass + if callback is not None: + par.Bind(wx.EVT_BUTTON, callback, bt) + return bt + + def getToggleBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): + if Type is not None: + label=CHAR[Type]+' '+label + bt=wx.ToggleButton(par,wx.ID_ANY, label, style=wx.BU_EXACTFIT) + if callback is not None: + par.Bind(wx.EVT_TOGGLEBUTTON, callback, bt) + return bt + + + +# --------------------------------------------------------------------------------} +# --- Log Dec +# --------------------------------------------------------------------------------{ +class LogDecToolPanel(GUIToolPanel): + def __init__(self, parent): + super(LogDecToolPanel,self).__init__(parent) + btClose = self.getBtBitmap(self,'Close' ,'close' ,self.destroy ) + btComp = self.getBtBitmap(self,'Compute','compute',self.onCompute) + self.lb = wx.StaticText( self, -1, ' ') + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btClose ,0, flag = wx.LEFT|wx.CENTER,border = 1) + self.sizer.Add(btComp ,0, flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(self.lb ,0, flag = wx.LEFT|wx.CENTER,border = 5) + self.SetSizer(self.sizer) + + def onCompute(self,event=None): + if len(self.parent.plotData)!=1: + Error(self,'Log Dec tool only works with a single plot.') + return + pd =self.parent.plotData[0] + try: + logdec,DampingRatio,T,fn,fd,IPos,INeg,epos,eneg=logDecFromDecay(pd.y,pd.x) + lab='LogDec.: {:.4f} - Damping ratio: {:.4f} - F_n: {:.4f} - F_d: {:.4f} - T:{:.3f}'.format(logdec,DampingRatio,fn,fd,T) + self.lb.SetLabel(lab) + self.sizer.Layout() + ax=self.parent.fig.axes[0] + ax.plot(pd.x[IPos],pd.y[IPos],'o') + ax.plot(pd.x[INeg],pd.y[INeg],'o') + ax.plot(pd.x ,epos,'k--') + ax.plot(pd.x ,eneg,'k--') + self.parent.canvas.draw() + except: + self.lb.SetLabel('Failed. The signal needs to look like the decay of a first order system.') + #self.parent.load_and_draw(); # DATA HAS CHANGED + +# --------------------------------------------------------------------------------} +# --- Outliers +# --------------------------------------------------------------------------------{ +class OutlierToolPanel(GUIToolPanel): + """ + A quick and dirty solution to manipulate plotData + I need to think of a better way to do that + """ + def __init__(self, parent): + super(OutlierToolPanel,self).__init__(parent) + self.parent = parent # parent is GUIPlotPanel + + # Setting default states to parent + if 'RemoveOutliers' not in self.parent.plotDataOptions.keys(): + self.parent.plotDataOptions['RemoveOutliers']=False + if 'OutliersMedianDeviation' not in self.parent.plotDataOptions.keys(): + self.parent.plotDataOptions['OutliersMedianDeviation']=5 + + btClose = self.getBtBitmap(self,'Close','close',self.destroy) + self.btComp = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleCompute) + + lb1 = wx.StaticText(self, -1, 'Median deviation:') +# self.tMD = wx.TextCtrl(self, wx.ID_ANY,, size = (30,-1), style=wx.TE_PROCESS_ENTER) + self.tMD = wx.SpinCtrlDouble(self, value='11', size=wx.Size(60,-1)) + self.tMD.SetValue(self.parent.plotDataOptions['OutliersMedianDeviation']) + self.tMD.SetRange(0.0, 1000) + self.tMD.SetIncrement(0.5) + + self.lb = wx.StaticText( self, -1, '') + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btClose ,0,flag = wx.LEFT|wx.CENTER,border = 1) + self.sizer.Add(self.btComp,0,flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(lb1 ,0,flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(self.tMD ,0,flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(self.lb ,0,flag = wx.LEFT|wx.CENTER,border = 5) + self.SetSizer(self.sizer) + + self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onMDChangeArrow, self.tMD) + self.Bind(wx.EVT_TEXT_ENTER, self.onMDChangeEnter, self.tMD) + + if platform.system()=='Windows': + # See issue https://github.com/wxWidgets/Phoenix/issues/1762 + self.spintxt = self.tMD.Children[0] + assert isinstance(self.spintxt, wx.TextCtrl) + self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onMDChangeChar) + + self.onToggleCompute(init=True) + + def destroy(self,event=None): + self.parent.plotDataOptions['RemoveOutliers']=False + super(OutlierToolPanel,self).destroy() + + def onToggleCompute(self,event=None, init=False): + self.parent.plotDataOptions['OutliersMedianDeviation'] = float(self.tMD.Value) + + if not init: + self.parent.plotDataOptions['RemoveOutliers']= not self.parent.plotDataOptions['RemoveOutliers'] + + if self.parent.plotDataOptions['RemoveOutliers']: + self.lb.SetLabel('Outliers are now removed on the fly. Click "Clear" to stop.') + self.btComp.SetLabel(CHAR['sun']+' Clear') + else: + self.lb.SetLabel('Click on "Apply" to remove outliers on the fly for all new plot.') + self.btComp.SetLabel(CHAR['cloud']+' Apply') + + if not init: + self.parent.load_and_draw() # Data will change + + def onMDChange(self, event=None): + #print(self.tMD.Value) + self.parent.plotDataOptions['OutliersMedianDeviation'] = float(self.tMD.Value) + if self.parent.plotDataOptions['RemoveOutliers']: + self.parent.load_and_draw() # Data will change + + def onMDChangeArrow(self, event): + self.onMDChange() + event.Skip() + + def onMDChangeEnter(self, event): + self.onMDChange() + event.Skip() + + def onMDChangeChar(self, event): + event.Skip() + code = event.GetKeyCode() + if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: + #print(self.spintxt.Value) + self.tMD.SetValue(self.spintxt.Value) + self.onMDChangeEnter(event) + + + +# --------------------------------------------------------------------------------} +# --- Filter Tool +# --------------------------------------------------------------------------------{ +class FilterToolPanel(GUIToolPanel): + """ + Moving average/Filters + A quick and dirty solution to manipulate plotData + I need to think of a better way to do that + """ + def __init__(self, parent): + from pydatview.tools.signal import FILTERS + super(FilterToolPanel,self).__init__(parent) + self.parent = parent # parent is GUIPlotPanel + + self._DEFAULT_FILTERS=FILTERS + + # Setting default states to parent + if 'Filter' not in self.parent.plotDataOptions.keys(): + self.parent.plotDataOptions['Filter']=None + self._filterApplied = type(self.parent.plotDataOptions['Filter'])==dict + + + self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) + self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) + self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) + self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear) + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) + self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) + + self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + + self.cbFilters = wx.ComboBox(self, choices=[filt['name'] for filt in self._DEFAULT_FILTERS], style=wx.CB_READONLY) + self.lbParamName = wx.StaticText(self, -1, ' :') + self.cbFilters.SetSelection(0) + self.tParam = wx.SpinCtrlDouble(self, value='11', size=wx.Size(60,-1)) + self.lbInfo = wx.StaticText( self, -1, '') + + + # --- Layout + btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) + btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) + + + horzSizerT = wx.BoxSizer(wx.HORIZONTAL) + horzSizerT.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) + horzSizerT.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) + + horzSizer = wx.BoxSizer(wx.HORIZONTAL) + horzSizer.Add(wx.StaticText(self, -1, 'Filter:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) + horzSizer.Add(self.cbFilters , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) + horzSizer.Add(self.lbParamName , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) + horzSizer.Add(self.tParam , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) + + + vertSizer = wx.BoxSizer(wx.VERTICAL) + vertSizer.Add(self.lbInfo ,0, flag = wx.LEFT ,border = 5) + vertSizer.Add(horzSizerT ,1, flag = wx.LEFT|wx.EXPAND,border = 1) + vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) + self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) + self.SetSizer(self.sizer) + + # --- Events + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + self.cbFilters.Bind(wx.EVT_COMBOBOX, self.onSelectFilt) + self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow, self.tParam) + self.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter, self.tParam) + if platform.system()=='Windows': + # See issue https://github.com/wxWidgets/Phoenix/issues/1762 + self.spintxt = self.tParam.Children[0] + assert isinstance(self.spintxt, wx.TextCtrl) + self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onParamChangeChar) + + # --- Init triggers + self.updateTabList() + self.onSelectFilt() + self.onToggleApply(init=True) + + def destroy(self,event=None): + super(FilterToolPanel,self).destroy() + + def onSelectFilt(self, event=None): + """ Select the filter, but does not applied it to the plotData + parentFilt is unchanged + But if the parent already has + """ + iFilt = self.cbFilters.GetSelection() + filt = self._DEFAULT_FILTERS[iFilt] + self.lbParamName.SetLabel(filt['paramName']+':') + self.tParam.SetRange(filt['paramRange'][0], filt['paramRange'][1]) + self.tParam.SetIncrement(filt['increment']) + + parentFilt=self.parent.plotDataOptions['Filter'] + # Value + if type(parentFilt)==dict and parentFilt['name']==filt['name']: + self.tParam.SetValue(parentFilt['param']) + else: + self.tParam.SetValue(filt['param']) + + def _GUI2Data(self): + iFilt = self.cbFilters.GetSelection() + filt = self._DEFAULT_FILTERS[iFilt].copy() + try: + filt['param']=np.float(self.spintxt.Value) + except: + print('[WARN] pyDatView: Issue on Mac: GUITools.py/_GUI2Data. Help needed.') + return filt + + def onToggleApply(self, event=None, init=False): + """ + apply Filter based on GUI Data + """ + parentFilt=self.parent.plotDataOptions['Filter'] + if not init: + self._filterApplied = not self._filterApplied + + if self._filterApplied: + self.parent.plotDataOptions['Filter'] =self._GUI2Data() + #print('Apply', self.parent.plotDataOptions['Filter']) + self.lbInfo.SetLabel( + 'Filter is now applied on the fly. Change parameter live. Click "Clear" to stop. ' + ) + self.btPlot.Enable(False) + self.btClear.Enable(False) + self.btApply.SetLabel(CHAR['sun']+' Clear') + else: + self.parent.plotDataOptions['Filter'] = None + self.lbInfo.SetLabel( + 'Click on "Apply" to set filter on the fly for all plots. '+ + 'Click on "Plot" to try a filter on the current plot.' + ) + self.btPlot.Enable(True) + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + + if not init: + self.parent.load_and_draw() # Data will change + self.updateTabList() + + def onAdd(self,event=None): + iSel = self.cbTabs.GetSelection() + tabList = self.parent.selPanel.tabList + mainframe = self.parent.mainframe + icol, colname = self.parent.selPanel.xCol + opt = self._GUI2Data() + errors=[] + if iSel==0: + dfs, names, errors = tabList.applyFiltering(icol, opt, bAdd=True) + mainframe.load_dfs(dfs,names,bAdd=True) + else: + df, name = tabList.get(iSel-1).applyFiltering(icol, opt, bAdd=True) + mainframe.load_df(df,name,bAdd=True) + self.updateTabList() + + if len(errors)>0: + raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) + + def onPlot(self, event=None): + """ + Overlay on current axis the filter + """ + from pydatview.tools.signal import applyFilter + if len(self.parent.plotData)!=1: + Error(self,'Plotting only works for a single plot. Plot less data.') + return + filt=self._GUI2Data() + + PD = self.parent.plotData[0] + y_filt = applyFilter(PD.x0, PD.y0, filt) + ax = self.parent.fig.axes[0] + + PD_new = PlotData() + PD_new.fromXY(PD.x0, y_filt) + self.parent.transformPlotData(PD_new) + ax.plot(PD_new.x, PD_new.y, '-') + self.parent.canvas.draw() + + def onClear(self, event): + self.parent.load_and_draw() # Data will change + + def onParamChange(self, event=None): + if self._filterApplied: + self.parent.plotDataOptions['Filter'] =self._GUI2Data() + #print('OnParamChange', self.parent.plotDataOptions['Filter']) + self.parent.load_and_draw() # Data will change + + def onParamChangeArrow(self, event): + self.onParamChange() + event.Skip() + + def onParamChangeEnter(self, event): + self.onParamChange() + event.Skip() + + def onParamChangeChar(self, event): + event.Skip() + code = event.GetKeyCode() + if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: + #print(self.spintxt.Value) + self.tParam.SetValue(self.spintxt.Value) + self.onParamChangeEnter(event) + + def onTabChange(self,event=None): + #tabList = self.parent.selPanel.tabList + #iSel=self.cbTabs.GetSelection() + pass + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + try: + iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + except RuntimeError: + pass + + def onHelp(self,event=None): + Info(self,"""Filtering. + +The filtering operation changes the "y" values of a table/plot, +applying a given filter (typically cutting off some frequencies). + +To filter perform the following step: + +- Chose a filtering method: + - Moving average: apply a moving average filter, with + a length specified by the window size (in indices) + - High pass 1st order: apply a first oder high-pass filter, + passing the frequencies above the cutoff frequency parameter. + - Low pass 1st order: apply a first oder low-pass filter, + passing the frequencies below the cutoff frequency parameter. + +- Click on one of the following buttons: + - Plot: will display the filtered data on the figure + - Apply: will perform the filtering on the fly for all new plots + - Add: will create new table(s) with filtered values for all + signals. This process might take some time. + Currently done for all tables. +""") + + +# --------------------------------------------------------------------------------} +# --- Resample +# --------------------------------------------------------------------------------{ +class ResampleToolPanel(GUIToolPanel): + def __init__(self, parent): + super(ResampleToolPanel,self).__init__(parent) + + # --- Data from other modules + from pydatview.tools.signal import SAMPLERS + self.parent = parent # parent is GUIPlotPanel + self._SAMPLERS=SAMPLERS + # Setting default states to parent + if 'Sampler' not in self.parent.plotDataOptions.keys(): + self.parent.plotDataOptions['Sampler']=None + self._applied = type(self.parent.plotDataOptions['Sampler'])==dict + + + # --- GUI elements + self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) + self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) + self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) + self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) + self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) + + #self.lb = wx.StaticText( self, -1, """ Click help """) + self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS], style=wx.CB_READONLY) + + self.lbNewX = wx.StaticText(self, -1, 'New x: ') + self.textNewX = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER) + self.textOldX = wx.TextCtrl(self, wx.ID_ANY|wx.TE_READONLY) + self.textOldX.Enable(False) + + # --- Layout + btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) + btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) + + msizer = wx.FlexGridSizer(rows=2, cols=4, hgap=2, vgap=0) + msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(wx.StaticText(self, -1, 'Current x: '), 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.textOldX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) + msizer.Add(wx.StaticText(self, -1, 'Method:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.cbMethods , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.lbNewX , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.textNewX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) + msizer.AddGrowableCol(3,1) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + self.sizer.Add(msizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) + self.SetSizer(self.sizer) + + # --- Events + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + self.cbMethods.Bind(wx.EVT_COMBOBOX, self.onMethodChange) + self.textNewX.Bind(wx.EVT_TEXT_ENTER,self.onParamChange) + + # --- Init triggers + self.cbMethods.SetSelection(3) + self.onMethodChange(init=True) + self.onToggleApply(init=True) + self.updateTabList() + self.textNewX.SetValue('2') + + def setCurrentX(self, x=None): + if x is None: + x= self.parent.plotData[0].x + if len(x)<50: + s=np.array2string(x, separator=', ') + else: + s =np.array2string(x[[0,1,2,3]], separator=', ') + s+=', ..., ' + s+=np.array2string(x[[-3,-2,-1]], separator=', ') + s=s.replace('[','').replace(']','').replace(' ','').replace(',',', ') + + self.textOldX.SetValue(s) + + def onMethodChange(self, event=None, init=True): + """ Select the method, but does not applied it to the plotData + User data and option is unchanged + But if the user already has some options, they are used + """ + iOpt = self.cbMethods.GetSelection() + opt = self._SAMPLERS[iOpt] + self.lbNewX.SetLabel(opt['paramName']+':') + + parentOpt=self.parent.plotDataOptions['Sampler'] + # Value + if len(self.textNewX.Value)==0: + if type(parentOpt)==dict: + self.textNewX.SetValue(str(parentOpt['param'])[1:-1]) + else: + self.textNewX.SetValue(str(opt['param'])[2:-2]) + self.onParamChange() + + def onParamChange(self, event=None): + if self._applied: + self.parent.plotDataOptions['Sampler'] =self._GUI2Data() + self.parent.load_and_draw() # Data will change + self.setCurrentX() + + def _GUI2Data(self): + iOpt = self.cbMethods.GetSelection() + opt = self._SAMPLERS[iOpt].copy() + s= self.textNewX.Value.strip().replace('[','').replace(']','') + if len(s)>0: + if s.find(','): + opt['param']=np.array(s.split(',')).astype(float) + else: + opt['param']=np.array(s.split('')).astype(float) + return opt + + def onToggleApply(self, event=None, init=False): + """ + apply sampler based on GUI Data + """ + parentFilt=self.parent.plotDataOptions['Sampler'] + if not init: + self._applied = not self._applied + + if self._applied: + self.parent.plotDataOptions['Sampler'] =self._GUI2Data() + #print('Apply', self.parent.plotDataOptions['Sampler']) + #self.lbInfo.SetLabel( + # 'Sampler is now applied on the fly. Change parameter live. Click "Clear" to stop. ' + # ) + self.btPlot.Enable(False) + self.btClear.Enable(False) + self.btApply.SetLabel(CHAR['sun']+' Clear') + else: + self.parent.plotDataOptions['Sampler'] = None + #self.lbInfo.SetLabel( + # 'Click on "Apply" to set filter on the fly for all plots. '+ + # 'Click on "Plot" to try a filter on the current plot.' + # ) + self.btPlot.Enable(True) + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + + if not init: + self.parent.load_and_draw() # Data will change + self.setCurrentX() + + def onAdd(self,event=None): + iSel = self.cbTabs.GetSelection() + tabList = self.parent.selPanel.tabList + mainframe = self.parent.mainframe + icol, colname = self.parent.selPanel.xCol + print(icol,colname) + opt = self._GUI2Data() + errors=[] + if iSel==0: + dfs, names, errors = tabList.applyResampling(icol, opt, bAdd=True) + mainframe.load_dfs(dfs,names,bAdd=True) + else: + df, name = tabList.get(iSel-1).applyResampling(icol, opt, bAdd=True) + mainframe.load_df(df,name,bAdd=True) + self.updateTabList() + + if len(errors)>0: + raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) + + def onPlot(self,event=None): + from pydatview.tools.signal import applySampler + if len(self.parent.plotData)!=1: + Error(self,'Plotting only works for a single plot. Plot less data.') + return + opts=self._GUI2Data() + PD = self.parent.plotData[0] + x_new, y_new = applySampler(PD.x0, PD.y0, opts) + ax = self.parent.fig.axes[0] + + PD_new = PlotData() + PD_new.fromXY(x_new, y_new) + self.parent.transformPlotData(PD_new) + ax.plot(PD_new.x, PD_new.y, '-') + self.setCurrentX(x_new) + + self.parent.canvas.draw() + + def onClear(self,event=None): + self.parent.load_and_draw() # Data will change + # Update Current X + self.setCurrentX() + # Update Table list + self.updateTabList() + + + def onTabChange(self,event=None): + #tabList = self.parent.selPanel.tabList + #iSel=self.cbTabs.GetSelection() + pass + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + try: + iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + except RuntimeError: + pass + + def onHelp(self,event=None): + Info(self,"""Resampling. + +The resampling operation changes the "x" values of a table/plot and +adapt the "y" values accordingly. + +To resample perform the following step: + +- Chose a resampling method: + - replace: specify all the new x-values + - insert : insert a list of x values to the existing ones + - delete : delete a list of x values from the existing ones + - every-n : use every n values + - time-based: downsample using sample averaging or upsample using + linear interpolation, x-axis must already be in seconds + - delta x : specify a delta for uniform spacing of x values + +- Specify the x values as a space or comma separated list + +- Click on one of the following buttons: + - Plot: will display the resampled data on the figure + - Apply: will perform the resampling on the fly for all new plots + - Add: will create new table(s) with resampled values for all + signals. This process might take some time. + Select a table or choose all (default) +""") + + + +# --------------------------------------------------------------------------------} +# --- Mask +# --------------------------------------------------------------------------------{ +class MaskToolPanel(GUIToolPanel): + def __init__(self, parent): + super(MaskToolPanel,self).__init__(parent) + + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + + allMask = tabList.commonMaskString + if len(allMask)==0: + allMask=self.guessMask(tabList) # no known mask, we guess one to help the user + self.applied=False + else: + self.applied=True + + btClose = self.getBtBitmap(self, 'Close','close', self.destroy) + btComp = self.getBtBitmap(self, u'Mask (add)','add' , self.onApply) + if self.applied: + self.btCompMask = self.getToggleBtBitmap(self, 'Clear','sun', self.onToggleApplyMask) + self.btCompMask.SetValue(True) + else: + self.btCompMask = self.getToggleBtBitmap(self, 'Mask','cloud', self.onToggleApplyMask) + + self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'")""") + self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) + self.cbTabs.SetSelection(0) + + self.textMask = wx.TextCtrl(self, wx.ID_ANY, allMask) + #self.textMask.SetValue('({Time}>100) & ({Time}<400)') + #self.textMask.SetValue("{Date} > '2018-10-01'") + + btSizer = wx.FlexGridSizer(rows=2, cols=2, hgap=2, vgap=0) + btSizer.Add(btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(wx.StaticText(self, -1, '') ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btComp ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btCompMask ,0,flag = wx.ALL|wx.EXPAND, border = 1) + + row_sizer = wx.BoxSizer(wx.HORIZONTAL) + row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) + row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Mask:'), 0, wx.CENTER|wx.LEFT, 5) + row_sizer.Add(self.textMask, 1, wx.CENTER|wx.LEFT|wx.EXPAND, 5) + + vert_sizer = wx.BoxSizer(wx.VERTICAL) + vert_sizer.Add(self.lb ,0, flag = wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) + vert_sizer.Add(row_sizer ,1, flag = wx.EXPAND|wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + self.sizer.Add(vert_sizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) + self.SetSizer(self.sizer) + self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) + + def onTabChange(self,event=None): + tabList = self.parent.selPanel.tabList + iSel=self.cbTabs.GetSelection() + if iSel==0: + maskString = tabList.commonMaskString + else: + maskString= tabList.get(iSel-1).maskString + if len(maskString)>0: + self.textMask.SetValue(maskString) + #else: + # self.textMask.SetValue('') # no known mask + # self.textMask.SetValue(self.guessMask) # no known mask + + def guessMask(self,tabList): + cols=[c.lower() for c in tabList.get(0).columns_clean] + if 'time' in cols: + return '{Time} > 100' + elif 'date' in cols: + return "{Date} > '2017-01-01" + else: + return '' + + def onClear(self,event=None): + iSel = self.cbTabs.GetSelection() + tabList = self.parent.selPanel.tabList + mainframe = self.parent.mainframe + if iSel==0: + tabList.clearCommonMask() + else: + tabList.get(iSel-1).clearMask() + + mainframe.redraw() + self.onTabChange() + + def onToggleApplyMask(self,event=None): + self.applied = not self.applied + if self.applied: + self.btCompMask.SetLabel(CHAR['sun']+' Clear') + else: + self.btCompMask.SetLabel(CHAR['cloud']+' Mask') + + if self.applied: + self.onApply(event,bAdd=False) + else: + self.onClear() + + def onApply(self,event=None,bAdd=True): + maskString = self.textMask.GetLineText(0) + iSel = self.cbTabs.GetSelection() + tabList = self.parent.selPanel.tabList + mainframe = self.parent.mainframe + if iSel==0: + dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=bAdd) + if bAdd: + mainframe.load_dfs(dfs,names,bAdd=bAdd) + else: + mainframe.redraw() + if len(errors)>0: + raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) + else: + dfs, name = tabList.get(iSel-1).applyMaskString(maskString, bAdd=bAdd) + if bAdd: + mainframe.load_df(df,name,bAdd=bAdd) + else: + mainframe.redraw() + self.updateTabList() + + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + try: + iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + except RuntimeError: + pass + +# --------------------------------------------------------------------------------} +# --- Radial +# --------------------------------------------------------------------------------{ +sAVG_METHODS = ['Last `n` seconds','Last `n` periods'] +AVG_METHODS = ['constantwindow','periods'] + +class RadialToolPanel(GUIToolPanel): + def __init__(self, parent): + super(RadialToolPanel,self).__init__(parent) + + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + + btClose = self.getBtBitmap(self,'Close' ,'close' , self.destroy) + btComp = self.getBtBitmap(self,'Average','compute', self.onApply) # ART_PLUS + + self.lb = wx.StaticText( self, -1, """Select tables, averaging method and average parameter (`Period` methods uses the `azimuth` signal) """) + self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) + self.cbMethod = wx.ComboBox(self, choices=sAVG_METHODS, style=wx.CB_READONLY) + self.cbTabs.SetSelection(0) + self.cbMethod.SetSelection(0) + + self.textAverageParam = wx.TextCtrl(self, wx.ID_ANY, '2',size = (36,-1), style=wx.TE_PROCESS_ENTER) + + btSizer = wx.FlexGridSizer(rows=2, cols=1, hgap=0, vgap=0) + #btSizer = wx.BoxSizer(wx.VERTICAL) + btSizer.Add(btClose ,0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btComp ,0, flag = wx.ALL|wx.EXPAND, border = 1) + + row_sizer = wx.BoxSizer(wx.HORIZONTAL) + row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) + row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Method:'), 0, wx.CENTER|wx.LEFT, 5) + row_sizer.Add(self.cbMethod , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Param:') , 0, wx.CENTER|wx.LEFT, 5) + row_sizer.Add(self.textAverageParam , 0, wx.CENTER|wx.LEFT|wx.RIGHT| wx.EXPAND, 2) + + vert_sizer = wx.BoxSizer(wx.VERTICAL) + vert_sizer.Add(self.lb ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) + vert_sizer.Add(row_sizer ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + self.sizer.Add(vert_sizer ,0, flag = wx.LEFT|wx.EXPAND,border = TOOL_BORDER) + self.SetSizer(self.sizer) + self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) + + def onTabChange(self,event=None): + tabList = self.parent.selPanel.tabList + + def onApply(self,event=None): + try: + avgParam = float(self.textAverageParam.GetLineText(0)) + except: + raise Exception('Error: the averaging parameter needs to be an integer or a float') + iSel = self.cbTabs.GetSelection() + avgMethod = AVG_METHODS[self.cbMethod.GetSelection()] + tabList = self.parent.selPanel.tabList + mainframe = self.parent.mainframe + if iSel==0: + dfs, names, errors = tabList.radialAvg(avgMethod,avgParam) + mainframe.load_dfs(dfs,names,bAdd=True) + if len(errors)>0: + raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) + else: + dfs, names = tabList.get(iSel-1).radialAvg(avgMethod,avgParam) + mainframe.load_dfs(dfs,names,bAdd=True) + + self.updateTabList() + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + + +# --------------------------------------------------------------------------------} +# --- Curve Fitting +# --------------------------------------------------------------------------------{ +MODELS_EXAMPLE =[ + {'label':'User defined model', 'id':'eval:', + 'formula':'{a}*x**2 + {b}', + 'coeffs':None, + 'consts':None, + 'bounds':None }, + ] +MODELS_EXTRA =[ +# {'label':'Exponential decay', 'id':'eval:', +# 'formula':'{A}*exp(-{k}*x)+{B}', +# 'coeffs' :'k=1, A=1, B=0', +# 'consts' :None, +# 'bounds' :None}, +] + +class CurveFitToolPanel(GUIToolPanel): + def __init__(self, parent): + super(CurveFitToolPanel,self).__init__(parent) + + # Data + self.x = None + self.y_fit = None + + # GUI Objecst + btClose = self.getBtBitmap(self, 'Close','close', self.destroy) + btClear = self.getBtBitmap(self, 'Clear','sun', self.onClear) # DELETE + btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) + btCompFit = self.getBtBitmap(self, 'Fit','check', self.onCurveFit) + btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) + + boldFont = self.GetFont().Bold() + lbOutputs = wx.StaticText(self, -1, 'Outputs') + lbInputs = wx.StaticText(self, -1, 'Inputs ') + lbOutputs.SetFont(boldFont) + lbInputs.SetFont(boldFont) + + self.textFormula = wx.TextCtrl(self, wx.ID_ANY, '') + self.textGuess = wx.TextCtrl(self, wx.ID_ANY, '') + self.textBounds = wx.TextCtrl(self, wx.ID_ANY, '') + self.textConstants = wx.TextCtrl(self, wx.ID_ANY, '') + + self.textFormulaNum = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) + self.textCoeffs = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) + self.textInfo = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) + + + self.Models=copy.deepcopy(MODELS_EXAMPLE) + copy.deepcopy(FITTERS) + copy.deepcopy(MODELS) + copy.deepcopy(MODELS_EXTRA) + sModels=[d['label'] for d in self.Models] + + + self.cbModels = wx.ComboBox(self, choices=sModels, style=wx.CB_READONLY) + self.cbModels.SetSelection(0) + + btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) + btSizer.Add(btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btClear ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btAdd ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btCompFit ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btHelp ,0,flag = wx.ALL|wx.EXPAND, border = 1) + + inputSizer = wx.FlexGridSizer(rows=5, cols=2, hgap=0, vgap=0) + inputSizer.Add(lbInputs ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.cbModels ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.Add(wx.StaticText(self, -1, 'Formula:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.textFormula ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.Add(wx.StaticText(self, -1, 'Guess:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.textGuess ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.Add(wx.StaticText(self, -1, 'Bounds:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.textBounds ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.Add(wx.StaticText(self, -1, 'Constants:'),0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.textConstants ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.AddGrowableCol(1,1) + + outputSizer = wx.FlexGridSizer(rows=5, cols=2, hgap=0, vgap=0) + outputSizer.Add(lbOutputs ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(wx.StaticText(self, -1, '') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(wx.StaticText(self, -1, 'Formula:'),0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(self.textFormulaNum ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + outputSizer.Add(wx.StaticText(self, -1, 'Parameters:') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(self.textCoeffs ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + outputSizer.Add(wx.StaticText(self, -1, 'Accuracy:') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(self.textInfo ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + outputSizer.AddGrowableCol(1,0.5) + + horzSizer = wx.BoxSizer(wx.HORIZONTAL) + horzSizer.Add(inputSizer ,1.0, flag = wx.LEFT|wx.EXPAND,border = 2) + horzSizer.Add(outputSizer ,1.0, flag = wx.LEFT|wx.EXPAND,border = 9) + + vertSizer = wx.BoxSizer(wx.VERTICAL) +# vertSizer.Add(self.lbHelp ,0, flag = wx.LEFT ,border = 1) + vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) +# self.sizer.Add(vertSizerCB ,0, flag = wx.LEFT ,border = 1) + self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) + self.SetSizer(self.sizer) + + self.Bind(wx.EVT_COMBOBOX, self.onModelChange, self.cbModels) + + self.onModelChange() + + def onModelChange(self,event=None): + iModel = self.cbModels.GetSelection() + d = self.Models[iModel] + self.textFormula.SetEditable(True) + + if d['id'].find('fitter:')==0 : + self.textGuess.Enable(False) + self.textGuess.SetValue('') + self.textFormula.Enable(False) + self.textFormula.SetValue(d['formula']) + self.textBounds.Enable(False) + self.textBounds.SetValue('') + self.textConstants.Enable(True) + # NOTE: conversion to string works with list, and tuples, not numpy array + val = ', '.join([k+'='+str(v) for k,v in d['consts'].items()]) + self.textConstants.SetValue(val) + else: + # Formula + if d['id'].find('eval:')==0 : + self.textFormula.Enable(True) + self.textFormula.SetEditable(True) + else: + #self.textFormula.Enable(False) + self.textFormula.Enable(True) + self.textFormula.SetEditable(False) + self.textFormula.SetValue(d['formula']) + + # Guess + if d['coeffs'] is None: + self.textGuess.SetValue('') + else: + self.textGuess.SetValue(d['coeffs']) + + # Constants + if d['consts'] is None or len(d['consts'].strip())==0: + self.textConstants.Enable(False) + self.textConstants.SetValue('') + else: + self.textConstants.Enable(True) + self.textConstants.SetValue(d['consts']) + + # Bounds + self.textBounds.Enable(True) + if d['bounds'] is None or len(d['bounds'].strip())==0: + self.textBounds.SetValue('all=(-np.inf, np.inf)') + else: + self.textBounds.SetValue(d['bounds']) + + # Outputs + self.textFormulaNum.SetValue('(Click on Fit)') + self.textCoeffs.SetValue('') + self.textInfo.SetValue('') + + def onCurveFit(self,event=None): + self.x = None + self.y_fit = None + if len(self.parent.plotData)!=1: + Error(self,'Curve fitting tool only works with a single curve. Plot less data.') + return + PD =self.parent.plotData[0] + ax =self.parent.fig.axes[0] + # Restricting data to axes visible bounds on the x axis + xlim= ax.get_xlim() + b=np.logical_and(PD.x>=xlim[0], PD.x<=xlim[1]) + + iModel = self.cbModels.GetSelection() + d = self.Models[iModel] + + if d['id'].find('fitter:')==0 : + sFunc=d['id'] + p0=None + bounds=None + fun_kwargs=extract_key_miscnum(self.textConstants.GetLineText(0).replace('np.inf','inf')) + else: + # Formula + sFunc=d['id'] + if sFunc=='eval:': + sFunc+=self.textFormula.GetLineText(0) + # Bounds + bounds=self.textBounds.GetLineText(0).replace('np.inf','inf') + # Guess + p0=self.textGuess.GetLineText(0).replace('np.inf','inf') + fun_kwargs=extract_key_num(self.textConstants.GetLineText(0).replace('np.inf','inf')) + #print('>>> Model fit sFunc :',sFunc ) + #print('>>> Model fit p0 :',p0 ) + #print('>>> Model fit bounds:',bounds ) + #print('>>> Model fit kwargs:',fun_kwargs) + # Performing fit + y_fit, pfit, fitter = model_fit(sFunc, PD.x[b], PD.y[b], p0=p0, bounds=bounds,**fun_kwargs) + + formatter = lambda x: pretty_num_short(x, digits=3) + formula_num = fitter.formula_num(fmt=formatter) + # Update info + self.textFormulaNum.SetValue(formula_num) + self.textCoeffs.SetValue(', '.join(['{}={:s}'.format(k,formatter(v)) for k,v in fitter.model['coeffs'].items()])) + self.textInfo.SetValue('R2 = {:.3f} '.format(fitter.model['R2'])) + + # Saving + d['formula'] = self.textFormula.GetLineText(0) + d['bounds'] = self.textBounds.GetLineText(0).replace('np.inf','inf') + d['coeffs'] = self.textGuess.GetLineText(0).replace('np.inf','inf') + if d['id'].find('fitter:')==0 : + d['consts'], _ = set_common_keys(d['consts'],fun_kwargs) + else: + d['consts']= self.textConstants.GetLineText(0).replace('np.inf','inf') + + + # Plot + ax=self.parent.fig.axes[0] + ax.plot(PD.x[b],y_fit,'o', ms=4) + self.parent.canvas.draw() + + self.x=PD.x[b] + self.y_fit=y_fit + self.sx=PD.sx + self.sy=PD.sy + + def onClear(self,event=None): + self.parent.load_and_draw() # DATA HAS CHANGED + self.onModelChange() + + def onAdd(self,event=None): + name='model_fit' + if self.x is not None and self.y_fit is not None: + df=pd.DataFrame({self.sx:self.x, self.sy:self.y_fit}) + self.parent.mainframe.load_df(df,name,bAdd=True) + + def onHelp(self,event=None): + Info(self,"""Curve fitting is still in beta. + +To perform a curve fit, adjusts the "Inputs section on the left": +- Select a predefined equation to fit, using the scrolldown menu. +- Adjust the initial gues for the parameters (if wanted) +- (Only for few models: set constants values) +- Click on "Fit" + +If you select a user-defined model: +- Equation parameters are specified using curly brackets +- Numpy functions are available using "np" + +Buttons: +- Clear: remove the fit from the plot +- Add: add the fit data to the list of tables (can then be exported) + +""") + + + +TOOLS={ + 'LogDec': LogDecToolPanel, + 'Outlier': OutlierToolPanel, + 'Filter': FilterToolPanel, + 'Resample': ResampleToolPanel, + 'Mask': MaskToolPanel, + 'FASTRadialAverage': RadialToolPanel, + 'CurveFitting': CurveFitToolPanel, +} diff --git a/pydatview/Tables.py b/pydatview/Tables.py index e77525b..7974b75 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -1,627 +1,731 @@ -import numpy as np -import os.path -from dateutil import parser -import pandas as pd -import pydatview.fast.fastlib as fastlib -import pydatview.fast.fastfarm as fastfarm -try: - from .common import no_unit, ellude_common, getDt -except: - from common import no_unit, ellude_common, getDt -try: - import weio # File Formats and File Readers -except: - print('') - print('Error: the python package `weio` was not imported successfully.\n') - print('Most likely the submodule `weio` was not cloned with `pyDatView`') - print('Type the following command to retrieve it:\n') - print(' git submodule update --init --recursive\n') - print('Alternatively re-clone this repository into a separate folder:\n') - print(' git clone --recurse-submodules https://github.com/ebranlard/pyDatView\n') - sys.exit(-1) - - - -# --------------------------------------------------------------------------------} -# --- TabList -# --------------------------------------------------------------------------------{ -class TableList(object): # todo inherit list - def __init__(self,tabs=[]): - self._tabs=tabs - self.Naming='Ellude' - - def append(self,t): - if isinstance(t,list): - self._tabs += t - else: - self._tabs += [t] - - - def from_dataframes(self, dataframes=[], names=[], bAdd=False): - if not bAdd: - self.clean() # TODO figure it out - # Returning a list of tables - for df,name in zip(dataframes, names): - if df is not None: - self.append(Table(data=df, name=name)) - - def load_tables_from_files(self, filenames=[], fileformat=None, bAdd=False): - """ load multiple files, only trigger the plot at the end """ - if not bAdd: - self.clean() # TODO figure it out - warnList=[] - for f in filenames: - if f in self.unique_filenames: - warn.append('Warn: Cannot add a file already opened ' + f) - elif len(f)==0: - pass - # warn+= 'Warn: an empty filename was skipped' +'\n' - else: - tabs, warnloc = self._load_file_tabs(f,fileformat=fileformat) - if len(warnloc)>0: - warnList.append(warnloc) - self.append(tabs) - - return warnList - - def _load_file_tabs(self,filename,fileformat=None): - """ load a single file, adds table, and potentially trigger plotting """ - # Returning a list of tables - tabs=[] - warn='' - if not os.path.isfile(filename): - warn = 'Error: File not found: `'+filename+'`\n' - return tabs, warn - try: - F = weio.read(filename,fileformat = fileformat) - dfs = F.toDataFrame() - except weio.FileNotFoundError as e: - warn = 'Error: A file was not found!\n\n While opening:\n\n {}\n\n the following file was not found:\n\n {}\n'.format(filename, e.filename) - except IOError: - warn = 'Error: IO Error thrown while opening file: '+filename+'\n' - except MemoryError: - warn='Error: Insufficient memory!\n\nFile: '+filename+'\n\nTry closing and reopening the program, or use a 64 bit version of this program (i.e. of python).\n' - except weio.EmptyFileError: - warn='Error: File empty!\n\nFile is empty: '+filename+'\n\nOpen a different file.\n' - except weio.FormatNotDetectedError: - warn='Error: File format not detected!\n\nFile: '+filename+'\n\nUse an explicit file-format from the list\n' - except weio.WrongFormatError as e: - warn='Error: Wrong file format!\n\nFile: '+filename+'\n\n' \ - 'The file parser for the selected format failed to open the file.\n\n'+ \ - 'The reported error was:\n'+e.args[0]+'\n\n' + \ - 'Double-check your file format and report this error if you think it''s a bug.\n' - except weio.BrokenFormatError as e: - warn = 'Error: Inconsistency in the file format!\n\nFile: '+filename+'\n\n' \ - 'The reported error was:\n\n'+e.args[0]+'\n\n' + \ - 'Double-check your file format and report this error if you think it''s a bug.' - except: - raise - if len(warn)>0: - return tabs, warn - - if dfs is None: - pass - elif not isinstance(dfs,dict): - if len(dfs)>0: - tabs=[Table(data=dfs, filename=filename, fileformat=F.formatName())] - else: - for k in list(dfs.keys()): - if len(dfs[k])>0: - tabs.append(Table(data=dfs[k], name=str(k), filename=filename, fileformat=F.formatName())) - if len(tabs)<=0: - warn='Warn: No dataframe found in file: '+filename+'\n' - return tabs, warn - - def getTabs(self): - # TODO remove me later - return self._tabs - - def len(self): - return len(self._tabs) - - def haveSameColumns(self,I=None): - if I is None: - I=list(range(len(self._tabs))) - A=[len(self._tabs[i].columns)==len(self._tabs[I[0]].columns) for i in I ] - if all(A): - B=[self._tabs[i].columns_clean==self._tabs[I[0]].columns_clean for i in I] #list comparison - return all(B) - else: - return False - - def renameTable(self, iTab, newName): - oldName = self._tabs[iTab].name - if newName in [t.name for t in self._tabs]: - raise Exception('Error: This table already exist, choose a different name.') - # Renaming table - self._tabs[iTab].rename(newName) - return oldName - - def sort(self, method='byName'): - if method=='byName': - tabnames_display=self.getDisplayTabNames() - self._tabs = [t for _,t in sorted(zip(tabnames_display,self._tabs))] - else: - raise Exception('Sorting method unknown: `{}`'.format(method)) - - def deleteTabs(self, I): - self._tabs = [t for i,t in enumerate(self._tabs) if i not in I] - - def setActiveNames(self,names): - for t,tn in zip(self._tabs,names): - t.active_name=tn - - def setNaming(self,naming): - self.Naming=naming - - def getDisplayTabNames(self): - if self.Naming=='Ellude': - # Temporary hack, using last names if all last names are unique - names = [t.raw_name for t in self._tabs] - last_names=[n.split('|')[-1] for n in names] - if len(np.unique(last_names)) == len(names): - return ellude_common(last_names) - else: - return ellude_common(names) - elif self.Naming=='FileNames': - return [os.path.splitext(os.path.basename(t.filename))[0] for t in self._tabs] - else: - raise Exception('Table naming unknown: {}'.format(self.Naming)) - - @property - def tabNames(self): - return [t.name for t in self._tabs] - - @property - def filenames(self): - return [t.filename for t in self._tabs] - - @property - def unique_filenames(self): - return list(set([t.filename for t in self._tabs])) - - def clean(self): - del self._tabs - self._tabs=[] - - def __repr__(self): - return '\n'.join([t.__repr__() for t in self._tabs]) - - # --- Mask related - @property - def maskStrings(self): - return [t.maskString for t in self._tabs] - - @property - def commonMaskString(self): - maskStrings=set(self.maskStrings) - if len(maskStrings) == 1: - return next(iter(maskStrings)) - else: - return '' - - def clearCommonMask(self): - for t in self._tabs: - t.clearMask() - - def applyCommonMaskString(self,maskString,bAdd=True): - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - df_new, name_new = t.applyMaskString(maskString, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) - except: - errors.append('Mask failed for table: '+t.active_name) # TODO - - return dfs_new, names_new, errors - - def applyResampling(self,iCol,sampDict,bAdd=True): - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): -# try: - df_new, name_new = t.applyResampling(iCol,sampDict, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) -# except: -# errors.append('Resampling failed for table: '+t.active_name) # TODO - - return dfs_new, names_new, errors - - - - - # --- Radial average related - def radialAvg(self,avgMethod,avgParam): - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - dfs, names = t.radialAvg(avgMethod,avgParam) - for df,n in zip(dfs,names): - if df is not None: - dfs_new.append(df) - names_new.append(n) - return dfs_new, names_new, errors - - - # --- Element--related functions - def get(self,i): - return self._tabs[i] - - - -# --------------------------------------------------------------------------------} -# --- Table -# --------------------------------------------------------------------------------{ -# TODO sort out the naming -# -# Main naming concepts: -# name : -# active_name : -# raw_name : -# filename : -class Table(object): - def __init__(self,data=None,name='',filename='',columns=[],fileformat=''): - # Default init - self.maskString='' - self.mask=None - - self.filename = filename - self.fileformat = fileformat - self.formulas = [] - - if not isinstance(data,pd.DataFrame): - # ndarray?? - raise NotImplementedError('Tables that are not dataframe not implemented.') - else: - # --- Pandas DataFrame - self.data = data - self.columns = self.columnsFromDF(data) - # --- Trying to figure out how to name this table - if name is None or len(str(name))==0: - if data.columns.name is not None: - name=data.columns.name - - self.setupName(name=str(name)) - - self.convertTimeColumns() - - - def setupName(self,name=''): - # Creates a "codename": path | basename | name | ext - splits=[] - ext='' - if len(self.filename)>0: - base_dir = os.path.dirname(self.filename) - if len(base_dir)==0: - base_dir= os.getcwd() - self.filename=os.path.join(base_dir,self.filename) - splits.append(base_dir.replace('/','|').replace('\\','|')) - basename,ext=os.path.splitext(os.path.basename(self.filename)) - if len(basename)>0: - splits.append(basename) - if name is not None and len(name)>0: - splits.append(name) - #if len(ext)>0: - # splits.append(ext) - self.extension=ext - name='|'.join(splits) - if len(name)==0: - name='default' - self.name=name - self.active_name=self.name - - - def __repr__(self): - return 'Tab {} ({}x{}) (raw: {}, active: {}, file: {})'.format(self.name,self.nCols,self.nRows,self.raw_name, self.active_name,self.filename) - - def columnsFromDF(self,df): - return [s.replace('_',' ') for s in df.columns.values.astype(str)] - - - def clearMask(self): - self.maskString='' - self.mask=None - - def addLabelToName(self,label): - print('raw_name',self.raw_name) - raw_name=self.raw_name - sp=raw_name.split('|') - print(sp) - - def applyMaskString(self,maskString,bAdd=True): - df = self.data - Index = np.array(range(df.shape[0])) - sMask=maskString.replace('{Index}','Index') - for i,c in enumerate(self.columns): - c_no_unit = no_unit(c).strip() - c_in_df = df.columns[i] - # TODO sort out the mess with asarray (introduced to have and/or - # as array won't work with date comparison - # NOTE: using iloc to avoid duplicates column issue - if isinstance(df.iloc[0,i], pd._libs.tslibs.timestamps.Timestamp): - sMask=sMask.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') - else: - sMask=sMask.replace('{'+c_no_unit+'}','np.asarray(df[\''+c_in_df+'\'])') - df_new = None - name_new = None - if len(sMask.strip())>0 and sMask.strip().lower()!='no mask': - try: - mask = np.asarray(eval(sMask)) - if bAdd: - df_new = df[mask] - name_new=self.raw_name+'_masked' - else: - self.mask=mask - self.maskString=maskString - except: - raise Exception('Error: The mask failed for table: '+self.name) - return df_new, name_new - - def applyResampling(self,iCol,sampDict,bAdd=True): - from pydatview.tools.signal import applySamplerDF - if iCol==0: - raise Exception('Cannot resample based on index') - colName=self.data.columns[iCol-1] - df_new =applySamplerDF(self.data, colName, sampDict=sampDict) - df_new - if bAdd: - name_new=self.raw_name+'_resampled' - else: - name_new=None - self.data=df_new - return df_new, name_new - - - def radialAvg(self,avgMethod, avgParam): - df = self.data - base,out_ext = os.path.splitext(self.filename) - - # --- Detect if it's a FAST Farm file - sCols = ''.join(df.columns) - if sCols.find('WkDf')>1 or sCols.find('CtT')>0: - # --- FAST FARM files - Files=[base+ext for ext in ['.fstf','.FSTF','.Fstf','.fmas','.FMAS','.Fmas'] if os.path.exists(base+ext)] - if len(Files)==0: - fst_in=None - #raise Exception('Error: No .fstf file found with name: '+base+'.fstf') - else: - fst_in=Files[0] - - dfRad,_,dfDiam = fastfarm.spanwisePostProFF(fst_in,avgMethod=avgMethod,avgParam=avgParam,D=1,df=df) - dfs_new = [dfRad,dfDiam] - names_new=[self.raw_name+'_rad',self.raw_name+'_diam'] - else: - # --- FAST files - - # HACK for AD file to find the right .fst file - iDotAD=base.lower().find('.ad') - if iDotAD>1: - base=base[:iDotAD] - # - Files=[base+ext for ext in ['.fst','.FST','.Fst','.dvr','.Dvr','.DVR'] if os.path.exists(base+ext)] - if len(Files)==0: - fst_in=None - #raise Exception('Error: No .fst file found with name: '+base+'.fst') - else: - fst_in=Files[0] - - - dfRadED, dfRadAD, dfRadBD= fastlib.spanwisePostPro(fst_in, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df = self.data) - dfs_new = [dfRadAD, dfRadED, dfRadBD] - names_new=[self.raw_name+'_AD', self.raw_name+'_ED', self.raw_name+'_BD'] - return dfs_new, names_new - - def convertTimeColumns(self): - if len(self.data)>0: - for i,c in enumerate(self.data.columns.values): - y = self.data.iloc[:,i] - if y.dtype == np.object: - if isinstance(y.values[0], str): - # tring to convert to date - try: - parser.parse(y.values[0]) - isDate=True - except: - if y.values[0]=='NaT': - isDate=True - else: - isDate=False - if isDate: - try: - self.data[c]=pd.to_datetime(self.data[c].values).to_pydatetime() - print('Column {} converted to datetime'.format(c)) - except: - # Happens if values are e.g. "Monday, Tuesday" - print('Conversion to datetime failed, column {} inferred as string'.format(c)) - else: - print('Column {} inferred as string'.format(c)) - elif isinstance(y.values[0], (float, int)): - try: - self.data[c]=self.data[c].astype(float) - print('Column {} converted to float (likely nan)'.format(c)) - except: - self.data[c]=self.data[c].astype(str) - print('Column {} inferred and converted to string'.format(c)) - else : - print('>> Unknown type:',type(y.values[0])) - #print(self.data.dtypes) - - def renameColumn(self,iCol,newName): - self.columns[iCol]=newName - self.data.columns.values[iCol]=newName - - def deleteColumns(self,ICol): - """ Delete columns by index, not column names which can have duplicates""" - IKeep =[i for i in np.arange(self.data.shape[1]) if i not in ICol] - self.data = self.data.iloc[:, IKeep] # Drop won't work for duplicates - for i in sorted(ICol, reverse=True): - del(self.columns[i]) - for f in self.formulas: - if f['pos'] == (i + 1): - self.formulas.remove(f) - break - for f in self.formulas: - if f['pos'] > (i + 1): - f['pos'] = f['pos'] - 1 - - def rename(self,new_name): - self.name='>'+new_name - - def addColumn(self,sNewName,NewCol,i=-1,sFormula=''): - if i<0: - i=self.data.shape[1] - self.data.insert(int(i),sNewName,NewCol) - self.columns=self.columnsFromDF(self.data) - for f in self.formulas: - if f['pos'] > i: - f['pos'] = f['pos'] + 1 - self.formulas.append({'pos': i+1, 'formula': sFormula, 'name': sNewName}) - - def setColumn(self,sNewName,NewCol,i,sFormula=''): - if i<1: - raise ValueError('Cannot set column at position ' + str(i)) - self.data = self.data.drop(columns=self.data.columns[i-1]) - self.data.insert(int(i-1),sNewName,NewCol) - self.columns=self.columnsFromDF(self.data) - for f in self.formulas: - if f['pos'] == i: - f['name'] = sNewName - f['formula'] = sFormula - - def getColumn(self,i): - """ Return column of data, where i=0 is the index column - If a mask exist, the mask is applied - """ - if i <= 0 : - x = np.array(range(self.data.shape[0])) - if self.mask is not None: - x=x[self.mask] - - c = None - isString = False - isDate = False - else: - if self.mask is not None: - c = self.data.iloc[self.mask, i-1] - x = self.data.iloc[self.mask, i-1].values - else: - c = self.data.iloc[:, i-1] - x = self.data.iloc[:, i-1].values - - isString = c.dtype == np.object and isinstance(c.values[0], str) - if isString: - x=x.astype(str) - isDate = np.issubdtype(c.dtype, np.datetime64) - if isDate: - dt=getDt(x) - if dt>1: - x=x.astype('datetime64[s]') - else: - x=x.astype('datetime64') - return x,isString,isDate,c - - - - def evalFormula(self,sFormula): - df = self.data - Index = np.array(range(df.shape[0])) - sFormula=sFormula.replace('{Index}','Index') - for i,c in enumerate(self.columns): - c_no_unit = no_unit(c).strip() - c_in_df = df.columns[i] - sFormula=sFormula.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') - #print(sFormula) - try: - NewCol=eval(sFormula) - return NewCol - except: - return None - - def addColumnByFormula(self,sNewName,sFormula,i=-1): - NewCol=self.evalFormula(sFormula) - if NewCol is None: - return False - else: - self.addColumn(sNewName,NewCol,i,sFormula) - return True - - def setColumnByFormula(self,sNewName,sFormula,i=-1): - NewCol=self.evalFormula(sFormula) - if NewCol is None: - return False - else: - self.setColumn(sNewName,NewCol,i,sFormula) - return True - - - def export(self,path): - if isinstance(self.data, pd.DataFrame): - try: - self.data.to_csv(path,sep=',',index=False) #python3 - except: - self.data.to_csv(path,sep=str(u',').encode('utf-8'),index=False) #python 2. - else: - raise NotImplementedError('Export of data that is not a dataframe') - - - - @property - def basename(self): - return os.path.splitext(os.path.basename(self.filename))[0] - - @property - def shapestring(self): - return '{}x{}'.format(self.nCols, self.nRows) - - @property - def shape(self): - return (self.nRows, self.nCols) - - - @property - def columns_clean(self): - return [no_unit(s) for s in self.columns] - - @property - def name(self): - if len(self.__name)<=0: - return '' - if self.__name[0]=='>': - return self.__name[1:] - else: - return self.__name - - @property - def raw_name(self): - return self.__name - - @name.setter - def name(self,new_name): - self.__name=new_name - - @property - def nCols(self): - return len(self.columns) - - @property - def nRows(self): - return len(self.data.iloc[:,0]) # TODO if not panda - - -if __name__ == '__main__': - import pandas as pd; - from Tables import Table - import numpy as np - - def OnTabPopup(event): - self.PopupMenu(TablePopup(self,selPanel.tabPanel.lbTab), event.GetPosition()) +import numpy as np +import os.path +from dateutil import parser +import pandas as pd +try: + from .common import no_unit, ellude_common, getDt +except: + from common import no_unit, ellude_common, getDt +try: + import weio.weio as weio# File Formats and File Readers +except: + print('') + print('Error: the python package `weio` was not imported successfully.\n') + print('Most likely the submodule `weio` was not cloned with `pyDatView`') + print('Type the following command to retrieve it:\n') + print(' git submodule update --init --recursive\n') + print('Alternatively re-clone this repository into a separate folder:\n') + print(' git clone --recurse-submodules https://github.com/ebranlard/pyDatView\n') + sys.exit(-1) + + + +# --------------------------------------------------------------------------------} +# --- TabList +# --------------------------------------------------------------------------------{ +class TableList(object): # todo inherit list + def __init__(self,tabs=[]): + self._tabs=tabs + self.Naming='Ellude' + + # --- behaves like a list... + def __iter__(self): + self.__n = 0 + return self + + def __next__(self): + if self.__n < len(self._tabs): + self.__n += 1 + return self._tabs[self.__n-1] + else: + raise StopIteration + + def append(self,t): + if isinstance(t,list): + self._tabs += t + else: + self._tabs += [t] + + # --- Main high level methods + def from_dataframes(self, dataframes=[], names=[], bAdd=False): + if not bAdd: + self.clean() # TODO figure it out + # Returning a list of tables + for df,name in zip(dataframes, names): + if df is not None: + self.append(Table(data=df, name=name)) + + def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False): + """ load multiple files into table list""" + if not bAdd: + self.clean() # TODO figure it out + + if fileformats is None: + fileformats=[None]*len(filenames) + assert type(fileformats) ==list, 'fileformats must be a list' + + # Loop through files, appending tables within files + warnList=[] + for f,ff in zip(filenames, fileformats): + if f in self.unique_filenames: + warnList.append('Warn: Cannot add a file already opened ' + f) + elif len(f)==0: + pass + # warn+= 'Warn: an empty filename was skipped' +'\n' + else: + tabs, warnloc = self._load_file_tabs(f,fileformat=ff) + if len(warnloc)>0: + warnList.append(warnloc) + self.append(tabs) + + return warnList + + def _load_file_tabs(self, filename, fileformat=None): + """ load a single file, adds table """ + # Returning a list of tables + tabs=[] + warn='' + if not os.path.isfile(filename): + warn = 'Error: File not found: `'+filename+'`\n' + return tabs, warn + try: + #F = weio.read(filename, fileformat = fileformat) + # --- Expanded version of weio.read + F = None + if fileformat is None: + fileformat, F = weio.detectFormat(filename) + # Reading the file with the appropriate class if necessary + if not isinstance(F, fileformat.constructor): + F=fileformat.constructor(filename=filename) + dfs = F.toDataFrame() + except weio.FileNotFoundError as e: + warn = 'Error: A file was not found!\n\n While opening:\n\n {}\n\n the following file was not found:\n\n {}\n'.format(filename, e.filename) + except IOError: + warn = 'Error: IO Error thrown while opening file: '+filename+'\n' + except MemoryError: + warn='Error: Insufficient memory!\n\nFile: '+filename+'\n\nTry closing and reopening the program, or use a 64 bit version of this program (i.e. of python).\n' + except weio.EmptyFileError: + warn='Error: File empty!\n\nFile is empty: '+filename+'\n\nOpen a different file.\n' + except weio.FormatNotDetectedError: + warn='Error: File format not detected!\n\nFile: '+filename+'\n\nUse an explicit file-format from the list\n' + except weio.WrongFormatError as e: + warn='Error: Wrong file format!\n\nFile: '+filename+'\n\n' \ + 'The file parser for the selected format failed to open the file.\n\n'+ \ + 'The reported error was:\n'+e.args[0]+'\n\n' + \ + 'Double-check your file format and report this error if you think it''s a bug.\n' + except weio.BrokenFormatError as e: + warn = 'Error: Inconsistency in the file format!\n\nFile: '+filename+'\n\n' \ + 'The reported error was:\n\n'+e.args[0]+'\n\n' + \ + 'Double-check your file format and report this error if you think it''s a bug.' + except: + raise + if len(warn)>0: + return tabs, warn + + if dfs is None: + pass + elif not isinstance(dfs,dict): + if len(dfs)>0: + tabs=[Table(data=dfs, filename=filename, fileformat=fileformat)] + else: + for k in list(dfs.keys()): + if len(dfs[k])>0: + tabs.append(Table(data=dfs[k], name=str(k), filename=filename, fileformat=fileformat)) + if len(tabs)<=0: + warn='Warn: No dataframe found in file: '+filename+'\n' + return tabs, warn + + def getTabs(self): + # TODO remove me later + return self._tabs + + def len(self): + return len(self._tabs) + + def haveSameColumns(self,I=None): + if I is None: + I=list(range(len(self._tabs))) + A=[len(self._tabs[i].columns)==len(self._tabs[I[0]].columns) for i in I ] + if all(A): + B=[self._tabs[i].columns_clean==self._tabs[I[0]].columns_clean for i in I] #list comparison + return all(B) + else: + return False + + def renameTable(self, iTab, newName): + oldName = self._tabs[iTab].name + if newName in [t.name for t in self._tabs]: + raise Exception('Error: This table already exist, choose a different name.') + # Renaming table + self._tabs[iTab].rename(newName) + return oldName + + def sort(self, method='byName'): + if method=='byName': + tabnames_display=self.getDisplayTabNames() + self._tabs = [t for _,t in sorted(zip(tabnames_display,self._tabs))] + else: + raise Exception('Sorting method unknown: `{}`'.format(method)) + + def deleteTabs(self, I): + self._tabs = [t for i,t in enumerate(self._tabs) if i not in I] + + def setActiveNames(self,names): + for t,tn in zip(self._tabs,names): + t.active_name=tn + + def setNaming(self,naming): + self.Naming=naming + + def getDisplayTabNames(self): + if self.Naming=='Ellude': + # Temporary hack, using last names if all last names are unique + names = [t.raw_name for t in self._tabs] + last_names=[n.split('|')[-1] for n in names] + if len(np.unique(last_names)) == len(names): + return ellude_common(last_names) + else: + return ellude_common(names) + elif self.Naming=='FileNames': + return [os.path.splitext(os.path.basename(t.filename))[0] for t in self._tabs] + else: + raise Exception('Table naming unknown: {}'.format(self.Naming)) + + # --- Properties + @property + def tabNames(self): + return [t.name for t in self._tabs] + + @property + def filenames(self): + return [t.filename for t in self._tabs] + + @property + def fileformats(self): + return [t.fileformat for t in self._tabs] + + @property + def unique_filenames(self): + return list(set([t.filename for t in self._tabs])) + + @property + def filenames_and_formats(self): + """ return unique list of filenames with associated fileformats """ + filenames = [] + fileformats = [] + for t in self._tabs: + if t.filename not in filenames: + filenames.append(t.filename) + fileformats.append(t.fileformat) + return filenames, fileformats + + def clean(self): + del self._tabs + self._tabs=[] + + def __repr__(self): + return '\n'.join([t.__repr__() for t in self._tabs]) + + # --- Mask related + @property + def maskStrings(self): + return [t.maskString for t in self._tabs] + + @property + def commonMaskString(self): + maskStrings=set(self.maskStrings) + if len(maskStrings) == 1: + return next(iter(maskStrings)) + else: + return '' + + def clearCommonMask(self): + for t in self._tabs: + t.clearMask() + + def applyCommonMaskString(self,maskString,bAdd=True): + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(self._tabs): + try: + df_new, name_new = t.applyMaskString(maskString, bAdd=bAdd) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) + except: + errors.append('Mask failed for table: '+t.active_name) # TODO + + return dfs_new, names_new, errors + + # --- Resampling TODO MOVE THIS OUT OF HERE OR UNIFY + def applyResampling(self,iCol,sampDict,bAdd=True): + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(self._tabs): +# try: + df_new, name_new = t.applyResampling(iCol,sampDict, bAdd=bAdd) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) +# except: +# errors.append('Resampling failed for table: '+t.active_name) # TODO + return dfs_new, names_new, errors + + # --- Filtering TODO MOVE THIS OUT OF HERE OR UNIFY + def applyFiltering(self,iCol,options,bAdd=True): + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(self._tabs): +# try: + df_new, name_new = t.applyFiltering(iCol, options, bAdd=bAdd) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) +# except: +# errors.append('Resampling failed for table: '+t.active_name) # TODO + return dfs_new, names_new, errors + + + + + # --- Radial average related + def radialAvg(self,avgMethod,avgParam): + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(self._tabs): + dfs, names = t.radialAvg(avgMethod,avgParam) + for df,n in zip(dfs,names): + if df is not None: + dfs_new.append(df) + names_new.append(n) + return dfs_new, names_new, errors + + + # --- Element--related functions + def get(self,i): + return self._tabs[i] + + + +# --------------------------------------------------------------------------------} +# --- Table +# --------------------------------------------------------------------------------{ +# +class Table(object): + """ + Main attributes: + - data + - columns + - name + - raw_name + - active_name + - filename + - fileformat + - fileformat_name + - nCols x nRows + - mask + - maskString + - formulas + """ + # TODO sort out the naming + # Main naming concepts: + # name : + # active_name : + # raw_name : + # filename : + def __init__(self,data=None,name='',filename='',columns=[], fileformat=None): + # Default init + self.maskString='' + self.mask=None + + self.filename = filename + self.fileformat = fileformat + if fileformat is not None: + self.fileformat_name = fileformat.name + else: + self.fileformat_name = '' + self.formulas = [] + + if not isinstance(data,pd.DataFrame): + # ndarray?? + raise NotImplementedError('Tables that are not dataframe not implemented.') + else: + # --- Pandas DataFrame + self.data = data + self.columns = self.columnsFromDF(data) + # --- Trying to figure out how to name this table + if name is None or len(str(name))==0: + if data.columns.name is not None: + name=data.columns.name + + self.setupName(name=str(name)) + + self.convertTimeColumns() + + + def setupName(self,name=''): + # Creates a "codename": path | basename | name | ext + splits=[] + ext='' + if len(self.filename)>0: + base_dir = os.path.dirname(self.filename) + if len(base_dir)==0: + base_dir= os.getcwd() + self.filename=os.path.join(base_dir,self.filename) + splits.append(base_dir.replace('/','|').replace('\\','|')) + basename,ext=os.path.splitext(os.path.basename(self.filename)) + if len(basename)>0: + splits.append(basename) + if name is not None and len(name)>0: + splits.append(name) + #if len(ext)>0: + # splits.append(ext) + self.extension=ext + name='|'.join(splits) + if len(name)==0: + name='default' + self.name=name + self.active_name=self.name + + + def __repr__(self): + s='Table object:\n' + s+=' - name: {}\n'.format(self.name) + s+=' - raw_name : {}\n'.format(self.raw_name) + s+=' - active_name: {}\n'.format(self.raw_name) + s+=' - filename : {}\n'.format(self.filename) + s+=' - fileformat : {}\n'.format(self.fileformat) + s+=' - fileformat_name : {}\n'.format(self.fileformat_name) + s+=' - columns : {}\n'.format(self.columns) + s+=' - nCols x nRows: {}x{}\n'.format(self.nCols, self.nRows) + return s + + def columnsFromDF(self,df): + return [s.replace('_',' ') for s in df.columns.values.astype(str)] + + # --- Mask + def clearMask(self): + self.maskString='' + self.mask=None + + def applyMaskString(self,maskString,bAdd=True): + df = self.data + Index = np.array(range(df.shape[0])) + sMask=maskString.replace('{Index}','Index') + for i,c in enumerate(self.columns): + c_no_unit = no_unit(c).strip() + c_in_df = df.columns[i] + # TODO sort out the mess with asarray (introduced to have and/or + # as array won't work with date comparison + # NOTE: using iloc to avoid duplicates column issue + if isinstance(df.iloc[0,i], pd._libs.tslibs.timestamps.Timestamp): + sMask=sMask.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') + else: + sMask=sMask.replace('{'+c_no_unit+'}','np.asarray(df[\''+c_in_df+'\'])') + df_new = None + name_new = None + if len(sMask.strip())>0 and sMask.strip().lower()!='no mask': + try: + mask = np.asarray(eval(sMask)) + if bAdd: + df_new = df[mask] + name_new=self.raw_name+'_masked' + else: + self.mask=mask + self.maskString=maskString + except: + raise Exception('Error: The mask failed for table: '+self.name) + return df_new, name_new + + # --- Important manipulation TODO MOVE THIS OUT OF HERE OR UNIFY + def applyResampling(self, iCol, sampDict, bAdd=True): + from pydatview.tools.signal import applySamplerDF + if iCol==0: + raise Exception('Cannot resample based on index') + colName=self.data.columns[iCol-1] + df_new =applySamplerDF(self.data, colName, sampDict=sampDict) + df_new + if bAdd: + name_new=self.raw_name+'_resampled' + else: + name_new=None + self.data=df_new + return df_new, name_new + + def applyFiltering(self, iCol, options, bAdd=True): + from pydatview.tools.signal import applyFilterDF + if iCol==0: + raise Exception('Cannot filter based on index') + colName=self.data.columns[iCol-1] + df_new =applyFilterDF(self.data, colName, options) + df_new + if bAdd: + name_new=self.raw_name+'_filtered' + else: + name_new=None + self.data=df_new + return df_new, name_new + + + def radialAvg(self,avgMethod, avgParam): + import pydatview.fast.fastlib as fastlib + import pydatview.fast.fastfarm as fastfarm + df = self.data + base,out_ext = os.path.splitext(self.filename) + + # --- Detect if it's a FAST Farm file + sCols = ''.join(df.columns) + if sCols.find('WkDf')>1 or sCols.find('CtT')>0: + # --- FAST FARM files + Files=[base+ext for ext in ['.fstf','.FSTF','.Fstf','.fmas','.FMAS','.Fmas'] if os.path.exists(base+ext)] + if len(Files)==0: + fst_in=None + #raise Exception('Error: No .fstf file found with name: '+base+'.fstf') + else: + fst_in=Files[0] + + dfRad,_,dfDiam = fastfarm.spanwisePostProFF(fst_in,avgMethod=avgMethod,avgParam=avgParam,D=1,df=df) + dfs_new = [dfRad,dfDiam] + names_new=[self.raw_name+'_rad',self.raw_name+'_diam'] + else: + # --- FAST files + + # HACK for AD file to find the right .fst file + iDotAD=base.lower().find('.ad') + if iDotAD>1: + base=base[:iDotAD] + # + Files=[base+ext for ext in ['.fst','.FST','.Fst','.dvr','.Dvr','.DVR'] if os.path.exists(base+ext)] + if len(Files)==0: + fst_in=None + #raise Exception('Error: No .fst file found with name: '+base+'.fst') + else: + fst_in=Files[0] + + + dfRadED, dfRadAD, dfRadBD= fastlib.spanwisePostPro(fst_in, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df = self.data) + dfs_new = [dfRadAD, dfRadED, dfRadBD] + names_new=[self.raw_name+'_AD', self.raw_name+'_ED', self.raw_name+'_BD'] + return dfs_new, names_new + + def changeUnits(self, flavor='WE'): + """ Change units of the table """ + # NOTE: moved to a plugin, but interface kept + from pydatview.plugins.data_standardizeUnits import changeUnits + changeUnits(self, flavor=flavor) + + def convertTimeColumns(self): + if len(self.data)>0: + for i,c in enumerate(self.data.columns.values): + y = self.data.iloc[:,i] + if y.dtype == object: + if isinstance(y.values[0], str): + # tring to convert to date + try: + parser.parse(y.values[0]) + isDate=True + except: + if y.values[0]=='NaT': + isDate=True + else: + isDate=False + if isDate: + try: + self.data[c]=pd.to_datetime(self.data[c].values).to_pydatetime() + print('Column {} converted to datetime'.format(c)) + except: + # Happens if values are e.g. "Monday, Tuesday" + print('Conversion to datetime failed, column {} inferred as string'.format(c)) + else: + print('Column {} inferred as string'.format(c)) + elif isinstance(y.values[0], (float, int)): + try: + self.data[c]=self.data[c].astype(float) + print('Column {} converted to float (likely nan)'.format(c)) + except: + self.data[c]=self.data[c].astype(str) + print('Column {} inferred and converted to string'.format(c)) + else : + print('>> Unknown type:',type(y.values[0])) + #print(self.data.dtypes) + + + # --- Column manipulations + def renameColumn(self,iCol,newName): + self.columns[iCol]=newName + self.data.columns.values[iCol]=newName + + def deleteColumns(self,ICol): + """ Delete columns by index, not column names which can have duplicates""" + IKeep =[i for i in np.arange(self.data.shape[1]) if i not in ICol] + self.data = self.data.iloc[:, IKeep] # Drop won't work for duplicates + for i in sorted(ICol, reverse=True): + del(self.columns[i]) + for f in self.formulas: + if f['pos'] == (i + 1): + self.formulas.remove(f) + break + for f in self.formulas: + if f['pos'] > (i + 1): + f['pos'] = f['pos'] - 1 + + def rename(self,new_name): + self.name='>'+new_name + + def addColumn(self,sNewName,NewCol,i=-1,sFormula=''): + if i<0: + i=self.data.shape[1] + elif i>self.data.shape[1]+1: + i=self.data.shape[1] + self.data.insert(int(i),sNewName,NewCol) + self.columns=self.columnsFromDF(self.data) + for f in self.formulas: + if f['pos'] > i: + f['pos'] = f['pos'] + 1 + self.formulas.append({'pos': i+1, 'formula': sFormula, 'name': sNewName}) + + def setColumn(self,sNewName,NewCol,i,sFormula=''): + if i<1: + raise ValueError('Cannot set column at position ' + str(i)) + self.data = self.data.drop(columns=self.data.columns[i-1]) + self.data.insert(int(i-1),sNewName,NewCol) + self.columns=self.columnsFromDF(self.data) + for f in self.formulas: + if f['pos'] == i: + f['name'] = sNewName + f['formula'] = sFormula + + def getColumn(self,i): + """ Return column of data, where i=0 is the index column + If a mask exist, the mask is applied + + TODO TODO TODO get rid of this! + """ + if i <= 0 : + x = np.array(range(self.data.shape[0])) + if self.mask is not None: + x=x[self.mask] + + c = None + isString = False + isDate = False + else: + if self.mask is not None: + c = self.data.iloc[self.mask, i-1] + x = self.data.iloc[self.mask, i-1].values + else: + c = self.data.iloc[:, i-1] + x = self.data.iloc[:, i-1].values + + isString = c.dtype == np.object and isinstance(c.values[0], str) + if isString: + x=x.astype(str) + isDate = np.issubdtype(c.dtype, np.datetime64) + if isDate: + dt=getDt(x) + if dt>1: + x=x.astype('datetime64[s]') + else: + x=x.astype('datetime64') + return x,isString,isDate,c + + + def evalFormula(self,sFormula): + df = self.data + Index = np.array(range(df.shape[0])) + sFormula=sFormula.replace('{Index}','Index') + for i,c in enumerate(self.columns): + c_no_unit = no_unit(c).strip() + c_in_df = df.columns[i] + sFormula=sFormula.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') + try: + NewCol=eval(sFormula) + return NewCol + except: + return None + + def addColumnByFormula(self,sNewName,sFormula,i=-1): + NewCol=self.evalFormula(sFormula) + if NewCol is None: + return False + else: + self.addColumn(sNewName,NewCol,i,sFormula) + return True + + def setColumnByFormula(self,sNewName,sFormula,i=-1): + NewCol=self.evalFormula(sFormula) + if NewCol is None: + return False + else: + self.setColumn(sNewName,NewCol,i,sFormula) + return True + + + def export(self,path): + if isinstance(self.data, pd.DataFrame): + try: + self.data.to_csv(path,sep=',',index=False) #python3 + except: + self.data.to_csv(path,sep=str(u',').encode('utf-8'),index=False) #python 2. + else: + raise NotImplementedError('Export of data that is not a dataframe') + + + + # --- Properties + @property + def basename(self): + return os.path.splitext(os.path.basename(self.filename))[0] + + @property + def shapestring(self): + return '{}x{}'.format(self.nCols, self.nRows) + + @property + def shape(self): + return (self.nRows, self.nCols) + + @property + def columns_clean(self): + return [no_unit(s) for s in self.columns] + + @property + def name(self): + if len(self.__name)<=0: + return '' + if self.__name[0]=='>': + return self.__name[1:] + else: + return self.__name + + @property + def raw_name(self): + return self.__name + + @name.setter + def name(self,new_name): + self.__name=new_name + + @property + def nCols(self): + return len(self.columns) + + @property + def nRows(self): + return len(self.data.iloc[:,0]) # TODO if not panda + + +if __name__ == '__main__': + import pandas as pd; + from Tables import Table + import numpy as np + + def OnTabPopup(event): + self.PopupMenu(TablePopup(self,selPanel.tabPanel.lbTab), event.GetPosition()) diff --git a/pydatview/appdata.py b/pydatview/appdata.py new file mode 100644 index 0000000..f801565 --- /dev/null +++ b/pydatview/appdata.py @@ -0,0 +1,147 @@ +import json +import os +from weio.weio import defaultUserDataDir +from .GUICommon import Error + + +def configFilePath(): + return os.path.join(defaultUserDataDir(), 'pyDatView', 'pyDatView.json') + +def loadAppData(mainframe): + configFile = configFilePath() + os.makedirs(os.path.dirname(configFile), exist_ok=True) + data = defaultAppData(mainframe) + #print('>>> configFile', configFile) + #print('Default Data content:\n') + #for k,v in data.items(): + # print('{:20s}: {}\n'.format(k,v)) + if os.path.exists(configFile): + sError='' + try: + with open(configFile) as f: + data2 = json.load(f) + except: + sError='Error: pyDatView config file is not properly formatted.\n\n' + sError+='The config file was at the following location:\n {}\n\n'.format(configFile) + + configFileBkp = configFile+'_bkp' + import shutil + try: + shutil.copy2(configFile, configFileBkp) + sError+='A backup of the file was made at the following location:\n {}\n\n'.format(configFileBkp) + backup=True + except: + backup=False + if backup: + try: + os.remove(configFile) + except: + sError+='To solve this issue, the config file was deleted.\n\n' + else: + sError+='A backup of the file could not be made and the file was not deleted\n\n' + sError+='If the problem persists, post an issue on the github repository\n' + #raise Exception(sError) + Error(mainframe, sError) + if len(sError)==0: + # Merging only what overlaps between default and user file + # --- Level 1 + for k1,v1 in data2.items(): + if k1 in data.keys(): + if type(data[k1]) is dict: + # --- Level 2 + for k2,v2 in v1.items(): + if k2 in data[k1].keys(): + if type(data[k1][k2]) is dict: + # --- Level 3 + for k3,v3 in v2.items(): + if k3 in data[k1][k2].keys(): + data[k1][k2][k3]=v3 + else: + data[k1][k2]=v2 + else: + data[k1]=v1 + #print('Data content on load:\n') + #for k,v in data.items(): + # print('{:20s}: {}\n'.format(k,v)) + return data + +def saveAppData(mainFrame, data): + from .GUICommon import getFontSize, getMonoFontSize + if not mainFrame.datareset: + # --- Import data from GUI + data['fontSize'] = int(getFontSize()) + data['monoFontSize'] = int(getMonoFontSize()) + if hasattr(mainFrame, 'plotPanel'): + savePlotPanelData(data['plotPanel'], mainFrame.plotPanel) + if hasattr(mainFrame, 'plotPanel'): + saveInfoPanelData(data['infoPanel'], mainFrame.infoPanel) + + # --- Write config file + configFile = configFilePath() + #print('>>> Writing configFile', configFile) + #print('Data content on close:\n') + #for k,v in data.items(): + # print('{:20s}: {}\n'.format(k,v)) + try: + os.makedirs(os.path.dirname(configFile), exist_ok=True) + with open(configFile, 'w') as f: + json.dump(data, f, indent=2) + except: + pass + +def defaultAppData(mainframe): + data={} + # --- Main frame data + data['windowSize'] = (900,700) + data['monoFontSize'] = mainframe.systemFontSize + data['fontSize'] = mainframe.systemFontSize + #SIDE_COL = [160,160,300,420,530] + #SIDE_COL_LARGE = [200,200,360,480,600] + #BOT_PANL =85 + data['plotPanel']=defaultPlotPanelData() + data['infoPanel']=defaultInfoPanelData() + return data + +# --- Plot Panel +# TODO put this into plotPanel file? +def savePlotPanelData(data, plotPanel): + data['Grid'] = plotPanel.cbGrid.IsChecked() + data['CrossHair'] = plotPanel.cbXHair.IsChecked() + data['plotStyle']['Font'] = plotPanel.esthPanel.cbFont.GetValue() + data['plotStyle']['LegendFont'] = plotPanel.esthPanel.cbLgdFont.GetValue() + data['plotStyle']['LegendPosition'] = plotPanel.esthPanel.cbLegend.GetValue() + data['plotStyle']['LineWidth'] = plotPanel.esthPanel.cbLW.GetValue() + data['plotStyle']['MarkerSize'] = plotPanel.esthPanel.cbMS.GetValue() + +def defaultPlotPanelData(): + data={} + data['CrossHair']=True + data['Grid']=False + plotStyle = dict() + plotStyle['Font'] = '11' + plotStyle['LegendFont'] = '11' + plotStyle['LegendPosition'] = 'Upper right' + plotStyle['LineWidth'] = '1.5' + plotStyle['MarkerSize'] = '2' + data['plotStyle']= plotStyle + return data + +# --- Info Panel +# TODO put this into infoPanel file? +def saveInfoPanelData(data, infoPanel): + data['ColumnsRegular'] = [c['name'] for c in infoPanel.ColsReg if c['s']] + data['ColumnsFFT'] = [c['name'] for c in infoPanel.ColsFFT if c['s']] + data['ColumnsMinMax'] = [c['name'] for c in infoPanel.ColsMinMax if c['s']] + data['ColumnsPDF'] = [c['name'] for c in infoPanel.ColsPDF if c['s']] + data['ColumnsCmp'] = [c['name'] for c in infoPanel.ColsCmp if c['s']] + +def defaultInfoPanelData(): + data={} + data['ColumnsRegular'] = ['Column','Mean','Std','Min','Max','Range','dx','n'] + data['ColumnsFFT'] = ['Column','Mean','Std','Min','Max','Min(FFT)','Max(FFT)',u'\u222By(FFT)','dx(FFT)','xMax(FFT)','n(FFT)','n'] + data['ColumnsMinMax'] = ['Column','Mean','Std','Min','Max','Mean(MinMax)','Std(MinMax)',u'\u222By(MinMax)','n'] + data['ColumnsPDF'] = ['Column','Mean','Std','Min','Max','Min(PDF)','Max(PDF)',u'\u222By(PDF)','n(PDF)'] + data['ColumnsCmp'] = ['Column','Mean(Cmp)','Std(Cmp)','Min(Cmp)','Max(Cmp)','n(Cmp)'] + return data + + diff --git a/pydatview/common.py b/pydatview/common.py index 8903ec3..59f1340 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -1,454 +1,440 @@ -import numpy as np -import pandas as pd -import os -import platform -import datetime -import re - -CHAR={ -'menu' : u'\u2630', -'tridot' : u'\u26EC', -'apply' : u'\u1809', -'compute' : u'\u2699', # gear -'close' : u'\u274C', -'add' : u'\u2795', -'add_small': u'\ufe62', -'clear' : u'-', -'sun' : u'\u2600', -'suncloud' : u'\u26C5', -'cloud' : u'\u2601', -'check' : u'\u2714', -'help' : u'\u2753', -'pencil' : u'\u270f', # draw -'pick' : u'\u26cf', -'hammer' : u'\U0001f528', -'wrench' : u'\U0001f527', -'ruler' : u'\U0001F4CF', # measure -'control_knobs' : u'\U0001F39b', -'python' : u'\U0001F40D', -'chart' : u'\U0001F4c8', -'chart_small': u'\U0001F5e0', -} -# --------------------------------------------------------------------------------} -# --- ellude -# --------------------------------------------------------------------------------{ -def common_start(*strings): - """ Returns the longest common substring - from the beginning of the `strings` - """ - if len(strings)==1: - strings=tuple(strings[0]) - def _iter(): - for z in zip(*strings): - if z.count(z[0]) == len(z): # check all elements in `z` are the same - yield z[0] - else: - return - return ''.join(_iter()) - -def common_end(*strings): - if len(strings)==1: - strings=strings[0] - else: - strings=list(strings) - strings = [s[-1::-1] for s in strings] - return common_start(strings)[-1::-1] - -def find_leftstop(s): - for i,c in enumerate(reversed(s)): - if c in ['.','_','|']: - i=i+1 - return s[:len(s)-i] - return s - -def ellude_common(strings,minLength=2): - """ - ellude the common parts of two strings - - minLength: - if -1, string might be elluded up until there are of 0 length - if 0 , if a string of zero length is obtained, it will be tried to be extended until a stop character is found - - """ - # Selecting only the strings that do not start with the safe '>' char - S = [s for i,s in enumerate(strings) if ((len(s)>0) and (s[0]!= '>'))] - if len(S)==0: - pass - elif len(S)==1: - ns=S[0].rfind('|')+1 - ne=0; - else: - ss = common_start(S) - se = common_end(S) - iu = ss[:-1].rfind('_') - ip = ss[:-1].rfind('_') - if iu > 0: - if ip>0: - if iu>ip: - ss=ss[:iu+1] - else: - ss=ss[:iu+1] - - iu = se[:-1].find('_') - if iu > 0: - se=se[iu:] - iu = se[:-1].find('.') - if iu > 0: - se=se[iu:] - ns=len(ss) - ne=len(se) - - # Reduce start length if some strings end up empty - # Look if any of the strings will end up empty - SSS=[len(s[ns:-ne].lstrip('_') if ne>0 else s[ns:].lstrip('_')) for s in S] - currentMinLength=np.min(SSS) - if currentMinLength0: - ss=ss[:-delta] - ns=len(ss) - #print('ss',ss) - ss=find_leftstop(ss) - #print('ss',ss) - if len(ss)==ns: - ns=0 - else: - ns=len(ss)+1 - - for i,s in enumerate(strings): - if len(s)>0 and s[0]=='>': - strings[i]=s[1:] - else: - s=s[ns:-ne] if ne>0 else s[ns:] - strings[i]=s.lstrip('_') - if len(strings[i])==0: - strings[i]='tab{}'.format(i) - return strings - - -# --------------------------------------------------------------------------------} -# --- Key value -# --------------------------------------------------------------------------------{ -def extract_key_tuples(text): - """ - all=(0.1,-2),b=(inf,0), c=(-inf,0.3e+10) - """ - regex = re.compile(r'(?P[\w\-]+)=\((?P[0-9+epinf.-]*?),(?P[0-9+epinf.-]*?)\)($|,)') - return {match.group("key"): (np.float(match.group("value1")),np.float(match.group("value2"))) for match in regex.finditer(text.replace(' ',''))} - - -def extract_key_num(text): - """ - all=0.1, b=inf, c=-0.3e+10 - """ - regex = re.compile(r'(?P[\w\-]+)=(?P[0-9+epinf.-]*?)($|,)') - return {match.group("key"): np.float(match.group("value")) for match in regex.finditer(text.replace(' ',''))} - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -# def getMonoFontAbs(): -# import wx -# #return wx.Font(9, wx.MODERN, wx.NORMAL, wx.NORMAL, False, u'Monospace') -# if os.name=='nt': -# font=wx.Font(9, wx.TELETYPE, wx.NORMAL, wx.NORMAL, False) -# elif os.name=='posix': -# font=wx.Font(10, wx.TELETYPE, wx.NORMAL, wx.NORMAL, False) -# else: -# font=wx.Font(8, wx.TELETYPE, wx.NORMAL, wx.NORMAL, False) -# return font -# -# def getMonoFont(widget): -# import wx -# font = widget.GetFont() -# font.SetFamily(wx.TELETYPE) -# if platform.system()=='Windows': -# pass -# elif platform.system()=='Linux': -# pass -# elif platform.system()=='Darwin': -# font.SetPointSize(font.GetPointSize()-1) -# else: -# pass -# return font - -def getDt(x): - """ returns dt in s """ - def myisnat(dt): - if isinstance(dt,pd._libs.tslibs.timedeltas.Timedelta): - try: - dt=pd.to_timedelta(dt) # pandas 1.0 - except: - dt=pd.to_timedelta(dt,box=False) # backward compatibility - - elif isinstance(dt,datetime.timedelta): - dt=np.array([dt],dtype='timedelta64')[0] - return pd.isna(dt) -# try: -# print('>>>', dt,type(dt)) -# isnat=np.isnat(dt) -# except: -# print(type(dt),type(dx)) -# isnat=False -# raise -# return isnat - - - - if len(x)<=1: - return np.NaN - if isinstance(x[0],float): - return x[1]-x[0] - if isinstance(x[0],int) or isinstance(x[0],np.int32) or isinstance(x[0],np.int64): - return x[1]-x[0] - # first try with seconds - #print('') - #print('getDT: dx:',x[1]-x[0]) - dx = x[1]-x[0] - #print(type(dx)) - if myisnat(dx): - # we try the last values (or while loop, but may take a while) - dx = x[-1]-x[-2] - if myisnat(dx): - return np.nan - dt=np.timedelta64(dx,'s').item().total_seconds() - if dt<1: - # try higher resolution - dt=np.timedelta64(dx,'ns').item()/10.**9 - # TODO if dt> int res... do something - return dt - -def getTabCommonColIndices(tabs): - cleanedColLists = [ [cleanCol(s) for s in t.columns] for t in tabs] - commonCols = cleanedColLists[0] - for i in np.arange(1,len(cleanedColLists)): - commonCols = list( set(commonCols) & set( cleanedColLists[i])) - # Keep original order - commonCols =[c for c in cleanedColLists[0] if c in commonCols] # Might have duplicates.. - IMissPerTab=[] - IKeepPerTab=[] - IDuplPerTab=[] # Duplicates amongst the "common" - for cleanedCols in cleanedColLists: - IKeep=[] - IMiss=[] - IDupl=[] - # Ugly for loop here since we have to account for dupplicates - for comcol in commonCols: - I = [i for i, c in enumerate(cleanedCols) if c == comcol] - if len(I)==0: - pass - else: - if I[0] not in IKeep: - IKeep.append(I[0]) - if len(I)>1: - IDupl=IDupl+I[1:] - IMiss=[i for i,_ in enumerate(cleanedCols) if (i not in IKeep) and (i not in IDupl)] - IMissPerTab.append(IMiss) - IKeepPerTab.append(IKeep) - IDuplPerTab.append(IDupl) - return IKeepPerTab, IMissPerTab, IDuplPerTab - - -def cleanCol(s): - s=no_unit(s).strip() - s=no_unit(s.replace('(',' [').replace(')',']')) - s=s.lower().strip().replace('_','').replace(' ','').replace('-','') - return s - -def no_unit(s): - s=s.replace('_[',' [') - iu=s.rfind(' [') - if iu>1: - return s[:iu] - else: - return s - -def unit(s): - iu=s.rfind('[') - if iu>1: - return s[iu+1:].replace(']','') - else: - return '' - -def inverse_unit(s): - u=unit(s).strip() - if u=='': - return '' - elif u=='-': - return '-' - elif len(u)==1: - return '1/'+u; - elif u=='m/s': - return 's/m'; - elif u=='deg': - return '1/deg'; - else: - return '1/('+u+')' - -def filter_list(L, string): - """ simple (not regex or fuzzy) filtering of a list of strings - Returns matched indices and strings - """ - ignore_case = string==string.lower() - if ignore_case: - I=[i for i,s in enumerate(L) if string in s.lower()] - else: - I=[i for i,s in enumerate(L) if string in s] - L_found =np.array(L)[I] - return L_found, I - -def unique(l): - """ Return unique values of a list""" - used=set() - return [x for x in l if x not in used and (used.add(x) or True)] - -# --------------------------------------------------------------------------------} -# --- geometry -# --------------------------------------------------------------------------------{ -def rectangleOverlap(BLx1, BLy1, TRx1, TRy1, BLx2, BLy2, TRx2, TRy2): - """ returns true if two rectangles overlap - BL: Bottom left - TR: top right - "1" rectangle 1 - "2" rectangle 2 - """ - return not (TRx1 < BLx2 or BLx1 > TRx2 or TRy1 < BLy2 or BLy1> TRy2) -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -def pretty_time(t): - # fPrettyTime: returns a 6-characters string corresponding to the input time in seconds. - # fPrettyTime(612)=='10m12s' - # AUTHOR: E. Branlard - if np.isnan(t): - return 'NaT'; - if(t<0): - return '------'; - elif (t<1) : - c=np.floor(t*100); - s='{:2d}.{:02d}s'.format(0,int(c)) - elif(t<60) : - s=np.floor(t); - c=np.floor((t-s)*100); - s='{:2d}.{:02d}s'.format(int(s),int(c)) - elif(t<3600) : - m=np.floor(t/60); - s=np.mod( np.floor(t), 60); - s='{:2d}m{:02d}s'.format(int(m),int(s)) - elif(t<86400) : - h=np.floor(t/3600); - m=np.floor(( np.mod( np.floor(t) , 3600))/60); - s='{:2d}h{:02d}m'.format(int(h),int(m)) - elif(t<8553600) : #below 3month - d=np.floor(t/86400); - h=np.floor( np.mod(np.floor(t), 86400)/3600); - s='{:2d}d{:02d}h'.format(int(d),int(h)) - elif(t<31536000): - m=t/(3600*24*30.5); - s='{:4.1f}mo'.format(m) - #s='+3mon.'; - else: - y=t/(3600*24*365.25); - s='{:.1f}y'.format(y) - return s - -def pretty_num(x): - if abs(x)<1000 and abs(x)>1e-4: - return "{:9.4f}".format(x) - else: - return '{:.3e}'.format(x) - -def pretty_num_short(x,digits=3): - if digits==4: - if abs(x)<1000 and abs(x)>1e-1: - return "{:.4f}".format(x) - else: - return "{:.4e}".format(x) - elif digits==3: - if abs(x)<1000 and abs(x)>1e-1: - return "{:.3f}".format(x) - else: - return "{:.3e}".format(x) - elif digits==2: - if abs(x)<1000 and abs(x)>1e-1: - return "{:.2f}".format(x) - else: - return "{:.2e}".format(x) - -# --------------------------------------------------------------------------------} -# --- Chinese characters -# --------------------------------------------------------------------------------{ -cjk_ranges = [ - ( 0x4E00, 0x62FF), - ( 0x6300, 0x77FF), - ( 0x7800, 0x8CFF), - ( 0x8D00, 0x9FCC), - ( 0x3400, 0x4DB5), - (0x20000, 0x215FF), - (0x21600, 0x230FF), - (0x23100, 0x245FF), - (0x24600, 0x260FF), - (0x26100, 0x275FF), - (0x27600, 0x290FF), - (0x29100, 0x2A6DF), - (0x2A700, 0x2B734), - (0x2B740, 0x2B81D), - (0x2B820, 0x2CEAF), - (0x2CEB0, 0x2EBEF), - (0x2F800, 0x2FA1F) - ] - -def has_chinese_char(s): - def is_cjk(char): - char = ord(char) - for bottom, top in cjk_ranges: - if char >= bottom and char <= top: - return True - return False - for c in s: - char=ord(c) - for bottom, top in cjk_ranges: - if char >= bottom and char <= top: - return True - return False - - -# --------------------------------------------------------------------------------} -# --- Helper functions -# --------------------------------------------------------------------------------{ -def YesNo(parent, question, caption = 'Yes or no?'): - import wx - dlg = wx.MessageDialog(parent, question, caption, wx.YES_NO | wx.ICON_QUESTION) - result = dlg.ShowModal() == wx.ID_YES - dlg.Destroy() - return result -def Info(parent, message, caption = 'Info'): - import wx - dlg = wx.MessageDialog(parent, message, caption, wx.OK | wx.ICON_INFORMATION) - dlg.ShowModal() - dlg.Destroy() -def Warn(parent, message, caption = 'Warning!'): - import wx - dlg = wx.MessageDialog(parent, message, caption, wx.OK | wx.ICON_WARNING) - dlg.ShowModal() - dlg.Destroy() -def Error(parent, message, caption = 'Error!'): - import wx - dlg = wx.MessageDialog(parent, message, caption, wx.OK | wx.ICON_ERROR) - dlg.ShowModal() - dlg.Destroy() - - - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ - -def isString(x): - b = x.dtype == np.object and isinstance(x.values[0], str) - return b - -def isDate(x): - return np.issubdtype(x.dtype, np.datetime64) - +import numpy as np +import pandas as pd +import os +import platform +import datetime +import re + +CHAR={ +'menu' : u'\u2630', +'tridot' : u'\u26EC', +'apply' : u'\u1809', +'compute' : u'\u2699', # gear +'close' : u'\u274C', +'add' : u'\u2795', +'add_small': u'\ufe62', +'clear' : u'-', +'sun' : u'\u2600', +'suncloud' : u'\u26C5', +'cloud' : u'\u2601', +'check' : u'\u2714', +'help' : u'\u2753', +'pencil' : u'\u270f', # draw +'pick' : u'\u26cf', +'hammer' : u'\U0001f528', +'wrench' : u'\U0001f527', +'ruler' : u'\U0001F4CF', # measure +'control_knobs' : u'\U0001F39b', +'python' : u'\U0001F40D', +'chart' : u'\U0001F4c8', +'chart_small': u'\U0001F5e0', +} +# --------------------------------------------------------------------------------} +# --- ellude +# --------------------------------------------------------------------------------{ +def common_start(*strings): + """ Returns the longest common substring + from the beginning of the `strings` + """ + if len(strings)==1: + strings=tuple(strings[0]) + def _iter(): + for z in zip(*strings): + if z.count(z[0]) == len(z): # check all elements in `z` are the same + yield z[0] + else: + return + return ''.join(_iter()) + +def common_end(*strings): + if len(strings)==1: + strings=strings[0] + else: + strings=list(strings) + strings = [s[-1::-1] for s in strings] + return common_start(strings)[-1::-1] + +def find_leftstop(s): + for i,c in enumerate(reversed(s)): + if c in ['.','_','|']: + i=i+1 + return s[:len(s)-i] + return s + +def ellude_common(strings,minLength=2): + """ + ellude the common parts of two strings + + minLength: + if -1, string might be elluded up until there are of 0 length + if 0 , if a string of zero length is obtained, it will be tried to be extended until a stop character is found + + """ + # Selecting only the strings that do not start with the safe '>' char + S = [s for i,s in enumerate(strings) if ((len(s)>0) and (s[0]!= '>'))] + if len(S)==0: + pass + elif len(S)==1: + ns=S[0].rfind('|')+1 + ne=0; + else: + ss = common_start(S) + se = common_end(S) + iu = ss[:-1].rfind('_') + ip = ss[:-1].rfind('_') + if iu > 0: + if ip>0: + if iu>ip: + ss=ss[:iu+1] + else: + ss=ss[:iu+1] + + iu = se[:-1].find('_') + if iu > 0: + se=se[iu:] + iu = se[:-1].find('.') + if iu > 0: + se=se[iu:] + ns=len(ss) + ne=len(se) + + # Reduce start length if some strings end up empty + # Look if any of the strings will end up empty + SSS=[len(s[ns:-ne].lstrip('_') if ne>0 else s[ns:].lstrip('_')) for s in S] + currentMinLength=np.min(SSS) + if currentMinLength0: + ss=ss[:-delta] + ns=len(ss) + #print('ss',ss) + ss=find_leftstop(ss) + #print('ss',ss) + if len(ss)==ns: + ns=0 + else: + ns=len(ss)+1 + + for i,s in enumerate(strings): + if len(s)>0 and s[0]=='>': + strings[i]=s[1:] + else: + s=s[ns:-ne] if ne>0 else s[ns:] + strings[i]=s.lstrip('_') + if len(strings[i])==0: + strings[i]='tab{}'.format(i) + return strings + + +# --------------------------------------------------------------------------------} +# --- Key value +# --------------------------------------------------------------------------------{ +def extract_key_tuples(text): + """ + all=(0.1,-2),b=(inf,0), c=(-inf,0.3e+10) + """ + regex = re.compile(r'(?P[\w\-]+)=\((?P[0-9+epinf.-]*?),(?P[0-9+epinf.-]*?)\)($|,)') + return {match.group("key"): (np.float(match.group("value1")),np.float(match.group("value2"))) for match in regex.finditer(text.replace(' ',''))} + + +def extract_key_num(text): + """ + all=0.1, b=inf, c=-0.3e+10 + """ + regex = re.compile(r'(?P[\w\-]+)=(?P[0-9+epinf.-]*?)($|,)') + return {match.group("key"): np.float(match.group("value")) for match in regex.finditer(text.replace(' ',''))} + +def getDt(x): + """ returns dt in s """ + def myisnat(dt): + if isinstance(dt,pd._libs.tslibs.timedeltas.Timedelta): + try: + dt=pd.to_timedelta(dt) # pandas 1.0 + except: + dt=pd.to_timedelta(dt,box=False) # backward compatibility + + elif isinstance(dt,datetime.timedelta): + dt=np.array([dt],dtype='timedelta64')[0] + return pd.isna(dt) +# try: +# print('>>>', dt,type(dt)) +# isnat=np.isnat(dt) +# except: +# print(type(dt),type(dx)) +# isnat=False +# raise +# return isnat + + + + if len(x)<=1: + return np.NaN + if isinstance(x[0],float): + return x[1]-x[0] + if isinstance(x[0],int) or isinstance(x[0],np.int32) or isinstance(x[0],np.int64): + return x[1]-x[0] + # first try with seconds + #print('') + #print('getDT: dx:',x[1]-x[0]) + dx = x[1]-x[0] + #print(type(dx)) + if myisnat(dx): + # we try the last values (or while loop, but may take a while) + dx = x[-1]-x[-2] + if myisnat(dx): + return np.nan + dt=np.timedelta64(dx,'s').item().total_seconds() + if dt<1: + # try higher resolution + dt=np.timedelta64(dx,'ns').item()/10.**9 + # TODO if dt> int res... do something + return dt + +def getTabCommonColIndices(tabs): + cleanedColLists = [ [cleanCol(s) for s in t.columns] for t in tabs] + nCols = np.array([len(cols) for cols in cleanedColLists]) + # Common columns between all column lists + commonCols = cleanedColLists[0] + for i in np.arange(1,len(cleanedColLists)): + commonCols = list( set(commonCols) & set( cleanedColLists[i])) + # Keep original order + commonCols =[c for c in cleanedColLists[0] if c in commonCols] # Might have duplicates.. + IMissPerTab=[] + IKeepPerTab=[] + IDuplPerTab=[] # Duplicates amongst the "common" + for cleanedCols in cleanedColLists: + IKeep=[] + IMiss=[] + IDupl=[] + # Ugly for loop here since we have to account for dupplicates + for comcol in commonCols: + I = [i for i, c in enumerate(cleanedCols) if c == comcol] + if len(I)==0: + pass + else: + if I[0] not in IKeep: + IKeep.append(I[0]) + if len(I)>1: + IDupl=IDupl+I[1:] + IMiss=[i for i,_ in enumerate(cleanedCols) if (i not in IKeep) and (i not in IDupl)] + IMissPerTab.append(IMiss) + IKeepPerTab.append(IKeep) + IDuplPerTab.append(IDupl) + return IKeepPerTab, IMissPerTab, IDuplPerTab, nCols + + +# --------------------------------------------------------------------------------} +# --- Units +# --------------------------------------------------------------------------------{ +def cleanCol(s): + s=no_unit(s).strip() + s=no_unit(s.replace('(',' [').replace(')',']')) + s=s.lower().strip().replace('_','').replace(' ','').replace('-','') + return s + +def no_unit(s): + s=s.replace('_[',' [') + iu=s.rfind(' [') + if iu>1: + return s[:iu] + else: + return s + +def unit(s): + iu=s.rfind('[') + if iu>1: + return s[iu+1:].replace(']','') + else: + return '' + +def splitunit(s): + iu=s.rfind('[') + if iu>1: + return s[:iu], s[iu+1:].replace(']','') + else: + return s, '' + +def inverse_unit(s): + u=unit(s).strip() + if u=='': + return '' + elif u=='-': + return '-' + elif len(u)==1: + return '1/'+u; + elif u=='m/s': + return 's/m'; + elif u=='deg': + return '1/deg'; + else: + return '1/('+u+')' + + + +def filter_list(L, string): + """ simple (not regex or fuzzy) filtering of a list of strings + Returns matched indices and strings + """ + ignore_case = string==string.lower() + if ignore_case: + I=[i for i,s in enumerate(L) if string in s.lower()] + else: + I=[i for i,s in enumerate(L) if string in s] + L_found =np.array(L)[I] + return L_found, I + +def unique(l): + """ Return unique values of a list""" + used=set() + return [x for x in l if x not in used and (used.add(x) or True)] + +# --------------------------------------------------------------------------------} +# --- geometry +# --------------------------------------------------------------------------------{ +def rectangleOverlap(BLx1, BLy1, TRx1, TRy1, BLx2, BLy2, TRx2, TRy2): + """ returns true if two rectangles overlap + BL: Bottom left + TR: top right + "1" rectangle 1 + "2" rectangle 2 + """ + return not (TRx1 < BLx2 or BLx1 > TRx2 or TRy1 < BLy2 or BLy1> TRy2) +# --------------------------------------------------------------------------------} +# --- +# --------------------------------------------------------------------------------{ +def pretty_time(t): + # fPrettyTime: returns a 6-characters string corresponding to the input time in seconds. + # fPrettyTime(612)=='10m12s' + # AUTHOR: E. Branlard + if np.isnan(t): + return 'NaT'; + if(t<0): + return '------'; + elif (t<1) : + c=np.floor(t*100); + s='{:2d}.{:02d}s'.format(0,int(c)) + elif(t<60) : + s=np.floor(t); + c=np.floor((t-s)*100); + s='{:2d}.{:02d}s'.format(int(s),int(c)) + elif(t<3600) : + m=np.floor(t/60); + s=np.mod( np.floor(t), 60); + s='{:2d}m{:02d}s'.format(int(m),int(s)) + elif(t<86400) : + h=np.floor(t/3600); + m=np.floor(( np.mod( np.floor(t) , 3600))/60); + s='{:2d}h{:02d}m'.format(int(h),int(m)) + elif(t<8553600) : #below 3month + d=np.floor(t/86400); + h=np.floor( np.mod(np.floor(t), 86400)/3600); + s='{:2d}d{:02d}h'.format(int(d),int(h)) + elif(t<31536000): + m=t/(3600*24*30.5); + s='{:4.1f}mo'.format(m) + #s='+3mon.'; + else: + y=t/(3600*24*365.25); + s='{:.1f}y'.format(y) + return s + +def pretty_num(x): + if abs(x)<1000 and abs(x)>1e-4: + return "{:9.4f}".format(x) + else: + return '{:.3e}'.format(x) + +def pretty_num_short(x,digits=3): + if digits==4: + if abs(x)<1000 and abs(x)>1e-1: + return "{:.4f}".format(x) + else: + return "{:.4e}".format(x) + elif digits==3: + if abs(x)<1000 and abs(x)>1e-1: + return "{:.3f}".format(x) + else: + return "{:.3e}".format(x) + elif digits==2: + if abs(x)<1000 and abs(x)>1e-1: + return "{:.2f}".format(x) + else: + return "{:.2e}".format(x) + +# --------------------------------------------------------------------------------} +# --- Chinese characters +# --------------------------------------------------------------------------------{ +cjk_ranges = [ + ( 0x4E00, 0x62FF), + ( 0x6300, 0x77FF), + ( 0x7800, 0x8CFF), + ( 0x8D00, 0x9FCC), + ( 0x3400, 0x4DB5), + (0x20000, 0x215FF), + (0x21600, 0x230FF), + (0x23100, 0x245FF), + (0x24600, 0x260FF), + (0x26100, 0x275FF), + (0x27600, 0x290FF), + (0x29100, 0x2A6DF), + (0x2A700, 0x2B734), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEAF), + (0x2CEB0, 0x2EBEF), + (0x2F800, 0x2FA1F) + ] + +def has_chinese_char(s): + def is_cjk(char): + char = ord(char) + for bottom, top in cjk_ranges: + if char >= bottom and char <= top: + return True + return False + for c in s: + char=ord(c) + for bottom, top in cjk_ranges: + if char >= bottom and char <= top: + return True + return False + + +# --------------------------------------------------------------------------------} +# --- Helper functions +# --------------------------------------------------------------------------------{ +def YesNo(parent, question, caption = 'Yes or no?'): + import wx + dlg = wx.MessageDialog(parent, question, caption, wx.YES_NO | wx.ICON_QUESTION) + result = dlg.ShowModal() == wx.ID_YES + dlg.Destroy() + return result +def Info(parent, message, caption = 'Info'): + import wx + dlg = wx.MessageDialog(parent, message, caption, wx.OK | wx.ICON_INFORMATION) + dlg.ShowModal() + dlg.Destroy() +def Warn(parent, message, caption = 'Warning!'): + import wx + dlg = wx.MessageDialog(parent, message, caption, wx.OK | wx.ICON_WARNING) + dlg.ShowModal() + dlg.Destroy() +def Error(parent, message, caption = 'Error!'): + import wx + dlg = wx.MessageDialog(parent, message, caption, wx.OK | wx.ICON_ERROR) + dlg.ShowModal() + dlg.Destroy() + + + +# --------------------------------------------------------------------------------} +# --- +# --------------------------------------------------------------------------------{ + +def isString(x): + b = x.dtype == object and isinstance(x.values[0], str) + return b + +def isDate(x): + return np.issubdtype(x.dtype, np.datetime64) + diff --git a/pydatview/fast/case_gen.py b/pydatview/fast/case_gen.py index a22c090..f9c53f0 100644 --- a/pydatview/fast/case_gen.py +++ b/pydatview/fast/case_gen.py @@ -1,586 +1,593 @@ -from __future__ import division, print_function -import os -import collections -import glob -import pandas as pd -import numpy as np -import shutil -import stat -import re - -# --- Misc fast libraries -import weio.weio.fast_input_file as fi -import pydatview.fast.runner as runner -import pydatview.fast.postpro as postpro -#import pyFAST.input_output.fast_input_file as fi -#import pyFAST.case_generation.runner as runner -#import pyFAST.input_output.postpro as postpro - - -# --------------------------------------------------------------------------------} -# --- Template replace -# --------------------------------------------------------------------------------{ -def handleRemoveReadonlyWin(func, path, exc_info): - """ - Error handler for ``shutil.rmtree``. - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - Usage : ``shutil.rmtree(path, onerror=onerror)`` - """ - if not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise - - -def copyTree(src, dst): - """ - Copy a directory to another one, overwritting files if necessary. - copy_tree from distutils and copytree from shutil fail on Windows (in particular on git files) - """ - def forceMergeFlatDir(srcDir, dstDir): - if not os.path.exists(dstDir): - os.makedirs(dstDir) - for item in os.listdir(srcDir): - srcFile = os.path.join(srcDir, item) - dstFile = os.path.join(dstDir, item) - forceCopyFile(srcFile, dstFile) - - def forceCopyFile (sfile, dfile): - # ---- Handling error due to wrong mod - if os.path.isfile(dfile): - if not os.access(dfile, os.W_OK): - os.chmod(dfile, stat.S_IWUSR) - #print(sfile, ' > ', dfile) - shutil.copy2(sfile, dfile) - - def isAFlatDir(sDir): - for item in os.listdir(sDir): - sItem = os.path.join(sDir, item) - if os.path.isdir(sItem): - return False - return True - - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isfile(s): - if not os.path.exists(dst): - os.makedirs(dst) - forceCopyFile(s,d) - if os.path.isdir(s): - isRecursive = not isAFlatDir(s) - if isRecursive: - copyTree(s, d) - else: - forceMergeFlatDir(s, d) - - -def templateReplaceGeneral(PARAMS, templateDir=None, outputDir=None, main_file=None, removeAllowed=False, removeRefSubFiles=False, oneSimPerDir=False): - """ Generate inputs files by replacing different parameters from a template file. - The generated files are placed in the output directory `outputDir` - The files are read and written using the library `weio`. - The template file is read and its content can be changed like a dictionary. - Each item of `PARAMS` correspond to a set of parameters that will be replaced - in the template file to generate one input file. - - For "FAST" input files, parameters can be changed recursively. - - - INPUTS: - PARAMS: list of dictionaries. Each key of the dictionary should be a key present in the - template file when read with `weio` (see: weio.read(main_file).keys() ) - - PARAMS[0]={'DT':0.1, 'EDFile|GBRatio':1, 'ServoFile|GenEff':0.8} - - templateDir: if provided, this directory and its content will be copied to `outputDir` - before doing the parametric substitution - - outputDir : directory where files will be generated. - """ - # --- Helper functions - def rebase_rel(wd,s,sid): - split = os.path.splitext(s) - return os.path.join(wd,split[0]+sid+split[1]) - - def get_strID(p) : - if '__name__' in p.keys(): - strID=p['__name__'] - else: - raise Exception('When calling `templateReplace`, provide the key `__name_` in the parameter dictionaries') - return strID - - def splitAddress(sAddress): - sp = sAddress.split('|') - if len(sp)==1: - return sp[0],[] - else: - return sp[0],sp[1:] - - def rebaseFileName(org_filename, workDir, strID): - new_filename_full = rebase_rel(workDir, org_filename,'_'+strID) - new_filename = os.path.relpath(new_filename_full,workDir).replace('\\','/') - return new_filename, new_filename_full - - def replaceRecurse(templatename_or_newname, FileKey, ParamKey, ParamValue, Files, strID, workDir, TemplateFiles): - """ - FileKey: a single key defining which file we are currently modifying e.g. :'AeroFile', 'EDFile','FVWInputFileName' - ParamKey: the address key of the parameter to be changed, relative to the current FileKey - e.g. 'EDFile|IntMethod' (if FileKey is '') - 'IntMethod' (if FileKey is 'EDFile') - ParamValue: the value to be used - Files: dict of files, as returned by weio, keys are "FileKeys" - """ - # --- Special handling for the root - if FileKey=='': - FileKey='Root' - # --- Open (or get if already open) file where a parameter needs to be changed - if FileKey in Files.keys(): - # The file was already opened, it's stored - f = Files[FileKey] - newfilename_full = f.filename - newfilename = os.path.relpath(newfilename_full,workDir).replace('\\','/') - - else: - templatefilename = templatename_or_newname - templatefilename_full = os.path.join(workDir,templatefilename) - TemplateFiles.append(templatefilename_full) - if FileKey=='Root': - # Root files, we start from strID - ext = os.path.splitext(templatefilename)[-1] - newfilename_full = os.path.join(wd,strID+ext) - newfilename = strID+ext - else: - newfilename, newfilename_full = rebaseFileName(templatefilename, workDir, strID) - #print('--------------------------------------------------------------') - #print('TemplateFile :', templatefilename) - #print('TemplateFileFull:', templatefilename_full) - #print('NewFile :', newfilename) - #print('NewFileFull :', newfilename_full) - shutil.copyfile(templatefilename_full, newfilename_full) - f= fi.FASTInputFile(newfilename_full) # open the template file for that filekey - Files[FileKey]=f # store it - - # --- Changing parameters in that file - NewFileKey_or_Key, ChildrenKeys = splitAddress(ParamKey) - if len(ChildrenKeys)==0: - # A simple parameter is changed - Key = NewFileKey_or_Key - #print('Setting', FileKey, '|',Key, 'to',ParamValue) - if Key=='OutList': - OutList=f[Key] - f[Key]=addToOutlist(OutList, ParamValue) - else: - f[Key] = ParamValue - else: - # Parameters needs to be changed in subfiles (children) - NewFileKey = NewFileKey_or_Key - ChildrenKey = '|'.join(ChildrenKeys) - child_templatefilename = f[NewFileKey].strip('"') # old filename that will be used as a template - baseparent = os.path.dirname(newfilename) - #print('Child templatefilename:',child_templatefilename) - #print('Parent base dir :',baseparent) - workDir = os.path.join(workDir, baseparent) - - # - newchildFilename, Files = replaceRecurse(child_templatefilename, NewFileKey, ChildrenKey, ParamValue, Files, strID, workDir, TemplateFiles) - #print('Setting', FileKey, '|',NewFileKey, 'to',newchildFilename) - f[NewFileKey] = '"'+newchildFilename+'"' - - return newfilename, Files - - - # --- Safety checks - if templateDir is None and outputDir is None: - raise Exception('Provide at least a template directory OR an output directory') - - if templateDir is not None: - if not os.path.exists(templateDir): - raise Exception('Template directory does not exist: '+templateDir) - - # Default value of outputDir if not provided - if templateDir[-1]=='/' or templateDir[-1]=='\\' : - templateDir=templateDir[0:-1] - if outputDir is None: - outputDir=templateDir+'_Parametric' - - # --- Main file use as "master" - if templateDir is not None: - main_file=os.path.join(outputDir, os.path.basename(main_file)) - else: - main_file=main_file - - # Params need to be a list - if not isinstance(PARAMS,list): - PARAMS=[PARAMS] - - if oneSimPerDir: - workDirS=[os.path.join(outputDir,get_strID(p)) for p in PARAMS] - else: - workDirS=[outputDir]*len(PARAMS) - # --- Creating outputDir - Copying template folder to outputDir if necessary - # Copying template folder to workDir - for wd in list(set(workDirS)): - if removeAllowed: - removeFASTOuputs(wd) - if os.path.exists(wd) and removeAllowed: - shutil.rmtree(wd, ignore_errors=False, onerror=handleRemoveReadonlyWin) - copyTree(templateDir, wd) - if removeAllowed: - removeFASTOuputs(wd) - - - TemplateFiles=[] - files=[] - for ip,(wd,p) in enumerate(zip(workDirS,PARAMS)): - if '__index__' not in p.keys(): - p['__index__']=ip - - main_file_base = os.path.basename(main_file) - strID = get_strID(p) - # --- Setting up files for this simulation - Files=dict() - for k,v in p.items(): - if k =='__index__' or k=='__name__': - continue - new_mainFile, Files = replaceRecurse(main_file_base, '', k, v, Files, strID, wd, TemplateFiles) - - # --- Writting files - for k,f in Files.items(): - if k=='Root': - files.append(f.filename) - f.write() - - # --- Remove extra files at the end - if removeRefSubFiles: - TemplateFiles, nCounts = np.unique(TemplateFiles, return_counts=True) - if not oneSimPerDir: - # we can only detele template files that were used by ALL simulations - TemplateFiles=[t for nc,t in zip(nCounts, TemplateFiles) if nc==len(PARAMS)] - for tf in TemplateFiles: - try: - os.remove(tf) - except: - print('[FAIL] Removing '+tf) - pass - return files - -def templateReplace(PARAMS, templateDir, outputDir=None, main_file=None, removeAllowed=False, removeRefSubFiles=False, oneSimPerDir=False): - """ Replace parameters in a fast folder using a list of dictionaries where the keys are for instance: - 'DT', 'EDFile|GBRatio', 'ServoFile|GenEff' - """ - # --- For backward compatibility, remove "FAST|" from the keys - for p in PARAMS: - old_keys=[ k for k,_ in p.items() if k.find('FAST|')==0] - for k_old in old_keys: - k_new=k_old.replace('FAST|','') - p[k_new] = p.pop(k_old) - - return templateReplaceGeneral(PARAMS, templateDir, outputDir=outputDir, main_file=main_file, - removeAllowed=removeAllowed, removeRefSubFiles=removeRefSubFiles, oneSimPerDir=oneSimPerDir) - -def removeFASTOuputs(workDir): - # Cleaning folder - for f in glob.glob(os.path.join(workDir,'*.out')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.outb')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.ech')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.sum')): - os.remove(f) - -# --------------------------------------------------------------------------------} -# --- Tools for template replacement -# --------------------------------------------------------------------------------{ -def paramsSteadyAero(p=dict()): - p['AeroFile|AFAeroMod']=1 # remove dynamic effects dynamic - p['AeroFile|WakeMod']=1 # remove dynamic inflow dynamic - p['AeroFile|TwrPotent']=0 # remove tower shadow - return p - -def paramsNoGen(p=dict()): - p['EDFile|GenDOF' ] = 'False' - return p - -def paramsGen(p=dict()): - p['EDFile|GenDOF' ] = 'True' - return p - -def paramsNoController(p=dict()): - p['ServoFile|PCMode'] = 0; - p['ServoFile|VSContrl'] = 0; - p['ServoFile|YCMode'] = 0; - return p - -def paramsControllerDLL(p=dict()): - p['ServoFile|PCMode'] = 5; - p['ServoFile|VSContrl'] = 5; - p['ServoFile|YCMode'] = 5; - p['EDFile|GenDOF'] = 'True'; - return p - - -def paramsStiff(p=dict()): - p['EDFile|FlapDOF1'] = 'False' - p['EDFile|FlapDOF2'] = 'False' - p['EDFile|EdgeDOF' ] = 'False' - p['EDFile|TeetDOF' ] = 'False' - p['EDFile|DrTrDOF' ] = 'False' - p['EDFile|YawDOF' ] = 'False' - p['EDFile|TwFADOF1'] = 'False' - p['EDFile|TwFADOF2'] = 'False' - p['EDFile|TwSSDOF1'] = 'False' - p['EDFile|TwSSDOF2'] = 'False' - p['EDFile|PtfmSgDOF'] = 'False' - p['EDFile|PtfmSwDOF'] = 'False' - p['EDFile|PtfmHvDOF'] = 'False' - p['EDFile|PtfmRDOF'] = 'False' - p['EDFile|PtfmPDOF'] = 'False' - p['EDFile|PtfmYDOF'] = 'False' - return p - -def paramsWS_RPM_Pitch(WS, RPM, Pitch, baseDict=None, flatInputs=False): - """ - Generate OpenFAST "parameters" (list of dictionaries with "address") - chaing the inputs in ElastoDyn, InflowWind for different wind speed, RPM and Pitch - """ - # --- Ensuring everythin is an iterator - def iterify(x): - if not isinstance(x, collections.Iterable): x = [x] - return x - WS = iterify(WS) - RPM = iterify(RPM) - Pitch = iterify(Pitch) - # --- If inputs are not flat but different vectors to length through, we flatten them (TODO: meshgrid and ravel?) - if not flatInputs : - WS_flat = [] - Pitch_flat = [] - RPM_flat = [] - for pitch in Pitch: - for rpm in RPM: - for ws in WS: - WS_flat.append(ws) - RPM_flat.append(rpm) - Pitch_flat.append(pitch) - else: - WS_flat, Pitch_flat, RPM_flat = WS, Pitch, RPM - - # --- Defining the parametric study - PARAMS=[] - i=0 - for ws,rpm,pitch in zip(WS_flat,RPM_flat,Pitch_flat): - if baseDict is None: - p=dict() - else: - p = baseDict.copy() - p['EDFile|RotSpeed'] = rpm - p['InflowFile|HWindSpeed'] = ws - p['InflowFile|WindType'] = 1 # Setting steady wind - p['EDFile|BlPitch(1)'] = pitch - p['EDFile|BlPitch(2)'] = pitch - p['EDFile|BlPitch(3)'] = pitch - - p['__index__'] = i - p['__name__'] = '{:03d}_ws{:04.1f}_pt{:04.2f}_om{:04.2f}'.format(p['__index__'],p['InflowFile|HWindSpeed'],p['EDFile|BlPitch(1)'],p['EDFile|RotSpeed']) - i=i+1 - PARAMS.append(p) - return PARAMS - -def paramsLinearTrim(p=dict()): - - # Set a few DOFs, move this to main file - p['Linearize'] = True - p['CalcSteady'] = True - p['TrimGain'] = 1e-4 - p['TrimTol'] = 1e-5 - p['CompMooring'] = 0 - p['CompHydro'] = 0 - p['LinOutJac'] = False - p['LinOutMod'] = False - p['OutFmt'] = '"ES20.12E3"' # Important for decent resolution - - p['AeroFile|AFAeroMod'] = 1 - p['AeroFile|CavitCheck'] = 'False' - p['AeroFile|CompAA'] = 'False' - - p['ServoFile|PCMode'] = 0 - p['ServoFile|VSContrl'] = 1 - - p['ServoFile|CompNTMD'] = 'False' - p['ServoFile|CompTTMD'] = 'False' - - # Set all DOFs off, enable as desired - p['EDFile|FlapDOF1'] = 'False' - p['EDFile|FlapDOF2'] = 'False' - p['EDFile|EdgeDOF'] = 'False' - p['EDFile|TeetDOF'] = 'False' - p['EDFile|DrTrDOF'] = 'False' - p['EDFile|GenDOF'] = 'False' - p['EDFile|YawDOF'] = 'False' - p['EDFile|TwFADOF1'] = 'False' - p['EDFile|TwFADOF2'] = 'False' - p['EDFile|TwSSDOF1'] = 'False' - p['EDFile|TwSSDOF2'] = 'False' - p['EDFile|PtfmSgDOF'] = 'False' - p['EDFile|PtfmSwDOF'] = 'False' - p['EDFile|PtfmHvDOF'] = 'False' - p['EDFile|PtfmRDOF'] = 'False' - p['EDFile|PtfmPDOF'] = 'False' - p['EDFile|PtfmYDOF'] = 'False' - - - return p - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -def createStepWind(filename,WSstep=1,WSmin=3,WSmax=25,tstep=100,dt=0.5,tmin=0,tmax=999): - f = weio.FASTWndFile() - Steps= np.arange(WSmin,WSmax+WSstep,WSstep) - print(Steps) - nCol = len(f.colNames) - nRow = len(Steps)*2 - M = np.zeros((nRow,nCol)); - M[0,0] = tmin - M[0,1] = WSmin - for i,s in enumerate(Steps[:-1]): - M[2*i+1,0] = tmin + (i+1)*tstep-dt - M[2*i+2,0] = tmin + (i+1)*tstep - M[2*i+1,1] = Steps[i] - if i0: - main_fastfile=os.path.basename(main_fastfile) - - # --- Reading main fast file to get rotor radius - fst = fi.FASTInputFile(os.path.join(refdir,main_fastfile)) - ed = fi.FASTInputFile(os.path.join(refdir,fst['EDFile'].replace('"',''))) - R = ed['TipRad'] - - # --- Making sure we have - if (Omega is not None): - if (Lambda is not None): - WS = np.ones(Omega.shape)*WS_default - elif (WS is not None): - if len(WS)!=len(Omega): - raise Exception('When providing Omega and WS, both vectors should have the same dimension') - else: - WS = np.ones(Omega.shape)*WS_default - else: - Omega = WS_default * Lambda/R*60/(2*np.pi) # TODO, use more realistic combinations of WS and Omega - WS = np.ones(Omega.shape)*WS_default - - - # --- Defining flat vectors of operating conditions - WS_flat = [] - RPM_flat = [] - Pitch_flat = [] - for pitch in Pitch: - for (rpm,ws) in zip(Omega,WS): - WS_flat.append(ws) - RPM_flat.append(rpm) - Pitch_flat.append(pitch) - # --- Setting up default options - baseDict={'TMax': TMax, 'DT': 0.01, 'DT_Out': 0.1} # NOTE: Tmax should be at least 2pi/Omega - baseDict = paramsNoController(baseDict) - if bStiff: - baseDict = paramsStiff(baseDict) - if bNoGen: - baseDict = paramsNoGen(baseDict) - if bSteadyAero: - baseDict = paramsSteadyAero(baseDict) - - # --- Creating set of parameters to be changed - # TODO: verify that RtAeroCp and RtAeroCt are present in AeroDyn outlist - PARAMS = paramsWS_RPM_Pitch(WS_flat,RPM_flat,Pitch_flat,baseDict=baseDict, FlatInputs=True) - - # --- Generating all files in a workDir - workDir = refdir.strip('/').strip('\\')+'_CPLambdaPitch' - print('>>> Generating inputs files in {}'.format(workDir)) - RemoveAllowed=reRun # If the user want to rerun, we can remove, otherwise we keep existing simulations - fastFiles=templateReplace(PARAMS, refdir, outputDir=workDir,removeRefSubFiles=True,removeAllowed=RemoveAllowed,main_file=main_fastfile) - - # --- Running fast simulations - print('>>> Running {} simulations...'.format(len(fastFiles))) - runner.run_fastfiles(fastFiles, showOutputs=showOutputs, fastExe=fastExe, nCores=nCores, reRun=reRun) - - # --- Postpro - Computing averages at the end of the simluation - print('>>> Postprocessing...') - outFiles = [os.path.splitext(f)[0]+'.outb' for f in fastFiles] - # outFiles = glob.glob(os.path.join(workDir,'*.outb')) - ColKeepStats = ['RotSpeed_[rpm]','BldPitch1_[deg]','RtAeroCp_[-]','RtAeroCt_[-]','Wind1VelX_[m/s]'] - result = postpro.averagePostPro(outFiles,avgMethod='periods',avgParam=1,ColKeep=ColKeepStats,ColSort='RotSpeed_[rpm]') - # print(result) - - # --- Adding lambda, sorting and keeping only few columns - result['lambda_[-]'] = result['RotSpeed_[rpm]']*R*2*np.pi/60/result['Wind1VelX_[m/s]'] - result.sort_values(['lambda_[-]','BldPitch1_[deg]'],ascending=[True,True],inplace=True) - ColKeepFinal=['lambda_[-]','BldPitch1_[deg]','RtAeroCp_[-]','RtAeroCt_[-]'] - result=result[ColKeepFinal] - print('>>> Done') - - # --- Converting to a matrices - CP = result['RtAeroCp_[-]'].values - CT = result['RtAeroCt_[-]'].values - MCP =CP.reshape((len(Lambda),len(Pitch))) - MCT =CT.reshape((len(Lambda),len(Pitch))) - LAMBDA, PITCH = np.meshgrid(Lambda, Pitch) - # --- CP max - i,j = np.unravel_index(MCP.argmax(), MCP.shape) - MaxVal={'CP_max':MCP[i,j],'lambda_opt':LAMBDA[j,i],'pitch_opt':PITCH[j,i]} - - return MCP,MCT,Lambda,Pitch,MaxVal,result - - -if __name__=='__main__': - # --- Test of templateReplace - PARAMS = {} - PARAMS['TMax'] = 10 - PARAMS['__name__'] = 'MyName' - PARAMS['DT'] = 0.01 - PARAMS['DT_Out'] = 0.1 - PARAMS['EDFile|RotSpeed'] = 100 - PARAMS['EDFile|BlPitch(1)'] = 1 - PARAMS['EDFile|GBoxEff'] = 0.92 - PARAMS['ServoFile|VS_Rgn2K'] = 0.00038245 - PARAMS['ServoFile|GenEff'] = 0.95 - PARAMS['InflowFile|HWindSpeed'] = 8 - templateReplace(PARAMS,refDir,RemoveRefSubFiles=True) - +from __future__ import division, print_function +import os +import collections +import glob +import pandas as pd +import numpy as np +import shutil +import stat +import re + +# --- Misc fast libraries +import weio.weio.fast_input_file as fi +import pydatview.fast.runner as runner +import pydatview.fast.postpro as postpro +#import pyFAST.input_output.fast_input_file as fi +#import pyFAST.case_generation.runner as runner +#import pyFAST.input_output.postpro as postpro + + +# --------------------------------------------------------------------------------} +# --- Template replace +# --------------------------------------------------------------------------------{ +def handleRemoveReadonlyWin(func, path, exc_info): + """ + Error handler for ``shutil.rmtree``. + If the error is due to an access error (read only file) + it attempts to add write permission and then retries. + Usage : ``shutil.rmtree(path, onerror=onerror)`` + """ + if not os.access(path, os.W_OK): + # Is the error an access error ? + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise + + +def forceCopyFile (sfile, dfile): + # ---- Handling error due to wrong mod + if os.path.isfile(dfile): + if not os.access(dfile, os.W_OK): + os.chmod(dfile, stat.S_IWUSR) + #print(sfile, ' > ', dfile) + shutil.copy2(sfile, dfile) + +def copyTree(src, dst): + """ + Copy a directory to another one, overwritting files if necessary. + copy_tree from distutils and copytree from shutil fail on Windows (in particular on git files) + """ + def forceMergeFlatDir(srcDir, dstDir): + if not os.path.exists(dstDir): + os.makedirs(dstDir) + for item in os.listdir(srcDir): + srcFile = os.path.join(srcDir, item) + dstFile = os.path.join(dstDir, item) + forceCopyFile(srcFile, dstFile) + + def isAFlatDir(sDir): + for item in os.listdir(sDir): + sItem = os.path.join(sDir, item) + if os.path.isdir(sItem): + return False + return True + + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isfile(s): + if not os.path.exists(dst): + os.makedirs(dst) + forceCopyFile(s,d) + if os.path.isdir(s): + isRecursive = not isAFlatDir(s) + if isRecursive: + copyTree(s, d) + else: + forceMergeFlatDir(s, d) + + +def templateReplaceGeneral(PARAMS, templateDir=None, outputDir=None, main_file=None, removeAllowed=False, removeRefSubFiles=False, oneSimPerDir=False): + """ Generate inputs files by replacing different parameters from a template file. + The generated files are placed in the output directory `outputDir` + The files are read and written using the library `weio`. + The template file is read and its content can be changed like a dictionary. + Each item of `PARAMS` correspond to a set of parameters that will be replaced + in the template file to generate one input file. + + For "FAST" input files, parameters can be changed recursively. + + + INPUTS: + PARAMS: list of dictionaries. Each key of the dictionary should be a key present in the + template file when read with `weio` (see: weio.read(main_file).keys() ) + + PARAMS[0]={'DT':0.1, 'EDFile|GBRatio':1, 'ServoFile|GenEff':0.8} + + templateDir: if provided, this directory and its content will be copied to `outputDir` + before doing the parametric substitution + + outputDir : directory where files will be generated. + """ + # --- Helper functions + def rebase_rel(wd,s,sid): + split = os.path.splitext(s) + return os.path.join(wd,split[0]+sid+split[1]) + + def get_strID(p) : + if '__name__' in p.keys(): + strID=p['__name__'] + else: + raise Exception('When calling `templateReplace`, provide the key `__name_` in the parameter dictionaries') + return strID + + def splitAddress(sAddress): + sp = sAddress.split('|') + if len(sp)==1: + return sp[0],[] + else: + return sp[0],sp[1:] + + def rebaseFileName(org_filename, workDir, strID): + new_filename_full = rebase_rel(workDir, org_filename,'_'+strID) + new_filename = os.path.relpath(new_filename_full,workDir).replace('\\','/') + return new_filename, new_filename_full + + def replaceRecurse(templatename_or_newname, FileKey, ParamKey, ParamValue, Files, strID, workDir, TemplateFiles): + """ + FileKey: a single key defining which file we are currently modifying e.g. :'AeroFile', 'EDFile','FVWInputFileName' + ParamKey: the address key of the parameter to be changed, relative to the current FileKey + e.g. 'EDFile|IntMethod' (if FileKey is '') + 'IntMethod' (if FileKey is 'EDFile') + ParamValue: the value to be used + Files: dict of files, as returned by weio, keys are "FileKeys" + """ + # --- Special handling for the root + if FileKey=='': + FileKey='Root' + # --- Open (or get if already open) file where a parameter needs to be changed + if FileKey in Files.keys(): + # The file was already opened, it's stored + f = Files[FileKey] + newfilename_full = f.filename + newfilename = os.path.relpath(newfilename_full,workDir).replace('\\','/') + + else: + templatefilename = templatename_or_newname + templatefilename_full = os.path.join(workDir,templatefilename) + TemplateFiles.append(templatefilename_full) + if FileKey=='Root': + # Root files, we start from strID + ext = os.path.splitext(templatefilename)[-1] + newfilename_full = os.path.join(wd,strID+ext) + newfilename = strID+ext + else: + newfilename, newfilename_full = rebaseFileName(templatefilename, workDir, strID) + #print('--------------------------------------------------------------') + #print('TemplateFile :', templatefilename) + #print('TemplateFileFull:', templatefilename_full) + #print('NewFile :', newfilename) + #print('NewFileFull :', newfilename_full) + shutil.copyfile(templatefilename_full, newfilename_full) + f= fi.FASTInputFile(newfilename_full) # open the template file for that filekey + Files[FileKey]=f # store it + + # --- Changing parameters in that file + NewFileKey_or_Key, ChildrenKeys = splitAddress(ParamKey) + if len(ChildrenKeys)==0: + # A simple parameter is changed + Key = NewFileKey_or_Key + #print('Setting', FileKey, '|',Key, 'to',ParamValue) + if Key=='OutList': + OutList=f[Key] + f[Key]=addToOutlist(OutList, ParamValue) + else: + f[Key] = ParamValue + else: + # Parameters needs to be changed in subfiles (children) + NewFileKey = NewFileKey_or_Key + ChildrenKey = '|'.join(ChildrenKeys) + child_templatefilename = f[NewFileKey].strip('"') # old filename that will be used as a template + baseparent = os.path.dirname(newfilename) + #print('Child templatefilename:',child_templatefilename) + #print('Parent base dir :',baseparent) + workDir = os.path.join(workDir, baseparent) + + # + newchildFilename, Files = replaceRecurse(child_templatefilename, NewFileKey, ChildrenKey, ParamValue, Files, strID, workDir, TemplateFiles) + #print('Setting', FileKey, '|',NewFileKey, 'to',newchildFilename) + f[NewFileKey] = '"'+newchildFilename+'"' + + return newfilename, Files + + + # --- Safety checks + if templateDir is None and outputDir is None: + raise Exception('Provide at least a template directory OR an output directory') + + if templateDir is not None: + if not os.path.exists(templateDir): + raise Exception('Template directory does not exist: '+templateDir) + + # Default value of outputDir if not provided + if templateDir[-1]=='/' or templateDir[-1]=='\\' : + templateDir=templateDir[0:-1] + if outputDir is None: + outputDir=templateDir+'_Parametric' + + # --- Main file use as "master" + if templateDir is not None: + main_file=os.path.join(outputDir, os.path.basename(main_file)) + else: + main_file=main_file + + # Params need to be a list + if not isinstance(PARAMS,list): + PARAMS=[PARAMS] + + if oneSimPerDir: + workDirS=[os.path.join(outputDir,get_strID(p)) for p in PARAMS] + else: + workDirS=[outputDir]*len(PARAMS) + # --- Creating outputDir - Copying template folder to outputDir if necessary + # Copying template folder to workDir + for wd in list(set(workDirS)): + if removeAllowed: + removeFASTOuputs(wd) + if os.path.exists(wd) and removeAllowed: + shutil.rmtree(wd, ignore_errors=False, onerror=handleRemoveReadonlyWin) + copyTree(templateDir, wd) + if removeAllowed: + removeFASTOuputs(wd) + + + TemplateFiles=[] + files=[] + for ip,(wd,p) in enumerate(zip(workDirS,PARAMS)): + if '__index__' not in p.keys(): + p['__index__']=ip + + main_file_base = os.path.basename(main_file) + strID = get_strID(p) + # --- Setting up files for this simulation + Files=dict() + for k,v in p.items(): + if k =='__index__' or k=='__name__': + continue + new_mainFile, Files = replaceRecurse(main_file_base, '', k, v, Files, strID, wd, TemplateFiles) + + # --- Writting files + for k,f in Files.items(): + if k=='Root': + files.append(f.filename) + f.write() + + # --- Remove extra files at the end + if removeRefSubFiles: + TemplateFiles, nCounts = np.unique(TemplateFiles, return_counts=True) + if not oneSimPerDir: + # we can only detele template files that were used by ALL simulations + TemplateFiles=[t for nc,t in zip(nCounts, TemplateFiles) if nc==len(PARAMS)] + for tf in TemplateFiles: + try: + os.remove(tf) + except: + print('[FAIL] Removing '+tf) + pass + return files + +def templateReplace(PARAMS, templateDir, outputDir=None, main_file=None, removeAllowed=False, removeRefSubFiles=False, oneSimPerDir=False): + """ Replace parameters in a fast folder using a list of dictionaries where the keys are for instance: + 'DT', 'EDFile|GBRatio', 'ServoFile|GenEff' + """ + # --- For backward compatibility, remove "FAST|" from the keys + for p in PARAMS: + old_keys=[ k for k,_ in p.items() if k.find('FAST|')==0] + for k_old in old_keys: + k_new=k_old.replace('FAST|','') + p[k_new] = p.pop(k_old) + + return templateReplaceGeneral(PARAMS, templateDir, outputDir=outputDir, main_file=main_file, + removeAllowed=removeAllowed, removeRefSubFiles=removeRefSubFiles, oneSimPerDir=oneSimPerDir) + +def removeFASTOuputs(workDir): + # Cleaning folder + for f in glob.glob(os.path.join(workDir,'*.out')): + os.remove(f) + for f in glob.glob(os.path.join(workDir,'*.outb')): + os.remove(f) + for f in glob.glob(os.path.join(workDir,'*.ech')): + os.remove(f) + for f in glob.glob(os.path.join(workDir,'*.sum')): + os.remove(f) + +# --------------------------------------------------------------------------------} +# --- Tools for template replacement +# --------------------------------------------------------------------------------{ +def paramsSteadyAero(p=None): + p = dict() if p is None else p + p['AeroFile|AFAeroMod']=1 # remove dynamic effects dynamic + p['AeroFile|WakeMod']=1 # remove dynamic inflow dynamic + p['AeroFile|TwrPotent']=0 # remove tower shadow + return p + +def paramsNoGen(p=None): + p = dict() if p is None else p + p['EDFile|GenDOF' ] = 'False' + return p + +def paramsGen(p=None): + p = dict() if p is None else p + p['EDFile|GenDOF' ] = 'True' + return p + +def paramsNoController(p=None): + p = dict() if p is None else p + p['ServoFile|PCMode'] = 0; + p['ServoFile|VSContrl'] = 0; + p['ServoFile|YCMode'] = 0; + return p + +def paramsControllerDLL(p=None): + p = dict() if p is None else p + p['ServoFile|PCMode'] = 5; + p['ServoFile|VSContrl'] = 5; + p['ServoFile|YCMode'] = 5; + p['EDFile|GenDOF'] = 'True'; + return p + + +def paramsStiff(p=None): + p = dict() if p is None else p + p['EDFile|FlapDOF1'] = 'False' + p['EDFile|FlapDOF2'] = 'False' + p['EDFile|EdgeDOF' ] = 'False' + p['EDFile|TeetDOF' ] = 'False' + p['EDFile|DrTrDOF' ] = 'False' + p['EDFile|YawDOF' ] = 'False' + p['EDFile|TwFADOF1'] = 'False' + p['EDFile|TwFADOF2'] = 'False' + p['EDFile|TwSSDOF1'] = 'False' + p['EDFile|TwSSDOF2'] = 'False' + p['EDFile|PtfmSgDOF'] = 'False' + p['EDFile|PtfmSwDOF'] = 'False' + p['EDFile|PtfmHvDOF'] = 'False' + p['EDFile|PtfmRDOF'] = 'False' + p['EDFile|PtfmPDOF'] = 'False' + p['EDFile|PtfmYDOF'] = 'False' + return p + +def paramsWS_RPM_Pitch(WS, RPM, Pitch, baseDict=None, flatInputs=False): + """ + Generate OpenFAST "parameters" (list of dictionaries with "address") + chaing the inputs in ElastoDyn, InflowWind for different wind speed, RPM and Pitch + """ + # --- Ensuring everythin is an iterator + def iterify(x): + if not isinstance(x, collections.Iterable): x = [x] + return x + WS = iterify(WS) + RPM = iterify(RPM) + Pitch = iterify(Pitch) + # --- If inputs are not flat but different vectors to length through, we flatten them (TODO: meshgrid and ravel?) + if not flatInputs : + WS_flat = [] + Pitch_flat = [] + RPM_flat = [] + for pitch in Pitch: + for rpm in RPM: + for ws in WS: + WS_flat.append(ws) + RPM_flat.append(rpm) + Pitch_flat.append(pitch) + else: + WS_flat, Pitch_flat, RPM_flat = WS, Pitch, RPM + + # --- Defining the parametric study + PARAMS=[] + i=0 + for ws,rpm,pitch in zip(WS_flat,RPM_flat,Pitch_flat): + if baseDict is None: + p=dict() + else: + p = baseDict.copy() + p['EDFile|RotSpeed'] = rpm + p['InflowFile|HWindSpeed'] = ws + p['InflowFile|WindType'] = 1 # Setting steady wind + p['EDFile|BlPitch(1)'] = pitch + p['EDFile|BlPitch(2)'] = pitch + p['EDFile|BlPitch(3)'] = pitch + + p['__index__'] = i + p['__name__'] = '{:03d}_ws{:04.1f}_pt{:04.2f}_om{:04.2f}'.format(p['__index__'],p['InflowFile|HWindSpeed'],p['EDFile|BlPitch(1)'],p['EDFile|RotSpeed']) + i=i+1 + PARAMS.append(p) + return PARAMS + +def paramsLinearTrim(p=None): + p = dict() if p is None else p + + # Set a few DOFs, move this to main file + p['Linearize'] = True + p['CalcSteady'] = True + p['TrimGain'] = 1e-4 + p['TrimTol'] = 1e-5 + p['CompMooring'] = 0 + p['CompHydro'] = 0 + p['LinOutJac'] = False + p['LinOutMod'] = False + p['OutFmt'] = '"ES20.12E3"' # Important for decent resolution + + p['AeroFile|AFAeroMod'] = 1 + p['AeroFile|CavitCheck'] = 'False' + p['AeroFile|CompAA'] = 'False' + + p['ServoFile|PCMode'] = 0 + p['ServoFile|VSContrl'] = 1 + + p['ServoFile|CompNTMD'] = 'False' + p['ServoFile|CompTTMD'] = 'False' + + # Set all DOFs off, enable as desired + p['EDFile|FlapDOF1'] = 'False' + p['EDFile|FlapDOF2'] = 'False' + p['EDFile|EdgeDOF'] = 'False' + p['EDFile|TeetDOF'] = 'False' + p['EDFile|DrTrDOF'] = 'False' + p['EDFile|GenDOF'] = 'False' + p['EDFile|YawDOF'] = 'False' + p['EDFile|TwFADOF1'] = 'False' + p['EDFile|TwFADOF2'] = 'False' + p['EDFile|TwSSDOF1'] = 'False' + p['EDFile|TwSSDOF2'] = 'False' + p['EDFile|PtfmSgDOF'] = 'False' + p['EDFile|PtfmSwDOF'] = 'False' + p['EDFile|PtfmHvDOF'] = 'False' + p['EDFile|PtfmRDOF'] = 'False' + p['EDFile|PtfmPDOF'] = 'False' + p['EDFile|PtfmYDOF'] = 'False' + + + return p + +# --------------------------------------------------------------------------------} +# --- +# --------------------------------------------------------------------------------{ +def createStepWind(filename,WSstep=1,WSmin=3,WSmax=25,tstep=100,dt=0.5,tmin=0,tmax=999): + f = weio.FASTWndFile() + Steps= np.arange(WSmin,WSmax+WSstep,WSstep) + print(Steps) + nCol = len(f.colNames) + nRow = len(Steps)*2 + M = np.zeros((nRow,nCol)); + M[0,0] = tmin + M[0,1] = WSmin + for i,s in enumerate(Steps[:-1]): + M[2*i+1,0] = tmin + (i+1)*tstep-dt + M[2*i+2,0] = tmin + (i+1)*tstep + M[2*i+1,1] = Steps[i] + if i0: + main_fastfile=os.path.basename(main_fastfile) + + # --- Reading main fast file to get rotor radius + fst = fi.FASTInputFile(os.path.join(refdir,main_fastfile)) + ed = fi.FASTInputFile(os.path.join(refdir,fst['EDFile'].replace('"',''))) + R = ed['TipRad'] + + # --- Making sure we have + if (Omega is not None): + if (Lambda is not None): + WS = np.ones(Omega.shape)*WS_default + elif (WS is not None): + if len(WS)!=len(Omega): + raise Exception('When providing Omega and WS, both vectors should have the same dimension') + else: + WS = np.ones(Omega.shape)*WS_default + else: + Omega = WS_default * Lambda/R*60/(2*np.pi) # TODO, use more realistic combinations of WS and Omega + WS = np.ones(Omega.shape)*WS_default + + + # --- Defining flat vectors of operating conditions + WS_flat = [] + RPM_flat = [] + Pitch_flat = [] + for pitch in Pitch: + for (rpm,ws) in zip(Omega,WS): + WS_flat.append(ws) + RPM_flat.append(rpm) + Pitch_flat.append(pitch) + # --- Setting up default options + baseDict={'TMax': TMax, 'DT': 0.01, 'DT_Out': 0.1} # NOTE: Tmax should be at least 2pi/Omega + baseDict = paramsNoController(baseDict) + if bStiff: + baseDict = paramsStiff(baseDict) + if bNoGen: + baseDict = paramsNoGen(baseDict) + if bSteadyAero: + baseDict = paramsSteadyAero(baseDict) + + # --- Creating set of parameters to be changed + # TODO: verify that RtAeroCp and RtAeroCt are present in AeroDyn outlist + PARAMS = paramsWS_RPM_Pitch(WS_flat,RPM_flat,Pitch_flat,baseDict=baseDict, FlatInputs=True) + + # --- Generating all files in a workDir + workDir = refdir.strip('/').strip('\\')+'_CPLambdaPitch' + print('>>> Generating inputs files in {}'.format(workDir)) + RemoveAllowed=reRun # If the user want to rerun, we can remove, otherwise we keep existing simulations + fastFiles=templateReplace(PARAMS, refdir, outputDir=workDir,removeRefSubFiles=True,removeAllowed=RemoveAllowed,main_file=main_fastfile) + + # --- Running fast simulations + print('>>> Running {} simulations...'.format(len(fastFiles))) + runner.run_fastfiles(fastFiles, showOutputs=showOutputs, fastExe=fastExe, nCores=nCores, reRun=reRun) + + # --- Postpro - Computing averages at the end of the simluation + print('>>> Postprocessing...') + outFiles = [os.path.splitext(f)[0]+'.outb' for f in fastFiles] + # outFiles = glob.glob(os.path.join(workDir,'*.outb')) + ColKeepStats = ['RotSpeed_[rpm]','BldPitch1_[deg]','RtAeroCp_[-]','RtAeroCt_[-]','Wind1VelX_[m/s]'] + result = postpro.averagePostPro(outFiles,avgMethod='periods',avgParam=1,ColKeep=ColKeepStats,ColSort='RotSpeed_[rpm]') + # print(result) + + # --- Adding lambda, sorting and keeping only few columns + result['lambda_[-]'] = result['RotSpeed_[rpm]']*R*2*np.pi/60/result['Wind1VelX_[m/s]'] + result.sort_values(['lambda_[-]','BldPitch1_[deg]'],ascending=[True,True],inplace=True) + ColKeepFinal=['lambda_[-]','BldPitch1_[deg]','RtAeroCp_[-]','RtAeroCt_[-]'] + result=result[ColKeepFinal] + print('>>> Done') + + # --- Converting to a matrices + CP = result['RtAeroCp_[-]'].values + CT = result['RtAeroCt_[-]'].values + MCP =CP.reshape((len(Lambda),len(Pitch))) + MCT =CT.reshape((len(Lambda),len(Pitch))) + LAMBDA, PITCH = np.meshgrid(Lambda, Pitch) + # --- CP max + i,j = np.unravel_index(MCP.argmax(), MCP.shape) + MaxVal={'CP_max':MCP[i,j],'lambda_opt':LAMBDA[j,i],'pitch_opt':PITCH[j,i]} + + return MCP,MCT,Lambda,Pitch,MaxVal,result + + +if __name__=='__main__': + # --- Test of templateReplace + PARAMS = {} + PARAMS['TMax'] = 10 + PARAMS['__name__'] = 'MyName' + PARAMS['DT'] = 0.01 + PARAMS['DT_Out'] = 0.1 + PARAMS['EDFile|RotSpeed'] = 100 + PARAMS['EDFile|BlPitch(1)'] = 1 + PARAMS['EDFile|GBoxEff'] = 0.92 + PARAMS['ServoFile|VS_Rgn2K'] = 0.00038245 + PARAMS['ServoFile|GenEff'] = 0.95 + PARAMS['InflowFile|HWindSpeed'] = 8 + templateReplace(PARAMS,refDir,RemoveRefSubFiles=True) + diff --git a/pydatview/fast/fastfarm.py b/pydatview/fast/fastfarm.py index b09df03..1c939c0 100644 --- a/pydatview/fast/fastfarm.py +++ b/pydatview/fast/fastfarm.py @@ -1,482 +1,489 @@ -import os -import glob -import numpy as np -import pandas as pd -try: - import weio -except: - raise Exception('Python package `weio` not found, please install it from https://github.com/ebranlard/weio ') - -from . import fastlib - -# --------------------------------------------------------------------------------} -# --- Small helper functions -# --------------------------------------------------------------------------------{ -def insertTN(s,i,nWT=1000): - """ insert turbine number in name """ - if nWT<10: - fmt='{:d}' - elif nWT<100: - fmt='{:02d}' - else: - fmt='{:03d}' - if s.find('T1')>=0: - s=s.replace('T1','T'+fmt.format(i)) - else: - sp=os.path.splitext(s) - s=sp[0]+'_T'+fmt.format(i)+sp[1] - return s -def forceCopyFile (sfile, dfile): - # ---- Handling error due to wrong mod - if os.path.isfile(dfile): - if not os.access(dfile, os.W_OK): - os.chmod(dfile, stat.S_IWUSR) - #print(sfile, ' > ', dfile) - shutil.copy2(sfile, dfile) - -# --------------------------------------------------------------------------------} -# --- Tools to create fast farm simulations -# --------------------------------------------------------------------------------{ -def writeFSTandDLL(FstT1Name, nWT): - """ - Write FST files for each turbine, with different ServoDyn files and DLL - FST files, ServoFiles, and DLL files will be written next to their turbine 1 - files, with name Ti. - - FstT1Name: absolute or relative path to the Turbine FST file - """ - - FstT1Full = os.path.abspath(FstT1Name).replace('\\','/') - FstDir = os.path.dirname(FstT1Full) - - fst=weio.read(FstT1Name) - SrvT1Name = fst['ServoFile'].strip('"') - SrvT1Full = os.path.join(FstDir, SrvT1Name).replace('\\','/') - SrvDir = os.path.dirname(SrvT1Full) - SrvT1RelFst = os.path.relpath(SrvT1Full,FstDir) - if os.path.exists(SrvT1Full): - srv=weio.read(SrvT1Full) - DLLT1Name = srv['DLL_FileName'].strip('"') - DLLT1Full = os.path.join(SrvDir, DLLT1Name) - if os.path.exists(DLLT1Full): - servo=True - else: - print('[Info] DLL file not found, not copying servo and dll files ({})'.format(DLLT1Full)) - servo=False - else: - print('[Info] ServoDyn file not found, not copying servo and dll files ({})'.format(SrvT1Full)) - servo=False - - #print(FstDir) - #print(FstT1Full) - #print(SrvT1Name) - #print(SrvT1Full) - #print(SrvT1RelFst) - - for i in np.arange(2,nWT+1): - FstName = insertTN(FstT1Name,i,nWT) - if servo: - # TODO handle the case where T1 not present - SrvName = insertTN(SrvT1Name,i,nWT) - DLLName = insertTN(DLLT1Name,i,nWT) - DLLFullName = os.path.join(SrvDir, DLLName) - - print('') - print('FstName: ',FstName) - if servo: - print('SrvName: ',SrvName) - print('DLLName: ',DLLName) - print('DLLFull: ',DLLFullName) - - # Changing main file - if servo: - fst['ServoFile']='"'+SrvName+'"' - fst.write(FstName) - if servo: - # Changing servo file - srv['DLL_FileName']='"'+DLLName+'"' - srv.write(SrvName) - # Copying dll - forceCopyFile(DLLT1Full, DLLFullName) - - - -def rectangularLayoutSubDomains(D,Lx,Ly): - """ Retuns position of turbines in a rectangular layout - TODO, unfinished function parameters - """ - # --- Parameters - D = 112 # turbine diameter [m] - Lx = 3840 # x dimension of precusor - Ly = 3840 # y dimension of precusor - Height = 0 # Height above ground, likely 0 [m] - nDomains_x = 2 # number of domains in x - nDomains_y = 2 # number of domains in y - # --- 36 WT - nx = 3 # number of turbines to be placed along x in one precursor domain - ny = 3 # number of turbines to be placed along y in one precursor domain - StartX = 1/2 # How close do we start from the x boundary - StartY = 1/2 # How close do we start from the y boundary - # --- Derived parameters - Lx_Domain = Lx * nDomains_x # Full domain size - Ly_Domain = Ly * nDomains_y - DeltaX = Lx / (nx) # Turbine spacing - DeltaY = Ly / (ny) - xWT = np.arange(DeltaX*StartX,Lx_Domain,DeltaX) # Turbine positions - yWT = np.arange(DeltaY*StartY,Ly_Domain,DeltaY) - - print('Full domain size [D] : {:.2f} x {:.2f} '.format(Lx_Domain/D, Ly_Domain/D)) - print('Turbine spacing [D] : {:.2f} x {:.2f} '.format(DeltaX/D,DeltaX/D)) - print('Number of turbines : {:d} x {:d} = {:d}'.format(len(xWT),len(yWT),len(xWT)*len(yWT))) - - XWT,YWT=np.meshgrid(xWT,yWT) - ZWT=XWT*0+Height - - # --- Export coordinates only - M=np.column_stack((XWT.ravel(),YWT.ravel(),ZWT.ravel())) - np.savetxt('Farm_Coordinates.csv', M, delimiter=',',header='X_[m], Y_[m], Z_[m]') - print(M) - - return XWT, YWT, ZWT - -def fastFarmTurbSimExtent(TurbSimFile, HubHeight, D, xWT, yWT, Cmeander=1.9, Chord_max=3, extent_X=1.2, extent_Y=1.2): - """ - Determines "Ambient Wind" box parametesr for FastFarm, based on a TurbSimFile ('bts') - """ - # --- TurbSim data - ts = weio.read(TurbSimFile) - iy,iz = ts.closestPoint(y=0,z=HubHeight) - meanU = ts['u'][0,:,iy,iz].mean() - dY_High = ts['y'][1]-ts['y'][0] - dZ_High = ts['z'][1]-ts['z'][0] - Z0_Low = ts['z'][0] - Z0_High = ts['z'][0] # we start at lowest to include tower - Width = ts['y'][-1]-ts['y'][0] - Height = ts['z'][-1]-ts['z'][0] - dT_High = ts['dt'] - effSimLength = ts['t'][-1]-ts['t'][0] + Width/meanU - - # Desired resolution, rule of thumbs - dX_High_desired = Chord_max - dX_Low_desired = Cmeander*D*meanU/150.0 - dt_des = Cmeander*D/(10.0*meanU) - - # --- High domain - ZMax_High = HubHeight+extent_Y*D/2.0 - # high-box extent in x and y [D] - Xdist_High = extent_X*D - Ydist_High = extent_Y*D - Zdist_High = ZMax_High-Z0_High # we include the tower - X0_rel = Xdist_High/2.0 - Y0_rel = Ydist_High/2.0 - Length = effSimLength*meanU - nx = int(round(effSimLength/dT_High)) - dx_TS = Length/(nx-1) - dX_High = round(dX_High_desired/dx_TS)*dx_TS - - nX_High = int(round(Xdist_High/dX_High)+1) - nY_High = int(round(Ydist_High/dY_High)+1) - nZ_High = int(round(Zdist_High/dZ_High)+1) - - # --- High extent per turbine - nTurbs = len(xWT) - X0_des = np.asarray(xWT)-X0_rel - Y0_des = np.asarray(yWT)-Y0_rel - X0_High = np.around(np.round(X0_des/dX_High)*dX_High,3) - Y0_High = np.around(np.round(Y0_des/dY_High)*dY_High,3) - - # --- Low domain - dT_Low = round(dt_des/dT_High)*dT_High - dx_des = dX_Low_desired - dy_des = dX_Low_desired - dz_des = dX_Low_desired - X0_Low = min(xWT)-2*D - Y0_Low = -Width/2 - dX_Low = round(dx_des/dX_High)*dX_High - dY_Low = round(dy_des/dY_High)*dY_High - dZ_Low = round(dz_des/dZ_High)*dZ_High - Xdist = max(xWT)+8.0*D-X0_Low # Maximum extent - Ydist = Width - Zdist = Height - - nX_Low = int(Xdist/dX_Low)+1; - nY_Low = int(Ydist/dY_Low)+1; - nZ_Low = int(Zdist/dZ_Low)+1; - - if (nX_Low*dX_Low>Xdist): - nX_Low=nX_Low-1 - if (nY_Low*dY_Low>Ydist): - nY_Low=nY_Low-1 - if (nZ_Low*dZ_Low>Zdist): - nZ_Low=nZ_Low-1 - - d = dict() - d['DT'] = np.around(dT_Low ,3) - d['DT_High'] = np.around(dT_High,3) - d['NX_Low'] = int(nX_Low) - d['NY_Low'] = int(nY_Low) - d['NZ_Low'] = int(nZ_Low) - d['X0_Low'] = np.around(X0_Low,3) - d['Y0_Low'] = np.around(Y0_Low,3) - d['Z0_Low'] = np.around(Z0_Low,3) - d['dX_Low'] = np.around(dX_Low,3) - d['dY_Low'] = np.around(dY_Low,3) - d['dZ_Low'] = np.around(dZ_Low,3) - d['NX_High'] = int(nX_High) - d['NY_High'] = int(nY_High) - d['NZ_High'] = int(nZ_High) - # --- High extent info for turbine outputs - d['dX_High'] = np.around(dX_High,3) - d['dY_High'] = np.around(dY_High,3) - d['dZ_High'] = np.around(dZ_High,3) - d['X0_High'] = X0_High - d['Y0_High'] = Y0_High - d['Z0_High'] = np.around(Z0_High,3) - - return d - -def writeFastFarm(outputFile, templateFile, xWT, yWT, zWT, FFTS=None, OutListT1=None): - """ Write FastFarm input file based on a template, a TurbSimFile and the Layout - - outputFile: .fstf file to be written - templateFile: .fstf file that will be used to generate the output_file - XWT,YWT,ZWT: positions of turbines - FFTS: FastFarm TurbSim parameters as returned by fastFarmTurbSimExtent - """ - # --- Read template fast farm file - fst=weio.FASTInputFile(templateFile) - # --- Replace box extent values - if FFTS is not None: - fst['Mod_AmbWind'] = 2 - for k in ['DT', 'DT_High', 'NX_Low', 'NY_Low', 'NZ_Low', 'X0_Low', 'Y0_Low', 'Z0_Low', 'dX_Low', 'dY_Low', 'dZ_Low', 'NX_High', 'NY_High', 'NZ_High']: - if isinstance(FFTS[k],int): - fst[k] = FFTS[k] - else: - fst[k] = np.around(FFTS[k],3) - fst['WrDisDT'] = FFTS['DT'] - - # --- Set turbine names, position, and box extent - nWT = len(xWT) - fst['NumTurbines'] = nWT - if FFTS is not None: - nCol= 10 - else: - nCol = 4 - ref_path = fst['WindTurbines'][0,3] - WT = np.array(['']*nWT*nCol,dtype='object').reshape((nWT,nCol)) - for iWT,(x,y,z) in enumerate(zip(xWT,yWT,zWT)): - WT[iWT,0]=x - WT[iWT,1]=y - WT[iWT,2]=z - WT[iWT,3]=insertTN(ref_path,iWT+1,nWT) - if FFTS is not None: - WT[iWT,4]=FFTS['X0_High'][iWT] - WT[iWT,5]=FFTS['Y0_High'][iWT] - WT[iWT,6]=FFTS['Z0_High'] - WT[iWT,7]=FFTS['dX_High'] - WT[iWT,8]=FFTS['dY_High'] - WT[iWT,9]=FFTS['dZ_High'] - fst['WindTurbines']=WT - - fst.write(outputFile) - if OutListT1 is not None: - setFastFarmOutputs(outputFile, OutListT1) - -def setFastFarmOutputs(fastFarmFile, OutListT1): - """ Duplicate the output list, by replacing "T1" with T1->Tn """ - fst = weio.read(fastFarmFile) - nWTOut = min(fst['NumTurbines'],9) # Limited to 9 turbines - OutList=[''] - for s in OutListT1: - s=s.strip('"') - if s.find('T1'): - OutList+=['"'+s.replace('T1','T{:d}'.format(iWT+1))+'"' for iWT in np.arange(nWTOut) ] - else: - OutList+='"'+s+'"' - fst['OutList']=OutList - fst.write(fastFarmFile) - - -def plotFastFarmSetup(fastFarmFile): - """ """ - import matplotlib.pyplot as plt - fst=weio.FASTInputFile(fastFarmFile) - - fig = plt.figure(figsize=(13.5,10)) - ax = fig.add_subplot(111,aspect="equal") - - WT=fst['WindTurbines'] - x = WT[:,0].astype(float) - y = WT[:,1].astype(float) - - if fst['Mod_AmbWind'] == 2: - xmax_low = fst['X0_Low']+fst['DX_Low']*fst['NX_Low'] - ymax_low = fst['Y0_Low']+fst['DY_Low']*fst['NY_Low'] - # low-res box - ax.plot([fst['X0_Low'],xmax_low,xmax_low,fst['X0_Low'],fst['X0_Low']], - [fst['Y0_Low'],fst['Y0_Low'],ymax_low,ymax_low,fst['Y0_Low']],'--k',lw=2,label='Low') - X0_High = WT[:,4].astype(float) - Y0_High = WT[:,5].astype(float) - dX_High = WT[:,7].astype(float)[0] - dY_High = WT[:,8].astype(float)[0] - nX_High = fst['NX_High'] - nY_High = fst['NY_High'] - # high-res boxes - for wt in range(len(x)): - xmax_high = X0_High[wt]+dX_High*nX_High - ymax_high = Y0_High[wt]+dY_High*nY_High - ax.plot([X0_High[wt],xmax_high,xmax_high,X0_High[wt],X0_High[wt]], - [Y0_High[wt],Y0_High[wt],ymax_high,ymax_high,Y0_High[wt]], - '-', - label="HighT{0}".format(wt+1)) - ax.plot(x[wt],y[wt],'x',ms=8,mew=2,label="WT{0}".format(wt+1)) - else: - for wt in range(len(x)): - ax.plot(x[wt],y[wt],'x',ms=8,mew=2,label="WT{0}".format(wt+1)) - # - plt.legend(bbox_to_anchor=(1.05,1.015),frameon=False) - ax.set_xlabel("x-location [m]") - ax.set_ylabel("y-location [m]") - fig.tight_layout - # fig.savefig('FFarmLayout.pdf',bbox_to_inches='tight',dpi=500) - -# --------------------------------------------------------------------------------} -# --- Tools for postpro -# --------------------------------------------------------------------------------{ - -def spanwiseColFastFarm(Cols, nWT=9, nD=9): - """ Return column info, available columns and indices that contain AD spanwise data""" - FFSpanMap=dict() - for i in np.arange(nWT): - FFSpanMap['^CtT{:d}N(\d*)_\[-\]'.format(i+1)]='CtT{:d}_[-]'.format(i+1) - for i in np.arange(nWT): - for k in np.arange(nD): - FFSpanMap['^WkDfVxT{:d}N(\d*)D{:d}_\[m/s\]'.format(i+1,k+1) ]='WkDfVxT{:d}D{:d}_[m/s]'.format(i+1, k+1) - for i in np.arange(nWT): - for k in np.arange(nD): - FFSpanMap['^WkDfVrT{:d}N(\d*)D{:d}_\[m/s\]'.format(i+1,k+1) ]='WkDfVrT{:d}D{:d}_[m/s]'.format(i+1, k+1) - - return fastlib.find_matching_columns(Cols, FFSpanMap) - -def diameterwiseColFastFarm(Cols, nWT=9): - """ Return column info, available columns and indices that contain AD spanwise data""" - FFDiamMap=dict() - for i in np.arange(nWT): - for x in ['X','Y','Z']: - FFDiamMap['^WkAxs{}T{:d}D(\d*)_\[-\]'.format(x,i+1)] ='WkAxs{}T{:d}_[-]'.format(x,i+1) - for i in np.arange(nWT): - for x in ['X','Y','Z']: - FFDiamMap['^WkPos{}T{:d}D(\d*)_\[m\]'.format(x,i+1)] ='WkPos{}T{:d}_[m]'.format(x,i+1) - for i in np.arange(nWT): - for x in ['X','Y','Z']: - FFDiamMap['^WkVel{}T{:d}D(\d*)_\[m/s\]'.format(x,i+1)] ='WkVel{}T{:d}_[m/s]'.format(x,i+1) - for i in np.arange(nWT): - for x in ['X','Y','Z']: - FFDiamMap['^WkDiam{}T{:d}D(\d*)_\[m\]'.format(x,i+1)] ='WkDiam{}T{:d}_[m]'.format(x,i+1) - return fastlib.find_matching_columns(Cols, FFDiamMap) - -def SensorsFARMRadial(nWT=3,nD=10,nR=30,signals=None): - """ Returns a list of FASTFarm sensors that are used for the radial distribution - of quantities (e.g. Ct, Wake Deficits). - If `signals` is provided, the output is the list of sensors within the list `signals`. - """ - WT = np.arange(nWT) - r = np.arange(nR) - D = np.arange(nD) - sens=[] - sens+=['CtT{:d}N{:02d}_[-]'.format(i+1,j+1) for i in WT for j in r] - sens+=['WkDfVxT{:d}N{:02d}D{:d}_[m/s]'.format(i+1,j+1,k+1) for i in WT for j in r for k in D] - sens+=['WkDfVrT{:d}N{:02d}D{:d}_[m/s]'.format(i+1,j+1,k+1) for i in WT for j in r for k in D] - if signals is not None: - sens = [c for c in sens if c in signals] - return sens - -def SensorsFARMDiam(nWT,nD): - """ Returns a list of FASTFarm sensors that contain quantities at different downstream diameters - (e.g. WkAxs, WkPos, WkVel, WkDiam) - If `signals` is provided, the output is the list of sensors within the list `signals`. - """ - WT = np.arange(nWT) - D = np.arange(nD) - XYZ = ['X','Y','Z'] - sens=[] - sens+=['WkAxs{}T{:d}D{:d}_[-]'.format(x,i+1,j+1) for x in XYZ for i in WT for j in D] - sens+=['WkPos{}T{:d}D{:d}_[m]'.format(x,i+1,j+1) for x in XYZ for i in WT for j in D] - sens+=['WkVel{}T{:d}D{:d}_[m/s]'.format(x,i+1,j+1) for x in XYZ for i in WT for j in D] - sens+=['WkDiam{}T{:d}D{:d}_[m]'.format(x,i+1,j+1) for x in XYZ for i in WT for j in D] - if signals is not None: - sens = [c for c in sens if c in signals] - return sens - - -def extractFFRadialData(fastfarm_out,fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1,df=None): - # LEGACY - return spanwisePostProFF(fastfarm_input,avgMethod=avgMethod,avgParam=avgParam,D=D,df=df,fastfarm_out=fastfarm_out) - - -def spanwisePostProFF(fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1,df=None,fastfarm_out=None): - """ - Opens a FASTFarm output file, extract the radial data, average them and returns spanwise data - - D: diameter TODO, extract it from the main file - - See faslibt.averageDF for `avgMethod` and `avgParam`. - """ - # --- Opening ouputfile - if df is None: - df=weio.read(fastfarm_out).toDataFrame() - - # --- Opening input file and extracting inportant variables - if fastfarm_input is None: - # We don't have an input file, guess numbers of turbine, diameters, Nodes... - cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'T(\d+)') - nWT = np.array(sIdx).astype(int).max() - cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'D(\d+)') - nD = np.array(sIdx).astype(int).max() - cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'N(\d+)') - nr = np.array(sIdx).astype(int).max() - vr=None - vD=None - D=0 - else: - main=weio.FASTInputFile(fastfarm_input) - iOut = main['OutRadii'] - dr = main['dr'] # Radial increment of radial finite-difference grid (m) - OutDist = main['OutDist'] # List of downstream distances for wake output for an individual rotor - WT = main['WindTurbines'] - nWT = len(WT) - vr = dr*np.array(iOut) - vD = np.array(OutDist) - nr=len(iOut) - nD=len(vD) - - - # --- Extracting time series of radial data only - colRadial = SensorsFARMRadial(nWT=nWT,nD=nD,nR=nr,signals=df.columns.values) - colRadial=['Time_[s]']+colRadial - dfRadialTime = df[colRadial] # TODO try to do some magic with it, display it with a slider - - # --- Averaging data - dfAvg = fastlib.averageDF(df,avgMethod=avgMethod,avgParam=avgParam) - - # --- Extract radial data - ColsInfo, nrMax = spanwiseColFastFarm(df.columns.values, nWT=nWT, nD=nD) - dfRad = fastlib.extract_spanwise_data(ColsInfo, nrMax, df=None, ts=dfAvg.iloc[0]) - #dfRad = fastlib.insert_radial_columns(dfRad, vr) - if vr is None: - dfRad.insert(0, 'i_[#]', np.arange(nrMax)+1) - else: - dfRad.insert(0, 'r_[m]', vr[:nrMax]) - dfRad['i/n_[-]']=np.arange(nrMax)/nrMax - - # --- Extract downstream data - ColsInfo, nDMax = diameterwiseColFastFarm(df.columns.values, nWT=nWT) - dfDiam = fastlib.extract_spanwise_data(ColsInfo, nDMax, df=None, ts=dfAvg.iloc[0]) - #dfDiam = fastlib.insert_radial_columns(dfDiam) - if vD is None: - dfDiam.insert(0, 'i_[#]', np.arange(nDMax)+1) - else: - dfDiam.insert(0, 'x_[m]', vD[:nDMax]) - dfDiam['i/n_[-]'] = np.arange(nDMax)/nDMax - return dfRad, dfRadialTime, dfDiam - +import os +import glob +import numpy as np +import pandas as pd +from weio.weio.fast_input_file import FASTInputFile +from weio.weio.fast_output_file import FASTOutputFile +from weio.weio.turbsim_file import TurbSimFile + +from . import fastlib + +# --------------------------------------------------------------------------------} +# --- Small helper functions +# --------------------------------------------------------------------------------{ +def insertTN(s,i,nWT=1000): + """ insert turbine number in name """ + if nWT<10: + fmt='{:d}' + elif nWT<100: + fmt='{:02d}' + else: + fmt='{:03d}' + if s.find('T1')>=0: + s=s.replace('T1','T'+fmt.format(i)) + else: + sp=os.path.splitext(s) + s=sp[0]+'_T'+fmt.format(i)+sp[1] + return s +def forceCopyFile (sfile, dfile): + # ---- Handling error due to wrong mod + if os.path.isfile(dfile): + if not os.access(dfile, os.W_OK): + os.chmod(dfile, stat.S_IWUSR) + #print(sfile, ' > ', dfile) + shutil.copy2(sfile, dfile) + +# --------------------------------------------------------------------------------} +# --- Tools to create fast farm simulations +# --------------------------------------------------------------------------------{ +def writeFSTandDLL(FstT1Name, nWT): + """ + Write FST files for each turbine, with different ServoDyn files and DLL + FST files, ServoFiles, and DLL files will be written next to their turbine 1 + files, with name Ti. + + FstT1Name: absolute or relative path to the Turbine FST file + """ + + FstT1Full = os.path.abspath(FstT1Name).replace('\\','/') + FstDir = os.path.dirname(FstT1Full) + + fst=FASTInputFile(FstT1Name) + SrvT1Name = fst['ServoFile'].strip('"') + SrvT1Full = os.path.join(FstDir, SrvT1Name).replace('\\','/') + SrvDir = os.path.dirname(SrvT1Full) + SrvT1RelFst = os.path.relpath(SrvT1Full,FstDir) + if os.path.exists(SrvT1Full): + srv=FASTInputFile(SrvT1Full) + DLLT1Name = srv['DLL_FileName'].strip('"') + DLLT1Full = os.path.join(SrvDir, DLLT1Name) + if os.path.exists(DLLT1Full): + servo=True + else: + print('[Info] DLL file not found, not copying servo and dll files ({})'.format(DLLT1Full)) + servo=False + else: + print('[Info] ServoDyn file not found, not copying servo and dll files ({})'.format(SrvT1Full)) + servo=False + + #print(FstDir) + #print(FstT1Full) + #print(SrvT1Name) + #print(SrvT1Full) + #print(SrvT1RelFst) + + for i in np.arange(2,nWT+1): + FstName = insertTN(FstT1Name,i,nWT) + if servo: + # TODO handle the case where T1 not present + SrvName = insertTN(SrvT1Name,i,nWT) + DLLName = insertTN(DLLT1Name,i,nWT) + DLLFullName = os.path.join(SrvDir, DLLName) + + print('') + print('FstName: ',FstName) + if servo: + print('SrvName: ',SrvName) + print('DLLName: ',DLLName) + print('DLLFull: ',DLLFullName) + + # Changing main file + if servo: + fst['ServoFile']='"'+SrvName+'"' + fst.write(FstName) + if servo: + # Changing servo file + srv['DLL_FileName']='"'+DLLName+'"' + srv.write(SrvName) + # Copying dll + forceCopyFile(DLLT1Full, DLLFullName) + + + +def rectangularLayoutSubDomains(D,Lx,Ly): + """ Retuns position of turbines in a rectangular layout + TODO, unfinished function parameters + """ + # --- Parameters + D = 112 # turbine diameter [m] + Lx = 3840 # x dimension of precusor + Ly = 3840 # y dimension of precusor + Height = 0 # Height above ground, likely 0 [m] + nDomains_x = 2 # number of domains in x + nDomains_y = 2 # number of domains in y + # --- 36 WT + nx = 3 # number of turbines to be placed along x in one precursor domain + ny = 3 # number of turbines to be placed along y in one precursor domain + StartX = 1/2 # How close do we start from the x boundary + StartY = 1/2 # How close do we start from the y boundary + # --- Derived parameters + Lx_Domain = Lx * nDomains_x # Full domain size + Ly_Domain = Ly * nDomains_y + DeltaX = Lx / (nx) # Turbine spacing + DeltaY = Ly / (ny) + xWT = np.arange(DeltaX*StartX,Lx_Domain,DeltaX) # Turbine positions + yWT = np.arange(DeltaY*StartY,Ly_Domain,DeltaY) + + print('Full domain size [D] : {:.2f} x {:.2f} '.format(Lx_Domain/D, Ly_Domain/D)) + print('Turbine spacing [D] : {:.2f} x {:.2f} '.format(DeltaX/D,DeltaX/D)) + print('Number of turbines : {:d} x {:d} = {:d}'.format(len(xWT),len(yWT),len(xWT)*len(yWT))) + + XWT,YWT=np.meshgrid(xWT,yWT) + ZWT=XWT*0+Height + + # --- Export coordinates only + M=np.column_stack((XWT.ravel(),YWT.ravel(),ZWT.ravel())) + np.savetxt('Farm_Coordinates.csv', M, delimiter=',',header='X_[m], Y_[m], Z_[m]') + print(M) + + return XWT, YWT, ZWT + +def fastFarmTurbSimExtent(TurbSimFilename, HubHeight, D, xWT, yWT, Cmeander=1.9, Chord_max=3, extent_X=1.2, extent_Y=1.2): + """ + Determines "Ambient Wind" box parametesr for FastFarm, based on a TurbSimFile ('bts') + """ + # --- TurbSim data + ts = TurbSimFile(TurbSimFilename) + #iy,iz = ts.closestPoint(y=0,z=HubHeight) + #iy,iz = ts.closestPoint(y=0,z=HubHeight) + zMid, uMid = ts.midValues() + #print('uMid',uMid) + #meanU = ts['u'][0,:,iy,iz].mean() + meanU = uMid + dY_High = ts['y'][1]-ts['y'][0] + dZ_High = ts['z'][1]-ts['z'][0] + Z0_Low = ts['z'][0] + Z0_High = ts['z'][0] # we start at lowest to include tower + Width = ts['y'][-1]-ts['y'][0] + Height = ts['z'][-1]-ts['z'][0] + dT_High = ts['dt'] + #effSimLength = ts['t'][-1]-ts['t'][0] + Width/meanU + effSimLength = ts['t'][-1]-ts['t'][0] + + # Desired resolution, rule of thumbs + dX_High_desired = Chord_max + dX_Low_desired = Cmeander*D*meanU/150.0 + dt_des = Cmeander*D/(10.0*meanU) + + # --- High domain + ZMax_High = HubHeight+extent_Y*D/2.0 + # high-box extent in x and y [D] + Xdist_High = extent_X*D + Ydist_High = extent_Y*D + Zdist_High = ZMax_High-Z0_High # we include the tower + X0_rel = Xdist_High/2.0 + Y0_rel = Ydist_High/2.0 + Length = effSimLength*meanU + nx = int(round(effSimLength/dT_High)) + dx_TS = Length/(nx-1) + #print('dx_TS',dx_TS) + dX_High = round(dX_High_desired/dx_TS)*dx_TS + #print('dX_High_desired',dX_High_desired, dX_High) + + nX_High = int(round(Xdist_High/dX_High)+1) + nY_High = int(round(Ydist_High/dY_High)+1) + nZ_High = int(round(Zdist_High/dZ_High)+1) + + # --- High extent per turbine + nTurbs = len(xWT) + X0_des = np.asarray(xWT)-X0_rel + Y0_des = np.asarray(yWT)-Y0_rel + X0_High = np.around(np.round(X0_des/dX_High)*dX_High,3) + Y0_High = np.around(np.round(Y0_des/dY_High)*dY_High,3) + + # --- Low domain + dT_Low = round(dt_des/dT_High)*dT_High + dx_des = dX_Low_desired + dy_des = dX_Low_desired + dz_des = dX_Low_desired + X0_Low = round( (min(xWT)-2*D)/dX_High) *dX_High + Y0_Low = round( -Width/2 /dY_High) *dY_High + dX_Low = round( dx_des /dX_High)*dX_High + dY_Low = round( dy_des /dY_High)*dY_High + dZ_Low = round( dz_des /dZ_High)*dZ_High + Xdist = max(xWT)+8.0*D-X0_Low # Maximum extent + Ydist = Width + Zdist = Height + #print('dX_Low',dX_Low, dX_Low/dx_TS, dX_High/dx_TS) + + nX_Low = int(Xdist/dX_Low)+1; + nY_Low = int(Ydist/dY_Low)+1; + nZ_Low = int(Zdist/dZ_Low)+1; + + if (nX_Low*dX_Low>Xdist): + nX_Low=nX_Low-1 + if (nY_Low*dY_Low>Ydist): + nY_Low=nY_Low-1 + if (nZ_Low*dZ_Low>Zdist): + nZ_Low=nZ_Low-1 + + d = dict() + d['DT'] = np.around(dT_Low ,3) + d['DT_High'] = np.around(dT_High,3) + d['NX_Low'] = int(nX_Low) + d['NY_Low'] = int(nY_Low) + d['NZ_Low'] = int(nZ_Low) + d['X0_Low'] = np.around(X0_Low,3) + d['Y0_Low'] = np.around(Y0_Low,3) + d['Z0_Low'] = np.around(Z0_Low,3) + d['dX_Low'] = np.around(dX_Low,3) + d['dY_Low'] = np.around(dY_Low,3) + d['dZ_Low'] = np.around(dZ_Low,3) + d['NX_High'] = int(nX_High) + d['NY_High'] = int(nY_High) + d['NZ_High'] = int(nZ_High) + # --- High extent info for turbine outputs + d['dX_High'] = np.around(dX_High,3) + d['dY_High'] = np.around(dY_High,3) + d['dZ_High'] = np.around(dZ_High,3) + d['X0_High'] = X0_High + d['Y0_High'] = Y0_High + d['Z0_High'] = np.around(Z0_High,3) + + return d + +def writeFastFarm(outputFile, templateFile, xWT, yWT, zWT, FFTS=None, OutListT1=None): + """ Write FastFarm input file based on a template, a TurbSimFile and the Layout + + outputFile: .fstf file to be written + templateFile: .fstf file that will be used to generate the output_file + XWT,YWT,ZWT: positions of turbines + FFTS: FastFarm TurbSim parameters as returned by fastFarmTurbSimExtent + """ + # --- Read template fast farm file + fst=FASTInputFile(templateFile) + # --- Replace box extent values + if FFTS is not None: + fst['Mod_AmbWind'] = 2 + for k in ['DT', 'DT_High', 'NX_Low', 'NY_Low', 'NZ_Low', 'X0_Low', 'Y0_Low', 'Z0_Low', 'dX_Low', 'dY_Low', 'dZ_Low', 'NX_High', 'NY_High', 'NZ_High']: + if isinstance(FFTS[k],int): + fst[k] = FFTS[k] + else: + fst[k] = np.around(FFTS[k],3) + fst['WrDisDT'] = FFTS['DT'] + + # --- Set turbine names, position, and box extent + nWT = len(xWT) + fst['NumTurbines'] = nWT + if FFTS is not None: + nCol= 10 + else: + nCol = 4 + ref_path = fst['WindTurbines'][0,3] + WT = np.array(['']*nWT*nCol,dtype='object').reshape((nWT,nCol)) + for iWT,(x,y,z) in enumerate(zip(xWT,yWT,zWT)): + WT[iWT,0]=x + WT[iWT,1]=y + WT[iWT,2]=z + WT[iWT,3]=insertTN(ref_path,iWT+1,nWT) + if FFTS is not None: + WT[iWT,4]=FFTS['X0_High'][iWT] + WT[iWT,5]=FFTS['Y0_High'][iWT] + WT[iWT,6]=FFTS['Z0_High'] + WT[iWT,7]=FFTS['dX_High'] + WT[iWT,8]=FFTS['dY_High'] + WT[iWT,9]=FFTS['dZ_High'] + fst['WindTurbines']=WT + + fst.write(outputFile) + if OutListT1 is not None: + setFastFarmOutputs(outputFile, OutListT1) + +def setFastFarmOutputs(fastFarmFile, OutListT1): + """ Duplicate the output list, by replacing "T1" with T1->Tn """ + fst = FASTInputFile(fastFarmFile) + nWTOut = min(fst['NumTurbines'],9) # Limited to 9 turbines + OutList=[''] + for s in OutListT1: + s=s.strip('"') + if s.find('T1'): + OutList+=['"'+s.replace('T1','T{:d}'.format(iWT+1))+'"' for iWT in np.arange(nWTOut) ] + else: + OutList+='"'+s+'"' + fst['OutList']=OutList + fst.write(fastFarmFile) + + +def plotFastFarmSetup(fastFarmFile): + """ """ + import matplotlib.pyplot as plt + fst=FASTInputFile(fastFarmFile) + + fig = plt.figure(figsize=(13.5,10)) + ax = fig.add_subplot(111,aspect="equal") + + WT=fst['WindTurbines'] + x = WT[:,0].astype(float) + y = WT[:,1].astype(float) + + if fst['Mod_AmbWind'] == 2: + xmax_low = fst['X0_Low']+fst['DX_Low']*fst['NX_Low'] + ymax_low = fst['Y0_Low']+fst['DY_Low']*fst['NY_Low'] + # low-res box + ax.plot([fst['X0_Low'],xmax_low,xmax_low,fst['X0_Low'],fst['X0_Low']], + [fst['Y0_Low'],fst['Y0_Low'],ymax_low,ymax_low,fst['Y0_Low']],'--k',lw=2,label='Low') + X0_High = WT[:,4].astype(float) + Y0_High = WT[:,5].astype(float) + dX_High = WT[:,7].astype(float)[0] + dY_High = WT[:,8].astype(float)[0] + nX_High = fst['NX_High'] + nY_High = fst['NY_High'] + # high-res boxes + for wt in range(len(x)): + xmax_high = X0_High[wt]+dX_High*nX_High + ymax_high = Y0_High[wt]+dY_High*nY_High + ax.plot([X0_High[wt],xmax_high,xmax_high,X0_High[wt],X0_High[wt]], + [Y0_High[wt],Y0_High[wt],ymax_high,ymax_high,Y0_High[wt]], + '-', + label="HighT{0}".format(wt+1)) + ax.plot(x[wt],y[wt],'x',ms=8,mew=2,label="WT{0}".format(wt+1)) + else: + for wt in range(len(x)): + ax.plot(x[wt],y[wt],'x',ms=8,mew=2,label="WT{0}".format(wt+1)) + # + plt.legend(bbox_to_anchor=(1.05,1.015),frameon=False) + ax.set_xlabel("x-location [m]") + ax.set_ylabel("y-location [m]") + fig.tight_layout + # fig.savefig('FFarmLayout.pdf',bbox_to_inches='tight',dpi=500) + +# --------------------------------------------------------------------------------} +# --- Tools for postpro +# --------------------------------------------------------------------------------{ + +def spanwiseColFastFarm(Cols, nWT=9, nD=9): + """ Return column info, available columns and indices that contain AD spanwise data""" + FFSpanMap=dict() + for i in np.arange(nWT): + FFSpanMap['^CtT{:d}N(\d*)_\[-\]'.format(i+1)]='CtT{:d}_[-]'.format(i+1) + for i in np.arange(nWT): + for k in np.arange(nD): + FFSpanMap['^WkDfVxT{:d}N(\d*)D{:d}_\[m/s\]'.format(i+1,k+1) ]='WkDfVxT{:d}D{:d}_[m/s]'.format(i+1, k+1) + for i in np.arange(nWT): + for k in np.arange(nD): + FFSpanMap['^WkDfVrT{:d}N(\d*)D{:d}_\[m/s\]'.format(i+1,k+1) ]='WkDfVrT{:d}D{:d}_[m/s]'.format(i+1, k+1) + + return fastlib.find_matching_columns(Cols, FFSpanMap) + +def diameterwiseColFastFarm(Cols, nWT=9): + """ Return column info, available columns and indices that contain AD spanwise data""" + FFDiamMap=dict() + for i in np.arange(nWT): + for x in ['X','Y','Z']: + FFDiamMap['^WkAxs{}T{:d}D(\d*)_\[-\]'.format(x,i+1)] ='WkAxs{}T{:d}_[-]'.format(x,i+1) + for i in np.arange(nWT): + for x in ['X','Y','Z']: + FFDiamMap['^WkPos{}T{:d}D(\d*)_\[m\]'.format(x,i+1)] ='WkPos{}T{:d}_[m]'.format(x,i+1) + for i in np.arange(nWT): + for x in ['X','Y','Z']: + FFDiamMap['^WkVel{}T{:d}D(\d*)_\[m/s\]'.format(x,i+1)] ='WkVel{}T{:d}_[m/s]'.format(x,i+1) + for i in np.arange(nWT): + for x in ['X','Y','Z']: + FFDiamMap['^WkDiam{}T{:d}D(\d*)_\[m\]'.format(x,i+1)] ='WkDiam{}T{:d}_[m]'.format(x,i+1) + return fastlib.find_matching_columns(Cols, FFDiamMap) + +def SensorsFARMRadial(nWT=3,nD=10,nR=30,signals=None): + """ Returns a list of FASTFarm sensors that are used for the radial distribution + of quantities (e.g. Ct, Wake Deficits). + If `signals` is provided, the output is the list of sensors within the list `signals`. + """ + WT = np.arange(nWT) + r = np.arange(nR) + D = np.arange(nD) + sens=[] + sens+=['CtT{:d}N{:02d}_[-]'.format(i+1,j+1) for i in WT for j in r] + sens+=['WkDfVxT{:d}N{:02d}D{:d}_[m/s]'.format(i+1,j+1,k+1) for i in WT for j in r for k in D] + sens+=['WkDfVrT{:d}N{:02d}D{:d}_[m/s]'.format(i+1,j+1,k+1) for i in WT for j in r for k in D] + if signals is not None: + sens = [c for c in sens if c in signals] + return sens + +def SensorsFARMDiam(nWT,nD): + """ Returns a list of FASTFarm sensors that contain quantities at different downstream diameters + (e.g. WkAxs, WkPos, WkVel, WkDiam) + If `signals` is provided, the output is the list of sensors within the list `signals`. + """ + WT = np.arange(nWT) + D = np.arange(nD) + XYZ = ['X','Y','Z'] + sens=[] + sens+=['WkAxs{}T{:d}D{:d}_[-]'.format(x,i+1,j+1) for x in XYZ for i in WT for j in D] + sens+=['WkPos{}T{:d}D{:d}_[m]'.format(x,i+1,j+1) for x in XYZ for i in WT for j in D] + sens+=['WkVel{}T{:d}D{:d}_[m/s]'.format(x,i+1,j+1) for x in XYZ for i in WT for j in D] + sens+=['WkDiam{}T{:d}D{:d}_[m]'.format(x,i+1,j+1) for x in XYZ for i in WT for j in D] + if signals is not None: + sens = [c for c in sens if c in signals] + return sens + + +def extractFFRadialData(fastfarm_out,fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1,df=None): + # LEGACY + return spanwisePostProFF(fastfarm_input,avgMethod=avgMethod,avgParam=avgParam,D=D,df=df,fastfarm_out=fastfarm_out) + + +def spanwisePostProFF(fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1,df=None,fastfarm_out=None): + """ + Opens a FASTFarm output file, extract the radial data, average them and returns spanwise data + + D: diameter TODO, extract it from the main file + + See faslibt.averageDF for `avgMethod` and `avgParam`. + """ + # --- Opening ouputfile + if df is None: + df=FASTOutputFile(fastfarm_out).toDataFrame() + + # --- Opening input file and extracting inportant variables + if fastfarm_input is None: + # We don't have an input file, guess numbers of turbine, diameters, Nodes... + cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'T(\d+)') + nWT = np.array(sIdx).astype(int).max() + cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'D(\d+)') + nD = np.array(sIdx).astype(int).max() + cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'N(\d+)') + nr = np.array(sIdx).astype(int).max() + vr=None + vD=None + D=0 + else: + main=FASTInputFile(fastfarm_input) + iOut = main['OutRadii'] + dr = main['dr'] # Radial increment of radial finite-difference grid (m) + OutDist = main['OutDist'] # List of downstream distances for wake output for an individual rotor + WT = main['WindTurbines'] + nWT = len(WT) + vr = dr*np.array(iOut) + vD = np.array(OutDist) + nr=len(iOut) + nD=len(vD) + + + # --- Extracting time series of radial data only + colRadial = SensorsFARMRadial(nWT=nWT,nD=nD,nR=nr,signals=df.columns.values) + colRadial=['Time_[s]']+colRadial + dfRadialTime = df[colRadial] # TODO try to do some magic with it, display it with a slider + + # --- Averaging data + dfAvg = fastlib.averageDF(df,avgMethod=avgMethod,avgParam=avgParam) + + # --- Extract radial data + ColsInfo, nrMax = spanwiseColFastFarm(df.columns.values, nWT=nWT, nD=nD) + dfRad = fastlib.extract_spanwise_data(ColsInfo, nrMax, df=None, ts=dfAvg.iloc[0]) + #dfRad = fastlib.insert_radial_columns(dfRad, vr) + if vr is None: + dfRad.insert(0, 'i_[#]', np.arange(nrMax)+1) + else: + dfRad.insert(0, 'r_[m]', vr[:nrMax]) + dfRad['i/n_[-]']=np.arange(nrMax)/nrMax + + # --- Extract downstream data + ColsInfo, nDMax = diameterwiseColFastFarm(df.columns.values, nWT=nWT) + dfDiam = fastlib.extract_spanwise_data(ColsInfo, nDMax, df=None, ts=dfAvg.iloc[0]) + #dfDiam = fastlib.insert_radial_columns(dfDiam) + if vD is None: + dfDiam.insert(0, 'i_[#]', np.arange(nDMax)+1) + else: + dfDiam.insert(0, 'x_[m]', vD[:nDMax]) + dfDiam['i/n_[-]'] = np.arange(nDMax)/nDMax + return dfRad, dfRadialTime, dfDiam + diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index b596633..8555d11 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -1,1183 +1,1449 @@ -# --- For cmd.py -from __future__ import division, print_function -import os -import pandas as pd -import numpy as np -import re - -# --- fast libraries -from weio.weio.fast_input_file import FASTInputFile -from weio.weio.fast_output_file import FASTOutputFile -from weio.weio.fast_input_deck import FASTInputDeck -# from pyFAST.input_output.fast_input_file import FASTInputFile -# from pyFAST.input_output.fast_output_file import FASTOutputFile -# from pyFAST.input_output.fast_input_deck import FASTInputDeck - -# --------------------------------------------------------------------------------} -# --- Tools for IO -# --------------------------------------------------------------------------------{ -def ED_BldStations(ED): - """ Returns ElastoDyn Blade Station positions, useful to know where the outputs are. - INPUTS: - - ED: either: - - a filename of a ElastoDyn input file - - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) - - OUTUPTS: - - bld_fract: fraction of the blade length were stations are defined - - r_nodes: spanwise position from the rotor apex of the Blade stations - """ - if hasattr(ED,'startswith'): # if string - ED = FASTInputFile(ED) - - nBldNodes = ED['BldNodes'] - bld_fract = np.arange(1./nBldNodes/2., 1, 1./nBldNodes) - r_nodes = bld_fract*(ED['TipRad']-ED['HubRad']) + ED['HubRad'] - return bld_fract, r_nodes - -def ED_TwrStations(ED): - """ Returns ElastoDyn Tower Station positions, useful to know where the outputs are. - INPUTS: - - ED: either: - - a filename of a ElastoDyn input file - - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) - - OUTPUTS: - - r_fract: fraction of the towet length were stations are defined - - h_nodes: height from the *ground* of the stations (not from the Tower base) - """ - if hasattr(ED,'startswith'): # if string - ED = FASTInputFile(ED) - - nTwrNodes = ED['TwrNodes'] - twr_fract = np.arange(1./nTwrNodes/2., 1, 1./nTwrNodes) - h_nodes = twr_fract*(ED['TowerHt']-ED['TowerBsHt']) + ED['TowerBsHt'] - return twr_fract, h_nodes - -def ED_BldGag(ED): - """ Returns the radial position of ElastoDyn blade gages - INPUTS: - - ED: either: - - a filename of a ElastoDyn input file - - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) - OUTPUTS: - - r_gag: The radial positions of the gages, given from the rotor apex - """ - if hasattr(ED,'startswith'): # if string - ED = FASTInputFile(ED) - _,r_nodes= ED_BldStations(ED) - - # if ED.hasNodal: - # return r_nodes, None - nOuts = ED['NBlGages'] - if nOuts<=0: - return np.array([]), np.array([]) - if type(ED['BldGagNd']) is list: - Inodes = np.asarray(ED['BldGagNd']) - else: - Inodes = np.array([ED['BldGagNd']]) - r_gag = r_nodes[ Inodes[:nOuts] -1] - return r_gag, Inodes - -def ED_TwrGag(ED): - """ Returns the heights of ElastoDyn blade gages - INPUTS: - - ED: either: - - a filename of a ElastoDyn input file - - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) - OUTPUTS: - - h_gag: The heights of the gages, given from the ground height (tower base + TowerBsHt) - """ - if hasattr(ED,'startswith'): # if string - ED = FASTInputFile(ED) - _,h_nodes= ED_TwrStations(ED) - nOuts = ED['NTwGages'] - if nOuts<=0: - return np.array([]) - if type(ED['TwrGagNd']) is list: - Inodes = np.asarray(ED['TwrGagNd']) - else: - Inodes = np.array([ED['TwrGagNd']]) - h_gag = h_nodes[ Inodes[:nOuts] -1] - return h_gag - - -def AD14_BldGag(AD): - """ Returns the radial position of AeroDyn 14 blade gages (based on "print" in column 6) - INPUTS: - - AD: either: - - a filename of a AeroDyn input file - - an instance of FileCl, as returned by reading the file, AD = weio.read(AD_filename) - OUTPUTS: - - r_gag: The radial positions of the gages, given from the blade root - """ - if hasattr(ED,'startswith'): # if string - AD = FASTInputFile(AD) - - Nodes=AD['BldAeroNodes'] - if Nodes.shape[1]==6: - doPrint= np.array([ n.lower().find('p')==0 for n in Nodes[:,5]]) - else: - doPrint=np.array([ True for n in Nodes[:,0]]) - - r_gag = Nodes[doPrint,0].astype(float) - IR = np.arange(1,len(Nodes)+1)[doPrint] - return r_gag, IR - -def AD_BldGag(AD,AD_bld,chordOut=False): - """ Returns the radial position of AeroDyn blade gages - INPUTS: - - AD: either: - - a filename of a AeroDyn input file - - an instance of FileCl, as returned by reading the file, AD = weio.read(AD_filename) - - AD_bld: either: - - a filename of a AeroDyn Blade input file - - an instance of FileCl, as returned by reading the file, AD_bld = weio.read(AD_bld_filename) - OUTPUTS: - - r_gag: The radial positions of the gages, given from the blade root - """ - if hasattr(AD,'startswith'): # if string - AD = FASTInputFile(AD) - if hasattr(AD_bld,'startswith'): # if string - AD_bld = FASTInputFile(AD_bld) - #print(AD_bld.keys()) - - nOuts=AD['NBlOuts'] - if nOuts<=0: - if chordOut: - return np.array([]), np.array([]) - else: - return np.array([]) - INodes = np.array(AD['BlOutNd'][:nOuts]) - r_gag = AD_bld['BldAeroNodes'][INodes-1,0] - if chordOut: - chord_gag = AD_bld['BldAeroNodes'][INodes-1,5] - return r_gag,chord_gag - else: - return r_gag - -def BD_BldGag(BD): - """ Returns the radial position of BeamDyn blade gages - INPUTS: - - BD: either: - - a filename of a BeamDyn input file - - an instance of FileCl, as returned by reading the file, BD = weio.read(BD_filename) - OUTPUTS: - - r_gag: The radial positions of the gages, given from the rotor apex - """ - if hasattr(BD,'startswith'): # if string - BD = FASTInputFile(BD) - - M = BD['MemberGeom'] - r_nodes = M[:,2] # NOTE: we select the z axis here, and we don't take curvilenear coord - nOuts = BD['NNodeOuts'] - if nOuts<=0: - nOuts=0 - if type(BD['OutNd']) is list: - Inodes = np.asarray(BD['OutNd']) - else: - Inodes = np.array([BD['OutNd']]) - r_gag = r_nodes[ Inodes[:nOuts] -1] - return r_gag, Inodes, r_nodes - -# -# -# 1, 7, 14, 21, 30, 36, 43, 52, 58 BldGagNd List of blade nodes that have strain gages [1 to BldNodes] (-) [unused if NBlGages=0] - -# --------------------------------------------------------------------------------} -# --- Helper functions for radial data -# --------------------------------------------------------------------------------{ -def _HarmonizeSpanwiseData(Name, Columns, vr, R, IR=None) : - """ helper function to use with spanwiseAD and spanwiseED """ - # --- Data present - data = [c for _,c in Columns if c is not None] - ColNames = [n for n,_ in Columns if n is not None] - Lengths = [len(d) for d in data] - if len(data)<=0: - print('[WARN] No spanwise data for '+Name) - return None, None, None - - # --- Harmonize data so that they all have the same length - nrMax = np.max(Lengths) - ids=np.arange(nrMax) - if vr is None: - bFakeVr=True - vr_bar = ids/(nrMax-1) - else: - vr_bar=vr/R - bFakeVr=False - if (nrMax)len(vr_bar): - raise Exception('Inconsitent length between radial stations and max index present in output chanels') - - for i in np.arange(len(data)): - d=data[i] - if len(d)len(vr_bar): - print(vr_bar) - raise Exception('Inconsitent length between radial stations ({:d}) and max index present in output chanels ({:d})'.format(len(vr_bar),nrMax)) - df.insert(0, 'r/R_[-]', vr_bar) - - if IR is not None: - df['Node_[#]']=IR[:nrMax] - df['i_[#]']=ids+1 - if vr is not None: - df['r_[m]'] = vr[:nrMax] - return df - -def find_matching_columns(Cols, PatternMap): - ColsInfo=[] - nrMax=0 - for colpattern,colmap in PatternMap.items(): - # Extracting columns matching pattern - cols, sIdx = find_matching_pattern(Cols, colpattern) - if len(cols)>0: - # Sorting by ID - cols = np.asarray(cols) - Idx = np.array([int(s) for s in sIdx]) - Isort = np.argsort(Idx) - Idx = Idx[Isort] - cols = cols[Isort] - col={'name':colmap,'Idx':Idx,'cols':cols} - nrMax=max(nrMax,np.max(Idx)) - ColsInfo.append(col) - return ColsInfo,nrMax - -def extract_spanwise_data(ColsInfo, nrMax, df=None,ts=None): - """ - Extract spanwise data based on some column info - ColsInfo: see find_matching_columns - """ - nCols = len(ColsInfo) - if nCols==0: - return None - if ts is not None: - Values = np.zeros((nrMax,nCols)) - Values[:] = np.nan - elif df is not None: - raise NotImplementedError() - - ColNames =[c['name'] for c in ColsInfo] - - for ic,c in enumerate(ColsInfo): - Idx, cols, colname = c['Idx'], c['cols'], c['name'] - for idx,col in zip(Idx,cols): - Values[idx-1,ic]=ts[col] - nMissing = np.sum(np.isnan(Values[:,ic])) - if len(cols)nrMax: - print('[WARN] More values found for {}, found {}/{}'.format(colname,len(cols),nrMax)) - df = pd.DataFrame(data=Values, columns=ColNames) - df = df.reindex(sorted(df.columns), axis=1) - return df - -def spanwiseColBD(Cols): - """ Return column info, available columns and indices that contain BD spanwise data""" - BDSpanMap=dict() - for sB in ['B1','B2','B3']: - BDSpanMap['^'+sB+r'N(\d)TDxr_\[m\]']=sB+'TDxr_[m]' - BDSpanMap['^'+sB+r'N(\d)TDyr_\[m\]']=sB+'TDyr_[m]' - BDSpanMap['^'+sB+r'N(\d)TDzr_\[m\]']=sB+'TDzr_[m]' - return find_matching_columns(Cols, BDSpanMap) - -def spanwiseColED(Cols): - """ Return column info, available columns and indices that contain ED spanwise data""" - EDSpanMap=dict() - # All Outs - for sB in ['B1','B2','B3']: - EDSpanMap['^[A]*'+sB+r'N(\d*)ALx_\[m/s^2\]' ] = sB+'ALx_[m/s^2]' - EDSpanMap['^[A]*'+sB+r'N(\d*)ALy_\[m/s^2\]' ] = sB+'ALy_[m/s^2]' - EDSpanMap['^[A]*'+sB+r'N(\d*)ALz_\[m/s^2\]' ] = sB+'ALz_[m/s^2]' - EDSpanMap['^[A]*'+sB+r'N(\d*)TDx_\[m\]' ] = sB+'TDx_[m]' - EDSpanMap['^[A]*'+sB+r'N(\d*)TDy_\[m\]' ] = sB+'TDy_[m]' - EDSpanMap['^[A]*'+sB+r'N(\d*)TDz_\[m\]' ] = sB+'TDz_[m]' - EDSpanMap['^[A]*'+sB+r'N(\d*)RDx_\[deg\]' ] = sB+'RDx_[deg]' - EDSpanMap['^[A]*'+sB+r'N(\d*)RDy_\[deg\]' ] = sB+'RDy_[deg]' - EDSpanMap['^[A]*'+sB+r'N(\d*)RDz_\[deg\]' ] = sB+'RDz_[deg]' - EDSpanMap['^[A]*'+sB+r'N(\d*)MLx_\[kN-m\]' ] = sB+'MLx_[kN-m]' - EDSpanMap['^[A]*'+sB+r'N(\d*)MLy_\[kN-m\]' ] = sB+'MLy_[kN-m]' - EDSpanMap['^[A]*'+sB+r'N(\d*)MLz_\[kN-m\]' ] = sB+'MLz_[kN-m]' - EDSpanMap['^[A]*'+sB+r'N(\d*)FLx_\[kN\]' ] = sB+'FLx_[kN]' - EDSpanMap['^[A]*'+sB+r'N(\d*)FLy_\[kN\]' ] = sB+'FLy_[kN]' - EDSpanMap['^[A]*'+sB+r'N(\d*)FLz_\[kN\]' ] = sB+'FLz_[kN]' - EDSpanMap['^[A]*'+sB+r'N(\d*)FLxNT_\[kN\]' ] = sB+'FLxNT_[kN]' - EDSpanMap['^[A]*'+sB+r'N(\d*)FLyNT_\[kN\]' ] = sB+'FLyNT_[kN]' - EDSpanMap['^[A]*'+sB+r'N(\d*)FlyNT_\[kN\]' ] = sB+'FLyNT_[kN]' # <<< Unfortunate - EDSpanMap['^[A]*'+sB+r'N(\d*)MLxNT_\[kN-m\]'] = sB+'MLxNT_[kN-m]' - EDSpanMap['^[A]*'+sB+r'N(\d*)MLyNT_\[kN-m\]'] = sB+'MLyNT_[kN-m]' - # Old - for sB in ['b1','b2','b3']: - SB=sB.upper() - EDSpanMap[r'^Spn(\d)ALx'+sB+r'_\[m/s^2\]']=SB+'ALx_[m/s^2]' - EDSpanMap[r'^Spn(\d)ALy'+sB+r'_\[m/s^2\]']=SB+'ALy_[m/s^2]' - EDSpanMap[r'^Spn(\d)ALz'+sB+r'_\[m/s^2\]']=SB+'ALz_[m/s^2]' - EDSpanMap[r'^Spn(\d)TDx'+sB+r'_\[m\]' ]=SB+'TDx_[m]' - EDSpanMap[r'^Spn(\d)TDy'+sB+r'_\[m\]' ]=SB+'TDy_[m]' - EDSpanMap[r'^Spn(\d)TDz'+sB+r'_\[m\]' ]=SB+'TDz_[m]' - EDSpanMap[r'^Spn(\d)RDx'+sB+r'_\[deg\]' ]=SB+'RDx_[deg]' - EDSpanMap[r'^Spn(\d)RDy'+sB+r'_\[deg\]' ]=SB+'RDy_[deg]' - EDSpanMap[r'^Spn(\d)RDz'+sB+r'_\[deg\]' ]=SB+'RDz_[deg]' - EDSpanMap[r'^Spn(\d)FLx'+sB+r'_\[kN\]' ]=SB+'FLx_[kN]' - EDSpanMap[r'^Spn(\d)FLy'+sB+r'_\[kN\]' ]=SB+'FLy_[kN]' - EDSpanMap[r'^Spn(\d)FLz'+sB+r'_\[kN\]' ]=SB+'FLz_[kN]' - EDSpanMap[r'^Spn(\d)MLy'+sB+r'_\[kN-m\]' ]=SB+'MLx_[kN-m]' - EDSpanMap[r'^Spn(\d)MLx'+sB+r'_\[kN-m\]' ]=SB+'MLy_[kN-m]' - EDSpanMap[r'^Spn(\d)MLz'+sB+r'_\[kN-m\]' ]=SB+'MLz_[kN-m]' - return find_matching_columns(Cols, EDSpanMap) - -def spanwiseColAD(Cols): - """ Return column info, available columns and indices that contain AD spanwise data""" - ADSpanMap=dict() - for sB in ['B1','B2','B3']: - ADSpanMap['^[A]*'+sB+r'N(\d*)Alpha_\[deg\]']=sB+'Alpha_[deg]' - ADSpanMap['^[A]*'+sB+r'N(\d*)AOA_\[deg\]' ]=sB+'Alpha_[deg]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)AxInd_\[-\]' ]=sB+'AxInd_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)TnInd_\[-\]' ]=sB+'TnInd_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[deg\]' ]=sB+'AxInd_[-]' # DBGOuts NOTE BUG Unit - ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[deg\]' ]=sB+'TnInd_[-]' # DBGOuts NOTE BUG Unit - ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[-\]' ]=sB+'AxInd_[-]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[-\]' ]=sB+'TnInd_[-]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Uin_\[m/s\]' ]=sB+'Uin_[m/s]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Uit_\[m/s\]' ]=sB+'Uit_[m/s]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Uir_\[m/s\]' ]=sB+'Uir_[m/s]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Cl_\[-\]' ]=sB+'Cl_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cd_\[-\]' ]=sB+'Cd_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cm_\[-\]' ]=sB+'Cm_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cx_\[-\]' ]=sB+'Cx_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cy_\[-\]' ]=sB+'Cy_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cn_\[-\]' ]=sB+'Cn_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Ct_\[-\]' ]=sB+'Ct_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Re_\[-\]' ]=sB+'Re_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vrel_\[m/s\]' ]=sB+'Vrel_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Theta_\[deg\]']=sB+'Theta_[deg]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Phi_\[deg\]' ]=sB+'Phi_[deg]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Twst_\[deg\]' ]=sB+'Twst_[deg]' #DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Curve_\[deg\]']=sB+'Curve_[deg]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vindx_\[m/s\]']=sB+'Vindx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vindy_\[m/s\]']=sB+'Vindy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fx_\[N/m\]' ]=sB+'Fx_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fy_\[N/m\]' ]=sB+'Fy_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fl_\[N/m\]' ]=sB+'Fl_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fd_\[N/m\]' ]=sB+'Fd_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fn_\[N/m\]' ]=sB+'Fn_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Ft_\[N/m\]' ]=sB+'Ft_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VUndx_\[m/s\]']=sB+'VUndx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VUndy_\[m/s\]']=sB+'VUndy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VUndz_\[m/s\]']=sB+'VUndz_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VDisx_\[m/s\]']=sB+'VDisx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VDisy_\[m/s\]']=sB+'VDisy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VDisz_\[m/s\]']=sB+'VDisz_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)STVx_\[m/s\]' ]=sB+'STVx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)STVy_\[m/s\]' ]=sB+'STVy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)STVz_\[m/s\]' ]=sB+'STVz_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vx_\[m/s\]' ]=sB+'Vx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vy_\[m/s\]' ]=sB+'Vy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vz_\[m/s\]' ]=sB+'Vz_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)DynP_\[Pa\]' ]=sB+'DynP_[Pa]' - ADSpanMap['^[A]*'+sB+r'N(\d*)M_\[-\]' ]=sB+'M_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Mm_\[N-m/m\]' ]=sB+'Mm_[N-m/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Gam_\[' ]=sB+'Gam_[m^2/s]' #DBGOuts - # --- AD 14 - ADSpanMap[r'^Alpha(\d*)_\[deg\]' ]='Alpha_[deg]' - ADSpanMap[r'^DynPres(\d*)_\[Pa\]' ]='DynPres_[Pa]' - ADSpanMap[r'^CLift(\d*)_\[-\]' ]='CLift_[-]' - ADSpanMap[r'^CDrag(\d*)_\[-\]' ]='CDrag_[-]' - ADSpanMap[r'^CNorm(\d*)_\[-\]' ]='CNorm_[-]' - ADSpanMap[r'^CTang(\d*)_\[-\]' ]='CTang_[-]' - ADSpanMap[r'^CMomt(\d*)_\[-\]' ]='CMomt_[-]' - ADSpanMap[r'^Pitch(\d*)_\[deg\]' ]='Pitch_[deg]' - ADSpanMap[r'^AxInd(\d*)_\[-\]' ]='AxInd_[-]' - ADSpanMap[r'^TanInd(\d*)_\[-\]' ]='TanInd_[-]' - ADSpanMap[r'^ForcN(\d*)_\[N\]' ]='ForcN_[N]' - ADSpanMap[r'^ForcT(\d*)_\[N\]' ]='ForcT_[N]' - ADSpanMap[r'^Pmomt(\d*)_\[N-m\]' ]='Pmomt_[N-N]' - ADSpanMap[r'^ReNum(\d*)_\[x10^6\]']='ReNum_[x10^6]' - ADSpanMap[r'^Gamma(\d*)_\[m^2/s\]']='Gamma_[m^2/s]' - - return find_matching_columns(Cols, ADSpanMap) - -def insert_extra_columns_AD(dfRad, tsAvg, vr=None, rho=None, R=None, nB=None, chord=None): - # --- Compute additional values (AD15 only) - if dfRad is None: - return None - if dfRad.shape[1]==0: - return dfRad - if chord is not None: - if vr is not None: - chord =chord[0:len(dfRad)] - for sB in ['B1','B2','B3']: - try: - vr_bar=vr/R - Fx = dfRad[sB+'Fx_[N/m]'] - U0 = tsAvg['Wind1VelX_[m/s]'] - Ct=nB*Fx/(0.5 * rho * 2 * U0**2 * np.pi * vr) - Ct[vr<0.01*R] = 0 - dfRad[sB+'Ctloc_[-]'] = Ct - CT=2*np.trapz(vr_bar*Ct,vr_bar) - dfRad[sB+'CtAvg_[-]']= CT*np.ones(vr.shape) - except: - pass - try: - dfRad[sB+'Gamma_[m^2/s]'] = 1/2 * chord* dfRad[sB+'Vrel_[m/s]'] * dfRad[sB+'Cl_[-]'] - except: - pass - try: - if not sB+'Vindx_[m/s]' in dfRad.columns: - dfRad[sB+'Vindx_[m/s]']= -dfRad[sB+'AxInd_[-]'].values * dfRad[sB+'Vx_[m/s]'].values - dfRad[sB+'Vindy_[m/s]']= dfRad[sB+'TnInd_[-]'].values * dfRad[sB+'Vy_[m/s]'].values - except: - pass - return dfRad - - - -def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='.outb',df=None): - """ - Postprocess FAST radial data - - INPUTS: - - FST_IN: Fast .fst input file - - avgMethod='periods', avgParam=2: average over 2 last periods, Needs Azimuth sensors!!! - - avgMethod='constantwindow', avgParam=5: average over 5s of simulation - - postprofile: outputfile to write radial data - """ - # --- Opens Fast output and performs averaging - if df is None: - df = FASTOutputFile(FST_In.replace('.fst',out_ext).replace('.dvr',out_ext)).toDataFrame() - returnDF=True - else: - returnDF=False - # NOTE: spanwise script doest not support duplicate columns - df = df.loc[:,~df.columns.duplicated()] - dfAvg = averageDF(df,avgMethod=avgMethod ,avgParam=avgParam) # NOTE: average 5 last seconds - - # --- Extract info (e.g. radial positions) from Fast input file - # We don't have a .fst input file, so we'll rely on some default values for "r" - rho = 1.225 - chord = None - # --- Extract radial positions of output channels - r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst = FASTRadialOutputs(FST_In, OutputCols=df.columns.values) - if R is None: - R=1 - try: - chord = fst.AD.Bld1['BldAeroNodes'][:,5] # Full span - except: - pass - try: - rho = fst.AD['Rho'] - except: - try: - rho = fst.AD['AirDens'] - except: - pass - #print('r_AD:', r_AD) - #print('r_ED:', r_ED) - #print('r_BD:', r_BD) - #print('I_AD:', IR_AD) - #print('I_ED:', IR_ED) - #print('I_BD:', IR_BD) - # --- Extract radial data and export to csv if needed - dfRad_AD = None - dfRad_ED = None - dfRad_BD = None - Cols=dfAvg.columns.values - # --- AD - ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) - dfRad_AD = extract_spanwise_data(ColsInfoAD, nrMaxAD, df=None, ts=dfAvg.iloc[0]) - dfRad_AD = insert_extra_columns_AD(dfRad_AD, dfAvg.iloc[0], vr=r_AD, rho=rho, R=R, nB=3, chord=chord) - dfRad_AD = insert_radial_columns(dfRad_AD, r_AD, R=R, IR=IR_AD) - # --- ED - ColsInfoED, nrMaxED = spanwiseColED(Cols) - dfRad_ED = extract_spanwise_data(ColsInfoED, nrMaxED, df=None, ts=dfAvg.iloc[0]) - dfRad_ED = insert_radial_columns(dfRad_ED, r_ED, R=R, IR=IR_ED) - # --- BD - ColsInfoBD, nrMaxBD = spanwiseColBD(Cols) - dfRad_BD = extract_spanwise_data(ColsInfoBD, nrMaxBD, df=None, ts=dfAvg.iloc[0]) - dfRad_BD = insert_radial_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) - if returnDF: - return dfRad_ED , dfRad_AD, dfRad_BD, df - else: - return dfRad_ED , dfRad_AD, dfRad_BD - - - -def spanwisePostProRows(df, FST_In=None): - """ - Returns a 3D matrix: n x nSpan x nColumn where df is of size n x nColumn - - NOTE: this is really not optimal. Spanwise columns should be extracted only once.. - """ - # --- Extract info (e.g. radial positions) from Fast input file - # We don't have a .fst input file, so we'll rely on some default values for "r" - rho = 1.225 - chord = None - # --- Extract radial positions of output channels - r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst = FASTRadialOutputs(FST_In, OutputCols=df.columns.values) - #print('r_AD:', r_AD) - #print('r_ED:', r_ED) - #print('r_BD:', r_BD) - if R is None: - R=1 - try: - chord = fst.AD.Bld1['BldAeroNodes'][:,5] # Full span - except: - pass - try: - rho = fst.AD['Rho'] - except: - try: - rho = fst.AD['AirDens'] - except: - pass - # --- Extract radial data for each azimuthal average - M_AD=None - M_ED=None - M_BD=None - Col_AD=None - Col_ED=None - Col_BD=None - v = df.index.values - - # --- Getting Column info - Cols=df.columns.values - if r_AD is not None: - ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) - if r_ED is not None: - ColsInfoED, nrMaxED = spanwiseColED(Cols) - if r_BD is not None: - ColsInfoBD, nrMaxBD = spanwiseColBD(Cols) - for i,val in enumerate(v): - if r_AD is not None: - dfRad_AD = extract_spanwise_data(ColsInfoAD, nrMaxAD, df=None, ts=df.iloc[i]) - dfRad_AD = insert_extra_columns_AD(dfRad_AD, df.iloc[i], vr=r_AD, rho=rho, R=R, nB=3, chord=chord) - dfRad_AD = insert_radial_columns(dfRad_AD, r_AD, R=R, IR=IR_AD) - if i==0: - M_AD = np.zeros((len(v), len(dfRad_AD), len(dfRad_AD.columns))) - Col_AD=dfRad_AD.columns.values - M_AD[i, :, : ] = dfRad_AD.values - if r_ED is not None and len(r_ED)>0: - dfRad_ED = extract_spanwise_data(ColsInfoED, nrMaxED, df=None, ts=df.iloc[i]) - dfRad_ED = insert_radial_columns(dfRad_ED, r_ED, R=R, IR=IR_ED) - if i==0: - M_ED = np.zeros((len(v), len(dfRad_ED), len(dfRad_ED.columns))) - Col_ED=dfRad_ED.columns.values - M_ED[i, :, : ] = dfRad_ED.values - if r_BD is not None and len(r_BD)>0: - dfRad_BD = extract_spanwise_data(ColsInfoBD, nrMaxBD, df=None, ts=df.iloc[i]) - dfRad_BD = insert_radial_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) - if i==0: - M_BD = np.zeros((len(v), len(dfRad_BD), len(dfRad_BD.columns))) - Col_BD=dfRad_BD.columns.values - M_BD[i, :, : ] = dfRad_BD.values - return M_AD, Col_AD, M_ED, Col_ED, M_BD, Col_BD - - -def FASTRadialOutputs(FST_In, OutputCols=None): - """ Returns radial positions where FAST has outputs - INPUTS: - FST_In: fast input file (.fst) - OUTPUTS: - r_AD: radial positions of FAST Outputs from the rotor center - """ - R = None - r_hub =0 - r_AD = None - r_ED = None - r_BD = None - IR_ED = None - IR_AD = None - IR_BD = None - fst=None - if FST_In is not None: - fst = FASTInputDeck(FST_In, readlist=['AD','ADbld','ED','BD']) - # NOTE: all this below should be in FASTInputDeck - if fst.version == 'F7': - # --- FAST7 - if not hasattr(fst,'AD'): - raise Exception('The AeroDyn file couldn''t be found or read, from main file: '+FST_In) - r_AD,IR_AD = AD14_BldGag(fst.AD) - R = fst.fst['TipRad'] - try: - rho = fst.AD['Rho'] - except: - rho = fst.AD['AirDens'] - else: - # --- OpenFAST 2 - R = None - - # --- ElastoDyn - if 'NumTurbines' in fst.fst.keys(): - # AeroDyn driver... - if 'HubRad(1)' in fst.fst.keys(): - r_hub = fst.fst['HubRad(1)'] - else: - r_hub = fst.fst['BldHubRad_bl(1_1)'] - - elif not hasattr(fst,'ED'): - print('[WARN] The Elastodyn file couldn''t be found or read, from main file: '+FST_In) - #raise Exception('The Elastodyn file couldn''t be found or read, from main file: '+FST_In) - else: - R = fst.ED['TipRad'] - r_hub = fst.ED['HubRad'] - if fst.ED.hasNodal: - _, r_ED = ED_BldStations(fst.ED) - IR_ED =None - else: - r_ED, IR_ED = ED_BldGag(fst.ED) - - # --- BeamDyn - if fst.BD is not None: - r_BD, IR_BD, r_BD_All = BD_BldGag(fst.BD) - r_BD= r_BD+r_hub - if R is None: - R = r_BD_All[-1] # just in case ED file missing - - # --- AeroDyn - if fst.AD is None: - print('[WARN] The AeroDyn file couldn''t be found or read, from main file: '+FST_In) - #raise Exception('The AeroDyn file couldn''t be found or read, from main file: '+FST_In) - else: - if fst.ADversion == 'AD15': - if fst.AD.Bld1 is None: - raise Exception('The AeroDyn blade file couldn''t be found or read, from main file: '+FST_In) - - if 'B1N001Cl_[-]' in OutputCols or np.any(np.char.find(list(OutputCols),'AB1N')==0): - # This was compiled with all outs - r_AD = fst.AD.Bld1['BldAeroNodes'][:,0] # Full span - r_AD += r_hub - IR_AD = None - else: - r_AD,_ = AD_BldGag(fst.AD,fst.AD.Bld1, chordOut = True) # Only at Gages locations - r_AD += r_hub - - if R is None: - # ElastoDyn was not read, we use R from AD - R = fst.AD.Bld1['BldAeroNodes'][-1,0] - - elif fst.ADversion == 'AD14': - r_AD,IR_AD = AD14_BldGag(fst.AD) - - else: - raise Exception('AeroDyn version unknown') - return r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst - - - -def addToOutlist(OutList, Signals): - if not isinstance(Signals,list): - raise Exception('Signals must be a list') - for s in Signals: - ss=s.split()[0].strip().strip('"').strip('\'') - AlreadyIn = any([o.find(ss)==1 for o in OutList ]) - if not AlreadyIn: - OutList.append(s) - return OutList - - - -# --------------------------------------------------------------------------------} -# --- Generic df -# --------------------------------------------------------------------------------{ -def remap_df(df, ColMap, bColKeepNewOnly=False, inPlace=False): - """ Add/rename columns of a dataframe, potentially perform operations between columns - - Example: - - ColumnMap={ - 'WS_[m/s]' : '{Wind1VelX_[m/s]}' , # create a new column from existing one - 'RtTSR_[-]' : '{RtTSR_[-]} * 2 + {RtAeroCt_[-]}' , # change value of column - 'RotSpeed_[rad/s]' : '{RotSpeed_[rpm]} * 2*np.pi/60 ', # new column [rpm] -> [rad/s] - } - # Read - df = weio.read('FASTOutBin.outb').toDataFrame() - # Change columns based on formulae, potentially adding new columns - df = fastlib.remap_df(df, ColumnMap, inplace=True) - - """ - if not inPlace: - df=df.copy() - ColMapMiss=[] - ColNew=[] - RenameMap=dict() - for k0,v in ColMap.items(): - k=k0.strip() - v=v.strip() - if v.find('{')>=0: - search_results = re.finditer(r'\{.*?\}', v) - expr=v - # For more advanced operations, we use an eval - bFail=False - for item in search_results: - col=item.group(0)[1:-1] - if col not in df.columns: - ColMapMiss.append(col) - bFail=True - expr=expr.replace(item.group(0),'df[\''+col+'\']') - #print(k0, '=', expr) - if not bFail: - df[k]=eval(expr) - ColNew.append(k) - else: - print('[WARN] Column not present in dataframe, cannot evaluate: ',expr) - else: - #print(k0,'=',v) - if v not in df.columns: - ColMapMiss.append(v) - print('[WARN] Column not present in dataframe: ',v) - else: - RenameMap[k]=v - - # Applying renaming only now so that expressions may be applied in any order - for k,v in RenameMap.items(): - k=k.strip() - iCol = list(df.columns).index(v) - df.columns.values[iCol]=k - ColNew.append(k) - df.columns = df.columns.values # Hack to ensure columns are updated - - if len(ColMapMiss)>0: - print('[FAIL] The following columns were not found in the dataframe:',ColMapMiss) - #print('Available columns are:',df.columns.values) - - if bColKeepNewOnly: - ColNew = [c for c,_ in ColMap.items() if c in ColNew]# Making sure we respec order from user - ColKeepSafe = [c for c in ColNew if c in df.columns.values] - ColKeepMiss = [c for c in ColNew if c not in df.columns.values] - if len(ColKeepMiss)>0: - print('[WARN] Signals missing and omitted for ColKeep:\n '+'\n '.join(ColKeepMiss)) - df=df[ColKeepSafe] - return df - - -# --------------------------------------------------------------------------------} -# --- Tools for PostProcessing one or several simulations -# --------------------------------------------------------------------------------{ -def _zero_crossings(y,x=None,direction=None): - """ - Find zero-crossing points in a discrete vector, using linear interpolation. - direction: 'up' or 'down', to select only up-crossings or down-crossings - Returns: - x values xzc such that y(yzc)==0 - indexes izc, such that the zero is between y[izc] (excluded) and y[izc+1] (included) - if direction is not provided, also returns: - sign, equal to 1 for up crossing - """ - y=np.asarray(y) - if x is None: - x=np.arange(len(y)) - - if np.any((x[1:] - x[0:-1]) <= 0.0): - raise Exception('x values need to be in ascending order') - - # Indices before zero-crossing - iBef = np.where(y[1:]*y[0:-1] < 0.0)[0] - - # Find the zero crossing by linear interpolation - xzc = x[iBef] - y[iBef] * (x[iBef+1] - x[iBef]) / (y[iBef+1] - y[iBef]) - - # Selecting points that are exactly 0 and where neighbor change sign - iZero = np.where(y == 0.0)[0] - iZero = iZero[np.where((iZero > 0) & (iZero < x.size-1))] - iZero = iZero[np.where(y[iZero-1]*y[iZero+1] < 0.0)] - - # Concatenate - xzc = np.concatenate((xzc, x[iZero])) - iBef = np.concatenate((iBef, iZero)) - - # Sort - iSort = np.argsort(xzc) - xzc, iBef = xzc[iSort], iBef[iSort] - - # Return up-crossing, down crossing or both - sign = np.sign(y[iBef+1]-y[iBef]) - if direction == 'up': - I= np.where(sign==1)[0] - return xzc[I],iBef[I] - elif direction == 'down': - I= np.where(sign==-1)[0] - return xzc[I],iBef[I] - elif direction is not None: - raise Exception('Direction should be either `up` or `down`') - return xzc, iBef, sign - -def find_matching_pattern(List, pattern): - """ Return elements of a list of strings that match a pattern - and return the first matching group - """ - reg_pattern=re.compile(pattern) - MatchedElements=[] - MatchedStrings=[] - for l in List: - match=reg_pattern.search(l) - if match: - MatchedElements.append(l) - if len(match.groups(1))>0: - MatchedStrings.append(match.groups(1)[0]) - else: - MatchedStrings.append('') - return MatchedElements, MatchedStrings - - - -def extractSpanTSReg(ts, col_pattern, colname, IR=None): - """ Helper function to extract spanwise results, like B1N1Cl B1N2Cl etc. - - Example - col_pattern: 'B1N(\d*)Cl_\[-\]' - colname : 'B1Cl_[-]' - """ - # Extracting columns matching pattern - cols, sIdx = find_matching_pattern(ts.keys(), col_pattern) - if len(cols) ==0: - return (None,None) - - # Sorting by ID - cols = np.asarray(cols) - Idx = np.array([int(s) for s in sIdx]) - Isort = np.argsort(Idx) - Idx = Idx[Isort] - cols = cols[Isort] - - nrMax = np.max(Idx) - Values = np.zeros((nrMax,1)) - Values[:] = np.nan -# if IR is None: -# cols = [col_pattern.format(ir+1) for ir in range(nr)] -# else: -# cols = [col_pattern.format(ir) for ir in IR] - for idx,col in zip(Idx,cols): - Values[idx-1]=ts[col] - nMissing = np.sum(np.isnan(Values)) - if nMissing==nrMax: - return (None,None) - if len(cols)nrMax: - print('[WARN] More values found for {}, found {}/{}'.format(colname,len(cols),nrMax)) - return (colname,Values) - -def extractSpanTS(ts, nr, col_pattern, colname, IR=None): - """ Helper function to extract spanwise results, like B1N1Cl B1N2Cl etc. - - Example - col_pattern: 'B1N{:d}Cl_[-]' - colname : 'B1Cl_[-]' - """ - Values=np.zeros((nr,1)) - if IR is None: - cols = [col_pattern.format(ir+1) for ir in range(nr)] - else: - cols = [col_pattern.format(ir) for ir in IR] - colsExist = [c for c in cols if c in ts.keys() ] - if len(colsExist)==0: - return (None,None) - - Values = [ts[c] if c in ts.keys() else np.nan for c in cols ] - nMissing = np.sum(np.isnan(Values)) - #Values = ts[cols].T - #nCoun=len(Values) - if nMissing==nr: - return (None,None) - if len(colsExist)nr: - print('[WARN] More values found for {}, found {}/{}'.format(colname,len(cols),nr)) - return (colname,Values) - -def radialInterpTS(df, r, varName, r_ref, blade=1, bldFmt='AB{:d}', ndFmt='N{:03d}', method='interp'): - """ - Interpolate a time series at a given radial position for a given variable (varName) - INPUTS: - - df : a dataframe (typically with OpenFAST time series) - - r : radial positions of node where data is to be interpolated - - varName: variable name (and unit) to be interpolated. - The dataframe column will be assumed to be "BldFmt"+"ndFmt"+varName - - r_ref : radial position of nodal data present in the dataframe - - bldFmt : format for blade number, e.g. 'B{:d}' or 'AB{:d}' - - ndFmt : format for node number, e.g. 'N{:d}' or 'N{:03d}' - OUTPUT: - - interpolated time series - """ - # --- Sanity checks - r_ref = np.asarray(r_ref) - if not np.all(r_ref[:-1] <= r_ref[1:]): - raise Exception('This function only works for ascending radial values') - - # No extrapolation - if rnp.max(r_ref): - raise Exception('Extrapolation not supported') - - # Exactly on first or last nodes - if r==r_ref[0]: - col=bldFmt.format(blade) + ndFmt.format(1) + varName - if col in df.columns.values: - return df[col] - else: - raise Exception('Column {} not found in dataframe'.format(col)) - elif r==r_ref[-1]: - col=bldFmt.format(blade) + ndFmt.format(len(r_ref)+1) + varName - if col in df.columns.values: - return df[col] - else: - raise Exception('Column {} not found in dataframe'.format(col)) - - if method=='interp': - # Interpolation - iBef = np.where(r_reftStart].copy() - - dfPsi= bin_mean_DF(df, psiBin, colPsi) - if np.any(dfPsi['Counts']<1): - print('[WARN] some bins have no data! Increase the bin size.') - - return dfPsi - - -def averageDF(df,avgMethod='periods',avgParam=None,ColMap=None,ColKeep=None,ColSort=None,stats=['mean']): - """ - See average PostPro for documentation, same interface, just does it for one dataframe - """ - def renameCol(x): - for k,v in ColMap.items(): - if x==v: - return k - return x - # Before doing the colomn map we store the time - time = df['Time_[s]'].values - timenoNA = time[~np.isnan(time)] - # Column mapping - if ColMap is not None: - ColMapMiss = [v for _,v in ColMap.items() if v not in df.columns.values] - if len(ColMapMiss)>0: - print('[WARN] Signals missing and omitted for ColMap:\n '+'\n '.join(ColMapMiss)) - df.rename(columns=renameCol,inplace=True) - ## Defining a window for stats (start time and end time) - if avgMethod.lower()=='constantwindow': - tEnd = timenoNA[-1] - if avgParam is None: - tStart=timenoNA[0] - else: - tStart =tEnd-avgParam - elif avgMethod.lower()=='periods': - # --- Using azimuth to find periods - if 'Azimuth_[deg]' not in df.columns: - raise Exception('The sensor `Azimuth_[deg]` does not appear to be in the output file. You cannot use the averaging method by `periods`, use `constantwindow` instead.') - # NOTE: potentially we could average over each period and then average - psi=df['Azimuth_[deg]'].values - _,iBef = _zero_crossings(psi-psi[-10],direction='up') - if len(iBef)==0: - _,iBef = _zero_crossings(psi-180,direction='up') - if len(iBef)==0: - print('[WARN] Not able to find a zero crossing!') - tEnd = time[-1] - iBef=[0] - else: - tEnd = time[iBef[-1]] - - if avgParam is None: - tStart=time[iBef[0]] - else: - avgParam=int(avgParam) - if len(iBef)-10: - print('[WARN] Signals missing and omitted for ColKeep:\n '+'\n '.join(ColKeepMiss)) - df=df[ColKeepSafe] - if tStart=tStart) & (time<=tEnd) & (~np.isnan(time)))[0] - iEnd = IWindow[-1] - iStart = IWindow[0] - ## Absolute and relative differences at window extremities - DeltaValuesAbs=(df.iloc[iEnd]-df.iloc[iStart]).abs() -# DeltaValuesRel=(df.iloc[iEnd]-df.iloc[iStart]).abs()/df.iloc[iEnd] - DeltaValuesRel=(df.iloc[IWindow].max()-df.iloc[IWindow].min())/df.iloc[IWindow].mean() - #EndValues=df.iloc[iEnd] - #if avgMethod.lower()=='periods_omega': - # if DeltaValuesRel['RotSpeed_[rpm]']*100>5: - # print('[WARN] Rotational speed vary more than 5% in averaging window ({}%) for simulation: {}'.format(DeltaValuesRel['RotSpeed_[rpm]']*100,f)) - ## Stats values during window - # MeanValues = df[IWindow].mean() - # StdValues = df[IWindow].std() - if 'mean' in stats: - MeanValues = pd.DataFrame(df.iloc[IWindow].mean()).transpose() - else: - raise NotImplementedError() - return MeanValues - - - -def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKeep=None,ColSort=None,stats=['mean']): - """ Opens a list of FAST output files, perform average of its signals and return a panda dataframe - For now, the scripts only computes the mean within a time window which may be a constant or a time that is a function of the rotational speed (see `avgMethod`). - The script only computes the mean for now. Other stats will be added - - `ColMap` : dictionary where the key is the new column name, and v the old column name. - Default: None, output is not sorted - NOTE: the mapping is done before sorting and `ColKeep` is applied - ColMap = {'WS':Wind1VelX_[m/s], 'RPM': 'RotSpeed_[rpm]'} - `ColKeep` : List of strings corresponding to the signals to analyse. - Default: None, all columns are analysed - Example: ColKeep=['RotSpeed_[rpm]','BldPitch1_[deg]','RtAeroCp_[-]'] - or: ColKeep=list(ColMap.keys()) - `avgMethod` : string defining the method used to determine the extent of the averaging window: - - 'periods': use a number of periods(`avgParam`), determined by the azimuth. - - 'periods_omega': use a number of periods(`avgParam`), determined by the mean RPM - - 'constantwindow': the averaging window is constant (defined by `avgParam`). - `avgParam`: based on `avgMethod` it is either - - for 'periods_*': the number of revolutions for the window. - Default: None, as many period as possible are used - - for 'constantwindow': the number of seconds for the window - Default: None, full simulation length is used - """ - result=None - invalidFiles =[] - # Loop trough files and populate result - for i,f in enumerate(outFiles): - try: - df=FASTOutputFile(f).toDataFrame() - except: - invalidFiles.append(f) - continue - postpro=averageDF(df, avgMethod=avgMethod, avgParam=avgParam, ColMap=ColMap, ColKeep=ColKeep,ColSort=ColSort,stats=stats) - MeanValues=postpro # todo - if result is None: - # We create a dataframe here, now that we know the colums - columns = MeanValues.columns - result = pd.DataFrame(np.nan, index=np.arange(len(outFiles)), columns=columns) - result.iloc[i,:] = MeanValues.copy().values - - if ColSort is not None: - # Sorting - result.sort_values([ColSort],inplace=True,ascending=True) - result.reset_index(drop=True,inplace=True) - - if len(invalidFiles)==len(outFiles): - raise Exception('None of the files can be read (or exist)!') - elif len(invalidFiles)>0: - print('[WARN] There were {} missing/invalid files: {}'.format(len(invalidFiles),invalidFiles)) - - - return result - - -if __name__ == '__main__': - main() +# --- For cmd.py +from __future__ import division, print_function +import os +import pandas as pd +import numpy as np +import re + +# --- fast libraries +from weio.weio.fast_input_file import FASTInputFile +from weio.weio.fast_output_file import FASTOutputFile +from weio.weio.fast_input_deck import FASTInputDeck +# from pyFAST.input_output.fast_input_file import FASTInputFile +# from pyFAST.input_output.fast_output_file import FASTOutputFile +# from pyFAST.input_output.fast_input_deck import FASTInputDeck + +# --------------------------------------------------------------------------------} +# --- Tools for IO +# --------------------------------------------------------------------------------{ +def ED_BldStations(ED): + """ Returns ElastoDyn Blade Station positions, useful to know where the outputs are. + INPUTS: + - ED: either: + - a filename of a ElastoDyn input file + - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) + + OUTUPTS: + - bld_fract: fraction of the blade length were stations are defined + - r_nodes: spanwise position from the rotor apex of the Blade stations + """ + if hasattr(ED,'startswith'): # if string + ED = FASTInputFile(ED) + + nBldNodes = ED['BldNodes'] + bld_fract = np.arange(1./nBldNodes/2., 1, 1./nBldNodes) + r_nodes = bld_fract*(ED['TipRad']-ED['HubRad']) + ED['HubRad'] + return bld_fract, r_nodes + +def ED_TwrStations(ED): + """ Returns ElastoDyn Tower Station positions, useful to know where the outputs are. + INPUTS: + - ED: either: + - a filename of a ElastoDyn input file + - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) + + OUTPUTS: + - r_fract: fraction of the towet length were stations are defined + - h_nodes: height from the *ground* of the stations (not from the Tower base) + """ + if hasattr(ED,'startswith'): # if string + ED = FASTInputFile(ED) + + nTwrNodes = ED['TwrNodes'] + twr_fract = np.arange(1./nTwrNodes/2., 1, 1./nTwrNodes) + h_nodes = twr_fract*(ED['TowerHt']-ED['TowerBsHt']) + ED['TowerBsHt'] + return twr_fract, h_nodes + +def ED_BldGag(ED): + """ Returns the radial position of ElastoDyn blade gages + INPUTS: + - ED: either: + - a filename of a ElastoDyn input file + - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) + OUTPUTS: + - r_gag: The radial positions of the gages, given from the rotor apex + """ + if hasattr(ED,'startswith'): # if string + ED = FASTInputFile(ED) + _,r_nodes= ED_BldStations(ED) + + # if ED.hasNodal: + # return r_nodes, None + nOuts = ED['NBlGages'] + if nOuts<=0: + return np.array([]), np.array([]) + if type(ED['BldGagNd']) is list: + Inodes = np.asarray(ED['BldGagNd']) + else: + Inodes = np.array([ED['BldGagNd']]) + r_gag = r_nodes[ Inodes[:nOuts] -1] + return r_gag, Inodes + +def ED_TwrGag(ED): + """ Returns the heights of ElastoDyn blade gages + INPUTS: + - ED: either: + - a filename of a ElastoDyn input file + - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) + OUTPUTS: + - h_gag: The heights of the gages, given from the ground height (tower base + TowerBsHt) + """ + if hasattr(ED,'startswith'): # if string + ED = FASTInputFile(ED) + _,h_nodes= ED_TwrStations(ED) + nOuts = ED['NTwGages'] + if nOuts<=0: + return np.array([]) + if type(ED['TwrGagNd']) is list: + Inodes = np.asarray(ED['TwrGagNd']) + else: + Inodes = np.array([ED['TwrGagNd']]) + h_gag = h_nodes[ Inodes[:nOuts] -1] + return h_gag + + +def AD14_BldGag(AD): + """ Returns the radial position of AeroDyn 14 blade gages (based on "print" in column 6) + INPUTS: + - AD: either: + - a filename of a AeroDyn input file + - an instance of FileCl, as returned by reading the file, AD = weio.read(AD_filename) + OUTPUTS: + - r_gag: The radial positions of the gages, given from the blade root + """ + if hasattr(ED,'startswith'): # if string + AD = FASTInputFile(AD) + + Nodes=AD['BldAeroNodes'] + if Nodes.shape[1]==6: + doPrint= np.array([ n.lower().find('p')==0 for n in Nodes[:,5]]) + else: + doPrint=np.array([ True for n in Nodes[:,0]]) + + r_gag = Nodes[doPrint,0].astype(float) + IR = np.arange(1,len(Nodes)+1)[doPrint] + return r_gag, IR + +def AD_BldGag(AD,AD_bld,chordOut=False): + """ Returns the radial position of AeroDyn blade gages + INPUTS: + - AD: either: + - a filename of a AeroDyn input file + - an instance of FileCl, as returned by reading the file, AD = weio.read(AD_filename) + - AD_bld: either: + - a filename of a AeroDyn Blade input file + - an instance of FileCl, as returned by reading the file, AD_bld = weio.read(AD_bld_filename) + OUTPUTS: + - r_gag: The radial positions of the gages, given from the blade root + """ + if hasattr(AD,'startswith'): # if string + AD = FASTInputFile(AD) + if hasattr(AD_bld,'startswith'): # if string + AD_bld = FASTInputFile(AD_bld) + #print(AD_bld.keys()) + + nOuts=AD['NBlOuts'] + if nOuts<=0: + if chordOut: + return np.array([]), np.array([]) + else: + return np.array([]) + INodes = np.array(AD['BlOutNd'][:nOuts]) + r_gag = AD_bld['BldAeroNodes'][INodes-1,0] + if chordOut: + chord_gag = AD_bld['BldAeroNodes'][INodes-1,5] + return r_gag,chord_gag + else: + return r_gag + +def BD_BldStations(BD, BDBld): + """ Returns BeamDyn Blade Station positions, useful to know where the outputs are. + INPUTS: + - BD: either: + - a filename of a ElastoDyn input file + - an instance of FileCl, as returned by reading the file, BD = weio.read(BD_filename) + OUTUPTS: + - r_nodes: spanwise position from the balde root of the Blade stations + """ + if hasattr(BD,'startswith'): # if string + BD = FASTInputFile(BD) + if hasattr(BD,'startswith'): # if string + BDBld = FASTInputFile(BDBld) + # BD['BldFile'].replace('"','')) + + z_kp = BD['MemberGeom'][:,2] + R = z_kp[-1]-z_kp[0] + r = BDBld['BeamProperties']['span']*R + quad = BD['quadrature'] + ref = BD['refine'] + if 'default' in str(ref).lower(): + ref = 1 + dr = np.diff(r)/ref + rmid = np.concatenate( [r[:-1]+dr*(iref+1) for iref in np.arange(ref-1) ]) + r = np.concatenate( (r, rmid)) + r = np.unique(np.sort(r)) + return r + +def BD_BldGag(BD): + """ Returns the radial position of BeamDyn blade gages + INPUTS: + - BD: either: + - a filename of a BeamDyn input file + - an instance of FileCl, as returned by reading the file, BD = weio.read(BD_filename) + OUTPUTS: + - r_gag: The radial positions of the gages, given from the rotor apex + """ + if hasattr(BD,'startswith'): # if string + BD = FASTInputFile(BD) + + M = BD['MemberGeom'] + r_nodes = M[:,2] # NOTE: we select the z axis here, and we don't take curvilenear coord + nOuts = BD['NNodeOuts'] + if nOuts<=0: + nOuts=0 + if type(BD['OutNd']) is list: + Inodes = np.asarray(BD['OutNd']) + else: + Inodes = np.array([BD['OutNd']]) + r_gag = r_nodes[ Inodes[:nOuts] -1] + return r_gag, Inodes, r_nodes + +# +# +# 1, 7, 14, 21, 30, 36, 43, 52, 58 BldGagNd List of blade nodes that have strain gages [1 to BldNodes] (-) [unused if NBlGages=0] + +# --------------------------------------------------------------------------------} +# --- Helper functions for radial data +# --------------------------------------------------------------------------------{ +def _HarmonizeSpanwiseData(Name, Columns, vr, R, IR=None) : + """ helper function to use with spanwiseAD and spanwiseED """ + # --- Data present + data = [c for _,c in Columns if c is not None] + ColNames = [n for n,_ in Columns if n is not None] + Lengths = [len(d) for d in data] + if len(data)<=0: + print('[WARN] No spanwise data for '+Name) + return None, None, None + + # --- Harmonize data so that they all have the same length + nrMax = np.max(Lengths) + ids=np.arange(nrMax) + if vr is None: + bFakeVr=True + vr_bar = ids/(nrMax-1) + else: + vr_bar=vr/R + bFakeVr=False + if (nrMax)len(vr_bar): + raise Exception('Inconsitent length between radial stations and max index present in output chanels') + + for i in np.arange(len(data)): + d=data[i] + if len(d)len(vr_bar): + raise Exception('Inconsitent length between radial stations ({:d}) and max index present in output chanels ({:d})'.format(len(vr_bar),nrMax)) + df.insert(0, 'r/R_[-]', vr_bar) + + if IR is not None: + df['Node_[#]']=IR[:nrMax] + df['i_[#]']=ids+1 + if vr is not None: + df['r_[m]'] = vr[:nrMax] + return df + +def find_matching_columns(Cols, PatternMap): + ColsInfo=[] + nrMax=0 + for colpattern,colmap in PatternMap.items(): + # Extracting columns matching pattern + cols, sIdx = find_matching_pattern(Cols, colpattern) + if len(cols)>0: + # Sorting by ID + cols = np.asarray(cols) + Idx = np.array([int(s) for s in sIdx]) + Isort = np.argsort(Idx) + Idx = Idx[Isort] + cols = cols[Isort] + col={'name':colmap,'Idx':Idx,'cols':cols} + nrMax=max(nrMax,np.max(Idx)) + ColsInfo.append(col) + return ColsInfo,nrMax + +def extract_spanwise_data(ColsInfo, nrMax, df=None,ts=None): + """ + Extract spanwise data based on some column info + ColsInfo: see find_matching_columns + """ + nCols = len(ColsInfo) + if nCols==0: + return None + if ts is not None: + Values = np.zeros((nrMax,nCols)) + Values[:] = np.nan + elif df is not None: + raise NotImplementedError() + + ColNames =[c['name'] for c in ColsInfo] + + for ic,c in enumerate(ColsInfo): + Idx, cols, colname = c['Idx'], c['cols'], c['name'] + for idx,col in zip(Idx,cols): + Values[idx-1,ic]=ts[col] + nMissing = np.sum(np.isnan(Values[:,ic])) + if len(cols)nrMax: + print('[WARN] More values found for {}, found {}/{}'.format(colname,len(cols),nrMax)) + df = pd.DataFrame(data=Values, columns=ColNames) + df = df.reindex(sorted(df.columns), axis=1) + return df + + +def _BDSpanMap(): + BDSpanMap=dict() + for sB in ['B1','B2','B3']: + # Old nodal outputs + BDSpanMap['^'+sB+r'N(\d)TDxr_\[m\]'] = sB+'TDxr_[m]' + BDSpanMap['^'+sB+r'N(\d)TDyr_\[m\]'] = sB+'TDyr_[m]' + BDSpanMap['^'+sB+r'N(\d)TDzr_\[m\]'] = sB+'TDzr_[m]' + # New nodal outputs + BDSpanMap['^'+sB+r'N(\d*)_FxL_\[N\]'] = sB+'FxL_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FxL_\[N\]'] = sB+'FxL_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FxL_\[N\]'] = sB+'FxL_[N]' + BDSpanMap['^'+sB+r'N(\d*)_MxL_\[N-m\]'] = sB+'MxL_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MxL_\[N-m\]'] = sB+'MxL_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MxL_\[N-m\]'] = sB+'MxL_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_Fxr_\[N\]'] = sB+'Fxr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_Fxr_\[N\]'] = sB+'Fxr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_Fxr_\[N\]'] = sB+'Fxr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_Mxr_\[N-m\]'] = sB+'Mxr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_Mxr_\[N-m\]'] = sB+'Mxr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_Mxr_\[N-m\]'] = sB+'Mxr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_TDxr_\[m\]'] = sB+'TDxr_[m]' + BDSpanMap['^'+sB+r'N(\d*)_TDyr_\[m\]'] = sB+'TDyr_[m]' + BDSpanMap['^'+sB+r'N(\d*)_TDzr_\[m\]'] = sB+'TDzr_[m]' + BDSpanMap['^'+sB+r'N(\d*)_RDxr_\[-\]'] = sB+'RDxr_[-]' + BDSpanMap['^'+sB+r'N(\d*)_RDyr_\[-\]'] = sB+'RDyr_[-]' + BDSpanMap['^'+sB+r'N(\d*)_RDzr_\[-\]'] = sB+'RDzr_[-]' + BDSpanMap['^'+sB+r'N(\d*)_AbsXg_\[m\]'] = sB+'AbsXg_[m]' + BDSpanMap['^'+sB+r'N(\d*)_AbsYg_\[m\]'] = sB+'AbsYg_[m]' + BDSpanMap['^'+sB+r'N(\d*)_AbsZg_\[m\]'] = sB+'AbsZg_[m]' + BDSpanMap['^'+sB+r'N(\d*)_AbsXr_\[m\]'] = sB+'AbsXr_[m]' + BDSpanMap['^'+sB+r'N(\d*)_AbsYr_\[m\]'] = sB+'AbsYr_[m]' + BDSpanMap['^'+sB+r'N(\d*)_AbsZr_\[m\]'] = sB+'AbsZr_[m]' + BDSpanMap['^'+sB+r'N(\d*)_TVxg_\[m/s\]'] = sB+'TVxg_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_TVyg_\[m/s\]'] = sB+'TVyg_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_TVzg_\[m/s\]'] = sB+'TVzg_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_TVxl_\[m/s\]'] = sB+'TVxl_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_TVyl_\[m/s\]'] = sB+'TVyl_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_TVzl_\[m/s\]'] = sB+'TVzl_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_TVxr_\[m/s\]'] = sB+'TVxr_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_TVyr_\[m/s\]'] = sB+'TVyr_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_TVzr_\[m/s\]'] = sB+'TVzr_[m/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVxg_\[deg/s\]'] = sB+'RVxg_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVyg_\[deg/s\]'] = sB+'RVyg_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVzg_\[deg/s\]'] = sB+'RVzg_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVxl_\[deg/s\]'] = sB+'RVxl_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVyl_\[deg/s\]'] = sB+'RVyl_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVzl_\[deg/s\]'] = sB+'RVzl_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVxr_\[deg/s\]'] = sB+'RVxr_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVyr_\[deg/s\]'] = sB+'RVyr_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_RVzr_\[deg/s\]'] = sB+'RVzr_[deg/s]' + BDSpanMap['^'+sB+r'N(\d*)_TAxl_\[m/s^2\]'] = sB+'TAxl_[m/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_TAyl_\[m/s^2\]'] = sB+'TAyl_[m/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_TAzl_\[m/s^2\]'] = sB+'TAzl_[m/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_TAxr_\[m/s^2\]'] = sB+'TAxr_[m/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_TAyr_\[m/s^2\]'] = sB+'TAyr_[m/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_TAzr_\[m/s^2\]'] = sB+'TAzr_[m/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_RAxl_\[deg/s^2\]'] = sB+'RAxl_[deg/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_RAyl_\[deg/s^2\]'] = sB+'RAyl_[deg/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_RAzl_\[deg/s^2\]'] = sB+'RAzl_[deg/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_RAxr_\[deg/s^2\]'] = sB+'RAxr_[deg/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_RAyr_\[deg/s^2\]'] = sB+'RAyr_[deg/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_RAzr_\[deg/s^2\]'] = sB+'RAzr_[deg/s^2]' + BDSpanMap['^'+sB+r'N(\d*)_PFxL_\[N\]'] = sB+'PFxL_[N]' + BDSpanMap['^'+sB+r'N(\d*)_PFyL_\[N\]'] = sB+'PFyL_[N]' + BDSpanMap['^'+sB+r'N(\d*)_PFzL_\[N\]'] = sB+'PFzL_[N]' + BDSpanMap['^'+sB+r'N(\d*)_PMxL_\[N-m\]'] = sB+'PMxL_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_PMyL_\[N-m\]'] = sB+'PMyL_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_PMzL_\[N-m\]'] = sB+'PMzL_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_DFxL_\[N/m\]'] = sB+'DFxL_[N/m]' + BDSpanMap['^'+sB+r'N(\d*)_DFyL_\[N/m\]'] = sB+'DFyL_[N/m]' + BDSpanMap['^'+sB+r'N(\d*)_DFzL_\[N/m\]'] = sB+'DFzL_[N/m]' + BDSpanMap['^'+sB+r'N(\d*)_DMxL_\[N-m/m\]'] = sB+'DMxL_[N-m/m]' + BDSpanMap['^'+sB+r'N(\d*)_DMyL_\[N-m/m\]'] = sB+'DMyL_[N-m/m]' + BDSpanMap['^'+sB+r'N(\d*)_DMzL_\[N-m/m\]'] = sB+'DMzL_[N-m/m]' + BDSpanMap['^'+sB+r'N(\d*)_DFxR_\[N/m\]'] = sB+'DFxR_[N/m]' + BDSpanMap['^'+sB+r'N(\d*)_DFyR_\[N/m\]'] = sB+'DFyR_[N/m]' + BDSpanMap['^'+sB+r'N(\d*)_DFzR_\[N/m\]'] = sB+'DFzR_[N/m]' + BDSpanMap['^'+sB+r'N(\d*)_DMxR_\[N-m/m\]'] = sB+'DMxR_[N-m/m]' + BDSpanMap['^'+sB+r'N(\d*)_DMyR_\[N-m/m\]'] = sB+'DMyR_[N-m/m]' + BDSpanMap['^'+sB+r'N(\d*)_DMzR_\[N-m/m\]'] = sB+'DMzR_[N-m/m]' + BDSpanMap['^'+sB+r'N(\d*)_FFbxl_\[N\]'] = sB+'FFbxl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFbyl_\[N\]'] = sB+'FFbyl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFbzl_\[N\]'] = sB+'FFbzl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFbxr_\[N\]'] = sB+'FFbxr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFbyr_\[N\]'] = sB+'FFbyr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFbzr_\[N\]'] = sB+'FFbzr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_MFbxl_\[N-m\]'] = sB+'MFbxl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFbyl_\[N-m\]'] = sB+'MFbyl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFbzl_\[N-m\]'] = sB+'MFbzl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFbxr_\[N-m\]'] = sB+'MFbxr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFbyr_\[N-m\]'] = sB+'MFbyr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFbzr_\[N-m\]'] = sB+'MFbzr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_FFcxl_\[N\]'] = sB+'FFcxl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFcyl_\[N\]'] = sB+'FFcyl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFczl_\[N\]'] = sB+'FFczl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFcxr_\[N\]'] = sB+'FFcxr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFcyr_\[N\]'] = sB+'FFcyr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFczr_\[N\]'] = sB+'FFczr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_MFcxl_\[N-m\]'] = sB+'MFcxl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFcyl_\[N-m\]'] = sB+'MFcyl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFczl_\[N-m\]'] = sB+'MFczl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFcxr_\[N-m\]'] = sB+'MFcxr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFcyr_\[N-m\]'] = sB+'MFcyr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFczr_\[N-m\]'] = sB+'MFczr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_FFdxl_\[N\]'] = sB+'FFdxl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFdyl_\[N\]'] = sB+'FFdyl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFdzl_\[N\]'] = sB+'FFdzl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFdxr_\[N\]'] = sB+'FFdxr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFdyr_\[N\]'] = sB+'FFdyr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFdzr_\[N\]'] = sB+'FFdzr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_MFdxl_\[N-m\]'] = sB+'MFdxl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFdyl_\[N-m\]'] = sB+'MFdyl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFdzl_\[N-m\]'] = sB+'MFdzl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFdxr_\[N-m\]'] = sB+'MFdxr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFdyr_\[N-m\]'] = sB+'MFdyr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFdzr_\[N-m\]'] = sB+'MFdzr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_FFgxl_\[N\]'] = sB+'FFgxl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFgyl_\[N\]'] = sB+'FFgyl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFgzl_\[N\]'] = sB+'FFgzl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFgxr_\[N\]'] = sB+'FFgxr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFgyr_\[N\]'] = sB+'FFgyr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFgzr_\[N\]'] = sB+'FFgzr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_MFgxl_\[N-m\]'] = sB+'MFgxl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFgyl_\[N-m\]'] = sB+'MFgyl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFgzl_\[N-m\]'] = sB+'MFgzl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFgxr_\[N-m\]'] = sB+'MFgxr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFgyr_\[N-m\]'] = sB+'MFgyr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFgzr_\[N-m\]'] = sB+'MFgzr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_FFixl_\[N\]'] = sB+'FFixl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFiyl_\[N\]'] = sB+'FFiyl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFizl_\[N\]'] = sB+'FFizl_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFixr_\[N\]'] = sB+'FFixr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFiyr_\[N\]'] = sB+'FFiyr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_FFizr_\[N\]'] = sB+'FFizr_[N]' + BDSpanMap['^'+sB+r'N(\d*)_MFixl_\[N-m\]'] = sB+'MFixl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFiyl_\[N-m\]'] = sB+'MFiyl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFizl_\[N-m\]'] = sB+'MFizl_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFixr_\[N-m\]'] = sB+'MFixr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFiyr_\[N-m\]'] = sB+'MFiyr_[N-m]' + BDSpanMap['^'+sB+r'N(\d*)_MFizr_\[N-m\]'] = sB+'MFizr_[N-m]' + return BDSpanMap + + +def spanwiseColBD(Cols): + """ Return column info, available columns and indices that contain BD spanwise data""" + BDSpanMap = _BDSpanMap() + return find_matching_columns(Cols, BDSpanMap) + +def spanwiseColED(Cols): + """ Return column info, available columns and indices that contain ED spanwise data""" + EDSpanMap=dict() + # All Outs + for sB in ['B1','B2','B3']: + EDSpanMap['^[A]*'+sB+r'N(\d*)ALx_\[m/s^2\]' ] = sB+'ALx_[m/s^2]' + EDSpanMap['^[A]*'+sB+r'N(\d*)ALy_\[m/s^2\]' ] = sB+'ALy_[m/s^2]' + EDSpanMap['^[A]*'+sB+r'N(\d*)ALz_\[m/s^2\]' ] = sB+'ALz_[m/s^2]' + EDSpanMap['^[A]*'+sB+r'N(\d*)TDx_\[m\]' ] = sB+'TDx_[m]' + EDSpanMap['^[A]*'+sB+r'N(\d*)TDy_\[m\]' ] = sB+'TDy_[m]' + EDSpanMap['^[A]*'+sB+r'N(\d*)TDz_\[m\]' ] = sB+'TDz_[m]' + EDSpanMap['^[A]*'+sB+r'N(\d*)RDx_\[deg\]' ] = sB+'RDx_[deg]' + EDSpanMap['^[A]*'+sB+r'N(\d*)RDy_\[deg\]' ] = sB+'RDy_[deg]' + EDSpanMap['^[A]*'+sB+r'N(\d*)RDz_\[deg\]' ] = sB+'RDz_[deg]' + EDSpanMap['^[A]*'+sB+r'N(\d*)MLx_\[kN-m\]' ] = sB+'MLx_[kN-m]' + EDSpanMap['^[A]*'+sB+r'N(\d*)MLy_\[kN-m\]' ] = sB+'MLy_[kN-m]' + EDSpanMap['^[A]*'+sB+r'N(\d*)MLz_\[kN-m\]' ] = sB+'MLz_[kN-m]' + EDSpanMap['^[A]*'+sB+r'N(\d*)FLx_\[kN\]' ] = sB+'FLx_[kN]' + EDSpanMap['^[A]*'+sB+r'N(\d*)FLy_\[kN\]' ] = sB+'FLy_[kN]' + EDSpanMap['^[A]*'+sB+r'N(\d*)FLz_\[kN\]' ] = sB+'FLz_[kN]' + EDSpanMap['^[A]*'+sB+r'N(\d*)FLxNT_\[kN\]' ] = sB+'FLxNT_[kN]' + EDSpanMap['^[A]*'+sB+r'N(\d*)FLyNT_\[kN\]' ] = sB+'FLyNT_[kN]' + EDSpanMap['^[A]*'+sB+r'N(\d*)FlyNT_\[kN\]' ] = sB+'FLyNT_[kN]' # <<< Unfortunate + EDSpanMap['^[A]*'+sB+r'N(\d*)MLxNT_\[kN-m\]'] = sB+'MLxNT_[kN-m]' + EDSpanMap['^[A]*'+sB+r'N(\d*)MLyNT_\[kN-m\]'] = sB+'MLyNT_[kN-m]' + # Old + for sB in ['b1','b2','b3']: + SB=sB.upper() + EDSpanMap[r'^Spn(\d)ALx'+sB+r'_\[m/s^2\]']=SB+'ALx_[m/s^2]' + EDSpanMap[r'^Spn(\d)ALy'+sB+r'_\[m/s^2\]']=SB+'ALy_[m/s^2]' + EDSpanMap[r'^Spn(\d)ALz'+sB+r'_\[m/s^2\]']=SB+'ALz_[m/s^2]' + EDSpanMap[r'^Spn(\d)TDx'+sB+r'_\[m\]' ]=SB+'TDx_[m]' + EDSpanMap[r'^Spn(\d)TDy'+sB+r'_\[m\]' ]=SB+'TDy_[m]' + EDSpanMap[r'^Spn(\d)TDz'+sB+r'_\[m\]' ]=SB+'TDz_[m]' + EDSpanMap[r'^Spn(\d)RDx'+sB+r'_\[deg\]' ]=SB+'RDx_[deg]' + EDSpanMap[r'^Spn(\d)RDy'+sB+r'_\[deg\]' ]=SB+'RDy_[deg]' + EDSpanMap[r'^Spn(\d)RDz'+sB+r'_\[deg\]' ]=SB+'RDz_[deg]' + EDSpanMap[r'^Spn(\d)FLx'+sB+r'_\[kN\]' ]=SB+'FLx_[kN]' + EDSpanMap[r'^Spn(\d)FLy'+sB+r'_\[kN\]' ]=SB+'FLy_[kN]' + EDSpanMap[r'^Spn(\d)FLz'+sB+r'_\[kN\]' ]=SB+'FLz_[kN]' + EDSpanMap[r'^Spn(\d)MLy'+sB+r'_\[kN-m\]' ]=SB+'MLx_[kN-m]' + EDSpanMap[r'^Spn(\d)MLx'+sB+r'_\[kN-m\]' ]=SB+'MLy_[kN-m]' + EDSpanMap[r'^Spn(\d)MLz'+sB+r'_\[kN-m\]' ]=SB+'MLz_[kN-m]' + return find_matching_columns(Cols, EDSpanMap) + +def spanwiseColAD(Cols): + """ Return column info, available columns and indices that contain AD spanwise data""" + ADSpanMap=dict() + for sB in ['B1','B2','B3']: + ADSpanMap['^[A]*'+sB+r'N(\d*)Alpha_\[deg\]']=sB+'Alpha_[deg]' + ADSpanMap['^[A]*'+sB+r'N(\d*)AOA_\[deg\]' ]=sB+'Alpha_[deg]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)AxInd_\[-\]' ]=sB+'AxInd_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)TnInd_\[-\]' ]=sB+'TnInd_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[deg\]' ]=sB+'AxInd_[-]' # DBGOuts NOTE BUG Unit + ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[deg\]' ]=sB+'TnInd_[-]' # DBGOuts NOTE BUG Unit + ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[-\]' ]=sB+'AxInd_[-]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[-\]' ]=sB+'TnInd_[-]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Uin_\[m/s\]' ]=sB+'Uin_[m/s]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Uit_\[m/s\]' ]=sB+'Uit_[m/s]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Uir_\[m/s\]' ]=sB+'Uir_[m/s]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Cl_\[-\]' ]=sB+'Cl_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cd_\[-\]' ]=sB+'Cd_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cm_\[-\]' ]=sB+'Cm_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cx_\[-\]' ]=sB+'Cx_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cy_\[-\]' ]=sB+'Cy_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cn_\[-\]' ]=sB+'Cn_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Ct_\[-\]' ]=sB+'Ct_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Re_\[-\]' ]=sB+'Re_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vrel_\[m/s\]' ]=sB+'Vrel_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Theta_\[deg\]']=sB+'Theta_[deg]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Phi_\[deg\]' ]=sB+'Phi_[deg]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Twst_\[deg\]' ]=sB+'Twst_[deg]' #DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Curve_\[deg\]']=sB+'Curve_[deg]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindx_\[m/s\]']=sB+'Vindx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindy_\[m/s\]']=sB+'Vindy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fx_\[N/m\]' ]=sB+'Fx_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fy_\[N/m\]' ]=sB+'Fy_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fl_\[N/m\]' ]=sB+'Fl_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fd_\[N/m\]' ]=sB+'Fd_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fn_\[N/m\]' ]=sB+'Fn_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Ft_\[N/m\]' ]=sB+'Ft_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndx_\[m/s\]']=sB+'VUndx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndy_\[m/s\]']=sB+'VUndy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndz_\[m/s\]']=sB+'VUndz_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisx_\[m/s\]']=sB+'VDisx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisy_\[m/s\]']=sB+'VDisy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisz_\[m/s\]']=sB+'VDisz_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVx_\[m/s\]' ]=sB+'STVx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVy_\[m/s\]' ]=sB+'STVy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVz_\[m/s\]' ]=sB+'STVz_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vx_\[m/s\]' ]=sB+'Vx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vy_\[m/s\]' ]=sB+'Vy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vz_\[m/s\]' ]=sB+'Vz_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)DynP_\[Pa\]' ]=sB+'DynP_[Pa]' + ADSpanMap['^[A]*'+sB+r'N(\d*)M_\[-\]' ]=sB+'M_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Mm_\[N-m/m\]' ]=sB+'Mm_[N-m/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Gam_\[' ]=sB+'Gam_[m^2/s]' #DBGOuts + # --- AD 14 + ADSpanMap[r'^Alpha(\d*)_\[deg\]' ]='Alpha_[deg]' + ADSpanMap[r'^DynPres(\d*)_\[Pa\]' ]='DynPres_[Pa]' + ADSpanMap[r'^CLift(\d*)_\[-\]' ]='CLift_[-]' + ADSpanMap[r'^CDrag(\d*)_\[-\]' ]='CDrag_[-]' + ADSpanMap[r'^CNorm(\d*)_\[-\]' ]='CNorm_[-]' + ADSpanMap[r'^CTang(\d*)_\[-\]' ]='CTang_[-]' + ADSpanMap[r'^CMomt(\d*)_\[-\]' ]='CMomt_[-]' + ADSpanMap[r'^Pitch(\d*)_\[deg\]' ]='Pitch_[deg]' + ADSpanMap[r'^AxInd(\d*)_\[-\]' ]='AxInd_[-]' + ADSpanMap[r'^TanInd(\d*)_\[-\]' ]='TanInd_[-]' + ADSpanMap[r'^ForcN(\d*)_\[N\]' ]='ForcN_[N]' + ADSpanMap[r'^ForcT(\d*)_\[N\]' ]='ForcT_[N]' + ADSpanMap[r'^Pmomt(\d*)_\[N-m\]' ]='Pmomt_[N-N]' + ADSpanMap[r'^ReNum(\d*)_\[x10^6\]']='ReNum_[x10^6]' + ADSpanMap[r'^Gamma(\d*)_\[m^2/s\]']='Gamma_[m^2/s]' + + return find_matching_columns(Cols, ADSpanMap) + +def insert_extra_columns_AD(dfRad, tsAvg, vr=None, rho=None, R=None, nB=None, chord=None): + # --- Compute additional values (AD15 only) + if dfRad is None: + return None + if dfRad.shape[1]==0: + return dfRad + if chord is not None: + if vr is not None: + chord =chord[0:len(dfRad)] + for sB in ['B1','B2','B3']: + try: + vr_bar=vr/R + Fx = dfRad[sB+'Fx_[N/m]'] + U0 = tsAvg['Wind1VelX_[m/s]'] + Ct=nB*Fx/(0.5 * rho * 2 * U0**2 * np.pi * vr) + Ct[vr<0.01*R] = 0 + dfRad[sB+'Ctloc_[-]'] = Ct + CT=2*np.trapz(vr_bar*Ct,vr_bar) + dfRad[sB+'CtAvg_[-]']= CT*np.ones(vr.shape) + except: + pass + try: + dfRad[sB+'Gamma_[m^2/s]'] = 1/2 * chord* dfRad[sB+'Vrel_[m/s]'] * dfRad[sB+'Cl_[-]'] + except: + pass + try: + if not sB+'Vindx_[m/s]' in dfRad.columns: + dfRad[sB+'Vindx_[m/s]']= -dfRad[sB+'AxInd_[-]'].values * dfRad[sB+'Vx_[m/s]'].values + dfRad[sB+'Vindy_[m/s]']= dfRad[sB+'TnInd_[-]'].values * dfRad[sB+'Vy_[m/s]'].values + except: + pass + return dfRad + + + +def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='.outb',df=None): + """ + Postprocess FAST radial data. Average the time series, return a dataframe nr x nColumns + + INPUTS: + - FST_IN: Fast .fst input file + - avgMethod='periods', avgParam=2: average over 2 last periods, Needs Azimuth sensors!!! + - avgMethod='constantwindow', avgParam=5: average over 5s of simulation + - postprofile: outputfile to write radial data + """ + # --- Opens Fast output and performs averaging + if df is None: + df = FASTOutputFile(FST_In.replace('.fst',out_ext).replace('.dvr',out_ext)).toDataFrame() + returnDF=True + else: + returnDF=False + # NOTE: spanwise script doest not support duplicate columns + df = df.loc[:,~df.columns.duplicated()] + dfAvg = averageDF(df,avgMethod=avgMethod ,avgParam=avgParam) # NOTE: average 5 last seconds + + # --- Extract info (e.g. radial positions) from Fast input file + # We don't have a .fst input file, so we'll rely on some default values for "r" + rho = 1.225 + chord = None + # --- Extract radial positions of output channels + r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst = FASTRadialOutputs(FST_In, OutputCols=df.columns.values) + if R is None: + R=1 + try: + chord = fst.AD.Bld1['BldAeroNodes'][:,5] # Full span + except: + pass + try: + rho = fst.AD['Rho'] + except: + try: + rho = fst.AD['AirDens'] + except: + pass + #print('r_AD:', r_AD) + #print('r_ED:', r_ED) + #print('r_BD:', r_BD) + #print('I_AD:', IR_AD) + #print('I_ED:', IR_ED) + #print('I_BD:', IR_BD) + # --- Extract radial data and export to csv if needed + dfRad_AD = None + dfRad_ED = None + dfRad_BD = None + Cols=dfAvg.columns.values + # --- AD + ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) + dfRad_AD = extract_spanwise_data(ColsInfoAD, nrMaxAD, df=None, ts=dfAvg.iloc[0]) + dfRad_AD = insert_extra_columns_AD(dfRad_AD, dfAvg.iloc[0], vr=r_AD, rho=rho, R=R, nB=3, chord=chord) + dfRad_AD = insert_radial_columns(dfRad_AD, r_AD, R=R, IR=IR_AD) + # --- ED + ColsInfoED, nrMaxED = spanwiseColED(Cols) + dfRad_ED = extract_spanwise_data(ColsInfoED, nrMaxED, df=None, ts=dfAvg.iloc[0]) + dfRad_ED = insert_radial_columns(dfRad_ED, r_ED, R=R, IR=IR_ED) + # --- BD + ColsInfoBD, nrMaxBD = spanwiseColBD(Cols) + dfRad_BD = extract_spanwise_data(ColsInfoBD, nrMaxBD, df=None, ts=dfAvg.iloc[0]) + dfRad_BD = insert_radial_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) + if returnDF: + return dfRad_ED , dfRad_AD, dfRad_BD, df + else: + return dfRad_ED , dfRad_AD, dfRad_BD + + + +def spanwisePostProRows(df, FST_In=None): + """ + Returns a 3D matrix: n x nSpan x nColumn where df is of size n x mColumn + + NOTE: this is really not optimal. Spanwise columns should be extracted only once.. + """ + # --- Extract info (e.g. radial positions) from Fast input file + # We don't have a .fst input file, so we'll rely on some default values for "r" + rho = 1.225 + chord = None + # --- Extract radial positions of output channels + r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst = FASTRadialOutputs(FST_In, OutputCols=df.columns.values) + #print('r_AD:', r_AD) + #print('r_ED:', r_ED) + #print('r_BD:', r_BD) + if R is None: + R=1 + try: + chord = fst.AD.Bld1['BldAeroNodes'][:,5] # Full span + except: + pass + try: + rho = fst.AD['Rho'] + except: + try: + rho = fst.AD['AirDens'] + except: + print('[WARN] Using default air density (1.225)') + pass + # --- Extract radial data for each azimuthal average + M_AD=None + M_ED=None + M_BD=None + Col_AD=None + Col_ED=None + Col_BD=None + v = df.index.values + + # --- Getting Column info + Cols=df.columns.values + if r_AD is not None: + ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) + if r_ED is not None: + ColsInfoED, nrMaxED = spanwiseColED(Cols) + if r_BD is not None: + ColsInfoBD, nrMaxBD = spanwiseColBD(Cols) + for i,val in enumerate(v): + if r_AD is not None: + dfRad_AD = extract_spanwise_data(ColsInfoAD, nrMaxAD, df=None, ts=df.iloc[i]) + dfRad_AD = insert_extra_columns_AD(dfRad_AD, df.iloc[i], vr=r_AD, rho=rho, R=R, nB=3, chord=chord) + dfRad_AD = insert_radial_columns(dfRad_AD, r_AD, R=R, IR=IR_AD) + if i==0: + M_AD = np.zeros((len(v), len(dfRad_AD), len(dfRad_AD.columns))) + Col_AD=dfRad_AD.columns.values + M_AD[i, :, : ] = dfRad_AD.values + if r_ED is not None and len(r_ED)>0: + dfRad_ED = extract_spanwise_data(ColsInfoED, nrMaxED, df=None, ts=df.iloc[i]) + dfRad_ED = insert_radial_columns(dfRad_ED, r_ED, R=R, IR=IR_ED) + if i==0: + M_ED = np.zeros((len(v), len(dfRad_ED), len(dfRad_ED.columns))) + Col_ED=dfRad_ED.columns.values + M_ED[i, :, : ] = dfRad_ED.values + if r_BD is not None and len(r_BD)>0: + dfRad_BD = extract_spanwise_data(ColsInfoBD, nrMaxBD, df=None, ts=df.iloc[i]) + dfRad_BD = insert_radial_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) + if i==0: + M_BD = np.zeros((len(v), len(dfRad_BD), len(dfRad_BD.columns))) + Col_BD=dfRad_BD.columns.values + M_BD[i, :, : ] = dfRad_BD.values + return M_AD, Col_AD, M_ED, Col_ED, M_BD, Col_BD + + +def FASTRadialOutputs(FST_In, OutputCols=None): + """ Returns radial positions where FAST has outputs + INPUTS: + FST_In: fast input file (.fst) + OUTPUTS: + r_AD: radial positions of FAST Outputs from the rotor center + """ + R = None + r_hub =0 + r_AD = None + r_ED = None + r_BD = None + IR_ED = None + IR_AD = None + IR_BD = None + fst=None + if FST_In is not None: + fst = FASTInputDeck(FST_In, readlist=['AD','ADbld','ED','BD','BDbld']) + # NOTE: all this below should be in FASTInputDeck + if fst.version == 'F7': + # --- FAST7 + if not hasattr(fst,'AD'): + raise Exception('The AeroDyn file couldn''t be found or read, from main file: '+FST_In) + r_AD,IR_AD = AD14_BldGag(fst.AD) + R = fst.fst['TipRad'] + try: + rho = fst.AD['Rho'] + except: + rho = fst.AD['AirDens'] + else: + # --- OpenFAST 2 + R = None + + # --- ElastoDyn + if 'NumTurbines' in fst.fst.keys(): + # AeroDyn driver... + if 'HubRad(1)' in fst.fst.keys(): + r_hub = fst.fst['HubRad(1)'] + else: + r_hub = fst.fst['BldHubRad_bl(1_1)'] + + elif not hasattr(fst,'ED'): + print('[WARN] The Elastodyn file couldn''t be found or read, from main file: '+FST_In) + #raise Exception('The Elastodyn file couldn''t be found or read, from main file: '+FST_In) + else: + R = fst.ED['TipRad'] + r_hub = fst.ED['HubRad'] + if fst.ED.hasNodal: + _, r_ED = ED_BldStations(fst.ED) + IR_ED =None + else: + r_ED, IR_ED = ED_BldGag(fst.ED) + + # --- BeamDyn + if fst.BD is not None: + if R is None: + R = r_BD_All[-1] # just in case ED file missing + if fst.BD.hasNodal: + r_BD = BD_BldStations(fst.BD, fst.BDbld) + else: + r_BD, IR_BD, r_BD_All = BD_BldGag(fst.BD) + r_BD= r_BD+r_hub + + # --- AeroDyn + if fst.AD is None: + print('[WARN] The AeroDyn file couldn''t be found or read, from main file: '+FST_In) + #raise Exception('The AeroDyn file couldn''t be found or read, from main file: '+FST_In) + else: + if fst.ADversion == 'AD15': + if fst.AD.Bld1 is None: + raise Exception('The AeroDyn blade file couldn''t be found or read, from main file: '+FST_In) + + if 'B1N001Cl_[-]' in OutputCols or np.any(np.char.find(list(OutputCols),'AB1N')==0): + # This was compiled with all outs + r_AD = fst.AD.Bld1['BldAeroNodes'][:,0] # Full span + r_AD += r_hub + IR_AD = None + else: + r_AD,_ = AD_BldGag(fst.AD,fst.AD.Bld1, chordOut = True) # Only at Gages locations + r_AD += r_hub + + if R is None: + # ElastoDyn was not read, we use R from AD + R = fst.AD.Bld1['BldAeroNodes'][-1,0] + + elif fst.ADversion == 'AD14': + r_AD,IR_AD = AD14_BldGag(fst.AD) + + else: + raise Exception('AeroDyn version unknown') + return r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst + + + +def addToOutlist(OutList, Signals): + if not isinstance(Signals,list): + raise Exception('Signals must be a list') + for s in Signals: + ss=s.split()[0].strip().strip('"').strip('\'') + AlreadyIn = any([o.find(ss)==1 for o in OutList ]) + if not AlreadyIn: + OutList.append(s) + return OutList + + + +# --------------------------------------------------------------------------------} +# --- Generic df +# --------------------------------------------------------------------------------{ +def remap_df(df, ColMap, bColKeepNewOnly=False, inPlace=False, dataDict=None, verbose=False): + """ Add/rename columns of a dataframe, potentially perform operations between columns + + dataDict: dicitonary of data to be made available as "variable" in the column mapping + + Example: + + ColumnMap={ + 'WS_[m/s]' : '{Wind1VelX_[m/s]}' , # create a new column from existing one + 'RtTSR_[-]' : '{RtTSR_[-]} * 2 + {RtAeroCt_[-]}' , # change value of column + 'RotSpeed_[rad/s]' : '{RotSpeed_[rpm]} * 2*np.pi/60 ', # new column [rpm] -> [rad/s] + } + # Read + df = weio.read('FASTOutBin.outb').toDataFrame() + # Change columns based on formulae, potentially adding new columns + df = fastlib.remap_df(df, ColumnMap, inplace=True) + + """ + # Insert dataDict into namespace + if dataDict is not None: + for k,v in dataDict.items(): + exec('{:s} = dataDict["{:s}"]'.format(k,k)) + + + if not inPlace: + df=df.copy() + ColMapMiss=[] + ColNew=[] + RenameMap=dict() + # Loop for expressions + for k0,v in ColMap.items(): + k=k0.strip() + v=v.strip() + if v.find('{')>=0: + search_results = re.finditer(r'\{.*?\}', v) + expr=v + if verbose: + print('Attempt to insert column {:15s} with expr {}'.format(k,v)) + # For more advanced operations, we use an eval + bFail=False + for item in search_results: + col=item.group(0)[1:-1] + if col not in df.columns: + ColMapMiss.append(col) + bFail=True + expr=expr.replace(item.group(0),'df[\''+col+'\']') + #print(k0, '=', expr) + if not bFail: + df[k]=eval(expr) + ColNew.append(k) + else: + print('[WARN] Column not present in dataframe, cannot evaluate: ',expr) + else: + #print(k0,'=',v) + if v not in df.columns: + ColMapMiss.append(v) + print('[WARN] Column not present in dataframe: ',v) + else: + RenameMap[k]=v + + # Applying renaming only now so that expressions may be applied in any order + for k,v in RenameMap.items(): + if verbose: + print('Renaming column {:15s} > {}'.format(v,k)) + k=k.strip() + iCol = list(df.columns).index(v) + df.columns.values[iCol]=k + ColNew.append(k) + df.columns = df.columns.values # Hack to ensure columns are updated + + if len(ColMapMiss)>0: + print('[FAIL] The following columns were not found in the dataframe:',ColMapMiss) + #print('Available columns are:',df.columns.values) + + if bColKeepNewOnly: + ColNew = [c for c,_ in ColMap.items() if c in ColNew]# Making sure we respec order from user + ColKeepSafe = [c for c in ColNew if c in df.columns.values] + ColKeepMiss = [c for c in ColNew if c not in df.columns.values] + if len(ColKeepMiss)>0: + print('[WARN] Signals missing and omitted for ColKeep:\n '+'\n '.join(ColKeepMiss)) + df=df[ColKeepSafe] + return df + + +# --------------------------------------------------------------------------------} +# --- Tools for PostProcessing one or several simulations +# --------------------------------------------------------------------------------{ +def _zero_crossings(y,x=None,direction=None): + """ + Find zero-crossing points in a discrete vector, using linear interpolation. + direction: 'up' or 'down', to select only up-crossings or down-crossings + Returns: + x values xzc such that y(yzc)==0 + indexes izc, such that the zero is between y[izc] (excluded) and y[izc+1] (included) + if direction is not provided, also returns: + sign, equal to 1 for up crossing + """ + y=np.asarray(y) + if x is None: + x=np.arange(len(y)) + + if np.any((x[1:] - x[0:-1]) <= 0.0): + raise Exception('x values need to be in ascending order') + + # Indices before zero-crossing + iBef = np.where(y[1:]*y[0:-1] < 0.0)[0] + + # Find the zero crossing by linear interpolation + xzc = x[iBef] - y[iBef] * (x[iBef+1] - x[iBef]) / (y[iBef+1] - y[iBef]) + + # Selecting points that are exactly 0 and where neighbor change sign + iZero = np.where(y == 0.0)[0] + iZero = iZero[np.where((iZero > 0) & (iZero < x.size-1))] + iZero = iZero[np.where(y[iZero-1]*y[iZero+1] < 0.0)] + + # Concatenate + xzc = np.concatenate((xzc, x[iZero])) + iBef = np.concatenate((iBef, iZero)) + + # Sort + iSort = np.argsort(xzc) + xzc, iBef = xzc[iSort], iBef[iSort] + + # Return up-crossing, down crossing or both + sign = np.sign(y[iBef+1]-y[iBef]) + if direction == 'up': + I= np.where(sign==1)[0] + return xzc[I],iBef[I] + elif direction == 'down': + I= np.where(sign==-1)[0] + return xzc[I],iBef[I] + elif direction is not None: + raise Exception('Direction should be either `up` or `down`') + return xzc, iBef, sign + +def find_matching_pattern(List, pattern, sort=False): + """ Return elements of a list of strings that match a pattern + and return the first matching group + """ + reg_pattern=re.compile(pattern) + MatchedElements=[] + MatchedStrings=[] + for l in List: + match=reg_pattern.search(l) + if match: + MatchedElements.append(l) + if len(match.groups(1))>0: + MatchedStrings.append(match.groups(1)[0]) + else: + MatchedStrings.append('') + + if sort: + # Sorting by Matched string, NOTE: assumes that MatchedStrings are int. + # that's probably not necessary since alphabetical/integer sorting should be the same + # but it might be useful if number of leading zero differs, which would skew the sorting.. + MatchedElements = np.asarray(MatchedElements) + MatchedStrings = np.asarray(MatchedStrings) + Idx = np.array([int(s) for s in MatchedStrings]) + Isort = np.argsort(Idx) + Idx = Idx[Isort] + MatchedElements = MatchedElements[Isort] + MatchedStrings = MatchedStrings[Isort] + + return MatchedElements, MatchedStrings + + +def extractSpanTS(df, pattern): + r""" + Extract spanwise time series of a given "type" (e.g. Cl for each radial node) + Return a dataframe of size nt x nr + + NOTE: time is not inserted in the output dataframe + + To find "r" use FASTRadialOutputs, it is different for AeroDyn/ElastoDyn/BeamDyn/ + There is no guarantee that the number of columns matching pattern will exactly + corresponds to the number of radial stations. That's the responsability of the + OpenFAST user. + + INPUTS: + - df : a dataframe of size nt x nColumns + - pattern: Pattern used to find "radial" columns amongst the dataframe columns + r'B1N(\d*)Cl_\[-\]' + r'^AB1N(\d*)Cl_\[-\]' -> to match AB1N001Cl_[-], AB1N002Cl_[-], etc. + OUTPUTS: + - dfOut : a dataframe of size nt x nr where nr is the number of radial stations matching the pattern. The radial stations are sorted. + """ + cols, sIdx = find_matching_pattern(df.columns, pattern, sort=True) + return df[cols] + + +def _extractSpanTSReg_Legacy(ts, col_pattern, colname, IR=None): + r""" Helper function to extract spanwise results, like B1N1Cl B1N2Cl etc. + + Example + col_pattern: r'B1N(\d*)Cl_\[-\]' + colname : r'B1Cl_[-]' + """ + # Extracting columns matching pattern + cols, sIdx = find_matching_pattern(ts.keys(), col_pattern, sort=True) + if len(cols) ==0: + return (None,None) + + nrMax = np.max(Idx) + Values = np.zeros((nrMax,1)) + Values[:] = np.nan +# if IR is None: +# cols = [col_pattern.format(ir+1) for ir in range(nr)] +# else: +# cols = [col_pattern.format(ir) for ir in IR] + for idx,col in zip(Idx,cols): + Values[idx-1]=ts[col] + nMissing = np.sum(np.isnan(Values)) + if nMissing==nrMax: + return (None,None) + if len(cols)nrMax: + print('[WARN] More values found for {}, found {}/{}'.format(colname,len(cols),nrMax)) + return (colname,Values) + +def _extractSpanTS_Legacy(ts, nr, col_pattern, colname, IR=None): + """ Helper function to extract spanwise results, like B1N1Cl B1N2Cl etc. + + Example + col_pattern: 'B1N{:d}Cl_[-]' + colname : 'B1Cl_[-]' + """ + Values=np.zeros((nr,1)) + if IR is None: + cols = [col_pattern.format(ir+1) for ir in range(nr)] + else: + cols = [col_pattern.format(ir) for ir in IR] + colsExist = [c for c in cols if c in ts.keys() ] + if len(colsExist)==0: + return (None,None) + + Values = [ts[c] if c in ts.keys() else np.nan for c in cols ] + nMissing = np.sum(np.isnan(Values)) + #Values = ts[cols].T + #nCoun=len(Values) + if nMissing==nr: + return (None,None) + if len(colsExist)nr: + print('[WARN] More values found for {}, found {}/{}'.format(colname,len(cols),nr)) + return (colname,Values) + +def radialInterpTS(df, r, varName, r_ref, blade=1, bldFmt='AB{:d}', ndFmt='N{:03d}', method='interp'): + """ + Interpolate a time series at a given radial position for a given variable (varName) + INPUTS: + - df : a dataframe (typically with OpenFAST time series) + - r : radial positions of node where data is to be interpolated + - varName: variable name (and unit) to be interpolated. + The dataframe column will be assumed to be "BldFmt"+"ndFmt"+varName + - r_ref : radial position of nodal data present in the dataframe + - bldFmt : format for blade number, e.g. 'B{:d}' or 'AB{:d}' + - ndFmt : format for node number, e.g. 'N{:d}' or 'N{:03d}' + OUTPUT: + - interpolated time series + """ + # --- Sanity checks + r_ref = np.asarray(r_ref) + if not np.all(r_ref[:-1] <= r_ref[1:]): + raise Exception('This function only works for ascending radial values') + + # No extrapolation + if rnp.max(r_ref): + raise Exception('Extrapolation not supported') + + # Exactly on first or last nodes + if r==r_ref[0]: + col=bldFmt.format(blade) + ndFmt.format(1) + varName + if col in df.columns.values: + return df[col] + else: + raise Exception('Column {} not found in dataframe'.format(col)) + elif r==r_ref[-1]: + col=bldFmt.format(blade) + ndFmt.format(len(r_ref)+1) + varName + if col in df.columns.values: + return df[col] + else: + raise Exception('Column {} not found in dataframe'.format(col)) + + if method=='interp': + # Interpolation + iBef = np.where(r_reftStart].copy() + + dfPsi= bin_mean_DF(df, psiBin, colPsi) + if np.any(dfPsi['Counts']<1): + print('[WARN] some bins have no data! Increase the bin size.') + + return dfPsi + + +def averageDF(df,avgMethod='periods',avgParam=None,ColMap=None,ColKeep=None,ColSort=None,stats=['mean']): + """ + See average PostPro for documentation, same interface, just does it for one dataframe + """ + def renameCol(x): + for k,v in ColMap.items(): + if x==v: + return k + return x + # Before doing the colomn map we store the time + time = df['Time_[s]'].values + timenoNA = time[~np.isnan(time)] + # Column mapping + if ColMap is not None: + ColMapMiss = [v for _,v in ColMap.items() if v not in df.columns.values] + if len(ColMapMiss)>0: + print('[WARN] Signals missing and omitted for ColMap:\n '+'\n '.join(ColMapMiss)) + df.rename(columns=renameCol,inplace=True) + ## Defining a window for stats (start time and end time) + if avgMethod.lower()=='constantwindow': + tEnd = timenoNA[-1] + if avgParam is None: + tStart=timenoNA[0] + else: + tStart =tEnd-avgParam + elif avgMethod.lower()=='periods': + # --- Using azimuth to find periods + if 'Azimuth_[deg]' not in df.columns: + raise Exception('The sensor `Azimuth_[deg]` does not appear to be in the output file. You cannot use the averaging method by `periods`, use `constantwindow` instead.') + # NOTE: potentially we could average over each period and then average + psi=df['Azimuth_[deg]'].values + _,iBef = _zero_crossings(psi-psi[-10],direction='up') + if len(iBef)==0: + _,iBef = _zero_crossings(psi-180,direction='up') + if len(iBef)==0: + print('[WARN] Not able to find a zero crossing!') + tEnd = time[-1] + iBef=[0] + else: + tEnd = time[iBef[-1]] + + if avgParam is None: + tStart=time[iBef[0]] + else: + avgParam=int(avgParam) + if len(iBef)-10: + print('[WARN] Signals missing and omitted for ColKeep:\n '+'\n '.join(ColKeepMiss)) + df=df[ColKeepSafe] + if tStart=tStart) & (time<=tEnd) & (~np.isnan(time)))[0] + iEnd = IWindow[-1] + iStart = IWindow[0] + ## Absolute and relative differences at window extremities + DeltaValuesAbs=(df.iloc[iEnd]-df.iloc[iStart]).abs() +# DeltaValuesRel=(df.iloc[iEnd]-df.iloc[iStart]).abs()/df.iloc[iEnd] + DeltaValuesRel=(df.iloc[IWindow].max()-df.iloc[IWindow].min())/df.iloc[IWindow].mean() + #EndValues=df.iloc[iEnd] + #if avgMethod.lower()=='periods_omega': + # if DeltaValuesRel['RotSpeed_[rpm]']*100>5: + # print('[WARN] Rotational speed vary more than 5% in averaging window ({}%) for simulation: {}'.format(DeltaValuesRel['RotSpeed_[rpm]']*100,f)) + ## Stats values during window + # MeanValues = df[IWindow].mean() + # StdValues = df[IWindow].std() + if 'mean' in stats: + MeanValues = pd.DataFrame(df.iloc[IWindow].mean()).transpose() + else: + raise NotImplementedError() + return MeanValues + + + +def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKeep=None,ColSort=None,stats=['mean']): + """ Opens a list of FAST output files, perform average of its signals and return a panda dataframe + For now, the scripts only computes the mean within a time window which may be a constant or a time that is a function of the rotational speed (see `avgMethod`). + The script only computes the mean for now. Other stats will be added + + `ColMap` : dictionary where the key is the new column name, and v the old column name. + Default: None, output is not sorted + NOTE: the mapping is done before sorting and `ColKeep` is applied + ColMap = {'WS':Wind1VelX_[m/s], 'RPM': 'RotSpeed_[rpm]'} + `ColKeep` : List of strings corresponding to the signals to analyse. + Default: None, all columns are analysed + Example: ColKeep=['RotSpeed_[rpm]','BldPitch1_[deg]','RtAeroCp_[-]'] + or: ColKeep=list(ColMap.keys()) + `avgMethod` : string defining the method used to determine the extent of the averaging window: + - 'periods': use a number of periods(`avgParam`), determined by the azimuth. + - 'periods_omega': use a number of periods(`avgParam`), determined by the mean RPM + - 'constantwindow': the averaging window is constant (defined by `avgParam`). + `avgParam`: based on `avgMethod` it is either + - for 'periods_*': the number of revolutions for the window. + Default: None, as many period as possible are used + - for 'constantwindow': the number of seconds for the window + Default: None, full simulation length is used + """ + result=None + invalidFiles =[] + # Loop trough files and populate result + for i,f in enumerate(outFiles): + try: + df=FASTOutputFile(f).toDataFrame() + except: + invalidFiles.append(f) + continue + postpro=averageDF(df, avgMethod=avgMethod, avgParam=avgParam, ColMap=ColMap, ColKeep=ColKeep,ColSort=ColSort,stats=stats) + MeanValues=postpro # todo + if result is None: + # We create a dataframe here, now that we know the colums + columns = MeanValues.columns + result = pd.DataFrame(np.nan, index=np.arange(len(outFiles)), columns=columns) + result.iloc[i,:] = MeanValues.copy().values + + if ColSort is not None: + # Sorting + result.sort_values([ColSort],inplace=True,ascending=True) + result.reset_index(drop=True,inplace=True) + + if len(invalidFiles)==len(outFiles): + raise Exception('None of the files can be read (or exist)!') + elif len(invalidFiles)>0: + print('[WARN] There were {} missing/invalid files: {}'.format(len(invalidFiles),invalidFiles)) + + + return result + + +def integrateMoment(r, F): + """ + Integrate moment from force and radial station + M_j = \int_{r_j}^(r_n) f(r) * (r-r_j) dr for j=1,nr + TODO: integrate analytically the "r" part + """ + M = np.zeros(len(r)-1) + for ir,_ in enumerate(r[:-1]): + M[ir] = np.trapz(F[ir:]*(r[ir:]-r[ir]), r[ir:]-r[ir]) + return M + +def integrateMomentTS(r, F): + """ + Integrate moment from time series of forces at nr radial stations + + Compute + M_j = \int_{r_j}^(r_n) f(r) * (r-r_j) dr for j=1,nr + M_j = \int_{r_j}^(r_n) f(r) *r*dr - r_j * \int_(r_j}^{r_n} f(r) dr + j are the columns of M + + NOTE: simply trapezoidal integration is used. + The "r" term is not integrated analytically. This can be improved! + + INPUTS: + - r: array of size nr, of radial stations (ordered) + - F: array nt x nr of time series of forces at each radial stations + OUTPUTS: + - M: array nt x nr of integrated moment at each radial station + + """ + import scipy.integrate as si + # Compute \int_{r_j}^{r_n} f(r) dr, with "j" each column + IF = np.fliplr(-si.cumtrapz(np.fliplr(F), r[-1::-1])) + # Compute \int_{r_j}^{r_n} f(r)*r dr, with "j" each column + FR = F * r + IFR = np.fliplr(-si.cumtrapz(np.fliplr(FR), r[-1::-1])) + # Compute x_j * \int_{r_j}^(r_n) f(r) * r dr + R_IF = IF * r[:-1] + # \int_{r_j}^(r_n) f(r) * (r-r_j) dr = IF + IFR + M = IFR - R_IF + + + # --- Sanity checks + M0 = integrateMoment(r, F[0,:]) + Mm1 = integrateMoment(r, F[-1,:]) + if np.max(np.abs(M0-M[0,:]))>1e-8: + raise Exception('>>> Inaccuracies in integrateMomentTS') + if np.max(np.abs(Mm1-M[-1,:]))>1e-8: + raise Exception('>>> Inaccuracies in integrateMomentTS') + + return M + +if __name__ == '__main__': + main() diff --git a/pydatview/fast/runner.py b/pydatview/fast/runner.py index f321a81..d707088 100644 --- a/pydatview/fast/runner.py +++ b/pydatview/fast/runner.py @@ -1,175 +1,193 @@ -# --- For cmd.py -from __future__ import division, print_function -import os -import subprocess -import multiprocessing - -import collections -import glob -import pandas as pd -import numpy as np -import shutil -import stat -import re - -# --- Fast libraries -from weio.weio.fast_input_file import FASTInputFile -from weio.weio.fast_output_file import FASTOutputFile -# from pyFAST.input_output.fast_input_file import FASTInputFile -# from pyFAST.input_output.fast_output_file import FASTOutputFile - -FAST_EXE='openfast' - -# --------------------------------------------------------------------------------} -# --- Tools for executing FAST -# --------------------------------------------------------------------------------{ -# --- START cmd.py -def run_cmds(inputfiles, exe, parallel=True, showOutputs=True, nCores=None, showCommand=True): - """ Run a set of simple commands of the form `exe input_file` - By default, the commands are run in "parallel" (though the method needs to be improved) - The stdout and stderr may be displayed on screen (`showOutputs`) or hidden. - A better handling is yet required. - """ - Failed=[] - def _report(p): - if p.returncode==0: - print('[ OK ] Input : ',p.input_file) - else: - Failed.append(p) - print('[FAIL] Input : ',p.input_file) - print(' Directory: '+os.getcwd()) - print(' Command : '+p.cmd) - print(' Use `showOutputs=True` to debug, or run the command above.') - #out, err = p.communicate() - #print('StdOut:\n'+out) - #print('StdErr:\n'+err) - ps=[] - iProcess=0 - if nCores is None: - nCores=multiprocessing.cpu_count() - if nCores<0: - nCores=len(inputfiles)+1 - for i,f in enumerate(inputfiles): - #print('Process {}/{}: {}'.format(i+1,len(inputfiles),f)) - ps.append(run_cmd(f, exe, wait=(not parallel), showOutputs=showOutputs, showCommand=showCommand)) - iProcess += 1 - # waiting once we've filled the number of cores - # TODO: smarter method with proper queue, here processes are run by chunks - if parallel: - if iProcess==nCores: - for p in ps: - p.wait() - for p in ps: - _report(p) - ps=[] - iProcess=0 - # Extra process if not multiptle of nCores (TODO, smarter method) - for p in ps: - p.wait() - for p in ps: - _report(p) - # --- Giving a summary - if len(Failed)==0: - print('[ OK ] All simulations run successfully.') - return True - else: - print('[FAIL] {}/{} simulations failed:'.format(len(Failed),len(inputfiles))) - for p in Failed: - print(' ',p.input_file) - return False - -def run_cmd(input_file_or_arglist, exe, wait=True, showOutputs=False, showCommand=True): - """ Run a simple command of the form `exe input_file` or `exe arg1 arg2` """ - # TODO Better capture STDOUT - if isinstance(input_file_or_arglist, list): - args= [exe] + input_file_or_arglist - input_file = ' '.join(input_file_or_arglist) - input_file_abs = input_file - else: - input_file=input_file_or_arglist - if not os.path.isabs(input_file): - input_file_abs=os.path.abspath(input_file) - else: - input_file_abs=input_file - if not os.path.exists(exe): - raise Exception('Executable not found: {}'.format(exe)) - args= [exe,input_file] - #args = 'cd '+workDir+' && '+ exe +' '+basename - shell=False - if showOutputs: - STDOut= None - else: - STDOut= open(os.devnull, 'w') - if showCommand: - print('Running: '+' '.join(args)) - if wait: - class Dummy(): - pass - p=Dummy() - p.returncode=subprocess.call(args , stdout=STDOut, stderr=subprocess.STDOUT, shell=shell) - else: - p=subprocess.Popen(args, stdout=STDOut, stderr=subprocess.STDOUT, shell=shell) - # Storing some info into the process - p.cmd = ' '.join(args) - p.args = args - p.input_file = input_file - p.input_file_abs = input_file_abs - p.exe = exe - return p -# --- END cmd.py - -def run_fastfiles(fastfiles, fastExe=None, parallel=True, showOutputs=True, nCores=None, showCommand=True, reRun=True): - if fastExe is None: - fastExe=FAST_EXE - if not reRun: - # Figure out which files exist - newfiles=[] - for f in fastfiles: - base=os.path.splitext(f)[0] - if os.path.exists(base+'.outb') or os.path.exists(base+'.out'): - print('>>> Skipping existing simulation for: ',f) - pass - else: - newfiles.append(f) - fastfiles=newfiles - - return run_cmds(fastfiles, fastExe, parallel=parallel, showOutputs=showOutputs, nCores=nCores, showCommand=showCommand) - -def run_fast(input_file, fastExe=None, wait=True, showOutputs=False, showCommand=True): - if fastExe is None: - fastExe=FAST_EXE - return run_cmd(input_file, fastExe, wait=wait, showOutputs=showOutputs, showCommand=showCommand) - - -def writeBatch(batchfile, fastfiles, fastExe=None): - """ Write batch file, everything is written relative to the batch file""" - if fastExe is None: - fastExe=FAST_EXE - fastExe_abs = os.path.abspath(fastExe) - batchfile_abs = os.path.abspath(batchfile) - batchdir = os.path.dirname(batchfile_abs) - fastExe_rel = os.path.relpath(fastExe_abs, batchdir) - with open(batchfile,'w') as f: - for ff in fastfiles: - ff_abs = os.path.abspath(ff) - ff_rel = os.path.relpath(ff_abs, batchdir) - l = fastExe_rel + ' '+ ff_rel - f.write("%s\n" % l) - - -def removeFASTOuputs(workDir): - # Cleaning folder - for f in glob.glob(os.path.join(workDir,'*.out')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.outb')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.ech')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.sum')): - os.remove(f) - -if __name__=='__main__': - run_cmds(['main1.fst','main2.fst'], './Openfast.exe', parallel=True, showOutputs=False, nCores=4, showCommand=True) - pass - # --- Test of templateReplace - +# --- For cmd.py +from __future__ import division, print_function +import os +import subprocess +import multiprocessing + +import collections +import glob +import pandas as pd +import numpy as np +import shutil +import stat +import re + +# --- Fast libraries +from weio.weio.fast_input_file import FASTInputFile +from weio.weio.fast_output_file import FASTOutputFile +# from pyFAST.input_output.fast_input_file import FASTInputFile +# from pyFAST.input_output.fast_output_file import FASTOutputFile + +FAST_EXE='openfast' + +# --------------------------------------------------------------------------------} +# --- Tools for executing FAST +# --------------------------------------------------------------------------------{ +# --- START cmd.py +def run_cmds(inputfiles, exe, parallel=True, showOutputs=True, nCores=None, showCommand=True, verbose=True): + """ Run a set of simple commands of the form `exe input_file` + By default, the commands are run in "parallel" (though the method needs to be improved) + The stdout and stderr may be displayed on screen (`showOutputs`) or hidden. + A better handling is yet required. + """ + Failed=[] + def _report(p): + if p.returncode==0: + if verbose: + print('[ OK ] Input : ',p.input_file) + else: + Failed.append(p) + if verbose: + print('[FAIL] Input : ',p.input_file) + print(' Directory: '+os.getcwd()) + print(' Command : '+p.cmd) + print(' Use `showOutputs=True` to debug, or run the command above.') + #out, err = p.communicate() + #print('StdOut:\n'+out) + #print('StdErr:\n'+err) + ps=[] + iProcess=0 + if nCores is None: + nCores=multiprocessing.cpu_count() + if nCores<0: + nCores=len(inputfiles)+1 + for i,f in enumerate(inputfiles): + #print('Process {}/{}: {}'.format(i+1,len(inputfiles),f)) + ps.append(run_cmd(f, exe, wait=(not parallel), showOutputs=showOutputs, showCommand=showCommand)) + iProcess += 1 + # waiting once we've filled the number of cores + # TODO: smarter method with proper queue, here processes are run by chunks + if parallel: + if iProcess==nCores: + for p in ps: + p.wait() + for p in ps: + _report(p) + ps=[] + iProcess=0 + # Extra process if not multiptle of nCores (TODO, smarter method) + for p in ps: + p.wait() + for p in ps: + _report(p) + # --- Giving a summary + if len(Failed)==0: + if verbose: + print('[ OK ] All simulations run successfully.') + return True, Failed + else: + print('[FAIL] {}/{} simulations failed:'.format(len(Failed),len(inputfiles))) + for p in Failed: + print(' ',p.input_file) + return False, Failed + +def run_cmd(input_file_or_arglist, exe, wait=True, showOutputs=False, showCommand=True): + """ Run a simple command of the form `exe input_file` or `exe arg1 arg2` """ + # TODO Better capture STDOUT + if isinstance(input_file_or_arglist, list): + args= [exe] + input_file_or_arglist + input_file = ' '.join(input_file_or_arglist) + input_file_abs = input_file + else: + input_file=input_file_or_arglist + if not os.path.isabs(input_file): + input_file_abs=os.path.abspath(input_file) + else: + input_file_abs=input_file + if not os.path.exists(exe): + raise Exception('Executable not found: {}'.format(exe)) + args= [exe,input_file] + #args = 'cd '+workDir+' && '+ exe +' '+basename + shell=False + if showOutputs: + STDOut= None + else: + STDOut= open(os.devnull, 'w') + if showCommand: + print('Running: '+' '.join(args)) + if wait: + class Dummy(): + pass + p=Dummy() + p.returncode=subprocess.call(args , stdout=STDOut, stderr=subprocess.STDOUT, shell=shell) + else: + p=subprocess.Popen(args, stdout=STDOut, stderr=subprocess.STDOUT, shell=shell) + # Storing some info into the process + p.cmd = ' '.join(args) + p.args = args + p.input_file = input_file + p.input_file_abs = input_file_abs + p.exe = exe + return p +# --- END cmd.py + +def run_fastfiles(fastfiles, fastExe=None, parallel=True, showOutputs=True, nCores=None, showCommand=True, reRun=True, verbose=True): + if fastExe is None: + fastExe=FAST_EXE + if not reRun: + # Figure out which files exist + newfiles=[] + for f in fastfiles: + base=os.path.splitext(f)[0] + if os.path.exists(base+'.outb') or os.path.exists(base+'.out'): + print('>>> Skipping existing simulation for: ',f) + pass + else: + newfiles.append(f) + fastfiles=newfiles + + return run_cmds(fastfiles, fastExe, parallel=parallel, showOutputs=showOutputs, nCores=nCores, showCommand=showCommand, verbose=verbose) + +def run_fast(input_file, fastExe=None, wait=True, showOutputs=False, showCommand=True): + if fastExe is None: + fastExe=FAST_EXE + return run_cmd(input_file, fastExe, wait=wait, showOutputs=showOutputs, showCommand=showCommand) + + +def writeBatch(batchfile, fastfiles, fastExe=None, nBatches=1, pause=False): + """ Write batch file, everything is written relative to the batch file""" + if fastExe is None: + fastExe=FAST_EXE + fastExe_abs = os.path.abspath(fastExe) + batchfile_abs = os.path.abspath(batchfile) + batchdir = os.path.dirname(batchfile_abs) + fastExe_rel = os.path.relpath(fastExe_abs, batchdir) + def writeb(batchfile, fastfiles): + with open(batchfile,'w') as f: + for ff in fastfiles: + ff_abs = os.path.abspath(ff) + ff_rel = os.path.relpath(ff_abs, batchdir) + l = fastExe_rel + ' '+ ff_rel + f.write("%s\n" % l) + if pause: + f.write("pause\n") # windows only.. + + if nBatches==1: + writeb(batchfile, fastfiles) + else: + splits = np.array_split(fastfiles,nBatches) + base, ext = os.path.splitext(batchfile) + for i in np.arange(nBatches): + writeb(base+'_{:d}'.format(i+1) + ext, splits[i]) + + + + + + +def removeFASTOuputs(workDir): + # Cleaning folder + for f in glob.glob(os.path.join(workDir,'*.out')): + os.remove(f) + for f in glob.glob(os.path.join(workDir,'*.outb')): + os.remove(f) + for f in glob.glob(os.path.join(workDir,'*.ech')): + os.remove(f) + for f in glob.glob(os.path.join(workDir,'*.sum')): + os.remove(f) + +if __name__=='__main__': + run_cmds(['main1.fst','main2.fst'], './Openfast.exe', parallel=True, showOutputs=False, nCores=4, showCommand=True) + pass + # --- Test of templateReplace + diff --git a/pydatview/main.py b/pydatview/main.py index 07eedce..483b880 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -1,719 +1,795 @@ -from __future__ import division, unicode_literals, print_function, absolute_import -from builtins import map, range, chr, str -from io import open -from future import standard_library -standard_library.install_aliases() - -import numpy as np -import os.path -try: - import pandas as pd -except: - print('') - print('') - print('Error: problem loading pandas package:') - print(' - Check if this package is installed ( e.g. type: `pip install pandas`)') - print(' - If you are using anaconda, try `conda update python.app`') - print(' - If none of the above work, contact the developer.') - print('') - print('') - sys.exit(-1) - #raise - -import sys -import traceback -import gc - -# GUI -import wx -from .GUIPlotPanel import PlotPanel -from .GUISelectionPanel import SelectionPanel,SEL_MODES,SEL_MODES_ID -from .GUISelectionPanel import ColumnPopup,TablePopup -from .GUIInfoPanel import InfoPanel -from .GUIToolBox import GetKeyString, TBAddTool -from .Tables import TableList, Table -# Helper -from .common import * -from .GUICommon import * - - - -# --------------------------------------------------------------------------------} -# --- GLOBAL -# --------------------------------------------------------------------------------{ -PROG_NAME='pyDatView' -PROG_VERSION='v0.2-local' -try: - import weio # File Formats and File Readers - FILE_FORMATS= weio.fileFormats() -except: - print('') - print('Error: the python package `weio` was not imported successfully.\n') - print('Most likely the submodule `weio` was not cloned with `pyDatView`') - print('Type the following command to retrieve it:\n') - print(' git submodule update --init --recursive\n') - print('Alternatively re-clone this repository into a separate folder:\n') - print(' git clone --recurse-submodules https://github.com/ebranlard/pyDatView\n') - sys.exit(-1) -FILE_FORMATS_EXTENSIONS = [['.*']]+[f.extensions for f in FILE_FORMATS] -FILE_FORMATS_NAMES = ['auto (any supported file)'] + [f.name for f in FILE_FORMATS] -FILE_FORMATS_NAMEXT =['{} ({})'.format(n,','.join(e)) for n,e in zip(FILE_FORMATS_NAMES,FILE_FORMATS_EXTENSIONS)] - -SIDE_COL = [160,160,300,420,530] -SIDE_COL_LARGE = [200,200,360,480,600] -BOT_PANL =85 - -#matplotlib.rcParams['text.usetex'] = False -# matplotlib.rcParams['font.sans-serif'] = 'DejaVu Sans' -#matplotlib.rcParams['font.family'] = 'Arial' -#matplotlib.rcParams['font.sans-serif'] = 'Arial' -# matplotlib.rcParams['font.family'] = 'sans-serif' - - - - - -# --------------------------------------------------------------------------------} -# --- Drag and drop -# --------------------------------------------------------------------------------{ -# Implement File Drop Target class -class FileDropTarget(wx.FileDropTarget): - def __init__(self, parent): - wx.FileDropTarget.__init__(self) - self.parent = parent - def OnDropFiles(self, x, y, filenames): - filenames = [f for f in filenames if not os.path.isdir(f)] - filenames.sort() - if len(filenames)>0: - # If Ctrl is pressed we add - bAdd= wx.GetKeyState(wx.WXK_CONTROL); - iFormat=self.parent.comboFormats.GetSelection() - if iFormat==0: # auto-format - Format = None - else: - Format = FILE_FORMATS[iFormat-1] - self.parent.load_files(filenames,fileformat=Format,bAdd=bAdd) - return True - - - - -# --------------------------------------------------------------------------------} -# --- Main Frame -# --------------------------------------------------------------------------------{ -class MainFrame(wx.Frame): - def __init__(self, filename=None): - # Parent constructor - wx.Frame.__init__(self, None, -1, PROG_NAME+' '+PROG_VERSION) - # Data - self.tabList=TableList() - self.restore_formulas = [] - - # Hooking exceptions to display them to the user - sys.excepthook = MyExceptionHook - # --- GUI - #font = self.GetFont() - #print(font.GetFamily(),font.GetStyle(),font.GetPointSize()) - #font.SetFamily(wx.FONTFAMILY_DEFAULT) - #font.SetFamily(wx.FONTFAMILY_MODERN) - #font.SetFamily(wx.FONTFAMILY_SWISS) - #font.SetPointSize(8) - #print(font.GetFamily(),font.GetStyle(),font.GetPointSize()) - #self.SetFont(font) - # --- Menu - menuBar = wx.MenuBar() - - fileMenu = wx.Menu() - loadMenuItem = fileMenu.Append(wx.ID_NEW,"Open file" ,"Open file" ) - exptMenuItem = fileMenu.Append(-1 ,"Export table" ,"Export table" ) - saveMenuItem = fileMenu.Append(wx.ID_SAVE,"Save figure" ,"Save figure" ) - exitMenuItem = fileMenu.Append(wx.ID_EXIT, 'Quit', 'Quit application') - menuBar.Append(fileMenu, "&File") - self.Bind(wx.EVT_MENU,self.onExit ,exitMenuItem) - self.Bind(wx.EVT_MENU,self.onLoad ,loadMenuItem) - self.Bind(wx.EVT_MENU,self.onExport,exptMenuItem) - self.Bind(wx.EVT_MENU,self.onSave ,saveMenuItem) - - dataMenu = wx.Menu() - menuBar.Append(dataMenu, "&Data") - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e, 'Mask') , dataMenu.Append(wx.ID_ANY, 'Mask')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Outlier'), dataMenu.Append(wx.ID_ANY, 'Outliers removal')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Filter') , dataMenu.Append(wx.ID_ANY, 'Filter')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Resample') , dataMenu.Append(wx.ID_ANY, 'Resample')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'FASTRadialAverage'), dataMenu.Append(wx.ID_ANY, 'FAST - Radial average')) - - toolMenu = wx.Menu() - menuBar.Append(toolMenu, "&Tools") - self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'CurveFitting'), toolMenu.Append(wx.ID_ANY, 'Curve fitting')) - self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'LogDec') , toolMenu.Append(wx.ID_ANY, 'Damping from decay')) - - helpMenu = wx.Menu() - aboutMenuItem = helpMenu.Append(wx.NewId(), 'About', 'About') - menuBar.Append(helpMenu, "&Help") - self.SetMenuBar(menuBar) - self.Bind(wx.EVT_MENU,self.onAbout,aboutMenuItem) - - # --- ToolBar - tb = self.CreateToolBar(wx.TB_HORIZONTAL|wx.TB_TEXT|wx.TB_HORZ_LAYOUT) - self.toolBar = tb - self.comboFormats = wx.ComboBox(tb, choices = FILE_FORMATS_NAMEXT, style=wx.CB_READONLY) - self.comboFormats.SetSelection(0) - self.comboMode = wx.ComboBox(tb, choices = SEL_MODES, style=wx.CB_READONLY) - self.comboMode.SetSelection(0) - self.Bind(wx.EVT_COMBOBOX, self.onModeChange, self.comboMode ) - tb.AddSeparator() - tb.AddControl( wx.StaticText(tb, -1, 'Mode: ' ) ) - tb.AddControl( self.comboMode ) - tb.AddStretchableSpace() - tb.AddControl( wx.StaticText(tb, -1, 'Format: ' ) ) - tb.AddControl(self.comboFormats ) - tb.AddSeparator() - #bmp = wx.Bitmap('help.png') #wx.Bitmap("NEW.BMP", wx.BITMAP_TYPE_BMP) - TBAddTool(tb,"Open" ,wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN),self.onLoad) - TBAddTool(tb,"Reload",wx.ArtProvider.GetBitmap(wx.ART_REDO),self.onReload) - try: - TBAddTool(tb,"Add" ,wx.ArtProvider.GetBitmap(wx.ART_PLUS),self.onAdd) - except: - TBAddTool(tb,"Add" ,wx.ArtProvider.GetBitmap(wx.FILE_OPEN),self.onAdd) - #self.AddTBBitmapTool(tb,"Debug" ,wx.ArtProvider.GetBitmap(wx.ART_ERROR),self.onDEBUG) - tb.AddStretchableSpace() - tb.Realize() - - # --- Status bar - self.statusbar=self.CreateStatusBar(3, style=0) - self.statusbar.SetStatusWidths([200, -1, 70]) - - # --- Main Panel and Notebook - self.MainPanel = wx.Panel(self) - #self.MainPanel = wx.Panel(self, style=wx.RAISED_BORDER) - #self.MainPanel.SetBackgroundColour((200,0,0)) - - #self.nb = wx.Notebook(self.MainPanel) - #self.nb.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.on_tab_change) - - - sizer = wx.BoxSizer() - #sizer.Add(self.nb, 1, flag=wx.EXPAND) - self.MainPanel.SetSizer(sizer) - - # --- Drag and drop - dd = FileDropTarget(self) - self.SetDropTarget(dd) - - # --- Main Frame (self) - self.FrameSizer = wx.BoxSizer(wx.VERTICAL) - slSep = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) - self.FrameSizer.Add(slSep ,0, flag=wx.EXPAND|wx.BOTTOM,border=0) - self.FrameSizer.Add(self.MainPanel,1, flag=wx.EXPAND,border=0) - self.SetSizer(self.FrameSizer) - - self.SetSize((900, 700)) - self.Center() - self.Show() - self.Bind(wx.EVT_SIZE, self.OnResizeWindow) - - # Shortcuts - idFilter=wx.NewId() - self.Bind(wx.EVT_MENU, self.onFilter, id=idFilter) - - accel_tbl = wx.AcceleratorTable( - [(wx.ACCEL_CTRL, ord('F'), idFilter )] - ) - self.SetAcceleratorTable(accel_tbl) - - - def onFilter(self,event): - if hasattr(self,'selPanel'): - self.selPanel.colPanel1.tFilter.SetFocus() - event.Skip() - - - - def clean_memory(self,bReload=False): - #print('Clean memory') - # force Memory cleanup - self.tabList.clean() - if not bReload: - if hasattr(self,'selPanel'): - self.selPanel.clean_memory() - if hasattr(self,'infoPanel'): - self.infoPanel.clean() - if hasattr(self,'plotPanel'): - self.plotPanel.cleanPlot() - gc.collect() - - def load_files(self, filenames=[], fileformat=None, bReload=False, bAdd=False): - """ load multiple files, only trigger the plot at the end """ - if bReload: - if hasattr(self,'selPanel'): - self.selPanel.saveSelection() # TODO move to tables - - if not bAdd: - self.clean_memory(bReload=bReload) - - base_filenames = [os.path.basename(f) for f in filenames] - filenames = [f for __, f in sorted(zip(base_filenames, filenames))] - # Load the tables - warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformat=fileformat, bAdd=bAdd) - if bReload: - # Restore formulas that were previously added - _ITab, _STab = self.selPanel.getAllTables() - ITab = [iTab for __, iTab in sorted(zip(_STab, _ITab))] - if len(ITab) != len(self.restore_formulas): - raise ValueError('Invalid length of tabs and formulas!') - for iTab, f_list in zip(ITab, self.restore_formulas): - for f in f_list: - self.tabList.get(iTab).addColumnByFormula(f['name'], f['formula']) - self.restore_formulas = [] - for warn in warnList: - Warn(self,warn) - if self.tabList.len()>0: - self.load_tabs_into_GUI(bReload=bReload, bAdd=bAdd, bPlot=True) - - def load_df(self, df, name=None, bAdd=False, bPlot=True): - if bAdd: - self.tabList.append(Table(data=df, name=name)) - else: - self.tabList = TableList( [Table(data=df, name=name)] ) - self.load_tabs_into_GUI(bAdd=bAdd, bPlot=bPlot) - if hasattr(self,'selPanel'): - self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) - - def load_dfs(self, dfs, names, bAdd=False): - self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) - self.load_tabs_into_GUI(bAdd=bAdd, bPlot=True) - if hasattr(self,'selPanel'): - self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) - - def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): - if bAdd: - if not hasattr(self,'selPanel'): - bAdd=False - - if (not bReload) and (not bAdd): - self.cleanGUI() - self.Freeze() - # Setting status bar - self.setStatusBar() - - if bReload or bAdd: - self.selPanel.update_tabs(self.tabList) - else: - mode = SEL_MODES_ID[self.comboMode.GetSelection()] - #self.vSplitter = wx.SplitterWindow(self.nb) - self.vSplitter = wx.SplitterWindow(self.MainPanel) - self.selPanel = SelectionPanel(self.vSplitter, self.tabList, mode=mode, mainframe=self) - self.tSplitter = wx.SplitterWindow(self.vSplitter) - #self.tSplitter.SetMinimumPaneSize(20) - self.infoPanel = InfoPanel(self.tSplitter) - self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, self) - self.tSplitter.SetSashGravity(0.9) - self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) - self.tSplitter.SetMinimumPaneSize(BOT_PANL) - self.tSplitter.SetSashGravity(1) - self.tSplitter.SetSashPosition(400) - - self.vSplitter.SplitVertically(self.selPanel, self.tSplitter) - self.vSplitter.SetMinimumPaneSize(SIDE_COL[0]) - self.tSplitter.SetSashPosition(SIDE_COL[0]) - - #self.nb.AddPage(self.vSplitter, "Plot") - #self.nb.SendSizeEvent() - - sizer = self.MainPanel.GetSizer() - sizer.Add(self.vSplitter, 1, flag=wx.EXPAND,border=0) - self.MainPanel.SetSizer(sizer) - self.FrameSizer.Layout() - - self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel1.comboX ) - self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel1.lbColumns) - self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel2.comboX ) - self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel2.lbColumns) - self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel3.comboX ) - self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel3.lbColumns) - self.Bind(wx.EVT_LISTBOX , self.onTabSelectionChange, self.selPanel.tabPanel.lbTab) - self.Bind(wx.EVT_SPLITTER_SASH_POS_CHANGED, self.onSashChangeMain, self.vSplitter) - - self.selPanel.tabPanel.lbTab.Bind(wx.EVT_RIGHT_DOWN, self.OnTabPopup) - - # plot trigger - if bPlot: - self.mainFrameUpdateLayout() - self.onColSelectionChange(event=None) - try: - self.Thaw() - except: - pass - # Hack - #self.onShowTool(tool='Resample') - - def setStatusBar(self, ISel=None): - nTabs=self.tabList.len() - if ISel is None: - ISel = list(np.arange(nTabs)) - if nTabs<0: - self.statusbar.SetStatusText('', 0) # Format - self.statusbar.SetStatusText('', 1) # Filenames - self.statusbar.SetStatusText('', 2) # Shape - elif nTabs==1: - self.statusbar.SetStatusText(self.tabList.get(0).fileformat, 0) - self.statusbar.SetStatusText(self.tabList.get(0).filename , 1) - self.statusbar.SetStatusText(self.tabList.get(0).shapestring, 2) - elif len(ISel)==1: - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).fileformat , 0) - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).filename , 1) - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).shapestring, 2) - else: - self.statusbar.SetStatusText('' ,0) - self.statusbar.SetStatusText(", ".join(list(set([self.tabList.filenames[i] for i in ISel]))),1) - self.statusbar.SetStatusText('',2) - - def renameTable(self, iTab, newName): - oldName = self.tabList.renameTable(iTab, newName) - self.selPanel.renameTable(iTab, oldName, newName) - - def sortTabs(self, method='byName'): - self.tabList.sort(method=method) - # Updating tables - self.selPanel.update_tabs(self.tabList) - # Trigger a replot - self.onTabSelectionChange() - - - def deleteTabs(self, I): - self.tabList.deleteTabs(I) - - # Invalidating selections - self.selPanel.tabPanel.lbTab.SetSelection(-1) - # Until we have something better, we empty plot - self.plotPanel.empty() - self.infoPanel.empty() - self.selPanel.clean_memory() - # Updating tables - self.selPanel.update_tabs(self.tabList) - # Trigger a replot - self.onTabSelectionChange() - - def exportTab(self, iTab): - tab=self.tabList.get(iTab) - default_filename=tab.basename +'.csv' - with wx.FileDialog(self, "Save to CSV file",defaultFile=default_filename, - style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as dlg: - #, wildcard="CSV files (*.csv)|*.csv", - dlg.CentreOnParent() - if dlg.ShowModal() == wx.ID_CANCEL: - return # the user changed their mind - tab.export(dlg.GetPath()) - - def onShowTool(self, event=None, tool=''): - """ - tool in 'Outlier', 'Filter', 'LogDec','FASTRadialAverage', 'Mask', 'CurveFitting' - """ - if not hasattr(self,'plotPanel'): - Error(self,'Plot some data first') - return - self.plotPanel.showTool(tool) - - def onSashChangeMain(self,event=None): - pass - # doent work because size is not communicated yet - #if hasattr(self,'selPanel'): - # print('ON SASH') - # self.selPanel.setEquiSash(event) - - def OnTabPopup(self,event): - menu = TablePopup(self,self.selPanel.tabPanel.lbTab) - self.PopupMenu(menu, event.GetPosition()) - menu.Destroy() - - def onTabSelectionChange(self,event=None): - # TODO This can be cleaned-up - ISel=self.selPanel.tabPanel.lbTab.GetSelections() - if len(ISel)>0: - # Letting seletion panel handle the change - self.selPanel.tabSelectionChanged() - # Update of status bar - self.setStatusBar(ISel) - # Trigger the colSelection Event - self.onColSelectionChange(event=None) - - def onColSelectionChange(self,event=None): - if hasattr(self,'plotPanel'): - # Letting selection panel handle the change - self.selPanel.colSelectionChanged() - # Redrawing - self.plotPanel.load_and_draw() - # --- Stats trigger - #self.showStats() - - def redraw(self): - if hasattr(self,'plotPanel'): - self.plotPanel.load_and_draw() -# def showStats(self): -# self.infoPanel.showStats(self.plotPanel.plotData,self.plotPanel.pltTypePanel.plotType()) - - def onExit(self, event): - self.Close() - - def cleanGUI(self, event=None): - if hasattr(self,'plotPanel'): - del self.plotPanel - if hasattr(self,'selPanel'): - del self.selPanel - if hasattr(self,'infoPanel'): - del self.infoPanel - #self.deletePages() - try: - self.MainPanel.GetSizer().Clear(delete_windows=True) # Delete Windows - except: - self.MainPanel.GetSizer().Clear() - self.FrameSizer.Layout() - gc.collect() - - def onSave(self, event=None): - # using the navigation toolbar save functionality - self.plotPanel.navTB.save_figure() - - def onAbout(self, event=None): - Info(self,PROG_NAME+' '+PROG_VERSION+'\n\nVisit http://github.com/ebranlard/pyDatView for documentation.') - - def onReload(self, event=None): - filenames = self.tabList.unique_filenames - filenames.sort() - if len(filenames)>0: - # Save formulas to restore them after reload with sorted tabs - _ITab, _STab = self.selPanel.getAllTables() - ITab = [iTab for __, iTab in sorted(zip(_STab, _ITab))] - self.restore_formulas = [] - for iTab in ITab: - f = self.tabList.get(iTab).formulas - f = sorted(f, key=lambda k: k['pos']) - self.restore_formulas.append(f) - iFormat=self.comboFormats.GetSelection() - if iFormat==0: # auto-format - Format = None - else: - Format = FILE_FORMATS[iFormat-1] - self.load_files(filenames,fileformat=Format,bReload=True,bAdd=False) - else: - Error(self,'Open one or more file first.') - - def onDEBUG(self, event=None): - #self.clean_memory() - self.plotPanel.ctrlPanel.Refresh() - self.plotPanel.cb_sizer.ForceRefresh() - - def onExport(self, event=None): - ISel=[] - try: - ISel = self.selPanel.tabPanel.lbTab.GetSelections() - except: - pass - if len(ISel)>0: - self.exportTab(ISel[0]) - else: - Error(self,'Open a file and select a table first.') - - def onLoad(self, event=None): - self.selectFile(bAdd=False) - - def onAdd(self, event=None): - self.selectFile(bAdd=self.tabList.len()>0) - - def selectFile(self,bAdd=False): - # --- File Format extension - iFormat=self.comboFormats.GetSelection() - sFormat=self.comboFormats.GetStringSelection() - if iFormat==0: # auto-format - Format = None - #wildcard = 'all (*.*)|*.*' - wildcard='|'.join([n+'|*'+';*'.join(e) for n,e in zip(FILE_FORMATS_NAMEXT,FILE_FORMATS_EXTENSIONS)]) - #wildcard = sFormat + extensions+'|all (*.*)|*.*' - else: - Format = FILE_FORMATS[iFormat-1] - extensions = '|*'+';*'.join(FILE_FORMATS[iFormat-1].extensions) - wildcard = sFormat + extensions+'|all (*.*)|*.*' - - with wx.FileDialog(self, "Open file", wildcard=wildcard, - style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST | wx.FD_MULTIPLE) as dlg: - #other options: wx.CHANGE_DIR - #dlg.SetSize((100,100)) - #dlg.Center() - if dlg.ShowModal() == wx.ID_CANCEL: - return # the user changed their mind - self.load_files(dlg.GetPaths(),fileformat=Format,bAdd=bAdd) - - def onModeChange(self, event=None): - if hasattr(self,'selPanel'): - self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) - self.mainFrameUpdateLayout() - # --- Trigger to check number of columns - self.onTabSelectionChange() - - def mainFrameUpdateLayout(self, event=None): - if hasattr(self,'selPanel'): - nWind=self.selPanel.splitter.nWindows - if self.Size[0]<=800: - sash=SIDE_COL[nWind] - else: - sash=SIDE_COL_LARGE[nWind] - self.resizeSideColumn(sash) - - def OnResizeWindow(self, event): - try: - self.mainFrameUpdateLayout() - self.Layout() - except: - pass - # NOTE: doesn't work... - #if hasattr(self,'plotPanel'): - # Subplot spacing changes based on figure size - #print('>>> RESIZE WINDOW') - #self.redraw() - - # --- Side column - def resizeSideColumn(self,width): - # To force the replot we do an epic unsplit/split... - #self.vSplitter.Unsplit() - #self.vSplitter.SplitVertically(self.selPanel, self.tSplitter) - self.vSplitter.SetMinimumPaneSize(width) - self.vSplitter.SetSashPosition(width) - #self.selPanel.splitter.setEquiSash() - - # --- NOTEBOOK - #def deletePages(self): - # for index in reversed(range(self.nb.GetPageCount())): - # self.nb.DeletePage(index) - # self.nb.SendSizeEvent() - # gc.collect() - #def on_tab_change(self, event=None): - # page_to_select = event.GetSelection() - # wx.CallAfter(self.fix_focus, page_to_select) - # event.Skip(True) - #def fix_focus(self, page_to_select): - # page = self.nb.GetPage(page_to_select) - # page.SetFocus() - -#---------------------------------------------------------------------- -def MyExceptionHook(etype, value, trace): - """ - Handler for all unhandled exceptions. - :param `etype`: the exception type (`SyntaxError`, `ZeroDivisionError`, etc...); - :type `etype`: `Exception` - :param string `value`: the exception error message; - :param string `trace`: the traceback header, if any (otherwise, it prints the - standard Python header: ``Traceback (most recent call last)``. - """ - # Printing exception - traceback.print_exception(etype, value, trace) - # Then showing to user the last error - frame = wx.GetApp().GetTopWindow() - tmp = traceback.format_exception(etype, value, trace) - if tmp[-1].find('Exception: Error:')==0: - Error(frame,tmp[-1][18:]) - elif tmp[-1].find('Exception: Warn:')==0: - Warn(frame,tmp[-1][17:]) - else: - exception = 'The following exception occured:\n\n'+ tmp[-1] + '\n'+tmp[-2].strip() - Error(frame,exception) - try: - frame.Thaw() # Make sure any freeze event is stopped - except: - pass - -# --------------------------------------------------------------------------------} -# --- Tests -# --------------------------------------------------------------------------------{ -def test(filenames=None): - if filenames is not None: - app = wx.App(False) - frame = MainFrame() - frame.load_files(filenames,fileformat=None) - return - -# --------------------------------------------------------------------------------} -# --- Wrapped WxApp -# --------------------------------------------------------------------------------{ -class MyWxApp(wx.App): - def __init__(self, redirect=False, filename=None): - try: - wx.App.__init__(self, redirect, filename) - except: - if wx.Platform == '__WXMAC__': - #msg = """This program needs access to the screen. - # Please run with 'pythonw', not 'python', and only when you are logged - # in on the main display of your Mac.""" - msg= """ -MacOS Error: - This program needs access to the screen. Please run with a - Framework build of python, and only when you are logged in - on the main display of your Mac. - -pyDatView help: - You see the error above because you are using a Mac and - the python executable you are using does not have access to - your screen. This is a Mac issue, not a pyDatView issue. - Instead of calling 'python pyDatView.py', you need to find - another python and do '/path/python pyDatView.py' - You can try './pythonmac pyDatView.py', a script provided - in this repository to detect the path (in some cases) - - You can find additional help in the file 'README.md'. - - For quick reference, here are some typical cases: - - Your python was installed with 'brew', then likely use - /usr/lib/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX; - - Your python is an anaconda python, use something like:; - /anaconda3/bin/python.app (NOTE: the '.app'! - - You are using a python 2 version, you can use the system one: - /Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX - /System/Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX -""" - - elif wx.Platform == '__WXGTK__': - msg =""" -Error: - Unable to access the X Display, is $DISPLAY set properly? - -pyDatView help: - You are probably running this application on a server accessed via ssh. - Use `ssh -X` or `ssh -Y` to access the server. - Else, try setting up $DISPLAY before doing the ssh connection. -""" - else: - msg = 'Unable to create GUI' # TODO: more description is needed for wxMSW... - raise SystemExit(msg) - -# --------------------------------------------------------------------------------} -# --- Mains -# --------------------------------------------------------------------------------{ -def showApp(firstArg=None,dataframe=None,filenames=[]): - """ - The main function to start the data frame GUI. - """ - app = MyWxApp(False) - frame = MainFrame() - # Optional first argument - if firstArg is not None: - if isinstance(firstArg,list): - filenames=firstArg - elif isinstance(firstArg,str): - filenames=[firstArg] - elif isinstance(firstArg, pd.DataFrame): - dataframe=firstArg - # - if (dataframe is not None) and (len(dataframe)>0): - #import time - #tstart = time.time() - frame.load_df(dataframe) - #tend = time.time() - #print('PydatView time: ',tend-tstart) - elif len(filenames)>0: - frame.load_files(filenames,fileformat=None) - app.MainLoop() - -def cmdline(): - if len(sys.argv)>1: - pydatview(filename=sys.argv[1]) - else: - pydatview() +from __future__ import division, unicode_literals, print_function, absolute_import +from builtins import map, range, chr, str +from io import open +from future import standard_library +standard_library.install_aliases() + +import numpy as np +import os.path +import sys +import traceback +import gc +try: + import pandas as pd +except: + print('') + print('') + print('Error: problem loading pandas package:') + print(' - Check if this package is installed ( e.g. type: `pip install pandas`)') + print(' - If you are using anaconda, try `conda update python.app`') + print(' - If none of the above work, contact the developer.') + print('') + print('') + sys.exit(-1) + #raise + + +# GUI +import wx +from .GUIPlotPanel import PlotPanel +from .GUISelectionPanel import SelectionPanel,SEL_MODES,SEL_MODES_ID +from .GUISelectionPanel import ColumnPopup,TablePopup +from .GUIInfoPanel import InfoPanel +from .GUIToolBox import GetKeyString, TBAddTool +from .Tables import TableList, Table +# Helper +from .common import * +from .GUICommon import * +# Pluggins +from .plugins import dataPlugins + +try: + import weio.weio as weio# File Formats and File Readers +except: + print('') + print('Error: the python package `weio` was not imported successfully.\n') + print('Most likely the submodule `weio` was not cloned with `pyDatView`') + print('Type the following command to retrieve it:\n') + print(' git submodule update --init --recursive\n') + print('Alternatively re-clone this repository into a separate folder:\n') + print(' git clone --recurse-submodules https://github.com/ebranlard/pyDatView\n') + sys.exit(-1) + +from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData + +# --------------------------------------------------------------------------------} +# --- GLOBAL +# --------------------------------------------------------------------------------{ +PROG_NAME='pyDatView' +PROG_VERSION='v0.3-local' +SIDE_COL = [160,160,300,420,530] +SIDE_COL_LARGE = [200,200,360,480,600] +BOT_PANL =85 + +#matplotlib.rcParams['text.usetex'] = False +# matplotlib.rcParams['font.sans-serif'] = 'DejaVu Sans' +#matplotlib.rcParams['font.family'] = 'Arial' +#matplotlib.rcParams['font.sans-serif'] = 'Arial' +# matplotlib.rcParams['font.family'] = 'sans-serif' + + + + + +# --------------------------------------------------------------------------------} +# --- Drag and drop +# --------------------------------------------------------------------------------{ +# Implement File Drop Target class +class FileDropTarget(wx.FileDropTarget): + def __init__(self, parent): + wx.FileDropTarget.__init__(self) + self.parent = parent + def OnDropFiles(self, x, y, filenames): + filenames = [f for f in filenames if not os.path.isdir(f)] + filenames.sort() + if len(filenames)>0: + # If Ctrl is pressed we add + bAdd= wx.GetKeyState(wx.WXK_CONTROL); + iFormat=self.parent.comboFormats.GetSelection() + if iFormat==0: # auto-format + Format = None + else: + Format = self.parent.FILE_FORMATS[iFormat-1] + self.parent.load_files(filenames, fileformats=[Format]*len(filenames), bAdd=bAdd) + return True + + + + +# --------------------------------------------------------------------------------} +# --- Main Frame +# --------------------------------------------------------------------------------{ +class MainFrame(wx.Frame): + def __init__(self, data=None): + # Parent constructor + wx.Frame.__init__(self, None, -1, PROG_NAME+' '+PROG_VERSION) + # Hooking exceptions to display them to the user + sys.excepthook = MyExceptionHook + # --- Data + self.tabList=TableList() + self.restore_formulas = [] + self.systemFontSize = self.GetFont().GetPointSize() + self.data = loadAppData(self) + self.datareset = False + # Global variables... + setFontSize(self.data['fontSize']) + setMonoFontSize(self.data['monoFontSize']) + + # --- GUI + #font = self.GetFont() + #print(font.GetFamily(),font.GetStyle(),font.GetPointSize()) + #font.SetFamily(wx.FONTFAMILY_DEFAULT) + #font.SetFamily(wx.FONTFAMILY_MODERN) + #font.SetFamily(wx.FONTFAMILY_SWISS) + #font.SetPointSize(8) + #print(font.GetFamily(),font.GetStyle(),font.GetPointSize()) + #self.SetFont(font) + self.SetFont(getFont(self)) + # --- Menu + menuBar = wx.MenuBar() + + fileMenu = wx.Menu() + loadMenuItem = fileMenu.Append(wx.ID_NEW,"Open file" ,"Open file" ) + exptMenuItem = fileMenu.Append(-1 ,"Export table" ,"Export table" ) + saveMenuItem = fileMenu.Append(wx.ID_SAVE,"Save figure" ,"Save figure" ) + exitMenuItem = fileMenu.Append(wx.ID_EXIT, 'Quit', 'Quit application') + menuBar.Append(fileMenu, "&File") + self.Bind(wx.EVT_MENU,self.onExit ,exitMenuItem) + self.Bind(wx.EVT_MENU,self.onLoad ,loadMenuItem) + self.Bind(wx.EVT_MENU,self.onExport,exptMenuItem) + self.Bind(wx.EVT_MENU,self.onSave ,saveMenuItem) + + dataMenu = wx.Menu() + menuBar.Append(dataMenu, "&Data") + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e, 'Mask') , dataMenu.Append(wx.ID_ANY, 'Mask')) + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Outlier'), dataMenu.Append(wx.ID_ANY, 'Outliers removal')) + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Filter') , dataMenu.Append(wx.ID_ANY, 'Filter')) + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Resample') , dataMenu.Append(wx.ID_ANY, 'Resample')) + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'FASTRadialAverage'), dataMenu.Append(wx.ID_ANY, 'FAST - Radial average')) + + # --- Data Plugins + for string, function, isPanel in dataPlugins: + self.Bind(wx.EVT_MENU, lambda e, s_loc=string: self.onDataPlugin(e, s_loc), dataMenu.Append(wx.ID_ANY, string)) + + toolMenu = wx.Menu() + menuBar.Append(toolMenu, "&Tools") + self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'CurveFitting'), toolMenu.Append(wx.ID_ANY, 'Curve fitting')) + self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'LogDec') , toolMenu.Append(wx.ID_ANY, 'Damping from decay')) + + helpMenu = wx.Menu() + aboutMenuItem = helpMenu.Append(wx.NewId(), 'About', 'About') + resetMenuItem = helpMenu.Append(wx.NewId(), 'Reset options', 'Rest options') + menuBar.Append(helpMenu, "&Help") + self.SetMenuBar(menuBar) + self.Bind(wx.EVT_MENU,self.onAbout, aboutMenuItem) + self.Bind(wx.EVT_MENU,self.onReset, resetMenuItem) + + + self.FILE_FORMATS, errors= weio.fileFormats(ignoreErrors=True, verbose=False) + if len(errors)>0: + for e in errors: + Warn(self, e) + + self.FILE_FORMATS_EXTENSIONS = [['.*']]+[f.extensions for f in self.FILE_FORMATS] + self.FILE_FORMATS_NAMES = ['auto (any supported file)'] + [f.name for f in self.FILE_FORMATS] + self.FILE_FORMATS_NAMEXT =['{} ({})'.format(n,','.join(e)) for n,e in zip(self.FILE_FORMATS_NAMES,self.FILE_FORMATS_EXTENSIONS)] + + # --- ToolBar + tb = self.CreateToolBar(wx.TB_HORIZONTAL|wx.TB_TEXT|wx.TB_HORZ_LAYOUT) + self.toolBar = tb + self.comboFormats = wx.ComboBox(tb, choices = self.FILE_FORMATS_NAMEXT, style=wx.CB_READONLY) + self.comboFormats.SetSelection(0) + self.comboMode = wx.ComboBox(tb, choices = SEL_MODES, style=wx.CB_READONLY) + self.comboMode.SetSelection(0) + self.Bind(wx.EVT_COMBOBOX, self.onModeChange, self.comboMode ) + tb.AddSeparator() + tb.AddControl( wx.StaticText(tb, -1, 'Mode: ' ) ) + tb.AddControl( self.comboMode ) + tb.AddStretchableSpace() + tb.AddControl( wx.StaticText(tb, -1, 'Format: ' ) ) + tb.AddControl(self.comboFormats ) + tb.AddSeparator() + #bmp = wx.Bitmap('help.png') #wx.Bitmap("NEW.BMP", wx.BITMAP_TYPE_BMP) + TBAddTool(tb,"Open" ,wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN),self.onLoad) + TBAddTool(tb,"Reload",wx.ArtProvider.GetBitmap(wx.ART_REDO),self.onReload) + try: + TBAddTool(tb,"Add" ,wx.ArtProvider.GetBitmap(wx.ART_PLUS),self.onAdd) + except: + TBAddTool(tb,"Add" ,wx.ArtProvider.GetBitmap(wx.FILE_OPEN),self.onAdd) + #self.AddTBBitmapTool(tb,"Debug" ,wx.ArtProvider.GetBitmap(wx.ART_ERROR),self.onDEBUG) + tb.AddStretchableSpace() + tb.Realize() + + # --- Status bar + self.statusbar=self.CreateStatusBar(3, style=0) + self.statusbar.SetStatusWidths([200, -1, 70]) + + # --- Main Panel and Notebook + self.MainPanel = wx.Panel(self) + #self.MainPanel = wx.Panel(self, style=wx.RAISED_BORDER) + #self.MainPanel.SetBackgroundColour((200,0,0)) + + #self.nb = wx.Notebook(self.MainPanel) + #self.nb.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.on_tab_change) + + + sizer = wx.BoxSizer() + #sizer.Add(self.nb, 1, flag=wx.EXPAND) + self.MainPanel.SetSizer(sizer) + + # --- Drag and drop + dd = FileDropTarget(self) + self.SetDropTarget(dd) + + # --- Main Frame (self) + self.FrameSizer = wx.BoxSizer(wx.VERTICAL) + slSep = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) + self.FrameSizer.Add(slSep ,0, flag=wx.EXPAND|wx.BOTTOM,border=0) + self.FrameSizer.Add(self.MainPanel,1, flag=wx.EXPAND,border=0) + self.SetSizer(self.FrameSizer) + + self.SetSize(self.data['windowSize']) + self.Center() + self.Show() + self.Bind(wx.EVT_SIZE, self.OnResizeWindow) + self.Bind(wx.EVT_CLOSE, self.onClose) + + # Shortcuts + idFilter=wx.NewId() + self.Bind(wx.EVT_MENU, self.onFilter, id=idFilter) + + accel_tbl = wx.AcceleratorTable( + [(wx.ACCEL_CTRL, ord('F'), idFilter )] + ) + self.SetAcceleratorTable(accel_tbl) + + def onFilter(self,event): + if hasattr(self,'selPanel'): + self.selPanel.colPanel1.tFilter.SetFocus() + event.Skip() + + def clean_memory(self,bReload=False): + #print('Clean memory') + # force Memory cleanup + self.tabList.clean() + if not bReload: + if hasattr(self,'selPanel'): + self.selPanel.clean_memory() + if hasattr(self,'infoPanel'): + self.infoPanel.clean() + if hasattr(self,'plotPanel'): + self.plotPanel.cleanPlot() + gc.collect() + + def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False): + """ load multiple files, only trigger the plot at the end """ + if bReload: + if hasattr(self,'selPanel'): + self.selPanel.saveSelection() # TODO move to tables + + if not bAdd: + self.clean_memory(bReload=bReload) + + + if fileformats is None: + fileformats=[None]*len(filenames) + assert type(fileformats)==list, 'fileformats must be a list' + assert len(fileformats)==len(filenames), 'fileformats and filenames must have the same lengths' + + # Sorting files in alphabetical order in base_filenames order + base_filenames = [os.path.basename(f) for f in filenames] + I = np.argsort(base_filenames) + filenames = list(np.array(filenames)[I]) + fileformats = list(np.array(fileformats)[I]) + #filenames = [f for __, f in sorted(zip(base_filenames, filenames))] + + # Load the tables + warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformats=fileformats, bAdd=bAdd) + if bReload: + # Restore formulas that were previously added + for tab in self.tabList: + if tab.raw_name in self.restore_formulas.keys(): + for f in self.restore_formulas[tab.raw_name]: + tab.addColumnByFormula(f['name'], f['formula'], f['pos']-1) + self.restore_formulas = {} + # Display warnings + for warn in warnList: + Warn(self,warn) + # Load tables into the GUI + if self.tabList.len()>0: + self.load_tabs_into_GUI(bReload=bReload, bAdd=bAdd, bPlot=True) + + def load_df(self, df, name=None, bAdd=False, bPlot=True): + if bAdd: + self.tabList.append(Table(data=df, name=name)) + else: + self.tabList = TableList( [Table(data=df, name=name)] ) + self.load_tabs_into_GUI(bAdd=bAdd, bPlot=bPlot) + if hasattr(self,'selPanel'): + self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) + + def load_dfs(self, dfs, names, bAdd=False): + self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) + self.load_tabs_into_GUI(bAdd=bAdd, bPlot=True) + if hasattr(self,'selPanel'): + self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) + + def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): + if bAdd: + if not hasattr(self,'selPanel'): + bAdd=False + + if (not bReload) and (not bAdd): + self.cleanGUI() + self.Freeze() + # Setting status bar + self.setStatusBar() + + if bReload or bAdd: + self.selPanel.update_tabs(self.tabList) + else: + mode = SEL_MODES_ID[self.comboMode.GetSelection()] + #self.vSplitter = wx.SplitterWindow(self.nb) + self.vSplitter = wx.SplitterWindow(self.MainPanel) + self.selPanel = SelectionPanel(self.vSplitter, self.tabList, mode=mode, mainframe=self) + self.tSplitter = wx.SplitterWindow(self.vSplitter) + #self.tSplitter.SetMinimumPaneSize(20) + self.infoPanel = InfoPanel(self.tSplitter, data=self.data['infoPanel']) + self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, self) + self.tSplitter.SetSashGravity(0.9) + self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) + self.tSplitter.SetMinimumPaneSize(BOT_PANL) + self.tSplitter.SetSashGravity(1) + self.tSplitter.SetSashPosition(400) + + self.vSplitter.SplitVertically(self.selPanel, self.tSplitter) + self.vSplitter.SetMinimumPaneSize(SIDE_COL[0]) + self.tSplitter.SetSashPosition(SIDE_COL[0]) + + #self.nb.AddPage(self.vSplitter, "Plot") + #self.nb.SendSizeEvent() + + sizer = self.MainPanel.GetSizer() + sizer.Add(self.vSplitter, 1, flag=wx.EXPAND,border=0) + self.MainPanel.SetSizer(sizer) + self.FrameSizer.Layout() + + self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel1.comboX ) + self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel1.lbColumns) + self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel2.comboX ) + self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel2.lbColumns) + self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel3.comboX ) + self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel3.lbColumns) + self.Bind(wx.EVT_LISTBOX , self.onTabSelectionChange, self.selPanel.tabPanel.lbTab) + self.Bind(wx.EVT_SPLITTER_SASH_POS_CHANGED, self.onSashChangeMain, self.vSplitter) + + self.selPanel.tabPanel.lbTab.Bind(wx.EVT_RIGHT_DOWN, self.OnTabPopup) + + # plot trigger + if bPlot: + self.mainFrameUpdateLayout() + self.onColSelectionChange(event=None) + try: + self.Thaw() + except: + pass + # Hack + #self.onShowTool(tool='Filter') + #self.onShowTool(tool='Resample') + #self.onDataPlugin(toolName='Bin data') + + def setStatusBar(self, ISel=None): + nTabs=self.tabList.len() + if ISel is None: + ISel = list(np.arange(nTabs)) + if nTabs<0: + self.statusbar.SetStatusText('', 0) # Format + self.statusbar.SetStatusText('', 1) # Filenames + self.statusbar.SetStatusText('', 2) # Shape + elif nTabs==1: + self.statusbar.SetStatusText(self.tabList.get(0).fileformat_name, 0) + self.statusbar.SetStatusText(self.tabList.get(0).filename , 1) + self.statusbar.SetStatusText(self.tabList.get(0).shapestring, 2) + elif len(ISel)==1: + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).fileformat_name , 0) + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).filename , 1) + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).shapestring, 2) + else: + self.statusbar.SetStatusText('' ,0) + self.statusbar.SetStatusText(", ".join(list(set([self.tabList.filenames[i] for i in ISel]))),1) + self.statusbar.SetStatusText('',2) + + def renameTable(self, iTab, newName): + oldName = self.tabList.renameTable(iTab, newName) + self.selPanel.renameTable(iTab, oldName, newName) + + def sortTabs(self, method='byName'): + self.tabList.sort(method=method) + # Updating tables + self.selPanel.update_tabs(self.tabList) + # Trigger a replot + self.onTabSelectionChange() + + + def deleteTabs(self, I): + self.tabList.deleteTabs(I) + + # Invalidating selections + self.selPanel.tabPanel.lbTab.SetSelection(-1) + # Until we have something better, we empty plot + self.plotPanel.empty() + self.infoPanel.empty() + self.selPanel.clean_memory() + # Updating tables + self.selPanel.update_tabs(self.tabList) + # Trigger a replot + self.onTabSelectionChange() + + def exportTab(self, iTab): + tab=self.tabList.get(iTab) + default_filename=tab.basename +'.csv' + with wx.FileDialog(self, "Save to CSV file",defaultFile=default_filename, + style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as dlg: + #, wildcard="CSV files (*.csv)|*.csv", + dlg.CentreOnParent() + if dlg.ShowModal() == wx.ID_CANCEL: + return # the user changed their mind + tab.export(dlg.GetPath()) + + def onShowTool(self, event=None, tool=''): + """ + Show tool + tool in 'Outlier', 'Filter', 'LogDec','FASTRadialAverage', 'Mask', 'CurveFitting' + """ + if not hasattr(self,'plotPanel'): + Error(self,'Plot some data first') + return + self.plotPanel.showTool(tool) + + def onDataPlugin(self, event=None, toolName=''): + """ + Dispatcher to apply plugins to data: + - simple plugins are directly exectued + - plugins that are panels are sent over to plotPanel to show them + TODO merge with onShowTool + """ + if not hasattr(self,'plotPanel'): + Error(self,'Plot some data first') + return + + for thisToolName, function, isPanel in dataPlugins: + if toolName == thisToolName: + if isPanel: + panelClass = function(self, event, toolName) # getting panelClass + self.plotPanel.showToolPanel(panelClass) + else: + function(self, event, toolName) # calling the data function + return + raise NotImplementedError('Tool: ',toolName) + + + def onSashChangeMain(self,event=None): + pass + # doent work because size is not communicated yet + #if hasattr(self,'selPanel'): + # print('ON SASH') + # self.selPanel.setEquiSash(event) + + def OnTabPopup(self,event): + menu = TablePopup(self,self.selPanel.tabPanel.lbTab) + self.PopupMenu(menu, event.GetPosition()) + menu.Destroy() + + def onTabSelectionChange(self,event=None): + # TODO This can be cleaned-up + ISel=self.selPanel.tabPanel.lbTab.GetSelections() + if len(ISel)>0: + # Letting seletion panel handle the change + self.selPanel.tabSelectionChanged() + # Update of status bar + self.setStatusBar(ISel) + # Trigger the colSelection Event + self.onColSelectionChange(event=None) + + def onColSelectionChange(self,event=None): + if hasattr(self,'plotPanel'): + # Letting selection panel handle the change + self.selPanel.colSelectionChanged() + # Redrawing + self.plotPanel.load_and_draw() + # --- Stats trigger + #self.showStats() + + def redraw(self): + if hasattr(self,'plotPanel'): + self.plotPanel.load_and_draw() +# def showStats(self): +# self.infoPanel.showStats(self.plotPanel.plotData,self.plotPanel.pltTypePanel.plotType()) + + def onExit(self, event): + self.Close() + + def onClose(self, event): + saveAppData(self, self.data) + event.Skip() + + def cleanGUI(self, event=None): + if hasattr(self,'plotPanel'): + del self.plotPanel + if hasattr(self,'selPanel'): + del self.selPanel + if hasattr(self,'infoPanel'): + del self.infoPanel + #self.deletePages() + try: + self.MainPanel.GetSizer().Clear(delete_windows=True) # Delete Windows + except: + self.MainPanel.GetSizer().Clear() + self.FrameSizer.Layout() + gc.collect() + + def onSave(self, event=None): + # using the navigation toolbar save functionality + self.plotPanel.navTB.save_figure() + + def onAbout(self, event=None): + defaultDir = weio.defaultUserDataDir() # TODO input file options + About(self,PROG_NAME+' '+PROG_VERSION+'\n\n' + 'pyDatView config file:\n {}\n'.format(configFilePath())+ + 'weio data directory: \n {}\n'.format(os.path.join(defaultDir,'weio'))+ + '\n\nVisit http://github.com/ebranlard/pyDatView for documentation.') + + def onReset (self, event=None): + configFile = configFilePath() + result = YesNo(self, + 'The options of pyDatView will be reset to default.\nThe changes will be noticeable the next time you open pyDatView.\n\n'+ + 'This action will overwrite the user settings file:\n {}\n\n'.format(configFile)+ + 'pyDatView will then close.\n\n' + 'Are you sure you want to continue?', caption = 'Reset settings?') + if result: + try: + os.remove(configFile) + except: + pass + self.data = defaultAppData(self) + self.datareset = True + self.onExit(event=None) + + def onReload(self, event=None): + filenames, fileformats = self.tabList.filenames_and_formats + if len(filenames)>0: + # Save formulas to restore them after reload with sorted tabs + self.restore_formulas = {} + for tab in self.tabList._tabs: + f = tab.formulas # list of dict('pos','formula','name') + f = sorted(f, key=lambda k: k['pos']) # Sort formulae by position in list of formua + self.restore_formulas[tab.raw_name]=f # we use raw_name as key + # Actually load files (read and add in GUI) + self.load_files(filenames, fileformats=fileformats, bReload=True,bAdd=False) + else: + Error(self,'Open one or more file first.') + + def onDEBUG(self, event=None): + #self.clean_memory() + self.plotPanel.ctrlPanel.Refresh() + self.plotPanel.cb_sizer.ForceRefresh() + + def onExport(self, event=None): + ISel=[] + try: + ISel = self.selPanel.tabPanel.lbTab.GetSelections() + except: + pass + if len(ISel)>0: + self.exportTab(ISel[0]) + else: + Error(self,'Open a file and select a table first.') + + def onLoad(self, event=None): + self.selectFile(bAdd=False) + + def onAdd(self, event=None): + self.selectFile(bAdd=self.tabList.len()>0) + + def selectFile(self,bAdd=False): + # --- File Format extension + iFormat=self.comboFormats.GetSelection() + sFormat=self.comboFormats.GetStringSelection() + if iFormat==0: # auto-format + Format = None + #wildcard = 'all (*.*)|*.*' + wildcard='|'.join([n+'|*'+';*'.join(e) for n,e in zip(self.FILE_FORMATS_NAMEXT,self.FILE_FORMATS_EXTENSIONS)]) + #wildcard = sFormat + extensions+'|all (*.*)|*.*' + else: + Format = FILE_FORMATS[iFormat-1] + extensions = '|*'+';*'.join(FILE_FORMATS[iFormat-1].extensions) + wildcard = sFormat + extensions+'|all (*.*)|*.*' + + with wx.FileDialog(self, "Open file", wildcard=wildcard, + style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST | wx.FD_MULTIPLE) as dlg: + #other options: wx.CHANGE_DIR + #dlg.SetSize((100,100)) + #dlg.Center() + if dlg.ShowModal() == wx.ID_CANCEL: + return # the user changed their mind + self.load_files(dlg.GetPaths(),fileformat=Format,bAdd=bAdd) + + def onModeChange(self, event=None): + if hasattr(self,'selPanel'): + self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) + self.mainFrameUpdateLayout() + # --- Trigger to check number of columns + self.onTabSelectionChange() + + def mainFrameUpdateLayout(self, event=None): + if hasattr(self,'selPanel'): + nWind=self.selPanel.splitter.nWindows + if self.Size[0]<=800: + sash=SIDE_COL[nWind] + else: + sash=SIDE_COL_LARGE[nWind] + self.resizeSideColumn(sash) + + def OnResizeWindow(self, event): + try: + self.mainFrameUpdateLayout() + self.Layout() + except: + pass + # NOTE: doesn't work... + #if hasattr(self,'plotPanel'): + # Subplot spacing changes based on figure size + #print('>>> RESIZE WINDOW') + #self.redraw() + + # --- Side column + def resizeSideColumn(self,width): + # To force the replot we do an epic unsplit/split... + #self.vSplitter.Unsplit() + #self.vSplitter.SplitVertically(self.selPanel, self.tSplitter) + self.vSplitter.SetMinimumPaneSize(width) + self.vSplitter.SetSashPosition(width) + #self.selPanel.splitter.setEquiSash() + + # --- NOTEBOOK + #def deletePages(self): + # for index in reversed(range(self.nb.GetPageCount())): + # self.nb.DeletePage(index) + # self.nb.SendSizeEvent() + # gc.collect() + #def on_tab_change(self, event=None): + # page_to_select = event.GetSelection() + # wx.CallAfter(self.fix_focus, page_to_select) + # event.Skip(True) + #def fix_focus(self, page_to_select): + # page = self.nb.GetPage(page_to_select) + # page.SetFocus() + +#---------------------------------------------------------------------- +def MyExceptionHook(etype, value, trace): + """ + Handler for all unhandled exceptions. + :param `etype`: the exception type (`SyntaxError`, `ZeroDivisionError`, etc...); + :type `etype`: `Exception` + :param string `value`: the exception error message; + :param string `trace`: the traceback header, if any (otherwise, it prints the + standard Python header: ``Traceback (most recent call last)``. + """ + from wx._core import wxAssertionError + # Printing exception + traceback.print_exception(etype, value, trace) + if etype==wxAssertionError: + if wx.Platform == '__WXMAC__': + # We skip these exceptions on macos (likely bitmap size 0) + return + # Then showing to user the last error + frame = wx.GetApp().GetTopWindow() + tmp = traceback.format_exception(etype, value, trace) + if tmp[-1].find('Exception: Error:')==0: + Error(frame,tmp[-1][18:]) + elif tmp[-1].find('Exception: Warn:')==0: + Warn(frame,tmp[-1][17:]) + else: + exception = 'The following exception occured:\n\n'+ tmp[-1] + '\n'+tmp[-2].strip() + Error(frame,exception) + try: + frame.Thaw() # Make sure any freeze event is stopped + except: + pass + +# --------------------------------------------------------------------------------} +# --- Tests +# --------------------------------------------------------------------------------{ +def test(filenames=None): + if filenames is not None: + app = wx.App(False) + frame = MainFrame() + frame.load_files(filenames,fileformat=None) + return + +# --------------------------------------------------------------------------------} +# --- Wrapped WxApp +# --------------------------------------------------------------------------------{ +class MyWxApp(wx.App): + def __init__(self, redirect=False, filename=None): + try: + wx.App.__init__(self, redirect, filename) + except: + if wx.Platform == '__WXMAC__': + #msg = """This program needs access to the screen. + # Please run with 'pythonw', not 'python', and only when you are logged + # in on the main display of your Mac.""" + msg= """ +MacOS Error: + This program needs access to the screen. Please run with a + Framework build of python, and only when you are logged in + on the main display of your Mac. + +pyDatView help: + You see the error above because you are using a Mac and + the python executable you are using does not have access to + your screen. This is a Mac issue, not a pyDatView issue. + Instead of calling 'python pyDatView.py', you need to find + another python and do '/path/python pyDatView.py' + You can try './pythonmac pyDatView.py', a script provided + in this repository to detect the path (in some cases) + + You can find additional help in the file 'README.md'. + + For quick reference, here are some typical cases: + - Your python was installed with 'brew', then likely use + /usr/lib/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX; + - Your python is an anaconda python, use something like:; + /anaconda3/bin/python.app (NOTE: the '.app'! + - You are using a python 2 version, you can use the system one: + /Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX + /System/Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX +""" + + elif wx.Platform == '__WXGTK__': + msg =""" +Error: + Unable to access the X Display, is $DISPLAY set properly? + +pyDatView help: + You are probably running this application on a server accessed via ssh. + Use `ssh -X` or `ssh -Y` to access the server. + Else, try setting up $DISPLAY before doing the ssh connection. +""" + else: + msg = 'Unable to create GUI' # TODO: more description is needed for wxMSW... + raise SystemExit(msg) + +# --------------------------------------------------------------------------------} +# --- Mains +# --------------------------------------------------------------------------------{ +def showApp(firstArg=None,dataframe=None,filenames=[]): + """ + The main function to start the data frame GUI. + """ + app = MyWxApp(False) + frame = MainFrame() + # Optional first argument + if firstArg is not None: + if isinstance(firstArg,list): + filenames=firstArg + elif isinstance(firstArg,str): + filenames=[firstArg] + elif isinstance(firstArg, pd.DataFrame): + dataframe=firstArg + # + if (dataframe is not None) and (len(dataframe)>0): + #import time + #tstart = time.time() + frame.load_df(dataframe) + #tend = time.time() + #print('PydatView time: ',tend-tstart) + elif len(filenames)>0: + frame.load_files(filenames, fileformats=None) + app.MainLoop() + +def cmdline(): + if len(sys.argv)>1: + pydatview(filename=sys.argv[1]) + else: + pydatview() diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index a97417c..ac147da 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -1,796 +1,808 @@ -from __future__ import absolute_import -import os -import numpy as np -from .common import no_unit, unit, inverse_unit, has_chinese_char -from .common import isString, isDate, getDt -from .common import unique, pretty_num, pretty_time -from .GUIMeasure import find_closest # Should not depend on wx - -class PlotData(): - """ - Class for plot data - - For now, relies on some "indices" related to Tables/Columns/ and maybe Selection panel - Not really elegant. These dependencies should be removed in the future - """ - def __init__(PD, x=None, y=None, sx='', sy=''): - """ Dummy init for now """ - PD.id=-1 - PD.it=-1 # tablx index - PD.ix=-1 # column index - PD.iy=-1 # column index - PD.sx='' # x label - PD.sy='' # y label - PD.st='' # table label - PD.syl='' # y label for legend - PD.filename = '' - PD.tabname = '' - PD.x =[] # x data - PD.y =[] # y data - PD.xIsString=False # true if strings - PD.xIsDate =False # true if dates - PD.yIsString=False # true if strings - PD.yIsDate =False # true if dates - - if x is not None and y is not None: - PD.fromXY(x,y,sx,sy) - - def fromIDs(PD, tabs, i, idx, SameCol, Options={}): - """ Nasty initialization of plot data from "IDs" """ - PD.id = i - PD.it = idx[0] # table index - PD.ix = idx[1] # x index - PD.iy = idx[2] # y index - PD.sx = idx[3] # x label - PD.sy = idx[4] # y label - PD.syl = '' # y label for legend - PD.st = idx[5] # table label - PD.filename = tabs[PD.it].filename - PD.tabname = tabs[PD.it].active_name - PD.SameCol = SameCol - PD.x, PD.xIsString, PD.xIsDate,_ = tabs[PD.it].getColumn(PD.ix) # actual x data, with info - PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info - PD.c =c # raw values, used by PDF - - PD._post_init(Options=Options) - - def fromXY(PD, x, y, sx='', sy=''): - PD.x = x - PD.y = y - PD.c = y - PD.sx = sx - PD.sy = sy - PD.xIsString = isString(x) - PD.yIsString = isString(y) - PD.xIsDate = isDate (x) - PD.yIsDate = isDate (y) - - PD._post_init() - - - def _post_init(PD, Options={}): - # --- Perform data manipulation on the fly - #print(Options) - keys=Options.keys() - if 'RemoveOutliers' in keys: - if Options['RemoveOutliers']: - from pydatview.tools.signal import reject_outliers - try: - PD.x, PD.y = reject_outliers(PD.y, PD.x, m=Options['OutliersMedianDeviation']) - except: - raise Exception('Warn: Outlier removal failed. Desactivate it or use a different signal. ') - if 'Filter' in keys: - if Options['Filter']: - from pydatview.tools.signal import applyFilter - PD.y = applyFilter(PD.x, PD.y, Options['Filter']) - - if 'Sampler' in keys: - if Options['Sampler']: - from pydatview.tools.signal import applySampler - PD.x, PD.y = applySampler(PD.x, PD.y, Options['Sampler']) - - # --- Store stats - n=len(PD.y) - if n>1000: - if (PD.xIsString): - raise Exception('Error: x values contain more than 1000 string. This is not suitable for plotting.\n\nPlease select another column for table: {}\nProblematic column: {}\n'.format(PD.st,PD.sx)) - if (PD.yIsString): - raise Exception('Error: y values contain more than 1000 string. This is not suitable for plotting.\n\nPlease select another column for table: {}\nProblematic column: {}\n'.format(PD.st,PD.sy)) - - PD.needChineseFont = has_chinese_char(PD.sy) or has_chinese_char(PD.sx) - # Stats of the raw data (computed once and for all, since it can be expensive for large dataset - PD.computeRange() - # Store the values of the original data (labelled "0"), since the data might be modified later by PDF or MinMax etc. - PD._y0Min = PD._yMin - PD._y0Max = PD._yMax - PD._x0Min = PD._xMin - PD._x0Max = PD._xMax - PD._x0AtYMin = PD._xAtYMin - PD._x0AtYMax = PD._xAtYMax - PD._y0Std = PD.yStd() - PD._y0Mean = PD.yMean() - PD._n0 = (n,'{:d}'.format(n)) - PD.x0 =PD.x - PD.y0 =PD.y - # Store xyMeas input values so we don't need to recompute xyMeas in case they didn't change - PD.xyMeasInput1, PD.xyMeasInput2 = None, None - PD.xyMeas1, PD.xyMeas2 = None, None - - def __repr__(s): - s1='id:{}, it:{}, ix:{}, iy:{}, sx:"{}", sy:"{}", st:{}, syl:{}\n'.format(s.id,s.it,s.ix,s.iy,s.sx,s.sy,s.st,s.syl) - return s1 - - def toPDF(PD, nBins=30, smooth=False): - """ Convert y-data to Probability density function (PDF) as function of x - Uses "stats" library (from welib/pybra) - NOTE: inPlace - """ - from pydatview.tools.stats import pdf_gaussian_kde, pdf_histogram - - n=len(PD.y) - if PD.yIsString: - if n>100: - raise Exception('Warn: Dataset has string format and is too large to display') - vc = PD.c.value_counts().sort_index() - PD.x = vc.keys().tolist() - PD.y = vc/n # TODO counts/PDF option - PD.yIsString=False - PD.xIsString=True - elif PD.yIsDate: - raise Exception('Warn: Cannot plot PDF of dates') - else: - if nBins>=n: - nBins=n - if smooth: - try: - PD.x, PD.y = pdf_gaussian_kde(PD.y, nOut=nBins) - except np.linalg.LinAlgError as e: - PD.x, PD.y = pdf_histogram(PD.y, nBins=nBins, norm=True, count=False) - else: - PD.x, PD.y = pdf_histogram(PD.y, nBins=nBins, norm=True, count=False) - PD.xIsString=False - PD.yIsString=False - - PD.sx = PD.sy; - PD.sy = 'PDF('+no_unit(PD.sy)+')' - iu = inverse_unit(PD.sy) - if len(iu)>0: - PD.sy += ' ['+ iu +']' - - # Compute min max once and for all - PD.computeRange() - - return nBins - - - def toMinMax(PD, xScale=False, yScale=True): - """ Convert plot data to MinMax data based on GUI options - NOTE: inPlace - """ - if yScale: - if PD.yIsString: - raise Exception('Warn: Cannot compute min-max for strings') - mi = PD._y0Min[0] #mi= np.nanmin(PD.y) - mx = PD._y0Max[0] #mx= np.nanmax(PD.y) - if mi == mx: - PD.y=PD.y*0 - else: - PD.y = (PD.y-mi)/(mx-mi) - PD._yMin=0,'0' - PD._yMax=1,'1' - if xScale: - if PD.xIsString: - raise Exception('Warn: Cannot compute min-max for strings') - mi= PD._x0Min[0] - mx= PD._x0Max[0] - if mi == mx: - PD.x=PD.x*0 - else: - PD.x = (PD.x-mi)/(mx-mi) - PD._xMin=0,'0' - PD._xMax=1,'1' - - # Compute min max once and for all - #PD.computeRange() - - return None - - - def toFFT(PD, yType='Amplitude', xType='1/x', avgMethod='Welch', avgWindow='Hamming', bDetrend=True, nExp=8): - """ - Uses spectral.fft_wrap to generate a "FFT" plot data, with various options: - yType : amplitude, PSD, f x PSD - xType : 1/x, x, 2pi/x - avgMethod : None, Welch - avgWindow : Hamming, Hann, Rectangular - see module spectral for more - - NOTE: inplace (modifies itself), does not return a new instance - """ - from pydatview.tools.spectral import fft_wrap - - # --- TODO, make this independent of GUI - if PD.yIsString or PD.yIsDate: - raise Exception('Warn: Cannot plot FFT of dates or strings') - elif PD.xIsString: - raise Exception('Warn: Cannot plot FFT if x axis is string') - - dt=None - if PD.xIsDate: - dt = getDt(PD.x) - # --- Computing fft - x is freq, y is Amplitude - PD.x, PD.y, Info = fft_wrap(PD.x, PD.y, dt=dt, output_type=yType,averaging=avgMethod, averaging_window=avgWindow,detrend=bDetrend,nExp=nExp) - # --- Setting plot options - PD._Info=Info - PD.xIsDate=False - # y label - if yType=='PSD': - PD.sy= 'PSD({}) [({})^2/{}]'.format(no_unit(PD.sy), unit(PD.sy), unit(PD.sx)) - elif yType=='f x PSD': - PD.sy= 'f-weighted PSD({}) [({})^2]'.format(no_unit(PD.sy), unit(PD.sy)) - elif yType=='Amplitude': - PD.sy= 'FFT({}) [{}]'.format(no_unit(PD.sy), unit(PD.sy)) - else: - raise Exception('Unsupported FFT type {} '.format(yType)) - # x label - if xType=='1/x': - if unit(PD.sx)=='s': - PD.sx= 'Frequency [Hz]' - else: - PD.sx= '' - elif xType=='x': - PD.x=1/PD.x - if unit(PD.sx)=='s': - PD.sx= 'Period [s]' - else: - PD.sx= '' - elif xType=='2pi/x': - PD.x=2*np.pi*PD.x - if unit(PD.sx)=='s': - PD.sx= 'Cyclic frequency [rad/s]' - else: - PD.sx= '' - else: - raise Exception('Unsupported x-type {} '.format(xType)) - - PD.computeRange() - return Info - - def computeRange(PD): - """ Compute min max of data once and for all and store - From the performance tests, this ends up having a non negligible cost for large dataset, - so we store it to reuse these as much as possible. - If possible, should be used for the plotting as well, so that matplotlib don't - have to compute them again - NOTE: each variable is a tuple (v,s), with a float and its string representation - """ - PD._xMin = PD._xMinCalc() - PD._xMax = PD._xMaxCalc() - PD._yMin = PD._yMinCalc() - PD._yMax = PD._yMaxCalc() - PD._xAtYMin = PD._xAtYMinCalc(PD._yMin[0]) - PD._xAtYMax = PD._xAtYMaxCalc(PD._yMax[0]) - - - # --------------------------------------------------------------------------------} - # --- Stats functions that should only becalled once, could maybe use @attributes.. - # --------------------------------------------------------------------------------{ - def _yMinCalc(PD): - if PD.yIsString: - return PD.y[0],PD.y[0].strip() - elif PD.yIsDate: - return PD.y[0],'{}'.format(PD.y[0]) - else: - v=np.nanmin(PD.y) - s=pretty_num(v) - return (v,s) - - def _yMaxCalc(PD): - if PD.yIsString: - return PD.y[-1],PD.y[-1].strip() - elif PD.yIsDate: - return PD.y[-1],'{}'.format(PD.y[-1]) - else: - v=np.nanmax(PD.y) - s=pretty_num(v) - return (v,s) - - def _xAtYMinCalc(PD, yMin): - if PD.xIsString: - return PD.x[0],PD.x[0].strip() - elif PD.xIsDate: - return PD.x[0],'{}'.format(PD.x[0]) - else: - v = PD.x[np.where(PD.y == yMin)[0][0]] - s=pretty_num(v) - return (v,s) - - def _xAtYMaxCalc(PD, yMax): - if PD.xIsString: - return PD.x[-1],PD.x[-1].strip() - elif PD.xIsDate: - return PD.x[-1],'{}'.format(PD.x[-1]) - else: - v = PD.x[np.where(PD.y == yMax)[0][0]] - s=pretty_num(v) - return (v,s) - - def _xMinCalc(PD): - if PD.xIsString: - return PD.x[0],PD.x[0].strip() - elif PD.xIsDate: - return PD.x[0],'{}'.format(PD.x[0]) - else: - v=np.nanmin(PD.x) - s=pretty_num(v) - return (v,s) - - def _xMaxCalc(PD): - if PD.xIsString: - return PD.x[-1],PD.x[-1].strip() - elif PD.xIsDate: - return PD.x[-1],'{}'.format(PD.x[-1]) - else: - v=np.nanmax(PD.x) - s=pretty_num(v) - return (v,s) - - def xMin(PD): - return PD._xMin - - def xMax(PD): - return PD._xMax - - def xAtYMin(PD): - return PD._xAtYMin - - def xAtYMax(PD): - return PD._xAtYMax - - def yMin(PD): - return PD._yMin - - def yMax(PD): - return PD._yMax - - def y0Min(PD): - return PD._y0Min - - def y0Max(PD): - return PD._y0Max - - def y0Mean(PD): - return PD._y0Mean - - def y0Std(PD): - return PD._y0Std - - def n0(PD): - return PD._n0 - - # --------------------------------------------------------------------------------} - # --- Stats functions - # --------------------------------------------------------------------------------{ - def yMean(PD): - if PD.yIsString or PD.yIsDate: - return None,'NA' - else: - v=np.nanmean(PD.y) - s=pretty_num(v) - return (v,s) - - def yMedian(PD): - if PD.yIsString or PD.yIsDate: - return None,'NA' - else: - v=np.nanmedian(PD.y) - s=pretty_num(v) - return (v,s) - - def yStd(PD): - if PD.yIsString or PD.yIsDate: - return None,'NA' - else: - v=np.nanstd(PD.y) - s=pretty_num(v) - return (v,s) - - def yName(PD): - return PD.sy, PD.sy - - def fileName(PD): - return os.path.basename(PD.filename), os.path.basename(PD.filename) - - def baseDir(PD): - return os.path.dirname(PD.filename),os.path.join(os.path.dirname(PD.filename),'') - - def tabName(PD): - return PD.tabname, PD.tabname - - def ylen(PD): - v=len(PD.y) - s='{:d}'.format(v) - return v,s - - - def y0Var(PD): - if PD._y0Std[0] is not None: - v=PD._y0Std[0]**2 - s=pretty_num(v) - else: - v=None - s='NA' - return v,s - - def y0TI(PD): - v=PD._y0Std[0]/PD._y0Mean[0] - s=pretty_num(v) - return v,s - - - def yRange(PD): - if PD.yIsString: - return 'NA','NA' - elif PD.yIsDate: - dtAll=getDt([PD.x[-1]-PD.x[0]]) - return '',pretty_time(dtAll) - else: - v=np.nanmax(PD.y)-np.nanmin(PD.y) - s=pretty_num(v) - return v,s - - def yAbsMax(PD): - if PD.yIsString or PD.yIsDate: - return 'NA','NA' - else: - v=max(np.abs(PD._y0Min[0]),np.abs(PD._y0Max[0])) - s=pretty_num(v) - return v,s - - - def xRange(PD): - if PD.xIsString: - return 'NA','NA' - elif PD.xIsDate: - dtAll=getDt([PD.x[-1]-PD.x[0]]) - return '',pretty_time(dtAll) - else: - v=np.nanmax(PD.x)-np.nanmin(PD.x) - s=pretty_num(v) - return v,s - - - def inty(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y,x=PD.x) - s=pretty_num(v) - return v,s - - def intyintdx(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y,x=PD.x)/np.trapz(y=PD.x*0+1,x=PD.x) - s=pretty_num(v) - return v,s - - def intyx1(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y*PD.x,x=PD.x) - s=pretty_num(v) - return v,s - - def intyx1_scaled(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y*PD.x,x=PD.x) - v=v/np.trapz(y=PD.y,x=PD.x) - s=pretty_num(v) - return v,s - - def intyx2(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y*PD.x**2,x=PD.x) - s=pretty_num(v) - return v,s - - def meas1(PD, xymeas1, xymeas2): - if PD.xyMeasInput1 is not None and PD.xyMeasInput1 == xymeas1: - yv = PD.xyMeas1[1] - s = pretty_num(yv) - else: - xv, yv, s = PD._meas(xymeas1) - PD.xyMeas1 = [xv, yv] - PD.xyMeasInput1 = xymeas1 - return yv, s - - def meas2(PD, xymeas1, xymeas2): - if PD.xyMeasInput2 is not None and PD.xyMeasInput2 == xymeas2: - yv = PD.xyMeas2[1] - s = pretty_num(yv) - else: - xv, yv, s = PD._meas(xymeas2) - PD.xyMeas2 = [xv, yv] - PD.xyMeasInput2 = xymeas2 - return yv, s - - def yMeanMeas(PD): - return PD._measCalc('mean') - - def yMinMeas(PD): - return PD._measCalc('min') - - def yMaxMeas(PD): - return PD._measCalc('max') - - def xAtYMinMeas(PD): - return PD._measCalc('xmin') - - def xAtYMaxMeas(PD): - return PD._measCalc('xmax') - - def _meas(PD, xymeas): - try: - xv, yv = 'NA', 'NA' - xy = np.array([PD.x, PD.y]).transpose() - points = find_closest(xy, [xymeas[0], xymeas[1]], False) - if points.ndim == 1: - xv, yv = points[0:2] - s = pretty_num(yv) - else: - xv, yv = points[0, 0], points[0, 1] - s = ' / '.join([str(p) for p in points[:, 1]]) - except (IndexError, TypeError): - xv, yv = 'NA', 'NA' - s='NA' - return xv, yv, s - - def _measCalc(PD, mode): - if PD.xyMeas1 is None or PD.xyMeas2 is None: - return 'NA', 'NA' - try: - v = 'NA' - left_index = np.where(PD.x == PD.xyMeas1[0])[0][0] - right_index = np.where(PD.x == PD.xyMeas2[0])[0][0] - if left_index == right_index: - raise IndexError - if left_index > right_index: - left_index, right_index = right_index, left_index - if mode == 'mean': - v = np.nanmean(PD.y[left_index:right_index]) - elif mode == 'min': - v = np.nanmin(PD.y[left_index:right_index]) - elif mode == 'max': - v = np.nanmax(PD.y[left_index:right_index]) - elif mode == 'xmin': - v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmin(PD.y[left_index:right_index]))[0][0]] - elif mode == 'xmax': - v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmax(PD.y[left_index:right_index]))[0][0]] - else: - raise NotImplementedError('Error: Mode ' + mode + ' not implemented') - s = pretty_num(v) - except (IndexError, TypeError): - v = 'NA' - s = 'NA' - return v, s - - def dx(PD): - if len(PD.x)<=1: - return 'NA','NA' - if PD.xIsString: - return None,'NA' - elif PD.xIsDate: - dt=getDt(PD.x) - return dt,pretty_time(dt) - else: - v=PD.x[1]-PD.x[0] - s=pretty_num(v) - return v,s - - def xMax(PD): - if PD.xIsString: - return PD.x[-1],PD.x[-1] - elif PD.xIsDate: - return PD.x[-1],'{}'.format(PD.x[-1]) - else: - v=np.nanmax(PD.x) - s=pretty_num(v) - return v,s - def xMin(PD): - if PD.xIsString: - return PD.x[0],PD.x[0] - elif PD.xIsDate: - return PD.x[0],'{}'.format(PD.x[0]) - else: - v=np.nanmin(PD.x) - s=pretty_num(v) - return v,s - - def leq(PD,m): - from pydatview.tools.fatigue import eq_load - if PD.yIsString or PD.yIsDate: - return 'NA','NA' - else: - T,_=PD.xRange() - v=eq_load(PD.y, m=m, neq=T)[0][0] - return v,pretty_num(v) - - def Info(PD,var): - if var=='LSeg': - return '','{:d}'.format(PD._Info.LSeg) - elif var=='LWin': - return '','{:d}'.format(PD._Info.LWin) - elif var=='LOvlp': - return '','{:d}'.format(PD._Info.LOvlp) - elif var=='nFFT': - return '','{:d}'.format(PD._Info.nFFT) - - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -def compareMultiplePD(PD, mode, sComp): - """ - PD: list of PlotData - sComp: string in ['Relative', '|Relative|', 'Ratio', 'Absolute' - mode: plot mode, nTabs_1Col, nTabs_SameCols, nTabs_SimCols - - return: - PD_comp : new PlotData list that compares the input list PD - - """ - # --- Helper function - def getError(y,yref,method): - if len(y)!=len(yref): - raise NotImplementedError('Cannot compare signals of different lengths') - if sComp=='Relative': - if np.mean(np.abs(yref))<1e-7: - Error=(y-yRef)/(yRef+1)*100 - else: - Error=(y-yRef)/yRef*100 - elif sComp=='|Relative|': - if np.mean(np.abs(yref))<1e-7: - Error=abs((y-yRef)/(yRef+1))*100 - else: - Error=abs((y-yRef)/yRef)*100 - elif sComp=='Ratio': - if np.mean(np.abs(yref))<1e-7: - Error=(y+1)/(yRef+1) - else: - Error=y/yRef - elif sComp=='Absolute': - Error=y-yRef - else: - raise Exception('Something wrong '+sComp) - return Error - - def getErrorLabel(ylab=''): - if len(ylab)>0: - ylab=no_unit(ylab) - ylab='in '+ylab+' ' - if sComp=='Relative': - return 'Relative error '+ylab+'[%]'; - elif sComp=='|Relative|': - return 'Abs. relative error '+ylab+'[%]'; - if sComp=='Ratio': - return 'Ratio '+ylab.replace('in','of')+'[-]'; - elif sComp=='Absolute': - usy = unique([pd.sy for pd in PD]) - yunits= unique([unit(sy) for sy in usy]) - if len(yunits)==1 and len(yunits[0])>0: - return 'Absolute error '+ylab+'['+yunits[0]+']' - else: - return 'Absolute error '+ylab; - elif sComp=='Y-Y': - return PD[0].sy - - xlabelAll=PD[0].sx - - - if any([pd.yIsString for pd in PD]): - raise Exception('Warn: Cannot compare strings') - if any([pd.yIsDate for pd in PD]): - raise Exception('Warn: Cannot compare dates with other values') - - if mode=='nTabs_1Col': - ylabelAll=getErrorLabel(PD[1].sy) - usy = unique([pd.sy for pd in PD]) - #print('Compare - different tabs - 1 col') - st = [pd.st for pd in PD] - if len(usy)==1: - SS=usy[0] + ', '+ ' wrt. '.join(st[::-1]) - if sComp=='Y-Y': - xlabelAll=PD[0].st+', '+PD[0].sy - ylabelAll=PD[1].st+', '+PD[1].sy - else: - SS=' wrt. '.join(usy[::-1]) - if sComp=='Y-Y': - xlabelAll=PD[0].sy - ylabelAll=PD[1].sy - - xRef = PD[0].x - yRef = PD[0].y - PD[1].syl=SS - y=np.interp(xRef,PD[1].x,PD[1].y) - if sComp=='Y-Y': - PD[1].x=yRef - PD[1].y=y - else: - Error = getError(y,yRef,sComp) - PD[1].x=xRef - PD[1].y=Error - PD[1].sx=xlabelAll - PD[1].sy=ylabelAll - PD_comp=[PD[1]] # return - - elif mode=='1Tab_nCols': - # --- Compare one table - different columns - #print('One Tab, different columns') - ylabelAll=getErrorLabel() - xRef = PD[0].x - yRef = PD[0].y - pdRef=PD[0] - for pd in PD[1:]: - if sComp=='Y-Y': - pd.syl = no_unit(pd.sy)+' wrt. '+no_unit(pdRef.sy) - pd.x = yRef - pd.sx = PD[0].sy - else: - pd.syl = no_unit(pd.sy)+' wrt. '+no_unit(pdRef.sy) - pd.sx = xlabelAll - pd.sy = ylabelAll - Error = getError(pd.y,yRef,sComp) - pd.x=xRef - pd.y=Error - PD_comp=PD[1:] - elif mode =='nTabs_SameCols': - # --- Compare different tables, same column - #print('Several Tabs, same columns') - uiy=unique([pd.iy for pd in PD]) - uit=unique([pd.it for pd in PD]) - PD_comp=[] - for iy in uiy: - PD_SameCol=[pd for pd in PD if pd.iy==iy] - xRef = PD_SameCol[0].x - yRef = PD_SameCol[0].y - ylabelAll=getErrorLabel(PD_SameCol[0].sy) - for pd in PD_SameCol[1:]: - if pd.xIsString: - if len(xRef)==len(pd.x): - pass # fine able to interpolate - else: - raise Exception('X values have different length and are strings, cannot interpolate string. Use `Index` for x instead.') - else: - pd.y=np.interp(xRef,pd.x,pd.y) - if sComp=='Y-Y': - pd.x=yRef - pd.sx=PD_SameCol[0].st+', '+PD_SameCol[0].sy - if len(PD_SameCol)==1: - pd.sy =pd.st+', '+pd.sy - else: - pd.syl= pd.st - else: - if len(uit)<=2: - pd.syl = pd.st+' wrt. '+PD_SameCol[0].st+', '+pd.sy - else: - pd.syl = pd.st+'|'+pd.sy - pd.sx = xlabelAll - pd.sy = ylabelAll - Error = getError(pd.y,yRef,sComp) - pd.x=xRef - pd.y=Error - PD_comp.append(pd) - elif mode =='nTabs_SimCols': - # --- Compare different tables, similar columns - print('Several Tabs, similar columns, TODO') - PD_comp=[] - - return PD_comp - +from __future__ import absolute_import +import os +import numpy as np +from .common import no_unit, unit, inverse_unit, has_chinese_char +from .common import isString, isDate, getDt +from .common import unique, pretty_num, pretty_time +from .GUIMeasure import find_closest # Should not depend on wx + +class PlotData(): + """ + Class for plot data + + For now, relies on some "indices" related to Tables/Columns/ and maybe Selection panel + Not really elegant. These dependencies should be removed in the future + """ + def __init__(PD, x=None, y=None, sx='', sy=''): + """ Dummy init for now """ + PD.id=-1 + PD.it=-1 # tablx index + PD.ix=-1 # column index + PD.iy=-1 # column index + PD.sx='' # x label + PD.sy='' # y label + PD.st='' # table label + PD.syl='' # y label for legend + PD.filename = '' + PD.tabname = '' + PD.x =[] # x data + PD.y =[] # y data + PD.xIsString=False # true if strings + PD.xIsDate =False # true if dates + PD.yIsString=False # true if strings + PD.yIsDate =False # true if dates + + if x is not None and y is not None: + PD.fromXY(x,y,sx,sy) + + def fromIDs(PD, tabs, i, idx, SameCol, Options={}): + """ Nasty initialization of plot data from "IDs" """ + PD.id = i + PD.it = idx[0] # table index + PD.ix = idx[1] # x index + PD.iy = idx[2] # y index + PD.sx = idx[3] # x label + PD.sy = idx[4] # y label + PD.syl = '' # y label for legend + PD.st = idx[5] # table label + PD.filename = tabs[PD.it].filename + PD.tabname = tabs[PD.it].active_name + PD.SameCol = SameCol + PD.x, PD.xIsString, PD.xIsDate,_ = tabs[PD.it].getColumn(PD.ix) # actual x data, with info + PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info + PD.c =c # raw values, used by PDF + + PD._post_init(Options=Options) + + def fromXY(PD, x, y, sx='', sy=''): + PD.x = x + PD.y = y + PD.c = y + PD.sx = sx + PD.sy = sy + PD.xIsString = isString(x) + PD.yIsString = isString(y) + PD.xIsDate = isDate (x) + PD.yIsDate = isDate (y) + + PD._post_init() + + + def _post_init(PD, Options={}): + # --- Perform data manipulation on the fly + #[print(k,v) for k,v in Options.items()] + keys=Options.keys() + # TODO setup an "Order" + if 'RemoveOutliers' in keys: + if Options['RemoveOutliers']: + from pydatview.tools.signal import reject_outliers + try: + PD.x, PD.y = reject_outliers(PD.y, PD.x, m=Options['OutliersMedianDeviation']) + except: + raise Exception('Warn: Outlier removal failed. Desactivate it or use a different signal. ') + if 'Filter' in keys: + if Options['Filter']: + from pydatview.tools.signal import applyFilter + PD.y = applyFilter(PD.x, PD.y, Options['Filter']) + + if 'Sampler' in keys: + if Options['Sampler']: + from pydatview.tools.signal import applySampler + PD.x, PD.y = applySampler(PD.x, PD.y, Options['Sampler']) + + if 'Binning' in keys: + if Options['Binning']: + if Options['Binning']['active']: + PD.x, PD.y = Options['Binning']['applyCallBack'](PD.x, PD.y, Options['Binning']) + + # --- Store stats + n=len(PD.y) + if n>1000: + if (PD.xIsString): + raise Exception('Error: x values contain more than 1000 string. This is not suitable for plotting.\n\nPlease select another column for table: {}\nProblematic column: {}\n'.format(PD.st,PD.sx)) + if (PD.yIsString): + raise Exception('Error: y values contain more than 1000 string. This is not suitable for plotting.\n\nPlease select another column for table: {}\nProblematic column: {}\n'.format(PD.st,PD.sy)) + + PD.needChineseFont = has_chinese_char(PD.sy) or has_chinese_char(PD.sx) + # Stats of the raw data (computed once and for all, since it can be expensive for large dataset + PD.computeRange() + # Store the values of the original data (labelled "0"), since the data might be modified later by PDF or MinMax etc. + PD._y0Min = PD._yMin + PD._y0Max = PD._yMax + PD._x0Min = PD._xMin + PD._x0Max = PD._xMax + PD._x0AtYMin = PD._xAtYMin + PD._x0AtYMax = PD._xAtYMax + PD._y0Std = PD.yStd() + PD._y0Mean = PD.yMean() + PD._n0 = (n,'{:d}'.format(n)) + PD.x0 =PD.x + PD.y0 =PD.y + # Store xyMeas input values so we don't need to recompute xyMeas in case they didn't change + PD.xyMeasInput1, PD.xyMeasInput2 = None, None + PD.xyMeas1, PD.xyMeas2 = None, None + + def __repr__(s): + s1='id:{}, it:{}, ix:{}, iy:{}, sx:"{}", sy:"{}", st:{}, syl:{}\n'.format(s.id,s.it,s.ix,s.iy,s.sx,s.sy,s.st,s.syl) + return s1 + + def toPDF(PD, nBins=30, smooth=False): + """ Convert y-data to Probability density function (PDF) as function of x + Uses "stats" library (from welib/pybra) + NOTE: inPlace + """ + from pydatview.tools.stats import pdf_gaussian_kde, pdf_histogram + + n=len(PD.y) + if PD.yIsString: + if n>100: + raise Exception('Warn: Dataset has string format and is too large to display') + vc = PD.c.value_counts().sort_index() + PD.x = vc.keys().tolist() + PD.y = vc/n # TODO counts/PDF option + PD.yIsString=False + PD.xIsString=True + elif PD.yIsDate: + raise Exception('Warn: Cannot plot PDF of dates') + else: + if nBins>=n: + nBins=n + if smooth: + try: + PD.x, PD.y = pdf_gaussian_kde(PD.y, nOut=nBins) + except np.linalg.LinAlgError as e: + PD.x, PD.y = pdf_histogram(PD.y, nBins=nBins, norm=True, count=False) + else: + PD.x, PD.y = pdf_histogram(PD.y, nBins=nBins, norm=True, count=False) + PD.xIsString=False + PD.yIsString=False + + PD.sx = PD.sy; + PD.sy = 'PDF('+no_unit(PD.sy)+')' + iu = inverse_unit(PD.sy) + if len(iu)>0: + PD.sy += ' ['+ iu +']' + + # Compute min max once and for all + PD.computeRange() + + return nBins + + + def toMinMax(PD, xScale=False, yScale=True): + """ Convert plot data to MinMax data based on GUI options + NOTE: inPlace + """ + if yScale: + if PD.yIsString: + raise Exception('Warn: Cannot compute min-max for strings') + mi = PD._y0Min[0] #mi= np.nanmin(PD.y) + mx = PD._y0Max[0] #mx= np.nanmax(PD.y) + if mi == mx: + PD.y=PD.y*0 + else: + PD.y = (PD.y-mi)/(mx-mi) + PD._yMin=0,'0' + PD._yMax=1,'1' + if xScale: + if PD.xIsString: + raise Exception('Warn: Cannot compute min-max for strings') + mi= PD._x0Min[0] + mx= PD._x0Max[0] + if mi == mx: + PD.x=PD.x*0 + else: + PD.x = (PD.x-mi)/(mx-mi) + PD._xMin=0,'0' + PD._xMax=1,'1' + + # Compute min max once and for all + #PD.computeRange() + + return None + + + def toFFT(PD, yType='Amplitude', xType='1/x', avgMethod='Welch', avgWindow='Hamming', bDetrend=True, nExp=8, nPerDecade=10): + """ + Uses spectral.fft_wrap to generate a "FFT" plot data, with various options: + yType : amplitude, PSD, f x PSD + xType : 1/x, x, 2pi/x + avgMethod : None, Welch + avgWindow : Hamming, Hann, Rectangular + see module spectral for more + + NOTE: inplace (modifies itself), does not return a new instance + """ + from pydatview.tools.spectral import fft_wrap + + # --- TODO, make this independent of GUI + if PD.yIsString or PD.yIsDate: + raise Exception('Warn: Cannot plot FFT of dates or strings') + elif PD.xIsString: + raise Exception('Warn: Cannot plot FFT if x axis is string') + + dt=None + if PD.xIsDate: + dt = getDt(PD.x) + # --- Computing fft - x is freq, y is Amplitude + PD.x, PD.y, Info = fft_wrap(PD.x, PD.y, dt=dt, output_type=yType,averaging=avgMethod, averaging_window=avgWindow,detrend=bDetrend,nExp=nExp, nPerDecade=nPerDecade) + # --- Setting plot options + PD._Info=Info + PD.xIsDate=False + # y label + if yType=='PSD': + PD.sy= 'PSD({}) [({})^2/{}]'.format(no_unit(PD.sy), unit(PD.sy), unit(PD.sx)) + elif yType=='f x PSD': + PD.sy= 'f-weighted PSD({}) [({})^2]'.format(no_unit(PD.sy), unit(PD.sy)) + elif yType=='Amplitude': + PD.sy= 'FFT({}) [{}]'.format(no_unit(PD.sy), unit(PD.sy)) + else: + raise Exception('Unsupported FFT type {} '.format(yType)) + # x label + if xType=='1/x': + if unit(PD.sx)=='s': + PD.sx= 'Frequency [Hz]' + else: + PD.sx= '' + elif xType=='x': + PD.x=1/PD.x + if unit(PD.sx)=='s': + PD.sx= 'Period [s]' + else: + PD.sx= '' + elif xType=='2pi/x': + PD.x=2*np.pi*PD.x + if unit(PD.sx)=='s': + PD.sx= 'Cyclic frequency [rad/s]' + else: + PD.sx= '' + else: + raise Exception('Unsupported x-type {} '.format(xType)) + + PD.computeRange() + return Info + + def computeRange(PD): + """ Compute min max of data once and for all and store + From the performance tests, this ends up having a non negligible cost for large dataset, + so we store it to reuse these as much as possible. + If possible, should be used for the plotting as well, so that matplotlib don't + have to compute them again + NOTE: each variable is a tuple (v,s), with a float and its string representation + """ + PD._xMin = PD._xMinCalc() + PD._xMax = PD._xMaxCalc() + PD._yMin = PD._yMinCalc() + PD._yMax = PD._yMaxCalc() + PD._xAtYMin = PD._xAtYMinCalc(PD._yMin[0]) + PD._xAtYMax = PD._xAtYMaxCalc(PD._yMax[0]) + + + # --------------------------------------------------------------------------------} + # --- Stats functions that should only becalled once, could maybe use @attributes.. + # --------------------------------------------------------------------------------{ + def _yMinCalc(PD): + if PD.yIsString: + return PD.y[0],PD.y[0].strip() + elif PD.yIsDate: + return PD.y[0],'{}'.format(PD.y[0]) + else: + v=np.nanmin(PD.y) + s=pretty_num(v) + return (v,s) + + def _yMaxCalc(PD): + if PD.yIsString: + return PD.y[-1],PD.y[-1].strip() + elif PD.yIsDate: + return PD.y[-1],'{}'.format(PD.y[-1]) + else: + v=np.nanmax(PD.y) + s=pretty_num(v) + return (v,s) + + def _xAtYMinCalc(PD, yMin): + if PD.xIsString: + return PD.x[0],PD.x[0].strip() + elif PD.xIsDate: + return PD.x[0],'{}'.format(PD.x[0]) + else: + try: + v = PD.x[np.where(PD.y == yMin)[0][0]] # Might fail if all nan + except: + v = PD.x[0] + s=pretty_num(v) + return (v,s) + + def _xAtYMaxCalc(PD, yMax): + if PD.xIsString: + return PD.x[-1],PD.x[-1].strip() + elif PD.xIsDate: + return PD.x[-1],'{}'.format(PD.x[-1]) + else: + try: + v = PD.x[np.where(PD.y == yMax)[0][0]] # Might fail if all nan + except: + v = PD.x[0] + s=pretty_num(v) + return (v,s) + + def _xMinCalc(PD): + if PD.xIsString: + return PD.x[0],PD.x[0].strip() + elif PD.xIsDate: + return PD.x[0],'{}'.format(PD.x[0]) + else: + v=np.nanmin(PD.x) + s=pretty_num(v) + return (v,s) + + def _xMaxCalc(PD): + if PD.xIsString: + return PD.x[-1],PD.x[-1].strip() + elif PD.xIsDate: + return PD.x[-1],'{}'.format(PD.x[-1]) + else: + v=np.nanmax(PD.x) + s=pretty_num(v) + return (v,s) + + def xMin(PD): + return PD._xMin + + def xMax(PD): + return PD._xMax + + def xAtYMin(PD): + return PD._xAtYMin + + def xAtYMax(PD): + return PD._xAtYMax + + def yMin(PD): + return PD._yMin + + def yMax(PD): + return PD._yMax + + def y0Min(PD): + return PD._y0Min + + def y0Max(PD): + return PD._y0Max + + def y0Mean(PD): + return PD._y0Mean + + def y0Std(PD): + return PD._y0Std + + def n0(PD): + return PD._n0 + + # --------------------------------------------------------------------------------} + # --- Stats functions + # --------------------------------------------------------------------------------{ + def yMean(PD): + if PD.yIsString or PD.yIsDate: + return None,'NA' + else: + v=np.nanmean(PD.y) + s=pretty_num(v) + return (v,s) + + def yMedian(PD): + if PD.yIsString or PD.yIsDate: + return None,'NA' + else: + v=np.nanmedian(PD.y) + s=pretty_num(v) + return (v,s) + + def yStd(PD): + if PD.yIsString or PD.yIsDate: + return None,'NA' + else: + v=np.nanstd(PD.y) + s=pretty_num(v) + return (v,s) + + def yName(PD): + return PD.sy, PD.sy + + def fileName(PD): + return os.path.basename(PD.filename), os.path.basename(PD.filename) + + def baseDir(PD): + return os.path.dirname(PD.filename),os.path.join(os.path.dirname(PD.filename),'') + + def tabName(PD): + return PD.tabname, PD.tabname + + def ylen(PD): + v=len(PD.y) + s='{:d}'.format(v) + return v,s + + + def y0Var(PD): + if PD._y0Std[0] is not None: + v=PD._y0Std[0]**2 + s=pretty_num(v) + else: + v=None + s='NA' + return v,s + + def y0TI(PD): + v=PD._y0Std[0]/PD._y0Mean[0] + s=pretty_num(v) + return v,s + + + def yRange(PD): + if PD.yIsString: + return 'NA','NA' + elif PD.yIsDate: + dtAll=getDt([PD.x[-1]-PD.x[0]]) + return '',pretty_time(dtAll) + else: + v=np.nanmax(PD.y)-np.nanmin(PD.y) + s=pretty_num(v) + return v,s + + def yAbsMax(PD): + if PD.yIsString or PD.yIsDate: + return 'NA','NA' + else: + v=max(np.abs(PD._y0Min[0]),np.abs(PD._y0Max[0])) + s=pretty_num(v) + return v,s + + + def xRange(PD): + if PD.xIsString: + return 'NA','NA' + elif PD.xIsDate: + dtAll=getDt([PD.x[-1]-PD.x[0]]) + return '',pretty_time(dtAll) + else: + v=np.nanmax(PD.x)-np.nanmin(PD.x) + s=pretty_num(v) + return v,s + + + def inty(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y,x=PD.x) + s=pretty_num(v) + return v,s + + def intyintdx(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y,x=PD.x)/np.trapz(y=PD.x*0+1,x=PD.x) + s=pretty_num(v) + return v,s + + def intyx1(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y*PD.x,x=PD.x) + s=pretty_num(v) + return v,s + + def intyx1_scaled(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y*PD.x,x=PD.x) + v=v/np.trapz(y=PD.y,x=PD.x) + s=pretty_num(v) + return v,s + + def intyx2(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y*PD.x**2,x=PD.x) + s=pretty_num(v) + return v,s + + def meas1(PD, xymeas1, xymeas2): + if PD.xyMeasInput1 is not None and PD.xyMeasInput1 == xymeas1: + yv = PD.xyMeas1[1] + s = pretty_num(yv) + else: + xv, yv, s = PD._meas(xymeas1) + PD.xyMeas1 = [xv, yv] + PD.xyMeasInput1 = xymeas1 + return yv, s + + def meas2(PD, xymeas1, xymeas2): + if PD.xyMeasInput2 is not None and PD.xyMeasInput2 == xymeas2: + yv = PD.xyMeas2[1] + s = pretty_num(yv) + else: + xv, yv, s = PD._meas(xymeas2) + PD.xyMeas2 = [xv, yv] + PD.xyMeasInput2 = xymeas2 + return yv, s + + def yMeanMeas(PD): + return PD._measCalc('mean') + + def yMinMeas(PD): + return PD._measCalc('min') + + def yMaxMeas(PD): + return PD._measCalc('max') + + def xAtYMinMeas(PD): + return PD._measCalc('xmin') + + def xAtYMaxMeas(PD): + return PD._measCalc('xmax') + + def _meas(PD, xymeas): + try: + xv, yv = 'NA', 'NA' + xy = np.array([PD.x, PD.y]).transpose() + points = find_closest(xy, [xymeas[0], xymeas[1]], False) + if points.ndim == 1: + xv, yv = points[0:2] + s = pretty_num(yv) + else: + xv, yv = points[0, 0], points[0, 1] + s = ' / '.join([str(p) for p in points[:, 1]]) + except (IndexError, TypeError): + xv, yv = 'NA', 'NA' + s='NA' + return xv, yv, s + + def _measCalc(PD, mode): + if PD.xyMeas1 is None or PD.xyMeas2 is None: + return 'NA', 'NA' + try: + v = 'NA' + left_index = np.where(PD.x == PD.xyMeas1[0])[0][0] + right_index = np.where(PD.x == PD.xyMeas2[0])[0][0] + if left_index == right_index: + raise IndexError + if left_index > right_index: + left_index, right_index = right_index, left_index + if mode == 'mean': + v = np.nanmean(PD.y[left_index:right_index]) + elif mode == 'min': + v = np.nanmin(PD.y[left_index:right_index]) + elif mode == 'max': + v = np.nanmax(PD.y[left_index:right_index]) + elif mode == 'xmin': + v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmin(PD.y[left_index:right_index]))[0][0]] + elif mode == 'xmax': + v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmax(PD.y[left_index:right_index]))[0][0]] + else: + raise NotImplementedError('Error: Mode ' + mode + ' not implemented') + s = pretty_num(v) + except (IndexError, TypeError): + v = 'NA' + s = 'NA' + return v, s + + def dx(PD): + if len(PD.x)<=1: + return 'NA','NA' + if PD.xIsString: + return None,'NA' + elif PD.xIsDate: + dt=getDt(PD.x) + return dt,pretty_time(dt) + else: + v=PD.x[1]-PD.x[0] + s=pretty_num(v) + return v,s + + def xMax(PD): + if PD.xIsString: + return PD.x[-1],PD.x[-1] + elif PD.xIsDate: + return PD.x[-1],'{}'.format(PD.x[-1]) + else: + v=np.nanmax(PD.x) + s=pretty_num(v) + return v,s + def xMin(PD): + if PD.xIsString: + return PD.x[0],PD.x[0] + elif PD.xIsDate: + return PD.x[0],'{}'.format(PD.x[0]) + else: + v=np.nanmin(PD.x) + s=pretty_num(v) + return v,s + + def leq(PD,m): + from pydatview.tools.fatigue import eq_load + if PD.yIsString or PD.yIsDate: + return 'NA','NA' + else: + T,_=PD.xRange() + v=eq_load(PD.y, m=m, neq=T)[0][0] + return v,pretty_num(v) + + def Info(PD,var): + if var=='LSeg': + return '','{:d}'.format(PD._Info.LSeg) + elif var=='LWin': + return '','{:d}'.format(PD._Info.LWin) + elif var=='LOvlp': + return '','{:d}'.format(PD._Info.LOvlp) + elif var=='nFFT': + return '','{:d}'.format(PD._Info.nFFT) + + +# --------------------------------------------------------------------------------} +# --- +# --------------------------------------------------------------------------------{ +def compareMultiplePD(PD, mode, sComp): + """ + PD: list of PlotData + sComp: string in ['Relative', '|Relative|', 'Ratio', 'Absolute' + mode: plot mode, nTabs_1Col, nTabs_SameCols, nTabs_SimCols + + return: + PD_comp : new PlotData list that compares the input list PD + + """ + # --- Helper function + def getError(y,yref,method): + if len(y)!=len(yref): + raise NotImplementedError('Cannot compare signals of different lengths') + if sComp=='Relative': + if np.mean(np.abs(yref))<1e-7: + Error=(y-yRef)/(yRef+1)*100 + else: + Error=(y-yRef)/yRef*100 + elif sComp=='|Relative|': + if np.mean(np.abs(yref))<1e-7: + Error=abs((y-yRef)/(yRef+1))*100 + else: + Error=abs((y-yRef)/yRef)*100 + elif sComp=='Ratio': + if np.mean(np.abs(yref))<1e-7: + Error=(y+1)/(yRef+1) + else: + Error=y/yRef + elif sComp=='Absolute': + Error=y-yRef + else: + raise Exception('Something wrong '+sComp) + return Error + + def getErrorLabel(ylab=''): + if len(ylab)>0: + ylab=no_unit(ylab) + ylab='in '+ylab+' ' + if sComp=='Relative': + return 'Relative error '+ylab+'[%]'; + elif sComp=='|Relative|': + return 'Abs. relative error '+ylab+'[%]'; + if sComp=='Ratio': + return 'Ratio '+ylab.replace('in','of')+'[-]'; + elif sComp=='Absolute': + usy = unique([pd.sy for pd in PD]) + yunits= unique([unit(sy) for sy in usy]) + if len(yunits)==1 and len(yunits[0])>0: + return 'Absolute error '+ylab+'['+yunits[0]+']' + else: + return 'Absolute error '+ylab; + elif sComp=='Y-Y': + return PD[0].sy + + xlabelAll=PD[0].sx + + + if any([pd.yIsString for pd in PD]): + raise Exception('Warn: Cannot compare strings') + if any([pd.yIsDate for pd in PD]): + raise Exception('Warn: Cannot compare dates with other values') + + if mode=='nTabs_1Col': + ylabelAll=getErrorLabel(PD[1].sy) + usy = unique([pd.sy for pd in PD]) + #print('Compare - different tabs - 1 col') + st = [pd.st for pd in PD] + if len(usy)==1: + SS=usy[0] + ', '+ ' wrt. '.join(st[::-1]) + if sComp=='Y-Y': + xlabelAll=PD[0].st+', '+PD[0].sy + ylabelAll=PD[1].st+', '+PD[1].sy + else: + SS=' wrt. '.join(usy[::-1]) + if sComp=='Y-Y': + xlabelAll=PD[0].sy + ylabelAll=PD[1].sy + + xRef = PD[0].x + yRef = PD[0].y + PD[1].syl=SS + y=np.interp(xRef,PD[1].x,PD[1].y) + if sComp=='Y-Y': + PD[1].x=yRef + PD[1].y=y + else: + Error = getError(y,yRef,sComp) + PD[1].x=xRef + PD[1].y=Error + PD[1].sx=xlabelAll + PD[1].sy=ylabelAll + PD_comp=[PD[1]] # return + + elif mode=='1Tab_nCols': + # --- Compare one table - different columns + #print('One Tab, different columns') + ylabelAll=getErrorLabel() + xRef = PD[0].x + yRef = PD[0].y + pdRef=PD[0] + for pd in PD[1:]: + if sComp=='Y-Y': + pd.syl = no_unit(pd.sy)+' wrt. '+no_unit(pdRef.sy) + pd.x = yRef + pd.sx = PD[0].sy + else: + pd.syl = no_unit(pd.sy)+' wrt. '+no_unit(pdRef.sy) + pd.sx = xlabelAll + pd.sy = ylabelAll + Error = getError(pd.y,yRef,sComp) + pd.x=xRef + pd.y=Error + PD_comp=PD[1:] + elif mode =='nTabs_SameCols': + # --- Compare different tables, same column + #print('Several Tabs, same columns') + uiy=unique([pd.iy for pd in PD]) + uit=unique([pd.it for pd in PD]) + PD_comp=[] + for iy in uiy: + PD_SameCol=[pd for pd in PD if pd.iy==iy] + xRef = PD_SameCol[0].x + yRef = PD_SameCol[0].y + ylabelAll=getErrorLabel(PD_SameCol[0].sy) + for pd in PD_SameCol[1:]: + if pd.xIsString: + if len(xRef)==len(pd.x): + pass # fine able to interpolate + else: + raise Exception('X values have different length and are strings, cannot interpolate string. Use `Index` for x instead.') + else: + pd.y=np.interp(xRef,pd.x,pd.y) + if sComp=='Y-Y': + pd.x=yRef + pd.sx=PD_SameCol[0].st+', '+PD_SameCol[0].sy + if len(PD_SameCol)==1: + pd.sy =pd.st+', '+pd.sy + else: + pd.syl= pd.st + else: + if len(uit)<=2: + pd.syl = pd.st+' wrt. '+PD_SameCol[0].st+', '+pd.sy + else: + pd.syl = pd.st+'|'+pd.sy + pd.sx = xlabelAll + pd.sy = ylabelAll + Error = getError(pd.y,yRef,sComp) + pd.x=xRef + pd.y=Error + PD_comp.append(pd) + elif mode =='nTabs_SimCols': + # --- Compare different tables, similar columns + print('Several Tabs, similar columns, TODO') + PD_comp=[] + + return PD_comp + diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py new file mode 100644 index 0000000..a4d272c --- /dev/null +++ b/pydatview/plugins/__init__.py @@ -0,0 +1,30 @@ +""" +Register your plugins in this file: + 1) add a function that calls your plugin. The signature needs to be: + def _function_name(mainframe, event=None, label='') + The corpus of this function should import your package + and call the main function of your package. Your are free + to use the signature your want for your package + + 2) add a tuple to the variable dataPlugins of the form + (string, _function_name) + + where string will be displayed under the data menu of pyDatView. + +See working examples in this file and this directory. +""" + +def _data_standardizeUnits(mainframe, event=None, label=''): + from .data_standardizeUnits import standardizeUnitsPlugin + standardizeUnitsPlugin(mainframe, event, label) + +def _data_binning(mainframe, event=None, label=''): + from .data_binning import BinningToolPanel + return BinningToolPanel + + +dataPlugins=[ + ('Bin data' , _data_binning , True ), + ('Standardize Units (SI)', _data_standardizeUnits, False), + ('Standardize Units (WE)', _data_standardizeUnits, False), + ] diff --git a/pydatview/plugins/data_binning.py b/pydatview/plugins/data_binning.py new file mode 100644 index 0000000..22a1c25 --- /dev/null +++ b/pydatview/plugins/data_binning.py @@ -0,0 +1,307 @@ +import wx +import numpy as np +import pandas as pd +# import copy +# import platform +# from collections import OrderedDict +# For log dec tool +from pydatview.GUITools import GUIToolPanel, TOOL_BORDER +from pydatview.common import CHAR, Error, Info, pretty_num_short +from pydatview.plotdata import PlotData +# from pydatview.tools.damping import logDecFromDecay +# from pydatview.tools.curve_fitting import model_fit, extract_key_miscnum, extract_key_num, MODELS, FITTERS, set_common_keys + + +# --------------------------------------------------------------------------------} +# --- GUI +# --------------------------------------------------------------------------------{ +class BinningToolPanel(GUIToolPanel): + def __init__(self, parent): + super(BinningToolPanel,self).__init__(parent) + + # --- Data from other modules + self.parent = parent # parent is GUIPlotPanel + # Getting states from parent + if 'Binning' not in self.parent.plotDataOptions.keys() or self.parent.plotDataOptions['Binning'] is None: + self.parent.plotDataOptions['Binning'] =_DEFAULT_DICT.copy() + self.data = self.parent.plotDataOptions['Binning'] + self.data['selectionChangeCallBack'] = self.selectionChange + + + # --- GUI elements + self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) + self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) + self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) + self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) + + #self.lb = wx.StaticText( self, -1, """ Click help """) + self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + self.scBins = wx.SpinCtrl(self, value='50', style=wx.TE_RIGHT, size=wx.Size(60,-1) ) + self.textXMin = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) + self.textXMax = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) + self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) + self.btXRange = self.getBtBitmap(self, 'Default','compute', self.reset) + self.lbDX = wx.StaticText(self, -1, '') + self.scBins.SetRange(3, 10000) + + boldFont = self.GetFont().Bold() + lbInputs = wx.StaticText(self, -1, 'Inputs: ') + lbInputs.SetFont(boldFont) + + # --- Layout + btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) + btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) + + msizer = wx.FlexGridSizer(rows=1, cols=3, hgap=2, vgap=0) + msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) +# msizer.Add(self.btXRange , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.LEFT, 1) + + msizer2 = wx.FlexGridSizer(rows=2, cols=5, hgap=2, vgap=1) + + msizer2.Add(lbInputs , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 0) + msizer2.Add(wx.StaticText(self, -1, '#bins: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 1) + msizer2.Add(self.scBins , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, 1) + msizer2.Add(wx.StaticText(self, -1, 'dx: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 8) + msizer2.Add(self.lbDX , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) + msizer2.Add(self.btXRange , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 0) + msizer2.Add(wx.StaticText(self, -1, 'xmin: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 1) + msizer2.Add(self.textXMin , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) + msizer2.Add(wx.StaticText(self, -1, 'xmax: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 8) + msizer2.Add(self.textXMax , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) + #msizer2.AddGrowableCol(4,1) + + vsizer = wx.BoxSizer(wx.VERTICAL) + vsizer.Add(msizer,0, flag = wx.TOP ,border = 1) + vsizer.Add(msizer2,0, flag = wx.TOP|wx.EXPAND ,border = 1) + + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + self.sizer.Add(vsizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) + #self.sizer.Add(msizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) + self.SetSizer(self.sizer) + + # --- Events + self.scBins.Bind(wx.EVT_TEXT, self.onParamChange) + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + self.textXMin.Bind(wx.EVT_TEXT_ENTER, self.onParamChange) + self.textXMax.Bind(wx.EVT_TEXT_ENTER, self.onParamChange) + + # --- Init triggers + if self.data['active']: + self.setXRange(x=[self.data['xMin'], self.data['xMax']]) + else: + self.setXRange() + self.scBins.SetValue(self.data['nBins']) + self.onToggleApply(init=True) + self.updateTabList() + self.onParamChange() + + def reset(self, event=None): + self.setXRange() + self.updateTabList() # might as well until we add a nice callback/button.. + + def setXRange(self, x=None): + if x is None: + x= self.parent.plotData[0].x0 + xmin, xmax = np.nanmin(x), np.nanmax(x) + self.textXMin.SetValue(pretty_num_short(xmin)) + self.textXMax.SetValue(pretty_num_short(xmax)) + + def onParamChange(self, event=None): + self._GUI2Data() + self.lbDX.SetLabel(pretty_num_short((self.data['xMax']- self.data['xMin'])/self.data['nBins'])) + + if self.data['active']: + self.parent.load_and_draw() # Data will change + + def selectionChange(self): + """ function called if user change tables/columns""" + print('>>> Binning selectionChange callback, TODO') + self.setXRange() + + def _GUI2Data(self): + def zero_if_empty(s): + return 0 if len(s)==0 else s + self.data['nBins'] = int (self.scBins.Value) + self.data['xMin'] = float(zero_if_empty(self.textXMin.Value)) + self.data['xMax'] = float(zero_if_empty(self.textXMax.Value)) + + def onToggleApply(self, event=None, init=False): + """ + apply sampler based on GUI Data + """ + if not init: + self.data['active'] = not self.data['active'] + + if self.data['active']: + self._GUI2Data() + self.btPlot.Enable(False) + self.btClear.Enable(False) + self.btApply.SetLabel(CHAR['sun']+' Clear') + else: + self.parent.plotDataOptions['Binning'] = None + self.btPlot.Enable(True) + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + + if not init: + self.parent.plotDataOptions['Binning'] = self.data + self.parent.load_and_draw() # Data will change based on plotData + + + def onAdd(self,event=None): + from pydatview.tools.stats import bin_DF + iSel = self.cbTabs.GetSelection() + tabList = self.parent.selPanel.tabList + mainframe = self.parent.mainframe + icol, colname = self.parent.selPanel.xCol + if self.parent.selPanel.currentMode=='simColumnsMode': + # The difficulty here is that we have to use + # self.parent.selPanel.IKeepPerTab + # or maybe just do it for the first table to get the x column name, + # but there is no guarantee that other tables will have the exact same column name. + Error(self, 'Cannot add tables in "simColumnsMode" for now. Go back to 1 table mode, and add tables individually.') + return + if icol==0: + Error(self, 'Cannot resample based on index') + return + + self._GUI2Data() + errors=[] + + if iSel==0: + # Looping on all tables and adding new table + dfs_new = [] + names_new = [] + for itab, tab in enumerate(tabList): + df_new, name_new = bin_tab(tab, icol, colname, self.data, bAdd=True) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) + else: + errors.append(tab.active_name) + mainframe.load_dfs(dfs_new, names_new, bAdd=True) + else: + tab = tabList.get(iSel-1) + df_new, name_new = bin_tab(tab, icol, colname, self.data, bAdd=True) + if df_new is not None: + mainframe.load_df(df_new, name_new, bAdd=True) + else: + errors.append(tab.active_name) + self.updateTabList() + + if len(errors)>0: + Error(self, 'The binning failed on some tables:\n\n'+'\n'.join(errors)) + return + + def onPlot(self,event=None): + if len(self.parent.plotData)!=1: + Error(self,'Plotting only works for a single plot. Plot less data.') + return + self._GUI2Data() + PD = self.parent.plotData[0] + x_new, y_new = bin_plot(PD.x0, PD.y0, self.data) + + ax = self.parent.fig.axes[0] + PD_new = PlotData() + PD_new.fromXY(x_new, y_new) + self.parent.transformPlotData(PD_new) + ax.plot(PD_new.x, PD_new.y, '-') + self.parent.canvas.draw() + + def onClear(self,event=None): + self.parent.load_and_draw() # Data will change + # Update Table list + self.updateTabList() + + def onTabChange(self,event=None): + #tabList = self.parent.selPanel.tabList + #iSel=self.cbTabs.GetSelection() + pass + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + try: + iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + except RuntimeError: + pass + + def onHelp(self,event=None): + Info(self,"""Binning. + +The binning operation computes average y values for a set of x ranges. + +To bin perform the following step: + +- Specify the number of bins (#bins) +- Specify the min and max of the x values (or click on "Default") + +- Click on one of the following buttons: + - Plot: will display the binned data on the figure + - Apply: will perform the binning on the fly for all new plots + (click on Clear to stop applying) + - Add: will create new table(s) with biined values for all + signals. This process might take some time. + Select a table or choose all (default) +""") + + +# --------------------------------------------------------------------------------} +# --- DATA +# --------------------------------------------------------------------------------{ +def bin_plot(x, y, opts): + from pydatview.tools.stats import bin_signal + xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) + if xBins[0]>xBins[1]: + raise Exception('xmin must be lower than xmax') + x_new, y_new = bin_signal(x, y, xbins=xBins) + return x_new, y_new + +def bin_tab(tab, iCol, colName, opts, bAdd=True): + # TODO, make it such as it's only handling a dataframe instead of a table + from pydatview.tools.stats import bin_DF + colName = tab.data.columns[iCol-1] + error='' + xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) +# try: + df_new =bin_DF(tab.data, xbins=xBins, colBin=colName) + # Setting bin column as first columns + colNames = list(df_new.columns.values) + colNames.remove(colName) + colNames.insert(0, colName) + df_new=df_new.reindex(columns=colNames) + if bAdd: + name_new=tab.raw_name+'_binned' + else: + name_new=None + tab.data=df_new +# except: +# df_new = None +# name_new = None + + return df_new, name_new + + +_DEFAULT_DICT={ + 'active':False, + 'xMin':None, + 'xMax':None, + 'nBins':50, + 'dx':0, + 'applyCallBack':bin_plot, + 'selectionChangeCallBack':None, +} + diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py new file mode 100644 index 0000000..b08fcbf --- /dev/null +++ b/pydatview/plugins/data_standardizeUnits.py @@ -0,0 +1,107 @@ +import unittest +import numpy as np +from pydatview.common import splitunit + +def standardizeUnitsPlugin(mainframe, event=None, label='Standardize Units (SI)'): + """ + Main entry point of the plugin + """ + flavor = label.split('(')[1][0:2] + + for t in mainframe.tabList: + changeUnits(t, flavor=flavor) + + if hasattr(mainframe,'selPanel'): + mainframe.selPanel.colPanel1.setColumns() + mainframe.selPanel.colPanel2.setColumns() + mainframe.selPanel.colPanel3.setColumns() + mainframe.onTabSelectionChange() # trigger replot + +def changeUnits(tab, flavor='SI'): + """ Change units of a table + NOTE: it relies on the Table class, which may change interface in the future.. + """ + if flavor=='WE': + for i, colname in enumerate(tab.columns): + colname, tab.data.iloc[:,i] = change_units_to_WE(colname, tab.data.iloc[:,i]) + tab.columns[i] = colname # TODO, use a dataframe everywhere.. + tab.data.columns = tab.columns + elif flavor=='SI': + for i, colname in enumerate(tab.columns): + colname, tab.data.iloc[:,i] = change_units_to_SI(colname, tab.data.iloc[:,i]) + tab.columns[i] = colname # TODO, use a dataframe everywhere.. + tab.data.columns = tab.columns + else: + raise NotImplementedError(flavor) + + +def change_units_to_WE(s, c): + """ + Change units to wind energy units + s: channel name (string) containing units, typically 'speed_[rad/s]' + c: channel (array) + """ + svar, u = splitunit(s) + u=u.lower() + scalings = {} + scalings['rad/s'] = (30/np.pi,'rpm') # TODO decide + scalings['rad' ] = (180/np.pi,'deg') + scalings['n'] = (1e-3, 'kN') + scalings['nm'] = (1e-3, 'kNm') + scalings['n-m'] = (1e-3, 'kNm') + scalings['n*m'] = (1e-3, 'kNm') + scalings['w'] = (1e-3, 'kW') + if u in scalings.keys(): + scale, new_unit = scalings[u] + s = svar+'['+new_unit+']' + c *= scale + return s, c + +def change_units_to_SI(s, c): + """ + Change units to SI units + TODO, a lot more units conversion needed...will add them as we go + s: channel name (string) containing units, typically 'speed_[rad/s]' + c: channel (array) + """ + svar, u = splitunit(s) + u=u.lower() + scalings = {} + scalings['rpm'] = (np.pi/30,'rad/s') + scalings['rad' ] = (180/np.pi,'deg') + scalings['kn'] = (1e3, 'N') + scalings['knm'] = (1e3, 'Nm') + scalings['kn-m'] = (1e3, 'Nm') + scalings['kn*m'] = (1e3, 'Nm') + scalings['kw'] = (1e3, 'W') + if u in scalings.keys(): + scale, new_unit = scalings[u] + s = svar+'['+new_unit+']' + c *= scale + return s, c + + + + + +class TestChangeUnits(unittest.TestCase): + + def test_change_units(self): + import pandas as pd + from pydatview.Tables import Table + data = np.ones((1,3)) + data[:,0] *= 2*np.pi/60 # rad/s + data[:,1] *= 2000 # N + data[:,2] *= 10*np.pi/180 # rad + df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) + tab=Table(data=df) + changeUnits(tab, flavor='WE') + np.testing.assert_almost_equal(tab.data.values[:,0],[1]) + np.testing.assert_almost_equal(tab.data.values[:,1],[2]) + np.testing.assert_almost_equal(tab.data.values[:,2],[10]) + self.assertEqual(tab.columns, ['om [rpm]', 'F [kN]', 'angle [deg]']) + raise Exception('>>>>>>>>>>>>') + + +if __name__ == '__main__': + unittest.main() diff --git a/pydatview/plugins/tests/__init__.py b/pydatview/plugins/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydatview/plugins/tests/test_standardizeUnits.py b/pydatview/plugins/tests/test_standardizeUnits.py new file mode 100644 index 0000000..35843c3 --- /dev/null +++ b/pydatview/plugins/tests/test_standardizeUnits.py @@ -0,0 +1,24 @@ +import unittest +import numpy as np +import pandas as pd +from pydatview.plugins.data_standardizeUnits import changeUnits + +class TestChangeUnits(unittest.TestCase): + + def test_change_units(self): + from pydatview.Tables import Table + data = np.ones((1,3)) + data[:,0] *= 2*np.pi/60 # rad/s + data[:,1] *= 2000 # N + data[:,2] *= 10*np.pi/180 # rad + df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) + tab=Table(data=df) + changeUnits(tab, flavor='WE') + np.testing.assert_almost_equal(tab.data.values[:,0],[1]) + np.testing.assert_almost_equal(tab.data.values[:,1],[2]) + np.testing.assert_almost_equal(tab.data.values[:,2],[10]) + self.assertEqual(tab.columns, ['om [rpm]', 'F [kN]', 'angle [deg]']) + + +if __name__ == '__main__': + unittest.main() diff --git a/pydatview/tools/curve_fitting.py b/pydatview/tools/curve_fitting.py index 9e7b04b..c03f158 100644 --- a/pydatview/tools/curve_fitting.py +++ b/pydatview/tools/curve_fitting.py @@ -1,1108 +1,1384 @@ -import numpy as np -import scipy.optimize as so -import scipy.stats as stats -import string -import re -from collections import OrderedDict -from numpy import sqrt, pi, exp, cos, sin, log, inf, arctan # for user convenience -import six - -__all__ = ['model_fit'] -__all__ += ['ModelFitter','ContinuousPolynomialFitter','DiscretePolynomialFitter'] -__all__ += ['fit_polynomial_continuous','fit_polynomial_discrete', 'fit_powerlaw_u_alpha'] -__all__ += ['extract_variables'] -__all__ += ['MODELS','FITTERS'] - -# --------------------------------------------------------------------------------} -# --- Predifined functions NOTE: they need to be registered in variable `MODELS` -# --------------------------------------------------------------------------------{ -def gaussian(x, p): - """ p = (mu,sigma) """ - return 1/(p[1]*np.sqrt(2*np.pi)) * np.exp(-1/2*((x-p[0])/p[1])**2) - -def gaussian_w_offset(x, p): - """ p = (mu,sigma,y0) """ - return 1/(p[1]*np.sqrt(2*np.pi)) * np.exp(-1/2*((x-p[0])/p[1])**2) + p[2] - -def logarithmic(x, p): - """ p = (a,b) """ - return p[0]*np.log(x)+p[1] - -def powerlaw_all(x, p): - """ p = (alpha,u_ref,z_ref) """ - return p[1] * (x / p[2]) ** p[0] - -def powerlaw_alpha(x, p, u_ref=10, z_ref=100): - """ p = alpha """ - return u_ref * (x / z_ref) ** p[0] - -def powerlaw_u_alpha(x, p, z_ref=100): - """ p = (alpha, u_ref) """ - return p[1] * (x / z_ref) ** p[0] - -def expdecay(x, p, z_ref=100): - """ p = (A, k, B) formula: {A}*exp(-{k}*x)+{B} """, - return p[0]* np.exp(-p[1]*x) + p[2] - -def weibull_pdf(x, p, z_ref=100): - """ p = (A, k) formula: {k}*x**({k}-1) / {A}**{k} * np.exp(-x/{A})**{k} """, - return p[1] * x ** (p[1] - 1) / p[0] ** p[1] * np.exp(-(x / p[0]) ** p[1]) - -def gentorque(x, p): - """ - x: generator or rotor speed - p= (RtGnSp, RtTq , Rgn2K , SlPc , SpdGenOn) - RtGnSp Rated generator speed for simple variable-speed generator control (HSS side) (rpm) - RtTq Rated generator torque/constant generator torque in Region 3 for simple variable-speed generator control (HSS side) (N-m) - Rgn2K Generator torque constant in Region 2 for simple variable-speed generator control (HSS side) (N-m/rpm^2) - SlPc Rated generator slip percentage in Region 2 1/2 for simple variable-speed generator control (%) - """ - - # Init - RtGnSp, RtTq , Rgn2K , SlPc, SpdGenOn = p - GenTrq=np.zeros(x.shape) - - xmin,xmax=np.min(x), np.max(x) -# if RtGnSp<(xmin+xmax)*0.4: -# return GenTrq - - # Setting up different regions - xR21_Start = RtGnSp*(1-SlPc/100) - bR0 = xSpdGenOn , x=xR21_Start , x<=RtGnSp) - bR3 = x>RtGnSp - # R21 - y1, y2 = Rgn2K*xR21_Start**2, RtTq - x1, x2 = xR21_Start , RtGnSp - m=(y2-y1)/(x2-x1) - GenTrq[bR21] = m*(x[bR21]-x1) + y1 # R21 - GenTrq[bR2] = Rgn2K * x[bR2]**2 # R2 - GenTrq[bR3] = RtTq # R3 - return GenTrq - - -MODELS =[ -# {'label':'User defined model', -# 'name':'eval:', -# 'formula':'{a}*x**2 + {b}', -# 'coeffs':None, -# 'consts':None, -# 'bounds':None }, -{'label':'Gaussian', 'handle':gaussian,'id':'predef: gaussian', -'formula':'1/({sigma}*sqrt(2*pi)) * exp(-1/2 * ((x-{mu})/{sigma})**2)', -'coeffs' :'mu=0, sigma=1', # Order Important -'consts' :None, -'bounds' :None}, -{'label':'Gaussian with y-offset','handle':gaussian_w_offset,'id':'predef: gaussian-yoff', -'formula':'1/({sigma}*sqrt(2*pi)) * exp(-1/2 * ((x-{mu})/{sigma})**2) + {y0}', -'coeffs' :'mu=0, sigma=1, y0=0', #Order Important -'consts' :None, -'bounds' :'sigma=(-inf,inf), mu=(-inf,inf), y0=(-inf,inf)'}, -{'label':'Exponential', 'handle': expdecay, 'id':'predef: expdecay', -'formula':'{A}*exp(-{k}*x)+{B}', -'coeffs' :'A=1, k=1, B=0', # Order Important -'consts' :None, -'bounds' :None}, -{'label':'Logarithmic', 'handle': logarithmic, 'id':'predef: logarithmic', -'formula':'{a}*log(x)+{b}', -'coeffs' :'a=1, b=0', # Order Important -'consts' :None, -'bounds' :None}, -# --- Wind Energy -{'label':'Power law (alpha)', 'handle':powerlaw_alpha, 'id':'predef: powerlaw_alpha', -'formula':'{u_ref} * (z / {z_ref}) ** {alpha}', -'coeffs' : 'alpha=0.1', # Order important -'consts' : 'u_ref=10, z_ref=100', -'bounds' : 'alpha=(-1,1)'}, -{'label':'Power law (alpha,u)', 'handle':powerlaw_u_alpha, 'id':'predef: powerlaw_u_alpha', -'formula':'{u_ref} * (z / {z_ref}) ** {alpha}', -'coeffs': 'alpha=0.1, u_ref=10', # Order important -'consts': 'z_ref=100', -'bounds': 'u_ref=(0,inf), alpha=(-1,1)'}, -# 'powerlaw_all':{'label':'Power law (alpha,u,z)', 'handle':powerlaw_all, # NOTE: not that useful -# 'formula':'{u_ref} * (z / {z_ref}) ** {alpha}', -# 'coeffs': 'alpha=0.1, u_ref=10, z_ref=100', -# 'consts': None, -# 'bounds': 'u_ref=(0,inf), alpha=(-1,1), z_ref=(0,inf)'}, -{'label':'Weibull PDF', 'handle': weibull_pdf, 'id':'predef: weibull_pdf', -'formula':'{k}*x**({k}-1) / {A}**{k} * np.exp(-x/{A})**{k}', -'coeffs' :'A=1, k=1', # Order Important -'consts' :None, -'bounds' :'A=(0.1,inf), k=(0,5)'}, -# {'label':'Generator Torque', 'handle': gentorque, 'id':'predef: gentorque', -# 'formula': '{RtGnSp} , {RtTq} , {Rgn2K} , {SlPc} , {SpdGenOn}', -# 'coeffs' : 'RtGnSp=100 , RtTq=1000 , Rgn2K=0.01 ,SlPc=5 , SpdGenOn=0', # Order Important -# 'consts' :None, -# 'bounds' :'RtGnSp=(0.1,inf) , RtTq=(1,inf), Rgn2K=(0.0,0.1) ,SlPc=(0,20) , SpdGenOn=(0,inf)'} -] - -# --------------------------------------------------------------------------------} -# --- Main function wrapper -# --------------------------------------------------------------------------------{ -def model_fit(func, x, y, p0=None, bounds=None, **fun_kwargs): - """ - Parameters - ---------- - func: string or function handle - - function handle - - string starting with "fitter: ": (see variable _FITTER) - - "fitter: polynomial_continuous 5' : polyfit order 5 - - "fitter: polynomial_discrete 0 2 3 ': fit polynomial of exponents 0 2 3 - - string providing an expression to evaluate, e.g.: - - "eval: {a}*x + {b}*x**2 " - - string starting with "predef": (see variable MODELS) - - "predef: powerlaw_alpha" : - - "predef: powerlaw_all" : - - "predef: gaussian " : - - Returns - ------- - y_fit: array with same shape as `x` - fitted data. - fitter: ModelFitter object - """ - - if isinstance(func,six.string_types) and func.find('fitter:')==0: - predef_fitters=[m['id'] for m in FITTERS] - if func not in predef_fitters: - raise Exception('Function `{}` not defined in curve_fitting module\n Available fitters: {}'.format(func,predef_fitters)) - i = predef_fitters.index(func) - FitterDict = FITTERS[i] - consts=FITTERS[i]['consts'] - args, missing = set_common_keys(consts, fun_kwargs) - if len(missing)>0: - raise Exception('Curve fitting with `{}` requires the following arguments {}. Missing: {}'.format(func,consts.keys(),missing)) - - fitter = FitterDict['handle'](x=x, y=y, p0=p0, bounds=bounds, **fun_kwargs) - else: - fitter = ModelFitter(func, x, y, p0=p0, bounds=bounds, **fun_kwargs) - - pfit = [v for _,v in fitter.model['coeffs'].items()] - return fitter.data['y_fit'], pfit , fitter - - -class ModelFitter(): - def __init__(self,func=None, x=None, y=None, p0=None, bounds=None, **fun_kwargs): - - self.model={ - 'name':None, 'model_function':None, 'consts':fun_kwargs, 'formula': 'unavailable', # model signature - 'coeffs':None, 'formula_num':'unavailable', 'fitted_function':None, 'coeffs_init':p0, 'bounds':bounds, # model fitting - 'R2':None, - } - self.data={'x':x,'y':y,'y_fit':None} - - if func is None: - return - self.set_model(func, **fun_kwargs) - - # Initialize function if present - # Perform fit if data and function is present - if x is not None and y is not None: - self.fit_data(x,y,p0,bounds) - - def set_model(self,func, **fun_kwargs): - if callable(func): - # We don't have much additional info - self.model['model_function'] = func - self.model['name'] = func.__name__ - pass - - elif isinstance(func,six.string_types): - if func.find('predef:')==0: - # --- Minimization from a predefined function - predef_models=[m['id'] for m in MODELS] - if func not in predef_models: - raise Exception('Predefined function `{}` not defined in curve_fitting module\n Available functions: {}'.format(func,predef_models)) - i = predef_models.index(func) - ModelDict = MODELS[i] - self.model['model_function'] = ModelDict['handle'] - self.model['name'] = ModelDict['label'] - self.model['formula'] = ModelDict['formula'] - self.model['coeffs'] = extract_key_num(ModelDict['coeffs']) - self.model['coeffs_init'] = self.model['coeffs'].copy() - self.model['consts'] = extract_key_num(ModelDict['consts']) - self.model['bounds'] = extract_key_tuples(ModelDict['bounds']) - - elif func.find('eval:')==0: - # --- Minimization from a eval string - formula=func[5:] - # Extract coeffs {a} {b} {c}, replace by p[0] - variables, formula_eval = extract_variables(formula) - nParams=len(variables) - if nParams==0: - raise Exception('Formula should contains parameters in curly brackets, e.g.: {a}, {b}, {u_1}. No parameters found in {}'.format(formula)) - - # Check that the formula evaluates - x=np.array([1,2,5])*np.sqrt(2) # some random evaluation vector.. - p=[np.sqrt(2)/4]*nParams # some random initial conditions - #print('p',p) - #print('f',formula_eval) - try: - y=eval(formula_eval) - y=np.asarray(y) - if y.shape!=x.shape: - raise Exception('The formula does not return an array of same size as the input variable x. The formula must include `x`: {}'.format(formula_eval)) - except SyntaxError: - raise Exception('The formula does not evaluate, syntax error raised: {}'.format(formula_eval)) - except ZeroDivisionError: - pass - - # Creating the actual function - def func(x, p): - return eval(formula_eval) - - self.model['model_function'] = func - self.model['name'] = 'user function' - self.model['formula'] = formula - self.model['coeffs'] = OrderedDict([(k,v) for k,v in zip(variables,p)]) - self.model['coeffs_init'] = self.model['coeffs'].copy() - self.model['consts'] = {} - self.model['bounds'] = None - - else: - raise Exception('func string needs to start with `eval:` of `predef:`, func: {}'.format(func)) - else: - raise Exception('func should be string or callable') - - if fun_kwargs is None: - return - if len(fun_kwargs)==0: - return - if self.model['consts'] is None: - raise Exception('Fun_kwargs provided, but no function constants were defined') - - self.model['consts'], missing = set_common_keys(self.model['consts'], fun_kwargs ) - if len(missing)>0: - raise Exception('Curve fitting with function `{}` requires the following arguments {}. Missing: {}'.format(func.__name__,consts.keys(),missing)) - - def setup_bounds(self,bounds,nParams): - if bounds is not None: - self.model['bounds']=bounds # store in model - bounds=self.model['bounds'] # usemodel bounds as default - if bounds is not None: - if isinstance(bounds ,six.string_types): - bounds=extract_key_tuples(bounds) - - if isinstance(bounds ,dict): - if len(bounds)==0 or 'all' in bounds.keys(): - bounds=([-np.inf]*nParams,[np.inf]*nParams) - elif self.model['coeffs'] is not None: - b1=[] - b2=[] - for k in self.model['coeffs'].keys(): - if k in bounds.keys(): - b1.append(bounds[k][0]) - b2.append(bounds[k][1]) - else: - raise Exception('Bounds dictionary is missing the key: `{}`'.format(k)) - bounds=(b1,b2) - else: - raise NotImplementedError('Bounds dictionary with no known model coeffs.') - else: - # so.curve_fit needs a 2-tuple - b1,b2=bounds[0],bounds[1] - if not hasattr(b1,'__len__'): - b1=[b1]*nParams - if not hasattr(b2,'__len__'): - b2=[b2]*nParams - bounds=(b1,b2) - else: - bounds=([-np.inf]*nParams,[np.inf]*nParams) - - self.model['bounds']=bounds # store in model - - def setup_guess(self,p0,bounds, nParams): - """ Setup initial values p0: - - if p0 is a string (e.g. " a=1, b=3"), it's converted to a dict - - if p0 is a dict, the ordered keys of model['coeffs'] are used to sort p0 - """ - if isinstance(p0 ,six.string_types): - p0=extract_key_num(p0) - if len(p0)==0: - p0=None - - if p0 is None: - # There is some tricky logic here between the priority of bounds and coeffs - if self.model['coeffs'] is not None: - # We rely on function to give us decent init coefficients - p0 = ([v for _,v in self.model['coeffs'].items()]) - elif bounds is None: - p0 = ([0]*nParams) - else: - # use middle of bounds - p0 = [0]*nParams - for i,(b1,b2) in enumerate(zip(bounds[0],bounds[1])): - if (b1,b2)==(-np.inf,np.inf): - p0[i]=0 - elif b1==-np.inf: - p0[i] = -abs(b2)*2 - elif b2== np.inf: - p0[i] = abs(b1)*2 - else: - p0[i] = (b1+b2)/2 - p0 = (p0) - elif isinstance(p0,dict): - # User supplied a dictionary, we use the ordered keys of coeffs to sort p0 - p0_dict=p0.copy() - if self.model['coeffs'] is not None: - p0=[] - for k in self.model['coeffs'].keys(): - if k in p0_dict.keys(): - p0.append(p0_dict[k]) - else: - raise Exception('Guess dictionary is missing the key: `{}`'.format(k)) - else: - raise NotImplementedError('Guess dictionary with no known model coeffs.') - - # TODO check that p0 is within bounds - - if not hasattr(p0,'__len__'): - p0=(p0,) - self.model['coeffs_init'] = p0 - - def fit(self, func, x, y, p0=None, bounds=None, **fun_kwargs): - """ Fit model defined by a function to data (x,y) """ - # Setup function - self.set_model(func, **fun_kwargs) - # Fit data to model - self.fit_data(x, y, p0, bounds) - - def clean_data(self,x,y): - x=np.asarray(x) - y=np.asarray(y) - bNaN=~np.isnan(y) - y=y[bNaN] - x=x[bNaN] - bNaN=~np.isnan(x) - y=y[bNaN] - x=x[bNaN] - self.data['x']=x - self.data['y']=y - return x,y - - def fit_data(self, x, y, p0=None, bounds=None): - """ fit data, assuming a model is already setup""" - if self.model['model_function'] is None: - raise Exceptioin('Call set_function first') - - # Cleaning data, and store it in object - x,y=self.clean_data(x,y) - - # nParams - if isinstance(p0 ,six.string_types): - p0=extract_key_num(p0) - if len(p0)==0: - p0=None - if p0 is not None: - if hasattr(p0,'__len__'): - nParams=len(p0) - else: - nParams=1 - elif self.model['coeffs'] is not None: - nParams=len(self.model['coeffs']) - else: - raise Exception('Initial guess `p0` needs to be provided since we cant infer the size of the model coefficients.') - if self.model['coeffs'] is not None: - if len(self.model['coeffs'])!=nParams: - raise Exception('Inconsistent dimension between model guess (size {}) and the model parameters (size {})'.format(nParams,len(self.model['coeffs']))) - - # Bounds - self.setup_bounds(bounds,nParams) - - # Initial conditions - self.setup_guess(p0,self.model['bounds'],nParams) - - # Fitting - minimize_me = lambda x, *p : self.model['model_function'](x, p, **self.model['consts']) - pfit, pcov = so.curve_fit(minimize_me, x, y, p0=self.model['coeffs_init'], bounds=self.model['bounds']) - - # --- Reporting information about the fit (after the fit) - y_fit = self.model['model_function'](x, pfit, **self.model['consts']) - self.store_fit_info(y_fit, pfit) - - # --- Return a fitted function - self.model['fitted_function'] = lambda xx: self.model['model_function'](xx, pfit, **self.model['consts']) - - def store_fit_info(self, y_fit, pfit): - # --- Reporting information about the fit (after the fit) - self.data['y_fit']=y_fit - self.model['R2'] = rsquare(self.data['y'], y_fit) - if self.model['coeffs'] is not None: - if not isinstance(self.model['coeffs'], OrderedDict): - raise Exception('Coeffs need to be of type OrderedDict') - for k,v in zip(self.model['coeffs'].keys(), pfit): - self.model['coeffs'][k]=v - - # Replace numerical values in formula - if self.model['formula'] is not None: - formula_num=self.model['formula'] - for k,v in self.model['coeffs'].items(): - formula_num = formula_num.replace('{'+k+'}',str(v)) - for k,v in self.model['consts'].items(): - formula_num = formula_num.replace('{'+k+'}',str(v)) - self.model['formula_num'] = formula_num - - def formula_num(self, fmt=None): - """ return formula with coeffs and consts evaluted numerically""" - if fmt is None: - fmt_fun = lambda x: str(x) - elif isinstance(fmt,six.string_types): - fmt_fun = lambda x: ('{'+fmt+'}').format(x) - elif callable(fmt): - fmt_fun = fmt - formula_num=self.model['formula'] - for k,v in self.model['coeffs'].items(): - formula_num = formula_num.replace('{'+k+'}',fmt_fun(v)) - for k,v in self.model['consts'].items(): - formula_num = formula_num.replace('{'+k+'}',fmt_fun(v)) - return formula_num - - def __repr__(self): - s='<{} object> with fields:\n'.format(type(self).__name__) - s+=' - data, dictionary with keys: \n' - s+=' - x: [{} ... {}], n: {} \n'.format(self.data['x'][0],self.data['x'][-1],len(self.data['x'])) - s+=' - y: [{} ... {}], n: {} \n'.format(self.data['y'][0],self.data['y'][-1],len(self.data['y'])) - s+=' - model, dictionary with keys: \n' - for k,v in self.model.items(): - s=s+' - {:15s}: {}\n'.format(k,v) - return s - - -# --------------------------------------------------------------------------------} -# --- Predefined fitter -# --------------------------------------------------------------------------------{ -class ContinuousPolynomialFitter(ModelFitter): - def __init__(self,order=None, x=None, y=None, p0=None, bounds=None): - ModelFitter.__init__(self,x=None, y=None, p0=p0, bounds=bounds) - self.setOrder(int(order)) - if order is not None and x is not None and y is not None: - self.fit_data(x,y,p0,bounds) - - def setOrder(self, order): - self.order=order - if order is not None: - variables= string.ascii_lowercase[:order+1] - self.model['coeffs'] = OrderedDict([(var,1) for i,var in enumerate(variables)]) - formula = ' + '.join(['{}*x**{}'.format('{'+var+'}',order-i) for i,var in enumerate(variables)]) - self.model['formula'] = _clean_formula(formula) - - def fit_data(self, x, y, p0=None, bounds=None): - if self.order is None: - raise Exception('Polynomial Fitter not set, call function `setOrder` to set order') - # Cleaning data - x,y=self.clean_data(x,y) - - nParams=self.order+1 - # Bounds - self.setup_bounds(bounds,nParams) # TODO - # Initial conditions - self.setup_guess(p0,bounds,nParams) # TODO - - # Fitting - pfit = np.polyfit(x,y,self.order) - - # --- Reporting information about the fit (after the fit) - y_fit = np.polyval(pfit,x) - self.store_fit_info(y_fit, pfit) - - # --- Return a fitted function - self.model['fitted_function']=lambda xx : np.polyval(pfit,xx) - - -class DiscretePolynomialFitter(ModelFitter): - def __init__(self,exponents=None, x=None, y=None, p0=None, bounds=None): - ModelFitter.__init__(self,x=None, y=None, p0=p0, bounds=bounds) - self.setExponents(exponents) - if exponents is not None and x is not None and y is not None: - self.fit_data(x,y,p0,bounds) - - def setExponents(self, exponents): - self.exponents=exponents - if exponents is not None: - #exponents=-np.sort(-np.asarray(exponents)) - self.exponents=exponents - variables= string.ascii_lowercase[:len(exponents)] - self.model['coeffs'] = OrderedDict([(var,1) for i,var in enumerate(variables)]) - formula = ' + '.join(['{}*x**{}'.format('{'+var+'}',e) for var,e in zip(variables,exponents)]) - self.model['formula'] = _clean_formula(formula) - - def fit_data(self, x, y, p0=None, bounds=None): - if self.exponents is None: - raise Exception('Polynomial Fitter not set, call function `setExponents` to set exponents') - # Cleaning data, and store it in object - x,y=self.clean_data(x,y) - - nParams=len(self.exponents) - # Bounds - self.setup_bounds(bounds,nParams) # TODO - # Initial conditions - self.setup_guess(p0,bounds,nParams) # TODO - - X_poly=np.array([]) - for i,e in enumerate(self.exponents): - if i==0: - X_poly = np.array([x**e]) - else: - X_poly = np.vstack((X_poly,x**e)) - try: - pfit = np.linalg.lstsq(X_poly.T, y, rcond=None)[0] - except: - pfit = np.linalg.lstsq(X_poly.T, y) - - # --- Reporting information about the fit (after the fit) - y_fit= np.dot(pfit, X_poly) - self.store_fit_info(y_fit, pfit) - - # --- Return a fitted function - def fitted_function(xx): - y=np.zeros(xx.shape) - for i,(e,c) in enumerate(zip(self.exponents,pfit)): - y += c*x**e - return y - self.model['fitted_function']=fitted_function - -class GeneratorTorqueFitter(ModelFitter): - def __init__(self,x=None, y=None, p0=None, bounds=None): - ModelFitter.__init__(self,x=None, y=None, p0=p0, bounds=bounds) - -# RtGnSp, RtTq , Rgn2K , SlPc , SpdGenOn = p -# {'label':'Generator Torque', 'handle': gentorque, 'id':'predef: gentorque', -# 'formula': '{RtGnSp} , {RtTq} , {Rgn2K} , {SlPc} , {SpdGenOn}', - self.model['coeffs']= extract_key_num('RtGnSp=100 , RtTq=1000 , Rgn2K=0.01 ,SlPc=5 , SpdGenOn=0') -# 'consts' :None, -# 'bounds' :'RtGnSp=(0.1,inf) , RtTq=(1,inf), Rgn2K=(0.0,0.1) ,SlPc=(0,20) , SpdGenOn=(0,inf)'} - if x is not None and y is not None: - self.fit_data(x,y,p0,bounds) - - def fit_data(self, x, y, p0=None, bounds=None): - #nParams=5 - ## Bounds - #self.setup_bounds(bounds,nParams) # TODO - ## Initial conditions - #self.setup_guess(p0,bounds,nParams) # TODO - - # Cleaning data, and store it in object - x,y=self.clean_data(x,y) - - I = np.argsort(x) - x=x[I] - y=y[I] - - # Estimating deltas - xMin, xMax=np.min(x),np.max(x) - yMin, yMax=np.min(y),np.max(y) - DeltaX = (xMax-xMin)*0.02 - DeltaY = (yMax-yMin)*0.02 - - # Binning data - x_bin=np.linspace(xMin,xMax,min(200,len(x))) - x_lin=x_bin[0:-1]+np.diff(x_bin) - #y_lin=np.interp(x_lin,x,y) # TODO replace by bining - y_lin = np.histogram(y, x_bin, weights=y)[0]/ np.histogram(y, x_bin)[0] - y_lin, _, _ = stats.binned_statistic(x, y, statistic='mean', bins=x_bin) - x_lin, _, _ = stats.binned_statistic(x, x, statistic='mean', bins=x_bin) - bNaN=~np.isnan(y_lin) - y_lin=y_lin[bNaN] - x_lin=x_lin[bNaN] - - # --- Find good guess of parameters based on data - # SpdGenOn - iOn = np.where(y>0)[0][0] - SpdGenOn_0 = x[iOn] - SpdGenOn_Bnds = (max(x[iOn]-DeltaX,xMin), min(x[iOn]+DeltaX,xMax)) - # Slpc - Slpc_0 = 5 - Slpc_Bnds = (0,10) - # RtTq - RtTq_0 = yMax - RtTq_Bnds = (yMax-DeltaY, yMax+DeltaY) - # RtGnSp - iCloseRt = np.where(y>yMax*0.50)[0][0] - RtGnSp_0 = x[iCloseRt] - RtGnSp_Bnds = ( RtGnSp_0 -DeltaX*2, RtGnSp_0+DeltaX*2) - # Rgn2K - #print('>>>',SpdGenOn_0, RtGnSp_0) - bR2=np.logical_and(x>SpdGenOn_0, x ['a','b'] - The variables are replaced with p[0],..,p[n] in order of appearance - """ - regex = r"\{(.*?)\}" - matches = re.finditer(regex, sFormula, re.DOTALL) - formula_eval=sFormula - variables=[] - ivar=0 - for i, match in enumerate(matches): - for groupNum in range(0, len(match.groups())): - var = match.group(1) - if var not in variables: - variables.append(var) - formula_eval = formula_eval.replace('{'+match.group(1)+'}','p[{:d}]'.format(ivar)) - ivar+=1 - return variables, formula_eval - - -def extract_key_tuples(text): - """ - all=(0.1,-2),b=(inf,0), c=(-inf,0.3e+10) - """ - if text is None: - return {} - regex = re.compile(r'(?P[\w\-]+)=\((?P[0-9+epinf.-]*?),(?P[0-9+epinf.-]*?)\)($|,)') - return {match.group("key"): (np.float(match.group("value1")),np.float(match.group("value2"))) for match in regex.finditer(text.replace(' ',''))} - -def extract_key_num(text): - """ - all=0.1, b=inf, c=-0.3e+10 - """ - if text is None: - return {} - regex = re.compile(r'(?P[\w\-]+)=(?P[0-9+epinf.-]*?)($|,)') - return OrderedDict([(match.group("key"), np.float(match.group("value"))) for match in regex.finditer(text.replace(' ',''))]) - -def extract_key_miscnum(text): - """ - all=0.1, b=(inf,0), c=[-inf,0.3e+10,10,11]) - """ - def isint(s): - try: - int(s) - return True - except: - return False - - if text is None: - return {} - sp=re.compile('([\w]+)=').split(text.replace(' ','')) - if len(sp)<3: - return {} - sp=sp[1:] - keys = sp[0::2] - values = sp[1::2] - d={} - for (k,v) in zip(keys,values): - if v.find('(')>=0: - v=v.replace('(','').replace(')','') - v=v.split(',') - vect=tuple([np.float(val) for val in v if len(val.strip())>0]) - elif v.find('[')>=0: - v=v.replace('[','').replace(']','') - v=v.split(',') - vect=[int(val) if isint(val) else np.float(val) for val in v if len(val.strip())>0] # NOTE returning lists - else: - v=v.replace(',','').strip() - vect=int(v) if isint(v) else np.float(v) - d[k]=vect - return d - -def set_common_keys(dict_target, dict_source): - """ Set a dictionary using another one, missing keys in source dictionary are reported""" - keys_missing=[] - for k in dict_target.keys(): - if k in dict_source.keys(): - dict_target[k]=dict_source[k] - else: - keys_missing.append(k) - return dict_target, keys_missing - -def _clean_formula(s): - return s.replace('+-','-').replace('**1','').replace('*x**0','') - -def rsquare(y, f): - """ Compute coefficient of determination of data fit model and RMSE - [r2] = rsquare(y,f) - RSQUARE computes the coefficient of determination (R-square) value from - actual data Y and model data F. - INPUTS - y : Actual data - f : Model fit - OUTPUT - R2 : Coefficient of determination - """ - # Compare inputs - if not np.all(y.shape == f.shape) : - raise Exception('Y and F must be the same size') - # Check for NaN - tmp = np.logical_not(np.logical_or(np.isnan(y),np.isnan(f))) - y = y[tmp] - f = f[tmp] - R2 = max(0,1-np.sum((y-f)**2)/np.sum((y-np.mean(y))** 2)) - return R2 - -# --------------------------------------------------------------------------------} -# --- Low level fitter -# --------------------------------------------------------------------------------{ -def fit_polynomial_continuous(x, y, order): - """Fit a polynomial with a continuous set of exponents up to a given order - - Parameters - ---------- - x,y: see `model_fit` - order: integer - Maximum order of polynomial, e.g. 2: for a x**0 + b x**1 + c x**2 - - Returns - ------- - see `model_fit` - """ - pfit = np.polyfit(x,y,order) - y_fit = np.polyval(pfit,x) - - # coeffs_dict, e.g. {'a':xxx, 'b':xxx}, formula = 'a*x + b' - variables = string.ascii_lowercase[:order+1] - coeffs_dict = OrderedDict([(var,coeff) for i,(coeff,var) in enumerate(zip(pfit,variables))]) - formula = ' + '.join(['{}*x**{}'.format(var,order-i) for i,var in enumerate(variables)]) - formula = _clean_formula(formula) - - - return y_fit,pfit,{'coeffs':coeffs_dict,'formula':formula,'fitted_function':lambda xx : np.polyval(pfit,xx)} - -def fit_polynomial_discrete(x, y, exponents): - """Fit a polynomial with a discrete set of exponents - - Parameters - ---------- - x,y: see `model_fit` - exponents: array-like - Exponents to be used. e.g. [0,2,5] for a x**0 + b x**2 + c x**5 - - Returns - ------- - see `model_fit` - """ - #exponents=-np.sort(-np.asarray(exponents)) - X_poly=np.array([]) - for i,e in enumerate(exponents): - if i==0: - X_poly = np.array([x**e]) - else: - X_poly = np.vstack((X_poly,x**e)) - try: - pfit = np.linalg.lstsq(X_poly.T, y, rcond=None)[0] - except: - pfit = np.linalg.lstsq(X_poly.T, y) - y_fit= np.dot(pfit, X_poly) - - variables = string.ascii_lowercase[:len(exponents)] - coeffs_dict = OrderedDict([(var,coeff) for i,(coeff,var) in enumerate(zip(pfit,variables))]) - formula = ' + '.join(['{}*x**{}'.format(var,e) for var,e in zip(variables,exponents)]) - formula = _clean_formula(formula) - - return y_fit,pfit,{'coeffs':coeffs_dict,'formula':formula} - - -def fit_powerlaw_u_alpha(x, y, z_ref=100, p0=(10,0.1)): - """ - p[0] : u_ref - p[1] : alpha - """ - pfit, _ = so.curve_fit(lambda x, *p : p[0] * (x / z_ref) ** p[1], x, y, p0=p0) - y_fit = pfit[0] * (x / z_ref) ** pfit[1] - coeffs_dict=OrderedDict([('u_ref',pfit[0]),('alpha',pfit[1])]) - formula = '{u_ref} * (z / {z_ref}) ** {alpha}' - fitted_fun = lambda xx: pfit[0] * (xx / z_ref) ** pfit[1] - return y_fit, pfit, {'coeffs':coeffs_dict,'formula':formula,'fitted_function':fitted_fun} - -# --------------------------------------------------------------------------------} -# --- Unittests -# --------------------------------------------------------------------------------{ -import unittest - -class TestFitting(unittest.TestCase): - - def test_gaussian(self): - mu,sigma=0.5,1.2 - x=np.linspace(0,1,10) - y=gaussian(x,(mu,sigma)) - y_fit, pfit, fitter = model_fit('predef: gaussian', x, y) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(mu ,fitter.model['coeffs']['mu']) - np.testing.assert_almost_equal(sigma,fitter.model['coeffs']['sigma']) - - def test_gaussian_w_offset(self): - mu,sigma,y0=0.5,1.2,10 - x=np.linspace(0,1,10) - y=gaussian_w_offset(x,(mu,sigma,y0)) - y_fit, pfit, fitter = model_fit('predef: gaussian-yoff', x, y) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(mu ,fitter.model['coeffs']['mu']) - np.testing.assert_almost_equal(sigma,fitter.model['coeffs']['sigma']) - np.testing.assert_almost_equal(y0 ,fitter.model['coeffs']['y0']) - - def test_powerlaw_alpha(self): - u_ref,z_ref,alpha=20,12,0.12 - x = np.linspace(0,1,10) - y=powerlaw_all(x,(alpha,u_ref,z_ref)) - - fun_kwargs = {'u_ref':u_ref,'z_ref':z_ref} - y_fit, pfit, fitter = model_fit('predef: powerlaw_alpha', x, y, p0=(0.1), **fun_kwargs) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(alpha ,fitter.model['coeffs']['alpha']) - - def test_powerlaw_u_alpha(self): - u_ref,z_ref,alpha=10,12,0.12 - x = np.linspace(0,1,10) - y=powerlaw_all(x,(alpha,u_ref,z_ref,alpha)) - - fun_kwargs = {'z_ref':z_ref} - y_fit, pfit, fitter = model_fit('predef: powerlaw_u_alpha', x, y, **fun_kwargs) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(alpha ,fitter.model['coeffs']['alpha']) - np.testing.assert_almost_equal(u_ref ,fitter.model['coeffs']['u_ref']) - -# def test_powerlaw_all(self): -# u_ref,z_ref,alpha=10,12,0.12 -# x = np.linspace(0,1,10) -# y=powerlaw_all(x,(alpha,u_ref,z_ref,alpha)) -# -# y_fit, pfit, fitter = model_fit('predef: powerlaw_all', x, y) -# np.testing.assert_array_almost_equal(y,y_fit) -# np.testing.assert_almost_equal(alpha ,fitter.model['coeffs']['alpha']) -# # NOTE: cannot test for u_ref or z - - def test_expdecay(self): - A,k,B=0.5,1.2,10 - x=np.linspace(0,1,10) - y=expdecay(x,(A,k,B)) - y_fit, pfit, fitter = model_fit('predef: expdecay', x, y) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(A,fitter.model['coeffs']['A']) - np.testing.assert_almost_equal(k,fitter.model['coeffs']['k']) - np.testing.assert_almost_equal(B,fitter.model['coeffs']['B']) - - def test_weibull(self): - A, k = 10, 2.3, - x=np.linspace(0,1,10) - y=weibull_pdf(x,(A,k)) - y_fit, pfit, fitter = model_fit('predef: weibull_pdf', x, y) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(A,fitter.model['coeffs']['A'],5) - np.testing.assert_almost_equal(k,fitter.model['coeffs']['k']) - - def test_gentorque(self): - pass # TODO -# GBRatio= 27.5647 #; % Gearbox ratio (-) -# SpdGenOn = 14*GBRatio# -# RtGnSp = 1207.61 # % Rated generator speed for simple variable-speed generator control (HSS side) (rpm) -# RtTq = 1790.49 # % Rated generator torque/constant generator torque in Region 3 for simple variable-speed generator control (HSS side) (N-m) -# Rgn2K = 0.0004128 # % Generator torque constant in Region 2 for simple variable-speed generator control (HSS side) (N-m/rpm^2) -# SlPc = 6 # % Rated generator slip percentage in Region 2 1/2 for simple variable-speed generator control (%) -# # x=np.linspace(300,1500,100) -# x=np.linspace(300,1000,100) -# y=gentorque(x, (RtGnSp, RtTq , Rgn2K , SlPc , SpdGenOn)) -# -# bounds='RtGnSp=(1200,1300) , RtTq=(1500,1800), Rgn2K=(0.0,0.01) ,SlPc=(0,20) , SpdGenOn=(10,500)' -# p0 = [1250, 1700,0.001, 10, 50] -# y_fit, pfit, fitter = model_fit('fitter: gentorque', x, y) -# -# y_fit, pfit, fitter = model_fit('predef: gentorque', x, y, bounds=bounds, p0=p0) -# # np.testing.assert_array_almost_equal(y,y_fit) -# print(fitter) -# import matplotlib.pyplot as plt -# -# fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) -# fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) -# ax.plot(x, y ,'o', label='') -# ax.plot(x, y_fit ,'-', label='') -# ax.plot(x, fitter.model['fitted_function'](x) ,'.', label='') -# ax.set_xlabel('') -# ax.set_ylabel('') -# ax.legend() -# ax.tick_params(direction='in') -# plt.show() - - def test_polycont(self): - k = 2.0 - x = np.linspace(0,1,10) - y = k * x**3 - y_fit, pfit, fitter = model_fit('fitter: polynomial_continuous', x, y, order=3) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(k ,fitter.model['coeffs']['a']) - np.testing.assert_almost_equal(0 ,fitter.model['coeffs']['b']) - np.testing.assert_almost_equal(0 ,fitter.model['coeffs']['c']) - np.testing.assert_almost_equal(0 ,fitter.model['coeffs']['d']) - - def test_polydisc(self): - exponents=[0,3,5] - a,b,c = 2.0, 3.0, 4.0 - x = np.linspace(0,1,10) - y = a + b*x**3 + c*x**5 - y_fit, pfit, fitter = model_fit('fitter: polynomial_discrete', x, y, exponents=exponents) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(a ,fitter.model['coeffs']['a']) - np.testing.assert_almost_equal(b ,fitter.model['coeffs']['b']) - np.testing.assert_almost_equal(c ,fitter.model['coeffs']['c']) - - def test_evalpoly(self): - exponents=[0,3,5] - a,b,c = 2.0, 3.0, 4.0 - x = np.linspace(0,1,10) - y = a + b*x**3 + c*x**5 - y_fit, pfit, fitter = model_fit('eval: {a} + {b}*x**3 + {c}*x**5', x, y) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(a ,fitter.model['coeffs']['a']) - np.testing.assert_almost_equal(b ,fitter.model['coeffs']['b']) - np.testing.assert_almost_equal(c ,fitter.model['coeffs']['c']) - - def test_evalpowerlaw(self): - u_ref,z_ref,alpha=10,12,0.12 - x = np.linspace(0,1,10) - y=powerlaw_all(x,(alpha,u_ref,z_ref)) - y_fit, pfit, fitter = model_fit('eval: {u_ref}*(x/{z_ref})**{alpha}', x, y, p0=(8,9,0.1), bounds=(0.001,100)) - np.testing.assert_array_almost_equal(y,y_fit) - - def test_lowlevelpoly(self): - x=np.linspace(0,1,10) - y=x**2 - exponents=[0,1,2] - y_fit, pfit, model = fit_polynomial_discrete(x, y, exponents) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(1 , model['coeffs']['c']) - np.testing.assert_almost_equal(0 , model['coeffs']['a']) - - y_fit, pfit, model = fit_polynomial_continuous(x, y, 3) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(1 , model['coeffs']['b']) - np.testing.assert_almost_equal(0 , model['coeffs']['a']) - - def test_lowlevelpowerlaw(self): - u_ref,z_ref,alpha=10,12,0.12 - x = np.linspace(0,1,10) - y=powerlaw_all(x,(alpha,u_ref,z_ref)) - - y_fit, pfit, model = fit_powerlaw_u_alpha(x, y, z_ref=z_ref, p0=(9,0.1)) - np.testing.assert_array_almost_equal(y,y_fit) - np.testing.assert_almost_equal(alpha , model['coeffs']['alpha']) - np.testing.assert_almost_equal(u_ref , model['coeffs']['u_ref']) - -# def test_debug(self): -# # --- Try Gaussian -# x=np.linspace(0,1,10) -# y=gaussian(x,(0.5,1.2)) -# y_fit, pfit, fitter = model_fit('predef: gaussian', x, y) #, p0=(0,1)) -# # fitter = ModelFitter('eval: {a}*(1.0/{b}+2/0)**{c}', x, y, p0=(8,9,0.1)) -# # fitter = ModelFitter('eval: {a}/x', x, y, p0=(8,9,0.1)) -# -# # --- Plot -# y_fit=fitter.data['y_fit'] -# print(fitter) -# -# import matplotlib.pyplot as plt -# fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) -# fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) -# ax.plot(x, y ,'o', label='') -# ax.plot(x, y_fit ,'-', label='') -# ax.plot(x, fitter.model['fitted_function'](x) ,'.', label='') -# ax.set_xlabel('') -# ax.set_ylabel('') -# ax.legend() -# ax.tick_params(direction='in') -# plt.show() - - def test_extract_var(self): - var, _ = extract_variables('{a}*x + {b}') - self.assertEqual(var,['a','b']) - - var, _ = extract_variables('{BB}*x + {a}*{BB}') - self.assertEqual(var,['BB','a']) - - var, _ = extract_variables('{a}*x + {{b}}') #< TODO Won't work - #self.assertEqual(var,['a','b']) - - def test_key_tuples(self): - self.assertEqual(extract_key_tuples('a=(1,2)'),{'a':(1,2)}) - - self.assertEqual(extract_key_tuples('a=(1, 2),b =(inf,0),c= ( -inf , 0.3e+10)'),{'a':(1,2),'b':(inf,0),'c':(-inf,0.3e+10)}) - - def test_key_num(self): - self.assertEqual(extract_key_num('a=2'),OrderedDict({'a':2})) - self.assertEqual(extract_key_num('all=0.1,b =inf, c= -0.3e+10'),OrderedDict({'all':0.1,'b':inf,'c':-0.3e+10})) - - def test_key_misc(self): - self.assertEqual(extract_key_miscnum('a=2'),{'a':2}) - - #np.testing.assert_almost_equal(d['a'],(2,3)) - d=extract_key_miscnum('a=(2,3)') - self.assertEqual(d['a'],(2,3)) - d=extract_key_miscnum('a=[2,3]') - np.testing.assert_almost_equal(d['a'],[2,3]) - - d=extract_key_miscnum('a=[2,3],b=3,c=(0,)') - np.testing.assert_almost_equal(d['a'],[2,3]) - self.assertEqual(d['b'],3) - self.assertEqual(d['c'],(0,)) - - - - -if __name__ == '__main__': -# TestFitting().test_debug() -# TestFitting().test_gentorque() - -# # Writing example models to file -# a,b,c = 2.0, 3.0, 4.0 -# u_ref,z_ref,alpha=10,12,0.12 -# mu,sigma=0.5,1.2 -# x = np.linspace(0.1,30,20) -# A,k,B=0.5,1.2,10 -# y_exp=expdecay(x,(A,k,B)) -# A, k = 10, 2.3, -# y_weib=weibull_pdf(x,(A,k)) -# y_log=logarithmic(x,(a,b)) -# exponents=[0,3,5] -# y_poly = a + b*x**3 + c*x**5 -# y_power=powerlaw_all(x,(alpha,u_ref,z_ref)) -# y_gauss=gaussian(x,(mu,sigma)) -# M=np.column_stack((x,y_poly,y_power,y_gauss,y_gauss+10,y_weib,y_exp,y_log)) -# np.savetxt('../TestFit.csv',M,header='x,poly,power,gauss,gauss_off,weib,expdecay,log',delimiter=',') -# - unittest.main() +""" +Set of tools to fit a model to data. + +The quality of a fit is usually a strong function of the initial guess. +Because of this this package contains different kind of "helpers" and "wrapper" tools. + +FUNCTIONS +--------- + +This package can help fitting using: + 1) High level functions, e.g. fit_sinusoid +OR using the `model_fit` function that handles: + 2) User defined "eval" model, e.g. the user sets a string '{a}*x + {b}*x**2' + 3) Predefined models, e.g. Gaussian, logarithmic, weibull_pdf, etc. + 4) Predefined fitters, e.g. SinusoidFitter, DiscretePolynomialFitter, ContinuousPolynomialFitter + +1) The high level fitting functions available are: + - fit_sinusoid + - fit_polynomial + - fit_gaussian + +2) User defined model, using the `model_fit_function`: + - model_fit('eval: {a} + {b}*x**3 + {c}*x**5', x, y) + - model_fit('eval: {u_ref}*(x/{z_ref})**{alpha}', x, y, p0=(8,9,0.1), bounds=(0.001,100)) + User defined models, will require the user to provide an initial guess and potentially bounds + +3) Fitting using predefined models using the `model_fit` function : + - model_fit('predef: gaussian', x, y) + - model_fit('predef: gaussian-yoff', x, y) + - model_fit('predef: powerlaw_alpha', x, y, p0=(0.1), **fun_kwargs) + - model_fit('predef: powerlaw_u_alpha', x, y, **fun_kwargs) + - model_fit('predef: expdecay', x, y) + - model_fit('predef: weibull_pdf', x, y) + Predefined models have default values for bounds and guesses that can be overriden. + +4) Predefined fitters, wrapped with the `model_fit` function: + - model_fit('fitter: sinusoid', x, y) + - model_fit('fitter: polynomial_discrete', x, y, exponents=[0,2,4]) + - model_fit('fitter: polynomial_continuous', x, y, order=3) + Predefined fitters can handle bounds/initial guess better + +INPUTS: +-------- +All functions have the following inputs: + - x: array on the x-axis + - y: values on the y-axis (to be fitted against a model) +Additionally some functions have the following inputs: + - p0: initial values for parameters, either a string or a dict: + - string: the string is converted to a dictionary, assuming key value pairs + example: 'a=0, b=1.3' + - dictionary, then keys should corresponds to the parameters of the model + example: {'a':0, 'b':1.3} + - bounds: bounds for each parameters, either a string or a dictionary. + NOTE: pi and inf are available to set bounds + - if a string, the string is converted to a dictionary assuming key value pairs + example: 'a=(0,3), b=(-inf,pi)' + - if a dictionary, the keys should corresponds to the parameters of the model + example: {'a':(0,3), 'b':(-inf,pi)} + +OUTPUTS: +-------- +All functions returns the same outputs: + - y_fit : the fit to the y data + - pfit : the list of parameters used + - fitter: a `ModelFitter` object useful to manipulate the fit, in particular: + - fitter.model: dictionary with readable versions of the parameters, formula, + function to reevaluate the fit on a different x, etc. + - fitter.data: data used for the fit + - fitter.fit_data: perform another fit using different data + +MISC +---- +High-level fitters, predefined models or fitters can be added to this class. + +""" +import numpy as np +import scipy.optimize as so +import scipy.stats as stats +import string +import re +from collections import OrderedDict +from numpy import sqrt, pi, exp, cos, sin, log, inf, arctan # for user convenience +import six + +# --------------------------------------------------------------------------------} +# --- High level fitters +# --------------------------------------------------------------------------------{ +def fit_sinusoid(x,y,physical=False): + """ Fits a sinusoid to y with formula: + if physical is False: y_fit=A*sin(omega*x+phi)+B + if physical is True: y_fit=A*sin(2*pi(f+phi/360))+B """ + y_fit, pfit, fitter = model_fit('fitter: sinusoid', x, y, physical=physical) + return y_fit, pfit, fitter + +def fit_polynomial(x, y, order=None, exponents=None): + """ Fits a polynomial to y, either: + - full up to a given order: y_fit= {a_i} x^i , i=0..order + - or using a discrete set of exponents: y_fit= {a_i} x^e[i], i=0,..len(exponents) + OPTIONAL INPUTS: + - order: integer + Maximum order of polynomial, e.g. 2: for a x**0 + b x**1 + c x**2 + - exponents: array-like + Exponents to be used. e.g. [0,2,5] for a x**0 + b x**2 + c x**5 + """ + if order is not None: + y_fit, pfit, fitter = model_fit('fitter: polynomial_continuous', x, y, order=order) + else: + y_fit, pfit, fitter = model_fit('fitter: polynomial_discrete', x, y, exponents=exponents) + return y_fit, pfit, fitter + +def fit_gaussian(x, y, offset=False): + """ Fits a gaussin to y, with the following formula: + offset is True : '1/({sigma}*sqrt(2*pi)) * exp(-1/2 * ((x-{mu})/{sigma})**2)' + offset is False: '1/({sigma}*sqrt(2*pi)) * exp(-1/2 * ((x-{mu})/{sigma})**2) + {y0}' + """ + if offset: + return model_fit('predef: gaussian-yoff', x, y) + else: + return model_fit('predef: gaussian', x, y) + +# --------------------------------------------------------------------------------} +# --- Simple mid level fitter +# --------------------------------------------------------------------------------{ +def fit_polynomial_continuous(x, y, order): + """Fit a polynomial with a continuous set of exponents up to a given order + + Parameters + ---------- + x,y: see `model_fit` + order: integer + Maximum order of polynomial, e.g. 2: for a x**0 + b x**1 + c x**2 + + Returns + ------- + see `model_fit` + """ + pfit = np.polyfit(x,y,order) + y_fit = np.polyval(pfit,x) + + # coeffs_dict, e.g. {'a':xxx, 'b':xxx}, formula = 'a*x + b' + variables = string.ascii_lowercase[:order+1] + coeffs_dict = OrderedDict([(var,coeff) for i,(coeff,var) in enumerate(zip(pfit,variables))]) + formula = ' + '.join(['{}*x**{}'.format(var,order-i) for i,var in enumerate(variables)]) + formula = _clean_formula(formula) + + return y_fit,pfit,{'coeffs':coeffs_dict,'formula':formula,'fitted_function':lambda xx : np.polyval(pfit,xx)} + +def fit_polynomial_discrete(x, y, exponents): + """Fit a polynomial with a discrete set of exponents + + Parameters + ---------- + x,y: see `model_fit` + exponents: array-like + Exponents to be used. e.g. [0,2,5] for a x**0 + b x**2 + c x**5 + + Returns + ------- + see `model_fit` + """ + #exponents=-np.sort(-np.asarray(exponents)) + X_poly=np.array([]) + for i,e in enumerate(exponents): + if i==0: + X_poly = np.array([x**e]) + else: + X_poly = np.vstack((X_poly,x**e)) + try: + pfit = np.linalg.lstsq(X_poly.T, y, rcond=None)[0] + except: + pfit = np.linalg.lstsq(X_poly.T, y) + y_fit= np.dot(pfit, X_poly) + + variables = string.ascii_lowercase[:len(exponents)] + coeffs_dict = OrderedDict([(var,coeff) for i,(coeff,var) in enumerate(zip(pfit,variables))]) + formula = ' + '.join(['{}*x**{}'.format(var,e) for var,e in zip(variables,exponents)]) + formula = _clean_formula(formula) + + return y_fit,pfit,{'coeffs':coeffs_dict,'formula':formula} + + +def fit_powerlaw_u_alpha(x, y, z_ref=100, p0=(10,0.1)): + """ + p[0] : u_ref + p[1] : alpha + """ + pfit, _ = so.curve_fit(lambda x, *p : p[0] * (x / z_ref) ** p[1], x, y, p0=p0) + y_fit = pfit[0] * (x / z_ref) ** pfit[1] + coeffs_dict=OrderedDict([('u_ref',pfit[0]),('alpha',pfit[1])]) + formula = '{u_ref} * (z / {z_ref}) ** {alpha}' + fitted_fun = lambda xx: pfit[0] * (xx / z_ref) ** pfit[1] + return y_fit, pfit, {'coeffs':coeffs_dict,'formula':formula,'fitted_function':fitted_fun} + + +# --------------------------------------------------------------------------------} +# --- Predifined functions NOTE: they need to be registered in variable `MODELS` +# --------------------------------------------------------------------------------{ +def gaussian(x, p): + """ p = (mu,sigma) """ + return 1/(p[1]*np.sqrt(2*np.pi)) * np.exp(-1/2*((x-p[0])/p[1])**2) + +def gaussian_w_offset(x, p): + """ p = (mu,sigma,y0) """ + return 1/(p[1]*np.sqrt(2*np.pi)) * np.exp(-1/2*((x-p[0])/p[1])**2) + p[2] + +def logarithmic(x, p): + """ p = (a,b) """ + return p[0]*np.log(x)+p[1] + +def powerlaw_all(x, p): + """ p = (alpha,u_ref,z_ref) """ + return p[1] * (x / p[2]) ** p[0] + +def powerlaw_alpha(x, p, u_ref=10, z_ref=100): + """ p = alpha """ + return u_ref * (x / z_ref) ** p[0] + +def powerlaw_u_alpha(x, p, z_ref=100): + """ p = (alpha, u_ref) """ + return p[1] * (x / z_ref) ** p[0] + +def expdecay(x, p, z_ref=100): + """ p = (A, k, B) formula: {A}*exp(-{k}*x)+{B} """, + return p[0]* np.exp(-p[1]*x) + p[2] + +def weibull_pdf(x, p, z_ref=100): + """ p = (A, k) formula: {k}*x**({k}-1) / {A}**{k} * np.exp(-x/{A})**{k} """, + # NOTE: if x is 0, a divide by zero error is incountered if p[1]-1<0 + p=list(p) + return p[1] * x ** (p[1] - 1) / p[0] ** p[1] * np.exp(-(x / p[0]) ** p[1]) + +def sinusoid(x, p): + """ p = (A,omega,phi,B) """ + return p[0]*np.sin(p[1]*x+p[2]) + p[3] +def sinusoid_f(x, p): + """ p = (A,f,phi_deg,B) """ + return p[0]*np.sin(2*pi*(p[1]*x+p[2]/360)) + p[3] + + + +def secondorder_impulse(t, p): + """ p = (A, omega0, zeta, B, t0) """ + A, omega0, zeta, B, t0 = p + omegad = omega0 * sqrt(1-zeta**2) + phi = np.arctan2(zeta, sqrt(1-zeta**2)) + x = np.zeros(t.shape) + bp = t>=t0 + t = t[bp]-t0 + x[bp] += A * sin(omegad * t) * exp(-zeta * omega0 * t) + x+=B + return x + +def secondorder_step(t, p): + """ p = (A, omega0, zeta, B, t0) """ + A, omega0, zeta, B, t0 = p + omegad = omega0 * sqrt(1-zeta**2) + phi = np.arctan2(zeta, sqrt(1-zeta**2)) + x = np.zeros(t.shape) + bp = t>=t0 + t = t[bp]-t0 + x[bp] += A * ( 1- exp(-zeta*omega0 *t)/sqrt(1-zeta**2) * cos(omegad*t - phi)) + x+=B + return x + + +def gentorque(x, p): + """ + INPUTS: + x: generator or rotor speed + p= (RtGnSp, RtTq , Rgn2K , SlPc , SpdGenOn) + RtGnSp Rated generator speed for simple variable-speed generator control (HSS side) (rpm) + RtTq Rated generator torque/constant generator torque in Region 3 for simple variable-speed generator control (HSS side) (N-m) + Rgn2K Generator torque constant in Region 2 for simple variable-speed generator control (HSS side) (N-m/rpm^2) + SlPc Rated generator slip percentage in Region 2 1/2 for simple variable-speed generator control (%) + + OUTPUTS: + GenTrq: Generator torque [Nm] + + """ + + # Init + RtGnSp, RtTq , Rgn2K , SlPc, SpdGenOn = p + GenTrq=np.zeros(x.shape) + + xmin,xmax=np.min(x), np.max(x) +# if RtGnSp<(xmin+xmax)*0.4: +# return GenTrq + + # Setting up different regions + xR21_Start = RtGnSp*(1-SlPc/100) + bR0 = xSpdGenOn , x=xR21_Start , x<=RtGnSp) + bR3 = x>RtGnSp + # R21 + y1, y2 = Rgn2K*xR21_Start**2, RtTq + x1, x2 = xR21_Start , RtGnSp + m=(y2-y1)/(x2-x1) + GenTrq[bR21] = m*(x[bR21]-x1) + y1 # R21 + GenTrq[bR2] = Rgn2K * x[bR2]**2 # R2 + GenTrq[bR3] = RtTq # R3 + return GenTrq + + +MODELS =[ +# {'label':'User defined model', +# 'name':'eval:', +# 'formula':'{a}*x**2 + {b}', +# 'coeffs':None, +# 'consts':None, +# 'bounds':None }, +{'label':'Gaussian', 'handle':gaussian,'id':'predef: gaussian', +'formula':'1/({sigma}*sqrt(2*pi)) * exp(-1/2 * ((x-{mu})/{sigma})**2)', +'coeffs' :'mu=0, sigma=1', # Order Important +'consts' :None, +'bounds' :None}, +{'label':'Gaussian with y-offset','handle':gaussian_w_offset,'id':'predef: gaussian-yoff', +'formula':'1/({sigma}*sqrt(2*pi)) * exp(-1/2 * ((x-{mu})/{sigma})**2) + {y0}', +'coeffs' :'mu=0, sigma=1, y0=0', #Order Important +'consts' :None, +'bounds' :'sigma=(-inf,inf), mu=(-inf,inf), y0=(-inf,inf)'}, +{'label':'Exponential', 'handle': expdecay, 'id':'predef: expdecay', +'formula':'{A}*exp(-{k}*x)+{B}', +'coeffs' :'A=1, k=1, B=0', # Order Important +'consts' :None, +'bounds' :None}, +{'label':'Logarithmic', 'handle': logarithmic, 'id':'predef: logarithmic', +'formula':'{a}*log(x)+{b}', +'coeffs' :'a=1, b=0', # Order Important +'consts' :None, +'bounds' :None}, +{'label':'2nd order impulse/decay (manual)', 'handle': secondorder_impulse, 'id':'predef: secondorder_impulse', +'formula':'{A}*exp(-{zeta}*{omega}*(x-{x0})) * sin({omega}*sqrt(1-{zeta}**2))) +{B}', +'coeffs' :'A=1, omega=1, zeta=0.001, B=0, x0=0', # Order Important +'consts' :None, +'bounds' :'A=(-inf,inf), omega=(0,100), zeta=(0,1), B=(-inf,inf), x0=(-inf,inf)'}, +{'label':'2nd order step (manual)', 'handle': secondorder_step, 'id':'predef: secondorder_step', +'formula':'{A}*(1-exp(-{zeta}*{omega}*(x-{x0}))/sqrt(1-{zeta}**2) * cos({omega}*sqrt(1-{zeta}**2)-arctan({zeta}/sqrt(1-{zeta}**2)))) +{B}', +'coeffs' :'A=1, omega=1, zeta=0.001, B=0, x0=0', # Order Important +'consts' :None, +'bounds' :'A=(-inf,inf), omega=(0,100), zeta=(0,1), B=(-inf,inf), x0=(-inf,inf)'}, + +# --- Wind Energy +{'label':'Power law (alpha)', 'handle':powerlaw_alpha, 'id':'predef: powerlaw_alpha', +'formula':'{u_ref} * (z / {z_ref}) ** {alpha}', +'coeffs' : 'alpha=0.1', # Order important +'consts' : 'u_ref=10, z_ref=100', +'bounds' : 'alpha=(-1,1)'}, +{'label':'Power law (alpha,u)', 'handle':powerlaw_u_alpha, 'id':'predef: powerlaw_u_alpha', +'formula':'{u_ref} * (z / {z_ref}) ** {alpha}', +'coeffs': 'alpha=0.1, u_ref=10', # Order important +'consts': 'z_ref=100', +'bounds': 'u_ref=(0,inf), alpha=(-1,1)'}, +# 'powerlaw_all':{'label':'Power law (alpha,u,z)', 'handle':powerlaw_all, # NOTE: not that useful +# 'formula':'{u_ref} * (z / {z_ref}) ** {alpha}', +# 'coeffs': 'alpha=0.1, u_ref=10, z_ref=100', +# 'consts': None, +# 'bounds': 'u_ref=(0,inf), alpha=(-1,1), z_ref=(0,inf)'}, +{'label':'Weibull PDF', 'handle': weibull_pdf, 'id':'predef: weibull_pdf', +'formula':'{k}*x**({k}-1) / {A}**{k} * np.exp(-x/{A})**{k}', +'coeffs' :'A=1, k=1', # Order Important +'consts' :None, +'bounds' :'A=(0.1,inf), k=(0,5)'}, +{'label':'Generator Torque', 'handle': gentorque, 'id':'predef: gentorque', +'formula': '{RtGnSp} , {RtTq} , {Rgn2K} , {SlPc} , {SpdGenOn}', +'coeffs' : 'RtGnSp=100 , RtTq=1000 , Rgn2K=0.01 ,SlPc=5 , SpdGenOn=0', # Order Important +'consts' :None, +'bounds' :'RtGnSp=(0.1,inf) , RtTq=(1,inf), Rgn2K=(0.0,0.1) ,SlPc=(0,20) , SpdGenOn=(0,inf)'} +] + +# --------------------------------------------------------------------------------} +# --- Main function wrapper +# --------------------------------------------------------------------------------{ +def model_fit(func, x, y, p0=None, bounds=None, **fun_kwargs): + """ + Parameters + ---------- + func: string or function handle + - function handle + - string starting with "fitter: ": (see variable FITTERS) + - "fitter: polynomial_continuous 5' : polyfit order 5 + - "fitter: polynomial_discrete 0 2 3 ': fit polynomial of exponents 0 2 3 + - string providing an expression to evaluate, e.g.: + - "eval: {a}*x + {b}*x**2 " + - string starting with "predef": (see variable MODELS) + - "predef: powerlaw_alpha" : + - "predef: powerlaw_all" : + - "predef: gaussian " : + + x: array of x values + y: array of y values + p0: initial values for parameters, either a string or a dict: + - if a string: the string is converted to a dictionary, assuming key value pairs + example: 'a=0, b=1.3' + - if a dictionary, then keys should corresponds to the parameters of the model + example: {'a':0, 'b':1.3} + bounds: bounds for each parameters, either a string or a dictionary. + NOTE: pi and inf are available to set bounds + - if a string, the string is converted to a dictionary assuming key value pairs + example: 'a=(0,3), b=(-inf,pi)' + - if a dictionary, the keys should corresponds to the parameters of the model + example: {'a':(0,3), 'b':(-inf,pi)} + + Returns + ------- + y_fit: array with same shape as `x` + fitted data. + pfit : fitted parameters + fitter: ModelFitter object + """ + + if isinstance(func,six.string_types) and func.find('fitter:')==0: + # --- This is a high level fitter, we call the class + # The info about the class are storred in the global variable FITTERS + # See e.g. SinusoidFitter, DiscretePolynomialFitter + predef_fitters=[m['id'] for m in FITTERS] + if func not in predef_fitters: + raise Exception('Function `{}` not defined in curve_fitting module\n Available fitters: {}'.format(func,predef_fitters)) + i = predef_fitters.index(func) + FitterDict = FITTERS[i] + consts = FITTERS[i]['consts'] + args, missing = set_common_keys(consts, fun_kwargs) + if len(missing)>0: + raise Exception('Curve fitting with `{}` requires the following arguments {}. Missing: {}'.format(func,consts.keys(),missing)) + # Calling the class + fitter = FitterDict['handle'](x=x, y=y, p0=p0, bounds=bounds, **fun_kwargs) + else: + fitter = ModelFitter(func, x, y, p0=p0, bounds=bounds, **fun_kwargs) + + pfit = [v for _,v in fitter.model['coeffs'].items()] + return fitter.data['y_fit'], pfit , fitter + + +# --------------------------------------------------------------------------------} +# --- Main Class +# --------------------------------------------------------------------------------{ +class ModelFitter(): + def __init__(self,func=None, x=None, y=None, p0=None, bounds=None, **fun_kwargs): + + self.model={ + 'name':None, 'model_function':None, 'consts':fun_kwargs, 'formula': 'unavailable', # model signature + 'coeffs':None, 'formula_num':'unavailable', 'fitted_function':None, 'coeffs_init':p0, 'bounds':bounds, # model fitting + 'R2':None, + } + self.data={'x':x,'y':y,'y_fit':None} + + if func is None: + return + self.set_model(func, **fun_kwargs) + + # Initialize function if present + # Perform fit if data and function is present + if x is not None and y is not None: + self.fit_data(x,y,p0,bounds) + + def set_model(self,func, **fun_kwargs): + if callable(func): + # We don't have much additional info + self.model['model_function'] = func + self.model['name'] = func.__name__ + pass + + elif isinstance(func,six.string_types): + if func.find('predef:')==0: + # --- Minimization from a predefined function + predef_models=[m['id'] for m in MODELS] + if func not in predef_models: + raise Exception('Predefined function `{}` not defined in curve_fitting module\n Available functions: {}'.format(func,predef_models)) + i = predef_models.index(func) + ModelDict = MODELS[i] + self.model['model_function'] = ModelDict['handle'] + self.model['name'] = ModelDict['label'] + self.model['formula'] = ModelDict['formula'] + self.model['coeffs'] = extract_key_num(ModelDict['coeffs']) + self.model['coeffs_init'] = self.model['coeffs'].copy() + self.model['consts'] = extract_key_num(ModelDict['consts']) + self.model['bounds'] = extract_key_tuples(ModelDict['bounds']) + + elif func.find('eval:')==0: + # --- Minimization from a eval string + formula=func[5:] + # Extract coeffs {a} {b} {c}, replace by p[0] + variables, formula_eval = extract_variables(formula) + nParams=len(variables) + if nParams==0: + raise Exception('Formula should contains parameters in curly brackets, e.g.: {a}, {b}, {u_1}. No parameters found in {}'.format(formula)) + + # Check that the formula evaluates + x=np.array([1,2,5])*np.sqrt(2) # some random evaluation vector.. + p=[np.sqrt(2)/4]*nParams # some random initial conditions + try: + y=eval(formula_eval) + y=np.asarray(y) + if y.shape!=x.shape: + raise Exception('The formula does not return an array of same size as the input variable x. The formula must include `x`: {}'.format(formula_eval)) + except SyntaxError: + raise Exception('The formula does not evaluate, syntax error raised: {}'.format(formula_eval)) + except ZeroDivisionError: + pass + + # Creating the actual function + def func(x, p): + return eval(formula_eval) + + self.model['model_function'] = func + self.model['name'] = 'user function' + self.model['formula'] = formula + self.model['coeffs'] = OrderedDict([(k,v) for k,v in zip(variables,p)]) + self.model['coeffs_init'] = self.model['coeffs'].copy() + self.model['consts'] = {} + self.model['bounds'] = None + + else: + raise Exception('func string needs to start with `eval:` of `predef:`, func: {}'.format(func)) + else: + raise Exception('func should be string or callable') + + if fun_kwargs is None: + return + if len(fun_kwargs)==0: + return + if self.model['consts'] is None: + raise Exception('Fun_kwargs provided, but no function constants were defined') + + self.model['consts'], missing = set_common_keys(self.model['consts'], fun_kwargs ) + if len(missing)>0: + raise Exception('Curve fitting with function `{}` requires the following arguments {}. Missing: {}'.format(func.__name__,consts.keys(),missing)) + + def setup_bounds(self, bounds, nParams): + if bounds is not None: + self.model['bounds']=bounds # store in model + bounds=self.model['bounds'] # usemodel bounds as default + if bounds is not None: + if isinstance(bounds ,six.string_types): + bounds=extract_key_tuples(bounds) + + if isinstance(bounds ,dict): + if len(bounds)==0 or 'all' in bounds.keys(): + bounds=([-np.inf]*nParams,[np.inf]*nParams) + elif self.model['coeffs'] is not None: + b1=[] + b2=[] + for k in self.model['coeffs'].keys(): + if k in bounds.keys(): + b1.append(bounds[k][0]) + b2.append(bounds[k][1]) + else: + # TODO merge default bounds + raise Exception('Bounds dictionary is missing the key: `{}`'.format(k)) + bounds=(b1,b2) + else: + raise NotImplementedError('Bounds dictionary with no known model coeffs.') + else: + # so.curve_fit needs a 2-tuple + b1,b2=bounds[0],bounds[1] + if not hasattr(b1,'__len__'): + b1=[b1]*nParams + if not hasattr(b2,'__len__'): + b2=[b2]*nParams + bounds=(b1,b2) + else: + bounds=([-np.inf]*nParams,[np.inf]*nParams) + + self.model['bounds']=bounds # store in model + + def setup_guess(self, p0, bounds, nParams): + """ + Setup initial parameter values for the fit, based on what the user provided, and potentially the bounds + + INPUTS: + - p0: initial parameter values for the fit + - if a string (e.g. " a=1, b=3"), it's converted to a dict + - if a dict, the ordered keys of model['coeffs'] are used to sort p0 + - bounds: tuple of lower and upper bounds for each parameters. + Parameters are ordered as function of models['coeffs'] + bounds[0]: lower bounds or all parameters + bounds[1]: upper bounds or all parameters + + We can assume that the bounds are set + """ + def middleOfBounds(i): + """ return middle of bounds for parameter `i`""" + bLow = bounds[0][i] + bHigh = bounds[0][2] + if (bLow,bHigh)==(-np.inf,np.inf): + p_i=0 + elif bLow==-np.inf: + p_i = -abs(bHigh)*2 + elif bHigh== np.inf: + p_i = abs(bLow)*2 + else: + p_i = (bLow+bHigh)/2 + return p_i + + if isinstance(p0 ,six.string_types): + p0=extract_key_num(p0) + if len(p0)==0: + p0=None + + if p0 is None: + # There is some tricky logic here between the priority of bounds and coeffs + if self.model['coeffs'] is not None: + # We rely on function to give us decent init coefficients + p0 = ([v for _,v in self.model['coeffs'].items()]) + elif bounds is None: + p0 = ([0]*nParams) + else: + # use middle of bounds + p0 = [0]*nParams + for i,(b1,b2) in enumerate(zip(bounds[0],bounds[1])): + p0[i] = middleOfBounds(i) + p0 = (p0) + elif isinstance(p0,dict): + # User supplied a dictionary, we use the ordered keys of coeffs to sort p0 + p0_dict=p0.copy() + if self.model['coeffs'] is not None: + p0=[] + for k in self.model['coeffs'].keys(): + if k in p0_dict.keys(): + p0.append(p0_dict[k]) + else: + raise Exception('Guess dictionary is missing the key: `{}`'.format(k)) + else: + raise NotImplementedError('Guess dictionary with no known model coeffs.') + + + if not hasattr(p0,'__len__'): + p0=(p0,) + + # --- Last check that p0 is within bounds + if bounds is not None: + for p,k,lb,ub in zip(p0, self.model['coeffs'].keys(), bounds[0], bounds[1]): + if pub: + raise Exception('Parameter `{}` has the guess value {}, which is larger than the upper bound ({})'.format(k,p,ub)) + # TODO potentially set it as middle of bounds + + # --- Finally, store the initial guesses in the model + self.model['coeffs_init'] = p0 + + def fit(self, func, x, y, p0=None, bounds=None, **fun_kwargs): + """ Fit model defined by a function to data (x,y) """ + # Setup function + self.set_model(func, **fun_kwargs) + # Fit data to model + self.fit_data(x, y, p0, bounds) + + def clean_data(self,x,y): + x=np.asarray(x) + y=np.asarray(y) + bNaN=~np.isnan(y) + y=y[bNaN] + x=x[bNaN] + bNaN=~np.isnan(x) + y=y[bNaN] + x=x[bNaN] + self.data['x']=x + self.data['y']=y + return x,y + + def fit_data(self, x, y, p0=None, bounds=None): + """ fit data, assuming a model is already setup""" + if self.model['model_function'] is None: + raise Exception('Call set_function first') + + # Cleaning data, and store it in object + x,y=self.clean_data(x,y) + + # nParams + if isinstance(p0 ,six.string_types): + p0=extract_key_num(p0) + if len(p0)==0: + p0=None + if p0 is not None: + if hasattr(p0,'__len__'): + nParams=len(p0) + else: + nParams=1 + elif self.model['coeffs'] is not None: + nParams=len(self.model['coeffs']) + else: + raise Exception('Initial guess `p0` needs to be provided since we cant infer the size of the model coefficients.') + if self.model['coeffs'] is not None: + if len(self.model['coeffs'])!=nParams: + raise Exception('Inconsistent dimension between model guess (size {}) and the model parameters (size {})'.format(nParams,len(self.model['coeffs']))) + + # Bounds + self.setup_bounds(bounds,nParams) + + # Initial conditions + self.setup_guess(p0,self.model['bounds'],nParams) + + # Fitting + minimize_me = lambda x, *p : self.model['model_function'](x, p, **self.model['consts']) + pfit, pcov = so.curve_fit(minimize_me, x, y, p0=self.model['coeffs_init'], bounds=self.model['bounds']) + + # --- Reporting information about the fit (after the fit) + y_fit = self.model['model_function'](x, pfit, **self.model['consts']) + self.store_fit_info(y_fit, pfit) + + # --- Return a fitted function + self.model['fitted_function'] = lambda xx: self.model['model_function'](xx, pfit, **self.model['consts']) + + def store_fit_info(self, y_fit, pfit): + # --- Reporting information about the fit (after the fit) + self.data['y_fit']=y_fit + self.model['R2'] = rsquare(self.data['y'], y_fit) + if self.model['coeffs'] is not None: + if not isinstance(self.model['coeffs'], OrderedDict): + raise Exception('Coeffs need to be of type OrderedDict') + for k,v in zip(self.model['coeffs'].keys(), pfit): + self.model['coeffs'][k]=v + + # Replace numerical values in formula + if self.model['formula'] is not None: + formula_num=self.model['formula'] + for k,v in self.model['coeffs'].items(): + formula_num = formula_num.replace('{'+k+'}',str(v)) + for k,v in self.model['consts'].items(): + formula_num = formula_num.replace('{'+k+'}',str(v)) + self.model['formula_num'] = formula_num + + def formula_num(self, fmt=None): + """ return formula with coeffs and consts evaluted numerically""" + if fmt is None: + fmt_fun = lambda x: str(x) + elif isinstance(fmt,six.string_types): + fmt_fun = lambda x: ('{'+fmt+'}').format(x) + elif callable(fmt): + fmt_fun = fmt + formula_num=self.model['formula'] + for k,v in self.model['coeffs'].items(): + formula_num = formula_num.replace('{'+k+'}',fmt_fun(v)) + for k,v in self.model['consts'].items(): + formula_num = formula_num.replace('{'+k+'}',fmt_fun(v)) + return formula_num + + + + def plot(self, x=None, fig=None, ax=None): + if x is None: + x=self.data['x'] + + sFormula = _clean_formula(self.model['formula'],latex=True) + + import matplotlib.pyplot as plt + import matplotlib.patches as mpatches + + if fig is None: + fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) + fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) + + ax.plot(self.data['x'], self.data['y'], '.', label='Data') + ax.plot(x, self.model['fitted_function'](x), '-', label='Model ' + sFormula) + + # Add extra info to the legend + handles, labels = ax.get_legend_handles_labels() # get existing handles and labels + empty_patch = mpatches.Patch(color='none', label='Extra label') # create a patch with no color + for k,v in self.model['coeffs'].items(): + handles.append(empty_patch) # add new patches and labels to list + labels.append(r'${:s}$ = {}'.format(pretty_param(k),pretty_num_short(v))) + handles.append(empty_patch) # add new patches and labels to list + labels.append('$R^2$ = {}'.format(pretty_num_short(self.model['R2']))) + ax.legend(handles, labels) + + + #ax.set_xlabel('') + #ax.set_ylabel('') + return fig,ax + + def print_guessbounds(self): + s='' + p0 = self.model['coeffs_init'] + bounds = self.model['bounds'] + for i,(k,v) in enumerate(self.model['coeffs'].items()): + print( (pretty_num(bounds[0][i]),pretty_num(p0[i]), pretty_num(bounds[1][i])) ) + s+='{:15s}: {:10s} < {:10s} < {:10s}\n'.format(k, pretty_num(bounds[0][i]),pretty_num(p0[i]), pretty_num(bounds[1][i])) + print(s) + + + def __repr__(self): + s='<{} object> with fields:\n'.format(type(self).__name__) + s+=' - data, dictionary with keys: \n' + s+=' - x: [{} ... {}], n: {} \n'.format(self.data['x'][0],self.data['x'][-1],len(self.data['x'])) + s+=' - y: [{} ... {}], n: {} \n'.format(self.data['y'][0],self.data['y'][-1],len(self.data['y'])) + s+=' - model, dictionary with keys: \n' + for k,v in self.model.items(): + s=s+' - {:15s}: {}\n'.format(k,v) + return s + + +# --------------------------------------------------------------------------------} +# --- Wrapper for predefined fitters +# --------------------------------------------------------------------------------{ +class PredefinedModelFitter(ModelFitter): + def __init__(self, x=None, y=None, p0=None, bounds=None, **kwargs): + ModelFitter.__init__(self,x=None, y=None, p0=p0, bounds=bounds) # NOTE: not passing data + + self.kwargs=kwargs + + if x is not None and y is not None: + self.fit_data(x,y,p0,bounds) + + def setup_model(self): + """ + Setup model: + - guess/coeffs_init: return params in format needed for curve_fit (p0,p1,p2,p3) + - bound : bounds in format needed for curve_fit ((low0,low1,low2), (high0, high1)) + - coeffs : OrderedDict, necessary for user print + - formula : necessary for user print + """ + #self.model['coeffs'] = OrderedDict([(var,1) for i,var in enumerate(variables)]) + #self.model['formula'] = '' + #self.model['coeffs_init']=p_guess + #self.model['bounds']=bounds_guess + raise NotImplementedError('To be implemented by child class') + + def model_function(self, x, p): + raise NotImplementedError('To be implemented by child class') + + def fit_data(self, x, y, p0=None, bounds=None): + # Cleaning data + x,y=self.clean_data(x,y) + + # --- setup model + # guess initial parameters, potential bounds, and set necessary data + self.setup_model() + + # --- Minimization + minimize_me = lambda x, *p : self.model_function(x, p) + if self.model['bounds'] is None: + pfit, pcov = so.curve_fit(minimize_me, x, y, p0=self.model['coeffs_init']) + else: + pfit, pcov = so.curve_fit(minimize_me, x, y, p0=self.model['coeffs_init'], bounds=self.model['bounds']) + # --- Reporting information about the fit (after the fit) + # And Return a fitted function + y_fit = self.model_function(x, pfit) + self.model['fitted_function']=lambda xx : self.model_function(xx, pfit) + self.store_fit_info(y_fit, pfit) + + def plot_guess(self, x=None, fig=None, ax=None): + """ plotthe guess values""" + if x is None: + x=self.data['x'] + import matplotlib.pyplot as plt + if fig is None: + fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) + fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) + + p_guess = self.model['coeffs_init'] + + ax.plot(self.data['x'], self.data['y'] , '.', label='Data') + ax.plot(x, self.model_function(x,p_guess), '-', label='Model at guessed parameters') + ax.legend() + + +# --------------------------------------------------------------------------------} +# --- Predefined fitters +# --------------------------------------------------------------------------------{ +class SecondOrderFitterImpulse(PredefinedModelFitter): + + def model_function(self, x, p): + return secondorder_impulse(x, p) + + def setup_model(self): + """ p = (A, omega0, zeta, B, t0) """ + self.model['coeffs'] = OrderedDict([('A',1),('omega',1),('zeta',0.01),('B',0),('t0',0)]) + self.model['formula'] = '{A}*exp(-{zeta}*{omega}*(x-{x0}))*sin({omega}*sqrt(1-{zeta}**2)))+{B}' + + # --- Guess Initial values + x, y = self.data['x'],self.data['y'] + # TODO use signal + dt = x[1]-x[0] + omega0 = main_frequency(x,y) + A = np.max(y) - np.min(y) + B = np.mean(y) + zeta = 0.1 + y_start = y[0]+0.01*A + bDeviate = np.argwhere(abs(y-y_start)>abs(y_start-y[0]))[0] + t0 = x[bDeviate[0]] + p_guess = np.array([A, omega0, zeta, B, t0]) + self.model['coeffs_init'] = p_guess + # --- Set Bounds + T = x[-1]-x[0] + dt = x[1]-x[0] + om_min = 2*np.pi/T/2 + om_max = 2*np.pi/dt/2 + b_A = (A*0.1,A*3) + b_om = (om_min,om_max) + b_zeta = (0,1) + b_B = (np.min(y),np.max(y)) + b_x0 = (np.min(x),np.max(x)) + self.model['bounds'] = ((b_A[0],b_om[0],b_zeta[0],b_B[0],b_x0[0]),(b_A[1],b_om[1],b_zeta[1],b_B[1],b_x0[1])) + #self.plot_guess(); import matplotlib.pyplot as plt; plt.show() + #self.print_guessbounds(); + +class SecondOrderFitterStep(PredefinedModelFitter): + + def model_function(self, x, p): + return secondorder_step(x, p) + + def setup_model(self): + """ p = (A, omega0, zeta, B, t0) """ + self.model['coeffs'] = OrderedDict([('A',1),('omega',1),('zeta',0.01),('B',0),('t0',0)]) + self.model['formula'] ='{A}*(1-exp(-{zeta}*{omega}*(x-{x0}))/sqrt(1-{zeta}**2) * cos({omega}*sqrt(1-{zeta}**2)-arctan({zeta}/sqrt(1-{zeta}**2)))) +{B}' + # --- Guess Initial values + x, y = self.data['x'],self.data['y'] + # TODO use signal + omega0 = main_frequency(x,y) + A = np.max(y) - np.min(y) + B = y[0] + zeta = 0.1 + y_start = y[0]+0.01*A + bDeviate = np.argwhere(abs(y-y_start)>abs(y_start-y[0]))[0] + t0 = x[bDeviate[0]] + p_guess = np.array([A, omega0, zeta, B, t0]) + self.model['coeffs_init'] = p_guess + # --- Set Bounds + T = x[-1]-x[0] + dt = x[1]-x[0] + om_min = 2*np.pi/T/2 + om_max = 2*np.pi/dt/2 + b_A = (A*0.1,A*3) + b_om = (om_min,om_max) + b_zeta = (0,1) + b_B = (np.min(y),np.max(y)) + b_x0 = (np.min(x),np.max(x)) + self.model['bounds'] = ((b_A[0],b_om[0],b_zeta[0],b_B[0],b_x0[0]),(b_A[1],b_om[1],b_zeta[1],b_B[1],b_x0[1])) + #self.plot_guess(); import matplotlib.pyplot as plt; plt.show() + #self.print_guessbounds(); + +# --------------------------------------------------------------------------------} +# --- Predefined fitter +# --------------------------------------------------------------------------------{ +class ContinuousPolynomialFitter(ModelFitter): + def __init__(self,order=None, x=None, y=None, p0=None, bounds=None): + ModelFitter.__init__(self,x=None, y=None, p0=p0, bounds=bounds) + self.setOrder(int(order)) + if order is not None and x is not None and y is not None: + self.fit_data(x,y,p0,bounds) + + def setOrder(self, order): + self.order=order + if order is not None: + variables= string.ascii_lowercase[:order+1] + self.model['coeffs'] = OrderedDict([(var,1) for i,var in enumerate(variables)]) + formula = ' + '.join(['{}*x**{}'.format('{'+var+'}',order-i) for i,var in enumerate(variables)]) + self.model['formula'] = _clean_formula(formula) + + def fit_data(self, x, y, p0=None, bounds=None): + if self.order is None: + raise Exception('Polynomial Fitter not set, call function `setOrder` to set order') + # Cleaning data + x,y=self.clean_data(x,y) + + nParams=self.order+1 + # Bounds + self.setup_bounds(bounds, nParams) # TODO + # Initial conditions + self.setup_guess(p0, bounds, nParams) # TODO + + # Fitting + pfit = np.polyfit(x,y,self.order) + + # --- Reporting information about the fit (after the fit) + y_fit = np.polyval(pfit,x) + self.store_fit_info(y_fit, pfit) + + # --- Return a fitted function + self.model['fitted_function']=lambda xx : np.polyval(pfit,xx) + + +class DiscretePolynomialFitter(ModelFitter): + def __init__(self,exponents=None, x=None, y=None, p0=None, bounds=None): + ModelFitter.__init__(self,x=None, y=None, p0=p0, bounds=bounds) + self.setExponents(exponents) + if exponents is not None and x is not None and y is not None: + self.fit_data(x,y,p0,bounds) + + def setExponents(self, exponents): + self.exponents=exponents + if exponents is not None: + #exponents=-np.sort(-np.asarray(exponents)) + self.exponents=exponents + variables= string.ascii_lowercase[:len(exponents)] + self.model['coeffs'] = OrderedDict([(var,1) for i,var in enumerate(variables)]) + formula = ' + '.join(['{}*x**{}'.format('{'+var+'}',e) for var,e in zip(variables,exponents)]) + self.model['formula'] = _clean_formula(formula) + + def fit_data(self, x, y, p0=None, bounds=None): + if self.exponents is None: + raise Exception('Polynomial Fitter not set, call function `setExponents` to set exponents') + # Cleaning data, and store it in object + x,y=self.clean_data(x,y) + + nParams=len(self.exponents) + # Bounds + self.setup_bounds(bounds, nParams) # TODO + # Initial conditions + self.setup_guess(p0, bounds, nParams) # TODO + + X_poly=np.array([]) + for i,e in enumerate(self.exponents): + if i==0: + X_poly = np.array([x**e]) + else: + X_poly = np.vstack((X_poly,x**e)) + try: + pfit = np.linalg.lstsq(X_poly.T, y, rcond=None)[0] + except: + pfit = np.linalg.lstsq(X_poly.T, y) + + # --- Reporting information about the fit (after the fit) + y_fit= np.dot(pfit, X_poly) + self.store_fit_info(y_fit, pfit) + + # --- Return a fitted function + def fitted_function(xx): + y=np.zeros(xx.shape) + for i,(e,c) in enumerate(zip(self.exponents,pfit)): + y += c*x**e + return y + self.model['fitted_function']=fitted_function + + +class SinusoidFitter(ModelFitter): + def __init__(self, physical=False, x=None, y=None, p0=None, bounds=None): + ModelFitter.__init__(self, x=None, y=None, p0=p0, bounds=bounds) + #self.setOrder(int(order)) + self.physical=physical + if physical: + self.model['coeffs'] = OrderedDict([('A',1),('f',1),('phi',0),('B',0)]) + self.model['formula'] = '{A} * sin(2*pi*({f}*x + {phi}/360)) + {B}' + else: + self.model['coeffs'] = OrderedDict([('A',1),('omega',1),('phi',0),('B',0)]) + self.model['formula'] = '{A} * sin({omega}*x + {phi}) + {B}' + + if x is not None and y is not None: + self.fit_data(x,y,p0,bounds) + + def fit_data(self, x, y, p0=None, bounds=None): + # Cleaning data + x,y=self.clean_data(x,y) + + # TODO use signal + guess_freq= main_frequency(x,y)/(2*np.pi) # [Hz] + guess_amp = np.std(y) * 2.**0.5 + guess_offset = np.mean(y) + if self.physical: + guess = np.array([guess_amp, guess_freq, 0., guess_offset]) + minimize_me = lambda x, *p : sinusoid_f(x, p) + else: + guess = np.array([guess_amp, 2.*np.pi*guess_freq, 0., guess_offset]) + minimize_me = lambda x, *p : sinusoid(x, p) + self.model['coeffs_init'] = guess + + pfit, pcov = so.curve_fit(minimize_me, x, y, p0=guess) + + # --- Reporting information about the fit (after the fit) + # And Return a fitted function + if self.physical: + y_fit = sinusoid_f(x, pfit) + self.model['fitted_function']=lambda xx : sinusoid_f(xx, pfit) + else: + y_fit = sinusoid(x, pfit) + self.model['fitted_function']=lambda xx : sinusoid(xx, pfit) + self.store_fit_info(y_fit, pfit) + + + +class GeneratorTorqueFitter(ModelFitter): + def __init__(self,x=None, y=None, p0=None, bounds=None): + ModelFitter.__init__(self,x=None, y=None, p0=p0, bounds=bounds) + +# RtGnSp, RtTq , Rgn2K , SlPc , SpdGenOn = p +# {'label':'Generator Torque', 'handle': gentorque, 'id':'predef: gentorque', +# 'formula': '{RtGnSp} , {RtTq} , {Rgn2K} , {SlPc} , {SpdGenOn}', + self.model['coeffs']= extract_key_num('RtGnSp=100 , RtTq=1000 , Rgn2K=0.01 ,SlPc=5 , SpdGenOn=0') +# 'consts' :None, +# 'bounds' :'RtGnSp=(0.1,inf) , RtTq=(1,inf), Rgn2K=(0.0,0.1) ,SlPc=(0,20) , SpdGenOn=(0,inf)'} + if x is not None and y is not None: + self.fit_data(x,y,p0,bounds) + + def fit_data(self, x, y, p0=None, bounds=None): + #nParams=5 + ## Bounds + #self.setup_bounds(bounds,nParams) # TODO + ## Initial conditions + #self.setup_guess(p0,bounds,nParams) # TODO + + # Cleaning data, and store it in object + x,y=self.clean_data(x,y) + + I = np.argsort(x) + x=x[I] + y=y[I] + + # Estimating deltas + xMin, xMax=np.min(x),np.max(x) + yMin, yMax=np.min(y),np.max(y) + DeltaX = (xMax-xMin)*0.02 + DeltaY = (yMax-yMin)*0.02 + + # Binning data + x_bin=np.linspace(xMin,xMax,min(200,len(x))) + x_lin=x_bin[0:-1]+np.diff(x_bin) + #y_lin=np.interp(x_lin,x,y) # TODO replace by bining + y_lin = np.histogram(y, x_bin, weights=y)[0]/ np.histogram(y, x_bin)[0] + y_lin, _, _ = stats.binned_statistic(x, y, statistic='mean', bins=x_bin) + x_lin, _, _ = stats.binned_statistic(x, x, statistic='mean', bins=x_bin) + bNaN=~np.isnan(y_lin) + y_lin=y_lin[bNaN] + x_lin=x_lin[bNaN] + + # --- Find good guess of parameters based on data + # SpdGenOn + iOn = np.where(y>0)[0][0] + SpdGenOn_0 = x[iOn] + SpdGenOn_Bnds = (max(x[iOn]-DeltaX,xMin), min(x[iOn]+DeltaX,xMax)) + # Slpc + Slpc_0 = 5 + Slpc_Bnds = (0,10) + # RtTq + RtTq_0 = yMax + RtTq_Bnds = (yMax-DeltaY, yMax+DeltaY) + # RtGnSp + iCloseRt = np.where(y>yMax*0.50)[0][0] + RtGnSp_0 = x[iCloseRt] + RtGnSp_Bnds = ( RtGnSp_0 -DeltaX*2, RtGnSp_0+DeltaX*2) + # Rgn2K + #print('>>>',SpdGenOn_0, RtGnSp_0) + bR2=np.logical_and(x>SpdGenOn_0, x ['a','b'] + The variables are replaced with p[0],..,p[n] in order of appearance + """ + regex = r"\{(.*?)\}" + matches = re.finditer(regex, sFormula, re.DOTALL) + formula_eval=sFormula + variables=[] + ivar=0 + for i, match in enumerate(matches): + for groupNum in range(0, len(match.groups())): + var = match.group(1) + if var not in variables: + variables.append(var) + formula_eval = formula_eval.replace('{'+match.group(1)+'}','p[{:d}]'.format(ivar)) + ivar+=1 + return variables, formula_eval + + +def extract_key_tuples(text): + """ + all=(0.1,-2),b=(inf,0), c=(-inf,0.3e+10) + """ + if text is None: + return {} + regex = re.compile(r'(?P[\w\-]+)=\((?P[0-9+epinf.-]*?),(?P[0-9+epinf.-]*?)\)($|,)') + return {match.group("key"): (float(match.group("value1")),float(match.group("value2"))) for match in regex.finditer(text.replace(' ',''))} + +def extract_key_num(text): + """ + all=0.1, b=inf, c=-0.3e+10 + """ + if text is None: + return {} + regex = re.compile(r'(?P[\w\-]+)=(?P[0-9+epinf.-]*?)($|,)') + return OrderedDict([(match.group("key"), float(match.group("value"))) for match in regex.finditer(text.replace(' ',''))]) + +def extract_key_miscnum(text): + """ + all=0.1, b=(inf,0), c=[-inf,0.3e+10,10,11]) + """ + def isint(s): + try: + int(s) + return True + except: + return False + + if text is None: + return {} + sp=re.compile('([\w]+)=').split(text.replace(' ','')) + if len(sp)<3: + return {} + sp=sp[1:] + keys = sp[0::2] + values = sp[1::2] + d={} + for (k,v) in zip(keys,values): + if v.find('(')>=0: + v=v.replace('(','').replace(')','') + v=v.split(',') + vect=tuple([float(val) for val in v if len(val.strip())>0]) + elif v.find('[')>=0: + v=v.replace('[','').replace(']','') + v=v.split(',') + vect=[int(val) if isint(val) else float(val) for val in v if len(val.strip())>0] # NOTE returning lists + elif v.find('True')>=0: + v=v.replace(',','').strip() + vect=True + elif v.find('False')>=0: + v=v.replace(',','').strip() + vect=False + else: + v=v.replace(',','').strip() + vect=int(v) if isint(v) else float(v) + d[k]=vect + return d + +def set_common_keys(dict_target, dict_source): + """ Set a dictionary using another one, missing keys in source dictionary are reported""" + keys_missing=[] + for k in dict_target.keys(): + if k in dict_source.keys(): + dict_target[k]=dict_source[k] + else: + keys_missing.append(k) + return dict_target, keys_missing + +def _clean_formula(s, latex=False): + s = s.replace('+-','-').replace('**1','').replace('*x**0','') + s = s.replace('np.','') + if latex: + #s = s.replace('{','$').replace('}','$') + s = s.replace('phi',r'\phi') + s = s.replace('alpha',r'\alpha') + s = s.replace('beta' ,r'\alpha') + s = s.replace('zeta' ,r'\zeta') + s = s.replace('mu' ,r'\mu' ) + s = s.replace('pi' ,r'\pi' ) + s = s.replace('sigma',r'\sigma') + s = s.replace('omega',r'\omega') + s = s.replace('_ref',r'_{ref}') # make this general + s = s.replace(r'(',r'{(') + s = s.replace(r')',r')}') + s = s.replace(r'**',r'^') + s = s.replace(r'*', '') + s = s.replace('sin',r'\sin') + s = s.replace('exp',r'\exp') + s = s.replace('sqrt',r'\sqrt') + s = r'$'+s+r'$' + else: + s = s.replace('{','').replace('}','') + return s + + +def main_frequency(t,y): + """ + Returns main frequency of a signal + NOTE: this tool below to welib.tools.signal, but put here for convenience + """ + dt = t[1]-t[0] # assume uniform spacing of time and frequency + om = np.fft.fftfreq(len(t), (dt))*2*np.pi + Fyy = abs(np.fft.fft(y)) + omega = abs(om[np.argmax(Fyy[1:])+1]) # exclude the zero frequency (mean) + return omega + +def rsquare(y, f): + """ Compute coefficient of determination of data fit model and RMSE + [r2] = rsquare(y,f) + RSQUARE computes the coefficient of determination (R-square) value from + actual data Y and model data F. + INPUTS + y : Actual data + f : Model fit + OUTPUT + R2 : Coefficient of determination + """ + # Compare inputs + if not np.all(y.shape == f.shape) : + raise Exception('Y and F must be the same size') + # Check for NaN + tmp = np.logical_not(np.logical_or(np.isnan(y),np.isnan(f))) + y = y[tmp] + f = f[tmp] + R2 = max(0,1-np.sum((y-f)**2)/np.sum((y-np.mean(y))** 2)) + return R2 + +def pretty_param(s): + if s in ['alpha','beta','delta','gamma','epsilon','zeta','lambda','mu','nu','pi','rho','sigma','phi','psi','omega']: + s = r'\{}'.format(s) + s = s.replace('_ref',r'_{ref}') # make this general.. + return s + +def pretty_num(x): + if abs(x)<1000 and abs(x)>1e-4: + return "{:9.4f}".format(x) + else: + return '{:.3e}'.format(x) + +def pretty_num_short(x,digits=3): + if digits==4: + if abs(x)<1000 and abs(x)>1e-1: + return "{:.4f}".format(x) + else: + return "{:.4e}".format(x) + elif digits==3: + if abs(x)<1000 and abs(x)>1e-1: + return "{:.3f}".format(x) + else: + return "{:.3e}".format(x) + elif digits==2: + if abs(x)<1000 and abs(x)>1e-1: + return "{:.2f}".format(x) + else: + return "{:.2e}".format(x) + + +if __name__ == '__main__': + # --- Writing example models to file for pyDatView tests + a,b,c = 2.0, 3.0, 4.0 + u_ref,z_ref,alpha=10,12,0.12 + mu,sigma=0.5,1.2 + x = np.linspace(0.1,30,20) + A,k,B=0.5,1.2,10 + y_exp=expdecay(x,(A,k,B)) + A, k = 10, 2.3, + y_weib=weibull_pdf(x,(A,k)) + y_log=logarithmic(x,(a,b)) + exponents=[0,3,5] + y_poly = a + b*x**3 + c*x**5 + y_power=powerlaw_all(x,(alpha,u_ref,z_ref)) + y_gauss=gaussian(x,(mu,sigma)) + A= 101; B= -200.5; omega = 0.4; phi = np.pi/3 + y_sin=sinusoid(x,(A,omega,phi,B)) + np.random.normal(0, 0.1, len(x)) + M=np.column_stack((x,y_poly,y_power,y_gauss,y_gauss+10,y_weib,y_exp,y_log,y_sin)) + np.savetxt('../TestFit.csv',M,header='x,poly,power,gauss,gauss_off,weib,expdecay,log,sin',delimiter=',') diff --git a/pydatview/tools/damping.py b/pydatview/tools/damping.py index 9127752..0264935 100644 --- a/pydatview/tools/damping.py +++ b/pydatview/tools/damping.py @@ -15,9 +15,9 @@ def indexes(y, thres=0.3, min_dist=1, thres_abs=False): ---------- y : ndarray (signed) 1D amplitude data to search for peaks. - thres : float between [0., 1.] - Normalized threshold. Only the peaks with amplitude higher than the + thres : float, defining threshold. Only the peaks with amplitude higher than the threshold will be detected. + if thres_abs is False: between [0., 1.], normalized threshold. min_dist : int Minimum distance between each detected peak. The peak with the highest amplitude is preferred to satisfy this constraint. diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index 973d142..3c53d9d 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -98,7 +98,7 @@ def rainflow_windap(signal, levels=255., thresshold=(255 / 50)): if np.nanmax(signal) > 0: gain = np.nanmax(signal) / levels signal = signal / gain - signal = np.round(signal).astype(np.int) + signal = np.round(signal).astype(int) # If possible the module is compiled using cython otherwise the python implementation is used @@ -468,7 +468,7 @@ def peak_trough(x, R): #cpdef np.ndarray[long,ndim=1] peak_trough(np.ndarray[lo MINZO = 1 MAXZO = 2 ENDZO = 3 - S = np.zeros(x.shape[0] + 1, dtype=np.int) + S = np.zeros(x.shape[0] + 1, dtype=int) L = x.shape[0] goto = BEGIN diff --git a/pydatview/tools/signal.py b/pydatview/tools/signal.py index 2ea999a..b8371a7 100644 --- a/pydatview/tools/signal.py +++ b/pydatview/tools/signal.py @@ -1,366 +1,708 @@ -from __future__ import division -import numpy as np -from numpy.random import rand -import pandas as pd - - -# --- List of available filters -FILTERS=[ - {'name':'Moving average','param':100,'paramName':'Window Size','paramRange':[0,100000],'increment':1}, - {'name':'Low pass 1st order','param':1.0,'paramName':'Cutoff Freq.','paramRange':[0.0001,100000],'increment':0.1}, - {'name':'High pass 1st order','param':1.0,'paramName':'Cutoff Freq.','paramRange':[0.0001,100000],'increment':0.1}, -] - -SAMPLERS=[ - {'name':'Replace', 'param':[], 'paramName':'New x'}, - {'name':'Insert', 'param':[], 'paramName':'Insert list'}, - {'name':'Remove', 'param':[], 'paramName':'Remove list'}, - {'name':'Every n', 'param':2 , 'paramName':'n'}, - {'name':'Time-based', 'param':0.01 , 'paramName':'Sample time (s)'}, - {'name':'Delta x', 'param':0.1, 'paramName':'dx'}, -] - - - -def reject_outliers(y, x=None, m = 2., replaceNaN=True): - """ Reject outliers: - If replaceNaN is true: they are replaced by NaN - Otherwise they are removed - """ - if m==0: - # No rejection... - pass - else: - dd = np.abs(y - np.nanmedian(y)) - mdev = np.nanmedian(dd) - if mdev: - ss = dd/mdev - b=ss=0, j< len(xp)-1) - bLower =j<0 - bUpper =j>=len(xp)-1 - jOK = j[bOK] - #import pdb; pdb.set_trace() - dd[bOK] = (x[bOK] - xp[jOK]) / (xp[jOK + 1] - xp[jOK]) - jBef=j - jAft=j+1 - # - # Use first and last values for anything beyond xp - jAft[bUpper] = len(xp)-1 - jBef[bUpper] = len(xp)-1 - jAft[bLower] = 0 - jBef[bLower] = 0 - if extrap=='bounded': - pass - # OK - elif extrap=='nan': - dd[~bOK] = np.nan - else: - raise NotImplementedError() - - return (1 - dd) * fp[:,jBef] + fp[:,jAft] * dd - -def resample_interp(x_old, x_new, y_old=None, df_old=None): - #x_new=np.sort(x_new) - if df_old is not None: - # --- Method 1 (pandas) - #df_new = df_old.copy() - #df_new = df_new.set_index(x_old) - #df_new = df_new.reindex(df_new.index | x_new) - #df_new = df_new.interpolate().loc[x_new] - #df_new = df_new.reset_index() - # --- Method 2 interp storing dx - data_new=multiInterp(x_new, x_old, df_old.values.T) - df_new = pd.DataFrame(data=data_new.T, columns=df_old.columns.values) - return x_new, df_new - - if y_old is not None: - return x_new, np.interp(x_new, x_old, y_old) - - -def applySamplerDF(df_old, x_col, sampDict): - x_old=df_old[x_col].values - x_new, df_new =applySampler(x_old, y_old=None, sampDict=sampDict, df_old=df_old) - df_new[x_col]=x_new - return df_new - - -def applySampler(x_old, y_old, sampDict, df_old=None): - - param = np.asarray(sampDict['param']).ravel() - - if sampDict['name']=='Replace': - if len(param)==0: - raise Exception('Error: At least one value is required to resample the x values with') - x_new = param - return resample_interp(x_old, x_new, y_old, df_old) - - elif sampDict['name']=='Insert': - if len(param)==0: - raise Exception('Error: provide a list of values to insert') - x_new = np.sort(np.concatenate((x_old.ravel(),param))) - return resample_interp(x_old, x_new, y_old, df_old) - - elif sampDict['name']=='Remove': - I=[] - if len(param)==0: - raise Exception('Error: provide a list of values to remove') - for d in param: - Ifound= np.where(np.abs(x_old-d)<1e-3)[0] - if len(Ifound)>0: - I+=list(Ifound.ravel()) - x_new=np.delete(x_old,I) - return resample_interp(x_old, x_new, y_old, df_old) - - elif sampDict['name']=='Delta x': - if len(param)==0: - raise Exception('Error: provide value for dx') - dx = param[0] - x_new = np.arange(x_old[0], x_old[-1]+dx/2, dx) - return resample_interp(x_old, x_new, y_old, df_old) - - elif sampDict['name']=='Every n': - if len(param)==0: - raise Exception('Error: provide value for n') - n = int(param[0]) - if n==0: - raise Exception('Error: |n| should be at least 1') - - x_new=x_old[::n] - if df_old is not None: - return x_new, (df_old.copy()).iloc[::n,:] - if y_old is not None: - return x_new, y_old[::n] - - elif sampDict['name'] == 'Time-based': - if len(param) == 0: - raise Exception('Error: provide value for new sampling time') - sample_time = float(param[0]) - if sample_time <= 0: - raise Exception('Error: sample time must be positive') - - time_index = pd.TimedeltaIndex(x_old, unit="S") - x_new = pd.Series(x_old, index=time_index).resample("{:f}S".format(sample_time)).mean().interpolate().values - - if df_old is not None: - df_new = df_old.set_index(time_index, inplace=False).resample("{:f}S".format(sample_time)).mean() - df_new = df_new.interpolate().reset_index(drop=True) - return x_new, df_new - if y_old is not None: - y_new = pd.Series(y_old, index=time_index).resample("{:f}S".format(sample_time)).mean() - y_new = y_new.interpolate().values - return x_new, y_new - - else: - raise NotImplementedError('{}'.format(sampDict)) - pass - -# --------------------------------------------------------------------------------} -# --- Filters -# --------------------------------------------------------------------------------{ -# def moving_average(x, w): -# #t_new = np.arange(0,Tmax,dt) -# #nt = len(t_new) -# #nw=400 -# #u_new = moving_average(np.floor(np.linspace(0,3,nt+nw-1))*3+3.5, nw) -# return np.convolve(x, np.ones(w), 'valid') / w -# def moving_average(x,N,mode='same'): -# y=np.convolve(x, np.ones((N,))/N, mode=mode) -# return y -def moving_average(a, n=3) : - """ - perform moving average, return a vector of same length as input - - NOTE: also in kalman.filters - """ - a = a.ravel() - a = np.concatenate(([a[0]]*(n-1),a)) # repeating first values - ret = np.cumsum(a, dtype = float) - ret[n:] = ret[n:] - ret[:-n] - ret=ret[n - 1:] / n - return ret - -def lowpass1(y, dt, fc=3) : - """ - 1st order low pass filter - """ - tau=1/(2*np.pi*fc) - alpha=dt/(tau+dt) - y_filt=np.zeros(y.shape) - y_filt[0]=y[0] - for i in np.arange(1,len(y)): - y_filt[i]=alpha*y[i] + (1-alpha)*y_filt[i-1] - return y_filt - -def highpass1(y, dt, fc=3) : - """ - 1st order high pass filter - """ - tau=1/(2*np.pi*fc) - alpha=tau/(tau+dt) - y_filt=np.zeros(y.shape) - y_filt[0]=0 - for i in np.arange(1,len(y)): - y_filt[i]=alpha*y_filt[i-1] + alpha*(y[i]-y[i-1]) - m0=np.mean(y) - m1=np.mean(y_filt) - y_filt+=m0-m1 - return y_filt - - -def applyFilter(x, y,filtDict): - if filtDict['name']=='Moving average': - return moving_average(y, n=np.round(filtDict['param']).astype(int)) - elif filtDict['name']=='Low pass 1st order': - dt = x[1]-x[0] - return lowpass1(y, dt=dt, fc=filtDict['param']) - elif filtDict['name']=='High pass 1st order': - dt = x[1]-x[0] - return highpass1(y, dt=dt, fc=filtDict['param']) - else: - raise NotImplementedError('{}'.format(filtDict)) - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -def zero_crossings(y,x=None,direction=None): - """ - Find zero-crossing points in a discrete vector, using linear interpolation. - - direction: 'up' or 'down', to select only up-crossings or down-crossings - - returns: - x values xzc such that y(yzc)==0 - indexes izc, such that the zero is between y[izc] (excluded) and y[izc+1] (included) - - if direction is not provided, also returns: - sign, equal to 1 for up crossing - """ - if x is None: - x=np.arange(len(y)) - - if np.any((x[1:] - x[0:-1]) <= 0.0): - raise Exception('x values need to be in ascending order') - - # Indices before zero-crossing - iBef = np.where(y[1:]*y[0:-1] < 0.0)[0] - - # Find the zero crossing by linear interpolation - xzc = x[iBef] - y[iBef] * (x[iBef+1] - x[iBef]) / (y[iBef+1] - y[iBef]) - - # Selecting points that are exactly 0 and where neighbor change sign - iZero = np.where(y == 0.0)[0] - iZero = iZero[np.where((iZero > 0) & (iZero < x.size-1))] - iZero = iZero[np.where(y[iZero-1]*y[iZero+1] < 0.0)] - - # Concatenate - xzc = np.concatenate((xzc, x[iZero])) - iBef = np.concatenate((iBef, iZero)) - - # Sort - iSort = np.argsort(xzc) - xzc, iBef = xzc[iSort], iBef[iSort] - - # Return up-crossing, down crossing or both - sign = np.sign(y[iBef+1]-y[iBef]) - if direction == 'up': - I= np.where(sign==1)[0] - return xzc[I],iBef[I] - elif direction == 'down': - I= np.where(sign==-1)[0] - return xzc[I],iBef[I] - elif direction is not None: - raise Exception('Direction should be either `up` or `down`') - return xzc, iBef, sign - - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -def correlation(x, nMax=80, dt=1, method='manual'): - """ - Compute auto correlation of a signal - """ - nvec = np.arange(0,nMax) - sigma2 = np.var(x) - R = np.zeros(nMax) - R[0] =1 - for i,nDelay in enumerate(nvec[1:]): - R[i+1] = np.mean( x[0:-nDelay] * x[nDelay:] ) / sigma2 - - tau = nvec*dt - return R, tau - - -def correlated_signal(coeff, n=1000): - """ - Create a correlated random signal of length `n` based on the correlation coefficient `coeff` - value[t] = coeff * value[t-1] + (1-coeff) * random - """ - if coeff<0 or coeff>1: - raise Exception('Correlation coefficient should be between 0 and 1') - - x = np.zeros(n) - rvec = rand(n) - x[0] = rvec[0] - for m in np.arange(1,n): - x[m] = coeff*x[m-1] + (1-coeff)*rvec[m] - x-=np.mean(x) - return x - - -if __name__=='__main__': - import numpy as np - import matplotlib.pyplot as plt - - # Input - dt = 1 - n = 10000 - coeff = 0.95 # 1:full corr, 00-corr - nMax = 180 - # Create a correlated time series - tvec = np.arange(0,n)*dt - ts = correlated_signal(coeff, n) - # --- Compute correlation coefficient - R, tau = correlation(x, nMax=nMax) - fig,axes = plt.subplots(2, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) - fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) - ax=axes[0] - # Plot time series - ax.plot(tvec,ts) - ax.set_xlabel('t [s]') - ax.set_ylabel('u [m/s]') - ax.tick_params(direction='in') - # Plot correlation - ax=axes[1] - ax.plot(tau, R ,'b-o', label='computed') - ax.plot(tau, coeff**(tau/dt) , 'r--' ,label='coeff^{tau/dt}') # analytical coeff^n trend - ax.set_xlabel(r'$\tau$ [s]') - ax.set_ylabel(r'$R(\tau)$ [-]') - ax.legend() - plt.show() - - - - - - +from __future__ import division +import numpy as np +from numpy.random import rand +import pandas as pd + + +# --- List of available filters +FILTERS=[ + {'name':'Moving average','param':100,'paramName':'Window Size','paramRange':[0,100000],'increment':1}, + {'name':'Low pass 1st order','param':1.0,'paramName':'Cutoff Freq.','paramRange':[0.0001,100000],'increment':0.1}, + {'name':'High pass 1st order','param':1.0,'paramName':'Cutoff Freq.','paramRange':[0.0001,100000],'increment':0.1}, +] + +SAMPLERS=[ + {'name':'Replace', 'param':[], 'paramName':'New x'}, + {'name':'Insert', 'param':[], 'paramName':'Insert list'}, + {'name':'Remove', 'param':[], 'paramName':'Remove list'}, + {'name':'Every n', 'param':2 , 'paramName':'n'}, + {'name':'Time-based', 'param':0.01 , 'paramName':'Sample time (s)'}, + {'name':'Delta x', 'param':0.1, 'paramName':'dx'}, +] + + + +def reject_outliers(y, x=None, m = 2., replaceNaN=True): + """ Reject outliers: + If replaceNaN is true: they are replaced by NaN + Otherwise they are removed + """ + if m==0: + # No rejection... + pass + else: + dd = np.abs(y - np.nanmedian(y)) + mdev = np.nanmedian(dd) + if mdev: + ss = dd/mdev + b=ss=0, j< len(xp)-1) + bLower =j<0 + bUpper =j>=len(xp)-1 + jOK = j[bOK] + dd[bOK] = (x[bOK] - xp[jOK]) / (xp[jOK + 1] - xp[jOK]) + jBef=j + jAft=j+1 + # + # Use first and last values for anything beyond xp + jAft[bUpper] = len(xp)-1 + jBef[bUpper] = len(xp)-1 + jAft[bLower] = 0 + jBef[bLower] = 0 + if extrap=='bounded': + pass + # OK + elif extrap=='nan': + dd[~bOK] = np.nan + else: + raise NotImplementedError() + + return (1 - dd) * fp[:,jBef] + fp[:,jAft] * dd + +def interpArray(x, xp, fp, extrap='bounded'): + """ + Interpolate all the columns of a matrix `fp` based on one new value `x` + INPUTS: + - x : scalar new values + - xp : array ( np ), old values + - fp : array ( nval, np), matrix values to be interpolated + """ + # Sanity + xp = np.asarray(xp) + assert fp.shape[1]==len(xp), 'Second dimension of fp should have the same length as xp' + + j = np.searchsorted(xp, x) - 1 + if j<0: + # Before bounds + if extrap=='bounded': + return fp[:,0] + elif extrap=='nan': + return fp[:,0]*np.nan + else: + raise NotImplementedError() + + elif j>=len(xp)-1: + # After bounds + if extrap=='bounded': + return fp[:,-1] + elif extrap=='nan': + return fp[:,-1]*np.nan + else: + raise NotImplementedError() + else: + # Normal case, within bounds + dd = (x- xp[j]) / (xp[j+1] - xp[j]) + return (1 - dd) * fp[:,j] + fp[:,j+1] * dd + + +def resample_interp(x_old, x_new, y_old=None, df_old=None): + #x_new=np.sort(x_new) + if df_old is not None: + # --- Method 1 (pandas) + #df_new = df_old.copy() + #df_new = df_new.set_index(x_old) + #df_new = df_new.reindex(df_new.index | x_new) + #df_new = df_new.interpolate().loc[x_new] + #df_new = df_new.reset_index() + # --- Method 2 interp storing dx + data_new=multiInterp(x_new, x_old, df_old.values.T) + df_new = pd.DataFrame(data=data_new.T, columns=df_old.columns.values) + return x_new, df_new + + if y_old is not None: + return x_new, np.interp(x_new, x_old, y_old) + + +def applySamplerDF(df_old, x_col, sampDict): + x_old=df_old[x_col].values + x_new, df_new =applySampler(x_old, y_old=None, sampDict=sampDict, df_old=df_old) + df_new[x_col]=x_new + return df_new + + +def applySampler(x_old, y_old, sampDict, df_old=None): + + param = np.asarray(sampDict['param']).ravel() + + if sampDict['name']=='Replace': + if len(param)==0: + raise Exception('Error: At least one value is required to resample the x values with') + x_new = param + return resample_interp(x_old, x_new, y_old, df_old) + + elif sampDict['name']=='Insert': + if len(param)==0: + raise Exception('Error: provide a list of values to insert') + x_new = np.sort(np.concatenate((x_old.ravel(),param))) + return resample_interp(x_old, x_new, y_old, df_old) + + elif sampDict['name']=='Remove': + I=[] + if len(param)==0: + raise Exception('Error: provide a list of values to remove') + for d in param: + Ifound= np.where(np.abs(x_old-d)<1e-3)[0] + if len(Ifound)>0: + I+=list(Ifound.ravel()) + x_new=np.delete(x_old,I) + return resample_interp(x_old, x_new, y_old, df_old) + + elif sampDict['name']=='Delta x': + if len(param)==0: + raise Exception('Error: provide value for dx') + dx = param[0] + x_new = np.arange(x_old[0], x_old[-1]+dx/2, dx) + return resample_interp(x_old, x_new, y_old, df_old) + + elif sampDict['name']=='Every n': + if len(param)==0: + raise Exception('Error: provide value for n') + n = int(param[0]) + if n==0: + raise Exception('Error: |n| should be at least 1') + + x_new=x_old[::n] + if df_old is not None: + return x_new, (df_old.copy()).iloc[::n,:] + if y_old is not None: + return x_new, y_old[::n] + + elif sampDict['name'] == 'Time-based': + if len(param) == 0: + raise Exception('Error: provide value for new sampling time') + sample_time = float(param[0]) + if sample_time <= 0: + raise Exception('Error: sample time must be positive') + + time_index = pd.TimedeltaIndex(x_old, unit="S") + x_new = pd.Series(x_old, index=time_index).resample("{:f}S".format(sample_time)).mean().interpolate().values + + if df_old is not None: + df_new = df_old.set_index(time_index, inplace=False).resample("{:f}S".format(sample_time)).mean() + df_new = df_new.interpolate().reset_index(drop=True) + return x_new, df_new + if y_old is not None: + y_new = pd.Series(y_old, index=time_index).resample("{:f}S".format(sample_time)).mean() + y_new = y_new.interpolate().values + return x_new, y_new + + else: + raise NotImplementedError('{}'.format(sampDict)) + pass + +# --------------------------------------------------------------------------------} +# --- Filters +# --------------------------------------------------------------------------------{ +# def moving_average(x, w): +# #t_new = np.arange(0,Tmax,dt) +# #nt = len(t_new) +# #nw=400 +# #u_new = moving_average(np.floor(np.linspace(0,3,nt+nw-1))*3+3.5, nw) +# return np.convolve(x, np.ones(w), 'valid') / w +# def moving_average(x,N,mode='same'): +# y=np.convolve(x, np.ones((N,))/N, mode=mode) +# return y +def moving_average(a, n=3) : + """ + perform moving average, return a vector of same length as input + + NOTE: also in kalman.filters + """ + a = a.ravel() + a = np.concatenate(([a[0]]*(n-1),a)) # repeating first values + ret = np.cumsum(a, dtype = float) + ret[n:] = ret[n:] - ret[:-n] + ret=ret[n - 1:] / n + return ret + +def lowpass1(y, dt, fc=3) : + """ + 1st order low pass filter + """ + tau=1/(2*np.pi*fc) + alpha=dt/(tau+dt) + y_filt=np.zeros(y.shape) + y_filt[0]=y[0] + for i in np.arange(1,len(y)): + y_filt[i]=alpha*y[i] + (1-alpha)*y_filt[i-1] + return y_filt + +def highpass1(y, dt, fc=3) : + """ + 1st order high pass filter + """ + tau=1/(2*np.pi*fc) + alpha=tau/(tau+dt) + y_filt=np.zeros(y.shape) + y_filt[0]=0 + for i in np.arange(1,len(y)): + y_filt[i]=alpha*y_filt[i-1] + alpha*(y[i]-y[i-1]) + m0=np.mean(y) + m1=np.mean(y_filt) + y_filt+=m0-m1 + return y_filt + + +def applyFilter(x, y, filtDict): + if filtDict['name']=='Moving average': + return moving_average(y, n=np.round(filtDict['param']).astype(int)) + elif filtDict['name']=='Low pass 1st order': + dt = x[1]-x[0] + return lowpass1(y, dt=dt, fc=filtDict['param']) + elif filtDict['name']=='High pass 1st order': + dt = x[1]-x[0] + return highpass1(y, dt=dt, fc=filtDict['param']) + else: + raise NotImplementedError('{}'.format(filtDict)) + +def applyFilterDF(df_old, x_col, options): + """ apply filter on a dataframe """ + # Brute force loop + df_new = df_old.copy() + x = df_new[x_col] + for (colName, colData) in df_new.iteritems(): + if colName != x_col: + df_new[colName] = applyFilter(x, colData, options) + return df_new + + +# --------------------------------------------------------------------------------} +# --- +# --------------------------------------------------------------------------------{ +def zero_crossings(y,x=None,direction=None): + """ + Find zero-crossing points in a discrete vector, using linear interpolation. + + direction: 'up' or 'down', to select only up-crossings or down-crossings + + returns: + x values xzc such that y(yzc)==0 + indexes izc, such that the zero is between y[izc] (excluded) and y[izc+1] (included) + + if direction is not provided, also returns: + sign, equal to 1 for up crossing + """ + if x is None: + x=np.arange(len(y)) + + if np.any((x[1:] - x[0:-1]) <= 0.0): + raise Exception('x values need to be in ascending order') + + # Indices before zero-crossing + iBef = np.where(y[1:]*y[0:-1] < 0.0)[0] + + # Find the zero crossing by linear interpolation + xzc = x[iBef] - y[iBef] * (x[iBef+1] - x[iBef]) / (y[iBef+1] - y[iBef]) + + # Selecting points that are exactly 0 and where neighbor change sign + iZero = np.where(y == 0.0)[0] + iZero = iZero[np.where((iZero > 0) & (iZero < x.size-1))] + iZero = iZero[np.where(y[iZero-1]*y[iZero+1] < 0.0)] + + # Concatenate + xzc = np.concatenate((xzc, x[iZero])) + iBef = np.concatenate((iBef, iZero)) + + # Sort + iSort = np.argsort(xzc) + xzc, iBef = xzc[iSort], iBef[iSort] + + # Return up-crossing, down crossing or both + sign = np.sign(y[iBef+1]-y[iBef]) + if direction == 'up': + I= np.where(sign==1)[0] + return xzc[I],iBef[I] + elif direction == 'down': + I= np.where(sign==-1)[0] + return xzc[I],iBef[I] + elif direction is not None: + raise Exception('Direction should be either `up` or `down`') + return xzc, iBef, sign + + +# --------------------------------------------------------------------------------} +# --- Correlation +# --------------------------------------------------------------------------------{ +def correlation(x, nMax=80, dt=1, method='manual'): + """ + Compute auto correlation of a signal + """ + nvec = np.arange(0,nMax) + sigma2 = np.var(x) + R = np.zeros(nMax) + R[0] =1 + for i,nDelay in enumerate(nvec[1:]): + R[i+1] = np.mean( x[0:-nDelay] * x[nDelay:] ) / sigma2 + + tau = nvec*dt + return R, tau + + +def correlated_signal(coeff, n=1000, seed=None): + """ + Create a correlated random signal of length `n` based on the correlation coefficient `coeff` + value[t] = coeff * value[t-1] + (1-coeff) * random + """ + if coeff<0 or coeff>1: + raise Exception('Correlation coefficient should be between 0 and 1') + if seed is not None: + np.random.seed(seed) + + x = np.zeros(n) + rvec = rand(n) + x[0] = rvec[0] + for m in np.arange(1,n): + x[m] = coeff*x[m-1] + (1-coeff)*rvec[m] + x-=np.mean(x) + return x + + +def find_time_offset(t, f, g, outputAll=False): + """ + Find time offset between two signals (may be negative) + + t_offset = find_time_offset(t, f, g) + f(t+t_offset) ~= g(t) + + """ + import scipy + from scipy.signal import correlate + # Remove mean and normalize by std + f = f.copy() + g = g.copy() + f -= f.mean() + g -= g.mean() + f /= f.std() + g /= g.std() + + # Find cross-correlation + xcorr = correlate(f, g) + + # Lags + n = len(f) + dt = t[1]-t[0] + lag = np.arange(1-n, n)*dt + + # Time offset is located at maximum correlation + t_offset = lag[xcorr.argmax()] + + if outputAll: + return t_offset, lag, xcorr + else: + return t_offset + +def sine_approx(t, x, method='least_square'): + """ + Sinusoidal approximation of input signal x + """ + if method=='least_square': + from welib.tools.curve_fitting import fit_sinusoid + y_fit, pfit, fitter = fit_sinusoid(t, x) + omega = fitter.model['coeffs']['omega'] + A = fitter.model['coeffs']['A'] + phi = fitter.model['coeffs']['phi'] + x2 = y_fit + else: + raise NotImplementedError() + + + return x2, omega, A, phi + + +# --------------------------------------------------------------------------------} +# --- Convolution +# --------------------------------------------------------------------------------{ +def convolution_integral(time, f, g, method='auto'): + """ + Compute convolution integral: + f * g = \int 0^t f(tau) g(t-tau) dtau = g * f + For now, only works for uniform time vector, an exception is raised otherwise + + method=['auto','direct','fft'], + see scipy.signal.convolve + see scipy.signal.fftconvolve + """ + from scipy.signal import convolve + dt = time[1]-time[0] + if len(np.unique(np.around(np.diff(time)/dt,3)))>1: + raise Exception('Convolution integral implemented for uniform time vector') + + #np.convolve(f.ravel(), g.ravel() )[:len(time)]*dt + return convolve(f.ravel(), g.ravel() )[:len(time)]*dt + + +# --------------------------------------------------------------------------------} +# --- Intervals/peaks +# --------------------------------------------------------------------------------{ +def intervals(b, min_length=1, forgivingJump=True, removeSmallRel=True, removeSmallFact=0.1, mergeCloseRel=False, mergeCloseFact=0.2): + """ + Describe intervals from a boolean vector where intervals are indicated by True + + INPUT: + - b : a logical vector, where 1 means, I'm in an interval. + - min_length: if provided, do not return intervals of length < min_length + - forgivingJump: if true, merge intervals that are separated by a distance < min_length + - removeSmallRel: remove intervals that have a small length compared to the max length of intervals + - removeSmallFact: factor used for removeSmallRel + - mergeCloseRel: merge intervals that are closer than a fraction of the typical distance between intervals + + OUTPUTS: + - IStart : ending indices + - IEnd : ending indices + - Length: interval lenghts (IEnd-IStart+1) + + IStart, IEnd, Lengths = intervals([False, True, True, False, True, True, True, False]) + np.testing.assert_equal(IStart , np.array([1,4])) + np.testing.assert_equal(IEnd , np.array([2,6])) + np.testing.assert_equal(Lengths, np.array([2,3])) + """ + b = np.asarray(b) + total = np.sum(b) + + min_length=max(min_length,1) + if forgivingJump: + min_jump=min_length + else: + min_jump=1 + + if total==0: + IStart = np.array([]) + IEnd = np.array([]) + Lengths= np.array([]) + return IStart, IEnd, Lengths + elif total==1: + i = np.where(b)[0][0] + IStart = np.array([i]) + IEnd = np.array([i]) + Lengths= np.array([1]) + else: + n = len(b) + Idx = np.arange(n)[b] + delta_Idx=np.diff(Idx) + jumps =np.where(delta_Idx>min_jump)[0] + if len(jumps)==0: + IStart = np.array([Idx[0]]) + IEnd = np.array([Idx[-1]]) + else: + istart=Idx[0] + jumps=np.concatenate(([-1],jumps,[len(Idx)-1])) + IStart = Idx[jumps[:-1]+1] # intervals start right after a jump + IEnd = Idx[jumps[1:]] # intervals stops at jump + Lengths = IEnd-IStart+1 + + # Removing intervals smaller than min_length + bKeep = Lengths>=min_length + IStart = IStart[bKeep] + IEnd = IEnd[bKeep] + Lengths = Lengths[bKeep] + # Removing intervals smaller than less than a fraction of the max interval + if removeSmallRel: + bKeep = Lengths>=removeSmallFact*np.max(Lengths) + IStart = IStart[bKeep] + IEnd = IEnd[bKeep] + Lengths = Lengths[bKeep] + + # Distances between intervals + if mergeCloseRel: + if len(IStart)<=2: + pass + else: + D = IStart[1:]-IEnd[0:-1] + #print('D',D,np.max(D),int(np.max(D) * mergeCloseFact)) + min_length = max(int(np.max(D) * mergeCloseFact), min_length) + if min_length<=1: + pass + else: + #print('Readjusting min_length to {} to accomodate for max interval spacing of {:.0f}'.format(min_length, np.mean(D))) + return intervals(b, min_length=min_length, forgivingJump=True, removeSmallRel=removeSmallRel, removeSmallFact=removeSmallFact, mergeCloseRel=False) + return IStart, IEnd, Lengths + +def peaks(x, threshold=0.3, threshold_abs=True, method='intervals', min_length=3, + mergeCloseRel=True, returnIntervals=False): + """ + Find peaks in a signal, above a given threshold + INPUTS: + - x : 1d-array, signal + - threshold : scalar, absolute or relative threshold beyond which peaks are looked for + relative threshold are proportion of the max-min of the signal (between 0-1) + - threshold_abs : boolean, specify whether the threshold is absolute or relative + - method : string, selects which method is used to find the peaks, between: + - 'interval' : one peak per interval above the threshold + - 'derivative': uses derivative to find maxima, may return more than one per interval + - min_length: + - if 'interval' method is used: minimum interval + - if 'derivative' method is used: minimum distance between two peaks + + OPTIONS for interval method: + - mergeCloseRel: logical, if True, attempts to merge intervals that are close to each other compare to the typical interval spacing + set to False if all peaks are wanted + - returnIntervals: logical, if true, return intervals used for interval method + OUTPUTS: + - I : index of the peaks + -[IStart, IEnd] if return intervals is true, see function `intervals` + + + see also: + scipy.signal.find_peaks + + """ + if not threshold_abs: + threshold = threshold * (np.max(y) - np.min(y)) + np.min(y) + + if method =='intervals': + IStart, IEnd, Lengths = intervals(x>threshold, min_length=min_length, mergeCloseRel=mergeCloseRel) + I = np.array([iS if L==1 else np.argmax(x[iS:iE+1])+iS for iS,iE,L in zip(IStart,IEnd,Lengths)]) + if returnIntervals: + return I, IStart, IEnd + else: + return I + + elif method =='derivative': + I = indexes(x, thres=threshold, thres_abs=True, min_dist=min_length) + return I + else: + raise NotImplementedError('Method {}'.format(method)) + + + +# --------------------------------------------------------------------------------} +# --- Simple signals +# --------------------------------------------------------------------------------{ +def impulse(time, t0=0, A=1, epsilon=None, **kwargs): + """ + returns a dirac function: + A/dt if t==t0 + 0 otherwise + + Since the impulse response is poorly defined in discrete space, it's recommended + to use a smooth_delta. See the welib.tools.functions.delta + """ + from .functions import delta + t=np.asarray(time)-t0 + y= delta(t, epsilon=epsilon, **kwargs)*A + return y + +def step(time, t0=0, valueAtStep=0, A=1): + """ + returns a step function: + 0 if tt0 + valueAtStep if t==t0 + + NOTE: see also welib.tools.functions.Pi + """ + return np.heaviside(time-t0, valueAtStep)*A + +def ramp(time, t0=0, valueAtStep=0, A=1): + """ + returns a ramp function: + 0 if t=t0 + + NOTE: see also welib.tools.functions.Pi + """ + t=np.asarray(time)-t0 + y=np.zeros(t.shape) + y[t>=0]=A*t[t>=0] + return y + + +def hat(time, T=1, t0=0, A=1, method='abs'): + """ + returns a hat function: + A*hat if |t-t0|n: - print('[WARN] Power of 2 value was too high and was reduced. Disable averaging to use the full spectrum.'); - nExp=int(np.log(nFFTAll)/np.log(2))-1 - nPerSeg=2**nExp - if averaging_window=='hamming': - window = hamming(nPerSeg, True)# True=Symmetric, like matlab - elif averaging_window=='hann': - window = hann(nPerSeg, True) - elif averaging_window=='rectangular': - window = boxcar(nPerSeg) - else: - raise Exception('Averaging window unknown {}'.format(averaging_window)) - frq, PSD, Info = pwelch(y, fs=Fs, window=window, detrend=detrend) - Info.nExp = nExp - else: - raise Exception('Averaging method unknown {}'.format(averaging)) - - # --- Formatting output - if output_type=='amplitude': - deltaf = frq[1]-frq[0] - Y = np.sqrt(PSD*2*deltaf) - # NOTE: the above should be the same as:Y=abs(Y[range(nhalf)])/n;Y[1:-1]=Y[1:-1]*2; - elif output_type=='psd': # one sided - Y = PSD - elif output_type=='f x psd': - Y = PSD*frq - else: - raise NotImplementedError('Contact developer') - if detrend: - frq= frq[1:] - Y = Y[1:] - return frq, Y, Info - - - -# --------------------------------------------------------------------------------} -# --- Spectral simple (averaging below) -# --------------------------------------------------------------------------------{ -def fft_amplitude(y, fs=1.0, detrend ='constant', return_onesided=True): - """ Returns FFT amplitude of signal """ - frq, PSD, Info = psd(y, fs=fs, detrend=detrend, return_onesided=return_onesided) - deltaf = frq[1]-frq[0] - Y = np.sqrt(PSD*2*deltaf) - return frq, Y, Info - -def psd(y, fs=1.0, detrend ='constant', return_onesided=True): - """ Perform PSD without averaging """ - if not return_onesided: - raise NotImplementedError('Double sided todo') - - if detrend is None: - detrend=False - - if detrend=='constant' or detrend==True: - m=np.mean(y); - else: - m=0; - - n = len(y) - if n%2==0: - nhalf = int(n/2+1) - else: - nhalf = int((n+1)/2) - - frq = np.arange(nhalf)*fs/n; - Y = np.fft.rfft(y-m) #Y = np.fft.fft(y) - PSD = abs(Y[range(nhalf)])**2 /(n*fs) # PSD - PSD[1:-1] = PSD[1:-1]*2; - class InfoClass(): - pass - Info = InfoClass(); - Info.df = frq[1]-frq[0] - Info.fMax = frq[-1] - Info.LFreq = len(frq) - Info.LSeg = len(Y) - Info.LWin = len(Y) - Info.LOvlp = 0 - Info.nFFT = len(Y) - Info.nseg = 1 - return frq, PSD, Info - - -# --------------------------------------------------------------------------------} -# --- Windows -# --------------------------------------------------------------------------------{ -"""The suite of window functions.""" -def fnextpow2(x): - return 2**np.ceil( np.log(x)*0.99999999999/np.log(2)); - -def fDefaultWinLen(x,overlap_frac=0.5): - return fnextpow2(np.sqrt(len(x)/(1-overlap_frac))) - -def fDefaultWinLenMatlab(x): - return np.fix((len(x)-3)*2./9.) - -def _len_guards(M): - """Handle small or incorrect window lengths""" - if int(M) != M or M < 0: - raise ValueError('Window length M must be a non-negative integer') - return M <= 1 - -def _extend(M, sym): - """Extend window by 1 sample if needed for DFT-even symmetry""" - if not sym: - return M + 1, True - else: - return M, False - -def _truncate(w, needed): - """Truncate window by 1 sample if needed for DFT-even symmetry""" - if needed: - return w[:-1] - else: - return w - -def general_cosine(M, a, sym=True): - if _len_guards(M): - return np.ones(M) - M, needs_trunc = _extend(M, sym) - - fac = np.linspace(-np.pi, np.pi, M) - w = np.zeros(M) - for k in range(len(a)): - w += a[k] * np.cos(k * fac) - - return _truncate(w, needs_trunc) - - -def boxcar(M, sym=True): - """Return a boxcar or rectangular window. - - Also known as a rectangular window or Dirichlet window, this is equivalent - to no window at all. - """ - if _len_guards(M): - return np.ones(M) - M, needs_trunc = _extend(M, sym) - - w = np.ones(M, float) - - return _truncate(w, needs_trunc) - -def hann(M, sym=True): # same as hanning(*args, **kwargs): - return general_hamming(M, 0.5, sym) - - -def general_hamming(M, alpha, sym=True): - r"""Return a generalized Hamming window. - The generalized Hamming window is constructed by multiplying a rectangular - window by one period of a cosine function [1]_. - w(n) = \alpha - \left(1 - \alpha\right) \cos\left(\frac{2\pi{n}}{M-1}\right) - \qquad 0 \leq n \leq M-1 - """ - return general_cosine(M, [alpha, 1. - alpha], sym) - - -def hamming(M, sym=True): - r"""Return a Hamming window. - The Hamming window is a taper formed by using a raised cosine with - non-zero endpoints, optimized to minimize the nearest side lobe. - w(n) = 0.54 - 0.46 \cos\left(\frac{2\pi{n}}{M-1}\right) - \qquad 0 \leq n \leq M-1 - """ - return general_hamming(M, 0.54, sym) - -_win_equiv_raw = { - ('boxcar', 'box', 'ones', 'rect', 'rectangular'): (boxcar, False), - ('hamming', 'hamm', 'ham'): (hamming, False), - ('hanning', 'hann', 'han'): (hann, False), -} - -# Fill dict with all valid window name strings -_win_equiv = {} -for k, v in _win_equiv_raw.items(): - for key in k: - _win_equiv[key] = v[0] - -# Keep track of which windows need additional parameters -_needs_param = set() -for k, v in _win_equiv_raw.items(): - if v[1]: - _needs_param.update(k) - - -def get_window(window, Nx, fftbins=True): - """ - Return a window. - - Parameters - ---------- - window : string, float, or tuple - The type of window to create. See below for more details. - Nx : int - The number of samples in the window. - fftbins : bool, optional - If True (default), create a "periodic" window, ready to use with - `ifftshift` and be multiplied by the result of an FFT (see also - `fftpack.fftfreq`). - If False, create a "symmetric" window, for use in filter design. - """ - sym = not fftbins - try: - beta = float(window) - except (TypeError, ValueError): - args = () - if isinstance(window, tuple): - winstr = window[0] - if len(window) > 1: - args = window[1:] - elif isinstance(window, string_types): - if window in _needs_param: - raise ValueError("The '" + window + "' window needs one or " - "more parameters -- pass a tuple.") - else: - winstr = window - else: - raise ValueError("%s as window type is not supported." % - str(type(window))) - - try: - winfunc = _win_equiv[winstr] - except KeyError: - raise ValueError("Unknown window type.") - - params = (Nx,) + args + (sym,) - else: - winfunc = kaiser - params = (Nx, beta, sym) - - return winfunc(*params) - - - - - - -# --------------------------------------------------------------------------------} -# --- Helpers -# --------------------------------------------------------------------------------{ -def odd_ext(x, n, axis=-1): - """ - Odd extension at the boundaries of an array - Generate a new ndarray by making an odd extension of `x` along an axis. - """ - if n < 1: - return x - if n > x.shape[axis] - 1: - raise ValueError(("The extension length n (%d) is too big. " + - "It must not exceed x.shape[axis]-1, which is %d.") - % (n, x.shape[axis] - 1)) - left_end = axis_slice(x, start=0, stop=1, axis=axis) - left_ext = axis_slice(x, start=n, stop=0, step=-1, axis=axis) - right_end = axis_slice(x, start=-1, axis=axis) - right_ext = axis_slice(x, start=-2, stop=-(n + 2), step=-1, axis=axis) - ext = np.concatenate((2 * left_end - left_ext, - x, - 2 * right_end - right_ext), - axis=axis) - return ext - - -def even_ext(x, n, axis=-1): - """ - Even extension at the boundaries of an array - Generate a new ndarray by making an even extension of `x` along an axis. - """ - if n < 1: - return x - if n > x.shape[axis] - 1: - raise ValueError(("The extension length n (%d) is too big. " + - "It must not exceed x.shape[axis]-1, which is %d.") - % (n, x.shape[axis] - 1)) - left_ext = axis_slice(x, start=n, stop=0, step=-1, axis=axis) - right_ext = axis_slice(x, start=-2, stop=-(n + 2), step=-1, axis=axis) - ext = np.concatenate((left_ext, - x, - right_ext), - axis=axis) - return ext - - -def const_ext(x, n, axis=-1): - """ - Constant extension at the boundaries of an array - Generate a new ndarray that is a constant extension of `x` along an axis. - The extension repeats the values at the first and last element of - the axis. - """ - if n < 1: - return x - left_end = axis_slice(x, start=0, stop=1, axis=axis) - ones_shape = [1] * x.ndim - ones_shape[axis] = n - ones = np.ones(ones_shape, dtype=x.dtype) - left_ext = ones * left_end - right_end = axis_slice(x, start=-1, axis=axis) - right_ext = ones * right_end - ext = np.concatenate((left_ext, - x, - right_ext), - axis=axis) - return ext - - -def zero_ext(x, n, axis=-1): - """ - Zero padding at the boundaries of an array - Generate a new ndarray that is a zero padded extension of `x` along - an axis. - """ - if n < 1: - return x - zeros_shape = list(x.shape) - zeros_shape[axis] = n - zeros = np.zeros(zeros_shape, dtype=x.dtype) - ext = np.concatenate((zeros, x, zeros), axis=axis) - return ext - -def signaltools_detrend(data, axis=-1, type='linear', bp=0): - """ - Remove linear trend along axis from data. - - Parameters - ---------- - data : array_like - The input data. - axis : int, optional - The axis along which to detrend the data. By default this is the - last axis (-1). - type : {'linear', 'constant'}, optional - The type of detrending. If ``type == 'linear'`` (default), - the result of a linear least-squares fit to `data` is subtracted - from `data`. - If ``type == 'constant'``, only the mean of `data` is subtracted. - bp : array_like of ints, optional - A sequence of break points. If given, an individual linear fit is - performed for each part of `data` between two break points. - Break points are specified as indices into `data`. - - Returns - ------- - ret : ndarray - The detrended input data. - """ - if type not in ['linear', 'l', 'constant', 'c']: - raise ValueError("Trend type must be 'linear' or 'constant'.") - data = np.asarray(data) - dtype = data.dtype.char - if dtype not in 'dfDF': - dtype = 'd' - if type in ['constant', 'c']: - #print('Removing mean') - ret = data - np.expand_dims(np.mean(data, axis), axis) - return ret - else: - #print('Removing linear?') - dshape = data.shape - N = dshape[axis] - bp = sort(unique(r_[0, bp, N])) - if np.any(bp > N): - raise ValueError("Breakpoints must be less than length " - "of data along given axis.") - Nreg = len(bp) - 1 - # Restructure data so that axis is along first dimension and - # all other dimensions are collapsed into second dimension - rnk = len(dshape) - if axis < 0: - axis = axis + rnk - newdims = r_[axis, 0:axis, axis + 1:rnk] - newdata = reshape(np.transpose(data, tuple(newdims)), - (N, _prod(dshape) // N)) - newdata = newdata.copy() # make sure we have a copy - if newdata.dtype.char not in 'dfDF': - newdata = newdata.astype(dtype) - # Find leastsq fit and remove it for each piece - for m in range(Nreg): - Npts = bp[m + 1] - bp[m] - A = ones((Npts, 2), dtype) - A[:, 0] = cast[dtype](np.arange(1, Npts + 1) * 1.0 / Npts) - sl = slice(bp[m], bp[m + 1]) - coef, resids, rank, s = np.linalg.lstsq(A, newdata[sl]) - newdata[sl] = newdata[sl] - dot(A, coef) - # Put data back in original shape. - tdshape = take(dshape, newdims, 0) - ret = np.reshape(newdata, tuple(tdshape)) - vals = list(range(1, rnk)) - olddims = vals[:axis] + [0] + vals[axis:] - ret = np.transpose(ret, tuple(olddims)) - return ret - - - -# --------------------------------------------------------------------------------} -# --- Spectral Averaging -# --------------------------------------------------------------------------------{ -"""Tools for spectral analysis. """ - -def welch(x, fs=1.0, window='hann', nperseg=None, noverlap=None, nfft=None, - detrend='constant', return_onesided=True, scaling='density', - axis=-1): - """Interface identical to scipy.signal """ - - if detrend==True: - detrend='constant' - - freqs, Pxx = csd(x, x, fs, window, nperseg, noverlap, nfft, detrend, return_onesided, scaling, axis) - return freqs, Pxx.real - -#>>>> -def pwelch(x, window='hamming', noverlap=None, nfft=None, fs=1.0, nperseg=None, - detrend=False, return_onesided=True, scaling='density', - axis=-1): - r""" - NOTE: interface and default options modified to match matlab's implementation - >> detrend: default to False - >> window : default to 'hamming' - >> window: if an integer, use 'hamming(window, sym=True)' - - - Estimate power spectral density using Welch's method. - - Welch's method [1]_ computes an estimate of the power spectral - density by dividing the data into overlapping segments, computing a - modified periodogram for each segment and averaging the - periodograms. - - Parameters - ---------- - x : array_like - Time series of measurement values - fs : float, optional - Sampling frequency of the `x` time series. Defaults to 1.0. - window : str or tuple or array_like, optional - Desired window to use. If `window` is a string or tuple, it is - passed to `get_window` to generate the window values, which are - DFT-even by default. See `get_window` for a list of windows and - required parameters. If `window` is array_like it will be used - directly as the window and its length must be nperseg. Defaults - to a Hann window. - nperseg : int, optional - Length of each segment. Defaults to None, but if window is str or - tuple, is set to 256, and if window is array_like, is set to the - length of the window. - noverlap : int, optional - Number of points to overlap between segments. If `None`, - ``noverlap = nperseg // 2``. Defaults to `None`. - nfft : int, optional - Length of the FFT used, if a zero padded FFT is desired. If - `None`, the FFT length is `nperseg`. Defaults to `None`. - detrend : str or function or `False`, optional - Specifies how to detrend each segment. If `detrend` is a - string, it is passed as the `type` argument to the `detrend` - function. If it is a function, it takes a segment and returns a - detrended segment. If `detrend` is `False`, no detrending is - done. Defaults to 'constant'. - return_onesided : bool, optional - If `True`, return a one-sided spectrum for real data. If - `False` return a two-sided spectrum. Note that for complex - data, a two-sided spectrum is always returned. - scaling : { 'density', 'spectrum' }, optional - Selects between computing the power spectral density ('density') - where `Pxx` has units of V**2/Hz and computing the power - spectrum ('spectrum') where `Pxx` has units of V**2, if `x` - is measured in V and `fs` is measured in Hz. Defaults to - 'density' - axis : int, optional - Axis along which the periodogram is computed; the default is - over the last axis (i.e. ``axis=-1``). - - Returns - ------- - f : ndarray - Array of sample frequencies. - Pxx : ndarray - Power spectral density or power spectrum of x. - - See Also - -------- - periodogram: Simple, optionally modified periodogram - lombscargle: Lomb-Scargle periodogram for unevenly sampled data - - Notes - ----- - An appropriate amount of overlap will depend on the choice of window - and on your requirements. For the default Hann window an overlap of - 50% is a reasonable trade off between accurately estimating the - signal power, while not over counting any of the data. Narrower - windows may require a larger overlap. - - If `noverlap` is 0, this method is equivalent to Bartlett's method - [2]_. - - .. versionadded:: 0.12.0 - - References - ---------- - .. [1] P. Welch, "The use of the fast Fourier transform for the - estimation of power spectra: A method based on time averaging - over short, modified periodograms", IEEE Trans. Audio - Electroacoust. vol. 15, pp. 70-73, 1967. - .. [2] M.S. Bartlett, "Periodogram Analysis and Continuous Spectra", - Biometrika, vol. 37, pp. 1-16, 1950. - - """ - import math - def fnextpow2(x): - return 2**math.ceil( math.log(x)*0.99999999999/math.log(2)); - - # MANU >>> CHANGE OF DEFAULT OPTIONS - # MANU - If a length is provided use symmetric hamming window - if type(window)==int: - window=hamming(window, True) - # MANU - do not use 256 as default - if isinstance(window, string_types) or isinstance(window, tuple): - if nperseg is None: - if noverlap is None: - overlap_frac=0.5 - elif noverlap is 0: - overlap_frac=0 - else: - raise NotImplementedError('TODO noverlap set but not nperseg') - #nperseg = 256 # then change to default - nperseg=fnextpow2(math.sqrt(x.shape[-1]/(1-overlap_frac))); - - # MANU accepting true as detrend - if detrend==True: - detrend='constant' - - freqs, Pxx, Info = csd(x, x, fs, window, nperseg, noverlap, nfft, detrend, - return_onesided, scaling, axis) - - return freqs, Pxx.real, Info - - -def csd(x, y, fs=1.0, window='hann', nperseg=None, noverlap=None, nfft=None, - detrend='constant', return_onesided=True, scaling='density', axis=-1): - r""" - Estimate the cross power spectral density, Pxy, using Welch's - method. - """ - - freqs, _, Pxy, Info = _spectral_helper(x, y, fs, window, nperseg, noverlap, nfft, - detrend, return_onesided, scaling, axis, - mode='psd') - - # Average over windows. - if len(Pxy.shape) >= 2 and Pxy.size > 0: - if Pxy.shape[-1] > 1: - Pxy = Pxy.mean(axis=-1) - else: - Pxy = np.reshape(Pxy, Pxy.shape[:-1]) - - return freqs, Pxy, Info - - - -def coherence(x, y, fs=1.0, window='hann', nperseg=None, noverlap=None, - nfft=None, detrend='constant', axis=-1): - r""" - Estimate the magnitude squared coherence estimate, Cxy, of - discrete-time signals X and Y using Welch's method. - - ``Cxy = abs(Pxy)**2/(Pxx*Pyy)``, where `Pxx` and `Pyy` are power - spectral density estimates of X and Y, and `Pxy` is the cross - spectral density estimate of X and Y. - """ - - freqs, Pxx, Infoxx = welch(x, fs, window, nperseg, noverlap, nfft, detrend, axis=axis) - _, Pyy, Infoyy = welch(y, fs, window, nperseg, noverlap, nfft, detrend, axis=axis) - _, Pxy, Infoxy = csd(x, y, fs, window, nperseg, noverlap, nfft, detrend, axis=axis) - - Cxy = np.abs(Pxy)**2 / Pxx / Pyy - - return freqs, Cxy, Infoxx - - -def _spectral_helper(x, y, fs=1.0, window='hann', nperseg=None, noverlap=None, - nfft=None, detrend='constant', return_onesided=True, - scaling='spectrum', axis=-1, mode='psd', boundary=None, - padded=False): - """ Calculate various forms of windowed FFTs for PSD, CSD, etc. """ - if mode not in ['psd', 'stft']: - raise ValueError("Unknown value for mode %s, must be one of: " - "{'psd', 'stft'}" % mode) - - - - - - boundary_funcs = {'even': even_ext, - 'odd': odd_ext, - 'constant': const_ext, - 'zeros': zero_ext, - None: None} - - if boundary not in boundary_funcs: - raise ValueError("Unknown boundary option '{0}', must be one of: {1}" - .format(boundary, list(boundary_funcs.keys()))) - - # If x and y are the same object we can save ourselves some computation. - same_data = y is x - - if not same_data and mode != 'psd': - raise ValueError("x and y must be equal if mode is 'stft'") - - axis = int(axis) - - # Ensure we have np.arrays, get outdtype - x = np.asarray(x) - if not same_data: - y = np.asarray(y) - outdtype = np.result_type(x, y, np.complex64) - else: - outdtype = np.result_type(x, np.complex64) - - if not same_data: - # Check if we can broadcast the outer axes together - xouter = list(x.shape) - youter = list(y.shape) - xouter.pop(axis) - youter.pop(axis) - try: - outershape = np.broadcast(np.empty(xouter), np.empty(youter)).shape - except ValueError: - raise ValueError('x and y cannot be broadcast together.') - - if same_data: - if x.size == 0: - return np.empty(x.shape), np.empty(x.shape), np.empty(x.shape) - else: - if x.size == 0 or y.size == 0: - outshape = outershape + (min([x.shape[axis], y.shape[axis]]),) - emptyout = np.rollaxis(np.empty(outshape), -1, axis) - return emptyout, emptyout, emptyout - - if x.ndim > 1: - if axis != -1: - x = np.rollaxis(x, axis, len(x.shape)) - if not same_data and y.ndim > 1: - y = np.rollaxis(y, axis, len(y.shape)) - - # Check if x and y are the same length, zero-pad if necessary - if not same_data: - if x.shape[-1] != y.shape[-1]: - if x.shape[-1] < y.shape[-1]: - pad_shape = list(x.shape) - pad_shape[-1] = y.shape[-1] - x.shape[-1] - x = np.concatenate((x, np.zeros(pad_shape)), -1) - else: - pad_shape = list(y.shape) - pad_shape[-1] = x.shape[-1] - y.shape[-1] - y = np.concatenate((y, np.zeros(pad_shape)), -1) - - if nperseg is not None: # if specified by user - nperseg = int(nperseg) - if nperseg < 1: - raise ValueError('nperseg must be a positive integer') - - # parse window; if array like, then set nperseg = win.shape - win, nperseg = _triage_segments(window, nperseg,input_length=x.shape[-1]) - - if nfft is None: - nfft = nperseg - elif nfft < nperseg: - raise ValueError('nfft must be greater than or equal to nperseg.') - else: - nfft = int(nfft) - - if noverlap is None: - noverlap = nperseg//2 - else: - noverlap = int(noverlap) - if noverlap >= nperseg: - raise ValueError('noverlap must be less than nperseg.') - nstep = nperseg - noverlap - - # Padding occurs after boundary extension, so that the extended signal ends - # in zeros, instead of introducing an impulse at the end. - # I.e. if x = [..., 3, 2] - # extend then pad -> [..., 3, 2, 2, 3, 0, 0, 0] - # pad then extend -> [..., 3, 2, 0, 0, 0, 2, 3] - - if boundary is not None: - ext_func = boundary_funcs[boundary] - x = ext_func(x, nperseg//2, axis=-1) - if not same_data: - y = ext_func(y, nperseg//2, axis=-1) - - if padded: - # Pad to integer number of windowed segments - # I.e make x.shape[-1] = nperseg + (nseg-1)*nstep, with integer nseg - nadd = (-(x.shape[-1]-nperseg) % nstep) % nperseg - zeros_shape = list(x.shape[:-1]) + [nadd] - x = np.concatenate((x, np.zeros(zeros_shape)), axis=-1) - if not same_data: - zeros_shape = list(y.shape[:-1]) + [nadd] - y = np.concatenate((y, np.zeros(zeros_shape)), axis=-1) - - # Handle detrending and window functions - if not detrend: - def detrend_func(d): - return d - elif not hasattr(detrend, '__call__'): - def detrend_func(d): - return signaltools_detrend(d, type=detrend, axis=-1) - elif axis != -1: - # Wrap this function so that it receives a shape that it could - # reasonably expect to receive. - def detrend_func(d): - d = np.rollaxis(d, -1, axis) - d = detrend(d) - return np.rollaxis(d, axis, len(d.shape)) - else: - detrend_func = detrend - - if np.result_type(win,np.complex64) != outdtype: - win = win.astype(outdtype) - - if scaling == 'density': - scale = 1.0 / (fs * (win*win).sum()) - elif scaling == 'spectrum': - scale = 1.0 / win.sum()**2 - else: - raise ValueError('Unknown scaling: %r' % scaling) - - if mode == 'stft': - scale = np.sqrt(scale) - - if return_onesided: - if np.iscomplexobj(x): - sides = 'twosided' - #warnings.warn('Input data is complex, switching to ' 'return_onesided=False') - else: - sides = 'onesided' - if not same_data: - if np.iscomplexobj(y): - sides = 'twosided' - #warnings.warn('Input data is complex, switching to return_onesided=False') - else: - sides = 'twosided' - - if sides == 'twosided': - raise Exception('NOT IMPLEMENTED') - #freqs = fftpack.fftfreq(nfft, 1/fs) - elif sides == 'onesided': - freqs = np.fft.rfftfreq(nfft, 1/fs) - - # Perform the windowed FFTs - result = _fft_helper(x, win, detrend_func, nperseg, noverlap, nfft, sides) - - if not same_data: - # All the same operations on the y data - result_y = _fft_helper(y, win, detrend_func, nperseg, noverlap, nfft, - sides) - result = np.conjugate(result) * result_y - elif mode == 'psd': - result = np.conjugate(result) * result - - result *= scale - if sides == 'onesided' and mode == 'psd': - if nfft % 2: - result[..., 1:] *= 2 - else: - # Last point is unpaired Nyquist freq point, don't double - result[..., 1:-1] *= 2 - - time = np.arange(nperseg/2, x.shape[-1] - nperseg/2 + 1, - nperseg - noverlap)/float(fs) - if boundary is not None: - time -= (nperseg/2) / fs - - result = result.astype(outdtype) - - # All imaginary parts are zero anyways - if same_data and mode != 'stft': - result = result.real - - # Output is going to have new last axis for time/window index, so a - # negative axis index shifts down one - if axis < 0: - axis -= 1 - - # Roll frequency axis back to axis where the data came from - result = np.rollaxis(result, -1, axis) - - # TODO - class InfoClass(): - pass - Info = InfoClass(); - Info.df=freqs[1]-freqs[0] - Info.fMax=freqs[-1] - Info.LFreq=len(freqs) - Info.LSeg=nperseg - Info.LWin=len(win) - Info.LOvlp=noverlap - Info.nFFT=nfft - Info.nseg=-1 - #print('df:{:.3f} - fm:{:.2f} - nseg:{} - Lf:{:5d} - Lseg:{:5d} - Lwin:{:5d} - Lovlp:{:5d} - Nfft:{:5d} - Lsig:{}'.format(freqs[1]-freqs[0],freqs[-1],-1,len(freqs),nperseg,len(win),noverlap,nfft,x.shape[-1])) - return freqs, time, result, Info - - -def _fft_helper(x, win, detrend_func, nperseg, noverlap, nfft, sides): - """ Calculate windowed FFT """ - # Created strided array of data segments - if nperseg == 1 and noverlap == 0: - result = x[..., np.newaxis] - else: - # http://stackoverflow.com/a/5568169 - step = nperseg - noverlap - shape = x.shape[:-1]+((x.shape[-1]-noverlap)//step, nperseg) - strides = x.strides[:-1]+(step*x.strides[-1], x.strides[-1]) - result = np.lib.stride_tricks.as_strided(x, shape=shape, - strides=strides) - - # Detrend each data segment individually - result = detrend_func(result) - - # Apply window by multiplication - result = win * result - - # Perform the fft. Acts on last axis by default. Zero-pads automatically - if sides == 'twosided': - raise Exception('NOT IMPLEMENTED') - #func = fftpack.fft - else: - result = result.real - func = np.fft.rfft - result = func(result, n=nfft) - - return result - -def _triage_segments(window, nperseg,input_length): - """ - Parses window and nperseg arguments for spectrogram and _spectral_helper. - This is a helper function, not meant to be called externally. - """ - - #parse window; if array like, then set nperseg = win.shape - if isinstance(window, string_types) or isinstance(window, tuple): - # if nperseg not specified - if nperseg is None: - nperseg = 256 # then change to default - if nperseg > input_length: - print('nperseg = {0:d} is greater than input length ' - ' = {1:d}, using nperseg = {1:d}' - .format(nperseg, input_length)) - nperseg = input_length - win = get_window(window, nperseg) - else: - win = np.asarray(window) - if len(win.shape) != 1: - raise ValueError('window must be 1-D') - if input_length < win.shape[-1]: - raise ValueError('window is longer than input signal') - if nperseg is None: - nperseg = win.shape[0] - elif nperseg is not None: - if nperseg != win.shape[0]: - raise ValueError("value specified for nperseg is different from" - " length of window") - - return win, nperseg - - - - - - -# --------------------------------------------------------------------------------} -# --- Unittests -# --------------------------------------------------------------------------------{ -import unittest - -class TestSpectral(unittest.TestCase): - - def test_fft_amplitude(self): - dt=0.1 - t=np.arange(0,10,dt); - f0=1; - A=5; - y=A*np.sin(2*np.pi*f0*t) - f,Y,_=fft_amplitude(y,fs=1/dt,detrend=False) - i=np.argmax(Y) - self.assertAlmostEqual(Y[i],A) - self.assertAlmostEqual(f[i],f0) - -if __name__ == '__main__': - unittest.main() - +# Tools for spectral analysis of a real valued signal. +# +# The functions in this file were adapted from the python package scipy according to the following license: +# +# License: +# Copyright 2001, 2002 Enthought, Inc. +# All rights reserved. +# +# Copyright 2003-2013 SciPy Developers. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +# Neither the name of Enthought nor the names of the SciPy Developers may be used to endorse or promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import division, print_function, absolute_import + +import numpy as np +import pandas as pd +from six import string_types + +__all__ = ['fft_wrap','welch', 'psd', 'fft_amplitude'] +__all__ += ['pwelch', 'csd', 'coherence'] +__all__ += ['fnextpow2'] +__all__ += ['hann','hamming','boxcar','general_hamming','get_window'] +__all__ += ['TestSpectral'] + + +# --------------------------------------------------------------------------------} +# --- FFT wrap +# --------------------------------------------------------------------------------{ +def fft_wrap(t,y,dt=None, output_type='amplitude',averaging='None',averaging_window='hamming',detrend=False,nExp=None, nPerDecade=None): + """ + Wrapper to compute FFT amplitude or power spectra, with averaging. + INPUTS: + output_type : amplitude, PSD, f x PSD + averaging : None, Welch, Binning + averaging_window : Hamming, Hann, Rectangular + OUTPUTS: + frq: vector of frequencies + Y : Amplitude spectrum, PSD, or f * PSD + Info: a dictionary of info values + """ + + # Formatting inputs + output_type = output_type.lower() + averaging = averaging.lower() + averaging_window = averaging_window.lower() + y = np.asarray(y) + y = y[~np.isnan(y)] + n = len(y) + + if dt is None: + dtDelta0 = t[1]-t[0] + # Hack to use a constant dt + dt = (np.max(t)-np.min(t))/(n-1) + if dtDelta0 !=dt: + print('[WARN] dt from tmax-tmin different from dt from t2-t1' ) + Fs = 1/dt + if averaging =='none': + frq, PSD, Info = psd(y, fs=Fs, detrend=detrend, return_onesided=True) + elif averaging =='binning': + frq, PSD, Info = psd_binned(y, fs=Fs, detrend=detrend, return_onesided=True, nPerDecade=nPerDecade) + elif averaging=='welch': + # --- Welch - PSD + #overlap_frac=0.5 + #return fnextpow2(np.sqrt(len(x)/(1-overlap_frac))) + nFFTAll=fnextpow2(n) + if nExp is None: + nExp=int(np.log(nFFTAll)/np.log(2))-1 + nPerSeg=2**nExp + if nPerSeg>n: + print('[WARN] Power of 2 value was too high and was reduced. Disable averaging to use the full spectrum.'); + nExp=int(np.log(nFFTAll)/np.log(2))-1 + nPerSeg=2**nExp + if averaging_window=='hamming': + window = hamming(nPerSeg, True)# True=Symmetric, like matlab + elif averaging_window=='hann': + window = hann(nPerSeg, True) + elif averaging_window=='rectangular': + window = boxcar(nPerSeg) + else: + raise Exception('Averaging window unknown {}'.format(averaging_window)) + frq, PSD, Info = pwelch(y, fs=Fs, window=window, detrend=detrend) + Info.nExp = nExp + else: + raise Exception('Averaging method unknown {}'.format(averaging)) + + # --- Formatting output + if output_type=='amplitude': + deltaf = frq[1]-frq[0] + Y = np.sqrt(PSD*2*deltaf) + # NOTE: the above should be the same as:Y=abs(Y[range(nhalf)])/n;Y[1:-1]=Y[1:-1]*2; + elif output_type=='psd': # one sided + Y = PSD + elif output_type=='f x psd': + Y = PSD*frq + else: + raise NotImplementedError('Contact developer') + if detrend: + frq= frq[1:] + Y = Y[1:] + return frq, Y, Info + + + +# --------------------------------------------------------------------------------} +# --- Spectral simple (averaging below) +# --------------------------------------------------------------------------------{ +def fft_amplitude(y, fs=1.0, detrend ='constant', return_onesided=True): + """ Returns FFT amplitude of signal """ + frq, PSD, Info = psd(y, fs=fs, detrend=detrend, return_onesided=return_onesided) + deltaf = frq[1]-frq[0] + Y = np.sqrt(PSD*2*deltaf) + return frq, Y, Info + + +def psd_binned(y, fs=1.0, nPerDecade=10, detrend ='constant', return_onesided=True): + """ + Return PSD binned with nPoints per decade + """ + # --- First return regular PSD + frq, PSD, Info = psd(y, fs=fs, detrend=detrend, return_onesided=return_onesided) + + add0=False + if frq[0]==0: + add0=True + f0 = 0 + PSD0 = PSD[0] + frq=frq[1:] + PSD=PSD[1:] + + # -- Then bin per decase + log_f = np.log10(frq) + ndecades = np.ceil(log_f[-1] -log_f[0]) + xbins = np.linspace(log_f[0], log_f[-1], int(ndecades*nPerDecade)) + + # Using Pandas to bin.. + df = pd.DataFrame(data=np.column_stack((log_f,PSD)), columns=['x','y']) + xmid = (xbins[:-1]+xbins[1:])/2 + df['Bin'] = pd.cut(df['x'], bins=xbins, labels=xmid ) # Adding a column that has bin attribute + df2 = df.groupby('Bin').mean() # Average by bin + df2 = df2.reindex(xmid) + log_f_bin = df2['x'].values + PSD_bin = df2['y'].values + frq2= 10**log_f_bin + PSD2= PSD_bin + if add0: + frq2=np.concatenate( ([f0 ], frq2) ) + PSD2=np.concatenate( ([PSD0], PSD2) ) + b = ~np.isnan(frq2) + frq2 = frq2[b] + PSD2 = PSD2[b] + + #import matplotlib.pyplot as plt + #fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) + #fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) + #ax.plot(log_f, PSD, label='') + #ax.plot(log_f_bin, PSD_bin, 'o', label='') + #for x in xbins: + # ax.axvline(x, ls=':', c=(0.5,0.5,0.5)) + #ax.set_xlabel('') + #ax.set_ylabel('') + #ax.legend() + #plt.show() + + #Info.df = frq[1]-frq[0] + #Info.fMax = frq[-1] + #Info.LFreq = len(frq) + #Info.LSeg = len(Y) + #Info.LWin = len(Y) + #Info.LOvlp = 0 + #Info.nFFT = len(Y) + #Info.nseg = 1 + Info.nPerDecade = nPerDecade + Info.xbins = xbins + + return frq2, PSD2, Info + + +def psd(y, fs=1.0, detrend ='constant', return_onesided=True): + """ Perform PSD without averaging """ + if not return_onesided: + raise NotImplementedError('Double sided todo') + + if detrend is None: + detrend=False + + if detrend=='constant' or detrend==True: + m=np.mean(y); + else: + m=0; + + n = len(y) + if n%2==0: + nhalf = int(n/2+1) + else: + nhalf = int((n+1)/2) + + frq = np.arange(nhalf)*fs/n; + Y = np.fft.rfft(y-m) #Y = np.fft.fft(y) + PSD = abs(Y[range(nhalf)])**2 /(n*fs) # PSD + PSD[1:-1] = PSD[1:-1]*2; + class InfoClass(): + pass + Info = InfoClass(); + Info.df = frq[1]-frq[0] + Info.fMax = frq[-1] + Info.LFreq = len(frq) + Info.LSeg = len(Y) + Info.LWin = len(Y) + Info.LOvlp = 0 + Info.nFFT = len(Y) + Info.nseg = 1 + return frq, PSD, Info + + +# --------------------------------------------------------------------------------} +# --- Windows +# --------------------------------------------------------------------------------{ +"""The suite of window functions.""" +def fnextpow2(x): + return 2**np.ceil( np.log(x)*0.99999999999/np.log(2)); + +def fDefaultWinLen(x,overlap_frac=0.5): + return fnextpow2(np.sqrt(len(x)/(1-overlap_frac))) + +def fDefaultWinLenMatlab(x): + return np.fix((len(x)-3)*2./9.) + +def _len_guards(M): + """Handle small or incorrect window lengths""" + if int(M) != M or M < 0: + raise ValueError('Window length M must be a non-negative integer') + return M <= 1 + +def _extend(M, sym): + """Extend window by 1 sample if needed for DFT-even symmetry""" + if not sym: + return M + 1, True + else: + return M, False + +def _truncate(w, needed): + """Truncate window by 1 sample if needed for DFT-even symmetry""" + if needed: + return w[:-1] + else: + return w + +def general_cosine(M, a, sym=True): + if _len_guards(M): + return np.ones(M) + M, needs_trunc = _extend(M, sym) + + fac = np.linspace(-np.pi, np.pi, M) + w = np.zeros(M) + for k in range(len(a)): + w += a[k] * np.cos(k * fac) + + return _truncate(w, needs_trunc) + + +def boxcar(M, sym=True): + """Return a boxcar or rectangular window. + + Also known as a rectangular window or Dirichlet window, this is equivalent + to no window at all. + """ + if _len_guards(M): + return np.ones(M) + M, needs_trunc = _extend(M, sym) + + w = np.ones(M, float) + + return _truncate(w, needs_trunc) + +def hann(M, sym=True): # same as hanning(*args, **kwargs): + return general_hamming(M, 0.5, sym) + + +def general_hamming(M, alpha, sym=True): + r"""Return a generalized Hamming window. + The generalized Hamming window is constructed by multiplying a rectangular + window by one period of a cosine function [1]_. + w(n) = \alpha - \left(1 - \alpha\right) \cos\left(\frac{2\pi{n}}{M-1}\right) + \qquad 0 \leq n \leq M-1 + """ + return general_cosine(M, [alpha, 1. - alpha], sym) + + +def hamming(M, sym=True): + r"""Return a Hamming window. + The Hamming window is a taper formed by using a raised cosine with + non-zero endpoints, optimized to minimize the nearest side lobe. + w(n) = 0.54 - 0.46 \cos\left(\frac{2\pi{n}}{M-1}\right) + \qquad 0 \leq n \leq M-1 + """ + return general_hamming(M, 0.54, sym) + +_win_equiv_raw = { + ('boxcar', 'box', 'ones', 'rect', 'rectangular'): (boxcar, False), + ('hamming', 'hamm', 'ham'): (hamming, False), + ('hanning', 'hann', 'han'): (hann, False), +} + +# Fill dict with all valid window name strings +_win_equiv = {} +for k, v in _win_equiv_raw.items(): + for key in k: + _win_equiv[key] = v[0] + +# Keep track of which windows need additional parameters +_needs_param = set() +for k, v in _win_equiv_raw.items(): + if v[1]: + _needs_param.update(k) + + +def get_window(window, Nx, fftbins=True): + """ + Return a window. + + Parameters + ---------- + window : string, float, or tuple + The type of window to create. See below for more details. + Nx : int + The number of samples in the window. + fftbins : bool, optional + If True (default), create a "periodic" window, ready to use with + `ifftshift` and be multiplied by the result of an FFT (see also + `fftpack.fftfreq`). + If False, create a "symmetric" window, for use in filter design. + """ + sym = not fftbins + try: + beta = float(window) + except (TypeError, ValueError): + args = () + if isinstance(window, tuple): + winstr = window[0] + if len(window) > 1: + args = window[1:] + elif isinstance(window, string_types): + if window in _needs_param: + raise ValueError("The '" + window + "' window needs one or " + "more parameters -- pass a tuple.") + else: + winstr = window + else: + raise ValueError("%s as window type is not supported." % + str(type(window))) + + try: + winfunc = _win_equiv[winstr] + except KeyError: + raise ValueError("Unknown window type.") + + params = (Nx,) + args + (sym,) + else: + winfunc = kaiser + params = (Nx, beta, sym) + + return winfunc(*params) + + + + + + +# --------------------------------------------------------------------------------} +# --- Helpers +# --------------------------------------------------------------------------------{ +def odd_ext(x, n, axis=-1): + """ + Odd extension at the boundaries of an array + Generate a new ndarray by making an odd extension of `x` along an axis. + """ + if n < 1: + return x + if n > x.shape[axis] - 1: + raise ValueError(("The extension length n (%d) is too big. " + + "It must not exceed x.shape[axis]-1, which is %d.") + % (n, x.shape[axis] - 1)) + left_end = axis_slice(x, start=0, stop=1, axis=axis) + left_ext = axis_slice(x, start=n, stop=0, step=-1, axis=axis) + right_end = axis_slice(x, start=-1, axis=axis) + right_ext = axis_slice(x, start=-2, stop=-(n + 2), step=-1, axis=axis) + ext = np.concatenate((2 * left_end - left_ext, + x, + 2 * right_end - right_ext), + axis=axis) + return ext + + +def even_ext(x, n, axis=-1): + """ + Even extension at the boundaries of an array + Generate a new ndarray by making an even extension of `x` along an axis. + """ + if n < 1: + return x + if n > x.shape[axis] - 1: + raise ValueError(("The extension length n (%d) is too big. " + + "It must not exceed x.shape[axis]-1, which is %d.") + % (n, x.shape[axis] - 1)) + left_ext = axis_slice(x, start=n, stop=0, step=-1, axis=axis) + right_ext = axis_slice(x, start=-2, stop=-(n + 2), step=-1, axis=axis) + ext = np.concatenate((left_ext, + x, + right_ext), + axis=axis) + return ext + + +def const_ext(x, n, axis=-1): + """ + Constant extension at the boundaries of an array + Generate a new ndarray that is a constant extension of `x` along an axis. + The extension repeats the values at the first and last element of + the axis. + """ + if n < 1: + return x + left_end = axis_slice(x, start=0, stop=1, axis=axis) + ones_shape = [1] * x.ndim + ones_shape[axis] = n + ones = np.ones(ones_shape, dtype=x.dtype) + left_ext = ones * left_end + right_end = axis_slice(x, start=-1, axis=axis) + right_ext = ones * right_end + ext = np.concatenate((left_ext, + x, + right_ext), + axis=axis) + return ext + + +def zero_ext(x, n, axis=-1): + """ + Zero padding at the boundaries of an array + Generate a new ndarray that is a zero padded extension of `x` along + an axis. + """ + if n < 1: + return x + zeros_shape = list(x.shape) + zeros_shape[axis] = n + zeros = np.zeros(zeros_shape, dtype=x.dtype) + ext = np.concatenate((zeros, x, zeros), axis=axis) + return ext + +def signaltools_detrend(data, axis=-1, type='linear', bp=0): + """ + Remove linear trend along axis from data. + + Parameters + ---------- + data : array_like + The input data. + axis : int, optional + The axis along which to detrend the data. By default this is the + last axis (-1). + type : {'linear', 'constant'}, optional + The type of detrending. If ``type == 'linear'`` (default), + the result of a linear least-squares fit to `data` is subtracted + from `data`. + If ``type == 'constant'``, only the mean of `data` is subtracted. + bp : array_like of ints, optional + A sequence of break points. If given, an individual linear fit is + performed for each part of `data` between two break points. + Break points are specified as indices into `data`. + + Returns + ------- + ret : ndarray + The detrended input data. + """ + if type not in ['linear', 'l', 'constant', 'c']: + raise ValueError("Trend type must be 'linear' or 'constant'.") + data = np.asarray(data) + dtype = data.dtype.char + if dtype not in 'dfDF': + dtype = 'd' + if type in ['constant', 'c']: + #print('Removing mean') + ret = data - np.expand_dims(np.mean(data, axis), axis) + return ret + else: + #print('Removing linear?') + dshape = data.shape + N = dshape[axis] + bp = sort(unique(r_[0, bp, N])) + if np.any(bp > N): + raise ValueError("Breakpoints must be less than length " + "of data along given axis.") + Nreg = len(bp) - 1 + # Restructure data so that axis is along first dimension and + # all other dimensions are collapsed into second dimension + rnk = len(dshape) + if axis < 0: + axis = axis + rnk + newdims = r_[axis, 0:axis, axis + 1:rnk] + newdata = reshape(np.transpose(data, tuple(newdims)), + (N, _prod(dshape) // N)) + newdata = newdata.copy() # make sure we have a copy + if newdata.dtype.char not in 'dfDF': + newdata = newdata.astype(dtype) + # Find leastsq fit and remove it for each piece + for m in range(Nreg): + Npts = bp[m + 1] - bp[m] + A = ones((Npts, 2), dtype) + A[:, 0] = cast[dtype](np.arange(1, Npts + 1) * 1.0 / Npts) + sl = slice(bp[m], bp[m + 1]) + coef, resids, rank, s = np.linalg.lstsq(A, newdata[sl]) + newdata[sl] = newdata[sl] - dot(A, coef) + # Put data back in original shape. + tdshape = take(dshape, newdims, 0) + ret = np.reshape(newdata, tuple(tdshape)) + vals = list(range(1, rnk)) + olddims = vals[:axis] + [0] + vals[axis:] + ret = np.transpose(ret, tuple(olddims)) + return ret + + + +# --------------------------------------------------------------------------------} +# --- Spectral Averaging +# --------------------------------------------------------------------------------{ +"""Tools for spectral analysis. """ + +def welch(x, fs=1.0, window='hann', nperseg=None, noverlap=None, nfft=None, + detrend='constant', return_onesided=True, scaling='density', + axis=-1): + """Interface identical to scipy.signal """ + + if detrend==True: + detrend='constant' + + freqs, Pxx = csd(x, x, fs, window, nperseg, noverlap, nfft, detrend, return_onesided, scaling, axis) + return freqs, Pxx.real + +#>>>> +def pwelch(x, window='hamming', noverlap=None, nfft=None, fs=1.0, nperseg=None, + detrend=False, return_onesided=True, scaling='density', + axis=-1): + r""" + NOTE: interface and default options modified to match matlab's implementation + >> detrend: default to False + >> window : default to 'hamming' + >> window: if an integer, use 'hamming(window, sym=True)' + + + Estimate power spectral density using Welch's method. + + Welch's method [1]_ computes an estimate of the power spectral + density by dividing the data into overlapping segments, computing a + modified periodogram for each segment and averaging the + periodograms. + + Parameters + ---------- + x : array_like + Time series of measurement values + fs : float, optional + Sampling frequency of the `x` time series. Defaults to 1.0. + window : str or tuple or array_like, optional + Desired window to use. If `window` is a string or tuple, it is + passed to `get_window` to generate the window values, which are + DFT-even by default. See `get_window` for a list of windows and + required parameters. If `window` is array_like it will be used + directly as the window and its length must be nperseg. Defaults + to a Hann window. + nperseg : int, optional + Length of each segment. Defaults to None, but if window is str or + tuple, is set to 256, and if window is array_like, is set to the + length of the window. + noverlap : int, optional + Number of points to overlap between segments. If `None`, + ``noverlap = nperseg // 2``. Defaults to `None`. + nfft : int, optional + Length of the FFT used, if a zero padded FFT is desired. If + `None`, the FFT length is `nperseg`. Defaults to `None`. + detrend : str or function or `False`, optional + Specifies how to detrend each segment. If `detrend` is a + string, it is passed as the `type` argument to the `detrend` + function. If it is a function, it takes a segment and returns a + detrended segment. If `detrend` is `False`, no detrending is + done. Defaults to 'constant'. + return_onesided : bool, optional + If `True`, return a one-sided spectrum for real data. If + `False` return a two-sided spectrum. Note that for complex + data, a two-sided spectrum is always returned. + scaling : { 'density', 'spectrum' }, optional + Selects between computing the power spectral density ('density') + where `Pxx` has units of V**2/Hz and computing the power + spectrum ('spectrum') where `Pxx` has units of V**2, if `x` + is measured in V and `fs` is measured in Hz. Defaults to + 'density' + axis : int, optional + Axis along which the periodogram is computed; the default is + over the last axis (i.e. ``axis=-1``). + + Returns + ------- + f : ndarray + Array of sample frequencies. + Pxx : ndarray + Power spectral density or power spectrum of x. + + See Also + -------- + periodogram: Simple, optionally modified periodogram + lombscargle: Lomb-Scargle periodogram for unevenly sampled data + + Notes + ----- + An appropriate amount of overlap will depend on the choice of window + and on your requirements. For the default Hann window an overlap of + 50% is a reasonable trade off between accurately estimating the + signal power, while not over counting any of the data. Narrower + windows may require a larger overlap. + + If `noverlap` is 0, this method is equivalent to Bartlett's method + [2]_. + + .. versionadded:: 0.12.0 + + References + ---------- + .. [1] P. Welch, "The use of the fast Fourier transform for the + estimation of power spectra: A method based on time averaging + over short, modified periodograms", IEEE Trans. Audio + Electroacoust. vol. 15, pp. 70-73, 1967. + .. [2] M.S. Bartlett, "Periodogram Analysis and Continuous Spectra", + Biometrika, vol. 37, pp. 1-16, 1950. + + """ + import math + def fnextpow2(x): + return 2**math.ceil( math.log(x)*0.99999999999/math.log(2)); + + # MANU >>> CHANGE OF DEFAULT OPTIONS + # MANU - If a length is provided use symmetric hamming window + if type(window)==int: + window=hamming(window, True) + # MANU - do not use 256 as default + if isinstance(window, string_types) or isinstance(window, tuple): + if nperseg is None: + if noverlap is None: + overlap_frac=0.5 + elif noverlap == 0: + overlap_frac=0 + else: + raise NotImplementedError('TODO noverlap set but not nperseg') + #nperseg = 256 # then change to default + nperseg=fnextpow2(math.sqrt(x.shape[-1]/(1-overlap_frac))); + + # MANU accepting true as detrend + if detrend==True: + detrend='constant' + + freqs, Pxx, Info = csd(x, x, fs, window, nperseg, noverlap, nfft, detrend, + return_onesided, scaling, axis, returnInfo=True) + + return freqs, Pxx.real, Info + + +def csd(x, y, fs=1.0, window='hann', nperseg=None, noverlap=None, nfft=None, + detrend='constant', return_onesided=True, scaling='density', axis=-1, + returnInfo=False + ): + r""" + Estimate the cross power spectral density, Pxy, using Welch's + method. + """ + + freqs, _, Pxy, Info = _spectral_helper(x, y, fs, window, nperseg, noverlap, nfft, + detrend, return_onesided, scaling, axis, + mode='psd') + + # Average over windows. + if len(Pxy.shape) >= 2 and Pxy.size > 0: + if Pxy.shape[-1] > 1: + Pxy = Pxy.mean(axis=-1) + else: + Pxy = np.reshape(Pxy, Pxy.shape[:-1]) + + if returnInfo: + return freqs, Pxy, Info + else: + return freqs, Pxy + + + +def coherence(x, y, fs=1.0, window='hann', nperseg=None, noverlap=None, + nfft=None, detrend='constant', axis=-1): + r""" + Estimate the magnitude squared coherence estimate, Cxy, of + discrete-time signals X and Y using Welch's method. + + ``Cxy = abs(Pxy)**2/(Pxx*Pyy)``, where `Pxx` and `Pyy` are power + spectral density estimates of X and Y, and `Pxy` is the cross + spectral density estimate of X and Y. + """ + + freqs, Pxx, Infoxx = welch(x, fs, window, nperseg, noverlap, nfft, detrend, axis=axis) + _, Pyy, Infoyy = welch(y, fs, window, nperseg, noverlap, nfft, detrend, axis=axis) + _, Pxy, Infoxy = csd(x, y, fs, window, nperseg, noverlap, nfft, detrend, axis=axis, returnInfo=True) + + Cxy = np.abs(Pxy)**2 / Pxx / Pyy + + return freqs, Cxy, Infoxx + + +def _spectral_helper(x, y, fs=1.0, window='hann', nperseg=None, noverlap=None, + nfft=None, detrend='constant', return_onesided=True, + scaling='spectrum', axis=-1, mode='psd', boundary=None, + padded=False): + """ Calculate various forms of windowed FFTs for PSD, CSD, etc. """ + if mode not in ['psd', 'stft']: + raise ValueError("Unknown value for mode %s, must be one of: " + "{'psd', 'stft'}" % mode) + + + + + + boundary_funcs = {'even': even_ext, + 'odd': odd_ext, + 'constant': const_ext, + 'zeros': zero_ext, + None: None} + + if boundary not in boundary_funcs: + raise ValueError("Unknown boundary option '{0}', must be one of: {1}" + .format(boundary, list(boundary_funcs.keys()))) + + # If x and y are the same object we can save ourselves some computation. + same_data = y is x + + if not same_data and mode != 'psd': + raise ValueError("x and y must be equal if mode is 'stft'") + + axis = int(axis) + + # Ensure we have np.arrays, get outdtype + x = np.asarray(x) + if not same_data: + y = np.asarray(y) + outdtype = np.result_type(x, y, np.complex64) + else: + outdtype = np.result_type(x, np.complex64) + + if not same_data: + # Check if we can broadcast the outer axes together + xouter = list(x.shape) + youter = list(y.shape) + xouter.pop(axis) + youter.pop(axis) + try: + outershape = np.broadcast(np.empty(xouter), np.empty(youter)).shape + except ValueError: + raise ValueError('x and y cannot be broadcast together.') + + if same_data: + if x.size == 0: + return np.empty(x.shape), np.empty(x.shape), np.empty(x.shape) + else: + if x.size == 0 or y.size == 0: + outshape = outershape + (min([x.shape[axis], y.shape[axis]]),) + emptyout = np.rollaxis(np.empty(outshape), -1, axis) + return emptyout, emptyout, emptyout + + if x.ndim > 1: + if axis != -1: + x = np.rollaxis(x, axis, len(x.shape)) + if not same_data and y.ndim > 1: + y = np.rollaxis(y, axis, len(y.shape)) + + # Check if x and y are the same length, zero-pad if necessary + if not same_data: + if x.shape[-1] != y.shape[-1]: + if x.shape[-1] < y.shape[-1]: + pad_shape = list(x.shape) + pad_shape[-1] = y.shape[-1] - x.shape[-1] + x = np.concatenate((x, np.zeros(pad_shape)), -1) + else: + pad_shape = list(y.shape) + pad_shape[-1] = x.shape[-1] - y.shape[-1] + y = np.concatenate((y, np.zeros(pad_shape)), -1) + + if nperseg is not None: # if specified by user + nperseg = int(nperseg) + if nperseg < 1: + raise ValueError('nperseg must be a positive integer') + + # parse window; if array like, then set nperseg = win.shape + win, nperseg = _triage_segments(window, nperseg,input_length=x.shape[-1]) + + if nfft is None: + nfft = nperseg + elif nfft < nperseg: + raise ValueError('nfft must be greater than or equal to nperseg.') + else: + nfft = int(nfft) + + if noverlap is None: + noverlap = nperseg//2 + else: + noverlap = int(noverlap) + if noverlap >= nperseg: + raise ValueError('noverlap must be less than nperseg.') + nstep = nperseg - noverlap + + # Padding occurs after boundary extension, so that the extended signal ends + # in zeros, instead of introducing an impulse at the end. + # I.e. if x = [..., 3, 2] + # extend then pad -> [..., 3, 2, 2, 3, 0, 0, 0] + # pad then extend -> [..., 3, 2, 0, 0, 0, 2, 3] + + if boundary is not None: + ext_func = boundary_funcs[boundary] + x = ext_func(x, nperseg//2, axis=-1) + if not same_data: + y = ext_func(y, nperseg//2, axis=-1) + + if padded: + # Pad to integer number of windowed segments + # I.e make x.shape[-1] = nperseg + (nseg-1)*nstep, with integer nseg + nadd = (-(x.shape[-1]-nperseg) % nstep) % nperseg + zeros_shape = list(x.shape[:-1]) + [nadd] + x = np.concatenate((x, np.zeros(zeros_shape)), axis=-1) + if not same_data: + zeros_shape = list(y.shape[:-1]) + [nadd] + y = np.concatenate((y, np.zeros(zeros_shape)), axis=-1) + + # Handle detrending and window functions + if not detrend: + def detrend_func(d): + return d + elif not hasattr(detrend, '__call__'): + def detrend_func(d): + return signaltools_detrend(d, type=detrend, axis=-1) + elif axis != -1: + # Wrap this function so that it receives a shape that it could + # reasonably expect to receive. + def detrend_func(d): + d = np.rollaxis(d, -1, axis) + d = detrend(d) + return np.rollaxis(d, axis, len(d.shape)) + else: + detrend_func = detrend + + if np.result_type(win,np.complex64) != outdtype: + win = win.astype(outdtype) + + if scaling == 'density': + scale = 1.0 / (fs * (win*win).sum()) + elif scaling == 'spectrum': + scale = 1.0 / win.sum()**2 + else: + raise ValueError('Unknown scaling: %r' % scaling) + + if mode == 'stft': + scale = np.sqrt(scale) + + if return_onesided: + if np.iscomplexobj(x): + sides = 'twosided' + #warnings.warn('Input data is complex, switching to ' 'return_onesided=False') + else: + sides = 'onesided' + if not same_data: + if np.iscomplexobj(y): + sides = 'twosided' + #warnings.warn('Input data is complex, switching to return_onesided=False') + else: + sides = 'twosided' + + if sides == 'twosided': + raise Exception('NOT IMPLEMENTED') + #freqs = fftpack.fftfreq(nfft, 1/fs) + elif sides == 'onesided': + freqs = np.fft.rfftfreq(nfft, 1/fs) + + # Perform the windowed FFTs + result = _fft_helper(x, win, detrend_func, nperseg, noverlap, nfft, sides) + + if not same_data: + # All the same operations on the y data + result_y = _fft_helper(y, win, detrend_func, nperseg, noverlap, nfft, + sides) + result = np.conjugate(result) * result_y + elif mode == 'psd': + result = np.conjugate(result) * result + + result *= scale + if sides == 'onesided' and mode == 'psd': + if nfft % 2: + result[..., 1:] *= 2 + else: + # Last point is unpaired Nyquist freq point, don't double + result[..., 1:-1] *= 2 + + time = np.arange(nperseg/2, x.shape[-1] - nperseg/2 + 1, + nperseg - noverlap)/float(fs) + if boundary is not None: + time -= (nperseg/2) / fs + + result = result.astype(outdtype) + + # All imaginary parts are zero anyways + if same_data and mode != 'stft': + result = result.real + + # Output is going to have new last axis for time/window index, so a + # negative axis index shifts down one + if axis < 0: + axis -= 1 + + # Roll frequency axis back to axis where the data came from + result = np.rollaxis(result, -1, axis) + + # TODO + class InfoClass(): + pass + Info = InfoClass(); + Info.df=freqs[1]-freqs[0] + Info.fMax=freqs[-1] + Info.LFreq=len(freqs) + Info.LSeg=nperseg + Info.LWin=len(win) + Info.LOvlp=noverlap + Info.nFFT=nfft + Info.nseg=-1 + #print('df:{:.3f} - fm:{:.2f} - nseg:{} - Lf:{:5d} - Lseg:{:5d} - Lwin:{:5d} - Lovlp:{:5d} - Nfft:{:5d} - Lsig:{}'.format(freqs[1]-freqs[0],freqs[-1],-1,len(freqs),nperseg,len(win),noverlap,nfft,x.shape[-1])) + return freqs, time, result, Info + + +def _fft_helper(x, win, detrend_func, nperseg, noverlap, nfft, sides): + """ Calculate windowed FFT """ + # Created strided array of data segments + if nperseg == 1 and noverlap == 0: + result = x[..., np.newaxis] + else: + # http://stackoverflow.com/a/5568169 + step = nperseg - noverlap + shape = x.shape[:-1]+((x.shape[-1]-noverlap)//step, nperseg) + strides = x.strides[:-1]+(step*x.strides[-1], x.strides[-1]) + result = np.lib.stride_tricks.as_strided(x, shape=shape, + strides=strides) + + # Detrend each data segment individually + result = detrend_func(result) + + # Apply window by multiplication + result = win * result + + # Perform the fft. Acts on last axis by default. Zero-pads automatically + if sides == 'twosided': + raise Exception('NOT IMPLEMENTED') + #func = fftpack.fft + else: + result = result.real + func = np.fft.rfft + result = func(result, n=nfft) + + return result + +def _triage_segments(window, nperseg,input_length): + """ + Parses window and nperseg arguments for spectrogram and _spectral_helper. + This is a helper function, not meant to be called externally. + """ + + #parse window; if array like, then set nperseg = win.shape + if isinstance(window, string_types) or isinstance(window, tuple): + # if nperseg not specified + if nperseg is None: + nperseg = 256 # then change to default + if nperseg > input_length: + print('nperseg = {0:d} is greater than input length ' + ' = {1:d}, using nperseg = {1:d}' + .format(nperseg, input_length)) + nperseg = input_length + win = get_window(window, nperseg) + else: + win = np.asarray(window) + if len(win.shape) != 1: + raise ValueError('window must be 1-D') + if input_length < win.shape[-1]: + raise ValueError('window is longer than input signal') + if nperseg is None: + nperseg = win.shape[0] + elif nperseg is not None: + if nperseg != win.shape[0]: + raise ValueError("value specified for nperseg is different from" + " length of window") + + return win, nperseg + + + + + + +# --------------------------------------------------------------------------------} +# --- Unittests +# --------------------------------------------------------------------------------{ +import unittest + +class TestSpectral(unittest.TestCase): + + def test_fft_amplitude(self): + dt=0.1 + t=np.arange(0,10,dt); + f0=1; + A=5; + y=A*np.sin(2*np.pi*f0*t) + f,Y,_=fft_amplitude(y,fs=1/dt,detrend=False) + i=np.argmax(Y) + self.assertAlmostEqual(Y[i],A) + self.assertAlmostEqual(f[i],f0) + + def test_fft_binning(self): + dt=0.1 + t=np.arange(0,10,dt); + f0=1; + A=5; + y=A*np.sin(2*np.pi*f0*t) + + f, Y, Info = psd_binned(y, fs=1/dt, nPerDecade=10, detrend ='constant') + f2, Y2, Info2 = psd (y, fs=1/dt, detrend ='constant') + #print(f) + #print(Y) + + #import matplotlib.pyplot as plt + #fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) + #fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) + #ax.plot( f2, Y2 , label='Full') + #ax.plot( f, Y , label='Binned') + #ax.set_xlabel('') + #ax.set_ylabel('') + #ax.legend() + #plt.show() + +if __name__ == '__main__': + #TestSpectral().test_fft_binning() + unittest.main() + diff --git a/pydatview/tools/stats.py b/pydatview/tools/stats.py index 915e04a..aaa66dc 100644 --- a/pydatview/tools/stats.py +++ b/pydatview/tools/stats.py @@ -1,195 +1,250 @@ -""" -Set of tools for statistics - - measures (R^2, RMSE) - - pdf distributions - - Binning - -""" -import numpy as np -import pandas as pd - -# --------------------------------------------------------------------------------} -# --- Stats measures -# --------------------------------------------------------------------------------{ -def rsquare(y,f, c = True): - """ Compute coefficient of determination of data fit model and RMSE - [r2 rmse] = rsquare(y,f) - [r2 rmse] = rsquare(y,f,c) - RSQUARE computes the coefficient of determination (R-square) value from - actual data Y and model data F. The code uses a general version of - R-square, based on comparing the variability of the estimation errors - with the variability of the original values. RSQUARE also outputs the - root mean squared error (RMSE) for the user's convenience. - Note: RSQUARE ignores comparisons involving NaN values. - INPUTS - Y : Actual data - F : Model fit - - # OPTION - C : Constant term in model - R-square may be a questionable measure of fit when no - constant term is included in the model. - [DEFAULT] TRUE : Use traditional R-square computation - FALSE : Uses alternate R-square computation for model - without constant term [R2 = 1 - NORM(Y-F)/NORM(Y)] - # OUTPUT - R2 : Coefficient of determination - RMSE : Root mean squared error """ - # Compare inputs - if not np.all(y.shape == f.shape) : - raise Exception('Y and F must be the same size') - # Check for NaN - tmp = np.logical_not(np.logical_or(np.isnan(y),np.isnan(f))) - y = y[tmp] - f = f[tmp] - if c: - r2 = max(0,1-np.sum((y-f)**2)/np.sum((y-np.mean(y))** 2)) - else: - r2 = 1 - np.sum((y - f) ** 2) / np.sum((y) ** 2) - if r2 < 0: - import warnings - warnings.warn('Consider adding a constant term to your model') - r2 = 0 - rmse = np.sqrt(np.mean((y - f) ** 2)) - return r2,rmse - -def mean_rel_err(t1, y1, t2, y2, method='mean'): - """ - Methods: - 'mean' : 100 * |y1-y2|/mean(y1) - 'meanabs': 100 * |y1-y2|/mean(|y1|) - 'minmax': y1 and y2 scaled between 0.001 and 1 - |y1s-y2s|/|y1| - """ - if len(y1)!=len(y2): - y2=np.interp(t1,t2,y2) - # Method 1 relative to mean - if method=='mean': - ref_val = np.mean(y1) - meanrelerr = np.mean(np.abs(y1-y2)/ref_val)*100 - elif method=='meanabs': - ref_val = np.mean(np.abs(y1)) - meanrelerr = np.mean(np.abs(y1-y2)/ref_val)*100 - elif method=='minmax': - # Method 2 scaling signals - Min=min(np.min(y1), np.min(y2)) - Max=max(np.max(y1), np.max(y2)) - y1=(y1-Min)/(Max-Min)+0.001 - y2=(y2-Min)/(Max-Min)+0.001 - meanrelerr = np.mean(np.abs(y1-y2)/np.abs(y1))*100 - #print('Mean rel error {:7.2f} %'.format( meanrelerr)) - return meanrelerr - - -# --------------------------------------------------------------------------------} -# --- PDF -# --------------------------------------------------------------------------------{ -def pdf_histogram(y,nBins=50, norm=True, count=False): - yh, xh = np.histogram(y[~np.isnan(y)], bins=nBins) - dx = xh[1] - xh[0] - xh = xh[:-1] + dx/2 - if count: - yh = yh / (len(n)*dx) # TODO DEBUG /VERIFY THIS - else: - yh = yh / (nBins*dx) - if norm: - yh=yh/np.trapz(yh,xh) - return xh,yh - -def pdf_gaussian_kde(data, bw='scott', nOut=100, cut=3, clip=(-np.inf,np.inf)): - """ - Returns a smooth probability density function (univariate kernel density estimate - kde) - Inspired from `_univariate_kdeplot` from `seaborn.distributions` - - INPUTS: - bw: float defining bandwidth or method (string) to find it (more or less sigma) - cut: number of bandwidth kept for x axis (e.g. 3 sigmas) - clip: (xmin, xmax) values - OUTPUTS: - x, y: where y(x) = pdf(data) - """ - from scipy import stats - from six import string_types - - data = np.asarray(data) - data = data[~np.isnan(data)] - # Gaussian kde - kde = stats.gaussian_kde(data, bw_method = bw) - # Finding a relevant support (i.e. x values) - if isinstance(bw, string_types): - bw_ = "scotts" if bw == "scott" else bw - bw = getattr(kde, "%s_factor" % bw_)() * np.std(data) - x_min = max(data.min() - bw * cut, clip[0]) - x_max = min(data.max() + bw * cut, clip[1]) - x = np.linspace(x_min, x_max, nOut) - # Computing kde on support - y = kde(x) - return x, y - - -def pdf_sklearn(y): - #from sklearn.neighbors import KernelDensity - #kde = KernelDensity(kernel='gaussian', bandwidth=0.75).fit(y) #you can supply a bandwidth - #x=np.linspace(0,5,100)[:, np.newaxis] - #log_density_values=kde.score_samples(x) - #density=np.exp(log_density) - pass - -def pdf_sns(y,nBins=50): - import seaborn.apionly as sns - hh=sns.distplot(y,hist=True,norm_hist=False).get_lines()[0].get_data() - xh=hh[0] - yh=hh[1] - return xh,yh - - - -# --------------------------------------------------------------------------------} -# --- Binning -# --------------------------------------------------------------------------------{ -def bin_DF(df, xbins, colBin): - """ - Perform bin averaging of a dataframe - INPUTS: - - df : pandas dataframe - - xBins: end points delimiting the bins, array of ascending x values) - - colBin: column name (string) of the dataframe, used for binning - OUTPUTS: - binned dataframe, with additional columns 'Counts' for the number - - """ - if colBin not in df.columns.values: - raise Exception('The column `{}` does not appear to be in the dataframe'.format(colBin)) - xmid = (xbins[:-1]+xbins[1:])/2 - df['Bin'] = pd.cut(df[colBin], bins=xbins, labels=xmid ) # Adding a column that has bin attribute - df2 = df.groupby('Bin').mean() # Average by bin - # also counting - df['Counts'] = 1 - dfCount=df[['Counts','Bin']].groupby('Bin').sum() - df2['Counts'] = dfCount['Counts'] - # Just in case some bins are missing (will be nan) - df2 = df2.reindex(xmid) - return df2 - -def azimuthal_average_DF(df, psiBin=None, colPsi='Azimuth_[deg]', tStart=None, colTime='Time_[s]'): - """ - Average a dataframe based on azimuthal value - Returns a dataframe with same amount of columns as input, and azimuthal values as index - """ - if psiBin is None: - psiBin = np.arange(0,360+1,10) - - if tStart is not None: - if colTime not in df.columns.values: - raise Exception('The column `{}` does not appear to be in the dataframe'.format(colTime)) - df=df[ df[colTime]>tStart].copy() - - dfPsi= bin_DF(df, psiBin, colPsi) - if np.any(dfPsi['Counts']<1): - print('[WARN] some bins have no data! Increase the bin size.') - - return dfPsi - - - - +""" +Set of tools for statistics + - measures (R^2, RMSE) + - pdf distributions + - Binning + +""" +import numpy as np +import pandas as pd + +# --------------------------------------------------------------------------------} +# --- Stats measures +# --------------------------------------------------------------------------------{ +def rsquare(y,f, c = True): + """ Compute coefficient of determination of data fit model and RMSE + [r2 rmse] = rsquare(y,f) + [r2 rmse] = rsquare(y,f,c) + RSQUARE computes the coefficient of determination (R-square) value from + actual data Y and model data F. The code uses a general version of + R-square, based on comparing the variability of the estimation errors + with the variability of the original values. RSQUARE also outputs the + root mean squared error (RMSE) for the user's convenience. + Note: RSQUARE ignores comparisons involving NaN values. + INPUTS + Y : Actual data + F : Model fit + + # OPTION + C : Constant term in model + R-square may be a questionable measure of fit when no + constant term is included in the model. + [DEFAULT] TRUE : Use traditional R-square computation + FALSE : Uses alternate R-square computation for model + without constant term [R2 = 1 - NORM(Y-F)/NORM(Y)] + # OUTPUT + R2 : Coefficient of determination + RMSE : Root mean squared error """ + # Compare inputs + if not np.all(y.shape == f.shape) : + raise Exception('Y and F must be the same size') + # Check for NaN + tmp = np.logical_not(np.logical_or(np.isnan(y),np.isnan(f))) + y = y[tmp] + f = f[tmp] + if c: + r2 = max(0,1-np.sum((y-f)**2)/np.sum((y-np.mean(y))** 2)) + else: + r2 = 1 - np.sum((y - f) ** 2) / np.sum((y) ** 2) + if r2 < 0: + import warnings + warnings.warn('Consider adding a constant term to your model') + r2 = 0 + rmse = np.sqrt(np.mean((y - f) ** 2)) + return r2,rmse + +def mean_rel_err(t1=None, y1=None, t2=None, y2=None, method='mean', verbose=False): + """ + return mean relative error in % + + Methods: + 'mean' : 100 * |y1-y2|/mean(y1) + 'meanabs': 100 * |y1-y2|/mean(|y1|) + 'minmax': y1 and y2 scaled between 0.5 and 1.5 + |y1s-y2s|/|y1| + '0-2': signals are scalled between 0 & 2 + """ + if t1 is None and t2 is None: + pass + else: + if len(y1)!=len(y2): + y2=np.interp(t1,t2,y2) + # Method 1 relative to mean + if method=='mean': + ref_val = np.mean(y1) + meanrelerr = np.mean(np.abs(y1-y2)/ref_val)*100 + elif method=='meanabs': + ref_val = np.mean(np.abs(y1)) + meanrelerr = np.mean(np.abs(y1-y2)/ref_val)*100 + elif method=='minmax': + # Method 2 scaling signals + Min=min(np.min(y1), np.min(y2)) + Max=max(np.max(y1), np.max(y2)) + y1=(y1-Min)/(Max-Min)+0.5 + y2=(y2-Min)/(Max-Min)+0.5 + meanrelerr = np.mean(np.abs(y1-y2)/np.abs(y1))*100 + elif method=='1-2': + # transform values from 1 to 2 + Min=min(np.min(y1), np.min(y2)) + Max=max(np.max(y1), np.max(y2)) + y1 = (y1-Min)/(Max-Min)+1 + y2 = (y2-Min)/(Max-Min)+1 + meanrelerr = np.mean(np.abs(y1-y2)/np.abs(y1))*100 + else: + raise Exception('Unknown method',method) + + if verbose: + print('Mean rel error {:7.2f} %'.format( meanrelerr)) + return meanrelerr + + +# --------------------------------------------------------------------------------} +# --- PDF +# --------------------------------------------------------------------------------{ +def pdf_histogram(y,nBins=50, norm=True, count=False): + yh, xh = np.histogram(y[~np.isnan(y)], bins=nBins) + dx = xh[1] - xh[0] + xh = xh[:-1] + dx/2 + if count: + yh = yh / (len(n)*dx) # TODO DEBUG /VERIFY THIS + else: + yh = yh / (nBins*dx) + if norm: + yh=yh/np.trapz(yh,xh) + return xh,yh + +def pdf_gaussian_kde(data, bw='scott', nOut=100, cut=3, clip=(-np.inf,np.inf)): + """ + Returns a smooth probability density function (univariate kernel density estimate - kde) + Inspired from `_univariate_kdeplot` from `seaborn.distributions` + + INPUTS: + bw: float defining bandwidth or method (string) to find it (more or less sigma) + cut: number of bandwidth kept for x axis (e.g. 3 sigmas) + clip: (xmin, xmax) values + OUTPUTS: + x, y: where y(x) = pdf(data) + """ + from scipy import stats + from six import string_types + + data = np.asarray(data) + data = data[~np.isnan(data)] + # Gaussian kde + kde = stats.gaussian_kde(data, bw_method = bw) + # Finding a relevant support (i.e. x values) + if isinstance(bw, string_types): + bw_ = "scotts" if bw == "scott" else bw + bw = getattr(kde, "%s_factor" % bw_)() * np.std(data) + x_min = max(data.min() - bw * cut, clip[0]) + x_max = min(data.max() + bw * cut, clip[1]) + x = np.linspace(x_min, x_max, nOut) + # Computing kde on support + y = kde(x) + return x, y + + +def pdf_sklearn(y): + #from sklearn.neighbors import KernelDensity + #kde = KernelDensity(kernel='gaussian', bandwidth=0.75).fit(y) #you can supply a bandwidth + #x=np.linspace(0,5,100)[:, np.newaxis] + #log_density_values=kde.score_samples(x) + #density=np.exp(log_density) + pass + +def pdf_sns(y,nBins=50): + import seaborn.apionly as sns + hh=sns.distplot(y,hist=True,norm_hist=False).get_lines()[0].get_data() + xh=hh[0] + yh=hh[1] + return xh,yh + + + +# --------------------------------------------------------------------------------} +# --- Binning +# --------------------------------------------------------------------------------{ +def bin_DF(df, xbins, colBin, stats='mean'): + """ + Perform bin averaging of a dataframe + INPUTS: + - df : pandas dataframe + - xBins: end points delimiting the bins, array of ascending x values) + - colBin: column name (string) of the dataframe, used for binning + OUTPUTS: + binned dataframe, with additional columns 'Counts' for the number + + """ + if colBin not in df.columns.values: + raise Exception('The column `{}` does not appear to be in the dataframe'.format(colBin)) + xmid = (xbins[:-1]+xbins[1:])/2 + df['Bin'] = pd.cut(df[colBin], bins=xbins, labels=xmid ) # Adding a column that has bin attribute + if stats=='mean': + df2 = df.groupby('Bin').mean() # Average by bin + elif stats=='std': + df2 = df.groupby('Bin').std() # std by bin + # also counting + df['Counts'] = 1 + dfCount=df[['Counts','Bin']].groupby('Bin').sum() + df2['Counts'] = dfCount['Counts'] + # Just in case some bins are missing (will be nan) + df2 = df2.reindex(xmid) + return df2 + +def bin_signal(x, y, xbins=None, stats='mean', nBins=None): + """ + Perform bin averaging of a dataframe + INPUTS: + - df : pandas dataframe + - xBins: end points delimiting the bins, array of ascending x values) + - colBin: column name (string) of the dataframe, used for binning + OUTPUTS: + binned dataframe, with additional columns 'Counts' for the number + + """ + if xbins is None: + xmin, xmax = np.min(x), np.max(x) + dx = (xmax-xmin)/nBins + xbins=np.arange(xmin, xmax+dx/2, dx) + df = pd.DataFrame(data=np.column_stack((x,y)), columns=['x','y']) + df2 = bin_DF(df, xbins, colBin='x', stats=stats) + return df2['x'].values, df2['y'].values + + + +def azimuthal_average_DF(df, psiBin=np.arange(0,360+1,10), colPsi='Azimuth_[deg]', tStart=None, colTime='Time_[s]'): + """ + Average a dataframe based on azimuthal value + Returns a dataframe with same amount of columns as input, and azimuthal values as index + """ + if tStart is not None: + if colTime not in df.columns.values: + raise Exception('The column `{}` does not appear to be in the dataframe'.format(colTime)) + df=df[ df[colTime]>tStart].copy() + + dfPsi= bin_DF(df, psiBin, colPsi, stats='mean') + if np.any(dfPsi['Counts']<1): + print('[WARN] some bins have no data! Increase the bin size.') + + return dfPsi + + +def azimuthal_std_DF(df, psiBin=np.arange(0,360+1,10), colPsi='Azimuth_[deg]', tStart=None, colTime='Time_[s]'): + """ + Average a dataframe based on azimuthal value + Returns a dataframe with same amount of columns as input, and azimuthal values as index + """ + if tStart is not None: + if colTime not in df.columns.values: + raise Exception('The column `{}` does not appear to be in the dataframe'.format(colTime)) + df=df[ df[colTime]>tStart].copy() + + dfPsi= bin_DF(df, psiBin, colPsi, stats='std') + if np.any(dfPsi['Counts']<1): + print('[WARN] some bins have no data! Increase the bin size.') + + return dfPsi + + + + diff --git a/ressources/pyDatView.ico b/ressources/pyDatView.ico new file mode 100644 index 0000000..692e3f9 Binary files /dev/null and b/ressources/pyDatView.ico differ diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 29712d6..0b11459 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -1,46 +1,83 @@ -import unittest -import numpy as np -import pandas as pd -from pydatview.Tables import Table -import os - - - -class TestTable(unittest.TestCase): - - @classmethod - def setUpClass(cls): - d ={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0} - cls.df1 = pd.DataFrame(data=d) - d ={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0} - cls.df2 = pd.DataFrame(data=d) - - def test_table_name(self): - print(' ') - print(' ') - t1=Table(data=self.df1) - print(t1) - print(' ') - print(' ') - # Typically pyDatView adds tables like this: - # - # self.tabList.load_tables_from_files(filenames=filenames, fileformat=fileformat, bAdd=bAdd) - # - # if len(dfs)>0: - # tabs=[Table(df=dfs, name='default', filename=filename, fileformat=F.formatName())] - # else: - # for k in list(dfs.keys()): - # if len(dfs[k])>0: - # tabs.append(Table(df=dfs[k], name=k, filename=filename, fileformat=F.formatName())) - # OR - # if bAdd: - # self.tabList.append(Table(df=df, name=name)) - # else: - # self.tabList = TableList( [Table(df=df, name=name)] ) - # - # Tools add dfs like this to the GUI: - # self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) - # - -if __name__ == '__main__': - unittest.main() +import unittest +import numpy as np +import pandas as pd +from pydatview.Tables import Table, TableList +import os + + + +class TestTable(unittest.TestCase): + + @classmethod + def setUpClass(cls): + d ={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0} + cls.df1 = pd.DataFrame(data=d) + d ={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0} + cls.df2 = pd.DataFrame(data=d) + + cls.scriptdir = os.path.dirname(__file__) + + def test_table_name(self): + t1=Table(data=self.df1) + self.assertEqual(t1.raw_name, 'default') + # Typically pyDatView adds tables like this: + # + # self.tabList.load_tables_from_files(filenames=filenames, fileformat=fileformat, bAdd=bAdd) + # + # if len(dfs)>0: + # tabs=[Table(df=dfs, name='default', filename=filename, fileformat=F.formatName())] + # else: + # for k in list(dfs.keys()): + # if len(dfs[k])>0: + # tabs.append(Table(df=dfs[k], name=k, filename=filename, fileformat=F.formatName())) + # OR + # if bAdd: + # self.tabList.append(Table(df=df, name=name)) + # else: + # self.tabList = TableList( [Table(df=df, name=name)] ) + # + # Tools add dfs like this to the GUI: + # self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) + # + + + def test_load_files_misc_formats(self): + tablist = TableList() + files =[ + os.path.join(self.scriptdir,'../weio/weio/tests/example_files/CSVComma.csv'), + os.path.join(self.scriptdir,'../weio/weio/tests/example_files/HAWCStab2.pwr') + ] + # --- First read without fileformats + tablist.load_tables_from_files(filenames=files, fileformats=None, bAdd=False) + #print(tablist.fileformats) + + # --- Test iteration on tablist in passing.. + ffname1=[tab.fileformat.name for tab in tablist] + + # --- Then read with prescribed fileformats + fileformats1 = tablist.fileformats + tablist.load_tables_from_files(filenames=files, fileformats=fileformats1, bAdd=False) + ffname2 = [ff.name for ff in tablist.fileformats] + + self.assertEqual(ffname1, ffname2) + + + def test_change_units(self): + data = np.ones((1,3)) + data[:,0] *= 2*np.pi/60 # rad/s + data[:,1] *= 2000 # N + data[:,2] *= 10*np.pi/180 # rad + df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) + tab=Table(data=df) + tab.changeUnits() + np.testing.assert_almost_equal(tab.data.values[:,0],[1]) + np.testing.assert_almost_equal(tab.data.values[:,1],[2]) + np.testing.assert_almost_equal(tab.data.values[:,2],[10]) + self.assertEqual(tab.columns, ['om [rpm]', 'F [kN]', 'angle [deg]']) + + +if __name__ == '__main__': +# TestTable.setUpClass() +# tt= TestTable() +# tt.test_load_files_misc_formats() + unittest.main() diff --git a/tests/test_common.py b/tests/test_common.py index 3f6421c..1e5ab9b 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,121 +1,128 @@ -# - *- coding: utf- 8 - *- -from __future__ import unicode_literals,print_function -import unittest -import numpy as np -import pandas as pd -from pydatview.common import unit,no_unit,ellude_common,getDt, find_leftstop -from pydatview.common import has_chinese_char -from pydatview.common import filter_list -from pydatview.common import rectangleOverlap -import datetime - -class TestCommon(unittest.TestCase): - def assertEqual(self, first, second, msg=None): - #print('>',first,'<',' >',second,'<') - super(TestCommon, self).assertEqual(first, second, msg) - - def test_unit(self): - self.assertEqual(unit ('speed [m/s]'),'m/s' ) - self.assertEqual(unit ('speed [m/s' ),'m/s' ) # ... - self.assertEqual(no_unit('speed [m/s]'),'speed') - - def test_date(self): - def test_dt(datestr,dt_ref): - def myassert(x): - if np.isnan(dt_ref): - self.assertTrue(np.isnan(getDt(x))) - else: - self.assertEqual(getDt(x),dt_ref) - # Type: Numpy array - Elements: datetime64 - if isinstance(datestr[0],int): - x=np.array(datestr, dtype='datetime64[s]') - myassert(x) - - x=np.array(datestr) - myassert(x) - elif isinstance(datestr[0],float): - x=np.array(datestr) - myassert(x) - else: - x=np.array(datestr, dtype='datetime64') - myassert(x) - # Type: Pandas DatetimeIndex - Elements: TimeSamp - df = pd.DataFrame(data=datestr) - x = pd.to_datetime(df.iloc[:,0].values) - myassert(x) - # Type: Numpy array - Elements: datetime.datetime - df = pd.DataFrame(data=datestr) - x = pd.to_datetime(df.iloc[:,0].values).to_pydatetime() - myassert(x) - - test_dt(['2008-01-01','2009-01-01'],24*366*3600); # year - test_dt(['2008-01-01','2008-02-01'],24*3600*31); #month - test_dt(['2000-10-15 01:00:00', '2000-10-15 02:00:00'],3600); # hour - test_dt(['2000-10-15 00:00:05.000001', '2000-10-15 00:00:05.000002'],0.000001);#mu s - test_dt([np.datetime64('NaT'),'2000-10-15 00:00:05.000001'],np.nan); - test_dt([np.datetime64('NaT'),'2000-10-15 00:00:05.000001', '2000-10-15 00:00:05.000002'],0.000001) - test_dt([0],np.nan) - test_dt([0.0],np.nan) -# test_dt([0,1],1) # TODO -# test_dt([0.0,1.0],1.0) # TODO - self.assertEqual(getDt([0.0,0.1]),0.1) - self.assertEqual(getDt(np.array([0.0,0.1])),0.1) - self.assertEqual(getDt([0,1]),1) - self.assertEqual(getDt(np.array([0,1])),1) - - - def test_leftstop(self): - self.assertEqual(find_leftstop('A' ),'A' ) - self.assertEqual(find_leftstop('_' ),'' ) - self.assertEqual(find_leftstop('A_' ),'A' ) - self.assertEqual(find_leftstop('_B' ),'' ) - self.assertEqual(find_leftstop('ABC' ),'ABC') - self.assertEqual(find_leftstop('AB_D'),'AB' ) - self.assertEqual(find_leftstop('AB.D'),'AB' ) - - - def test_ellude(self): - print('') - print('') - self.assertListEqual(ellude_common(['>AA' ,'>AB'] ),['AA' ,'AB'] ) - self.assertListEqual(ellude_common(['AAA' ,'AAA_raw']),['AAA' ,'AAA_raw']) - self.assertListEqual(ellude_common(['A_.txt','A.txt'] ),['A_' ,'A'] ) - self.assertListEqual(ellude_common(['A_' ,'A'] ),['A_' ,'A'] ) - self.assertListEqual(ellude_common(['ABCDA_','ABCDAA'] ),['ABCDA_','ABCDAA'] ) - S=['C:|A_BD', 'C:|A_BD_bld|DC', 'C:|A_BD_bld|BP'] - self.assertListEqual(ellude_common(S),['BD','BD_bld|DC','BD_bld|BP'] ) - self.assertListEqual(ellude_common(['C|FO' , 'C|FO_HD']) , ['FO' , 'FO_HD'] ) - self.assertListEqual(ellude_common(['CT_0.11' , 'CT_0.22']) , ['11' , '22'] ) # Unfortunate - self.assertListEqual(ellude_common(['CT_0.1' , 'CT_0.9']) , ['0.1' , '0.9'] ) - self.assertListEqual(ellude_common(['CT=0.1' , 'CT=0.9']) , ['CT=0.1' , 'CT=0.9'] ) - self.assertListEqual(ellude_common(['AAA' , 'ABA'] , minLength=-1) , ['A' , 'B'] ) - #print(ellude_common(['Farm.ifw.T1','Farm.ifw.T2'],minLength=2)) - #print('') - #print('') - - def test_chinese_char(self): - self.assertEqual(has_chinese_char('') ,False) - self.assertEqual(has_chinese_char('aaaa'),False) - self.assertEqual(has_chinese_char('aa时'),True ) - self.assertEqual(has_chinese_char('a时a'),True ) - - def test_filter(self): - L=['RotTrq_[kNm]','B1RootMy_[kNm]','B2RootMy_[kNm]','Power_[kW]'] - Lf, If = filter_list(L,'Root') - self.assertEqual(If,[1,2]) - Lf, If = filter_list(L,'ro') - self.assertEqual(If,[0,1,2]) - self.assertEqual(Lf[0],'RotTrq_[kNm]') - Lf, If = filter_list(L,'Kro') - self.assertEqual(len(If),0) - self.assertEqual(len(Lf),0) - - def test_rectangleOverlap(self): - self.assertEqual(rectangleOverlap(0,0,1,1,0,0,2,2) ,True) # rect1 contained - self.assertEqual(rectangleOverlap(-2,-2,1,1,0,0,1,1) ,True) # rect2 contained - self.assertEqual(rectangleOverlap(-2,-2, 1, 1,0,0,2,2),True) # overlap corner2 in - self.assertEqual(rectangleOverlap(-2,-2, 1, 1,-3,0,2,2),True) # overlap - self.assertEqual(rectangleOverlap(-2,-2,-1,-1,0,0,1,1),False) - -if __name__ == '__main__': - unittest.main() +# - *- coding: utf- 8 - *- +from __future__ import unicode_literals,print_function +import unittest +import numpy as np +import pandas as pd +from pydatview.common import unit, no_unit, splitunit +from pydatview.common import ellude_common, getDt, find_leftstop +from pydatview.common import has_chinese_char +from pydatview.common import filter_list +from pydatview.common import rectangleOverlap +import datetime + +class TestCommon(unittest.TestCase): + def assertEqual(self, first, second, msg=None): + #print('>',first,'<',' >',second,'<') + super(TestCommon, self).assertEqual(first, second, msg) + + def test_unit(self): + self.assertEqual(unit ('speed [m/s]'),'m/s' ) + self.assertEqual(unit ('speed [m/s' ),'m/s' ) # ... + self.assertEqual(no_unit('speed [m/s]'),'speed') + + def test_splitunit(self): + self.assertEqual(splitunit ('speed [m/s]'),('speed ','m/s' )) + self.assertEqual(splitunit ('speed [m/s' ),('speed ','m/s' )) + self.assertEqual(splitunit ('speed_[m/s]'),('speed_','m/s' )) + self.assertEqual(splitunit ('speed'),('speed','' )) + + def test_date(self): + def test_dt(datestr,dt_ref): + def myassert(x): + if np.isnan(dt_ref): + self.assertTrue(np.isnan(getDt(x))) + else: + self.assertEqual(getDt(x),dt_ref) + # Type: Numpy array - Elements: datetime64 + if isinstance(datestr[0],int): + x=np.array(datestr, dtype='datetime64[s]') + myassert(x) + + x=np.array(datestr) + myassert(x) + elif isinstance(datestr[0],float): + x=np.array(datestr) + myassert(x) + else: + x=np.array(datestr, dtype='datetime64') + myassert(x) + # Type: Pandas DatetimeIndex - Elements: TimeSamp + df = pd.DataFrame(data=datestr) + x = pd.to_datetime(df.iloc[:,0].values) + myassert(x) + # Type: Numpy array - Elements: datetime.datetime + df = pd.DataFrame(data=datestr) + x = pd.to_datetime(df.iloc[:,0].values).to_pydatetime() + myassert(x) + + test_dt(['2008-01-01','2009-01-01'],24*366*3600); # year + test_dt(['2008-01-01','2008-02-01'],24*3600*31); #month + test_dt(['2000-10-15 01:00:00', '2000-10-15 02:00:00'],3600); # hour + test_dt(['2000-10-15 00:00:05.000001', '2000-10-15 00:00:05.000002'],0.000001);#mu s + test_dt([np.datetime64('NaT'),'2000-10-15 00:00:05.000001'],np.nan); + test_dt([np.datetime64('NaT'),'2000-10-15 00:00:05.000001', '2000-10-15 00:00:05.000002'],0.000001) + test_dt([0],np.nan) + test_dt([0.0],np.nan) +# test_dt([0,1],1) # TODO +# test_dt([0.0,1.0],1.0) # TODO + self.assertEqual(getDt([0.0,0.1]),0.1) + self.assertEqual(getDt(np.array([0.0,0.1])),0.1) + self.assertEqual(getDt([0,1]),1) + self.assertEqual(getDt(np.array([0,1])),1) + + + def test_leftstop(self): + self.assertEqual(find_leftstop('A' ),'A' ) + self.assertEqual(find_leftstop('_' ),'' ) + self.assertEqual(find_leftstop('A_' ),'A' ) + self.assertEqual(find_leftstop('_B' ),'' ) + self.assertEqual(find_leftstop('ABC' ),'ABC') + self.assertEqual(find_leftstop('AB_D'),'AB' ) + self.assertEqual(find_leftstop('AB.D'),'AB' ) + + + def test_ellude(self): + print('') + print('') + self.assertListEqual(ellude_common(['>AA' ,'>AB'] ),['AA' ,'AB'] ) + self.assertListEqual(ellude_common(['AAA' ,'AAA_raw']),['AAA' ,'AAA_raw']) + self.assertListEqual(ellude_common(['A_.txt','A.txt'] ),['A_' ,'A'] ) + self.assertListEqual(ellude_common(['A_' ,'A'] ),['A_' ,'A'] ) + self.assertListEqual(ellude_common(['ABCDA_','ABCDAA'] ),['ABCDA_','ABCDAA'] ) + S=['C:|A_BD', 'C:|A_BD_bld|DC', 'C:|A_BD_bld|BP'] + self.assertListEqual(ellude_common(S),['BD','BD_bld|DC','BD_bld|BP'] ) + self.assertListEqual(ellude_common(['C|FO' , 'C|FO_HD']) , ['FO' , 'FO_HD'] ) + self.assertListEqual(ellude_common(['CT_0.11' , 'CT_0.22']) , ['11' , '22'] ) # Unfortunate + self.assertListEqual(ellude_common(['CT_0.1' , 'CT_0.9']) , ['0.1' , '0.9'] ) + self.assertListEqual(ellude_common(['CT=0.1' , 'CT=0.9']) , ['CT=0.1' , 'CT=0.9'] ) + self.assertListEqual(ellude_common(['AAA' , 'ABA'] , minLength=-1) , ['A' , 'B'] ) + #print(ellude_common(['Farm.ifw.T1','Farm.ifw.T2'],minLength=2)) + #print('') + #print('') + + def test_chinese_char(self): + self.assertEqual(has_chinese_char('') ,False) + self.assertEqual(has_chinese_char('aaaa'),False) + self.assertEqual(has_chinese_char('aa时'),True ) + self.assertEqual(has_chinese_char('a时a'),True ) + + def test_filter(self): + L=['RotTrq_[kNm]','B1RootMy_[kNm]','B2RootMy_[kNm]','Power_[kW]'] + Lf, If = filter_list(L,'Root') + self.assertEqual(If,[1,2]) + Lf, If = filter_list(L,'ro') + self.assertEqual(If,[0,1,2]) + self.assertEqual(Lf[0],'RotTrq_[kNm]') + Lf, If = filter_list(L,'Kro') + self.assertEqual(len(If),0) + self.assertEqual(len(Lf),0) + + def test_rectangleOverlap(self): + self.assertEqual(rectangleOverlap(0,0,1,1,0,0,2,2) ,True) # rect1 contained + self.assertEqual(rectangleOverlap(-2,-2,1,1,0,0,1,1) ,True) # rect2 contained + self.assertEqual(rectangleOverlap(-2,-2, 1, 1,0,0,2,2),True) # overlap corner2 in + self.assertEqual(rectangleOverlap(-2,-2, 1, 1,-3,0,2,2),True) # overlap + self.assertEqual(rectangleOverlap(-2,-2,-1,-1,0,0,1,1),False) + +if __name__ == '__main__': + unittest.main() diff --git a/weio b/weio index b1addf6..8f5b2b8 160000 --- a/weio +++ b/weio @@ -1 +1 @@ -Subproject commit b1addf69acf817ec42de62711dd8dff94c8e9f4b +Subproject commit 8f5b2b8a387789910f6fc59b0f9466c3a160b4b6