From 9e49b8a5069ec488026f9479c21e7faae9fbe0e5 Mon Sep 17 00:00:00 2001 From: SimonHH <38310787+SimonHH@users.noreply.github.com> Date: Wed, 22 Jun 2022 19:24:35 +0200 Subject: [PATCH 001/178] x-axis beyond 300 sensors add [...] as 301th sensor. When it is selected show all sensors. --- pydatview/GUISelectionPanel.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index a623bee..82bd112 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -755,7 +755,14 @@ def setGUIColumns(self, xSel=-1, ySel=[]): # NOTE: limiting to 300 for now.. I'm not sure anywant would want to scroll more than that # Consider adding a "more button" # see e.g. https://comp.soft-sys.wxwindows.narkive.com/gDfA1Ds5/long-load-time-in-wxpython - self.comboX.Set(columnsX[:300]) # non filtered + if self.comboX.GetCurrentSelection()==300: + self.comboX.Set(columnsX) + else: + if len(columnsX)>300: + columnsX_show=np.append(columnsX[:300],'[...]') + else: + columnsX_show=columnsX + self.comboX.Set(columnsX_show) # non filtered # Set selection for y, if any, and considering filtering for iFull in ySel: @@ -812,6 +819,9 @@ def getColumnSelection(self): SY = [self.lbColumns.GetString(i) for i in IY] iXFull = iX # NOTE: x is always in full IYFull = [self.Filt2Full[iY] for iY in IY] + + if self.comboX.GetCurrentSelection()==300: + self.setGUIColumns(xSel=iXFull, ySel=IYFull) return iXFull,IYFull,sX,SY def onClearFilter(self, event=None): From 4034b1f3cfce7541abfc4103ee9e10f026365f5f Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 22 Jun 2022 13:29:44 -0600 Subject: [PATCH 002/178] Update of GH tests --- .github/workflows/tests.yml | 54 ++++++++++++++++++++----------------- Makefile | 2 +- _tools/NewRelease.md | 38 ++++++++++++++++++++++++-- _tools/setVersion.sh | 9 ++++++- 4 files changed, 75 insertions(+), 28 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 09d5540..eef91c1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,6 +21,32 @@ jobs: with: submodules: recursive + # --- Versioning + - name: Versioning + id: versioning + run: | + git fetch --unshallow + export VERSION_TAG="" + export CURRENT_DEV_TAG="v0.3-dev" + export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` + echo "GIT DESCRIBE: `git describe`" + echo "GITHUB_REF: $GITHUB_REF" + echo "VERSION_NAME $VERSION_NAME" + # Check if current version corresponds to a tagged commit + if [[ $GITHUB_REF == *"tags"* ]]; then export VERSION_TAG=${GITHUB_REF/refs\/tags\//} ; fi + echo "VERSION_TAG $VERSION_TAG" + if [[ "$VERSION_TAG" == "" ]]; then export VERSION_TAG=$CURRENT_DEV_TAG ; fi + if [[ "$VERSION_TAG" != "$CURRENT_DEV_TAG" ]]; then export VERSION_NAME=$VERSION_TAG ; fi + echo "VERSION_NAME: $VERSION_NAME" + echo "VERSION_TAG : $VERSION_TAG" + if [[ "$VERSION_TAG" != "$CURRENT_DEV_TAG" ]]; then export FULL_VERSION_NAME="version $VERSION_NAME" ; fi + if [[ "$VERSION_TAG" == "$CURRENT_DEV_TAG" ]]; then export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" ; fi + echo "VERSION_NAME: $VERSION_NAME" + echo "FULL_VERSION_NAME: $FULL_VERSION_NAME" + echo "::set-output name=FULL_VERSION_NAME::$FULL_VERSION_NAME" + echo "::set-output name=VERSION_NAME::$VERSION_NAME" + echo "::set-output name=VERSION_TAG::$VERSION_TAG" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -64,27 +90,6 @@ jobs: if: ${{ steps.check_deploy.outputs.GO == '1'}} run: sudo apt-get install nsis - - name: Versioning - if: ${{ steps.check_deploy.outputs.GO == '1'}} - id: versioning - run: | - git fetch --unshallow - export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` - export FULL_VERSION_NAME="version $VERSION_NAME" - echo "GITHUB_REF: $GITHUB_REF" - echo "VERSION_NAME $VERSION_NAME" - echo "FULL_VERSION_NAME $FULL_VERSION_NAME" - if [[ $GITHUB_REF == *"tags" ]]; then export VERSION_TAG=${GITHUB_REF/refs\/tags\//} ; fi - if [[ $GITHUB_REF != *"tags" ]]; then export VERSION_TAG="" ; fi - echo "VERSION_TAG $VERSION_TAG" - if [[ "$VERSION_TAG" == "" ]]; then export VERSION_TAG="vdev" ; fi - if [[ "$VERSION_TAG" == "vdev" ]]; then export VERSION_NAME=$VERSION_NAME"-dev" ; fi - if [[ "$VERSION_TAG" == "vdev" ]]; then export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" ; fi - echo "VERSION_NAME: $VERSION_NAME" - echo "FULL_VERSION_NAME: $FULL_VERSION_NAME" - echo "::set-output name=FULL_VERSION_NAME::$FULL_VERSION_NAME" - echo "::set-output name=VERSION_NAME::$VERSION_NAME" - echo "::set-output name=VERSION_TAG::$VERSION_TAG" - name: Before deploy if: ${{ steps.check_deploy.outputs.GO == '1'}} @@ -101,6 +106,7 @@ jobs: pip install distlib git clone https://github.com/takluyver/pynsist mv pynsist/nsist nsist + sh _tools/setVersion.sh $VERSION_NAME make installer mv build/nsis/pyDatView_setup.exe "pyDatView_"$VERSION_NAME"_setup.exe" mv _tools/pyDatView.cmd build/nsis/ @@ -109,8 +115,8 @@ jobs: mv build/nsis build/pyDatView_$VERSION_NAME cd build && zip -r "../pyDatView_"$VERSION_NAME"_portable.zip" pyDatView_$VERSION_NAME cd .. - cp "pyDatView_"$VERSION_NAME"_setup.exe" "pyDatView_LatestVersion_setup.exe" - cp "pyDatView_"$VERSION_NAME"_portable.zip" "pyDatView_LatestVersion_portable.zip" + if [[ "$VERSION_TAG" == *"-dev"* ]]; then cp "pyDatView_"$VERSION_NAME"_setup.exe" "pyDatView_LatestVersion_setup.exe" ;fi + if [[ "$VERSION_TAG" == *"-dev"* ]]; then cp "pyDatView_"$VERSION_NAME"_portable.zip" "pyDatView_LatestVersion_portable.zip" ;fi ls - name: Deploy @@ -124,7 +130,7 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: pyDatView_*.* release_name: ${{steps.versioning.outputs.FULL_VERSION_NAME}} - tag: vdev + tag: ${{steps.versioning.outputs.VERSION_TAG}} overwrite: true file_glob: true body: | diff --git a/Makefile b/Makefile index b3893d6..f79965f 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ else @sh _tools/setVersion.sh endif -installer: version +installer: python -m nsist installer.cfg diff --git a/_tools/NewRelease.md b/_tools/NewRelease.md index 4f14fc9..13a10e5 100644 --- a/_tools/NewRelease.md +++ b/_tools/NewRelease.md @@ -1,9 +1,43 @@ +# Creating a new release Steps: - Change PROG\_VERSION in pydateview/main.py - Change version in installler.cfg +- Change CURRENT\_DEV\_TAG in .github/workflows/tests.yml (not pretty...) - Commit changes and push to pull request - Merge pull request from old to new version -- Tag new version v0.x `git tag -a v0.x` -- Push tags: `git push --tags` + +- Tag new version v0.x: + git tag -a v0.x +- Push tags: + git push --tags +- Github actions looks at GITHUB_REF to see if it's a tag. If it's a tag, it should push to it. + Otherwise, it pushes to the tag defined by CURRENT_DEV_TAG + +- Make sure things worked + +- checkout dev and create a new dev tag, e.g. v0.3-dev + + +- Rename vdev to v0.3-dev + git tag v0.3-dev vdev + git tag -d vdev + git push origin v0.3-dev :vdev + + + + + +#Misc notes: + + +## Delete a tag locally and remote +git tag -d vtag +git push --delete origin vtag + +## Rename a tag locally and remote +git tag new old +git tag -d old +git push origin new :old + diff --git a/_tools/setVersion.sh b/_tools/setVersion.sh index da2abc4..4c9b9b8 100644 --- a/_tools/setVersion.sh +++ b/_tools/setVersion.sh @@ -1,4 +1,11 @@ #!/bin/bash -export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` + +if [ $# -eq 0 ] + then + # "No arguments supplied" + export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` +else + export VERSION_NAME=$1 +fi echo "Setting version to $VERSION_NAME" sed -i "s/PROG_VERSION=.*$/PROG_VERSION='${VERSION_NAME}'/" pydatview/main.py From e6833d619215f5a26f24a0559b6083cf2c2901b9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 22 Jun 2022 14:06:13 -0600 Subject: [PATCH 003/178] GH: attempt multiline if --- .github/workflows/tests.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eef91c1..35b9c4a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,14 +33,17 @@ jobs: echo "GITHUB_REF: $GITHUB_REF" echo "VERSION_NAME $VERSION_NAME" # Check if current version corresponds to a tagged commit - if [[ $GITHUB_REF == *"tags"* ]]; then export VERSION_TAG=${GITHUB_REF/refs\/tags\//} ; fi + if [[ $GITHUB_REF == *"tags"* ]]; then + echo ">>> This is a tagged version" + export VERSION_TAG=${GITHUB_REF/refs\/tags\//} + export VERSION_NAME=$VERSION_TAG + export FULL_VERSION_NAME="version $VERSION_NAME" + else + echo ">>> This is not a tagged version" + export VERSION_TAG=$CURRENT_DEV_TAG ; + export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" + fi echo "VERSION_TAG $VERSION_TAG" - if [[ "$VERSION_TAG" == "" ]]; then export VERSION_TAG=$CURRENT_DEV_TAG ; fi - if [[ "$VERSION_TAG" != "$CURRENT_DEV_TAG" ]]; then export VERSION_NAME=$VERSION_TAG ; fi - echo "VERSION_NAME: $VERSION_NAME" - echo "VERSION_TAG : $VERSION_TAG" - if [[ "$VERSION_TAG" != "$CURRENT_DEV_TAG" ]]; then export FULL_VERSION_NAME="version $VERSION_NAME" ; fi - if [[ "$VERSION_TAG" == "$CURRENT_DEV_TAG" ]]; then export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" ; fi echo "VERSION_NAME: $VERSION_NAME" echo "FULL_VERSION_NAME: $FULL_VERSION_NAME" echo "::set-output name=FULL_VERSION_NAME::$FULL_VERSION_NAME" From 3911eadd7412820be4f109849cd62f8fb0001d16 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 22 Jun 2022 14:41:14 -0600 Subject: [PATCH 004/178] GH: only deploy on push --- .github/workflows/tests.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 35b9c4a..b52b160 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,12 +26,9 @@ jobs: id: versioning run: | git fetch --unshallow - export VERSION_TAG="" export CURRENT_DEV_TAG="v0.3-dev" - export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` echo "GIT DESCRIBE: `git describe`" echo "GITHUB_REF: $GITHUB_REF" - echo "VERSION_NAME $VERSION_NAME" # Check if current version corresponds to a tagged commit if [[ $GITHUB_REF == *"tags"* ]]; then echo ">>> This is a tagged version" @@ -41,6 +38,7 @@ jobs: else echo ">>> This is not a tagged version" export VERSION_TAG=$CURRENT_DEV_TAG ; + export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" fi echo "VERSION_TAG $VERSION_TAG" @@ -83,8 +81,12 @@ jobs: echo "GH_EVENT : $GH_EVENT" echo "PY_VERSION : $PY_VERSION" export OK=0 - if [[ $PY_VERSION == "3.6" ]]; then export OK=1 ; fi - if [[ "$OK" == "1" && $GH_EVENT == "push" ]]; then export OK=1 ; fi + # Only deploy for push events + if [[ $PY_VERSION == "3.6" ]]; then + if [[ $GH_EVENT == "push" ]]; then + export OK=1 ; + fi + fi echo "DEPLOY : $OK" echo "::set-output name=GO::$OK" From 6b9a9270a5c78cae3fcaf8604c4c7417ab126114 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 22 Jun 2022 15:23:16 -0600 Subject: [PATCH 005/178] IO: replace weio by pydatview.io (#118) --- .github/workflows/tests.yml | 1 - .gitmodules | 3 - Makefile | 2 +- README.md | 13 +- _tools/travis_requirements.txt | 3 +- example_files/CSVComma.csv | 5 + example_files/FASTIn_arf_coords.txt | 407 +++++++ example_files/HAWCStab2.pwr | 46 + installer.cfg | 7 +- pydatview/Tables.py | 12 +- pydatview/appdata.py | 2 +- pydatview/fast/case_gen.py | 2 +- pydatview/fast/fastfarm.py | 6 +- pydatview/fast/postpro.py | 6 +- pydatview/fast/runner.py | 4 +- pydatview/io/__init__.py | 264 ++++ pydatview/io/bladed_out_file.py | 380 ++++++ pydatview/io/bmodes_out_file.py | 152 +++ pydatview/io/cactus_element_file.py | 107 ++ pydatview/io/cactus_file.py | 386 ++++++ pydatview/io/csv_file.py | 292 +++++ pydatview/io/excel_file.py | 88 ++ pydatview/io/fast_input_deck.py | 442 +++++++ pydatview/io/fast_input_file.py | 1288 ++++++++++++++++++++ pydatview/io/fast_input_file_graph.py | 330 +++++ pydatview/io/fast_linearization_file.py | 348 ++++++ pydatview/io/fast_output_file.py | 498 ++++++++ pydatview/io/fast_summary_file.py | 267 ++++ pydatview/io/fast_wind_file.py | 84 ++ pydatview/io/file.py | 191 +++ pydatview/io/file_formats.py | 29 + pydatview/io/flex_blade_file.py | 142 +++ pydatview/io/flex_doc_file.py | 209 ++++ pydatview/io/flex_out_file.py | 200 +++ pydatview/io/flex_profile_file.py | 147 +++ pydatview/io/flex_wavekin_file.py | 104 ++ pydatview/io/hawc2_ae_file.py | 71 ++ pydatview/io/hawc2_dat_file.py | 156 +++ pydatview/io/hawc2_htc_file.py | 117 ++ pydatview/io/hawc2_pc_file.py | 77 ++ pydatview/io/hawc2_st_file.py | 64 + pydatview/io/hawcstab2_cmb_file.py | 36 + pydatview/io/hawcstab2_ind_file.py | 68 ++ pydatview/io/hawcstab2_pwr_file.py | 56 + pydatview/io/mannbox_file.py | 334 +++++ pydatview/io/mini_yaml.py | 98 ++ pydatview/io/netcdf_file.py | 50 + pydatview/io/parquet_file.py | 45 + pydatview/io/raawmat_file.py | 211 ++++ pydatview/io/rosco_performance_file.py | 239 ++++ pydatview/io/tdms_file.py | 65 + pydatview/io/tecplot_file.py | 222 ++++ pydatview/io/tools/__init__.py | 0 pydatview/io/tools/graph.py | 672 ++++++++++ pydatview/io/turbsim_file.py | 770 ++++++++++++ pydatview/io/turbsim_ts_file.py | 101 ++ pydatview/io/user.py | 8 + pydatview/io/vtk_file.py | 1349 +++++++++++++++++++++ pydatview/io/wetb/__init__.py | 0 pydatview/io/wetb/hawc2/Hawc2io.py | 351 ++++++ pydatview/io/wetb/hawc2/__init__.py | 18 + pydatview/io/wetb/hawc2/ae_file.py | 132 ++ pydatview/io/wetb/hawc2/htc_contents.py | 489 ++++++++ pydatview/io/wetb/hawc2/htc_extensions.py | 152 +++ pydatview/io/wetb/hawc2/htc_file.py | 585 +++++++++ pydatview/io/wetb/hawc2/htc_file_set.py | 55 + pydatview/io/wetb/hawc2/pc_file.py | 170 +++ pydatview/io/wetb/hawc2/st_file.py | 274 +++++ pydatview/main.py | 21 +- requirements.txt | 2 +- setup.py | 11 - tests/prof_all.py | 2 +- tests/test_Tables.py | 4 +- weio | 1 - 74 files changed, 13474 insertions(+), 69 deletions(-) delete mode 100644 .gitmodules create mode 100644 example_files/CSVComma.csv create mode 100644 example_files/FASTIn_arf_coords.txt create mode 100644 example_files/HAWCStab2.pwr create mode 100644 pydatview/io/__init__.py create mode 100644 pydatview/io/bladed_out_file.py create mode 100644 pydatview/io/bmodes_out_file.py create mode 100644 pydatview/io/cactus_element_file.py create mode 100644 pydatview/io/cactus_file.py create mode 100644 pydatview/io/csv_file.py create mode 100644 pydatview/io/excel_file.py create mode 100644 pydatview/io/fast_input_deck.py create mode 100644 pydatview/io/fast_input_file.py create mode 100644 pydatview/io/fast_input_file_graph.py create mode 100644 pydatview/io/fast_linearization_file.py create mode 100644 pydatview/io/fast_output_file.py create mode 100644 pydatview/io/fast_summary_file.py create mode 100644 pydatview/io/fast_wind_file.py create mode 100644 pydatview/io/file.py create mode 100644 pydatview/io/file_formats.py create mode 100644 pydatview/io/flex_blade_file.py create mode 100644 pydatview/io/flex_doc_file.py create mode 100644 pydatview/io/flex_out_file.py create mode 100644 pydatview/io/flex_profile_file.py create mode 100644 pydatview/io/flex_wavekin_file.py create mode 100644 pydatview/io/hawc2_ae_file.py create mode 100644 pydatview/io/hawc2_dat_file.py create mode 100644 pydatview/io/hawc2_htc_file.py create mode 100644 pydatview/io/hawc2_pc_file.py create mode 100644 pydatview/io/hawc2_st_file.py create mode 100644 pydatview/io/hawcstab2_cmb_file.py create mode 100644 pydatview/io/hawcstab2_ind_file.py create mode 100644 pydatview/io/hawcstab2_pwr_file.py create mode 100644 pydatview/io/mannbox_file.py create mode 100644 pydatview/io/mini_yaml.py create mode 100644 pydatview/io/netcdf_file.py create mode 100644 pydatview/io/parquet_file.py create mode 100644 pydatview/io/raawmat_file.py create mode 100644 pydatview/io/rosco_performance_file.py create mode 100644 pydatview/io/tdms_file.py create mode 100644 pydatview/io/tecplot_file.py create mode 100644 pydatview/io/tools/__init__.py create mode 100644 pydatview/io/tools/graph.py create mode 100644 pydatview/io/turbsim_file.py create mode 100644 pydatview/io/turbsim_ts_file.py create mode 100644 pydatview/io/user.py create mode 100644 pydatview/io/vtk_file.py create mode 100644 pydatview/io/wetb/__init__.py create mode 100644 pydatview/io/wetb/hawc2/Hawc2io.py create mode 100644 pydatview/io/wetb/hawc2/__init__.py create mode 100644 pydatview/io/wetb/hawc2/ae_file.py create mode 100644 pydatview/io/wetb/hawc2/htc_contents.py create mode 100644 pydatview/io/wetb/hawc2/htc_extensions.py create mode 100644 pydatview/io/wetb/hawc2/htc_file.py create mode 100644 pydatview/io/wetb/hawc2/htc_file_set.py create mode 100644 pydatview/io/wetb/hawc2/pc_file.py create mode 100644 pydatview/io/wetb/hawc2/st_file.py delete mode 160000 weio diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b52b160..8425cb1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,7 +57,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -r _tools/travis_requirements.txt - pip install -r weio/requirements.txt - name: System info run: | diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index c9f2115..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "weio"] - path = weio - url = https://github.com/ebranlard/weio.git diff --git a/Makefile b/Makefile index f79965f..3804fb4 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ else detected_OS := $(patsubst MINGW%,MSYS,$(detected_OS)) endif -testfile= weio/weio/tests/example_files/FASTIn_arf_coords.txt +testfile= example_files/FASTIn_arf_coords.txt all: ifeq ($(detected_OS),Darwin) # Mac OS X ./pythonmac pyDatView.py $(testfile) diff --git a/README.md b/README.md index 637ab40..22cb984 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ For **Windows** users, an installer executable is available [here](https://githu **Linux** and **MacOS** users can use the command lines below. **Linux** users may need to install the package python-wxgtk\* (e.g. `python-gtk3.0`) from their distribution. **MacOS** users can use a `brew`, `anaconda` or `virtualenv` version of python and pip, but the final version of python that calls the script needs to have access to the screen (see [details for MacOS](#macos-installation)). The main commands for **Linux** and **MacOS** users are: ```bash -git clone --recurse-submodules https://github.com/ebranlard/pyDatView +git clone https://github.com/ebranlard/pyDatView cd pyDatView python -m pip install --user -r requirements.txt make # executes: 'python pyDatView.py' (on linux) or './pythonmac pyDatView.py' (on Mac) @@ -152,7 +152,7 @@ For Windows users, installer executables are available [here](https://github.com The script is compatible python 2.7 and python 3 and relies on the following python packages: `numpy` `matplotlib`, `pandas`, `wxpython`. To download the code and install the dependencies (with pip) run the following: ```bash -git clone --recurse-submodules https://github.com/ebranlard/pyDatView +git clone https://github.com/ebranlard/pyDatView cd pyDatView python -m pip install --user -r requirements.txt ``` @@ -176,7 +176,7 @@ python setup.py install The installation works with python2 and python3, with `brew` (with or without a `virtualenv`) or `anaconda`. First, download the source code: ```bash -git clone --recurse-submodules https://github.com/ebranlard/pyDatView +git clone https://github.com/ebranlard/pyDatView cd pyDatView ``` Before installing the requirements, you need to be aware of the two following issues with MacOS: @@ -232,8 +232,11 @@ python setup.py install ## Adding more file formats -File formats can be added by implementing a subclass of `weio/File.py`, for instance `weio/VTKFile.py`. Existing examples are found in the folder `weio`. -Once implemented the fileformat needs to be registered in `weio/__init__.py` by adding an import line at the beginning of this script and adding a line in the function `fileFormats()` of the form `formats.append(FileFormat(VTKFile))` +File formats can be added by implementing a subclass of `pydatview/io/File.py`, for instance `pydatview/io/VTKFile.py`. Existing examples are found in the folder `pydatview/io`. +Once implemented the fileformat needs to be registered in `pydatview/io/__init__.py` by adding an import line at the beginning of this script and adding a line in the function `fileFormats()` of the form `formats.append(FileFormat(VTKFile))` + +If you believe your fileformat will be beneficial to the wind energy community, we recommend to also add your file format to the [weio](http://github.com/ebranlard/weio/) repository. +Follow the procedure mentioned in the README of the weio repository (in particualr adding unit tests and minimalistic example files). diff --git a/_tools/travis_requirements.txt b/_tools/travis_requirements.txt index 687f9f9..6aba4ee 100644 --- a/_tools/travis_requirements.txt +++ b/_tools/travis_requirements.txt @@ -4,7 +4,8 @@ numpy ; python_version<"3.0" pandas>=1.0.1; python_version>"3.0" pandas; python_version<="3.0" xlrd==1.2.0; python_version<"3.0" -pyarrow +pyarrow # for parquet files +matplotlib future chardet scipy diff --git a/example_files/CSVComma.csv b/example_files/CSVComma.csv new file mode 100644 index 0000000..35ea236 --- /dev/null +++ b/example_files/CSVComma.csv @@ -0,0 +1,5 @@ +ColA,ColB +1,4 +2,9 +3,6 +4,6 diff --git a/example_files/FASTIn_arf_coords.txt b/example_files/FASTIn_arf_coords.txt new file mode 100644 index 0000000..aa31ac2 --- /dev/null +++ b/example_files/FASTIn_arf_coords.txt @@ -0,0 +1,407 @@ + 400 NumCoords ! The number of coordinates in the airfoil shape file (including an extra coordinate for airfoil reference). Set to zero if coordinates not included. +! ......... x-y coordinates are next if NumCoords > 0 ............. +! x-y coordinate of airfoil reference +! x/c y/c +0.25 0 +! coordinates of airfoil shape; data from TU Delft as posted here: https://wind.nrel.gov/forum/wind/viewtopic.php?f=2&t=440 +! NACA 64-618 (interpolated to 399 points) +! x/c y/c +1.000000 0.000000 +0.990000 0.003385 +0.980000 0.006126 +0.975000 0.007447 +0.970000 0.008767 +0.965000 0.010062 +0.960000 0.011357 +0.955000 0.012639 +0.950000 0.013921 +0.945000 0.015200 +0.940000 0.016478 +0.935000 0.017757 +0.930000 0.019036 +0.925000 0.020317 +0.920000 0.021598 +0.915000 0.022881 +0.910000 0.024163 +0.905000 0.025448 +0.900000 0.026733 +0.887500 0.029951 +0.875000 0.033169 +0.862500 0.036386 +0.850000 0.039603 +0.837500 0.042804 +0.825000 0.046004 +0.812500 0.049171 +0.800000 0.052337 +0.787500 0.055452 +0.775000 0.058566 +0.762500 0.061611 +0.750000 0.064656 +0.737500 0.067615 +0.725000 0.070573 +0.712500 0.073429 +0.700000 0.076285 +0.687500 0.079029 +0.675000 0.081773 +0.662500 0.084393 +0.650000 0.087012 +0.637500 0.089490 +0.625000 0.091967 +0.612500 0.094283 +0.600000 0.096599 +0.587500 0.098743 +0.575000 0.100887 +0.562500 0.102843 +0.550000 0.104799 +0.537500 0.106549 +0.525000 0.108299 +0.512500 0.109830 +0.500000 0.111360 +0.487500 0.112649 +0.475000 0.113937 +0.462500 0.114964 +0.450000 0.115990 +0.445000 0.116320 +0.440000 0.116650 +0.435000 0.116931 +0.430000 0.117211 +0.425000 0.117439 +0.420000 0.117667 +0.415000 0.117835 +0.410000 0.118003 +0.405000 0.118104 +0.400000 0.118204 +0.395000 0.118231 +0.390000 0.118258 +0.385000 0.118213 +0.380000 0.118168 +0.375000 0.118057 +0.370000 0.117946 +0.365000 0.117777 +0.360000 0.117607 +0.355000 0.117383 +0.350000 0.117159 +0.345000 0.116881 +0.340000 0.116603 +0.335000 0.116273 +0.330000 0.115942 +0.325000 0.115562 +0.320000 0.115181 +0.315000 0.114750 +0.310000 0.114319 +0.305000 0.113838 +0.300000 0.113356 +0.295000 0.112824 +0.290000 0.112292 +0.285000 0.111710 +0.280000 0.111127 +0.275000 0.110495 +0.270000 0.109863 +0.265000 0.109180 +0.260000 0.108497 +0.255000 0.107762 +0.250000 0.107027 +0.245000 0.106241 +0.240000 0.105454 +0.235000 0.104614 +0.230000 0.103774 +0.225000 0.102880 +0.220000 0.101985 +0.215000 0.101035 +0.210000 0.100084 +0.205000 0.099076 +0.200000 0.098068 +0.195000 0.097001 +0.190000 0.095934 +0.185000 0.094805 +0.180000 0.093676 +0.175000 0.092484 +0.170000 0.091291 +0.165000 0.090032 +0.160000 0.088772 +0.155000 0.087441 +0.150000 0.086110 +0.145000 0.084704 +0.140000 0.083298 +0.135000 0.081814 +0.130000 0.080329 +0.125000 0.078759 +0.120000 0.077188 +0.115000 0.075525 +0.110000 0.073862 +0.105000 0.072098 +0.100000 0.070334 +0.097500 0.069412 +0.095000 0.068489 +0.092500 0.067537 +0.090000 0.066584 +0.087500 0.065601 +0.085000 0.064617 +0.082500 0.063600 +0.080000 0.062583 +0.077500 0.061531 +0.075000 0.060478 +0.072500 0.059388 +0.070000 0.058297 +0.067500 0.057165 +0.065000 0.056032 +0.062500 0.054854 +0.060000 0.053676 +0.057500 0.052447 +0.055000 0.051218 +0.052500 0.049933 +0.050000 0.048647 +0.047500 0.047299 +0.045000 0.045950 +0.042500 0.044530 +0.040000 0.043110 +0.037500 0.041606 +0.035000 0.040102 +0.032500 0.038501 +0.030000 0.036899 +0.027500 0.035177 +0.025000 0.033454 +0.022500 0.031574 +0.020000 0.029694 +0.018750 0.028680 +0.017500 0.027666 +0.016250 0.026589 +0.015000 0.025511 +0.013750 0.024354 +0.012500 0.023197 +0.011250 0.021936 +0.010000 0.020674 +0.009500 0.020131 +0.009000 0.019587 +0.008500 0.019017 +0.008000 0.018447 +0.007500 0.017844 +0.007000 0.017241 +0.006500 0.016598 +0.006000 0.015955 +0.005500 0.015260 +0.005000 0.014565 +0.004500 0.013801 +0.004000 0.013037 +0.003500 0.012167 +0.003000 0.011296 +0.002500 0.010262 +0.002000 0.009227 +0.001875 0.008930 +0.001750 0.008633 +0.001625 0.008315 +0.001500 0.007997 +0.001375 0.007655 +0.001250 0.007312 +0.001125 0.006934 +0.001000 0.006555 +0.000875 0.006125 +0.000750 0.005695 +0.000625 0.005184 +0.000500 0.004672 +0.000400 0.004190 +0.000350 0.003913 +0.000300 0.003636 +0.000200 0.002970 +0.000100 0.002104 +0.000050 0.001052 +0.000000 0.000000 +0.000050 -0.001046 +0.000100 -0.002092 +0.000200 -0.002954 +0.000300 -0.003613 +0.000350 -0.003891 +0.000400 -0.004169 +0.000500 -0.004658 +0.000625 -0.005178 +0.000750 -0.005698 +0.000875 -0.006135 +0.001000 -0.006572 +0.001125 -0.006956 +0.001250 -0.007340 +0.001375 -0.007684 +0.001500 -0.008027 +0.001625 -0.008341 +0.001750 -0.008654 +0.001875 -0.008943 +0.002000 -0.009231 +0.002500 -0.010204 +0.003000 -0.011176 +0.003500 -0.011953 +0.004000 -0.012729 +0.004500 -0.013380 +0.005000 -0.014030 +0.005500 -0.014595 +0.006000 -0.015160 +0.006500 -0.015667 +0.007000 -0.016174 +0.007500 -0.016636 +0.008000 -0.017098 +0.008500 -0.017526 +0.009000 -0.017953 +0.009500 -0.018352 +0.010000 -0.018750 +0.011250 -0.019644 +0.012500 -0.020537 +0.013750 -0.021322 +0.015000 -0.022107 +0.016250 -0.022812 +0.017500 -0.023517 +0.018750 -0.024160 +0.020000 -0.024803 +0.022500 -0.025948 +0.025000 -0.027092 +0.027500 -0.028097 +0.030000 -0.029102 +0.032500 -0.030003 +0.035000 -0.030904 +0.037500 -0.031725 +0.040000 -0.032546 +0.042500 -0.033304 +0.045000 -0.034061 +0.047500 -0.034767 +0.050000 -0.035472 +0.052500 -0.036132 +0.055000 -0.036792 +0.057500 -0.037414 +0.060000 -0.038035 +0.062500 -0.038622 +0.065000 -0.039209 +0.067500 -0.039766 +0.070000 -0.040322 +0.072500 -0.040852 +0.075000 -0.041381 +0.077500 -0.041885 +0.080000 -0.042389 +0.082500 -0.042870 +0.085000 -0.043350 +0.087500 -0.043809 +0.090000 -0.044268 +0.092500 -0.044707 +0.095000 -0.045145 +0.097500 -0.045566 +0.100000 -0.045987 +0.105000 -0.046782 +0.110000 -0.047576 +0.115000 -0.048313 +0.120000 -0.049050 +0.125000 -0.049734 +0.130000 -0.050417 +0.135000 -0.051053 +0.140000 -0.051688 +0.145000 -0.052278 +0.150000 -0.052868 +0.155000 -0.053418 +0.160000 -0.053967 +0.165000 -0.054478 +0.170000 -0.054988 +0.175000 -0.055461 +0.180000 -0.055934 +0.185000 -0.056373 +0.190000 -0.056811 +0.195000 -0.057216 +0.200000 -0.057621 +0.205000 -0.057993 +0.210000 -0.058365 +0.215000 -0.058705 +0.220000 -0.059045 +0.225000 -0.059355 +0.230000 -0.059664 +0.235000 -0.059944 +0.240000 -0.060224 +0.245000 -0.060474 +0.250000 -0.060723 +0.255000 -0.060943 +0.260000 -0.061163 +0.265000 -0.061354 +0.270000 -0.061545 +0.275000 -0.061708 +0.280000 -0.061871 +0.285000 -0.062004 +0.290000 -0.062137 +0.295000 -0.062240 +0.300000 -0.062343 +0.305000 -0.062417 +0.310000 -0.062490 +0.315000 -0.062534 +0.320000 -0.062577 +0.325000 -0.062590 +0.330000 -0.062602 +0.335000 -0.062583 +0.340000 -0.062563 +0.345000 -0.062512 +0.350000 -0.062460 +0.355000 -0.062374 +0.360000 -0.062287 +0.365000 -0.062164 +0.370000 -0.062040 +0.375000 -0.061878 +0.380000 -0.061716 +0.385000 -0.061509 +0.390000 -0.061301 +0.395000 -0.061040 +0.400000 -0.060778 +0.405000 -0.060458 +0.410000 -0.060138 +0.415000 -0.059763 +0.420000 -0.059388 +0.425000 -0.058966 +0.430000 -0.058544 +0.435000 -0.058083 +0.440000 -0.057622 +0.445000 -0.057127 +0.450000 -0.056632 +0.462500 -0.055265 +0.475000 -0.053897 +0.487500 -0.052374 +0.500000 -0.050850 +0.512500 -0.049195 +0.525000 -0.047539 +0.537500 -0.045777 +0.550000 -0.044014 +0.562500 -0.042165 +0.575000 -0.040316 +0.587500 -0.038401 +0.600000 -0.036486 +0.612500 -0.034526 +0.625000 -0.032565 +0.637500 -0.030575 +0.650000 -0.028585 +0.662500 -0.026594 +0.675000 -0.024603 +0.687500 -0.022632 +0.700000 -0.020660 +0.712500 -0.018728 +0.725000 -0.016795 +0.737500 -0.014922 +0.750000 -0.013048 +0.762500 -0.011260 +0.775000 -0.009472 +0.787500 -0.007797 +0.800000 -0.006122 +0.812500 -0.004594 +0.825000 -0.003065 +0.837500 -0.001721 +0.850000 -0.000376 +0.862500 0.000742 +0.875000 0.001859 +0.887500 0.002698 +0.900000 0.003536 +0.905000 0.003780 +0.910000 0.004023 +0.915000 0.004205 +0.920000 0.004387 +0.925000 0.004504 +0.930000 0.004620 +0.935000 0.004661 +0.940000 0.004702 +0.945000 0.004658 +0.950000 0.004614 +0.955000 0.004476 +0.960000 0.004338 +0.965000 0.004084 +0.970000 0.003829 +0.975000 0.003436 +0.980000 0.003042 +0.990000 0.001910 +1.000000 0.000000 diff --git a/example_files/HAWCStab2.pwr b/example_files/HAWCStab2.pwr new file mode 100644 index 0000000..5ab6ead --- /dev/null +++ b/example_files/HAWCStab2.pwr @@ -0,0 +1,46 @@ + # V [m/s] 1 P [kW] 2 T [kN] 3 Cp [-] 4 Ct [-] 5 Pitch Q [Nm] 6 Flap M [kNm] 7 Edge M [kNm] 8 Pitch [deg] 9 Speed [rpm] 10 Tip x [m] 11 Tip y [m] 12 Tip z [m] 13 J_rot [kg*m^2] 14 J_DT [kg*m^2] 15 + 0.3000000000E+01 0.5398423625E+02 0.2226372226E+03 0.7285076413E-01 0.9013348850E+00 -0.7673842965E+05 -0.6556375541E+04 0.4457925824E+02 0.3670255000E+01 0.5000000000E+01 -0.2883777069E+00 -0.1234314003E+02 0.1194277603E+03 0.3101094143E+09 0.3181180646E+09 + 0.3500000000E+01 0.3055236102E+03 0.3084687127E+03 0.2596407088E+00 0.9175023268E+00 -0.6240956989E+05 -0.8818192799E+04 0.2050822920E+03 0.3331074000E+01 0.5000000000E+01 -0.2647987833E+00 -0.1234477340E+02 0.1194276461E+03 0.3101082489E+09 0.3181168992E+09 + 0.4000000000E+01 0.6334956030E+03 0.4056710702E+03 0.3606593633E+00 0.9238205865E+00 -0.4302175020E+05 -0.1137430002E+05 0.4142740044E+03 0.2890489000E+01 0.5000000000E+01 -0.2341568017E+00 -0.1234668712E+02 0.1194275123E+03 0.3101067445E+09 0.3181153948E+09 + 0.4500000000E+01 0.1047009028E+04 0.5146749406E+03 0.4186470321E+00 0.9260685320E+00 -0.1765533271E+05 -0.1423918181E+05 0.6779437287E+03 0.2339542000E+01 0.5000000000E+01 -0.1958200501E+00 -0.1234874939E+02 0.1194273681E+03 0.3101048782E+09 0.3181135285E+09 + 0.5000000000E+01 0.1555115338E+04 0.6360831474E+03 0.4533036789E+00 0.9270657417E+00 0.1466925262E+05 -0.1743327924E+05 0.1001847856E+04 0.1665880000E+01 0.5000000000E+01 -0.1489201279E+00 -0.1235077113E+02 0.1194272267E+03 0.3101026185E+09 0.3181112688E+09 + 0.5500000000E+01 0.2165559979E+04 0.7646686981E+03 0.4742637926E+00 0.9210554048E+00 0.5426722692E+05 -0.2081137919E+05 0.1390894878E+04 0.9443540000E+00 0.5000000000E+01 -0.9866528191E-01 -0.1235232627E+02 0.1194271180E+03 0.3101002253E+09 0.3181088755E+09 + 0.6000000000E+01 0.2882026589E+04 0.9005565172E+03 0.4861639926E+00 0.9114797629E+00 0.1017326476E+06 -0.2438175118E+05 0.1847416804E+04 0.1542180000E+00 0.5000000000E+01 -0.4361403711E-01 -0.1235330493E+02 0.1194270495E+03 0.3100976362E+09 0.3181062864E+09 + 0.6500000000E+01 0.3692420034E+04 0.9941112677E+03 0.4899025091E+00 0.8573278226E+00 0.1488893921E+06 -0.2674400569E+05 0.2363612242E+04 0.5350000000E-03 0.5000000000E+01 -0.3290517114E-01 -0.1235340730E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.7000000000E+01 0.4577667090E+04 0.1083006329E+04 0.4862825892E+00 0.8053293919E+00 0.1956107083E+06 -0.2898210632E+05 0.2905933493E+04 0.5350000000E-03 0.5037432000E+01 -0.3290516996E-01 -0.1235340729E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.7500000000E+01 0.5630330419E+04 0.1243247156E+04 0.4862825968E+00 0.8053294529E+00 0.2245530801E+06 -0.3327027783E+05 0.3335892831E+04 0.5350000000E-03 0.5397249000E+01 -0.3290516865E-01 -0.1235340728E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.8000000000E+01 0.6833135916E+04 0.1414539080E+04 0.4862826035E+00 0.8053295063E+00 0.2554914757E+06 -0.3785418548E+05 0.3795504524E+04 0.5350000000E-03 0.5757066000E+01 -0.3290516724E-01 -0.1235340727E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.8500000000E+01 0.8196092792E+04 0.1596881853E+04 0.4862825936E+00 0.8053294276E+00 0.2884259718E+06 -0.4273382208E+05 0.4284769125E+04 0.5350000000E-03 0.6116882000E+01 -0.3290516575E-01 -0.1235340725E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.9000000000E+01 0.9729211025E+04 0.1790275957E+04 0.4862825998E+00 0.8053294764E+00 0.3233564196E+06 -0.4790920158E+05 0.4803685559E+04 0.5350000000E-03 0.6476699000E+01 -0.3290516416E-01 -0.1235340724E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.9500000000E+01 0.1144249953E+05 0.1994720880E+04 0.4862825911E+00 0.8053294075E+00 0.3602829768E+06 -0.5338030918E+05 0.5352254967E+04 0.5350000000E-03 0.6836515000E+01 -0.3290516248E-01 -0.1235340722E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.1000000000E+02 0.1334596840E+05 0.2210217165E+04 0.4862825968E+00 0.8053294524E+00 0.3992054767E+06 -0.5914716055E+05 0.5930476144E+04 0.5350000000E-03 0.7196332000E+01 -0.3290516071E-01 -0.1235340720E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.1050000000E+02 0.1544962683E+05 0.2436764548E+04 0.4862826018E+00 0.8053294931E+00 0.4401240005E+06 -0.6520974805E+05 0.6538349675E+04 0.5350000000E-03 0.7556149000E+01 -0.3290515886E-01 -0.1235340718E+02 0.1194270424E+03 0.3100971364E+09 0.3181057867E+09 + 0.1100000000E+02 0.1553233025E+05 0.2004220536E+04 0.4251977357E+00 0.6035211860E+00 0.3733916395E+06 -0.5257641630E+05 0.6569236087E+04 0.3260913000E+01 0.7560000000E+01 -0.2599201809E+00 -0.1234509376E+02 0.1194276237E+03 0.3101080087E+09 0.3181166589E+09 + 0.1150000000E+02 0.1553454072E+05 0.1807041362E+04 0.3721604140E+00 0.4978490608E+00 0.3470905986E+06 -0.4662693815E+05 0.6569848555E+04 0.4962560000E+01 0.7560000000E+01 -0.3781154569E+00 -0.1233564154E+02 0.1194282847E+03 0.3101139129E+09 0.3181225632E+09 + 0.1200000000E+02 0.1553668727E+05 0.1672004622E+04 0.3275916269E+00 0.4230513528E+00 0.3308277450E+06 -0.4244271254E+05 0.6570551632E+04 0.6259375000E+01 0.7560000000E+01 -0.4679729902E+00 -0.1232608942E+02 0.1194289526E+03 0.3101185213E+09 0.3181271716E+09 + 0.1250000000E+02 0.1553587252E+05 0.1567638222E+04 0.2898120694E+00 0.3655414885E+00 0.3192121404E+06 -0.3912971776E+05 0.6570072251E+04 0.7356196000E+01 0.7560000000E+01 -0.5437874744E+00 -0.1231642859E+02 0.1194296282E+03 0.3101224936E+09 0.3181311439E+09 + 0.1300000000E+02 0.1553793135E+05 0.1482683528E+04 0.2576716732E+00 0.3196430708E+00 0.3105306194E+06 -0.3636830509E+05 0.6570847970E+04 0.8327868000E+01 0.7560000000E+01 -0.6107858262E+00 -0.1230666208E+02 0.1194303111E+03 0.3101260706E+09 0.3181347209E+09 + 0.1350000000E+02 0.1553915341E+05 0.1410787911E+04 0.2301029369E+00 0.2820267518E+00 0.3036953253E+06 -0.3397823704E+05 0.6571304067E+04 0.9215340000E+01 0.7560000000E+01 -0.6718254509E+00 -0.1229675271E+02 0.1194310041E+03 0.3101293857E+09 0.3181380360E+09 + 0.1400000000E+02 0.1553812391E+05 0.1348398530E+04 0.2063018144E+00 0.2506402257E+00 0.2981346027E+06 -0.3185933084E+05 0.6570837279E+04 0.1004158900E+02 0.7560000000E+01 -0.7285096231E+00 -0.1228668018E+02 0.1194317084E+03 0.3101325138E+09 0.3181411640E+09 + 0.1450000000E+02 0.1553566421E+05 0.1293419337E+04 0.1856550377E+00 0.2241218839E+00 0.2935460743E+06 -0.2995292716E+05 0.6569789098E+04 0.1082011400E+02 0.7560000000E+01 -0.7817813104E+00 -0.1227644424E+02 0.1194324242E+03 0.3101354983E+09 0.3181441485E+09 + 0.1500000000E+02 0.1553741219E+05 0.1244799684E+04 0.1677173840E+00 0.2015533965E+00 0.2898940243E+06 -0.2822975735E+05 0.6570531406E+04 0.1155827800E+02 0.7560000000E+01 -0.8321580883E+00 -0.1226607292E+02 0.1194331494E+03 0.3101383616E+09 0.3181470118E+09 + 0.1550000000E+02 0.1553992008E+05 0.1201171543E+04 0.1520263951E+00 0.1821406783E+00 0.2869036370E+06 -0.2665087608E+05 0.6571608897E+04 0.1226443500E+02 0.7560000000E+01 -0.8802213602E+00 -0.1225554616E+02 0.1194338855E+03 0.3101411314E+09 0.3181497817E+09 + 0.1600000000E+02 0.1553746044E+05 0.1161336836E+04 0.1381902522E+00 0.1652629716E+00 0.2842673439E+06 -0.2518224662E+05 0.6570607077E+04 0.1294548300E+02 0.7560000000E+01 -0.9264490592E+00 -0.1224483468E+02 0.1194346345E+03 0.3101438315E+09 0.3181524818E+09 + 0.1650000000E+02 0.1553736589E+05 0.1125225021E+04 0.1260012320E+00 0.1505638541E+00 0.2821292276E+06 -0.2382132731E+05 0.6570614033E+04 0.1360275500E+02 0.7560000000E+01 -0.9709388581E+00 -0.1223397807E+02 0.1194353937E+03 0.3101464642E+09 0.3181551145E+09 + 0.1700000000E+02 0.1553739690E+05 0.1092182507E+04 0.1152053178E+00 0.1376697120E+00 0.2803551186E+06 -0.2254910427E+05 0.6570685121E+04 0.1424017300E+02 0.7560000000E+01 -0.1013962785E+01 -0.1222296380E+02 0.1194361639E+03 0.3101490427E+09 0.3181576930E+09 + 0.1750000000E+02 0.1553682718E+05 0.1061770096E+04 0.1056041310E+00 0.1262952772E+00 0.2788813841E+06 -0.2135294170E+05 0.6570513029E+04 0.1486036800E+02 0.7560000000E+01 -0.1055703800E+01 -0.1221178949E+02 0.1194369453E+03 0.3101515756E+09 0.3181602259E+09 + 0.1800000000E+02 0.1553697057E+05 0.1033744727E+04 0.9704499257E-01 0.1162231388E+00 0.2777065794E+06 -0.2022649772E+05 0.6570650496E+04 0.1546481100E+02 0.7560000000E+01 -0.1096265676E+01 -0.1220046591E+02 0.1194377371E+03 0.3101540672E+09 0.3181627175E+09 + 0.1850000000E+02 0.1553725172E+05 0.1007795832E+04 0.8938709102E-01 0.1072617525E+00 0.2767720829E+06 -0.1916199996E+05 0.6570854485E+04 0.1605511900E+02 0.7560000000E+01 -0.1135761281E+01 -0.1218899574E+02 0.1194385392E+03 0.3101565225E+09 0.3181651727E+09 + 0.1900000000E+02 0.1553734594E+05 0.9836820133E+03 0.8251315727E-01 0.9925552735E-01 0.2760406205E+06 -0.1815269166E+05 0.6570987389E+04 0.1663265700E+02 0.7560000000E+01 -0.1174285841E+01 -0.1217738144E+02 0.1194393513E+03 0.3101589458E+09 0.3181675960E+09 + 0.1950000000E+02 0.1553664238E+05 0.9611924466E+03 0.7632236060E-01 0.9207454594E-01 0.2754655445E+06 -0.1719208349E+05 0.6570791588E+04 0.1719873100E+02 0.7560000000E+01 -0.1211929940E+01 -0.1216562233E+02 0.1194401736E+03 0.3101613413E+09 0.3181699916E+09 + 0.2000000000E+02 0.1553689749E+05 0.9402894474E+03 0.7073982340E-01 0.8562315543E-01 0.2750768912E+06 -0.1627839639E+05 0.6571006104E+04 0.1775377200E+02 0.7560000000E+01 -0.1248725502E+01 -0.1215373272E+02 0.1194410050E+03 0.3101637098E+09 0.3181723601E+09 + 0.2050000000E+02 0.1553665423E+05 0.9207692424E+03 0.6568658493E-01 0.7980385066E-01 0.2748089678E+06 -0.1540551990E+05 0.6571016636E+04 0.1829895300E+02 0.7560000000E+01 -0.1284753351E+01 -0.1214170872E+02 0.1194418458E+03 0.3101660552E+09 0.3181747055E+09 + 0.2100000000E+02 0.1553628908E+05 0.9025349727E+03 0.6110281233E-01 0.7454134767E-01 0.2746608791E+06 -0.1457045048E+05 0.6570981274E+04 0.1883473000E+02 0.7560000000E+01 -0.1320046435E+01 -0.1212955946E+02 0.1194426953E+03 0.3101683786E+09 0.3181770289E+09 + 0.2150000000E+02 0.1553620034E+05 0.8854909974E+03 0.5693668373E-01 0.6977019959E-01 0.2746375155E+06 -0.1377059404E+05 0.6571067218E+04 0.1936168500E+02 0.7560000000E+01 -0.1354645815E+01 -0.1211728963E+02 0.1194435533E+03 0.3101706815E+09 0.3181793318E+09 + 0.2200000000E+02 0.1553653707E+05 0.8695323130E+03 0.5314223125E-01 0.6543256807E-01 0.2747337666E+06 -0.1300341730E+05 0.6571337140E+04 0.1988044100E+02 0.7560000000E+01 -0.1388594946E+01 -0.1210490115E+02 0.1194444196E+03 0.3101729660E+09 0.3181816163E+09 + 0.2250000000E+02 0.1553627038E+05 0.8545191736E+03 0.4967565567E-01 0.6147537226E-01 0.2749103342E+06 -0.1226491816E+05 0.6571357861E+04 0.2039156100E+02 0.7560000000E+01 -0.1421933038E+01 -0.1209239563E+02 0.1194452941E+03 0.3101752336E+09 0.3181838839E+09 + 0.2300000000E+02 0.1553618504E+05 0.8404088845E+03 0.4650460765E-01 0.5785888640E-01 0.2751820024E+06 -0.1155398168E+05 0.6571459965E+04 0.2089548700E+02 0.7560000000E+01 -0.1454691131E+01 -0.1207977617E+02 0.1194461765E+03 0.3101774858E+09 0.3181861361E+09 + 0.2350000000E+02 0.1553594362E+05 0.8271185911E+03 0.4359732102E-01 0.5454535992E-01 0.2755296674E+06 -0.1086827176E+05 0.6571501392E+04 0.2139252400E+02 0.7560000000E+01 -0.1486891181E+01 -0.1206704816E+02 0.1194470666E+03 0.3101797232E+09 0.3181883735E+09 + 0.2400000000E+02 0.1553574070E+05 0.8145976274E+03 0.4092743933E-01 0.5150352948E-01 0.2759513338E+06 -0.1020655254E+05 0.6571563879E+04 0.2188293700E+02 0.7560000000E+01 -0.1518552444E+01 -0.1205421714E+02 0.1194479638E+03 0.3101819464E+09 0.3181905967E+09 + 0.2450000000E+02 0.1553700833E+05 0.8028470384E+03 0.3847476720E-01 0.4870880739E-01 0.2764871857E+06 -0.9568702524E+04 0.6572250353E+04 0.2236688800E+02 0.7560000000E+01 -0.1549687466E+01 -0.1204129063E+02 0.1194488677E+03 0.3101841555E+09 0.3181928058E+09 + 0.2500000000E+02 0.1553480512E+05 0.7916154036E+03 0.3620621170E-01 0.4612448412E-01 0.2769887995E+06 -0.8947269030E+04 0.6571479526E+04 0.2284547500E+02 0.7560000000E+01 -0.1580368674E+01 -0.1202824984E+02 0.1194497796E+03 0.3101863550E+09 0.3181950053E+09 diff --git a/installer.cfg b/installer.cfg index f41020c..6f49f55 100644 --- a/installer.cfg +++ b/installer.cfg @@ -46,13 +46,10 @@ pypi_wheels = # PyYAML==5.1.2 -packages=weio +packages= future -exclude=weio/.git* - weio/tests - pkgs/weio/examples - pkgs/weio/weio/tests +exclude= pkgs/numpy/core/include pkgs/numpy/doc pkgs/numpy/f2py diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 7974b75..3e940e1 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -6,17 +6,7 @@ from .common import no_unit, ellude_common, getDt except: from common import no_unit, ellude_common, getDt -try: - import weio.weio as weio# File Formats and File Readers -except: - print('') - print('Error: the python package `weio` was not imported successfully.\n') - print('Most likely the submodule `weio` was not cloned with `pyDatView`') - print('Type the following command to retrieve it:\n') - print(' git submodule update --init --recursive\n') - print('Alternatively re-clone this repository into a separate folder:\n') - print(' git clone --recurse-submodules https://github.com/ebranlard/pyDatView\n') - sys.exit(-1) +import pydatview.io as weio # File Formats and File Readers diff --git a/pydatview/appdata.py b/pydatview/appdata.py index f801565..db0816a 100644 --- a/pydatview/appdata.py +++ b/pydatview/appdata.py @@ -1,6 +1,6 @@ import json import os -from weio.weio import defaultUserDataDir +from pydatview.io import defaultUserDataDir from .GUICommon import Error diff --git a/pydatview/fast/case_gen.py b/pydatview/fast/case_gen.py index f9c53f0..4b7076e 100644 --- a/pydatview/fast/case_gen.py +++ b/pydatview/fast/case_gen.py @@ -9,7 +9,7 @@ import re # --- Misc fast libraries -import weio.weio.fast_input_file as fi +import pydatview.io.fast_input_file as fi import pydatview.fast.runner as runner import pydatview.fast.postpro as postpro #import pyFAST.input_output.fast_input_file as fi diff --git a/pydatview/fast/fastfarm.py b/pydatview/fast/fastfarm.py index 1c939c0..072f10e 100644 --- a/pydatview/fast/fastfarm.py +++ b/pydatview/fast/fastfarm.py @@ -2,9 +2,9 @@ import glob import numpy as np import pandas as pd -from weio.weio.fast_input_file import FASTInputFile -from weio.weio.fast_output_file import FASTOutputFile -from weio.weio.turbsim_file import TurbSimFile +from pydatview.io.fast_input_file import FASTInputFile +from pydatview.io.fast_output_file import FASTOutputFile +from pydatview.io.turbsim_file import TurbSimFile from . import fastlib diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 8555d11..3eb7f89 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -6,9 +6,9 @@ import re # --- fast libraries -from weio.weio.fast_input_file import FASTInputFile -from weio.weio.fast_output_file import FASTOutputFile -from weio.weio.fast_input_deck import FASTInputDeck +from pydatview.io.fast_input_file import FASTInputFile +from pydatview.io.fast_output_file import FASTOutputFile +from pydatview.io.fast_input_deck import FASTInputDeck # from pyFAST.input_output.fast_input_file import FASTInputFile # from pyFAST.input_output.fast_output_file import FASTOutputFile # from pyFAST.input_output.fast_input_deck import FASTInputDeck diff --git a/pydatview/fast/runner.py b/pydatview/fast/runner.py index d707088..eaddce1 100644 --- a/pydatview/fast/runner.py +++ b/pydatview/fast/runner.py @@ -13,8 +13,8 @@ import re # --- Fast libraries -from weio.weio.fast_input_file import FASTInputFile -from weio.weio.fast_output_file import FASTOutputFile +from pydatview.io.fast_input_file import FASTInputFile +from pydatview.io.fast_output_file import FASTOutputFile # from pyFAST.input_output.fast_input_file import FASTInputFile # from pyFAST.input_output.fast_output_file import FASTOutputFile diff --git a/pydatview/io/__init__.py b/pydatview/io/__init__.py new file mode 100644 index 0000000..73737be --- /dev/null +++ b/pydatview/io/__init__.py @@ -0,0 +1,264 @@ +from .file import File, WrongFormatError, BrokenFormatError, FileNotFoundError, EmptyFileError +from .file_formats import FileFormat, isRightFormat +import sys +import os +import numpy as np + +class FormatNotDetectedError(Exception): + pass + +class UserFormatImportError(Exception): + pass + + +_FORMATS=None + +def fileFormats(userpath=None, ignoreErrors=False, verbose=False): + """ return list of fileformats supported by the library + If userpath is provided, + + """ + global _FORMATS + if _FORMATS is not None: + return _FORMATS + # --- Library formats + from .fast_input_file import FASTInputFile + from .fast_output_file import FASTOutputFile + from .csv_file import CSVFile + from .fast_wind_file import FASTWndFile + from .fast_linearization_file import FASTLinearizationFile + from .fast_summary_file import FASTSummaryFile + from .bmodes_out_file import BModesOutFile + from .hawc2_pc_file import HAWC2PCFile + from .hawc2_ae_file import HAWC2AEFile + from .hawc2_dat_file import HAWC2DatFile + from .hawc2_htc_file import HAWC2HTCFile + from .hawc2_st_file import HAWC2StFile + from .hawcstab2_pwr_file import HAWCStab2PwrFile + from .hawcstab2_ind_file import HAWCStab2IndFile + from .hawcstab2_cmb_file import HAWCStab2CmbFile + from .mannbox_file import MannBoxFile + from .flex_blade_file import FLEXBladeFile + from .flex_profile_file import FLEXProfileFile + from .flex_out_file import FLEXOutFile + from .flex_doc_file import FLEXDocFile + from .flex_wavekin_file import FLEXWaveKinFile + from .excel_file import ExcelFile + from .turbsim_ts_file import TurbSimTSFile + from .turbsim_file import TurbSimFile + from .netcdf_file import NetCDFFile + from .tdms_file import TDMSFile + from .tecplot_file import TecplotFile + from .vtk_file import VTKFile + from .bladed_out_file import BladedFile + from .parquet_file import ParquetFile + from .cactus_file import CactusFile + from .raawmat_file import RAAWMatFile + from .rosco_performance_file import ROSCOPerformanceFile + priorities = [] + formats = [] + def addFormat(priority, fmt): + priorities.append(priority) + formats.append(fmt) + addFormat(0, FileFormat(CSVFile)) + addFormat(0, FileFormat(ExcelFile)) + addFormat(10, FileFormat(TecplotFile)) + addFormat(10, FileFormat(BladedFile)) + addFormat(20, FileFormat(FASTInputFile)) + addFormat(20, FileFormat(FASTOutputFile)) + addFormat(20, FileFormat(FASTWndFile)) + addFormat(20, FileFormat(FASTLinearizationFile)) + addFormat(20, FileFormat(FASTSummaryFile)) + addFormat(20, FileFormat(TurbSimTSFile)) + addFormat(20, FileFormat(TurbSimFile)) + addFormat(30, FileFormat(HAWC2DatFile)) + addFormat(30, FileFormat(HAWC2HTCFile)) + addFormat(30, FileFormat(HAWC2StFile)) + addFormat(30, FileFormat(HAWC2PCFile)) + addFormat(30, FileFormat(HAWC2AEFile)) + addFormat(30, FileFormat(HAWCStab2PwrFile)) + addFormat(30, FileFormat(HAWCStab2IndFile)) + addFormat(30, FileFormat(HAWCStab2CmbFile)) + addFormat(30, FileFormat(MannBoxFile)) + addFormat(40, FileFormat(FLEXBladeFile)) + addFormat(40, FileFormat(FLEXProfileFile)) + addFormat(40, FileFormat(FLEXOutFile)) + addFormat(40, FileFormat(FLEXWaveKinFile)) + addFormat(40, FileFormat(FLEXDocFile)) + addFormat(50, FileFormat(BModesOutFile)) + addFormat(50, FileFormat(ROSCOPerformanceFile)) + addFormat(60, FileFormat(NetCDFFile)) + addFormat(60, FileFormat(VTKFile)) + addFormat(60, FileFormat(TDMSFile)) + addFormat(60, FileFormat(ParquetFile)) + addFormat(70, FileFormat(CactusFile)) + addFormat(70, FileFormat(RAAWMatFile)) + + # --- User defined formats from user path + UserClasses, UserPaths, UserModules, UserModuleNames, errors = userFileClasses(userpath, ignoreErrors, verbose=verbose) + for cls, f in zip(UserClasses, UserPaths): + try: + ff = FileFormat(cls) + except Exception as e: + s='Error registering a user fileformat.\n\nThe module location was: {}\n\nThe class name was: {}\n\nMake sure the class has `defaultExtensions` and `formatName` as static methods.\n\nThe exception was:\n{}'.format(f, cls.__name__, e) + if ignoreErrors: + errors.append(s) + continue + else: + raise UserFormatImportError(s) + # Use class.priority + try: + priority = cls.priority() + except: + priority=2 + addFormat(priority, ff) + + # --- Sort fileformats by priorities + formats = np.asarray(formats)[np.argsort(priorities, kind='stable')] + + _FORMATS=formats + if ignoreErrors: + return formats, errors + else: + return formats + + + +def userFileClasses(userpath=None, ignoreErrors=False, verbose=True): + """ return list of user file class in UserData folder""" + if userpath is None: + dataDir = defaultUserDataDir() + userpath = os.path.join(dataDir, 'weio') + errors = [] + UserClasses = [] + UserPaths = [] + UserModules = [] + UserModuleNames = [] + if os.path.exists(userpath): + if verbose: + print('>>> Looking for user modules in folder:',userpath) + import glob + from importlib.machinery import SourceFileLoader + import inspect + pyfiles = glob.glob(os.path.join(userpath,'*.py')) + # Loop through files, look for classes of the form ClassNameFile, + for f in pyfiles: + if f in ['__init__.py']: + continue + mod_name = os.path.basename(os.path.splitext(f)[0]) + try: + if verbose: + print('>>> Trying to load user module:',f) + module = SourceFileLoader(mod_name,f).load_module() + except Exception as e: + s='Error importing a user module.\n\nThe module location was: {}\n\nTry importing this module to debug it.\n\nThe Exception was:\n{}'.format(f, e) + if ignoreErrors: + errors.append(s) + continue + else: + raise UserFormatImportError(s) + found=False + for name, obj in inspect.getmembers(module): + if inspect.isclass(obj): + classname = obj.__name__.lower() + if classname!='file' and classname.find('file')>=0 and classname.find('error')<0: + if verbose: + print(' Found File class with name:',obj.__name__) + UserClasses.append(obj) + UserPaths.append(f) + UserModules.append(module) + UserModuleNames.append(mod_name) + found=True # allowing only one class per file for now.. + break + if not found: + s='Error finding a class named "*File" in the user module.\n\nThe module location was: {}\n\nNo class containing the string "File" in its name was found.'.format(f) + if ignoreErrors: + errors.append(s) + else: + raise UserFormatImportError(s) + return UserClasses, UserPaths, UserModules, UserModuleNames, errors + + +def defaultUserDataDir(): + """ + Returns a parent directory path + where persistent application data can be stored. + # linux: ~/.local/share + # macOS: ~/Library/Application Support + # windows: C:/Users//AppData/Roaming + """ + home = os.path.expanduser('~') + ptfm = sys.platform + if ptfm == "win32": + return os.path.join(home , 'AppData','Roaming') + elif ptfm.startswith("linux"): + return os.path.join(home, '.local', 'share') + elif ptfm == "darwin": + return os.path.join(home, 'Library','Application Support') + else: + print('>>>>>>>>>>>>>>>>> Unknown Platform', sys.platform) + return './UserData' + + + +def detectFormat(filename, **kwargs): + """ Detect the file formats by looping through the known list. + The method may simply try to open the file, if that's the case + the read file is returned. """ + import os + import re + global _FORMATS + if _FORMATS is None: + formats=fileFormats() + else: + formats=_FORMATS + ext = os.path.splitext(filename.lower())[1] + detected = False + i = 0 + while not detected and i0: + extPatMatch = [re.match(pat, ext) is not None for pat in extPatterns] + extMatch = any(extPatMatch) + else: + extMatch = False + if extMatch: # we have a match on the extension + valid, F = isRightFormat(myformat, filename, **kwargs) + if valid: + #print('File detected as :',myformat) + detected=True + return myformat,F + + i += 1 + + if not detected: + raise FormatNotDetectedError('The file format could not be detected for the file: '+filename) + +def read(filename, fileformat=None, **kwargs): + F = None + # Detecting format if necessary + if fileformat is None: + fileformat,F = detectFormat(filename, **kwargs) + # Reading the file with the appropriate class if necessary + if not isinstance(F, fileformat.constructor): + F=fileformat.constructor(filename=filename) + return F + + +# --- For legacy code +def FASTInputFile(*args,**kwargs): + from .fast_input_file import FASTInputFile as fi + return fi(*args,**kwargs) +def FASTOutputFile(*args,**kwargs): + from .fast_output_file import FASTOutputFile as fo + return fo(*args,**kwargs) +def CSVFile(*args,**kwargs): + from .csv_file import CSVFile as csv + return csv(*args,**kwargs) + + diff --git a/pydatview/io/bladed_out_file.py b/pydatview/io/bladed_out_file.py new file mode 100644 index 0000000..1f7a6c1 --- /dev/null +++ b/pydatview/io/bladed_out_file.py @@ -0,0 +1,380 @@ +# -*- coding: utf-8 -*- +import os +import numpy as np +import re +import pandas as pd +import glob +import shlex +# try: +from .file import File, WrongFormatError, BrokenFormatError, isBinary +# except: +# EmptyFileError = type('EmptyFileError', (Exception,),{}) +# WrongFormatError = type('WrongFormatError', (Exception,),{}) +# BrokenFormatError = type('BrokenFormatError', (Exception,),{}) +# File=dict + + +# --------------------------------------------------------------------------------} +# --- Helper functions +# --------------------------------------------------------------------------------{ +def read_bladed_sensor_file(sensorfile): + """ + Extract relevant informations from a bladed sensor file + """ + with open(sensorfile, 'r') as fid: + sensorLines = fid.readlines() + + dat=dict() # relevant info in sensor file + + ## read sensor file line by line (just read up to line 20) + #while i < 17: + for i, t_line in enumerate(sensorLines): + if i>30: + break + t_line = t_line.replace('\t',' ') + + if t_line.startswith('NDIMENS'): + # check what is matrix dimension of the file. For blade & tower, + # the matrix is 3-dimensional. + temp = t_line[7:].strip().split() + dat['NDIMENS'] = int(temp[-1]); + + elif t_line.startswith('DIMENS'): + # check what is the size of the matrix + # for example, it can be 11x52500 or 12x4x52500 + temp = t_line[6:].strip().split() + dat['nSensors'] = int(temp[0]) + dat['nMajor'] = int(temp[dat['NDIMENS']-1]) + if dat['NDIMENS'] == 2: + dat['nSections'] = 1 + dat['SectionList'] = [] + + elif t_line.startswith('FORMAT'): + # precision: n/a, R*4, R*8, I*4 + temp = t_line[7:].strip() + dat['Precision'] = np.float32 + if temp[-1] == '8': + dat['Precision'] = np.float64 + + elif t_line.startswith('GENLAB'): + # category of the file you are reading: + dat['category'] = t_line[6:].strip().replace('\'','') + + elif t_line.startswith('AXIVAL'): + # Section on the 3rd dimension you are reading + # sometimes, the info is written on "AXITICK" + temp = t_line[7:].split() + dat['SectionList'] = np.array(temp, dtype=float) + dat['nSections'] = len(dat['SectionList']) + + elif t_line.startswith('AXITICK'): + # Section on the 3rd dimension you are reading + # sometimes, the info is written on "AXIVAL" + # Check next line, we concatenate if doesnt start with AXISLAB (Might need more cases) + try: + nextLine=sensorLines[i+1].strip() + if not nextLine.startswith('AXISLAB'): + t_line = t_line.strip()+' '+nextLine + except: + pass + + temp = t_line[7:].strip() + temp = temp.strip('\'').split('\' \'') + dat['SectionList'] = np.array(temp, dtype=str) + dat['nSections'] = len(dat['SectionList']) + + elif t_line.startswith('VARIAB'): + # channel names, NOTE: either quoted, non-quoted, and a mix of both + # Check next line, we concatenate if doesnt start with AXISLAB (Might need more cases) + try: + nextLine=sensorLines[i+1].strip() + if not nextLine.startswith('VARUNIT'): + t_line = t_line.strip()+' '+nextLine + except: + pass + dat['ChannelName'] = shlex.split(t_line[6:]) + + elif t_line.startswith('VARUNIT'): + # channel units: + # Check next line, we concatenate if doesnt start with AXISLAB (Might need more cases) + try: + nextLine=sensorLines[i+1].strip() + if not nextLine.startswith('AXISLAB'): + t_line = t_line.strip()+' '+nextLine + except: + pass + def repUnits(s): + s = s.replace('TT','s^2').replace('T','s').replace('A','rad') + s = s.replace('P','W').replace('L','m').replace('F','N').replace('M','kg') + return s + dat['ChannelUnit']=[repUnits(s) for s in shlex.split(t_line[7:].strip())] + + elif t_line.startswith('MIN '): + dat['MIN'] = float(t_line[3:].strip()) # Start time? + + elif t_line.startswith('STEP'): + dat['STEP'] = float(t_line[4:].strip()) # DT? + + + NeededKeys=['ChannelName','nSensors','nMajor','nSections'] + if not all(key in dat.keys() for key in NeededKeys): + raise BrokenFormatError('Broken or unsupported format. Some necessary keys where not found in the bladed sensor file: {}'.format(sensorfile)) + + if len(dat['ChannelName']) != dat['nSensors']: + raise BrokenFormatError('Broken or unsupported format. Wrong number of channels while reading bladed sensor file: {}'.format(sensorfile)) + # if number of channel names are not matching with Sensor number then create dummy ones: + #dat['ChannelName'] = ['Channel' + str(ss) for ss in range(dat['nSensors'])] + + + return dat + +def OrgData(data, **info): + """ Flatten 3D field into 2D table""" + # since some of the matrices are 3 dimensional, we want to make all + # to 2d matrix, so I am organizing them here: + if info['NDIMENS'] == 3: + SName = [] + SUnit = [] + dataOut = np.zeros( (info['nMajor'],len(info['SectionList'])*len(info['ChannelName'])) ) + + col_vec = -1 + for isec,sec in enumerate(info['SectionList']): + for ichan,(chan,unit) in enumerate(zip(info['ChannelName'], info['ChannelUnit'])): + try: + SName.append(str(np.around(float(sec),2)) + 'm-' + chan) + except ValueError: + SName.append(str(sec) + '-' + chan) + SUnit.append(unit) + col_vec +=1 + dataOut[:,col_vec] = data[:,isec,ichan] + + data = dataOut + info['ChannelName'] = SName + info['ChannelUnit'] = SUnit + else: + pass # Nothing to do for 2D + + return data, info + + + +def read_bladed_output(sensorFilename, readTimeFilesOnly=False): + """ + read a bladed sensor file and data file, reorganize a 3D file into 2D table + """ + # --- Read sensor file and extract relevant informations + sensorInfo = read_bladed_sensor_file(sensorFilename) + nSensors = sensorInfo['nSensors'] + nMajor = sensorInfo['nMajor'] + nSections = sensorInfo['nSections'] + hasTime = 'MIN' and 'STEP' in sensorInfo.keys() + # --- Return if caller only wants time series + if readTimeFilesOnly and not hasTime: + return [], {} + + # --- Read data file + dataFilename = sensorFilename.replace('%','$') + + if isBinary(dataFilename): # it is binary + + with open(os.path.join(dataFilename), 'rb') as fid_2: + data = np.fromfile(fid_2, sensorInfo['Precision']) + + try: + if sensorInfo['NDIMENS'] == 3: + data = np.reshape(data,(nMajor, nSections, nSensors), order='C') + + elif sensorInfo['NDIMENS'] == 2: + data = np.reshape(data,(nMajor,nSensors), order='C') + except: + print('>>> Failed to reshape binary file {}'.format(dataFilename)) + raise + + + else: + #print('it is ascii', NDIMENS) + if sensorInfo['NDIMENS'] == 2: + try: + # Data is stored as time, signal, we reshape to signal, time + data = np.loadtxt(dataFilename) + except ValueError as e: + # Most likely this was a binary file... + data = np.empty((nMajor, nSensors)) * np.nan + print('>>> Value error while reading 2d ascii file: {}'.format(dataFilename)) + raise e + except: + data = np.empty((nMajor, nSensors)) * np.nan + print('>>> Failed to read 2d ascii file: {}'.format(dataFilename)) + raise + + + elif sensorInfo['NDIMENS'] == 3: + try: + # Data is stored as sections, time, signal, we reshape to signal, section, time + data = np.loadtxt(dataFilename).reshape((nMajor, nSections, nSensors),order='C') + except: + data = np.empty((nMajor, nSections, nSensors)) * np.nan + print('>>> Failed to read 3d ascii file: {}'.format(dataFilename)) + + return OrgData(data, **sensorInfo) + + +class BladedFile(File): + r""" + Read a Bladed out put file (current version is only binary files) + + Main methods: + read: it finds all % and $ files based on selected .$PJ file and calls "DataValue" to read data from all those files + toDataFrame: create Pandas dataframe output + + Main data stored: + self.dataSets: dictionary of datasets, for each "length" of data + + example: + filename = r'h:\004_Loads\Sim\Bladed\003\Ramp_up\Bladed_out_ascii.$04' + f = BladedFile(filename) + print(f.dataSets.keys()) + df = f.toDataFrame() + + """ + @staticmethod + def defaultExtensions(): + return ['.%*', '.$*'] + + @staticmethod + def formatName(): + return 'Bladed output file' + + def __init__(self, filename=None, **kwargs): + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ read self, or read filename if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # Calling children function + self._read(**kwargs) + + def _read(self): + """ + Read a bladed output file, data are in *.$II and sensors in *%II. + - If the file is a *$PJ file, all output files are read + - Otherwise only the current file is read + """ + + basename, ext = os.path.splitext(self.filename) + if ext.lower()=='.$pj': + readTimeFilesOnly=True + searchPattern = basename + '.%[0-9][0-9]*' # find all files in the folder + else: + readTimeFilesOnly=False + searchPattern = basename + ext.replace('$','%') # sensor file name + + # Look for files matching pattern + files = glob.glob(searchPattern) + + # We'll store the data in "dataSets",dictionaries + dataSets={} + + if len(files)==0: + e= FileNotFoundError(searchPattern) + e.filename=(searchPattern) + raise e + elif len(files)==1: + readTimeFilesOnly=False + + files.sort() + + for i,filename in enumerate(files): + + dataFilename = filename.replace('%','$') + try: + # Call "Read_bladed_file" function to Read and store data: + data, info = read_bladed_output(filename, readTimeFilesOnly=readTimeFilesOnly) + except FileNotFoundError as e: + print('>>> Missing datafile: {}'.format(e.filename)) + if len(files)==1: + raise e + continue + except ValueError as e: + print('>>> ValueError while reading: {}'.format(dataFilename)) + if len(files)==1: + raise e + continue + except: + raise + print('>>> Misc error while reading: {}'.format(dataFilename)) + if len(files)==1: + raise + continue + if len(data)==0: + print('>>> Skipping file since no time present {}'.format(filename)) + continue + + # we use number of data as key, but we'll use "name" later + key = info['nMajor'] + + if key in dataSets.keys(): + # dataset with this length are already present, we concatenate + dset = dataSets[key] + dset['data'] = np.column_stack((dset['data'], data)) + dset['sensors'] += info['ChannelName'] + dset['units'] += info['ChannelUnit'] + dset['name'] = 'Misc_'+str(key) + + else: + # We add a new dataset for this length + dataSets[key] = {} + dset = dataSets[key] + # We force a time vector when possible + if 'MIN' and 'STEP' in info.keys(): + time = np.arange(info['nMajor'])*info['STEP'] + info['MIN'] + data = np.column_stack((time, data)) + info['ChannelName'].insert(0, 'Time') + info['ChannelUnit'].insert(0, 's') + + dset['data'] = data + dset['sensors'] = info['ChannelName'] + dset['units'] = info['ChannelUnit'] + dset['name'] = info['category'] + + # Check if we have "many" misc, if only one, replace by "Misc" + keyMisc = [k for k,v in dataSets.items() if v['name'].startswith('Misc_')] + if len(keyMisc)==1: + #dataSets[keyMisc[0]]['name']='Misc' + # We keep only one dataset for simplicity + self.dataSets= {'Misc': dataSets[keyMisc[0]]} + else: + # Instead of using nMajor as key, we use the "name" + self.dataSets= {v['name']: v for (k, v) in dataSets.items()} + + + def toDataFrame(self): + dfs={} + for k,dset in self.dataSets.items(): + BL_ChannelUnit = [ name+' ['+unit+']' for name,unit in zip(dset['sensors'],dset['units'])] + df = pd.DataFrame(data=dset['data'], columns=BL_ChannelUnit) + # remove duplicate columns + df = df.loc[:,~df.columns.duplicated()] + df.columns.name = k # hack for pyDatView when one dataframe is returned + dfs[k] = df + if len(dfs)==1: + return dfs[next(iter(dfs))] + else: + return dfs + +if __name__ == '__main__': + pass + #filename = r'E:\Work_Google Drive\Bladed_Sims\Bladed_out_binary.$41' + #Output = BladedFile(filename) + #df = Output.toDataFrame() + + diff --git a/pydatview/io/bmodes_out_file.py b/pydatview/io/bmodes_out_file.py new file mode 100644 index 0000000..337d284 --- /dev/null +++ b/pydatview/io/bmodes_out_file.py @@ -0,0 +1,152 @@ +""" +Input/output class for the BModes output files +""" +import numpy as np +import pandas as pd +import os + +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + File=dict + +class BModesOutFile(File): + """ + Read/write a BModes output file. The object behaves as a dictionary. + + Main methods + ------------ + - read, write, toDataFrame, keys + + Examples + -------- + f = BModesOutFile('file.out') + print(f.keys()) + print(f.toDataFrame().columns) + + """ + + @staticmethod + def defaultExtensions(): + """ List of file extensions expected for this fileformat""" + return ['.out'] + + @staticmethod + def formatName(): + """ Short string (~100 char) identifying the file format""" + return 'BModes output file' + + def __init__(self, filename=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # --- Calling (children) function to read + self._read(**kwargs) + + def write(self, filename=None): + """ Rewrite object to file, or write object to `filename` if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + # Calling (children) function to write + self._write() + + def _read(self): + """ Reads self.filename and stores data into self. Self is (or behaves like) a dictionary""" + # --- Example: + #self['data']=[] + #with open(self.filename, 'r', errors="surrogateescape") as f: + # for i, line in enumerate(f): + # self['data'].append(line) + + with open(self.filename) as f: + self['frequencies'] = [] + self['mode_shapes'] = [] + row_string = f.readline() + if row_string.find('BModes')<0: + raise WrongFormatError('This file was not generated by BModes, "BModes" is not found on first line') + while row_string: + row_string = f.readline() + freq_id = row_string.find('freq = ') + if freq_id > 0: + self['frequencies'].append(float(row_string[freq_id+7:freq_id+19])) + f.readline() + f.readline() + f.readline() + data=[] + while True: + row_data = f.readline() + if len(row_data.strip())==0 or row_data.find('===')==0: + break + else: + data.append(row_data.split()) + self['mode_shapes'].append(np.asarray(data).astype(float)) + + def _write(self): + """ Writes to self.filename""" + # --- Example: + #with open(self.filename,'w') as f: + # f.write(self.toString) + raise NotImplementedError() + + def toDataFrame(self): + """ Returns object into one DataFrame, or a dictionary of DataFrames""" + dfs={} + cols =['span_loc','s-s disp','s-s slope','f-a disp','f-a slope','twist'] + for iMode,mode in enumerate(self['mode_shapes']): + dfs['Mode_{}'.format(iMode+1)]= pd.DataFrame(data=mode,columns=cols) + return dfs + + # --- Optional functions + def __repr__(self): + """ String that is written to screen when the user calls `print()` on the object. + Provide short and relevant information to save time for the user. + """ + s='<{} object>:\n'.format(type(self).__name__) + s+='|Main attributes:\n' + s+='| - filename: {}\n'.format(self.filename) + # --- Example printing some relevant information for user + s+='|Main keys:\n' + s+='| - frequencies: {}\n'.format(self['frequencies']) + s+='| - mode_shapes : {} shapes of shape {}x{}\n'.format(len(self['mode_shapes']), *self['mode_shapes'][0].shape) + s+='|Main methods:\n' + s+='| - read, write, toDataFrame, keys' + return s + + def _get_modal_coefficients(x, y, deg=[2, 3, 4, 5, 6]): + # Normalize x input + xn = (x - x.min()) / (x.max() - x.min()) + # Get coefficients to 6th order polynomial + p6 = np.polynomial.polynomial.polyfit(xn, y, deg) + return p6 + + def _identify(self): + """ identify modes""" + pass + + + +if __name__ == '__main__': + f = BModesOutFile('tests/example_files/BModesOut.out') + df = f.toDataFrame() + print(f) + print(df) + diff --git a/pydatview/io/cactus_element_file.py b/pydatview/io/cactus_element_file.py new file mode 100644 index 0000000..08e7fbf --- /dev/null +++ b/pydatview/io/cactus_element_file.py @@ -0,0 +1,107 @@ +import numpy as np +import pandas as pd +import os + +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + File=dict + +class CactusElementFile(File): + + @staticmethod + def defaultExtensions(): + return ['.in'] + + @staticmethod + def formatName(): + return 'CACTUS file' + + def __init__(self,filename=None,**kwargs): + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ read self, or read filename if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # Calling children function + self._read(**kwargs) + + def write(self, filename=None): + """ write self, or to filename if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + # Calling children function + self._write() + + def _read(self): + """ """ + import f90nml + from .csv_file import CSVFile + + filepath = self.filename + basepath = '_'.join(filepath.split('_')[:-1]) + basename = os.path.basename(basepath) + parentdir = os.path.dirname(basepath) + mainfile = parentdir[:-6]+basename+'.in' + print(basename) + print(basepath) + print(parentdir) + print(mainfile) + print(os.path.dirname(basepath)) + + + + +# elemfile= +# if len(df.columns)!=len(cols): +# print('column for rename:',cols) +# print('columns in file :',df.columns) +# print(len(df.columns)) +# print(len(cols)) +# raise Exception('Problem with number of columns') +# df.columns=cols +# +# # --- Read elem +# elemfile = f.replace('TimeData','ElementData') +# dsfile = f.replace('TimeData','DSData') +# dfElem = weio.read(elemfile).toDataFrame() + + #with open(self.filename, 'r', errors="surrogateescape") as f: + # for i, line in enumerate(f): + # data.append(line) + + def _write(self): + """ """ + with open(self.filename,'w') as f: + f.write(self.toString) + + def toDataFrame(self): + #cols=['Alpha_[deg]','Cl_[-]','Cd_[-]','Cm_[-]'] + #dfs[name] = pd.DataFrame(data=..., columns=cols) + #df=pd.DataFrame(data=,columns=) + return + + + def toString(self): + s='' + return s + + def __repr__(self): + s ='Class XXXX (attributes: data)\n' + return s + + diff --git a/pydatview/io/cactus_file.py b/pydatview/io/cactus_file.py new file mode 100644 index 0000000..272e575 --- /dev/null +++ b/pydatview/io/cactus_file.py @@ -0,0 +1,386 @@ +import numpy as np +import pandas as pd +import os + +try: + from .file import File, WrongFormatError, BrokenFormatError, EmptyFileError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + File=dict + +class CactusFile(File): + + @staticmethod + def defaultExtensions(): + return ['.in'] + + @staticmethod + def formatName(): + return 'CACTUS file' + + def __init__(self,filename=None,**kwargs): + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ read self, or read filename if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # Calling children function + self._read(**kwargs) + + def write(self, filename=None): + """ write self, or to filename if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + # Calling children function + self._write() + + def _read(self): + """ """ + import f90nml + from .csv_file import CSVFile + + filepath = self.filename + basepath = os.path.splitext(filepath)[0] + basename = os.path.basename(basepath) + parentdir = os.path.dirname(basepath) + + # --- Read main input file + nml = f90nml.read(filepath) + for k in ['configinputs','caseinputs']: + self[k] = nml[k] + + # --- Try to read geometry file + arfoilfile = os.path.join(parentdir, nml['caseinputs']['afdpath']) + geomfile = os.path.join(parentdir, nml['caseinputs']['geomfilepath']) + if os.path.exists(geomfile): + with open(geomfile, 'r', errors="surrogateescape") as fid: + geom=dict() + nMax=10 + for i, line in enumerate(fid): + # remove comment + line = line.strip().split('!')[0] + sp = line.strip().split(':') + key = sp[0].strip().lower() + if len(key)>0: + strvalue = sp[1] + try: + value = np.asarray(strvalue.split()).astype(float) + if len(value)==1: + value = value[0] + except: + value = strvalue + geom[key]=value + if i==nMax: + break + self['geom']=geom + self['geom']['file']=geomfile + else: + print('[FAIL] Geom file not found (quantites will be pooorly scaled):',geomfile) + self['geom']={'nblade':1, 'refr':3.28084, 'refar':2, 'file':None} + + # --- Try to read element time data file + timefile = os.path.join(parentdir, 'output', basename+'_TimeData.csv') + if os.path.exists(timefile): + df = CSVFile(timefile).toDataFrame() + nBlades = list(df.columns).count('Blade Fx Coeff. (-)') + self['geom']['nblade'] = list(df.columns).count('Blade Fx Coeff. (-)') + cols=list(df.columns[:8]) + bldCols=['Blade{:d} Fx Coeff. (-)', 'Blade{:d} Fy Coeff. (-)', 'Blade{:d} Fz Coeff. (-)', 'Blade{:d} Torque Coeff. (-)'] + for ib in range(self['geom']['nblade']): + cols+=[b.format(ib+1) for b in bldCols] + df.columns=cols + self['dfTime']=df + + else: + self['dfTime']=None + print('TimeData file not found:',timefile) + + # --- Try to read element data file + elemfile = os.path.join(parentdir, 'output', basename+'_ElementData.csv') + if os.path.exists(elemfile): + dfElem = CSVFile(elemfile).toDataFrame() + self['dfElem'] = dfElem + else: + self['dfElem'] = None + print('ElementData file not found:',elemfile) + + + # --- Read DS file + dsfile = os.path.join(parentdir, 'output', basename+'_DSData.csv') + try: + dfDS =CSVFile(dsfile).toDataFrame() + self['dfDS'] = dfDS + except (FileNotFoundError, EmptyFileError): + self['dfDS'] = None + print('DSData file not found or empty:',dsfile) + + + @property + def omega(self): + return self['caseinputs']['rpm']*2*np.pi/60 + + @property + def TSR(self): + return self['caseinputs']['ut'] + + @property + def RPM(self): + return self['caseinputs']['rpm'] + + @property + def dt(self): + nRot = self['configinputs']['nr'] + nPerRot = self['configinputs']['nti'] + T = 2*np.pi/(self.omega) + return T/nPerRot + + @property + def R(self): + if self['geom']['file'] is not None: + R = self['geom']['refr']/3.28084 # feet to m + else: + R=1 + return R + + @property + def A(self): + # NOTE: Turbine reference area (for force/torque/power normalization) divided by reference radius squared. + if self['geom']['refar'] is not None: + #A = self['geom']['refar']/(3.28084**2) # feet^2 to m^2 + A = self['geom']['refar']*self['geom']['refr']**2 + A /=(3.28084**2) # feet^2 to m^2 + else: + A = (2*self.R)**2 # D^2 + return A + + @property + def U(self): + return self.omega*self.R/self.TSR + + + @property + def time(self): + nRot = self['configinputs']['nr'] + nPerRot = self['configinputs']['nti'] + timeSteps = np.arange(0,nRot*nPerRot) + T = 2*np.pi/(self.omega) + return timeSteps*self.dt + + + def timeDataToOpenFAST(self, df): + """ Convert to similar labels as OpenFAST""" + if df is None: + return None + nRot = self['configinputs']['nr'] + nPerRot = self['configinputs']['nti'] + TSR = self.TSR + CTExcrM = self['caseinputs']['ctexcrm'] + rho = self['caseinputs']['rho']*1.2/0.0023280000 + + time = self.time + if df.shape[0]>> Inconsistent shape, ',iB,ie) + else: + # TODO x/y/R + uix_g=dfSec['IndU (-)'].values*U + uiy_g=dfSec['IndV (-)'].values*U + uiz_g=dfSec['IndW (-)'].values*U + + uix=-(np.cos(psi)*uiz_g + np.sin(psi)*uix_g) + uiy= (np.cos(psi)*uix_g - np.sin(psi)*uiz_g) + + df.insert(c , 'AB{:d}N{:03d}Alpha_[deg]'.format(iB,ie) , alphaSign*dfSec['AOA25 (deg)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}Alpha50_[deg]'.format(iB,ie) , alphaSign*dfSec['AOA50 (deg)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}Alpha75_[deg]'.format(iB,ie) , alphaSign*dfSec['AOA75 (deg)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}Cl_[-]' .format(iB,ie) , alphaSign*dfSec['CL (-)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}Cd_[-]' .format(iB,ie) , dfSec['CD (-)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}Cm_[-]' .format(iB,ie) , alphaSign*dfSec['CM25 (-)'].values); c+=1 + + #BladeElemOutData(BladeElemOutRow,24)=CN ! Element normal force coefficient (per span) based on local chord and flow velocity + #BladeElemOutData(BladeElemOutRow,25)=CT ! Element tangential force coefficient (per span) based on local chord and flow velocity + df.insert(c , 'AB{:d}N{:03d}Cn_[-]' .format(iB,ie) , alphaSign*dfSec['CN (-)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}Ct_[-]' .format(iB,ie) , alphaSign*dfSec['CT (-)'].values); c+=1 + # CT=-CL5*sin(alpha25)+CD5*cos(alpha25) + # CT=-CL5*sin(alpha50)+CD5*cos(alpha50) + Cl= dfSec['CL (-)'].values; Cd= dfSec['CD (-)'].values; alpha= dfSec['AOA25 (deg)'].values*np.pi/180 + df.insert(c , 'AB{:d}N{:03d}Ct2_[-]' .format(iB,ie) , alphaSign*(-Cl*np.sin(alpha) + Cd*np.cos(alpha))); c+=1 + + df.insert(c , 'AB{:d}N{:03d}Cxg_[-]' .format(iB,ie) , dfSec['Fx (-)'].values); c+=1 # TODO, this is likely coefficients related to global coords + df.insert(c , 'AB{:d}N{:03d}Cyg_[-]' .format(iB,ie) , - dfSec['Fz (-)'].values); c+=1 # TODO + df.insert(c , 'AB{:d}N{:03d}Czg_[-]' .format(iB,ie) , dfSec['Fy (-)'].values); c+=1 # TODO + df.insert(c , 'AB{:d}N{:03d}ClC_[-]' .format(iB,ie) , alphaSign*dfSec['CLCirc (-)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}Re_[-]' .format(iB,ie) , dfSec['Re (-)'].values/1e6); c+=1 + df.insert(c , 'AB{:d}N{:03d}Gam_[m^2/s]'.format(iB,ie) , alphaSign*dfSec['GB (?)'].values*U*R); c+=1 # TODO + df.insert(c , 'AB{:d}N{:03d}Vrel_[m/s]' .format(iB,ie) , dfSec['Ur (-)'].values*U); c+=1 + df.insert(c , 'AB{:d}N{:03d}Vindx_[m/s]'.format(iB,ie) , uix ); c+=1 # TODO + df.insert(c , 'AB{:d}N{:03d}Vindy_[m/s]'.format(iB,ie) , uiy ); c+=1 # TODO + + if dfDS is not None: + dfSecDS = dfBld_DS[dfBld_DS['Element']==ie] + df.insert(c , 'AB{:d}N{:03d}alpha_34_[deg]'.format(iB,ie) , alphaSign*dfSecDS['alpha (deg)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}alphaE_[deg]'.format(iB,ie) , alphaSign*dfSecDS['alrefL (deg)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}alphaED_[deg]'.format(iB,ie) , alphaSign*dfSecDS['alrefD (deg)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}adotnorm_[-]'.format(iB,ie) , alphaSign*dfSecDS['adotnorm (-)'].values); c+=1 + #df.insert(c , 'AB{:d}N{:03d}AlphaDot_[-]'.format(iB,ie) , alphaSign*dfSec['AdotNorm (-)'].values); c+=1 # TODO + try: + df.insert(c , 'AB{:d}N{:03d}activeL_[-]'.format(iB,ie) , dfSecDS['DynamicFlagL'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}activeD_[-]'.format(iB,ie) , dfSecDS['DynamicFlagD'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}alphaLagD_[deg]'.format(iB,ie), alphaSign*dfSecDS['alphaLagD (deg)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}delP_[-]'.format(iB,ie) , dfSecDS['delN'].values); c+=1 # NOTE SWAPPING N AND P!!!! + df.insert(c , 'AB{:d}N{:03d}delN_[-]'.format(iB,ie) , dfSecDS['delP'].values); c+=1 # NOTE SWAPPING N AND P!!!! + df.insert(c , 'AB{:d}N{:03d}transA_[-]'.format(iB,ie) , dfSecDS['transA'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}gammaL_[-]'.format(iB,ie) , dfSecDS['gammaL'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}gammaD_[-]'.format(iB,ie) , dfSecDS['gammaD'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}dalphaL_[deg]'.format(iB,ie) , alphaSign*dfSecDS['dalphaL (deg)'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}dalphaD_[deg]'.format(iB,ie) , alphaSign*(dfSecDS['alpha (deg)'].values -dfSecDS['alrefD (deg)'].values)) + df.insert(c , 'AB{:d}N{:03d}Tu_[s]'.format(iB,ie) , dfSecDS['Tu'].values); c+=1 # TODO TODO WRONG + df.insert(c , 'AB{:d}N{:03d}alphaDot_[rad/s]'.format(iB,ie),alphaSign*dfSecDS['alphadot'].values); c+=1 + df.insert(c , 'AB{:d}N{:03d}alphaDot2_[rad/s]'.format(iB,ie),np.concatenate(([0],np.diff(-dfSecDS['alpha (deg)'].values))))*np.pi/180; c+=1 # TODO TODO WRONG + except: + pass + return df,c + + + def _write(self): + """ """ + with open(self.filename,'w') as f: + f.write(self.toString) + + def toDataFrame(self, format='OpenFAST', alphaSign=-1): + # --- + df,c = self.timeDataToOpenFAST(df = self['dfTime']) + df,c = self.elemDataToOpenFAST(dfElem=self['dfElem'], df=df, c=c, dfDS=self['dfDS'], alphaSign=alphaSign) + return df + + + def toString(self): + s='' + return s + + def __repr__(self): + s ='Class XXXX (attributes: data)\n' + return s + + diff --git a/pydatview/io/csv_file.py b/pydatview/io/csv_file.py new file mode 100644 index 0000000..1b45578 --- /dev/null +++ b/pydatview/io/csv_file.py @@ -0,0 +1,292 @@ +from __future__ import division,unicode_literals,print_function,absolute_import +from builtins import map, range, chr, str +from io import open +from future import standard_library +standard_library.install_aliases() +import os + +from .file import File, WrongFormatError +import pandas as pd + +class CSVFile(File): + """ + Read/write a CSV file. + + Main methods + ------------ + read, write, toDataFrame + + Examples + -------- + + # Read a csv file and convert it to a pandas dataframe + f = CSVFile('test.csv') + df = f.toDataFrame() + + """ + + @staticmethod + def defaultExtensions(): + return ['.csv','.txt'] + + @staticmethod + def formatName(): + return 'CSV file' + + def __init__(self, filename=None, sep=None, colNames=None, commentChar=None, commentLines=None,\ + colNamesLine=None, detectColumnNames=True, header=None, **kwargs): + colNames = [] if colNames is None else colNames + commentLines = [] if commentLines is None else commentLines + self.sep = sep + self.colNames = colNames + self.commentChar = commentChar + self.commentLines = commentLines + self.colNamesLine = colNamesLine + self.detectColumnNames = detectColumnNames + self.data=[] + if header is None: + self.header=[] + else: + if not hasattr(header, '__len__'): + self.header=[header] + else: + self.header=header + self.nHeader=0 + if (len(self.commentLines)>0) and (self.commentChar is not None): + raise Exception('Provide either `commentChar` or `commentLines` for CSV file types') + if (len(self.colNames)>0) and (self.colNamesLine is not None): + raise Exception('Provide either `colNames` or `colNamesLine` for CSV file types') + super(CSVFile, self).__init__(filename=filename,**kwargs) + + def _read(self): + COMMENT_CHAR=['#','!',';'] + # --- Detecting encoding + # NOTE: done by parent class method + + # --- Subfunctions + def readFirstLines(nLines): + lines=[] + with open(self.filename, 'r', encoding=self.encoding, errors="surrogateescape") as fid: + for i, line in enumerate(fid): + lines.append(line.strip()) + if i==nLines: + break + return lines + + def readline(iLine): + with open(self.filename,'r',encoding=self.encoding) as f: + for i, line in enumerate(f): + if i==iLine: + return line.strip() + elif i>iLine: + break + def split(s): + if s is None: + return [] + if self.sep==r'\s+': + return s.strip().split() + else: + return [c.strip() for c in s.strip().split(self.sep)] + def strIsFloat(s): + try: + float(s) + return True + except: + return False + # --- Safety + if self.sep=='' or self.sep==' ': + self.sep=r'\s+' + + iStartLine=0 + + # --- Exclude some files from the CSV reader --- + line=readline(iStartLine) + words=line.split() + if len(words)>1: + try: + int(words[0]) + word0int = True + except: + word0int = False + if word0int and words[1].isalpha(): + raise WrongFormatError('Input File {}: '.format(self.filename) + 'is not likely a CSV file' ) + + # --- Headers (i.e. comments) + # TODO: read few headers lines instead of multiple read below.. + + self.header = [] + if len(self.commentLines)>0: + # We read the lines + with open(self.filename,'r',encoding=self.encoding) as f: + for i in range(max(self.commentLines)+1): + l = f.readline() + if i in self.commentLines: + self.header.append(l.strip()) + elif self.commentChar is not None: + # we detect the comments lines that start with comment char + with open(self.filename,'r',encoding=self.encoding) as f: + n=0 + while n<100: + l = f.readline().strip() + if (not l) or (l+'_dummy')[0] != self.commentChar[0]: + break + self.header.append(l.strip()) + n+=1 + self.commentLines=list(range(len(self.header))) + else: + # We still believe that some characters are comments + line=readline(iStartLine) + line=str(line).strip() + if len(line)>0 and line[0] in COMMENT_CHAR: + self.commentChar=line[0] + # Nasty copy paste from above + with open(self.filename,'r',encoding=self.encoding) as f: + n=0 + while n<100: + l = f.readline().strip() + if (not l) or (l+'_dummy')[0] != self.commentChar[0]: + break + self.header.append(l.strip()) + n+=1 + + iStartLine = len(self.header) + + # --- File separator + if self.sep is None: + # Detecting separator by reading first lines of the file + try: + with open(self.filename,'r',encoding=self.encoding) as f: + dummy=[next(f).strip() for x in range(iStartLine)] + head=[next(f).strip() for x in range(2)] + # comma, semi columns or tab + if head[1].find(',')>0: + self.sep=',' + elif head[1].find(';')>0: + self.sep=';' + elif head[1].find('\t')>0: + self.sep='\t' + else: + self.sep=r'\s+' + except: + # most likely an empty file + pass + + # --- ColumnNames + if self.colNamesLine is not None: + if self.colNamesLine<0: + # The column names are hidden somwhere in the header + line=readline(iStartLine+self.colNamesLine).strip() + # Removing comment if present (should be present..) + if self.commentChar is not None: + if line.find(self.commentChar)==0: + line=line[len(self.commentChar):].strip() + self.colNames = split(line) + else: + line=readline(self.colNamesLine) + self.colNames=split(line) + iStartLine = max(iStartLine,self.colNamesLine+1) + elif len(self.colNames)>0: + pass + elif not self.detectColumnNames: + pass + else: + # Looking at first line of data, if mainly floats -> it's not the column names + colNames = split(readline(iStartLine)) + nFloat = sum([strIsFloat(s) for s in colNames]) + if nFloat ==0 or (len(colNames)>2 and nFloat <= len(colNames)/2): + # We assume that the line contains the column names + self.colNames=colNames + self.colNamesLine = iStartLine + iStartLine = iStartLine+1 + # --- Now, maybe the user has put some units below + first_line = readline(iStartLine) + #print('>>> first line',first_line) + first_cols = split(first_line) + nFloat = sum([strIsFloat(s) for s in first_cols]) + nPa = first_line.count('(')+first_line.count('[') + #if nFloat == 0 or nPa>len(self.colNames)/2: + if nPa>len(self.colNames)/2: + # that's definitely some units + if len(first_cols)==len(self.colNames): + self.colNames=[c.strip()+'_'+u.strip() for c,u in zip(self.colNames, first_cols)] + iStartLine = iStartLine+1 + elif len(self.header)>0: + # Maybe the columns names are in the header + if self.sep is not None: + first_line = readline(iStartLine) + first_cols = split(first_line) + #print('CommentChar:',self.commentChar) + #print('First line:',first_line) + #print('First col :',first_cols) + for l in self.header: + if self.commentChar is not None: + if len(self.commentChar)>0: + l=l[len(self.commentChar):] + cols=split(l) + nFloat = sum([strIsFloat(s) for s in cols]) + if len(cols)==len(first_cols) and nFloat <= len(colNames)-1: + self.colNames = cols + break + # --- Reading data + skiprows = list(range(iStartLine)) + if (self.colNamesLine is not None): + skiprows.append(self.colNamesLine) + if (self.commentLines is not None) and len(self.commentLines)>0: + skiprows = skiprows + self.commentLines + skiprows =list(sorted(set(skiprows))) + if self.sep is not None: + if self.sep=='\t': + self.sep=r'\s+' + #print(skiprows) + try: +# self.data = pd.read_csv(self.filename,sep=self.sep,skiprows=skiprows,header=None,comment=self.commentChar,encoding=self.encoding) + with open(self.filename,'r',encoding=self.encoding) as f: + self.data = pd.read_csv(f,sep=self.sep,skiprows=skiprows,header=None,comment=self.commentChar) + except pd.errors.ParserError as e: + raise WrongFormatError('CSV File {}: '.format(self.filename)+e.args[0]) + + if (len(self.colNames)==0) or (len(self.colNames)!=len(self.data.columns)): + self.colNames=['C{}'.format(i) for i in range(len(self.data.columns))] + self.data.columns = self.colNames; + self.data.rename(columns=lambda x: x.strip(),inplace=True) + #import pdb + #pdb.set_trace() + + def _write(self): + # --- Safety + if self.sep==r'\s+' or self.sep=='': + self.sep='\t' + # Write + if len(self.header)>0: + with open(self.filename, 'w', encoding='utf-8') as f: + f.write('\n'.join(self.header)+'\n') + with open(self.filename, 'a', encoding='utf-8') as f: + try: + self.data.to_csv(f, sep=self.sep, index=False,header=False, line_terminator='\n') + except TypeError: + print('[WARN] CSVFile: Pandas failed, likely encoding error. Attempting a quick and dirty fix.') + s='' + vals=self.data.values + for l in vals: + sLine=(self.sep).join([str(v) for v in l]) + s+=sLine+'\n' + f.write(s) + else: + self.data.to_csv(self.filename,sep=self.sep,index=False) + + def __repr__(self): + s = 'CSVFile: {}\n'.format(self.filename) + s += 'sep=`{}` commentChar=`{}`\ncolNamesLine={}'.format(self.sep,self.commentChar,self.colNamesLine) + s += ', encoding={}'.format(self.encoding)+'\n' + s += 'commentLines={}'.format(self.commentLines)+'\n' + s += 'colNames={}'.format(self.colNames) + s += '\n' + if len(self.header)>0: + s += 'header:\n'+ '\n'.join(self.header)+'\n' + if len(self.data)>0: + s += 'size: {}x{}'.format(len(self.data),len(self.data.columns)) + return s + + def _toDataFrame(self): + return self.data + diff --git a/pydatview/io/excel_file.py b/pydatview/io/excel_file.py new file mode 100644 index 0000000..df28931 --- /dev/null +++ b/pydatview/io/excel_file.py @@ -0,0 +1,88 @@ +from __future__ import division,unicode_literals,print_function,absolute_import +from builtins import map, range, chr, str +from io import open +from future import standard_library +standard_library.install_aliases() + +from .file import File, WrongFormatError, BrokenFormatError +import numpy as np +import pandas as pd + +# from pandas import ExcelWriter +from pandas import ExcelFile + +class ExcelFile(File): + + @staticmethod + def defaultExtensions(): + return ['.xls','.xlsx'] + + @staticmethod + def formatName(): + return 'Excel file' + + def _read(self): + self.data=dict() + # Reading all sheets + try: + xls = pd.ExcelFile(self.filename, engine='openpyxl') + except: + xls = pd.ExcelFile(self.filename) + dfs = {} + for sheet_name in xls.sheet_names: + # Reading sheet + df = xls.parse(sheet_name, header=None) + # TODO detect sub tables + # Dropping empty rows and cols + df.dropna(how='all',axis=0,inplace=True) + df.dropna(how='all',axis=1,inplace=True) + #print(df.shape) + if df.shape[0]>0: + # Setting first row as header + df=df.rename(columns=df.iloc[0]).drop(df.index[0]).reset_index(drop=True) + #print(df) + self.data[sheet_name]=df + + #def toString(self): + # s='' + # return s + + def _write(self): + # Create a Pandas Excel writer using XlsxWriter as the engine. + writer = pd.ExcelWriter(self.filename, engine='xlsxwriter') + # Convert the dataframe to an XlsxWriter Excel object. + for k,_ in self.data.items(): + df = self.data[k] + df.to_excel(writer, sheet_name=k, index=False) + # # Account info columns (set size) + # worksheet.set_column('B:D', 20) + # # Total formatting + # total_fmt = workbook.add_format({'align': 'right', 'num_format': '$#,##0', + # 'bold': True, 'bottom':6}) + # # Total percent format + # total_percent_fmt = workbook.add_format({'align': 'right', 'num_format': '0.0%', + # 'bold': True, 'bottom':6}) + # workbook = writer.book + # worksheet = writer.sheets['report'] + # Highlight the top 5 values in Green + #worksheet.conditional_format(color_range, {'type': 'top', + # 'value': '5', + # 'format': format2}) + ## Highlight the bottom 5 values in Red + #worksheet.conditional_format(color_range, {'type': 'bottom', + # 'value': '5', + # 'format': format1}) + # Close the Pandas Excel writer and output the Excel file. + writer.save() + + def __repr__(self): + s ='Class XXXX (attributes: data)\n' + return s + + + def _toDataFrame(self): + #cols=['Alpha_[deg]','Cl_[-]','Cd_[-]','Cm_[-]'] + #dfs[name] = pd.DataFrame(data=..., columns=cols) + #df=pd.DataFrame(data=,columns=) + return self.data + diff --git a/pydatview/io/fast_input_deck.py b/pydatview/io/fast_input_deck.py new file mode 100644 index 0000000..48531e7 --- /dev/null +++ b/pydatview/io/fast_input_deck.py @@ -0,0 +1,442 @@ +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from io import open +from builtins import range +from builtins import str +from future import standard_library +standard_library.install_aliases() +import os +import numpy as np +import re +import pandas as pd + +from .fast_input_file import FASTInputFile + +__all__ = ['FASTInputDeck'] +# --------------------------------------------------------------------------------} +# --- Full FAST input deck +# --------------------------------------------------------------------------------{ +class FASTInputDeck(dict): + """Container for input files that make up a FAST input deck""" + + def __init__(self, fullFstPath='', readlist=['all'], verbose=False): + """Read FAST master file and read inputs for FAST modules + + INPUTS: + - fullFstPath: + - readlist: list of module files to be read, or ['all'], modules are identified as follows: + ['Fst','ED','AD','BD','BDbld','EDtwr','EDbld','ADbld','AF','AC','IW','HD','SrvD','SD','MD'] + where: + AF: airfoil polars + AC: airfoil coordinates (if present) + + """ + # Sanity + if type(verbose) is not bool: + raise Exception('`verbose` arguments needs to be a boolean') + + self.filename = fullFstPath + self.verbose = verbose + self.readlist = readlist + if not type(self.readlist) is list: + self.readlist=[readlist] + if 'all' in self.readlist: + self.readlist = ['Fst','ED','AD','BD','BDbld','EDtwr','EDbld','ADbld','AF','AC','IW','HD','SrvD','SD','MD'] + else: + self.readlist = ['Fst']+self.readlist + + self.inputfiles = {} + + # --- Harmonization with AeroElasticSE + self.FAST_ver = 'OPENFAST' + self.path2dll = None # Path to dll file + + self.fst_vt={} + self.fst_vt['description'] = '' + self.fst_vt['Fst'] = None + self.fst_vt['ElastoDyn'] = None + self.fst_vt['ElastoDynBlade'] = None + self.fst_vt['ElastoDynTower'] = None + self.fst_vt['InflowWind'] = None + self.fst_vt['AeroDyn14'] = None + self.fst_vt['AeroDyn15'] = None + self.fst_vt['AeroDynBlade'] = None + self.fst_vt['AeroDynTower'] = None + self.fst_vt['AeroDynPolar'] = None + self.fst_vt['ServoDyn'] = None + self.fst_vt['DISCON_in'] = None + self.fst_vt['HydroDyn'] = None + self.fst_vt['MoorDyn'] = None + self.fst_vt['SubDyn'] = None + self.fst_vt['MAP'] = None + self.fst_vt['BeamDyn'] = None + self.fst_vt['BeamDynBlade'] = None # Small change of interface + self.fst_vt['af_data'] = [] # Small change of interface + self.fst_vt['ac_data'] = [] # TODO, how is it stored in WEIS? + + + self.ADversion='' + + # Read all inputs files + if len(fullFstPath)>0: + self.read() + + + def readAD(self, filename=None, readlist=None, verbose=False, key='AeroDyn15'): + """ + readlist: 'AD','AF','AC' + """ + if readlist is not None: + readlist_bkp = self.readlist + self.readlist=readlist + if not type(self.readlist) is list: + self.readlist=[readlist] + if 'all' in self.readlist: + self.readlist = ['Fst','ED','AD','BD','BDbld','EDtwr','EDbld','ADbld','AF','AC','IW','HD','SrvD','SD','MD'] + + if filename is None: + filename = self.fst_vt['Fst']['AeroFile'] + baseDir = os.path.dirname(self.fst_vt['Fst']['AeroFile']) + else: + baseDir = os.path.dirname(filename) + + self.verbose = verbose + + self.fst_vt[key] = self._read(filename,'AD') + + if self.fst_vt[key] is not None: + # Blades + bld_file = os.path.join(baseDir, self.fst_vt[key]['ADBlFile(1)']) + self.fst_vt['AeroDynBlade'] = self._read(bld_file,'ADbld') + #self.fst_vt['AeroDynBlade'] = [] + #for i in range(3): + # bld_file = os.path.join(os.path.dirname(self.fst_vt['Fst']['AeroFile']), self.fst_vt[key]['ADBlFile({})'.format(i+1)]) + # self.fst_vt['AeroDynBlade'].append(self._read(bld_file,'ADbld')) + # Polars + self.fst_vt['af_data']=[] # TODO add to "AeroDyn" + for afi, af_filename in enumerate(self.fst_vt['AeroDyn15']['AFNames']): + af_filename = os.path.join(baseDir,af_filename).replace('"','') + try: + polar = self._read(af_filename, 'AF') + except: + polar=None + print('[FAIL] reading polar {}'.format(af_filename)) + self.fst_vt['af_data'].append(polar) + if polar is not None: + coordFile = polar['NumCoords'] + if isinstance(coordFile,str): + coordFile = coordFile.replace('"','') + baseDirCoord=os.path.dirname(af_filename) + if coordFile[0]=='@': + ac_filename = os.path.join(baseDirCoord,coordFile[1:]) + coords = self._read(ac_filename, 'AC') + self.fst_vt['ac_data'].append(coords) + + # --- Backward compatibility + self.AD = self.fst_vt[key] + self.ADversion='AD15' if key=='AeroDyn15' else 'AD14' + + if readlist is not None: + self.readlist=readlist_bkp + + @property + def FAST_InputFile(self): + return os.path.basename(self.filename) # FAST input file (ext=.fst) + @property + def FAST_directory(self): + return os.path.dirname(self.filename) # Path to fst directory files + + + @property + def inputFiles(self): + files=[] + files+=[self.ED_path, self.ED_twr_path, self.ED_bld_path] + files+=[self.BD_path, self.BD_bld_path] + return [f for f in files if f not in self.unusedNames] + + + @property + def ED_relpath(self): + try: + return self.fst_vt['Fst']['EDFile'].replace('"','') + except: + return 'none' + + @property + def ED_twr_relpath(self): + try: + return os.path.join(os.path.dirname(self.fst_vt['Fst']['EDFile']).replace('"',''), self.fst_vt['ElastoDyn']['TwrFile'].replace('"','')) + except: + return 'none' + + @property + def ED_bld_relpath(self): + try: + if 'BldFile(1)' in self.fst_vt['ElastoDyn'].keys(): + return os.path.join(os.path.dirname(self.fst_vt['Fst']['EDFile'].replace('"','')), self.fst_vt['ElastoDyn']['BldFile(1)'].replace('"','')) + else: + return os.path.join(os.path.dirname(self.fst_vt['Fst']['EDFile'].replace('"','')), self.fst_vt['ElastoDyn']['BldFile1'].replace('"','')) + except: + return 'none' + + @property + def BD_relpath(self): + try: + return self.fst_vt['Fst']['BDBldFile(1)'].replace('"','') + except: + return 'none' + + @property + def BD_bld_relpath(self): + try: + return os.path.join(os.path.dirname(self.fst_vt['Fst']['BDBldFile(1)'].replace('"','')), self.fst_vt['BeamDyn']['BldFile'].replace('"','')) + except: + return 'none' + + @property + def ED_path(self): return self._fullpath(self.ED_relpath) + @property + def BD_path(self): return self._fullpath(self.BD_relpath) + @property + def BD_bld_path(self): return self._fullpath(self.BD_bld_relpath) + @property + def ED_twr_path(self): return self._fullpath(self.ED_twr_relpath) + @property + def ED_bld_path(self): return self._fullpath(self.ED_bld_relpath) + + + + def _fullpath(self, relfilepath): + relfilepath = relfilepath.replace('"','') + basename = os.path.basename(relfilepath) + if basename.lower() in self.unusedNames: + return 'none' + else: + return os.path.join(self.FAST_directory, relfilepath) + + + def read(self, filename=None): + if filename is not None: + self.filename = filename + + # Read OpenFAST files + self.fst_vt['Fst'] = self._read(self.FAST_InputFile, 'Fst') + if self.fst_vt['Fst'] is None: + raise Exception('Error reading main file {}'.format(self.filename)) + keys = self.fst_vt['Fst'].keys() + + + if 'NumTurbines' in keys: + self.version='AD_driver' + elif 'InterpOrder' in self.fst_vt['Fst'].keys(): + self.version='OF2' + else: + self.version='F7' + + + if self.version=='AD_driver': + # ---- AD Driver + # InflowWind + if self.fst_vt['Fst']['CompInflow']>0: + self.fst_vt['InflowWind'] = self._read(self.fst_vt['Fst']['InflowFile'],'IW') + + self.readAD(key='AeroDyn15') + + elif self.version=='OF2': + # ---- Regular OpenFAST file + # ElastoDyn + if 'EDFile' in self.fst_vt['Fst'].keys(): + self.fst_vt['ElastoDyn'] = self._read(self.fst_vt['Fst']['EDFile'],'ED') + if self.fst_vt['ElastoDyn'] is not None: + twr_file = self.ED_twr_relpath + bld_file = self.ED_bld_relpath + self.fst_vt['ElastoDynTower'] = self._read(twr_file,'EDtwr') + self.fst_vt['ElastoDynBlade'] = self._read(bld_file,'EDbld') + + # InflowWind + if self.fst_vt['Fst']['CompInflow']>0: + self.fst_vt['InflowWind'] = self._read(self.fst_vt['Fst']['InflowFile'],'IW') + + # AeroDyn + if self.fst_vt['Fst']['CompAero']>0: + key = 'AeroDyn14' if self.fst_vt['Fst']['CompAero']==1 else 'AeroDyn15' + self.readAD(key=key, readlist=self.readlist) + + # ServoDyn + if self.fst_vt['Fst']['CompServo']>0: + self.fst_vt['ServoDyn'] = self._read(self.fst_vt['Fst']['ServoFile'],'SrvD') + # TODO Discon + + # HydroDyn + if self.fst_vt['Fst']['CompHydro']== 1: + self.fst_vt['HydroDyn'] = self._read(self.fst_vt['Fst']['HydroFile'],'HD') + + # SubDyn + if self.fst_vt['Fst']['CompSub'] == 1: + self.fst_vt['SubDyn'] = self._read(self.fst_vt['Fst']['SubFile'],'HD') + + # Mooring + if self.fst_vt['Fst']['CompMooring']==1: + self.fst_vt['MAP'] = self._read(self.fst_vt['Fst']['MooringFile'],'MD') + if self.fst_vt['Fst']['CompMooring']==2: + self.fst_vt['MoorDyn'] = self._read(self.fst_vt['Fst']['MooringFile'],'MD') + + # BeamDyn + if self.fst_vt['Fst']['CompElast'] == 2: + self.fst_vt['BeamDyn'] = self._read(self.fst_vt['Fst']['BDBldFile(1)'],'BD') + if self.fst_vt['BeamDyn'] is not None: + # Blades + bld_file = os.path.join(os.path.dirname(self.fst_vt['Fst']['BDBldFile(1)']), self.fst_vt['BeamDyn']['BldFile']) + self.fst_vt['BeamDynBlade']= self._read(bld_file,'BDbld') + + # --- Backward compatibility + self.fst = self.fst_vt['Fst'] + self.ED = self.fst_vt['ElastoDyn'] + if not hasattr(self,'AD'): + self.AD = None + if self.AD is not None: + self.AD.Bld1 = self.fst_vt['AeroDynBlade'] + self.AD.AF = self.fst_vt['af_data'] + self.IW = self.fst_vt['InflowWind'] + self.BD = self.fst_vt['BeamDyn'] + self.BDbld = self.fst_vt['BeamDynBlade'] + + @ property + def unusedNames(self): + return ['unused','nan','na','none'] + + def _read(self, relfilepath, shortkey): + """ read any openfast input """ + relfilepath =relfilepath.replace('"','') + basename = os.path.basename(relfilepath) + + # Only read what the user requested to be read + if shortkey not in self.readlist: + if self.verbose: + print('>>> Skipping ',shortkey) + return None + + # Skip "unused" and "NA" + if basename.lower() in self.unusedNames: + if self.verbose: + print('>>> Unused ',shortkey) + return None + + # Attempt reading + fullpath =os.path.join(self.FAST_directory, relfilepath) + try: + data = FASTInputFile(fullpath) + if self.verbose: + print('>>> Read: ',fullpath) + self.inputfiles[shortkey] = fullpath + return data + except FileNotFoundError: + print('[WARN] File not found '+fullpath) + return None + + + + def write(self, filename=None, prefix='', suffix='', directory=None): + """ Write a standardized input file deck""" + if filename is None: + filename=self.filename # Overwritting + self.filename=filename + if directory is None: + directory = os.path.dirname(filename) + basename = os.path.splitext(os.path.basename(filename))[0] + + + fst = self.fst_vt['Fst'] + + # Filenames + filename_ED = os.path.join(directory,prefix+'ED'+suffix+'.dat') if fst['CompElast']>0 else 'none' + filename_IW = os.path.join(directory,prefix+'IW'+suffix+'.dat') if fst['CompInflow']>0 else 'none' + filename_BD = os.path.join(directory,prefix+'BD'+suffix+'.dat') if fst['CompElast']==2 else 'none' + filename_AD = os.path.join(directory,prefix+'AD'+suffix+'.dat') if fst['CompAero']>0 else 'none' + filename_HD = os.path.join(directory,prefix+'HD'+suffix+'.dat') if fst['CompHydro']>0 else 'none' + filename_SD = os.path.join(directory,prefix+'SD'+suffix+'.dat') if fst['CompSub']>0 else 'none' + filename_MD = os.path.join(directory,prefix+'MD'+suffix+'.dat') if fst['CompMooring']>0 else 'none' + filename_SvD = os.path.join(directory,prefix+'SvD'+suffix+'.dat') if fst['CompServo']>0 else 'none' + filename_Ice = os.path.join(directory,prefix+'Ice'+suffix+'.dat') if fst['CompIce']>0 else 'none' + filename_ED_bld = os.path.join(directory,prefix+'ED_bld'+suffix+'.dat') if fst['CompElast']>0 else 'none' + filename_ED_twr = os.path.join(directory,prefix+'ED_twr'+suffix+'.dat') if fst['CompElast']>0 else 'none' + filename_BD_bld = os.path.join(directory,prefix+'BD_bld'+suffix+'.dat') if fst['CompElast']>0 else 'none' + # TODO AD Profiles and OLAF + + fst['EDFile'] = '"' + os.path.basename(filename_ED) + '"' + fst['BDBldFile(1)'] = '"' + os.path.basename(filename_BD) + '"' + fst['BDBldFile(2)'] = '"' + os.path.basename(filename_BD) + '"' + fst['BDBldFile(3)'] = '"' + os.path.basename(filename_BD) + '"' + fst['InflowFile'] = '"' + os.path.basename(filename_IW) + '"' + fst['AeroFile'] = '"' + os.path.basename(filename_AD) + '"' + fst['ServoFile'] = '"' + os.path.basename(filename_AD) + '"' + fst['HydroFile'] = '"' + os.path.basename(filename_HD) + '"' + fst['SubFile'] = '"' + os.path.basename(filename_SD) + '"' + fst['MooringFile'] = '"' + os.path.basename(filename_MD) + '"' + fst['IceFile'] = '"' + os.path.basename(filename_Ice)+ '"' + fst.write(filename) + + + ED = self.fst_vt['ElastoDyn'] + if fst['CompElast']>0: + ED['TwrFile'] = '"' + os.path.basename(filename_ED_twr)+ '"' + self.fst_vt['ElastoDynTower'].write(filename_ED_twr) + if fst['CompElast']==1: + if 'BldFile1' in ED.keys(): + ED['BldFile1'] = '"' + os.path.basename(filename_ED_bld)+ '"' + ED['BldFile2'] = '"' + os.path.basename(filename_ED_bld)+ '"' + ED['BldFile3'] = '"' + os.path.basename(filename_ED_bld)+ '"' + else: + ED['BldFile(1)'] = '"' + os.path.basename(filename_ED_bld)+ '"' + ED['BldFile(2)'] = '"' + os.path.basename(filename_ED_bld)+ '"' + ED['BldFile(3)'] = '"' + os.path.basename(filename_ED_bld)+ '"' + self.fst_vt['ElastoDynBlade'].write(filename_ED_bld) + + elif fst['CompElast']==2: + BD = self.fst_vt['BeamDyn'] + BD['BldFile'] = '"'+os.path.basename(filename_BD_bld)+'"' + self.fst_vt['BeamDynBlade'].write(filename_BD_bld) # TODO TODO pick up the proper blade file! + BD.write(filename_BD) + ED.write(filename_ED) + + + if fst['CompInflow']>0: + self.fst_vt['InflowWind'].write(filename_IW) + + if fst['CompAero']>0: + self.fst_vt['AeroDyn15'].write(filename_AD) + # TODO other files + + if fst['CompServo']>0: + self.fst_vt['ServoDyn'].write(filename_SvD) + + if fst['CompHydro']==1: + self.fst_vt['HydroDyn'].write(filename_HD) + + if fst['CompSub']==1: + self.fst_vt['SubDyn'].write(filename_SD) + elif fst['CompSub']==2: + raise NotImplementedError() + + if fst['CompMooring']==1: + self.fst_vt['MAP'].write(filename_MD) + if self.fst_vt['Fst']['CompMooring']==2: + self.fst_vt['MoorDyn'].write(filename_MD) + + + + def __repr__(self): + s=''+'\n' + s+='filename : '+self.filename+'\n' + s+='version : '+self.version+'\n' + s+='AD version : '+self.ADversion+'\n' + s+='fst_vt : dict{'+','.join([k for k,v in self.fst_vt.items() if v is not None])+'}\n' + s+='inputFiles : {}\n'.format(self.inputFiles) + s+='\n' + return s + +if __name__ == "__main__": + fst=FASTInputDeck('NREL5MW.fst') + print(fst) diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py new file mode 100644 index 0000000..a975148 --- /dev/null +++ b/pydatview/io/fast_input_file.py @@ -0,0 +1,1288 @@ +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from io import open +from builtins import range +from builtins import str +from future import standard_library +standard_library.install_aliases() +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + # --- Allowing this file to be standalone.. + class WrongFormatError(Exception): + pass + class BrokenFormatError(Exception): + pass + File = dict +import os +import numpy as np +import re +import pandas as pd + +__all__ = ['FASTInputFile'] + +TABTYPE_NOT_A_TAB = 0 +TABTYPE_NUM_WITH_HEADER = 1 +TABTYPE_NUM_WITH_HEADERCOM = 2 +TABTYPE_NUM_NO_HEADER = 4 +TABTYPE_NUM_BEAMDYN = 5 +TABTYPE_MIX_WITH_HEADER = 6 +TABTYPE_FIL = 3 +TABTYPE_FMT = 9999 # TODO + +# --------------------------------------------------------------------------------} +# --- INPUT FILE +# --------------------------------------------------------------------------------{ +class FASTInputFile(File): + """ + Read/write an OpenFAST input file. The object behaves like a dictionary. + + Main methods + ------------ + - read, write, toDataFrame, keys + + Main keys + --------- + The keys correspond to the keys used in the file. For instance for a .fst file: 'DT','TMax' + + Examples + -------- + + filename = 'AeroDyn.dat' + f = FASTInputFile(filename) + f['TwrAero'] = True + f['AirDens'] = 1.225 + f.write('AeroDyn_Changed.dat') + + """ + + @staticmethod + def defaultExtensions(): + return ['.dat','.fst','.txt','.fstf'] + + @staticmethod + def formatName(): + return 'FAST input file' + + def __init__(self, filename=None, **kwargs): + self._size=None + self._encoding=None + if filename: + self.filename = filename + self.read() + else: + self.filename = None + + def keys(self): + self.labels = [ d['label'] for d in self.data if not d['isComment'] ] + return self.labels + + def getID(self,label): + i=self.getIDSafe(label) + if i<0: + raise KeyError('Variable `'+ label+'` not found in FAST file:'+self.filename) + else: + return i + + def getIDs(self,label): + I=[] + # brute force search + for i in range(len(self.data)): + d = self.data[i] + if d['label'].lower()==label.lower(): + I.append(i) + if len(I)<0: + raise KeyError('Variable `'+ label+'` not found in FAST file:'+self.filename) + else: + return I + + def getIDSafe(self,label): + # brute force search + for i in range(len(self.data)): + d = self.data[i] + if d['label'].lower()==label.lower(): + return i + return -1 + + # Making object an iterator + def __iter__(self): + self.iCurrent=-1 + self.iMax=len(self.data)-1 + return self + + def __next__(self): # Python 2: def next(self) + if self.iCurrent > self.iMax: + raise StopIteration + else: + self.iCurrent += 1 + return self.data[self.iCurrent] + + # Making it behave like a dictionary + def __setitem__(self,key,item): + I = self.getIDs(key) + for i in I: + self.data[i]['value'] = item + + def __getitem__(self,key): + i = self.getID(key) + return self.data[i]['value'] + + def __repr__(self): + s ='Fast input file: {}\n'.format(self.filename) + return s+'\n'.join(['{:15s}: {}'.format(d['label'],d['value']) for i,d in enumerate(self.data)]) + + def addKeyVal(self,key,val,descr=None): + d=getDict() + d['label']=key + d['value']=val + if descr is not None: + d['descr']=descr + self.data.append(d) + + def read(self, filename=None): + if filename: + self.filename = filename + if self.filename: + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + self._read() + else: + raise Exception('No filename provided') + + def _read(self): + + # --- Tables that can be detected based on the "Value" (first entry on line) + # TODO members for BeamDyn with mutliple key point ####### TODO PropSetID is Duplicate SubDyn and used in HydroDyn + NUMTAB_FROM_VAL_DETECT = ['HtFract' , 'TwrElev' , 'BlFract' , 'Genspd_TLU' , 'BlSpn' , 'WndSpeed' , 'HvCoefID' , 'AxCoefID' , 'JointID' , 'Dpth' , 'FillNumM' , 'MGDpth' , 'SimplCd' , 'RNodes' , 'kp_xr' , 'mu1' , 'TwrHtFr' , 'TwrRe' , 'WT_X'] + NUMTAB_FROM_VAL_DIM_VAR = ['NTwInpSt' , 'NumTwrNds' , 'NBlInpSt' , 'DLL_NumTrq' , 'NumBlNds' , 'NumCases' , 'NHvCoef' , 'NAxCoef' , 'NJoints' , 'NCoefDpth' , 'NFillGroups' , 'NMGDepths' , 1 , 'BldNodes' , 'kp_total' , 1 , 'NTwrHt' , 'NTwrRe' , 'NumTurbines'] + NUMTAB_FROM_VAL_VARNAME = ['TowProp' , 'TowProp' , 'BldProp' , 'DLLProp' , 'BldAeroNodes' , 'Cases' , 'HvCoefs' , 'AxCoefs' , 'Joints' , 'DpthProp' , 'FillGroups' , 'MGProp' , 'SmplProp' , 'BldAeroNodes' , 'MemberGeom' , 'DampingCoeffs' , 'TowerProp' , 'TowerRe', 'WindTurbines'] + NUMTAB_FROM_VAL_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 1 , 2 , 2 , 1 , 1 , 2 ] + NUMTAB_FROM_VAL_TYPE = ['num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' , 'num' , 'mix'] + # SubDyn + NUMTAB_FROM_VAL_DETECT += [ 'RJointID' , 'IJointID' , 'COSMID' , 'CMJointID' ] + NUMTAB_FROM_VAL_DIM_VAR += [ 'NReact' , 'NInterf' , 'NCOSMs' , 'NCmass' ] + NUMTAB_FROM_VAL_VARNAME += [ 'BaseJoints' , 'InterfaceJoints' , 'MemberCosineMatrix' , 'ConcentratedMasses'] + NUMTAB_FROM_VAL_NHEADER += [ 2 , 2 , 2 , 2 ] + NUMTAB_FROM_VAL_TYPE += [ 'mix' , 'num' , 'num' , 'num' ] + + + # --- Tables that can be detected based on the "Label" (second entry on line) + # NOTE: MJointID1, used by SubDyn and HydroDyn + NUMTAB_FROM_LAB_DETECT = ['NumAlf' , 'F_X' , 'MemberCd1' , 'MJointID1' , 'NOutLoc' , 'NOutCnt' , 'PropD' ,'Diam' ,'Type' ,'LineType' ] + NUMTAB_FROM_LAB_DIM_VAR = ['NumAlf' , 'NKInpSt' , 'NCoefMembers' , 'NMembers' , 'NMOutputs' , 'NMOutputs' , 'NPropSets' ,'NTypes' ,'NConnects' ,'NLines' ] + NUMTAB_FROM_LAB_VARNAME = ['AFCoeff' , 'TMDspProp' , 'MemberProp' , 'Members' , 'MemberOuts' , 'MemberOuts' , 'SectionProp' ,'LineTypes' ,'ConnectionProp' ,'LineProp' ] + NUMTAB_FROM_LAB_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 ] + NUMTAB_FROM_LAB_NOFFSET = [0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ] + NUMTAB_FROM_LAB_TYPE = ['num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' ,'mix' ,'mix' ,'mix' ] + # SubDyn + NUMTAB_FROM_LAB_DETECT += ['GuyanDampSize' , 'YoungE' , 'YoungE' , 'EA' , 'MatDens' ] + NUMTAB_FROM_LAB_DIM_VAR += [6 , 'NPropSets', 'NXPropSets', 'NCablePropSets' , 'NRigidPropSets'] + NUMTAB_FROM_LAB_VARNAME += ['GuyanDampMatrix' , 'BeamProp' , 'BeamPropX' , 'CableProp' , 'RigidProp' ] + NUMTAB_FROM_LAB_NHEADER += [0 , 2 , 2 , 2 , 2 ] + NUMTAB_FROM_LAB_NOFFSET += [1 , 0 , 0 , 0 , 0 ] + NUMTAB_FROM_LAB_TYPE += ['num' , 'num' , 'num' , 'num' , 'num' ] + # OLAF + NUMTAB_FROM_LAB_DETECT += ['GridName' ] + NUMTAB_FROM_LAB_DIM_VAR += ['nGridOut' ] + NUMTAB_FROM_LAB_VARNAME += ['GridOutputs'] + NUMTAB_FROM_LAB_NHEADER += [0 ] + NUMTAB_FROM_LAB_NOFFSET += [2 ] + NUMTAB_FROM_LAB_TYPE += ['mix' ] + + FILTAB_FROM_LAB_DETECT = ['FoilNm' ,'AFNames'] + FILTAB_FROM_LAB_DIM_VAR = ['NumFoil','NumAFfiles'] + FILTAB_FROM_LAB_VARNAME = ['FoilNm' ,'AFNames'] + + # Using lower case to be more tolerant.. + NUMTAB_FROM_VAL_DETECT_L = [s.lower() for s in NUMTAB_FROM_VAL_DETECT] + NUMTAB_FROM_LAB_DETECT_L = [s.lower() for s in NUMTAB_FROM_LAB_DETECT] + FILTAB_FROM_LAB_DETECT_L = [s.lower() for s in FILTAB_FROM_LAB_DETECT] + + self.data = [] + self.hasNodal=False + self.module = None + #with open(self.filename, 'r', errors="surrogateescape") as f: + with open(self.filename, 'r', errors="surrogateescape") as f: + lines=f.read().splitlines() + # IF NEEDED> DO THE FOLLOWING FORMATTING: + #lines = [str(l).encode('utf-8').decode('ascii','ignore') for l in f.read().splitlines()] + + # Fast files start with ! or - + #if lines[0][0]!='!' and lines[0][0]!='-': + # raise Exception('Fast file do not start with ! or -, is it the right format?') + + # Special filetypes + if self.detectAndReadExtPtfmSE(lines): + return + if self.detectAndReadAirfoil(lines): + return + + # Parsing line by line, storing each line into a dictionary + i=0 + nComments = 0 + nWrongLabels = 0 + allowSpaceSeparatedList=False + while i0 \ + or line.upper().find('MESH-BASED OUTPUTS')>0 \ + or line.upper().find('OUTPUT CHANNELS' )>0: # "OutList - The next line(s) contains a list of output parameters. See OutListParameters.xlsx for a listing of available output channels, (-)'" + # TODO, lazy implementation so far, MAKE SUB FUNCTION + parts = re.match(r'^\W*\w+', line) + if parts: + firstword = parts.group(0).strip() + else: + raise NotImplementedError + remainer = re.sub(r'^\W*\w+\W*', '', line) + # Parsing outlist, and then we continue at a new "i" (to read END etc.) + OutList,i = parseFASTOutList(lines,i+1) + d = getDict() + if self.hasNodal: + d['label'] = firstword+'_Nodal' + else: + d['label'] = firstword + d['descr'] = remainer + d['tabType'] = TABTYPE_FIL # TODO + d['value'] = ['']+OutList + self.data.append(d) + if i>=len(lines): + break + # --- Here we cheat and force an exit of the input file + # The reason for this is that some files have a lot of things after the END, which will result in the file being intepreted as a wrong format due to too many comments + if i+20 or lines[i+2].lower().find('bldnd_bloutnd')>0): + self.hasNodal=True + else: + self.data.append(parseFASTInputLine('END of input file (the word "END" must appear in the first 3 columns of this last OutList line)',i+1)) + self.data.append(parseFASTInputLine('---------------------------------------------------------------------------------------',i+2)) + break + elif line.upper().find('SSOUTLIST' )>0 or line.upper().find('SDOUTLIST' )>0: + # SUBDYN Outlist doesn not follow regular format + self.data.append(parseFASTInputLine(line,i)) + # OUTLIST Exception for BeamDyn + OutList,i = parseFASTOutList(lines,i+1) + # TODO + for o in OutList: + d = getDict() + d['isComment'] = True + d['value']=o + self.data.append(d) + # --- Here we cheat and force an exit of the input file + self.data.append(parseFASTInputLine('END of input file (the word "END" must appear in the first 3 columns of this last OutList line)',i+1)) + self.data.append(parseFASTInputLine('---------------------------------------------------------------------------------------',i+2)) + break + elif line.upper().find('ADDITIONAL STIFFNESS')>0: + # TODO, lazy implementation so far, MAKE SUB FUNCTION + self.data.append(parseFASTInputLine(line,i)) + i +=1 + KDAdd = [] + for _ in range(19): + KDAdd.append(lines[i]) + i +=1 + d = getDict() + d['label'] = 'KDAdd' # TODO + d['tabType'] = TABTYPE_FIL # TODO + d['value'] = KDAdd + self.data.append(d) + if i>=len(lines): + break + elif line.upper().find('DISTRIBUTED PROPERTIES')>0: + self.data.append(parseFASTInputLine(line,i)); + i+=1; + self.readBeamDynProps(lines,i) + return + + # --- Parsing of standard lines: value(s) key comment + line = lines[i] + d = parseFASTInputLine(line,i,allowSpaceSeparatedList) + + # --- Handling of special files + if d['label'].lower()=='kp_total': + # BeamDyn has weird space speparated list around keypoint definition + allowSpaceSeparatedList=True + elif d['label'].lower()=='numcoords': + # TODO, lazy implementation so far, MAKE SUB FUNCTION + if isStr(d['value']): + if d['value'][0]=='@': + # it's a ref to the airfoil coord file + pass + else: + if not strIsInt(d['value']): + raise WrongFormatError('Wrong value of NumCoords') + if int(d['value'])<=0: + pass + else: + self.data.append(d); i+=1; + # 3 comment lines + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + splits=cleanAfterChar(cleanLine(lines[i]),'!').split() + # Airfoil ref point + try: + pos=[float(splits[0]), float(splits[1])] + except: + raise WrongFormatError('Wrong format while reading coordinates of airfoil reference') + i+=1 + d = getDict() + d['label'] = 'AirfoilRefPoint' + d['value'] = pos + self.data.append(d) + # 2 comment lines + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + # Table of coordinats itself + d = getDict() + d['label'] = 'AirfoilCoord' + d['tabDimVar'] = 'NumCoords' + d['tabType'] = TABTYPE_NUM_WITH_HEADERCOM + nTabLines = self[d['tabDimVar']]-1 # SOMEHOW ONE DATA POINT LESS + d['value'], d['tabColumnNames'],_ = parseFASTNumTable(self.filename,lines[i:i+nTabLines+1],nTabLines,i,1) + d['tabUnits'] = ['(-)','(-)'] + self.data.append(d) + break + + + + #print('label>',d['label'],'<',type(d['label'])); + #print('value>',d['value'],'<',type(d['value'])); + #print(isStr(d['value'])) + #if isStr(d['value']): + # print(d['value'].lower() in NUMTAB_FROM_VAL_DETECT_L) + + + # --- Handling of tables + if isStr(d['value']) and d['value'].lower() in NUMTAB_FROM_VAL_DETECT_L: + # Table with numerical values, + ii = NUMTAB_FROM_VAL_DETECT_L.index(d['value'].lower()) + tab_type = NUMTAB_FROM_VAL_TYPE[ii] + if tab_type=='num': + d['tabType'] = TABTYPE_NUM_WITH_HEADER + else: + d['tabType'] = TABTYPE_MIX_WITH_HEADER + d['label'] = NUMTAB_FROM_VAL_VARNAME[ii] + d['tabDimVar'] = NUMTAB_FROM_VAL_DIM_VAR[ii] + nHeaders = NUMTAB_FROM_VAL_NHEADER[ii] + nTabLines=0 + if isinstance(d['tabDimVar'],int): + nTabLines = d['tabDimVar'] + else: + nTabLines = self[d['tabDimVar']] + #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); + d['value'], d['tabColumnNames'], d['tabUnits'] = parseFASTNumTable(self.filename,lines[i:i+nTabLines+nHeaders], nTabLines, i, nHeaders, tableType=tab_type, varNumLines=d['tabDimVar']) + i += nTabLines+nHeaders-1 + + # --- Temporary hack for e.g. SubDyn, that has duplicate table, impossible to detect in the current way... + # So we remove the element form the list one read + del NUMTAB_FROM_VAL_DETECT[ii] + del NUMTAB_FROM_VAL_DIM_VAR[ii] + del NUMTAB_FROM_VAL_VARNAME[ii] + del NUMTAB_FROM_VAL_NHEADER[ii] + del NUMTAB_FROM_VAL_TYPE [ii] + del NUMTAB_FROM_VAL_DETECT_L[ii] + + elif isStr(d['label']) and d['label'].lower() in NUMTAB_FROM_LAB_DETECT_L: + ii = NUMTAB_FROM_LAB_DETECT_L.index(d['label'].lower()) + tab_type = NUMTAB_FROM_LAB_TYPE[ii] + # Special case for airfoil data, the table follows NumAlf, so we add d first + if d['label'].lower()=='numalf': + d['tabType']=TABTYPE_NOT_A_TAB + self.data.append(d) + # Creating a new dictionary for the table + d = {'value':None, 'label':'NumAlf', 'isComment':False, 'descr':'', 'tabType':None} + i += 1 + nHeaders = NUMTAB_FROM_LAB_NHEADER[ii] + nOffset = NUMTAB_FROM_LAB_NOFFSET[ii] + if nOffset>0: + # Creating a dictionary for that entry + dd = {'value':d['value'], 'label':d['label'], 'isComment':False, 'descr':d['descr'], 'tabType':TABTYPE_NOT_A_TAB} + self.data.append(dd) + + d['label'] = NUMTAB_FROM_LAB_VARNAME[ii] + d['tabDimVar'] = NUMTAB_FROM_LAB_DIM_VAR[ii] + if d['label'].lower()=='afcoeff' : + d['tabType'] = TABTYPE_NUM_WITH_HEADERCOM + else: + if tab_type=='num': + d['tabType'] = TABTYPE_NUM_WITH_HEADER + else: + d['tabType'] = TABTYPE_MIX_WITH_HEADER + if isinstance(d['tabDimVar'],int): + nTabLines = d['tabDimVar'] + else: + nTabLines = self[d['tabDimVar']] + #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); + d['value'], d['tabColumnNames'], d['tabUnits'] = parseFASTNumTable(self.filename,lines[i:i+nTabLines+nHeaders+nOffset],nTabLines,i, nHeaders, tableType=tab_type, nOffset=nOffset, varNumLines=d['tabDimVar']) + i += nTabLines+1-nOffset + + # --- Temporary hack for e.g. SubDyn, that has duplicate table, impossible to detect in the current way... + # So we remove the element form the list one read + del NUMTAB_FROM_LAB_DETECT[ii] + del NUMTAB_FROM_LAB_DIM_VAR[ii] + del NUMTAB_FROM_LAB_VARNAME[ii] + del NUMTAB_FROM_LAB_NHEADER[ii] + del NUMTAB_FROM_LAB_NOFFSET[ii] + del NUMTAB_FROM_LAB_TYPE [ii] + del NUMTAB_FROM_LAB_DETECT_L[ii] + + elif isStr(d['label']) and d['label'].lower() in FILTAB_FROM_LAB_DETECT_L: + ii = FILTAB_FROM_LAB_DETECT_L.index(d['label'].lower()) + d['label'] = FILTAB_FROM_LAB_VARNAME[ii] + d['tabDimVar'] = FILTAB_FROM_LAB_DIM_VAR[ii] + d['tabType'] = TABTYPE_FIL + nTabLines = self[d['tabDimVar']] + #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); + d['value'] = parseFASTFilTable(lines[i:i+nTabLines],nTabLines,i) + i += nTabLines-1 + + + + self.data.append(d) + i += 1 + # --- Safety checks + if d['isComment']: + #print(line) + nComments +=1 + else: + if hasSpecialChars(d['label']): + nWrongLabels +=1 + #print('label>',d['label'],'<',type(d['label']),line); + if i>3: # first few lines may be comments, we allow it + #print('Line',i,'Label:',d['label']) + raise WrongFormatError('Special Character found in Label: `{}`, for line: `{}`'.format(d['label'],line)) + if len(d['label'])==0: + nWrongLabels +=1 + if nComments>len(lines)*0.35: + #print('Comment fail',nComments,len(lines),self.filename) + raise WrongFormatError('Most lines were read as comments, probably not a FAST Input File') + if nWrongLabels>len(lines)*0.10: + #print('Label fail',nWrongLabels,len(lines),self.filename) + raise WrongFormatError('Too many lines with wrong labels, probably not a FAST Input File') + + # --- PostReading checks + labels = self.keys() + duplicates = set([x for x in labels if (labels.count(x) > 1) and x!='OutList' and x.strip()!='-']) + if len(duplicates)>0: + print('[WARN] Duplicate labels found in file: '+self.filename) + print(' Duplicates: '+', '.join(duplicates)) + print(' It\'s strongly recommended to make them unique! ') +# except WrongFormatError as e: +# raise WrongFormatError('Fast File {}: '.format(self.filename)+'\n'+e.args[0]) +# except Exception as e: +# raise e +# # print(e) +# raise Exception('Fast File {}: '.format(self.filename)+'\n'+e.args[0]) + + + def toString(self): + s='' + # Special file formats, TODO subclass + if self.module=='ExtPtfm': + s+='!Comment\n' + s+='!Comment Flex 5 Format\n' + s+='!Dimension: {}\n'.format(self['nDOF']) + s+='!Time increment in simulation: {}\n'.format(self['dt']) + s+='!Total simulation time in file: {}\n'.format(self['T']) + + s+='\n!Mass Matrix\n' + s+='!Dimension: {}\n'.format(self['nDOF']) + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['MassMatrix']) + + s+='\n\n!Stiffness Matrix\n' + s+='!Dimension: {}\n'.format(self['nDOF']) + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['StiffnessMatrix']) + + s+='\n\n!Damping Matrix\n' + s+='!Dimension: {}\n'.format(self['nDOF']) + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['DampingMatrix']) + + s+='\n\n!Loading and Wave Elevation\n' + s+='!Dimension: 1 time column - {} force columns\n'.format(self['nDOF']) + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['Loading']) + return s + + def toStringVLD(val,lab,descr): + val='{}'.format(val) + lab='{}'.format(lab) + if len(val)<13: + val='{:13s}'.format(val) + if len(lab)<13: + lab='{:13s}'.format(lab) + return val+' '+lab+' - '+descr.strip().strip('-').strip()+'\n' + + def beamdyn_section_mat_tostring(x,K,M): + def mat_tostring(M,fmt='24.16e'): + return '\n'.join([' '+' '.join(['{:24.16E}'.format(m) for m in M[i,:]]) for i in range(np.size(M,1))]) + s='' + s+='{:.6f}\n'.format(x) + s+=mat_tostring(K) + #s+=np.array2string(K) + s+='\n' + s+='\n' + s+=mat_tostring(M) + #s+=np.array2string(M) + s+='\n' + s+='\n' + return s + + + for i in range(len(self.data)): + d=self.data[i] + if d['isComment']: + s+='{}'.format(d['value']) + elif d['tabType']==TABTYPE_NOT_A_TAB: + if isinstance(d['value'], list): + sList=', '.join([str(x) for x in d['value']]) + s+='{} {} {}'.format(sList,d['label'],d['descr']) + else: + s+=toStringVLD(d['value'],d['label'],d['descr']).strip() + elif d['tabType']==TABTYPE_NUM_WITH_HEADER: + if d['tabColumnNames'] is not None: + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) + #s+=d['descr'] # Not ready for that + if d['tabUnits'] is not None: + s+='\n' + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) + newline='\n' + else: + newline='' + if np.size(d['value'],0) > 0 : + s+=newline + s+='\n'.join('\t'.join( ('{:15.0f}'.format(x) if int(x)==x else '{:15.8e}'.format(x) ) for x in y) for y in d['value']) + elif d['tabType']==TABTYPE_MIX_WITH_HEADER: + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) + if d['tabUnits'] is not None: + s+='\n' + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) + if np.size(d['value'],0) > 0 : + s+='\n' + s+='\n'.join('\t'.join('{}'.format(x) for x in y) for y in d['value']) + elif d['tabType']==TABTYPE_NUM_WITH_HEADERCOM: + s+='! {}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) + s+='! {}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) + s+='\n'.join('\t'.join('{:15.8e}'.format(x) for x in y) for y in d['value']) + elif d['tabType']==TABTYPE_FIL: + #f.write('{} {} {}\n'.format(d['value'][0],d['tabDetect'],d['descr'])) + s+='{} {} {}\n'.format(d['value'][0],d['label'],d['descr']) # TODO? + s+='\n'.join(fil for fil in d['value'][1:]) + elif d['tabType']==TABTYPE_NUM_BEAMDYN: + data = d['value'] + Cols =['Span'] + Cols+=['K{}{}'.format(i+1,j+1) for i in range(6) for j in range(6)] + Cols+=['M{}{}'.format(i+1,j+1) for i in range(6) for j in range(6)] + for i in np.arange(len(data['span'])): + x = data['span'][i] + K = data['K'][i] + M = data['M'][i] + s += beamdyn_section_mat_tostring(x,K,M) + else: + raise Exception('Unknown table type for variable {}'.format(d)) + if i0: + # Hack for blade files, we add the modes + x=Val[:,0] + Modes=np.zeros((x.shape[0],3)) + Modes[:,0] = x**2 * self['BldFl1Sh(2)'] \ + + x**3 * self['BldFl1Sh(3)'] \ + + x**4 * self['BldFl1Sh(4)'] \ + + x**5 * self['BldFl1Sh(5)'] \ + + x**6 * self['BldFl1Sh(6)'] + Modes[:,1] = x**2 * self['BldFl2Sh(2)'] \ + + x**3 * self['BldFl2Sh(3)'] \ + + x**4 * self['BldFl2Sh(4)'] \ + + x**5 * self['BldFl2Sh(5)'] \ + + x**6 * self['BldFl2Sh(6)'] + Modes[:,2] = x**2 * self['BldEdgSh(2)'] \ + + x**3 * self['BldEdgSh(3)'] \ + + x**4 * self['BldEdgSh(4)'] \ + + x**5 * self['BldEdgSh(5)'] \ + + x**6 * self['BldEdgSh(6)'] + Val = np.hstack((Val,Modes)) + Cols = Cols + ['ShapeFlap1_[-]','ShapeFlap2_[-]','ShapeEdge1_[-]'] + + elif self.getIDSafe('TwFAM1Sh(2)')>0: + # Hack for tower files, we add the modes + x=Val[:,0] + Modes=np.zeros((x.shape[0],4)) + Modes[:,0] = x**2 * self['TwFAM1Sh(2)'] \ + + x**3 * self['TwFAM1Sh(3)'] \ + + x**4 * self['TwFAM1Sh(4)'] \ + + x**5 * self['TwFAM1Sh(5)'] \ + + x**6 * self['TwFAM1Sh(6)'] + Modes[:,1] = x**2 * self['TwFAM2Sh(2)'] \ + + x**3 * self['TwFAM2Sh(3)'] \ + + x**4 * self['TwFAM2Sh(4)'] \ + + x**5 * self['TwFAM2Sh(5)'] \ + + x**6 * self['TwFAM2Sh(6)'] + Modes[:,2] = x**2 * self['TwSSM1Sh(2)'] \ + + x**3 * self['TwSSM1Sh(3)'] \ + + x**4 * self['TwSSM1Sh(4)'] \ + + x**5 * self['TwSSM1Sh(5)'] \ + + x**6 * self['TwSSM1Sh(6)'] + Modes[:,3] = x**2 * self['TwSSM2Sh(2)'] \ + + x**3 * self['TwSSM2Sh(3)'] \ + + x**4 * self['TwSSM2Sh(4)'] \ + + x**5 * self['TwSSM2Sh(5)'] \ + + x**6 * self['TwSSM2Sh(6)'] + Val = np.hstack((Val,Modes)) + Cols = Cols + ['ShapeForeAft1_[-]','ShapeForeAft2_[-]','ShapeSideSide1_[-]','ShapeSideSide2_[-]'] + elif d['label']=='AFCoeff': + try: + pol = d['value'] + alpha = pol[:,0]*np.pi/180. + Cl = pol[:,1] + Cd = pol[:,2] + Cd0 = self['Cd0'] + # Cn (with or without Cd0) + Cn1 = Cl*np.cos(alpha)+ (Cd-Cd0)*np.sin(alpha) + Cn = Cl*np.cos(alpha)+ Cd*np.sin(alpha) + Val=np.column_stack((Val,Cn)); Cols+=['Cn_[-]'] + Val=np.column_stack((Val,Cn1)); Cols+=['Cn_Cd0off_[-]'] + + CnLin = self['C_nalpha']*(alpha-self['alpha0']*np.pi/180.) + CnLin[alpha<-20*np.pi/180]=np.nan + CnLin[alpha> 30*np.pi/180]=np.nan + Val=np.column_stack((Val,CnLin)); Cols+=['Cn_pot_[-]'] + + # Highlighting points surrounding 0 1 2 Cn points + CnPoints = Cn*np.nan + iBef2 = np.where(alpha0: + if l[0]=='!': + if l.find('!dimension')==0: + self.addKeyVal('nDOF',int(l.split(':')[1])) + nDOFCommon=self['nDOF'] + elif l.find('!time increment')==0: + self.addKeyVal('dt',float(l.split(':')[1])) + elif l.find('!total simulation time')==0: + self.addKeyVal('T',float(l.split(':')[1])) + elif len(l.strip())==0: + pass + else: + raise BrokenFormatError('Unexcepted content found on line {}'.format(i)) + i+=1 + except BrokenFormatError as e: + raise e + except: + raise + + + return True + + + + def detectAndReadAirfoil(self,lines): + if len(lines)<14: + return False + # Reading number of tables + L3 = lines[2].strip().split() + if len(L3)<=0: + return False + if not strIsInt(L3[0]): + return False + nTables=int(L3[0]) + # Reading table ID + L4 = lines[3].strip().split() + if len(L4)<=nTables: + return False + TableID=L4[:nTables] + if nTables==1: + TableID=[''] + # Keywords for file format + KW1=lines[12].strip().split() + KW2=lines[13].strip().split() + if len(KW1)>nTables and len(KW2)>nTables: + if KW1[nTables].lower()=='angle' and KW2[nTables].lower()=='minimum': + d = getDict(); d['isComment'] = True; d['value'] = lines[0]; self.data.append(d); + d = getDict(); d['isComment'] = True; d['value'] = lines[1]; self.data.append(d); + for i in range(2,14): + splits = lines[i].split() + #print(splits) + d = getDict() + d['label'] = ' '.join(splits[1:]) # TODO + d['descr'] = ' '.join(splits[1:]) # TODO + d['value'] = float(splits[0]) + self.data.append(d) + #pass + #for i in range(2,14): + nTabLines=0 + while 14+nTabLines0 : + nTabLines +=1 + #data = np.array([lines[i].strip().split() for i in range(14,len(lines)) if len(lines[i])>0]).astype(float) + #data = np.array([lines[i].strip().split() for i in takewhile(lambda x: len(lines[i].strip())>0, range(14,len(lines)-1))]).astype(float) + data = np.array([lines[i].strip().split() for i in range(14,nTabLines+14)]).astype(float) + #print(data) + d = getDict() + d['label'] = 'Polar' + d['tabDimVar'] = nTabLines + d['tabType'] = TABTYPE_NUM_NO_HEADER + d['value'] = data + if np.size(data,1)==1+nTables*3: + d['tabColumnNames'] = ['Alpha']+[n+l for l in TableID for n in ['Cl','Cd','Cm']] + d['tabUnits'] = ['(deg)']+['(-)' , '(-)' , '(-)']*nTables + elif np.size(data,1)==1+nTables*2: + d['tabColumnNames'] = ['Alpha']+[n+l for l in TableID for n in ['Cl','Cd']] + d['tabUnits'] = ['(deg)']+['(-)' , '(-)']*nTables + else: + d['tabColumnNames'] = ['col{}'.format(j) for j in range(np.size(data,1))] + self.data.append(d) + return True + + def readBeamDynProps(self,lines,iStart): + nStations=self['station_total'] + #M=np.zeros((nStations,1+36+36)) + M = np.zeros((nStations,6,6)) + K = np.zeros((nStations,6,6)) + span = np.zeros(nStations) + i=iStart; + try: + for j in range(nStations): + # Read span location + span[j]=float(lines[i]); i+=1; + # Read stiffness matrix + K[j,:,:]=np.array((' '.join(lines[i:i+6])).split()).astype(float).reshape(6,6) + i+=7 + # Read mass matrix + M[j,:,:]=np.array((' '.join(lines[i:i+6])).split()).astype(float).reshape(6,6) + i+=7 + except: + raise WrongFormatError('An error occured while reading section {}/{}'.format(j+1,nStations)) + + d = getDict() + d['label'] = 'BeamProperties' + d['descr'] = '' + d['tabType'] = TABTYPE_NUM_BEAMDYN + d['value'] = {'span':span, 'K':K, 'M':M} + self.data.append(d) + +# --------------------------------------------------------------------------------} +# --- Helper functions +# --------------------------------------------------------------------------------{ +def isStr(s): + # Python 2 and 3 compatible + # Two options below + # NOTE: all this avoided since we import str from builtins + # --- Version 2 + # isString = False; + # if(isinstance(s, str)): + # isString = True; + # try: + # if(isinstance(s, basestring)): # todo unicode as well + # isString = True; + # except NameError: + # pass; + # return isString + # --- Version 1 + # try: + # basestring # python 2 + # return isinstance(s, basestring) or isinstance(s,unicode) + # except NameError: + # basestring=str #python 3 + # return isinstance(s, str) + return isinstance(s, str) + +def strIsFloat(s): + #return s.replace('.',',1').isdigit() + try: + float(s) + return True + except: + return False + +def strIsBool(s): + return s.lower() in ['true','false','t','f'] + +def strIsInt(s): + s = str(s) + if s[0] in ('-', '+'): + return s[1:].isdigit() + return s.isdigit() + +def strToBool(s): + return s.lower() in ['true','t'] + +def hasSpecialChars(s): + # fast allows for parenthesis + # For now we allow for - but that's because of BeamDyn geometry members + return not re.match("^[\"\'a-zA-Z0-9_()-]*$", s) + +def cleanLine(l): + # makes a string single space separated + l = l.replace('\t',' ') + l = ' '.join(l.split()) + l = l.strip() + return l + +def cleanAfterChar(l,c): + # remove whats after a character + n = l.find(c); + if n>0: + return l[:n] + else: + return l + +def getDict(): + return {'value':None, 'label':'', 'isComment':False, 'descr':'', 'tabType':TABTYPE_NOT_A_TAB} + +def _merge_value(splits): + + merged = splits.pop(0) + if merged[0] == '"': + while merged[-1] != '"': + merged += " "+splits.pop(0) + splits.insert(0, merged) + + + + +def parseFASTInputLine(line_raw,i,allowSpaceSeparatedList=False): + d = getDict() + #print(line_raw) + try: + # preliminary cleaning (Note: loss of formatting) + line = cleanLine(line_raw) + # Comment + if any(line.startswith(c) for c in ['#','!','--','==']) or len(line)==0: + d['isComment']=True + d['value']=line_raw + return d + if line.lower().startswith('end'): + sp =line.split() + if len(sp)>2 and sp[1]=='of': + d['isComment']=True + d['value']=line_raw + + # Detecting lists + List=[]; + iComma=line.find(',') + if iComma>0 and iComma<30: + fakeline=line.replace(' ',',') + fakeline=re.sub(',+',',',fakeline) + csplits=fakeline.split(',') + # Splitting based on comma and looping while it's numbers of booleans + ii=0 + s=csplits[ii] + #print(csplits) + while strIsFloat(s) or strIsBool(s) and ii=len(csplits): + raise WrongFormatError('Wrong number of list values') + s = csplits[ii] + #print('[INFO] Line {}: Found list: '.format(i),List) + # Defining value and remaining splits + if len(List)>=2: + d['value']=List + line_remaining=line + # eating line, removing each values + for iii in range(ii): + sValue=csplits[iii] + ipos=line_remaining.find(sValue) + line_remaining = line_remaining[ipos+len(sValue):] + splits=line_remaining.split() + iNext=0 + else: + # It's not a list, we just use space as separators + splits=line.split(' ') + _merge_value(splits) + s=splits[0] + + if strIsInt(s): + d['value']=int(s) + if allowSpaceSeparatedList and len(splits)>1: + if strIsInt(splits[1]): + d['value']=splits[0]+ ' '+splits[1] + elif strIsFloat(s): + d['value']=float(s) + elif strIsBool(s): + d['value']=strToBool(s) + else: + d['value']=s + iNext=1 + #import pdb ; pdb.set_trace(); + + # Extracting label (TODO, for now only second split) + bOK=False + while (not bOK) and iNext comment assumed'.format(i+1)) + d['isComment']=True + d['value']=line_raw + iNext = len(splits)+1 + + # Recombining description + if len(splits)>=iNext+1: + d['descr']=' '.join(splits[iNext:]) + except WrongFormatError as e: + raise WrongFormatError('Line {}: '.format(i+1)+e.args[0]) + except Exception as e: + raise Exception('Line {}: '.format(i+1)+e.args[0]) + + return d + +def parseFASTOutList(lines,iStart): + OutList=[] + i = iStart + MAX=200 + while iMAX : + raise Exception('More that 200 lines found in outlist') + if i>=len(lines): + print('[WARN] End of file reached while reading Outlist') + #i=min(i+1,len(lines)) + return OutList,iStart+len(OutList) + + +def extractWithinParenthesis(s): + mo = re.search(r'\((.*)\)', s) + if mo: + return mo.group(1) + return '' + +def extractWithinBrackets(s): + mo = re.search(r'\((.*)\)', s) + if mo: + return mo.group(1) + return '' + +def detectUnits(s,nRef): + nPOpen=s.count('(') + nPClos=s.count(')') + nBOpen=s.count('[') + nBClos=s.count(']') + + sep='!#@#!' + if (nPOpen == nPClos) and (nPOpen>=nRef): + #that's pretty good + Units=s.replace('(','').replace(')',sep).split(sep)[:-1] + elif (nBOpen == nBClos) and (nBOpen>=nRef): + Units=s.replace('[','').replace(']',sep).split(sep)[:-1] + else: + Units=s.split() + return Units + + +def parseFASTNumTable(filename,lines,n,iStart,nHeaders=2,tableType='num',nOffset=0, varNumLines=''): + """ + First lines of data starts at: nHeaders+nOffset + + """ + Tab = None + ColNames = None + Units = None + + + if len(lines)!=n+nHeaders+nOffset: + raise BrokenFormatError('Not enough lines in table: {} lines instead of {}\nFile:{}'.format(len(lines)-nHeaders,n,filename)) + try: + if nHeaders==0: + # Extract number of values from number of numerical values on first line + numeric_const_pattern = r'[-+]? (?: (?: \d* \. \d+ ) | (?: \d+ \.? ) )(?: [Ee] [+-]? \d+ ) ?' + rx = re.compile(numeric_const_pattern, re.VERBOSE) + header = cleanAfterChar(lines[nOffset], '!') + if tableType=='num': + dat= np.array(rx.findall(header)).astype(float) + ColNames=['C{}'.format(j) for j in range(len(dat))] + else: + raise NotImplementedError('Reading FAST tables with no headers for type different than num not implemented yet') + + elif nHeaders>=1: + # Extract column names + i = 0 + sTmp = cleanLine(lines[i]) + sTmp = cleanAfterChar(sTmp,'[') + sTmp = cleanAfterChar(sTmp,'(') + sTmp = cleanAfterChar(sTmp,'!') + sTmp = cleanAfterChar(sTmp,'#') + if sTmp.startswith('!'): + sTmp=sTmp[1:].strip() + ColNames=sTmp.split() + if nHeaders>=2: + # Extract units + i = 1 + sTmp = cleanLine(lines[i]) + sTmp = cleanAfterChar(sTmp,'!') + sTmp = cleanAfterChar(sTmp,'#') + if sTmp.startswith('!'): + sTmp=sTmp[1:].strip() + + Units = detectUnits(sTmp,len(ColNames)) + Units = ['({})'.format(u.strip()) for u in Units] + # Forcing user to match number of units and column names + if len(ColNames) != len(Units): + print(ColNames) + print(Units) + print('[WARN] {}: Line {}: Number of column names different from number of units in table'.format(filename, iStart+i+1)) + + nCols=len(ColNames) + + if tableType=='num': + if n==0: + Tab = np.zeros((n, nCols)) + for i in range(nHeaders+nOffset,n+nHeaders+nOffset): + l = cleanAfterChar(lines[i].lower(),'!') + l = cleanAfterChar(l,'#') + v = l.split() + if len(v) != nCols: + # Discarding SubDyn special cases + if ColNames[-1].lower() not in ['nodecnt']: + print('[WARN] {}: Line {}: number of data different from number of column names. ColumnNames: {}'.format(filename, iStart+i+1, ColNames)) + if i==nHeaders+nOffset: + # Node Cnt + if len(v) != nCols: + if ColNames[-1].lower()== 'nodecnt': + ColNames = ColNames+['Col']*(len(v)-nCols) + Units = Units+['Col']*(len(v)-nCols) + + nCols=len(v) + Tab = np.zeros((n, nCols)) + # Accounting for TRUE FALSE and converting to float + v = [s.replace('true','1').replace('false','0').replace('noprint','0').replace('print','1') for s in v] + v = [float(s) for s in v[0:nCols]] + if len(v) < nCols: + raise Exception('Number of data is lower than number of column names') + Tab[i-nHeaders-nOffset,:] = v + elif tableType=='mix': + # a mix table contains a mixed of strings and floats + # For now, we are being a bit more relaxed about the number of columns + if n==0: + Tab = np.zeros((n, nCols)).astype(object) + for i in range(nHeaders+nOffset,n+nHeaders+nOffset): + l = lines[i] + l = cleanAfterChar(l,'!') + l = cleanAfterChar(l,'#') + v = l.split() + if l.startswith('---'): + raise BrokenFormatError('Error reading line {} while reading table. Is the variable `{}` set correctly?'.format(iStart+i+1, varNumLines)) + if len(v) != nCols: + # Discarding SubDyn special cases + if ColNames[-1].lower() not in ['cosmid', 'ssifile']: + print('[WARN] {}: Line {}: Number of data is different than number of column names. Column Names: {}'.format(filename,iStart+1+i, ColNames)) + if i==nHeaders+nOffset: + if len(v)>nCols: + ColNames = ColNames+['Col']*(len(v)-nCols) + Units = Units+['Col']*(len(v)-nCols) + nCols=len(v) + Tab = np.zeros((n, nCols)).astype(object) + v=v[0:min(len(v),nCols)] + Tab[i-nHeaders-nOffset,0:len(v)] = v + # If all values are float, we convert to float + if all([strIsFloat(x) for x in Tab.ravel()]): + Tab=Tab.astype(float) + else: + raise Exception('Unknown table type') + + ColNames = ColNames[0:nCols] + if Units is not None: + Units = Units[0:nCols] + Units = ['('+u.replace('(','').replace(')','')+')' for u in Units] + if nHeaders==0: + ColNames=None + + except Exception as e: + raise BrokenFormatError('Line {}: {}'.format(iStart+i+1,e.args[0])) + return Tab, ColNames, Units + + +def parseFASTFilTable(lines,n,iStart): + Tab = [] + try: + i=0 + if len(lines)!=n: + raise WrongFormatError('Not enough lines in table: {} lines instead of {}'.format(len(lines),n)) + for i in range(n): + l = lines[i].split() + #print(l[0].strip()) + Tab.append(l[0].strip()) + + except Exception as e: + raise Exception('Line {}: '.format(iStart+i+1)+e.args[0]) + return Tab + + +if __name__ == "__main__": + pass + #B=FASTIn('Turbine.outb') + + + diff --git a/pydatview/io/fast_input_file_graph.py b/pydatview/io/fast_input_file_graph.py new file mode 100644 index 0000000..cfb9767 --- /dev/null +++ b/pydatview/io/fast_input_file_graph.py @@ -0,0 +1,330 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +# Local +try: + from .tools.graph import * +except ImportError: + from welib.FEM.graph import * + + +# --------------------------------------------------------------------------------} +# --- Wrapper to convert a "fast" input file dictionary into a graph +# --------------------------------------------------------------------------------{ +def fastToGraph(data): + if 'BeamProp' in data.keys(): + return subdynToGraph(data) + + if 'SmplProp' in data.keys(): + return hydrodynToGraph(data) + + if 'DOF2Nodes' in data.keys(): + return subdynSumToGraph(data) + + raise NotImplementedError('Graph for object with keys: {}'.format(data.keys())) + +# --------------------------------------------------------------------------------} +# --- SubDyn +# --------------------------------------------------------------------------------{ +def subdynToGraph(sd): + """ + sd: dict-like object as returned by weio + """ + type2Color=[ + (0.1,0.1,0.1), # Watchout based on background + (0.753,0.561,0.05), # 1 Beam + (0.541,0.753,0.05), # 2 Cable + (0.753,0.05,0.204), # 3 Rigid + (0.918,0.702,0.125), # 3 Rigid + ] + + Graph = GraphModel() + # --- Properties + if 'BeamProp' in sd.keys(): + BProps = sd['BeamProp'] + Graph.addNodePropertySet('Beam') + for ip,P in enumerate(BProps): + prop= NodeProperty(ID=P[0], E=P[1], G=P[2], rho=P[3], D=P[4], t=P[5] ) + Graph.addNodeProperty('Beam',prop) + + if 'CableProp' in sd.keys(): + CProps = sd['CableProp'] + Graph.addNodePropertySet('Cable') + for ip,P in enumerate(CProps): + Chan = -1 if len(P)<5 else P[4] + prop= NodeProperty(ID=P[0], EA=P[1], rho=P[2], T0=P[3], Chan=Chan) + Graph.addNodeProperty('Cable',prop) + + if 'RigidProp' in sd.keys(): + RProps = sd['RigidProp'] + Graph.addNodePropertySet('Rigid') + for ip,P in enumerate(RProps): + prop= NodeProperty(ID=P[0], rho=P[1]) + Graph.addNodeProperty('Rigid',prop) + + # --- Nodes and DOFs + Nodes = sd['Joints'] + for iNode,N in enumerate(Nodes): + Type= 1 if len(N)<=4 else N[4] + node = Node(ID=N[0], x=N[1], y=N[2], z=N[3], Type=Type) + Graph.addNode(node) + + # --- Elements + Members = sd['Members'].astype(int) + PropSets = ['Beam','Cable','Rigid'] + for ie,E in enumerate(Members): + Type=1 if len(E)==5 else E[5] + #elem= Element(E[0], E[1:3], propset=PropSets[Type-1], propIDs=E[3:5]) + elem= Element(E[0], E[1:3], Type=PropSets[Type-1], propIDs=E[3:5], propset=PropSets[Type-1]) + elem.data['object']='cylinder' + elem.data['color'] = type2Color[Type] + Graph.addElement(elem) + # Nodal prop data + #Graph.setElementNodalProp(elem, propset=PropSets[Type-1], propIDs=E[3:5]) + + # --- Concentrated Masses (in global coordinates), node data + for iC, CM in enumerate(sd['ConcentratedMasses']): + #CMJointID, JMass, JMXX, JMYY, JMZZ, JMXY, JMXZ, JMYZ, MCGX, MCGY, MCGZ + nodeID = CM[0] + n = Graph.getNode(nodeID) + M66 = np.zeros((6,6)) + if len(CM)==11: + m = CM[1] + x, y ,z = (CM[8], CM[9], CM[10]) + Jxx = CM[2]; Jyy = CM[3]; Jzz = CM[4] + Jxy = CM[5]; Jxz = CM[6]; Jyz = CM[7]; + else: + raise NotImplementedError('TODO legacy') + m = CM[1] + Jxx = CM[2]; Jyy = CM[3]; Jzz = CM[4] + Jxy = 0; Jyz =0; Jzz = 0; x,y,z=0,0,0 + M66[0, :] =[ m , 0 , 0 , 0 , z*m , -y*m ] + M66[1, :] =[ 0 , m , 0 , -z*m , 0 , x*m ] + M66[2, :] =[ 0 , 0 , m , y*m , -x*m , 0 ] + M66[3, :] =[ 0 , -z*m , y*m , Jxx + m*(y**2+z**2) , Jxy - m*x*y , Jxz - m*x*z ] + M66[4, :] =[ z*m , 0 , -x*m , Jxy - m*x*y , Jyy + m*(x**2+z**2) , Jyz - m*y*z ] + M66[5, :] =[ -y*m , x*m , 0 , Jxz - m*x*z , Jyz - m*y*z , Jzz + m*(x**2+y**2) ] + n.setData({'addedMassMatrix':M66}) + + # Nodal data + for iN,N in enumerate(sd['InterfaceJoints']): + nodeID = int(N[0]) + Graph.setNodalData(nodeID,IBC=N[1:]) + for iN,N in enumerate(sd['BaseJoints']): + NN=[int(n) if i<7 else n for i,n in enumerate(N)] + nodeID = NN[0] + Graph.setNodalData(nodeID,RBC=NN[1:]) + # print('CMass') + # print(sd['ConcentratedMasses']) + + return Graph + + + + + +# --------------------------------------------------------------------------------} +# --- HydroDyn +# --------------------------------------------------------------------------------{ +def hydrodynToGraph(hd): + """ + hd: dict-like object as returned by weio + """ + def type2Color(Pot): + if Pot: + return (0.753,0.05,0.204), # Pot flow + else: + return (0.753,0.561,0.05), # Morison + + + Graph = GraphModel() + + # --- Properties + if 'SectionProp' in hd.keys(): + # NOTE: setting it as element property since two memebrs may connect on the same node with different diameters/thicknesses + Graph.addNodePropertySet('Section') + for ip,P in enumerate(hd['SectionProp']): + # PropSetID PropD PropThck + prop= NodeProperty(ID=P[0], D=P[1], t=P[2]) + Graph.addNodeProperty('Section',prop) + + # --- Hydro Coefs - will be stored in AxCoefs, SimpleCoefs, DepthCoefs, MemberCoefs + if 'AxCoefs' in hd.keys(): + Graph.addNodePropertySet('AxCoefs') + for ip,P in enumerate(hd['AxCoefs']): + prop= NodeProperty(ID=P[0], JAxCd=P[1], JAxCa=P[2], JAxCp=P[3]) + Graph.addNodeProperty('AxCoefs',prop) + if 'SmplProp' in hd.keys(): + Graph.addNodePropertySet('SimpleCoefs') + for ip,P in enumerate(hd['SmplProp']): + # SimplCd SimplCdMG SimplCa SimplCaMG SimplCp SimplCpMG SimplAxCd SimplAxCdMG SimplAxCa SimplAxCaMG SimplAxCp SimplAxCpMG + if len(P)==12: + prop= NodeProperty(ID=ip+1, Cd=P[0], CdMG=P[1], Ca=P[2], CaMG=P[3], Cp=P[4], CpMG=P[5], AxCd=P[6], AxCdMG=P[7], AxCa=P[8], AxCaMG=P[9], AxCp=P[10], AxCpMG=P[11]) + elif len(P)==10: + prop= NodeProperty(ID=ip+1, Cd=P[0], CdMG=P[1], Ca=P[2], CaMG=P[3], Cp=P[4], CpMG=P[5], AxCa=P[6], AxCaMG=P[7], AxCp=P[8], AxCpMG=P[9]) + else: + raise NotImplementedError() + Graph.addNodeProperty('SimpleCoefs',prop) + if 'DpthProp' in hd.keys(): + Graph.addMiscPropertySet('DepthCoefs') + for ip,P in enumerate(hd['DpthProp']): + # Dpth DpthCd DpthCdMG DpthCa DpthCaMG DpthCp DpthCpMG DpthAxCd DpthAxCdMG DpthAxCa DpthAxCaMG DpthAxCp DpthAxCpMG + prop= Property(ID=ip+1, Dpth=P[0], Cd=P[1], CdMG=P[2], Ca=P[3], CaMG=P[4], Cp=P[5], CpMG=P[6], AxCd=P[7], AxCdMG=P[8], AxCa=P[9], AxCaMG=P[10], AxCp=P[11], AxCpMG=P[12]) + Graph.addMiscProperty('DepthCoefs',prop) + if 'MemberProp' in hd.keys(): + # Member-based hydro coefficinet + Graph.addMiscPropertySet('MemberCoefs') + for ip,P in enumerate(hd['MemberProp']): + # MemberID MemberCd1 MemberCd2 MemberCdMG1 MemberCdMG2 MemberCa1 MemberCa2 MemberCaMG1 MemberCaMG2 MemberCp1 MemberCp2 MemberCpMG1 MemberCpMG2 MemberAxCd1 MemberAxCd2 MemberAxCdMG1 MemberAxCdMG2 MemberAxCa1 MemberAxCa2 MemberAxCaMG1 MemberAxCaMG2 MemberAxCp1 MemberAxCp2 MemberAxCpMG1 MemberAxCpMG2 + prop = Property(ID=ip+1, MemberID=P[0], Cd1=P[1], Cd2=P[2], CdMG1=P[3], CdMG2=P[4], Ca1=P[5], Ca2=P[6], CaMG1=P[7], CaMG2=P[8], Cp1=P[9], Cp2=P[10], CpMG1=P[11], CpMG2=P[12], AxCd1=P[14], AxCd2=P[15], axCdMG1=P[16], axCdMG2=P[17], AxCa1=P[18], AxCa2=P[19], AxCaMG1=P[20], AxCaMG2=P[21], AxCp1=P[22], AxCp2=P[23]) + Graph.addMiscProperty('MemberCoefs',prop) + # --- + if 'FillGroups' in hd.keys(): + # Filled members + Graph.addMiscPropertySet('FillGroups') + for ip,P in enumerate(hd['FillGroups']): + # FillNumM FillMList FillFSLoc FillDens + raise NotImplementedError('hydroDynToGraph, Fill List might not be properly set, verify below') + prop = MiscProperty(ID=ip+1, FillNumM=P[0], FillMList=P[1], FillFSLoc=P[2], FillDens=P[3]) + Graph.addMiscProperty('FillGroups',prop) + + if 'MGProp' in hd.keys(): + # Marine Growth + Graph.addMiscPropertySet('MG') + for ip,P in enumerate(hd['MGProp']): + # MGDpth MGThck MGDens + # (m) (m) (kg/m^3) + prop = Property(ID=ip+1, MGDpth=P[0], MGThck=P[1], MGDens=P[2]) + Graph.addMiscProperty('FillGroups',prop) + + # --- Nodes + Nodes = hd['Joints'] + for iNode,N in enumerate(Nodes): + node = Node(ID=N[0], x=N[1], y=N[2], z=N[3]) + Graph.addNode(node) + Graph.setNodeNodalProp(node, 'AxCoefs', N[4]) + + # --- Elements + PropSets=['SimpleCoefs','DepthCoefs','MemberCoefs'] + Members = hd['Members'] + for ie,E in enumerate(Members): + # MemberID MJointID1 MJointID2 MPropSetID1 MPropSetID2 MDivSize MCoefMod PropPot + EE = E[:5].astype(int) + Type = int(E[6]) # MCoefMod + Pot = E[7].lower()[0]=='t' + elem= Element(ID=EE[0], nodeIDs=EE[1:3], propIDs=EE[3:5], propset='Section', CoefMod=PropSets[Type-1], DivSize=float(E[5]), Pot=Pot) + elem.data['object']='cylinder' + elem.data['color'] = type2Color(Pot) + Graph.addElement(elem) + # Nodal prop data NOTE: can't do that anymore for memebrs with different diameters at the same node + #Graph.setElementNodalProp(elem, propset='Section', propIDs=EE[3:5]) + if Type==1: + # Simple + Graph.setElementNodalProp(elem, propset='SimpleCoefs', propIDs=[1,1]) + else: + print('>>> TODO type DepthCoefs and MemberCoefs') + + return Graph + + +# --------------------------------------------------------------------------------} +# --- SubDyn Summary file +# --------------------------------------------------------------------------------{ +def subdynSumToGraph(data): + """ + data: dict-like object as returned by weio + """ + type2Color=[ + (0.1,0.1,0.1), # Watchout based on background + (0.753,0.561,0.05), # 1 Beam + (0.541,0.753,0.05), # 2 Cable + (0.753,0.05,0.204), # 3 Rigid + (0.918,0.702,0.125), # 3 Rigid + ] + + #print(data.keys()) + DOF2Nodes = data['DOF2Nodes'] + nDOF = data['nDOF_red'] + + Graph = GraphModel() + + # --- Nodes and DOFs + Nodes = data['Nodes'] + for iNode,N in enumerate(Nodes): + if len(N)==9: # Temporary fix + #N[4]=np.float(N[4].split()[0]) + N=N.astype(np.float32) + ID = int(N[0]) + nodeDOFs=DOF2Nodes[(DOF2Nodes[:,1]==ID),0] # NOTE: these were reindex to start at 0 + node = Node(ID=ID, x=N[1], y=N[2], z=N[3], Type=int(N[4]), DOFs=nodeDOFs) + Graph.addNode(node) + + # --- Elements + Elements = data['Elements'] + for ie,E in enumerate(Elements): + nodeIDs=[int(E[1]),int(E[2])] + # shear_[-] Ixx_[m^4] Iyy_[m^4] Jzz_[m^4] T0_[N] + D = np.sqrt(E[7]/np.pi)*4 # <<< Approximation basedon area TODO use I as well + elem= Element(int(E[0]), nodeIDs, Type=int(E[5]), Area=E[7], rho=E[8], E=E[7], G=E[8], D=D) + elem.data['object']='cylinder' + elem.data['color'] = type2Color[int(E[5])] + Graph.addElement(elem) + + #print(self.extent) + #print(self.maxDimension) + + # --- Graph Modes + # Very important sortDims should be None to respect order of nodes + dispGy, posGy, InodesGy, dispCB, posCB, InodesCB = data.getModes(sortDim=None) + for iMode in range(dispGy.shape[2]): + Graph.addMode(displ=dispGy[:,:,iMode],name='GY{:d}'.format(iMode+1), freq=1/(2*np.pi)) + + for iMode in range(dispCB.shape[2]): + Graph.addMode(displ=dispCB[:,:,iMode],name='CB{:d}'.format(iMode+1), freq=data['CB_frequencies'][iMode]) + + #print(Graph.toJSON()) + + + return Graph + + + +if __name__ == '__main__': + from .fast_input_file import FASTInputFile + + filename='../../_data/Monopile/MT100_SD.dat' + # filename='../../_data/Monopile/TetraSpar_SubDyn_v3.dat' + + sd = FASTInputFile(filename) +# sd.write('OutMT.dat') + Graph = sd.toGraph() + Graph.divideElements(2) + print(Graph) + print(Graph.sortNodesBy('z')) + # print(Graph.nodalDataFrame(sortBy='z')) + print(Graph.points) + print(Graph.connectivity) + print(Graph) + +# import numpy as np +# import matplotlib.pyplot as plt +# from matplotlib import collections as mc +# from mpl_toolkits.mplot3d import Axes3D +# fig = plt.figure() +# ax = fig.add_subplot(1,2,1,projection='3d') +# +# lines=Graph.toLines(output='coord') +# for l in lines: +# # ax.add_line(l) +# ax.plot(l[:,0],l[:,1],l[:,2]) +# +# ax.autoscale() +# ax.set_xlim([-40,40]) +# ax.set_ylim([-40,40]) +# ax.set_zlim([-40,40]) +# # ax.margins(0.1) +# +# plt.show() + + diff --git a/pydatview/io/fast_linearization_file.py b/pydatview/io/fast_linearization_file.py new file mode 100644 index 0000000..3409201 --- /dev/null +++ b/pydatview/io/fast_linearization_file.py @@ -0,0 +1,348 @@ +from __future__ import division +from __future__ import print_function +from __future__ import absolute_import +from io import open +from .file import File, isBinary, WrongFormatError, BrokenFormatError +import pandas as pd +import numpy as np +import re + +class FASTLinearizationFile(File): + """ + Read/write an OpenFAST linearization file. The object behaves like a dictionary. + + Main keys + --------- + - 'x', 'xdot' 'u', 'y', 'A', 'B', 'C', 'D' + + Main methods + ------------ + - read, write, toDataFrame, keys, xdescr, ydescr, udescr + + Examples + -------- + + f = FASTLinearizationFile('5MW.1.lin') + print(f.keys()) + print(f['u']) # input operating point + print(f.udescr()) # description of inputs + + # use a dataframe with "named" columns and rows + df = f.toDataFrame() + print(df['A'].columns) + print(df['A']) + + """ + @staticmethod + def defaultExtensions(): + return ['.lin'] + + @staticmethod + def formatName(): + return 'FAST linearization output' + + def _read(self, *args, **kwargs): + self['header']=[] + + def extractVal(lines, key): + for l in lines: + if l.find(key)>=0: + return l.split(key)[1].split()[0] + return None + + def readToMarker(fid, marker, nMax): + lines=[] + for i, line in enumerate(fid): + if i>nMax: + raise BrokenFormatError('`{}` not found in file'.format(marker)) + if line.find(marker)>=0: + break + lines.append(line.strip()) + return lines, line + + def readOP(fid, n): + OP=[] + Var = {'RotatingFrame': [], 'DerivativeOrder': [], 'Description': []} + colNames=fid.readline().strip() + dummy= fid.readline().strip() + bHasDeriv= colNames.find('Derivative Order')>=0 + for i, line in enumerate(fid): + sp=line.strip().split() + if sp[1].find(',')>=0: + # Most likely this OP has three values (e.g. orientation angles) + # For now we discard the two other values + OP.append(float(sp[1][:-1])) + iRot=4 + else: + OP.append(float(sp[1])) + iRot=2 + Var['RotatingFrame'].append(sp[iRot]) + if bHasDeriv: + Var['DerivativeOrder'].append(int(sp[iRot+1])) + Var['Description'].append(' '.join(sp[iRot+2:]).strip()) + else: + Var['DerivativeOrder'].append(-1) + Var['Description'].append(' '.join(sp[iRot+1:]).strip()) + if i>=n-1: + break + return OP, Var + + def readMat(fid, n, m): + vals=[f.readline().strip().split() for i in np.arange(n)] +# try: + return np.array(vals).astype(float) +# except ValueError: +# import pdb; pdb.set_trace() + + # Reading + with open(self.filename, 'r', errors="surrogateescape") as f: + # --- Reader header + self['header'], lastLine=readToMarker(f, 'Jacobians included', 30) + self['header'].append(lastLine) + nx = int(extractVal(self['header'],'Number of continuous states:')) + nxd = int(extractVal(self['header'],'Number of discrete states:' )) + nz = int(extractVal(self['header'],'Number of constraint states:')) + nu = int(extractVal(self['header'],'Number of inputs:' )) + ny = int(extractVal(self['header'],'Number of outputs:' )) + bJac = extractVal(self['header'],'Jacobians included in this file?') + try: + self['Azimuth'] = float(extractVal(self['header'],'Azimuth:')) + except: + self['Azimuth'] = None + try: + self['RotSpeed'] = float(extractVal(self['header'],'Rotor Speed:')) # rad/s + except: + self['RotSpeed'] = None + try: + self['WindSpeed'] = float(extractVal(self['header'],'Wind Speed:')) + except: + self['WindSpeed'] = None + + KEYS=['Order of','A:','B:','C:','D:','ED M:', 'dUdu','dUdy'] + + for i, line in enumerate(f): + line = line.strip() + KeyFound=any([line.find(k)>=0 for k in KEYS]) + if KeyFound: + if line.find('Order of continuous states:')>=0: + self['x'], self['x_info'] = readOP(f, nx) + elif line.find('Order of continuous state derivatives:')>=0: + self['xdot'], self['xdot_info'] = readOP(f, nx) + elif line.find('Order of inputs')>=0: + self['u'], self['u_info'] = readOP(f, nu) + elif line.find('Order of outputs')>=0: + self['y'], self['y_info'] = readOP(f, ny) + elif line.find('A:')>=0: + self['A'] = readMat(f, nx, nx) + elif line.find('B:')>=0: + self['B'] = readMat(f, nx, nu) + elif line.find('C:')>=0: + self['C'] = readMat(f, ny, nx) + elif line.find('D:')>=0: + self['D'] = readMat(f, ny, nu) + elif line.find('dUdu:')>=0: + self['dUdu'] = readMat(f, nu, nu) + elif line.find('dUdy:')>=0: + self['dUdy'] = readMat(f, nu, ny) + elif line.find('ED M:')>=0: + self['EDDOF'] = line[5:].split() + self['M'] = readMat(f, 24, 24) + + def toString(self): + s='' + return s + + def _write(self): + with open(self.filename,'w') as f: + f.write(self.toString()) + + def short_descr(self,slist): + def shortname(s): + s=s.strip() + s = s.replace('(m/s)' , '_[m/s]' ); + s = s.replace('(kW)' , '_[kW]' ); + s = s.replace('(deg)' , '_[deg]' ); + s = s.replace('(N)' , '_[N]' ); + s = s.replace('(kN-m)' , '_[kNm]' ); + s = s.replace('(N-m)' , '_[Nm]' ); + s = s.replace('(kN)' , '_[kN]' ); + s = s.replace('(rpm)' , '_[rpm]' ); + s = s.replace('(rad)' , '_[rad]' ); + s = s.replace('(rad/s)' , '_[rad/s]' ); + s = s.replace('(rad/s^2)', '_[rad/s^2]' ); + s = s.replace('(m/s^2)' , '_[m/s^2]'); + s = s.replace('(deg/s^2)','_[deg/s^2]'); + s = s.replace('(m)' , '_[m]' ); + s = s.replace(', m/s/s','_[m/s^2]'); + s = s.replace(', m/s^2','_[m/s^2]'); + s = s.replace(', m/s','_[m/s]'); + s = s.replace(', m','_[m]'); + s = s.replace(', rad/s/s','_[rad/s^2]'); + s = s.replace(', rad/s^2','_[rad/s^2]'); + s = s.replace(', rad/s','_[rad/s]'); + s = s.replace(', rad','_[rad]'); + s = s.replace(', -','_[-]'); + s = s.replace(', Nm/m','_[Nm/m]'); + s = s.replace(', Nm','_[Nm]'); + s = s.replace(', N/m','_[N/m]'); + s = s.replace(', N','_[N]'); + s = s.replace('(1)','1') + s = s.replace('(2)','2') + s = s.replace('(3)','3') + s= re.sub(r'\([^)]*\)','', s) # remove parenthesis + s = s.replace('ED ',''); + s = s.replace('BD_','BD_B'); + s = s.replace('IfW ',''); + s = s.replace('Extended input: ','') + s = s.replace('1st tower ','qt1'); + s = s.replace('2nd tower ','qt2'); + nd = s.count('First time derivative of ') + if nd>=0: + s = s.replace('First time derivative of ' ,''); + if nd==1: + s = 'd_'+s.strip() + elif nd==2: + s = 'dd_'+s.strip() + s = s.replace('Variable speed generator DOF ','psi_rot'); # NOTE: internally in FAST this is the azimuth of the rotor + s = s.replace('fore-aft bending mode DOF ' ,'FA' ); + s = s.replace('side-to-side bending mode DOF','SS' ); + s = s.replace('bending-mode DOF of blade ' ,'' ); + s = s.replace(' rotational-flexibility DOF, rad','-ROT' ); + s = s.replace('rotational displacement in ','rot' ); + s = s.replace('Drivetrain','DT' ); + s = s.replace('translational displacement in ','trans' ); + s = s.replace('finite element node ','N' ); + s = s.replace('-component position of node ','posN') + s = s.replace('-component inflow on tower node','TwrN') + s = s.replace('-component inflow on blade 1, node','Bld1N') + s = s.replace('-component inflow on blade 2, node','Bld2N') + s = s.replace('-component inflow on blade 3, node','Bld3N') + s = s.replace('-component inflow velocity at node','N') + s = s.replace('X translation displacement, node','TxN') + s = s.replace('Y translation displacement, node','TyN') + s = s.replace('Z translation displacement, node','TzN') + s = s.replace('X translation velocity, node','TVxN') + s = s.replace('Y translation velocity, node','TVyN') + s = s.replace('Z translation velocity, node','TVzN') + s = s.replace('X translation acceleration, node','TAxN') + s = s.replace('Y translation acceleration, node','TAyN') + s = s.replace('Z translation acceleration, node','TAzN') + s = s.replace('X orientation angle, node' ,'RxN') + s = s.replace('Y orientation angle, node' ,'RyN') + s = s.replace('Z orientation angle, node' ,'RzN') + s = s.replace('X rotation velocity, node' ,'RVxN') + s = s.replace('Y rotation velocity, node' ,'RVyN') + s = s.replace('Z rotation velocity, node' ,'RVzN') + s = s.replace('X rotation acceleration, node' ,'RAxN') + s = s.replace('Y rotation acceleration, node' ,'RAyN') + s = s.replace('Z rotation acceleration, node' ,'RAzN') + s = s.replace('X force, node','FxN') + s = s.replace('Y force, node','FyN') + s = s.replace('Z force, node','FzN') + s = s.replace('X moment, node','MxN') + s = s.replace('Y moment, node','MyN') + s = s.replace('Z moment, node','MzN') + s = s.replace('FX', 'Fx') + s = s.replace('FY', 'Fy') + s = s.replace('FZ', 'Fz') + s = s.replace('MX', 'Mx') + s = s.replace('MY', 'My') + s = s.replace('MZ', 'Mz') + s = s.replace('FKX', 'FKx') + s = s.replace('FKY', 'FKy') + s = s.replace('FKZ', 'FKz') + s = s.replace('MKX', 'MKx') + s = s.replace('MKY', 'MKy') + s = s.replace('MKZ', 'MKz') + s = s.replace('Nodes motion','') + s = s.replace('cosine','cos' ); + s = s.replace('sine','sin' ); + s = s.replace('collective','coll.'); + s = s.replace('Blade','Bld'); + s = s.replace('rotZ','TORS-R'); + s = s.replace('transX','FLAP-D'); + s = s.replace('transY','EDGE-D'); + s = s.replace('rotX','EDGE-R'); + s = s.replace('rotY','FLAP-R'); + s = s.replace('flapwise','FLAP'); + s = s.replace('edgewise','EDGE'); + s = s.replace('horizontal surge translation DOF','Surge'); + s = s.replace('horizontal sway translation DOF','Sway'); + s = s.replace('vertical heave translation DOF','Heave'); + s = s.replace('roll tilt rotation DOF','Roll'); + s = s.replace('pitch tilt rotation DOF','Pitch'); + s = s.replace('yaw rotation DOF','Yaw'); + s = s.replace('vertical power-law shear exponent','alpha') + s = s.replace('horizontal wind speed ','WS') + s = s.replace('propagation direction','WD') + s = s.replace(' pitch command','pitch') + s = s.replace('HSS_','HSS') + s = s.replace('Bld','B') + s = s.replace('tower','Twr') + s = s.replace('Tower','Twr') + s = s.replace('Nacelle','Nac') + s = s.replace('Platform','Ptfm') + s = s.replace('SrvD','SvD') + s = s.replace('Generator torque','Qgen') + s = s.replace('coll. blade-pitch command','PitchColl') + s = s.replace('wave elevation at platform ref point','WaveElevRefPoint') + s = s.replace('1)','1'); + s = s.replace('2)','2'); + s = s.replace('3)','3'); + s = s.replace(',',''); + s = s.replace(' ',''); + s=s.strip() + return s + return [shortname(s) for s in slist] + + def xdescr(self): + return self.short_descr(self['x_info']['Description']) + + def xdotdescr(self): + return self.short_descr(self['xdot_info']['Description']) + + def ydescr(self): + if 'y_info' in self.keys(): + return self.short_descr(self['y_info']['Description']) + else: + return [] + def udescr(self): + if 'u_info' in self.keys(): + return self.short_descr(self['u_info']['Description']) + else: + return [] + + def _toDataFrame(self): + dfs={} + + xdescr_short = self.xdescr() + xdotdescr_short = self.xdotdescr() + ydescr_short = self.ydescr() + udescr_short = self.udescr() + + if 'A' in self.keys(): + dfs['A'] = pd.DataFrame(data = self['A'], index=xdescr_short, columns=xdescr_short) + if 'B' in self.keys(): + dfs['B'] = pd.DataFrame(data = self['B'], index=xdescr_short, columns=udescr_short) + if 'C' in self.keys(): + dfs['C'] = pd.DataFrame(data = self['C'], index=ydescr_short, columns=xdescr_short) + if 'D' in self.keys(): + dfs['D'] = pd.DataFrame(data = self['D'], index=ydescr_short, columns=udescr_short) + if 'x' in self.keys(): + dfs['x'] = pd.DataFrame(data = np.asarray(self['x']).reshape((1,-1)), columns=xdescr_short) + if 'xdot' in self.keys(): + dfs['xdot'] = pd.DataFrame(data = np.asarray(self['xdot']).reshape((1,-1)), columns=xdotdescr_short) + if 'u' in self.keys(): + dfs['u'] = pd.DataFrame(data = np.asarray(self['u']).reshape((1,-1)), columns=udescr_short) + if 'y' in self.keys(): + dfs['y'] = pd.DataFrame(data = np.asarray(self['y']).reshape((1,-1)), columns=ydescr_short) + if 'M' in self.keys(): + dfs['M'] = pd.DataFrame(data = self['M'], index=self['EDDOF'], columns=self['EDDOF']) + if 'dUdu' in self.keys(): + dfs['dUdu'] = pd.DataFrame(data = self['dUdu'], index=udescr_short, columns=udescr_short) + if 'dUdy' in self.keys(): + dfs['dUdy'] = pd.DataFrame(data = self['dUdy'], index=udescr_short, columns=ydescr_short) + + return dfs + + diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py new file mode 100644 index 0000000..8f1d982 --- /dev/null +++ b/pydatview/io/fast_output_file.py @@ -0,0 +1,498 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from io import open +from builtins import map +from builtins import range +from builtins import chr +from builtins import str +from future import standard_library +standard_library.install_aliases() + +from itertools import takewhile + +# try: +from .file import File, WrongFormatError, BrokenReaderError, EmptyFileError +# except: +# # --- Allowing this file to be standalone.. +# class WrongFormatError(Exception): +# pass +# class WrongReaderError(Exception): +# pass +# class EmptyFileError(Exception): +# pass +# File = dict +try: + from .csv_file import CSVFile +except: + print('CSVFile not available') +import numpy as np +import pandas as pd +import struct +import os +import re + + +# --------------------------------------------------------------------------------} +# --- OUT FILE +# --------------------------------------------------------------------------------{ +class FASTOutputFile(File): + """ + Read an OpenFAST ouput file (.out, .outb, .elev). + + Main methods + ------------ + - read, write, toDataFrame + + Examples + -------- + + # read an output file, convert it to pandas dataframe, modify it, write it back + f = FASTOutputFile('5MW.outb') + df=f.toDataFrame() + time = df['Time_[s]'] + Omega = df['RotSpeed_[rpm]'] + df['Time_[s]'] -=100 + f.writeDataFrame(df, '5MW_TimeShifted.outb') + + """ + + @staticmethod + def defaultExtensions(): + return ['.out','.outb','.elm','.elev'] + + @staticmethod + def formatName(): + return 'FAST output file' + + def _read(self): + def readline(iLine): + with open(self.filename) as f: + for i, line in enumerate(f): + if i==iLine-1: + return line.strip() + elif i>=iLine: + break + + ext = os.path.splitext(self.filename.lower())[1] + self.info={} + self['binary']=False + try: + if ext in ['.out','.elev']: + self.data, self.info = load_ascii_output(self.filename) + elif ext=='.outb': + self.data, self.info = load_binary_output(self.filename) + self['binary']=True + elif ext=='.elm': + F=CSVFile(filename=self.filename, sep=' ', commentLines=[0,2],colNamesLine=1) + self.data = F.data + del F + self.info['attribute_units']=readline(3).replace('sec','s').split() + self.info['attribute_names']=self.data.columns.values + else: + self.data, self.info = load_output(self.filename) + except MemoryError as e: + raise BrokenReaderError('FAST Out File {}: Memory error encountered\n{}'.format(self.filename,e)) + except Exception as e: + raise WrongFormatError('FAST Out File {}: {}'.format(self.filename,e.args)) + if self.data.shape[0]==0: + raise EmptyFileError('This FAST output file contains no data: {}'.format(self.filename)) + + if self.info['attribute_units'] is not None: + self.info['attribute_units'] = [re.sub(r'[()\[\]]','',u) for u in self.info['attribute_units']] + + + def _write(self): + if self['binary']: + channels = self.data + chanNames = self.info['attribute_names'] + chanUnits = self.info['attribute_units'] + descStr = self.info['description'] + writeBinary(self.filename, channels, chanNames, chanUnits, fileID=2, descStr=descStr) + else: + # ascii output + with open(self.filename,'w') as f: + f.write('\t'.join(['{:>10s}'.format(c) for c in self.info['attribute_names']])+'\n') + f.write('\t'.join(['{:>10s}'.format('('+u+')') for u in self.info['attribute_units']])+'\n') + # TODO better.. + f.write('\n'.join(['\t'.join(['{:10.4f}'.format(y[0])]+['{:10.3e}'.format(x) for x in y[1:]]) for y in self.data])) + + def _toDataFrame(self): + if self.info['attribute_units'] is not None: + cols=[n+'_['+u.replace('sec','s')+']' for n,u in zip(self.info['attribute_names'],self.info['attribute_units'])] + else: + cols=self.info['attribute_names'] + if isinstance(self.data, pd.DataFrame): + df= self.data + df.columns=cols + else: + df = pd.DataFrame(data=self.data,columns=cols) + + return df + + def writeDataFrame(self, df, filename, binary=True): + writeDataFrame(df, filename, binary=binary) + +# -------------------------------------------------------------------------------- +# --- Helper low level functions +# -------------------------------------------------------------------------------- +def load_output(filename): + """Load a FAST binary or ascii output file + + Parameters + ---------- + filename : str + filename + + Returns + ------- + data : ndarray + data values + info : dict + info containing: + - name: filename + - description: description of dataset + - attribute_names: list of attribute names + - attribute_units: list of attribute units + """ + + assert os.path.isfile(filename), "File, %s, does not exists" % filename + with open(filename, 'r') as f: + try: + f.readline() + except UnicodeDecodeError: + return load_binary_output(filename) + return load_ascii_output(filename) + +def load_ascii_output(filename): + with open(filename) as f: + info = {} + info['name'] = os.path.splitext(os.path.basename(filename))[0] + # Header is whatever is before the keyword `time` + in_header = True + header = [] + while in_header: + l = f.readline() + if not l: + raise Exception('Error finding the end of FAST out file header. Keyword Time missing.') + in_header= (l+' dummy').lower().split()[0] != 'time' + if in_header: + header.append(l) + else: + info['description'] = header + info['attribute_names'] = l.split() + info['attribute_units'] = [unit[1:-1] for unit in f.readline().split()] + # --- + # Data, up to end of file or empty line (potential comment line at the end) +# data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(np.float) + # --- + data = np.loadtxt(f, comments=('This')) # Adding "This" for the Hydro Out files.. + return data, info + + +def load_binary_output(filename, use_buffer=True): + """ + 03/09/15: Ported from ReadFASTbinary.m by Mads M Pedersen, DTU Wind + 24/10/18: Low memory/buffered version by E. Branlard, NREL + 18/01/19: New file format for exctended channels, by E. Branlard, NREL + + Info about ReadFASTbinary.m: + % Author: Bonnie Jonkman, National Renewable Energy Laboratory + % (c) 2012, National Renewable Energy Laboratory + % + % Edited for FAST v7.02.00b-bjj 22-Oct-2012 + """ + def fread(fid, n, type): + fmt, nbytes = {'uint8': ('B', 1), 'int16':('h', 2), 'int32':('i', 4), 'float32':('f', 4), 'float64':('d', 8)}[type] + return struct.unpack(fmt * n, fid.read(nbytes * n)) + + def freadRowOrderTableBuffered(fid, n, type_in, nCols, nOff=0, type_out='float64'): + """ + Reads of row-ordered table from a binary file. + + Read `n` data of type `type_in`, assumed to be a row ordered table of `nCols` columns. + Memory usage is optimized by allocating the data only once. + Buffered reading is done for improved performances (in particular for 32bit python) + + `nOff` allows for additional column space at the begining of the storage table. + Typically, `nOff=1`, provides a column at the beginning to store the time vector. + + @author E.Branlard, NREL + + """ + fmt, nbytes = {'uint8': ('B', 1), 'int16':('h', 2), 'int32':('i', 4), 'float32':('f', 4), 'float64':('d', 8)}[type_in] + nLines = int(n/nCols) + GoodBufferSize = 4096*40 + nLinesPerBuffer = int(GoodBufferSize/nCols) + BufferSize = nCols * nLinesPerBuffer + nBuffer = int(n/BufferSize) + # Allocation of data + data = np.zeros((nLines,nCols+nOff), dtype = type_out) + # Reading + try: + nIntRead = 0 + nLinesRead = 0 + while nIntRead0: + op,cl = chars + iu=c.rfind(op) + if iu>1: + name = c[:iu] + unit = c[iu+1:].replace(cl,'') + if name[-1]=='_': + name=name[:-1] + + chanNames.append(name) + chanUnits.append(unit) + + if binary: + writeBinary(filename, channels, chanNames, chanUnits) + else: + NotImplementedError() + + +def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr=''): + """ + Write an OpenFAST binary file. + + Based on contributions from + Hugo Castro, David Schlipf, Hochschule Flensburg + + Input: + FileName - string: contains file name to open + Channels - 2-D array: dimension 1 is time, dimension 2 is channel + ChanName - cell array containing names of output channels + ChanUnit - cell array containing unit names of output channels, preferably surrounded by parenthesis + FileID - constant that determines if the time is stored in the + output, indicating possible non-constant time step + DescStr - String describing the file + """ + # Data sanitization + chanNames = list(chanNames) + channels = np.asarray(channels) + if chanUnits[0][0]!='(': + chanUnits = ['('+u+')' for u in chanUnits] # units surrounded by parenthesis to match OpenFAST convention + + nT, nChannelsWithTime = np.shape(channels) + nChannels = nChannelsWithTime - 1 + + # For FileID =2, time needs to be present and at the first column + try: + iTime = chanNames.index('Time') + except ValueError: + raise Exception('`Time` needs to be present in channel names' ) + if iTime!=0: + raise Exception('`Time` needs to be the first column of `chanName`' ) + + time = channels[:,iTime] + timeStart = time[0] + timeIncr = time[1]-time[0] + dataWithoutTime = channels[:,1:] + + # Compute data range, scaling and offsets to convert to int16 + # To use the int16 range to its fullest, the max float is matched to 2^15-1 and the + # the min float is matched to -2^15. Thus, we have the to equations we need + # to solve to get scaling and offset, see line 120 of ReadFASTbinary: + # Int16Max = FloatMax * Scaling + Offset + # Int16Min = FloatMin * Scaling + Offset + int16Max = np.single( 32767.0) # Largest integer represented in 2 bytes, 2**15 - 1 + int16Min = np.single(-32768.0) # Smallest integer represented in 2 bytes -2**15 + int16Rng = np.single(int16Max - int16Min) # Max Range of 2 byte integer + mins = np.min(dataWithoutTime, axis=0) + ranges = np.single(np.max(dataWithoutTime, axis=0) - mins) + ranges[ranges==0]=1 # range set to 1 for constant channel. In OpenFAST: /sqrt(epsilon(1.0_SiKi)) + ColScl = np.single(int16Rng/ranges) + ColOff = np.single(int16Min - np.single(mins)*ColScl) + + #Just available for fileID + if fileID != 2: + print("current version just works with FileID = 2") + + else: + with open(fileName,'wb') as fid: + # Notes on struct: + # @ is used for packing in native byte order + # B - unsigned integer 8 bits + # h - integer 16 bits + # i - integer 32 bits + # f - float 32 bits + # d - float 64 bits + + # Write header informations + fid.write(struct.pack('@h',fileID)) + fid.write(struct.pack('@i',nChannels)) + fid.write(struct.pack('@i',nT)) + fid.write(struct.pack('@d',timeStart)) + fid.write(struct.pack('@d',timeIncr)) + fid.write(struct.pack('@{}f'.format(nChannels), *ColScl)) + fid.write(struct.pack('@{}f'.format(nChannels), *ColOff)) + descStrASCII = [ord(char) for char in descStr] + fid.write(struct.pack('@i',len(descStrASCII))) + fid.write(struct.pack('@{}B'.format(len((descStrASCII))), *descStrASCII)) + + # Write channel names + for chan in chanNames: + ordchan = [ord(char) for char in chan]+ [32]*(10-len(chan)) + fid.write(struct.pack('@10B', *ordchan)) + + # Write channel units + for unit in chanUnits: + ordunit = [ord(char) for char in unit]+ [32]*(10-len(unit)) + fid.write(struct.pack('@10B', *ordunit)) + + # Pack data + packedData=np.zeros((nT, nChannels), dtype=np.int16) + for iChan in range(nChannels): + packedData[:,iChan] = np.clip( ColScl[iChan]*dataWithoutTime[:,iChan]+ColOff[iChan], int16Min, int16Max) + + # Write data + fid.write(struct.pack('@{}h'.format(packedData.size), *packedData.flatten())) + fid.close() + + +if __name__ == "__main__": + B=FASTOutputFile('tests/example_files/FASTOutBin.outb') + df=B.toDataFrame() + B.writeDataFrame(df, 'tests/example_files/FASTOutBin_OUT.outb') + + diff --git a/pydatview/io/fast_summary_file.py b/pydatview/io/fast_summary_file.py new file mode 100644 index 0000000..d3004d5 --- /dev/null +++ b/pydatview/io/fast_summary_file.py @@ -0,0 +1,267 @@ +import numpy as np +import pandas as pd +from io import open +import os +# Local +from .mini_yaml import yaml_read + +try: + from .file import File, EmptyFileError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + File=dict + +# --------------------------------------------------------------------------------} +# --- Main Class +# --------------------------------------------------------------------------------{ +class FASTSummaryFile(File): + """ + Read an OpenFAST summary file (.sum, .yaml). The object behaves as a dictionary. + NOTE: open new subdyn format supported. + + Main methods + ------------ + - read, toDataFrame + + Examples + -------- + + # read a subdyn summary file + sum = FASTSummaryFile('5MW.SD.sum.yaml') + print(sum['module']) # SubDyn + M = sum['M'] # Mass matrix + K = sum['K'] # stiffness matrix + + """ + + @staticmethod + def defaultExtensions(): + return ['.sum','.yaml'] + + @staticmethod + def formatName(): + return 'FAST summary file' + + def __init__(self,filename=None, **kwargs): + self.filename = None + if filename: + self.read(filename, **kwargs) + + def read(self, filename=None, header_only=False): + """ """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + + with open(self.filename, 'r', errors="surrogateescape") as fid: + header= readFirstLines(fid, 4) + if any(['subdyn' in s.lower() for s in header]): + self['module']='SubDyn' + readSubDynSum(self) + else: + raise NotImplementedError('This summary file format is not yet supported') + + def toDataFrame(self): + if 'module' not in self.keys(): + raise Exception(''); + if self['module']=='SubDyn': + raise Exception('This should not happen since class was added to subdyn object') + # dfs=subDynToDataFrame(self) + return dfs + + def toGraph(self): + from .fast_input_file_graph import fastToGraph + return fastToGraph(self) + + + +# --------------------------------------------------------------------------------} +# --- Helper functions +# --------------------------------------------------------------------------------{ +def readFirstLines(fid, nLines): + lines=[] + for i, line in enumerate(fid): + lines.append(line.strip()) + if i==nLines: + break + return lines + +# --------------------------------------------------------------------------------} +# --- Sub-reader/class for SubDyn summary files +# --------------------------------------------------------------------------------{ +def readSubDynSum(self): + + # Read data + #T=yaml.load(fid, Loader=yaml.SafeLoader) + yaml_read(self.filename, self) + + # --- Treatement of useful data + if self['DOF2Nodes'].shape[1]==3: + self['DOF2Nodes']=np.column_stack((np.arange(self['DOF2Nodes'].shape[0])+1,self['DOF2Nodes'])) + # NOTE: DOFs are reindexed to start at 0 + self['DOF2Nodes'][:,0]-=1 + self['DOF___L'] -=1 # internal DOFs + self['DOF___B'] -=1 # internal + self['DOF___F'] -=1 # fixed DOFs + + self['CB_frequencies']=self['CB_frequencies'].ravel() + self['X'] = self['Nodes'][:,1].astype(float) + self['Y'] = self['Nodes'][:,2].astype(float) + self['Z'] = self['Nodes'][:,3].astype(float) + + # --- Useful methods that will be added to the class + def NodesDisp(self, IDOF, UDOF, maxDisp=None, sortDim=None): + DOF2Nodes = self['DOF2Nodes'] + # NOTE: SubDyn nodes in the summary files are sorted + # so the position we give are for all Nodes + INodes = list(np.sort(np.unique(DOF2Nodes[IDOF,1]))) # Sort + nShapes = UDOF.shape[1] + disp=np.empty((len(INodes),3,nShapes)); disp.fill(np.nan) + pos=np.empty((len(INodes),3)) ; pos.fill(np.nan) + # TODO + # handle T_red for rigid and joints + for i,iDOF in enumerate(IDOF): + iNode = DOF2Nodes[iDOF,1] + nDOFPerNode = DOF2Nodes[iDOF,2] + nodeDOF = DOF2Nodes[iDOF,3] + iiNode = INodes.index(iNode) + if nodeDOF<=3: + pos[iiNode, 0]=self['X'][iNode-1] + pos[iiNode, 1]=self['Y'][iNode-1] + pos[iiNode, 2]=self['Z'][iNode-1] + for iShape in np.arange(nShapes): + disp[iiNode, nodeDOF-1, iShape] = UDOF[i, iShape] + # Scaling + if maxDisp is not None: + for iShape in np.arange(nShapes): + mD=np.nanmax(np.abs(disp[:, :, iShape])) + if mD>1e-5: + disp[:, :, iShape] *= maxDisp/mD + # Sorting according to a dimension + if sortDim is not None: + I=np.argsort(pos[:,sortDim]) + INodes = np.array(INodes)[I] + disp = disp[I,:,:] + pos = pos[I,:] + return disp, pos, INodes + + def getModes(data, maxDisp=None, sortDim=2): + """ return Guyan and CB modes""" + if maxDisp is None: + #compute max disp such as it's 10% of maxdimension + dx = np.max(self['X'])-np.min(self['X']) + dy = np.max(self['Y'])-np.min(self['Y']) + dz = np.max(self['Z'])-np.min(self['Z']) + maxDisp = np.max([dx,dy,dz])*0.1 + + # NOTE: DOF have been reindexed -1 + DOF_B = data['DOF___B'].ravel() + DOF_F = data['DOF___F'].ravel() + DOF_K = (np.concatenate((DOF_B,data['DOF___L'].ravel(), DOF_F))).astype(int) + + # CB modes + PhiM = data['PhiM'] + Phi_CB = np.vstack((np.zeros((len(DOF_B),PhiM.shape[1])),PhiM, np.zeros((len(DOF_F),PhiM.shape[1])))) + dispCB, posCB, INodesCB = data.NodesDisp(DOF_K, Phi_CB, maxDisp=maxDisp, sortDim=sortDim) + # Guyan modes + PhiR = data['PhiR'] + Phi_Guyan = np.vstack((np.eye(len(DOF_B)),PhiR, np.zeros((len(DOF_F),PhiR.shape[1])))) + dispGy, posGy, INodesGy = data.NodesDisp(DOF_K, Phi_Guyan, maxDisp=maxDisp, sortDim=sortDim) + + return dispGy, posGy, INodesGy, dispCB, posCB, INodesCB + + + def subDynToJson(data, outfile=None): + """ Convert to a "JSON" format + + TODO: convert to graph and use graph.toJSON + + """ + + dispGy, posGy, _, dispCB, posCB, _ = data.getModes() + + Nodes = self['Nodes'] + Elements = self['Elements'] + Elements[:,0]-=1 + Elements[:,1]-=1 + Elements[:,2]-=1 + CB_freq = data['CB_frequencies'].ravel() + + d=dict(); + d['Connectivity']=Elements[:,[1,2]].astype(int).tolist(); + d['Nodes']=Nodes[:,[1,2,3]].tolist() + d['ElemProps']=[{'shape':'cylinder','type':int(Elements[iElem,5]),'Diam':np.sqrt(Elements[iElem,7]/np.pi)*4} for iElem in range(len(Elements))] # NOTE: diameter is cranked up + # disp[iiNode, nodeDOF-1, iShape] = UDOF[i, iShape] + + d['Modes']=[ + { + 'name':'GY{:d}'.format(iMode+1), + 'omega':1, + 'Displ':dispGy[:,:,iMode].tolist() + } for iMode in range(dispGy.shape[2]) ] + d['Modes']+=[ + { + 'name':'CB{:d}'.format(iMode+1), + 'omega':CB_freq[iMode]*2*np.pi, #in [rad/s] + 'Displ':dispCB[:,:,iMode].tolist() + } for iMode in range(dispCB.shape[2]) ] + d['groundLevel']=np.min(data['Z']) # TODO + + if outfile is not None: + import json + with open(outfile, 'w', encoding='utf-8') as f: + try: + f.write(unicode(json.dumps(d, ensure_ascii=False))) #, indent=2) + except: + json.dump(d, f, indent=2) + return d + + + def subDynToDataFrame(data): + """ Convert to DataFrame containing nodal displacements """ + def toDF(pos,disp,preffix=''): + disp[np.isnan(disp)]=0 + disptot=disp.copy() + columns=[] + for ishape in np.arange(disp.shape[2]): + disptot[:,:,ishape]= pos + disp[:,:,ishape] + sMode=preffix+'Mode{:d}'.format(ishape+1) + columns+=[sMode+'x_[m]',sMode+'y_[m]',sMode+'z_[m]'] + disptot= np.moveaxis(disptot,2,1).reshape(disptot.shape[0],disptot.shape[1]*disptot.shape[2]) + disp = np.moveaxis(disp,2,1).reshape(disp.shape[0],disp.shape[1]*disp.shape[2]) + df= pd.DataFrame(data = disptot ,columns = columns) + # remove zero + dfDisp= pd.DataFrame(data = disp ,columns = columns) + df = df.loc[:, (dfDisp != 0).any(axis=0)] + dfDisp = dfDisp.loc[:, (dfDisp != 0).any(axis=0)] + dfDisp.columns = [c.replace('Mode','Disp') for c in dfDisp.columns.values] + return df, dfDisp + + dispGy, posGy, _, dispCB, posCB, _ = data.getModes() + + columns = ['z_[m]','x_[m]','y_[m]'] + dataZXY = np.column_stack((posGy[:,2],posGy[:,0],posGy[:,1])) + dfZXY = pd.DataFrame(data = dataZXY, columns=columns) + df1, df1d = toDF(posGy, dispGy,'Guyan') + df2, df2d = toDF(posCB, dispCB,'CB') + df = pd.concat((dfZXY, df1, df2, df1d, df2d), axis=1) + return df + + # adding method to class dynamically to give it a "SubDyn Summary flavor" + setattr(FASTSummaryFile, 'NodesDisp' , NodesDisp) + setattr(FASTSummaryFile, 'toDataFrame', subDynToDataFrame) + setattr(FASTSummaryFile, 'toJSON' , subDynToJson) + setattr(FASTSummaryFile, 'getModes' , getModes) + + return self + + +if __name__=='__main__': + T=FASTSummaryFile('../Pendulum.SD.sum.yaml') + df=T.toDataFrame() + print(df) diff --git a/pydatview/io/fast_wind_file.py b/pydatview/io/fast_wind_file.py new file mode 100644 index 0000000..b642d89 --- /dev/null +++ b/pydatview/io/fast_wind_file.py @@ -0,0 +1,84 @@ +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from io import open +from builtins import map +from builtins import range +from builtins import chr +from builtins import str +from future import standard_library +standard_library.install_aliases() + +from .csv_file import CSVFile +from .file import isBinary, WrongFormatError +import numpy as np +import pandas as pd + +class FASTWndFile(CSVFile): + + @staticmethod + def defaultExtensions(): + return ['.wnd'] + + @staticmethod + def formatName(): + return 'FAST determ. wind file' + + def __init__(self, *args, **kwargs): + self.colNames=['Time','WindSpeed','WindDir','VertSpeed','HorizShear','VertShear','LinVShear','GustSpeed'] + self.units=['[s]','[m/s]','[deg]','[m/s]','[-]','[-]','[-]','[m/s]'] + Cols=['{}_{}'.format(c,u) for c,u in zip(self.colNames,self.units)] + + header=[] + header+=['!Wind file.'] + header+=['!Time Wind Wind Vert. Horiz. Vert. LinV Gust'] + header+=['! Speed Dir Speed Shear Shear Shear Speed'] + + super(FASTWndFile, self).__init__(sep=' ',commentChar='!',colNames=Cols, header=header, *args, **kwargs) + + def _read(self, *args, **kwargs): + if isBinary(self.filename): + raise WrongFormatError('This is a binary file (turbulence file?) not a FAST ascii determinisctic wind file') + super(FASTWndFile, self)._read(*args, **kwargs) + + def _write(self, *args, **kwargs): + super(FASTWndFile, self)._write(*args, **kwargs) + + + def _toDataFrame(self): + return self.data + + +# --------------------------------------------------------------------------------} +# --- Functions specific to file type +# --------------------------------------------------------------------------------{ + def stepWind(self,WSstep=1,WSmin=3,WSmax=25,tstep=100,dt=0.5,tmin=0,tmax=999): + """ Set the wind file to a step wind + tstep: can be an array of size 2 [tstepmax tstepmin] + + + """ + + Steps= np.arange(WSmin,WSmax+WSstep,WSstep) + if hasattr(tstep,'__len__'): + tstep = np.around(np.linspace(tstep[0], tstep[1], len(Steps)),0) + else: + tstep = len(Steps)*[tstep] + nCol = len(self.colNames) + nRow = len(Steps)*2 + M = np.zeros((nRow,nCol)); + M[0,0] = tmin + M[0,1] = WSmin + for i,s in enumerate(Steps[:-1]): + M[2*i+1,0] = tmin + tstep[i]-dt + M[2*i+2,0] = tmin + tstep[i] + tmin +=tstep[i] + M[2*i+1,1] = Steps[i] + if i14 and code<31): + return True + return False + except UnicodeDecodeError: + return True + +def ascii_comp(file1,file2,bDelete=False): + """ Compares two ascii files line by line. + Comparison is done ignoring multiple white spaces for now""" + # --- Read original as ascii + with open(file1, 'r') as f1: + lines1 = f1.read().splitlines(); + lines1 = '|'.join([l.replace('\t',' ').strip() for l in lines1]) + lines1 = ' '.join(lines1.split()) + # --- Read second file as ascii + with open(file2, 'r') as f2: + lines2 = f2.read().splitlines(); + lines2 = '|'.join([l.replace('\t',' ').strip() for l in lines2]) + lines2 = ' '.join(lines2.split()) + + if lines1 == lines2: + if bDelete: + os.remove(file2) + return True + else: + return False diff --git a/pydatview/io/file_formats.py b/pydatview/io/file_formats.py new file mode 100644 index 0000000..daa5400 --- /dev/null +++ b/pydatview/io/file_formats.py @@ -0,0 +1,29 @@ +from .file import WrongFormatError + +def isRightFormat(fileformat, filename, **kwargs): + """ Tries to open a file, return true and the file if it succeeds """ + #raise NotImplementedError("Method must be implemented in the subclass") + try: + F=fileformat.constructor(filename=filename, **kwargs) + return True,F + except MemoryError: + raise + except WrongFormatError: + return False,None + except: + raise + +class FileFormat(): + def __init__(self,fileclass=None): + self.constructor = fileclass + if fileclass is None: + self.extensions = [] + self.name = '' + else: + self.extensions = fileclass.defaultExtensions() + self.name = fileclass.formatName() + + + def __repr__(self): + return 'FileFormat object: {} ({})'.format(self.name,self.extensions[0]) + diff --git a/pydatview/io/flex_blade_file.py b/pydatview/io/flex_blade_file.py new file mode 100644 index 0000000..4442ffa --- /dev/null +++ b/pydatview/io/flex_blade_file.py @@ -0,0 +1,142 @@ +from __future__ import division,unicode_literals,print_function,absolute_import +from builtins import map, range, chr, str +from io import open +from future import standard_library +standard_library.install_aliases() + +from .file import File, WrongFormatError, BrokenFormatError +import numpy as np +import pandas as pd +import os + +#from .wetb.fast import fast_io + + +class FLEXBladeFile(File): + + @staticmethod + def defaultExtensions(): + return ['.bld','.bla','.00X'] #'.001 etc..' + + @staticmethod + def formatName(): + return 'FLEX blade file' + + def _read(self): + headers_all = ['r_[m]','EIFlap_[Nm2]','EIEdge_[Nm2]','GKt_[Nm2]','Mass_[kg/m]','Jxx_[kg.m]','PreBendFlap_[m]','PreBendEdge_[m]'\ + ,'Str.Twist_[deg]','PhiOut_[deg]','Ycog_[m]','Yshc_[m]','CalcOutput_[0/1]'\ + ,'Chord_[m]','AeroTwist_[deg]','RelThickness_[%]','AeroCenter_[m]','AeroTorsion_[0/1]','ProfileSet_[#]'] + with open(self.filename, 'r', errors="surrogateescape") as f: + try: + firstline = f.readline().strip() + nSections = int(f.readline().strip().split()[0]) + except: + raise WrongFormatError('Unable to read first two lines of blade file') + try: + self.version=int(firstline[1:4]) + except: + self.version=0 + # --- Different handling depending on version + if self.version==0: + # Version 0 struct has no GKt + # Version 0 aero has no profile set, no TorsionAero + nColsStruct = 8 + nColsAero = 5 + struct_headers = ['r_[m]','EIFlap_[Nm2]','EIEdge_[Nm2]','Mass_[kg/m]','Str.Twist_[deg]','CalcOutput_[0/1]','PreBendFlap_[m]','PreBendEdge_[m]'] + aero_headers = ['X_BladeRoot_[m]','Chord_[m]','AeroTwist_[deg]','RelThickness_[%]','AeroCenter_[m]'] + elif self.version==1: + # Version 1 struct has GKt + # Version 1 aero has no profile set + nColsStruct = 8 + nColsAero = 6 + struct_headers = ['r_[m]','EIFlap_[Nm2]','EIEdge_[Nm2]','Mass_[kg/m]','Str.Twist_[deg]','CalcOutput_[0/1]','PreBendFlap_[m]','PreBendEdge_[m]'] + aero_headers = ['X_BladeRoot_[m]','Chord_[m]','AeroTwist_[deg]','RelThickness_[%]','AeroCenter_[m]','AeroTorsion_[0/1]'] + elif self.version==2: + nColsStruct = 9 + nColsAero = 7 + struct_headers = ['r_[m]','EIFlap_[Nm2]','EIEdge_[Nm2]','Mass_[kg/m]','Str.Twist_[deg]','CalcOutput_[0/1]','PreBendFlap_[m]','PreBendEdge_[m]','GKt_[Nm2]'] + aero_headers = ['X_BladeRoot_[m]','Chord_[m]','AeroTwist_[deg]','RelThickness_[%]','AeroCenter_[m]','AeroTorsion_[0/1]','ProfileSet_[#]'] + elif self.version==3: + nColsStruct = 13 + nColsAero = 7 + struct_headers = ['r_[m]','EIFlap_[Nm2]','EIEdge_[Nm2]','GKt_[Nm2]','Mass_[kg/m]','Jxx_[kg.m]','PreBendFlap_[m]','PreBendEdge_[m]','Str.Twist_[deg]','PhiOut_[deg]','Ycog_[m]','Yshc_[m]','CalcOutput_[0/1]'] + aero_headers = ['X_BladeRoot_[m]','Chord_[m]','AeroTwist_[deg]','RelThickness_[%]','AeroCenter_[m]','AeroTorsion_[0/1]','ProfileSet_[#]'] + else: + raise BrokenFormatError('Blade format not implemented') + + struct = np.zeros((nSections,nColsStruct)) + aero = np.zeros((nSections,nColsAero)) + + # --- Structural data + try: + for iSec in range(nSections): + vals=f.readline().split() + #if len(vals)>=nColsStruct: + struct[iSec,:]=np.array(vals[0:nColsStruct]).astype(float) + #elif self.version==1: + # # version 1 has either 8 or 9 columns + # nColsStruct=nColsStruct-1 + # struct_headers=struct_headers[0:-1] + # struct =struct[:,:-1] + # struct[iSec,:]=np.array(vals[0:nColsStruct]).astype(float) + except: + raise WrongFormatError('Unable to read structural data') + try: + self.BetaC = float(f.readline().strip().split()[0]) + if self.version==3: + f.readline() + self.FlapDamping = [float(v) for v in f.readline().strip().split(';')[0].split()] + self.EdgeDamping = [float(v) for v in f.readline().strip().split(';')[0].split()] + self.TorsDamping = [float(v) for v in f.readline().strip().split(';')[0].split()] + f.readline() + f.readline() + else: + Damping = [float(v) for v in f.readline().strip().split()[0:4]] + self.FlapDamping = Damping[0:2] + self.EdgeDamping = Damping[2:4] + self.TorsDamping = [] + except: + raise + raise WrongFormatError('Unable to read damping data') + + # --- Aero + try: + for iSec in range(nSections): + vals=f.readline().split()[0:nColsAero] + aero[iSec,:]=np.array(vals).astype(float) + except: + raise WrongFormatError('Unable to read aerodynamic data') + + self.ProfileFile=f.readline().strip() + + # --- Concatenating aero and structural data + self._cols = struct_headers+aero_headers[1:] + data = np.column_stack((struct,aero[:,1:])) + dataMiss=pd.DataFrame(data=data, columns=self._cols) + self._nColsStruct=nColsStruct # to remember where to split + # --- Making sure all columns are present, irrespectively of version + self.data=pd.DataFrame(data=[], columns=headers_all) + for c in self._cols: + self.data[c]=dataMiss[c] + +# def toString(self): +# s='' +# if len(self.ProfileSets)>0: +# prefix='PROFILE SET ' +# else: +# prefix='' +# for pset in self.ProfileSets: +# s+=pset.toString(prefix) +# return s +# +# def _write(self): +# with open(self.filename,'w') as f: +# f.write(self.toString) +# + def __repr__(self): + s ='Class FLEXBladeFile (attributes: data, BetaC, FlapDamping, EdgeDamping, ProfileFile)\n' + return s + + def _toDataFrame(self): + return self.data + diff --git a/pydatview/io/flex_doc_file.py b/pydatview/io/flex_doc_file.py new file mode 100644 index 0000000..d3e656d --- /dev/null +++ b/pydatview/io/flex_doc_file.py @@ -0,0 +1,209 @@ +from __future__ import division,unicode_literals,print_function,absolute_import +from builtins import map, range, chr, str +from io import open +from future import standard_library +standard_library.install_aliases() + +from .file import File, WrongFormatError, BrokenFormatError +import numpy as np +import pandas as pd +import os +import re + +class FLEXDocFile(File): + + @staticmethod + def defaultExtensions(): + return ['.out','doc'] + + @staticmethod + def formatName(): + return 'FLEX WaveKin file' + + def _read(self): + with open(self.filename, 'r', errors="surrogateescape") as f: + line1=f.readline().strip() + if line1.find('#Program')!=0: + raise WrongFormatError() + lines=f.read().splitlines()+[line1] + + ArraysHeader=[ + '#Blade_ShapeFunction_DOF1_Shape', + '#Blade_ShapeFunction_DOF2_Shape', + '#Blade_ShapeFunction_DOF3_Shape', + '#Blade_ShapeFunction_DOF4_Shape', + '#Blade_ShapeFunction_DOF5_Shape', + '#Blade_ShapeFunction_DOF6_Shape', + '#Blade_ShapeFunction_DOF7_Shape', + '#Blade_ShapeFunction_DOF8_Shape', + '#Blade_ShapeFunction_DOF9_Shape', + '#Blade_ShapeFunction_DOF10_Shape', + '#Tower_SectionData', + '#Tower_ShapeFunction_DOF1_Shape', + '#Tower_ShapeFunction_DOF2_Shape', + '#Tower_ShapeFunction_DOF3_Shape', + '#Tower_ShapeFunction_DOF4_Shape', + '#Tower_ShapeFunction_DOF5_Shape', + '#Tower_ShapeFunction_DOF6_Shape', + '#Tower_ShapeFunction_DOF7_Shape', + '#Tower_ShapeFunction_DOF8_Shape', + '#Foundation_SectionData', + '#Foundation_ShapeFunction_DOF1_Shape', + '#Foundation_ShapeFunction_DOF2_Shape', + '#Global_Mode1_Shape', + ] + + ArraysNoHeader=[ + '#Blade_IsolatedMassMatrix' + '#Blade_IsolatedStiffnessMatrix', + '#Blade_IsolatedDampingMatrix', + '#Foundation_IsolatedMassMatrix', + '#Foundation_IsolatedStiffnessMatrix', + '#Foundation_IsolatedStiffnessMatrixCorrection', + '#Foundation_IsolatedDampingMatrix', + '#Tower_MassMatrix', + '#Tower_IsolatedMassMatrix', + '#Tower_IsolatedStiffnessMatrix', + '#Tower_IsolatedStiffnessMatrixCorrection', + '#Tower_IsolatedDampingMatrix', + '#EVA_MassMatrix', + '#EVA_StiffnessMatrix', + '#EVA_DampingMatrix', + '#EVA_Eigenvectors', + '#EVA_Eigenfrequencies', + '#EVA_Eigenvalues', + '#EVA_Damping', + '#EVA_LogDec', + ] + + numeric_const_pattern = r'[-+]? (?: (?: \d* \. \d+ ) | (?: \d+ \.? ) )(?: [Ee] [+-]? \d+ ) ?' + rx = re.compile(numeric_const_pattern, re.VERBOSE) + + i=0 + while i0 and lines[i][0]!='#': + array_lines.append(np.array([float(v) if v not in ['Fnd','Twr','COG'] else ['Fnd','Twr','COG'].index(v) for v in lines[i].split()])) + i=i+1 + # --- Process array + M = np.array(array_lines) + # --- Process header + cols=header.split() + try: + ii=int(cols[0]) + header=' '.join(cols[1:]) + except: + pass + if header.find('[')<=0: + cols=header.split() + else: + header=header.replace('rough','rough_[-]') + header=header.replace('n ','n_[-] ') + spcol=header.split(']') + cols= [v.strip().replace(' ','_').replace('[','_[').replace('__','_').replace('__','_')+']' for v in spcol[:-1]] + + if len(cols)!=M.shape[1]: + cols=['C{}'.format(j) for j in range(M.shape[1])] + # --- Store + keys = sp[0].split('_') + keys[0]=keys[0][1:] + if keys[0] not in self.keys(): + self[keys[0]] = dict() + subkey = '_'.join(keys[1:]) + df = pd.DataFrame(data = M, columns = cols) + self[keys[0]][subkey] = df + continue + # --- Array with no header + elif sp[0] in ArraysNoHeader: + array_lines=[] + i=i+1 + header=lines[i] + i=i+1 + while i0 and lines[i][0]!='#': + array_lines.append(np.array([float(v) for v in lines[i].split()])) + i=i+1 + # --- Process array + M = np.array(array_lines) + # --- Store + keys = sp[0].split('_') + keys[0]=keys[0][1:] + if keys[0] not in self.keys(): + self[keys[0]] = dict() + subkey = '_'.join(keys[1:]) + self[keys[0]][subkey] = M + continue + else: + # --- Regular + keys = sp[0].split('_') + key=keys[0][1:] + subkey = '_'.join(keys[1:]) + values= ' '.join(sp[1:]) + try: + dat= np.array(rx.findall(values)).astype(float) + if len(dat)==1: + dat=dat[0] + except: + dat = values + + if len(key.strip())>0: + if len(subkey)==0: + if key not in self.keys(): + self[key] = dat + else: + print('>>> line',i,l) + print(self.keys()) + raise Exception('Duplicate singleton key:',key) + else: + if key not in self.keys(): + self[key] = dict() + self[key][subkey]=dat + i+=1 + + # --- Some adjustements + try: + df=self['Global']['Mode1_Shape'] + df=df.drop('C0',axis=1) + df.columns=['H_[m]','U_FA_[-]','U_SS_[]'] + self['Global']['Mode1_Shape']=df + except: + pass + +# def _write(self): +# with open(self.filename,'w') as f: +# f.write(self.toString) + + def __repr__(self): + s='<{} object> with keys:\n'.format(type(self).__name__) + for k in self.keys(): + if type(self[k]) is dict: + s+='{:15s}: dict with keys {}\n'.format(k, list(self[k].keys())) + else: + s+='{:15s} : {}\n'.format(k,self[k]) + return s + + def _toDataFrame(self): + dfs={} + for k,v in self.items(): + if type(v) is pd.DataFrame: + dfs[k]=v + #if type(v) is np.ndarray: + # if len(v.shape)>1: + # dfs[k]=v + if type(self[k]) is dict: + for k2,v2 in self[k].items(): + if type(v2) is pd.DataFrame: + dfs[k+'_'+k2]=v2 + return dfs + diff --git a/pydatview/io/flex_out_file.py b/pydatview/io/flex_out_file.py new file mode 100644 index 0000000..d9c95e5 --- /dev/null +++ b/pydatview/io/flex_out_file.py @@ -0,0 +1,200 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from io import open +from builtins import map +from builtins import range +from builtins import chr +from builtins import str +from future import standard_library +standard_library.install_aliases() + +from .file import File, WrongFormatError, BrokenFormatError +import numpy as np +import pandas as pd +import os + +#from .wetb.fast import fast_io + + + +# --------------------------------------------------------------------------------} +# --- OUT FILE +# --------------------------------------------------------------------------------{ +class FLEXOutFile(File): + + @staticmethod + def defaultExtensions(): + return ['.res','.int'] + + @staticmethod + def formatName(): + return 'FLEX output file' + + def _read(self): + # --- First read the binary file + dtype=np.float32; # Flex internal data is stored in single precision + try: + self.data,self.tmin,self.dt,self.Version,self.DateID,self.title=read_flex_res(self.filename, dtype=dtype) + except WrongFormatError as e: + raise WrongFormatError('FLEX File {}: '.format(self.filename)+'\n'+e.args[0]) + self.nt = np.size(self.data,0) + self.nSensors = np.size(self.data,1) + self.time = np.arange(self.tmin, self.tmin + self.nt * self.dt, self.dt).reshape(self.nt,1).astype(dtype) + + # --- Then the sensor file + sensor_filename = os.path.join(os.path.dirname(self.filename), "sensor") + if not os.path.isfile(sensor_filename): + # we are being nice and create some fake sensors info + self.sensors=read_flex_sensor_fake(self.nSensors) + else: + self.sensors=read_flex_sensor(sensor_filename) + if len(self.sensors['ID'])!=self.nSensors: + raise BrokenFormatError('Inconsistent number of sensors: {} (sensor file) {} (out file), for file: {}'.format(len(self.sensors['ID']),self.nSensors,self.filename)) + + #def _write(self): # TODO + # pass + + def __repr__(self): + return 'Flex Out File: {}\nVersion:{} - DateID:{} - Title:{}\nSize:{}x{} - tmin:{} - dt:{}]\nSensors:{}'.format(self.filename,self.Version,self.DateID,self.title,self.nt,self.nSensors,self.tmin,self.dt,self.sensors['Name']) + + def _toDataFrame(self): + # Appending time to form the dataframe + names = ['Time'] + self.sensors['Name'] + units = ['s'] + self.sensors['Unit'] + units = [u.replace('(','').replace(')','').replace('[','').replace(']','') for u in units] + data = np.concatenate((self.time, self.data), axis=1) + cols=[n+'_['+u+']' for n,u in zip(names,units)] + return pd.DataFrame(data=data,columns=cols) + +# --------------------------------------------------------------------------------} +# --- Helper Functions +# --------------------------------------------------------------------------------{ +def read_flex_res(filename, dtype=np.float32): + # Read flex file + with open(filename,'rb') as fid: + #_ = struct.unpack('i', fid.read(4)) # Dummy + _ = np.fromfile(fid, 'int32', 1) # Dummy + # --- Trying to get DateID + fid.seek(4) # + DateID=np.fromfile(fid, 'int32', 6) + if DateID[0]<32 and DateID[1]<13 and DateID[3]<25 and DateID[4]<61: + # OK, DateID was present + title = fid.read(40).strip() + else: + fid.seek(4) # + DateID = np.fromfile(fid, 'int32', 1) + title = fid.read(60).strip() + _ = np.fromfile(fid, 'int32', 2) # Dummy + # FILE POSITION <<< fid.seek(4 * 19) + nSensors = np.fromfile(fid, 'int32', 1)[0] + IDs = np.fromfile(fid, 'int32', nSensors) + _ = np.fromfile(fid, 'int32', 1) # Dummy + # FILE POSITION <<< fid.seek(4*nSensors+4*21) + Version = np.fromfile(fid, 'int32', 1)[0] + # FILE POSITION <<< fid.seek(4*(nSensors)+4*22) + if Version == 12: + raise NotImplementedError('Flex out file with version 12, TODO. Implement it!') + # TODO + #fseek(o.fid,4*(21+o.nSensors),-1);% seek to the data from beginning of file + #RL=o.nSensors+5; % calculate the length of each row + #A = fread(o.fid,[RL,inf],'single'); % read whole file + #t=A(2,:);% time vector contained in row 2 + #o.SensorData=A(5:end,:); + # save relevant information + #o.tmin = t(1) ; + #o.dt = t(2)-t(1); + #o.t = t ; + #o.nt = length(t); + elif Version in [0,2,3]: + tmin = np.fromfile(fid, 'f', 1)[0] # Dummy + dt = np.fromfile(fid, 'f', 1)[0] # Dummy + scale_factors = np.fromfile(fid, 'f', nSensors).astype(dtype) + # --- Reading Time series + # FILE POSITION <<< fid.seek(8*nSensors + 48*2) + data = np.fromfile(fid, 'int16').astype(dtype) #data = np.fromstring(fid.read(), 'int16').astype(dtype) + nt = int(len(data) / nSensors) + try: + if Version ==3: + data = data.reshape(nSensors, nt).transpose() + else: + data = data.reshape(nt, nSensors) + except ValueError: + raise WrongFormatError("Flat data length {} is not compatible with {}x{} (nt x nSensors)".format(len(data),nt,nSensors)) + for i in range(nSensors): + data[:, i] *= scale_factors[i] + + return (data,tmin,dt,Version,DateID,title) + + +def read_flex_sensor(sensor_file): + with open(sensor_file, encoding="utf-8") as fid: + sensor_info_lines = fid.readlines()[2:] + sensor_info = [] + d=dict({ 'ID':[],'Gain':[],'Offset':[],'Unit':[],'Name':[],'Description':[]}); + for line in sensor_info_lines: + line = line.strip().split() + d['ID'] .append(int(line[0])) + d['Gain'] .append(float(line[1])) + d['Offset'] .append(float(line[2])) + d['Unit'] .append(line[5]) + d['Name'] .append(line[6]) + d['Description'] .append(' '.join(line[7:])) + return d + +def read_flex_sensor_fake(nSensors): + d=dict({ 'ID':[],'Gain':[],'Offset':[],'Unit':[],'Name':[],'Description':[]}); + for i in range(nSensors): + d['ID'] .append(i+1) + d['Gain'] .append(1.0) + d['Offset'] .append(0.0) + d['Unit'] .append('(NA)') + d['Name'] .append('S{:04d}'.format(i+1)) + d['Description'] .append('NA') + return d + + + + + + +# def write_flex_file(filename,data,tmin,dt): +# ds = dataset +# # Write int data file +# f = open(filename, 'wb') +# f.write(struct.pack('ii', 0, 0)) # 2x empty int +# title = ("%-60s" % str(ds.name)).encode() +# f.write(struct.pack('60s', title)) # title +# f.write(struct.pack('ii', 0, 0)) # 2x empty int +# ns = len(sensors) +# f.write(struct.pack('i', ns)) +# f.write(struct.pack('i' * ns, *range(1, ns + 1))) # sensor number +# f.write(struct.pack('ii', 0, 0)) # 2x empty int +# time = ds.basis_attribute() +# f.write(struct.pack('ff', time[0], time[1] - time[0])) # start time and time step +# +# scale_factors = np.max(np.abs(data), 0) / 32000 +# f.write(struct.pack('f' * len(scale_factors), *scale_factors)) +# # avoid dividing by zero +# not0 = np.where(scale_factors != 0) +# data[:, not0] /= scale_factors[not0] +# #flatten and round +# data = np.round(data.flatten()).astype(np.int16) +# f.write(struct.pack('h' * len(data), *data.tolist())) +# f.close() +# +# # write sensor file +# f = open(os.path.join(os.path.dirname(filename), 'sensor'), 'w') +# f.write("Sensor list for %s\n" % filename) +# f.write(" No forst offset korr. c Volt Unit Navn Beskrivelse------------\n") +# sensorlineformat = "%3s %.3f %.3f 1.00 0.00 %7s %-8s %s\n" +# +# if isinstance(ds, FLEX4Dataset): +# gains = np.r_[ds.gains[1:], np.ones(ds.shape[1] - len(ds.gains))] +# offsets = np.r_[ds.offsets[1:], np.zeros(ds.shape[1] - len(ds.offsets))] +# sensorlines = [sensorlineformat % ((nr + 1), gain, offset, att.unit[:7], att.name.replace(" ", "_")[:8], att.description[:512]) for nr, att, gain, offset in zip(range(ns), sensors, gains, offsets)] +# else: +# sensorlines = [sensorlineformat % ((nr + 1), 1, 0, att.unit[:7], att.name.replace(" ", "_")[:8], att.description[:512]) for nr, att in enumerate(sensors)] +# f.writelines(sensorlines) +# f.close() diff --git a/pydatview/io/flex_profile_file.py b/pydatview/io/flex_profile_file.py new file mode 100644 index 0000000..9a29f17 --- /dev/null +++ b/pydatview/io/flex_profile_file.py @@ -0,0 +1,147 @@ +from __future__ import division,unicode_literals,print_function,absolute_import +from builtins import map, range, chr, str +from io import open +from future import standard_library +standard_library.install_aliases() + +from .file import File, WrongFormatError, BrokenFormatError +import numpy as np +import pandas as pd +import os + +#from .wetb.fast import fast_io + +class ProfileSet(): + def __init__(self,header,thickness,polars,polar_headers): + self.header = header + self.polars = polars + self.thickness = thickness + self.polar_headers = polar_headers + + def toString(self,PREFIX=''): + s =PREFIX+self.header+'\n' + s+=' '.join([str(t) for t in self.thickness])+'\n' + s+=str(self.polars[0].shape[0])+'\n' + for ph,t,polar in zip(self.polar_headers,self.thickness,self.polars): + s+=ph+'\n' + s+='\n'.join([' '.join(['{:15.7e}'.format(v) for v in line]) for line in polar]) +# s+=ph+'\n' + return s + + def __repr__(self): + s ='Class ProfileSet (attributes: header, polars, thickness, polar_headers)\n' + s+=' header : '+self.header+'\n' + s+=' thickness : '+str(self.thickness)+'\n' + s+=' Number of polars: '+str(len(self.thickness))+'\n' + s+=' Alpha values : '+str(self.polars[0].shape[0])+'\n' + for ip,(ph,t) in enumerate(zip(self.polar_headers,self.thickness)): + s+= ' Polar: {}, Thickness: {}, Header: {}\n'.format(ip+1,t,ph) + return s + +class FLEXProfileFile(File): + + @staticmethod + def defaultExtensions(): + return ['.pro','.00X'] #'.001 etc..' + + @staticmethod + def formatName(): + return 'FLEX profile file' + + def _read(self): + self.ProfileSets=[] + setNumber=1 + with open(self.filename, 'r', errors="surrogateescape") as f: + def read_header(allow_empty=False): + """ Reads the header of a profile set (4 first lines) + - The first line may start with "Profile set I:" to indicate a set number + - Second line is number of thicknesses + - Third is thicnkesses + - Fourth is number of alpha values + """ + header=[] + for i, line in enumerate(f): + header.append(line.strip()) + if i==3: + break + if len(header)<4: + if allow_empty: + return [],[],'',False + else: + raise WrongFormatError('A Flex profile file needs at leats 4 lines of headers') + try: + nThickness=int(header[1]) + except: + raise WrongFormatError('Number of thicknesses (integer) should be on line 2') + try: + thickness=np.array(header[2].split()).astype(float) + except: + raise WrongFormatError('Number of thicknesses (integer) should be on line 2') + if len(thickness)!=nThickness: + raise WrongFormatError('Number of thicknesses read ({}) different from the number reported ({})'.format(len(thickness),nThickness)) + try: + nAlpha=int(header[3]) + except: + raise WrongFormatError('Number of alpha values (integer) should be on line 4') + if header[0].lower().find('profile set')==0: + header[0]=header[0][11:] + bHasSets=True + else: + bHasSets=False + return nAlpha,thickness,header[0],bHasSets + + def read_polars(nAlpha,thickness): + polars=[] + polar_headers=[] + for it,t in enumerate(thickness): + polar_headers.append(f.readline().strip()) + polars.append(np.zeros((nAlpha,4))) + try: + for ia in range(nAlpha): + polars[it][ia,:]=np.array([f.readline().split()]).astype(float) + except: + raise BrokenFormatError('An error occured while reading set number {}, polar number {}, (thickness {}), value number {}.'.format(setNumber,it+1,t,ia+1)) + + return polars,polar_headers + + # Reading headers and polars + while True: + nAlpha,thickness,Header,bHasSets = read_header(allow_empty=setNumber>1) + if len(thickness)==0: + break + polars,polar_headers = read_polars(nAlpha,thickness) + PSet= ProfileSet(Header,thickness,polars,polar_headers) + self.ProfileSets.append(PSet) + setNumber=setNumber+1 + + def toString(self): + s='' + if len(self.ProfileSets)>0: + prefix='PROFILE SET ' + else: + prefix='' + for pset in self.ProfileSets: + s+=pset.toString(prefix) + return s + + def _write(self): + with open(self.filename,'w') as f: + f.write(self.toString) + + def __repr__(self): + s ='Class FlexProfileFile (attributes: ProfileSets)\n' + s+=' Number of profiles sets: '+str(len(self.ProfileSets))+'\n' + for ps in self.ProfileSets: + s+=ps.__repr__() + return s + + + def _toDataFrame(self): + cols=['Alpha_[deg]','Cl_[-]','Cd_[-]','Cm_[-]'] + dfs = {} + for iset,pset in enumerate(self.ProfileSets): + for ipol,(thickness,polar) in enumerate(zip(pset.thickness,pset.polars)): + name='pc_set_{}_t_{}'.format(iset+1,thickness) + dfs[name] = pd.DataFrame(data=polar, columns=cols) + return dfs + diff --git a/pydatview/io/flex_wavekin_file.py b/pydatview/io/flex_wavekin_file.py new file mode 100644 index 0000000..684e8f8 --- /dev/null +++ b/pydatview/io/flex_wavekin_file.py @@ -0,0 +1,104 @@ +from __future__ import division,unicode_literals,print_function,absolute_import +from builtins import map, range, chr, str +from io import open +from future import standard_library +standard_library.install_aliases() + +from .file import File, WrongFormatError, BrokenFormatError +from .csv_file import CSVFile +import numpy as np +import pandas as pd +import os +import re + +#from .wetb.fast import fast_io + + +class FLEXWaveKinFile(File): + + @staticmethod + def defaultExtensions(): + return ['.wko'] #'.001 etc..' + + @staticmethod + def formatName(): + return 'FLEX WaveKin file' + + def _read(self): + numeric_const_pattern = r'[-+]? (?: (?: \d* \. \d+ ) | (?: \d+ \.? ) )(?: [Ee] [+-]? \d+ ) ?' + rx = re.compile(numeric_const_pattern, re.VERBOSE) + def extract_floats(s): + v=np.array(rx.findall(s)) + return v + + + try: + csv = CSVFile(self.filename, sep=' ', commentLines=list(np.arange(11)),detectColumnNames=False) + except: + raise WrongFormatError('Unable to parse Flex WaveKin file as CSV with 11 header lines') + + header = csv.header + self['header'] = csv.header[0:2] + self['data'] = csv.data + try: + self['MaxCrestHeight'] = float(extract_floats(header[2])[0]) + self['MaxLongiVel'] = float(extract_floats(header[3])[0]) + self['MaxLongiAcc'] = float(extract_floats(header[4])[0]) + dat = extract_floats(header[5]).astype(float) + self['WaterDepth'] = dat[0] + self['Hs'] = dat[1] + self['Tp'] = dat[2] + self['SpecType'] = dat[3] + except: + raise BrokenFormatError('Unable to parse floats from header lines 3-6') + + try: + nDisp = int(extract_floats(header[6])[0]) + nRelD = int(extract_floats(header[8])[0]) + except: + raise BrokenFormatError('Unable to parse int from header lines 7 and 9') + + try: + displ = extract_floats(header[7]).astype(float) + depth = extract_floats(header[9]).astype(float) + except: + raise BrokenFormatError('Unable to parse displacements or depths from header lines 8 and 10') + if len(displ)!=nDisp: + print(displ) + raise BrokenFormatError('Number of displacements ({}) does not match number provided ({})'.format(nDisp, len(displ))) + if len(depth)!=nRelD: + print(depth) + raise BrokenFormatError('Number of rel depth ({}) does not match number provided ({})'.format(nRelD, len(depth))) + + self['RelDepth'] = depth + self['Displacements'] = displ + + cols=['Time_[s]', 'WaveElev_[m]'] + for j,x in enumerate(displ): + for i,z in enumerate(depth): + cols+=['u_z={:.1f}_x={:.1f}_[m/s]'.format(z*self['WaterDepth']*-1,x)] + for i,z in enumerate(depth): + cols+=['a_z={:.1f}_x={:.1f}_[m/s^2]'.format(z*self['WaterDepth'],x)] + + if len(cols)!=len(self['data'].columns): + raise BrokenFormatError('Number of columns not valid') + self['data'].columns = cols + +# def _write(self): +# with open(self.filename,'w') as f: +# f.write(self.toString) + + def __repr__(self): + s='<{} object> with keys:\n'.format(type(self).__name__) + + for k in ['MaxCrestHeight','MaxLongiVel','MaxLongiAcc','WaterDepth','Hs','Tp','SpecType','RelDepth','Displacements']: + s += '{:15s}: {}\n'.format(k,self[k]) + if len(self['header'])>0: + s += 'header : '+ ' ,'.join(self['header'])+'\n' + if len(self['data'])>0: + s += 'data size : {}x{}'.format(self['data'].shape[0],self['data'].shape[1]) + return s + + def _toDataFrame(self): + return self['data'] + diff --git a/pydatview/io/hawc2_ae_file.py b/pydatview/io/hawc2_ae_file.py new file mode 100644 index 0000000..46e7f13 --- /dev/null +++ b/pydatview/io/hawc2_ae_file.py @@ -0,0 +1,71 @@ +""" +Hawc2 AE file +""" +import os +import pandas as pd + +try: + from .file import File, WrongFormatError, EmptyFileError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + +from .wetb.hawc2.ae_file import AEFile + +class HAWC2AEFile(File): + + @staticmethod + def defaultExtensions(): + return ['.dat','.ae','.txt'] + + @staticmethod + def formatName(): + return 'HAWC2 AE file' + + def __init__(self,filename=None,**kwargs): + if filename: + self.filename = filename + self.read(**kwargs) + else: + self.filename = None + self.data = AEFile() + + def read(self, filename=None, **kwargs): + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # --- + try: + self.data = AEFile(self.filename) + except Exception as e: + raise WrongFormatError('AE File {}: '.format(self.filename)+e.args[0]) + + def write(self, filename=None): + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + # --- + self.data.save(self.filename) + + def toDataFrame(self): + cols=['radius_[m]','chord_[m]','thickness_[%]','pc_set_[#]'] + nset = len(self.data.ae_sets) + if nset == 1: + return pd.DataFrame(data=self.data.ae_sets[1], columns=cols) + else: + dfs = {} + for iset,aeset in enumerate(self.data.ae_sets): + name='ae_set_{}'.format(iset+1) + dfs[name] = pd.DataFrame(data=self.data.ae_sets[iset+1], columns=cols) + return dfs + + # --- Convenient utils + def add_set(self, **kwargs): + self.data.add_set(**kwargs) + diff --git a/pydatview/io/hawc2_dat_file.py b/pydatview/io/hawc2_dat_file.py new file mode 100644 index 0000000..3a3bab4 --- /dev/null +++ b/pydatview/io/hawc2_dat_file.py @@ -0,0 +1,156 @@ +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from io import open +from builtins import map +from builtins import range +from builtins import chr +from builtins import str +from future import standard_library +standard_library.install_aliases() +import os +import numpy as np + +from .file import File, WrongFormatError, FileNotFoundError +import pandas as pd + +from .wetb.hawc2.Hawc2io import ReadHawc2 + + +class HAWC2DatFile(File): + + @staticmethod + def defaultExtensions(): + return ['.dat','.sel'] + + @staticmethod + def formatName(): + return 'HAWC2 dat file' + + def __init__(self, filename=None, **kwargs): + self.info={} + self.data=np.array([]) + self.bHawc=False + super(HAWC2DatFile, self).__init__(filename=filename,**kwargs) + + def _read(self): + try: + res_file = ReadHawc2(self.filename) + self.data = res_file.ReadAll() + self.info['attribute_names'] = res_file.ChInfo[0] + self.info['attribute_units'] = res_file.ChInfo[1] + self.info['attribute_descr'] = res_file.ChInfo[2] + if res_file.FileFormat=='BHAWC_ASCII': + self.bHawc=True + except FileNotFoundError: + raise + #raise WrongFormatError('HAWC2 dat File {}: '.format(self.filename)+' File Not Found:'+e.filename) + except Exception as e: +# raise e + raise WrongFormatError('HAWC2 dat File {}: '.format(self.filename)+e.args[0]) + + #def _write(self): + #self.data.to_csv(self.filename,sep=self.false,index=False) + + def _toDataFrame(self): + import re + + # Simplify output names + names=self.info['attribute_names'] + for i,desc in enumerate(self.info['attribute_descr']): + elem = re.findall(r'E-nr:\s*(\d+)', desc) + zrel = re.findall(r'Z-rel:\s*(\d+.\d+)', desc) + node = re.findall(r'nodenr:\s*(\d+)', desc) + mbdy = re.findall(r'Mbdy:([-a-zA-Z0-9_.]*) ', desc) + s = re.findall(r's=\s*(\d+.\d+)\[m\]', desc) + sS = re.findall(r's/S=\s*(\d+.\d+)', desc) + + pref='' + names[i] = names[i].replace(' ','') + names[i] = names[i].replace('coo:global','g').strip() + names[i] = names[i].replace('Statepos','').strip() + names[i] = names[i].replace('axisangle','rot_').strip() + + if len(mbdy)==1: + names[i] = names[i].replace('coo:'+mbdy[0],'b').strip() + pref += mbdy[0] + + if len(zrel)==1 and len(elem)==1: + ielem=int(elem[0]) + fzrel=float(zrel[0]) + if fzrel==0: + pref+= 'N'+str(ielem) + elif fzrel==1: + pref+='N'+str(ielem+1) + else: + pref+='N'+str(ielem+fzrel) + if len(s)==1 and len(sS)==1: + pref+='r'+str(sS[0]) + + if len(node)==1: + pref+='N'+node[0] + names[i]=pref+names[i] + + if self.info['attribute_units'] is not None: + units = [u.replace('(','').replace(')','').replace('[','').replace(']','') for u in self.info['attribute_units']] + + cols=[n+'_['+u+']' for n,u in zip(names,units)] + else: + cols=names + + return pd.DataFrame(data=self.data,columns=cols) +# + def _write(self): + filename=self.filename + ext = os.path.splitext(filename)[-1].lower() + if ext=='.dat': + datfilename=filename + elif ext=='.sel': + datfilename=filename.replace(ext,'.dat') + else: + datfilename=filename+'.dat' + selfilename=datfilename[:-4]+'.sel' + nScans = self.data.shape[0] + nChannels = self.data.shape[1] + SimTime = self.data[-1,0] #-self.data[0,0] + # --- dat file + np.savetxt(datfilename, self.data, fmt=b'%16.8e') + # --- Sel file + with open(selfilename, 'w') as f: + if self.bHawc: + f.write('BHawC channel reference file (sel):\n') + f.write('+===================== (Name) =================== (Time stamp) ========= (Path) ==========================================================+\n') + f.write('Original BHAWC file : NA.dat 2001.01.01 00:00:00 C:\\\n') + f.write('Channel reference file : NA.sel 2001.01.01 00:00:00 C:\\\n') + f.write('Result file : NA.dat 2001.01.01 00:00:00 C:\\\n') + f.write('+=========================================================================================================================================+\n') + f.write('Scans \tChannels \tTime [sec] \tCoordinate convention: Siemens\n') + f.write('{:19s}\t{:25s}\t{:25s}\n\n'.format(str(nScans),str(nChannels),'{:.3f}'.format(SimTime))) + f.write('{:19s}\t{:25s}\t{:25s}\t{:25s}\n'.format('Channel','Variable descriptions','Labels','Units')) + for chan,(label,descr,unit) in enumerate(zip(self.info['attribute_names'],self.info['attribute_descr'],self.info['attribute_units'])): + unit=unit.replace('(','').replace(')','').replace('[','').replace(']','') + f.write('{:19s}\t{:25s}\t{:25s}\t{:25s}\n'.format(str(chan+1),descr[0:26],label[0:26],'['+unit+']')) + else: + + f.write('________________________________________________________________________________________________________________________\n') + f.write(' Version ID : NA\n') + f.write(' Time : 00:00:00\n') + f.write(' Date : 01:01.2001\n') + f.write('________________________________________________________________________________________________________________________\n') + f.write(' Result file : {:s}\n'.format(os.path.basename(datfilename))) + f.write('________________________________________________________________________________________________________________________\n') + f.write(' Scans Channels Time [sec] Format\n') + f.write('{:12s}{:12s}{:16s}{:s}\n'.format(str(nScans),str(nChannels),'{:.3f}'.format(SimTime),'ASCII')) + f.write('\n') + f.write('{:12s}{:31s}{:11s}{:s}'.format('Channel','Name','Unit','Variable Description\n')) + f.write('\n') + for chan,(label,descr,unit) in enumerate(zip(self.info['attribute_names'],self.info['attribute_descr'],self.info['attribute_units'])): + unit=unit.replace('(','').replace(')','').replace('[','').replace(']','') + f.write('{:12s}{:31s}{:11s}{:s}\n'.format(str(chan+1),label[0:30],unit,descr)) + f.write('________________________________________________________________________________________________________________________\n'); + +class BHAWCDatFile(HAWC2DatFile): + def __init__(self, filename=None, **kwargs): + super(HAWC2DatFile, self).__init__(filename=filename,**kwargs) + self.bHawc=False diff --git a/pydatview/io/hawc2_htc_file.py b/pydatview/io/hawc2_htc_file.py new file mode 100644 index 0000000..51fb1ce --- /dev/null +++ b/pydatview/io/hawc2_htc_file.py @@ -0,0 +1,117 @@ +""" +Wrapper around wetb to read/write htc files. +TODO: rewrite of c2_def might not be obvious +""" +from .file import File + +import numpy as np +import pandas as pd +import os + +from .wetb.hawc2.htc_file import HTCFile +from .hawc2_st_file import HAWC2StFile + +class HAWC2HTCFile(File): + + @staticmethod + def defaultExtensions(): + return ['.htc'] + + @staticmethod + def formatName(): + return 'HAWC2 htc file' + + def _read(self): + self.data = HTCFile(self.filename) + self.data.contents # trigger read + + def _write(self): + self.data.save(self.filename) + + def __repr__(self): + s='<{} object> with attribute `data`\n'.format(type(self).__name__) + return s + + def bodyByName(self, bodyname): + """ return body inputs given a body name""" + struct = self.data.new_htc_structure + bodyKeys = [k for k in struct.keys() if k.startswith('main_body')] + bdies = [struct[k] for k in bodyKeys if struct[k].name[0]==bodyname] + if len(bdies)==1: + return bdies[0] + elif len(bdies)==0: + raise Exception('No body found with name {} in file {}'.format(bodyname,self.filename)) + else: + raise Exception('Several bodies found with name {} in file {}'.format(bodyname,self.filename)) + + def bodyC2(self, bdy): + """ return body C2_def given body inputs""" + try: + nsec = bdy.c2_def.nsec[0] + except: + raise Exception('body has no c2_def section') + val = np.array([bdy.c2_def[k].values[0:] for k in bdy.c2_def.keys() if k.startswith('sec')]) + val = val.reshape((-1,5)).astype(float) + val = val[np.argsort(val[:,0]),:] + return val + + def setBodyC2(self, bdy, val): + """ set body C2_def given body inputs and new c2def""" + # TODO different number of section + nsec = bdy.c2_def.nsec[0] + nsec_new = val.shape[0] + sec_keys = [k for k in bdy.c2_def.keys() if k.startswith('sec')] + bdy.c2_def.nsec = nsec_new + if nsec != nsec_new: + if nsec_new -Ly/2 but we flip this to -Ly/2 -> Ly/2 + """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + + if N is None: + # try to infer N's from filename with format 'stringN1xN2xN3' + basename = os.path.splitext(os.path.basename(self.filename))[0] + splits = basename.split('x') + temp = re.findall(r'\d+', basename) + res = list(map(int, temp)) + if len(res)>=3: + N=res[-3:] + else: + raise BrokenFormatError('Reading a Mann box requires the knowledge of the dimensions. The dimensions can be inferred from the filename, for instance: `filebase_1024x32x32.u`. Try renaming your file such that the three last digits are the dimensions in x, y and z.') + nx,ny,nz=N + + def _read_buffered(): + data=np.zeros((nx,ny,nz),dtype=np.float32) + with open(self.filename, mode='rb') as f: + for ix in range(nx): + Buffer = np.frombuffer(f.read(4*ny*nz), dtype=np.float32) # 4-bytes + data[ix,:,:] = np.flip(Buffer.reshape((ny,nz)),0) + return data + + def _read_nonbuffered(): + data = np.fromfile(self.filename, np.dtype(' -Ly/2 + # So we flip the y-axis, so that the field is consistent with typical y values + return np.flip(data, 1) # i.e. data=data[:,::-1,:] + +# self['field']= _read_nonbuffered() + self['field']= _read_buffered() + self['dy']=dy + self['dz']=dz + self['y0']=y0 + self['z0']=z0 + self['zMid']=zMid +# print('1',self['field'][:,-1,0]) +# print('2',self['field'][0,-1::-1,0]) +# print('3',self['field'][0,-1,:]) + + + def write(self, filename=None): + """ Write mann box """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + nx,ny,nz = self['field'].shape + sfmt='<{:d}f'.format(ny*nz) + with open(self.filename, mode='wb') as f: + for ix in np.arange(nx): + data = np.flip(self['field'][ix,:,:],0).ravel() # We have to flip the y axis again + f.write(struct.pack(sfmt, *data)) + + + def __repr__(self): + s='<{} object> with keys:\n'.format(type(self).__name__) + s+='| - filename: {}\n'.format(self.filename) + s+='| - field: shape {}x{}x{}\n'.format(self['field'].shape[0],self['field'].shape[1],self['field'].shape[2]) + s+='| min: {}, max: {}, mean: {} \n'.format(np.min(self['field']), np.max(self['field']), np.mean(self['field'])) + s+='| - dy, dz: {}, {}\n'.format(self['dy'], self['dz']) + s+='| - y0, z0 zMid: {}, {}, {}\n'.format(self['y0'], self['z0'], self['zMid']) + s+='|useful getters: y, z, _iMid, fromTurbSim\n' + z=self.z + y=self.y + s+='| y: [{} ... {}], dy: {}, n: {} \n'.format(y[0],y[-1],self['dy'],len(y)) + s+='| z: [{} ... {}], dz: {}, n: {} \n'.format(z[0],z[-1],self['dz'],len(z)) + return s + + + @property + def z(self): + zmax = self['z0'] + (self['field'].shape[2]-1+0.1)*self['dz'] + z = np.arange(self['z0'], zmax, self['dz']) + if self['zMid'] is not None: + z+= self['zMid']-np.mean(z) + return z + + @property + def y(self): + if self['y0'] is not None: + ymax = self['y0'] + (self['field'].shape[1]-1+0.1)*self['dy'] + y = np.arange(self['y0'], ymax, self['dy']) + else: + ymax = (self['field'].shape[1]-1+0.1)*self['dy'] + y = np.arange(0, ymax, self['dy']) + y -= np.mean(y) + return y + + def t(self, dx, U): + # 1.5939838 dx - distance (in meters) between points in the x direction (m) + # 99.5 RefHt_Hawc - reference height; the height (in meters) of the vertical center of the grid (m) + # 6.26 URef - Mean u-component wind speed at the reference height (m/s) + dt = dx/U + nt = self['field'].shape[0] + return np.arange(0, dt*(nt-0.5), dt) + + # --------------------------------------------------------------------------------} + # --- Extracting relevant data + # --------------------------------------------------------------------------------{ + def valuesAt(self, y, z, method='nearest'): + """ return wind speed time series at a point """ + if method == 'nearest': + iy, iz = self.closestPoint(y, z) + u = self['field'][:,iy,iz] + else: + raise NotImplementedError() + return u + + def closestPoint(self, y, z): + iy = np.argmin(np.abs(self.y-y)) + iz = np.argmin(np.abs(self.z-z)) + return iy,iz + + def _iMid(self): + _, ny, nz = self['field'].shape + return int(ny/2), int(nz/2) + + @property + def vertProfile(self): + iy, iz = self._iMid() + m = np.mean(self['field'][:,iy,:], axis=0) + s = np.std (self['field'][:,iy,:], axis=0) + return self.z,m,s + + + def toDataFrame(self): + dfs={} + ny = len(self.y) + nz = len(self.z) + # Index at mid box + iy,iz = self._iMid() + + # Mean vertical profile + z, m, s = self.vertProfile + ti = s/m*100 + cols=['z_[m]','vel_[m/s]','sigma_[m/s]','TI_[%]'] + data = np.column_stack((z,m[:],s[:],ti[:])) + dfs['VertProfile'] = pd.DataFrame(data = data ,columns = cols) + + # Mid time series + u = self['field'][:,iy,iz] + cols=['t/T_[-]','vel_[m/s]'] + fake_t = np.linspace(0, 1, len(u)) + data = np.column_stack((fake_t,u[:])) + dfs['ZMidLine'] = pd.DataFrame(data = data ,columns = cols) + + + # ZMin YEnd time series + u = self['field'][:,-1,iz] + cols=['t/T_[-]','vel_[m/s]'] + fake_t = np.linspace(0, 1, len(u)) + data = np.column_stack((fake_t,u[:])) + dfs['ZMidYEndLine'] = pd.DataFrame(data = data ,columns = cols) + + # ZMin YStart time series + u = self['field'][:,0,iz] + cols=['t/T_[-]','vel_[m/s]'] + fake_t = np.linspace(0, 1, len(u)) + data = np.column_stack((fake_t,u[:])) + dfs['ZMidYStartLine'] = pd.DataFrame(data = data ,columns = cols) + + + +# # Mid crosscorr y +# y, rho_uu_y, rho_vv_y, rho_ww_y = self.crosscorr_y() +# cols = ['y_[m]', 'rho_uu_[-]','rho_vv_[-]','rho_ww_[-]'] +# data = np.column_stack((y, rho_uu_y, rho_vv_y, rho_ww_y)) +# dfs['Mid_xcorr_y'] = pd.DataFrame(data = data ,columns = cols) +# +# # Mid crosscorr z +# z, rho_uu_z, rho_vv_z, rho_ww_z = self.crosscorr_z() +# cols = ['z_[m]', 'rho_uu_[-]','rho_vv_[-]','rho_ww_[-]'] +# data = np.column_stack((z, rho_uu_z, rho_vv_z, rho_ww_z)) +# dfs['Mid_xcorr_z'] = pd.DataFrame(data = data ,columns = cols) +# +# # Mid csd +# fc, chi_uu, chi_vv, chi_ww = self.csd_longi() +# cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] +# data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) +# dfs['Mid_csd_longi'] = pd.DataFrame(data = data ,columns = cols) +# +# # Mid csd +# fc, chi_uu, chi_vv, chi_ww = self.csd_lat() +# cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] +# data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) +# dfs['Mid_csd_lat'] = pd.DataFrame(data = data ,columns = cols) +# +# # Mid csd +# fc, chi_uu, chi_vv, chi_ww = self.csd_vert() +# cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] +# data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) +# dfs['Mid_csd_vert'] = pd.DataFrame(data = data ,columns = cols) + return dfs + + + + # Useful converters + def fromTurbSim(self, u, icomp=0, removeConstant=None, removeAllMean=False): + """ + Assumes: + u (3 x nt x ny x nz) + Removes the mean of the turbsim file for the "u" component. + """ + if icomp==0: + if removeAllMean is True: + self['field'] = u[icomp, :, : ,: ]-np.mean(u[icomp,:,:,:],axis=0) + elif removeConstant is not None: + self['field'] = u[icomp, :, : ,: ]-removeConstant + else: + self['field'] = u[icomp, :, : ,: ] + else: + self['field'] = u[icomp, :, : ,: ] + return self + +if __name__=='__main__': + mb = MannBoxFile('mini-u_1024x32x32.bin') +# mb = MannBoxFile('mann_bin/mini-u.bin', N=(2,4,8)) +# F1=mb['field'].ravel() +# mb.write('mann_bin/mini-u-out.bin') +# +# mb2= MannBoxFile('mann_bin/mini-u-out.bin', N=(2,4,8)) +# F2=mb2['field'].ravel() +# print(F1-F2) diff --git a/pydatview/io/mini_yaml.py b/pydatview/io/mini_yaml.py new file mode 100644 index 0000000..04269b9 --- /dev/null +++ b/pydatview/io/mini_yaml.py @@ -0,0 +1,98 @@ +from __future__ import unicode_literals +from __future__ import print_function +from io import open +import numpy as np + +def yaml_read(filename,dictIn=None): + """ + read yaml files only supports: + - Key value pairs: + key: value + - Key with lists of lists: + key: + - [0,1] + - [0,1] + - Comments are stripped based on first # found (in string or not) + - Keys are found based on first : found (in string or not) + """ + # Read all lines at once + with open(filename, 'r', errors="surrogateescape") as f: + lines=f.read().splitlines() + + + if dictIn is None: + d=dict() + else: + d=dictIn + + def cleanComment(l): + """ remove comments from a line""" + return l.split('#')[0].strip() + + def readDashList(iStart): + """ """ + i=iStart + while i0]) + try: + FirstElems=FirstElems.astype(int) + mytype=int + except: + try: + FirstElems=FirstElems.astype(float) + mytype=float + except: + raise Exception('Cannot convert line to float or int: {}'.format(lines[iStart])) + M = np.zeros((n,len(FirstElems)), mytype) + if len(FirstElems)>0: + for i in np.arange(iStart,iEnd+1): + elem = cleanComment(lines[i])[1:].replace(']','').replace('[','').split(',') + M[i-iStart,:] = np.array([v.strip() for v in elem if len(v)>0]).astype(mytype) + return M, iEnd+1 + + i=0 + while i (len(list_50hz)+len(list_1hz)): + print('WARNING: channel size in file is greater than size of channel names lists!') + # find any missing channels from file + missing50hz = list(set(list_50hz)-set(names_50hz)) + missing1hz = list(set(list_1hz)-set(names_1hz)) + + if output == 'xarray': + import xarray as xr + # add missing columns filled with NaNs + nan50 = np.empty(30000) + nan50[:] = np.nan + nan1 = np.empty(600) + nan1[:] = np.nan + try: + if missing50hz: + for x in missing50hz: dat_50hz.append(nan50) + names_50hz.append(missing50hz) + if missing1hz: + for y in missing1hz: dat_1hz.append(nan1) + names_1hz.append(missing1hz) + except: + pass + # convert lists into xarray + df50hz = xr.DataArray(np.transpose(dat_50hz), dims=('timestamp','channel'),coords={'timestamp':tstamp50hz,'channel':names_50hz}) + df1hz = xr.DataArray(np.transpose(dat_1hz), dims=('timestamp','channel'),coords={'timestamp':tstamp1hz,'channel':names_1hz}) + # store in dict + ds = {'data50hz': df50hz, 'data1hz': df1hz} + return ds + + if output == 'pandas': + # convert lists into pandas + df_50hz = pd.DataFrame(dat_50hz).T + df_50hz.columns = names_50hz + df_1hz = pd.DataFrame(dat_1hz).T + df_1hz.columns = names_1hz + df_50hz.index = tstamp50hz + df_1hz.index = tstamp1hz + try: + # add columns with missing channels filled with NaNs + for x in missing50hz: df_50hz[x] = np.nan + for y in missing1hz: df_1hz[y] = np.nan + except: + pass + # store in dict + ds = {'data50hz': df_50hz, 'data1hz': df_1hz} + return ds + +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + File=dict + +class RAAWMatFile(File): + """ + Read a RAAW .mat file. The object behaves as a dictionary. + + Main methods + ------------ + - read, toDataFrame, keys + + Examples + -------- + f = RAAWMatFile('file.mat') + print(f.keys()) + print(f.toDataFrame().columns) + + """ + + @staticmethod + def defaultExtensions(): + """ List of file extensions expected for this fileformat""" + return ['.mat'] + + @staticmethod + def formatName(): + """ Short string (~100 char) identifying the file format""" + return 'RAAW .mat file' + + def __init__(self, filename=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # --- Calling (children) function to read + self._read(**kwargs) + + def write(self, filename=None): + """ Rewrite object to file, or write object to `filename` if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + # Calling (children) function to write + self._write() + + def _read(self): + """ Reads self.filename and stores data into self. Self is (or behaves like) a dictionary""" + self['data'] = matfile(self.filename,output='pandas') + + def _write(self): + """ Writes to self.filename""" + # --- Example: + #with open(self.filename,'w') as f: + # f.write(self.toString) + raise NotImplementedError() + + def toDataFrame(self): + """ Returns object into one DataFrame, or a dictionary of DataFrames""" + return self['data'] + + # --- Optional functions + def __repr__(self): + """ String that is written to screen when the user calls `print()` on the object. + Provide short and relevant information to save time for the user. + """ + s='<{} object>:\n'.format(type(self).__name__) + s+='|Main attributes:\n' + s+='| - filename: {}\n'.format(self.filename) + # --- Example printing some relevant information for user + #s+='|Main keys:\n' + #s+='| - ID: {}\n'.format(self['ID']) + #s+='| - data : shape {}\n'.format(self['data'].shape) + s+='|Main methods:\n' + s+='| - read, toDataFrame, keys\n' + s+='|Info:\n' + d1=self['data']['data1hz'] + d5=self['data']['data50hz'] + s+='| - data1hz : {} to {} (n:{}, T:{}, dt:{})\n'.format(d1.index[0], d1.index[-1], len(d1), (d1.index[-1]-d1.index[0]).total_seconds(), (d1.index[1]-d1.index[0]).total_seconds()) + s+='| - data50hz: {} to {} (n:{}, T:{}, dt:{})\n'.format(d5.index[0], d5.index[-1], len(d5), (d5.index[-1]-d5.index[0]).total_seconds(), (d5.index[1]-d5.index[0]).total_seconds()) + return s diff --git a/pydatview/io/rosco_performance_file.py b/pydatview/io/rosco_performance_file.py new file mode 100644 index 0000000..fac98b5 --- /dev/null +++ b/pydatview/io/rosco_performance_file.py @@ -0,0 +1,239 @@ +""" +Input/output class for the ROSCO performance (Cp,Ct,Cq) fileformat +""" +import numpy as np +import pandas as pd +import os + +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + File=dict + +class ROSCOPerformanceFile(File): + """ + Read/write a ROSCO performance file. The object behaves as a dictionary. + + Main methods + ------------ + - read, write, toDataFrame, keys + + Examples + -------- + f = ROSCOPerformanceFile('Cp_Ct_Cq.txt') + print(f.keys()) + print(f.toDataFrame().columns) + + """ + + @staticmethod + def defaultExtensions(): + """ List of file extensions expected for this fileformat""" + return ['.txt'] + + @staticmethod + def formatName(): + """ Short string (~100 char) identifying the file format""" + return 'ROSCO Performance file' + + def __init__(self, filename=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # --- Calling (children) function to read + self._read(**kwargs) + + def write(self, filename=None): + """ Rewrite object to file, or write object to `filename` if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + # Calling (children) function to write + self._write() + + def _read(self): + """ Reads self.filename and stores data into self. Self is (or behaves like) a dictionary""" + # --- Example: + pitch, TSR, WS, Cp, Ct, Cq = load_from_txt(self.filename) + self['pitch'] = pitch + self['TSR'] = TSR + self['WS'] = WS + self['CP'] = Cp + self['CT'] = Ct + self['CQ'] = Cq + + def _write(self): + """ Writes to self.filename""" + # --- Example: + write_rotor_performance(self.filename, self['pitch'], self['TSR'], self['CP'],self['CT'], self['CQ'], self['WS'], TurbineName='') + + def toDataFrame(self): + """ Returns object into dictionary of DataFrames""" + dfs={} + columns = ['TSR_[-]']+['Pitch_{:.2f}_[deg]'.format(p) for p in self['pitch']] + dfs['CP'] = pd.DataFrame(np.column_stack((self['TSR'], self['CP'])), columns=columns) + dfs['CT'] = pd.DataFrame(np.column_stack((self['TSR'], self['CT'])), columns=columns) + dfs['CQ'] = pd.DataFrame(np.column_stack((self['TSR'], self['CQ'])), columns=columns) + return dfs + + # --- Optional functions + def __repr__(self): + """ String that is written to screen when the user calls `print()` on the object. + Provide short and relevant information to save time for the user. + """ + s='<{} object>:\n'.format(type(self).__name__) + s+='|Main attributes:\n' + s+='| - filename: {}\n'.format(self.filename) + # --- Example printing some relevant information for user + s+='|Main keys:\n' + s+='| - pitch: {}\n'.format(self['pitch']) + s+='| - TSR: {}\n'.format(self['TSR']) + s+='| - WS: {}\n'.format(self['WS']) + s+='| - CP,CT,CQ : shape {}\n'.format(self['CP'].shape) + s+='|Main methods:\n' + s+='| - read, write, toDataFrame, keys' + return s + + + + +def load_from_txt(txt_filename): + ''' + Taken from ROSCO_toolbox/utitities.py by Nikhar Abbas + https://github.com/NREL/ROSCO + Apache 2.0 License + + Load rotor performance data from a *.txt file. + Parameters: + ----------- + txt_filename: str + Filename of the text containing the Cp, Ct, and Cq data. This should be in the format printed by the write_rotorperformance function + ''' + + pitch = None + TSR = None + WS = None + + with open(txt_filename) as pfile: + for iline, line in enumerate(pfile): + # Read Blade Pitch Angles (degrees) + if 'Pitch angle' in line: + pitch = np.array([float(x) for x in pfile.readline().strip().split()]) + + # Read Tip Speed Ratios (rad) + elif 'TSR' in line: + TSR = np.array([float(x) for x in pfile.readline().strip().split()]) + + #Read WS + elif 'Wind speed' in line: + WS = np.array([float(x) for x in pfile.readline().strip().split()]) + + # Read Power Coefficients + elif 'Power' in line: + pfile.readline() + Cp = np.empty((len(TSR),len(pitch))) + for tsr_i in range(len(TSR)): + Cp[tsr_i] = np.array([float(x) for x in pfile.readline().strip().split()]) + + # Read Thrust Coefficients + elif 'Thrust' in line: + pfile.readline() + Ct = np.empty((len(TSR),len(pitch))) + for tsr_i in range(len(TSR)): + Ct[tsr_i] = np.array([float(x) for x in pfile.readline().strip().split()]) + + # Read Torque Coefficients + elif 'Torque' in line: + pfile.readline() + Cq = np.empty((len(TSR),len(pitch))) + for tsr_i in range(len(TSR)): + Cq[tsr_i] = np.array([float(x) for x in pfile.readline().strip().split()]) + + if pitch is None and iline>10: + raise WrongFormatError('This does not appear to be a ROSCO performance file, Pitch vector not found') + + return pitch, TSR, WS, Cp, Ct, Cq + + +def write_rotor_performance(txt_filename, pitch, TSR, CP, CT, CQ, WS=None, TurbineName=''): + ''' + Taken from ROSCO_toolbox/utitities.py by Nikhar Abbas + https://github.com/NREL/ROSCO + Apache 2.0 License + + Write text file containing rotor performance data + Parameters: + ------------ + txt_filename: str, optional + Desired output filename to print rotor performance data. Default is Cp_Ct_Cq.txt + ''' + file = open(txt_filename,'w') + # Headerlines + file.write('# ----- Rotor performance tables for the {} wind turbine ----- \n'.format(TurbineName)) + file.write('# ------------ Written on {} using the ROSCO toolbox ------------ \n\n'.format(now.strftime('%b-%d-%y'))) + + # Pitch angles, TSR, and wind speed + file.write('# Pitch angle vector, {} entries - x axis (matrix columns) (deg)\n'.format(len(pitch))) + for i in range(len(pitch)): + file.write('{:0.4} '.format(pitch[i])) + file.write('\n# TSR vector, {} entries - y axis (matrix rows) (-)\n'.format(len(TSR))) + for i in range(len(TSR)): + file.write('{:0.4} '.format(TSR[i])) + if WS is not None: + file.write('\n# Wind speed vector - z axis (m/s)\n') + for i in range(len(WS)): + file.write('{:0.4} '.format(WS[i])) + file.write('\n') + + # Cp + file.write('\n# Power coefficient\n\n') + for i in range(len(TSR)): + for j in range(len(pitch)): + file.write('{0:.6f} '.format(CP[i,j])) + file.write('\n') + file.write('\n') + + # Ct + file.write('\n# Thrust coefficient\n\n') + for i in range(len(TSR)): + for j in range(len(pitch)): + file.write('{0:.6f} '.format(CT[i,j])) + file.write('\n') + file.write('\n') + + # Cq + file.write('\n# Torque coefficient\n\n') + for i in range(len(TSR)): + for j in range(len(pitch)): + file.write('{0:.6f} '.format(CQ[i,j])) + file.write('\n') + file.write('\n') + file.close() + + + + +if __name__ == '__main__': + f = ROSCOPerformanceFile('./tests/example_files/RoscoPerformance_CpCtCq.txt') + print(f) + dfs = f.toDataFrame() + print(dfs['CP']) + diff --git a/pydatview/io/tdms_file.py b/pydatview/io/tdms_file.py new file mode 100644 index 0000000..d05e1a9 --- /dev/null +++ b/pydatview/io/tdms_file.py @@ -0,0 +1,65 @@ +from .file import File, WrongFormatError, BrokenFormatError +import numpy as np +import pandas as pd + +class TDMSFile(File): + + @staticmethod + def defaultExtensions(): + return ['.tdms'] + + @staticmethod + def formatName(): + return 'TDMS file' + + def _read(self): + try: + from nptdms import TdmsFile + except: + raise Exception('Install the library nptdms to read this file') + + fh = TdmsFile(self.filename, read_metadata_only=False) + channels_address = list(fh.objects.keys()) + channels_address = [ s.replace("'",'') for s in channels_address] + channel_keys= [ s.split('/')[1:] for s in channels_address if len(s.split('/'))==3] + # --- Setting up list of signals and times + signals=[] + times=[] + for i,ck in enumerate(channel_keys): + channel = fh.object(ck[0],ck[1]) + signals.append(channel.data) + times.append (channel.time_track()) + + lenTimes = [len(time) for time in times] + minTimes = [np.min(time) for time in times] + maxTimes = [np.max(time) for time in times] + if len(np.unique(lenTimes))>1: + print(lenTimes) + raise NotImplementedError('Different time length') + # NOTE: could use fh.as_dataframe + if len(np.unique(minTimes))>1: + print(minTimes) + raise NotImplementedError('Different time span') + if len(np.unique(maxTimes))>1: + print(maxTimes) + raise NotImplementedError('Different time span') + # --- Gathering into a data frame with time + time =times[0] + signals = [time]+signals + M = np.column_stack(signals) + colnames = ['Time_[s]'] + [ck[1] for ck in channel_keys] + self['data'] = pd.DataFrame(data = M, columns=colnames) + +# def toString(self): +# s='' +# return s +# def _write(self): +# pass + + def __repr__(self): + s ='Class TDMS (key: data)\n' + return s + + def _toDataFrame(self): + return self['data'] + diff --git a/pydatview/io/tecplot_file.py b/pydatview/io/tecplot_file.py new file mode 100644 index 0000000..1c9aa2d --- /dev/null +++ b/pydatview/io/tecplot_file.py @@ -0,0 +1,222 @@ +""" +Read/Write TecPto ascii files +sea read_tecplot documentation below + +Part of weio library: https://github.com/ebranlard/weio + +""" +import pandas as pd +import numpy as np +import os +import struct + +try: + from .file import File, EmptyFileError, WrongFormatError, BrokenFormatError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + File=dict + + + +Keywords=['title','variables','zone','text','geometry','datasetauxdata','customlabels','varauxdata'] +# --------------------------------------------------------------------------------} +# --- Helper functions +# --------------------------------------------------------------------------------{ +def is_number(s): + try: + float(s) + return True + except ValueError: + pass + + try: + import unicodedata + unicodedata.numeric(s) + return True + except (TypeError, ValueError): + pass + + return False + + +def _process_merged_line(line, section, dict_out): + n = len(section) + line = line[n:].strip() + if section=='title': + dict_out[section]=line + elif section=='variables': + line = line.replace('=','').strip() + line = line.replace(',',' ').strip() + line = line.replace(' ',' ').strip() + line = line.replace('[','_[').strip() + line = line.replace('(','_(').strip() + line = line.replace('__','_').strip() + if line.find('"')==0: + line = line.replace('" "',',') + line = line.replace('"','') + sp=line.split(',') + else: + sp=line.split() + dict_out[section]=sp + elif section=='datasetauxdata': + if section not in dict_out.keys(): + dict_out[section]={} # initialixe an empty directory + sp = line.split('=') + key = sp[0] + value = sp[1].replace('"','').strip() + if is_number(value): + value=float(value) + dict_out[section][key]=value + + elif section=='zone': + if section not in dict_out.keys(): + dict_out[section]={} # initialixe an empty directory + sp = line.split('=') + key = sp[0] + value = sp[1].replace('"','').strip() + if is_number(value): + value=float(value) + dict_out[section][key]=value + + else: + print('!!! Reading of section not implemented:') + print('Processing section {}:'.format(section),line) + dict_out[section]=line + +def read_tecplot(filename, dict_out={}): + """ Reads a tecplot file + Limited support: + - title optional + - variables mandatory + - Lines may be continued to next line, stopping when a predefined keyword is detected + For now, assumes that only one section of numerical data is present + """ + + merged_line='' + current_section='' + variables=[] + with open(filename, "r") as f: + dfs = [] # list of dataframes + iline=0 + while True: + line= f.readline().strip() + iline+=1 + if not line: + break + l=line.lower().strip() + # Comment + if l[0]=='#': + continue + new_section = [k for k in Keywords if l.find(k)==0 ] + + if len(new_section)==1: + # --- Start of a new section + # First, process the previous section + if len(merged_line)>0: + _process_merged_line(merged_line, current_section, dict_out) + # Then start the new section + current_section=new_section[0] + merged_line =line + elif len(current_section)==0: + raise WrongFormatError('No section detected') + else: + if current_section=='title' or current_section=='variables': + # OK + pass + else: + if 'variables' not in dict_out.keys(): + raise WrongFormatError('The `variables` section should be present') + sp = l.split() + if is_number(sp[0]): + if len(merged_line)>0: + _process_merged_line(merged_line, current_section, dict_out) + # --- Special case of numerical values outside of zone + f.close() + M = np.loadtxt(filename, skiprows = iline-1) + if M.shape[1]!=len(dict_out['variables']): + raise BrokenFormatError('Number of columns of data does not match number of variables') + dict_out['data']=M + break + else: + # --- Continuation of previous section + merged_line +=' '+line + return dict_out + + +class TecplotFile(File): + + @staticmethod + def defaultExtensions(): + return ['.dat'] + + @staticmethod + def formatName(): + return 'Tecplot ASCII file' + + def __init__(self,filename=None,**kwargs): + self.filename = None + if filename: + self.read(filename=filename,**kwargs) + + def read(self, filename=None): + """ read a tecplot ascii file + sea `read_tecplot` documentation above + """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + + try: + read_tecplot(filename,self) + except BrokenFormatError: + raise + except WrongFormatError: + raise + except Exception as e: + raise WrongFormatError('Tecplot dat File {}: '.format(self.filename)+e.args[0]) + + def write(self, filename=None, precision=None): + """ Write tecplot ascii file """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + + with open(self.filename, mode='w') as f: + if 'title' in self.keys(): + f.write('TITLE = {}\n'.format(self['title'])) + f.write('VARIABLES = ' + ','.join(['"{}"'.format(col) for col in self['variables'] ]) + '\n') + for k in Keywords[2:]: + if k in self.keys(): + f.write('{} = {}\n'.format(k,self[k])) + # Data + if 'data' in self.keys(): + for row in self['data']: + srow = np.array2string(row, edgeitems=0, separator=' ', precision=precision) + f.write(srow[1:-1]+'\n') + + + def __repr__(self): + s='<{} object> with keys:\n'.format(type(self).__name__) + for k,v in self.items(): + s+=' - {}: {}\n'.format(k,v) + return s + + def toDataFrame(self): + return pd.DataFrame(data=self['data'],columns=self['variables']) + +if __name__=='__main__': + mb = MannBoxFile('mann_bin/mini-u.bin', N=(2,4,8)) + F1=mb['field'].ravel() + mb.write('mann_bin/mini-u-out.bin') + + mb2= MannBoxFile('mann_bin/mini-u-out.bin', N=(2,4,8)) + F2=mb2['field'].ravel() +# print(F1-F2) diff --git a/pydatview/io/tools/__init__.py b/pydatview/io/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydatview/io/tools/graph.py b/pydatview/io/tools/graph.py new file mode 100644 index 0000000..e924ef1 --- /dev/null +++ b/pydatview/io/tools/graph.py @@ -0,0 +1,672 @@ +""" +Basics Classes for a "geometrical" graph model: + - nodes have a position (x,y,z), and some data (taken from a list of properties) + - elements (links) connect nodes, they contain some data (taken from a list of properties) + +An ordering of Elements, Nodes, and Properties is present, but whenever possible, +the "ID" is used to identify them, instead of their index. + + +Nodes: + Node.ID: unique ID (int) of the node. IDs never change. + Node.x,y,z: coordinate of the nodes + Node.data : dictionary of data stored at the node + +Elements: + Elem.ID: unique ID (int) of the element. IDs never change. + Elem.nodeIDs: list of node IDs making up the element + Elem.nodes : list of nodes making the element (by reference) # NOTE: this has cross reference! + Elem.nodeProps : properties # Nodal properties. NOTE: cannot be transfered to node because of how SubDyn handles it.. + Elem.data : dictionary of data stored at the element + # Optional + Elem.propset: string referring to the property set in the dictionary of properties + Elem.propIDs: IDs used for the properties of this element at each node + +NodePropertySets: dictionary of NodeProperties + Node Property: + NProp.ID: unique ID of the node proprety + NProp.data: dictionary of data + + +ElemPropertySets: dictionary of ElemProperties + +""" + +import numpy as np +import pandas as pd + + +# --------------------------------------------------------------------------------} +# --- Node +# --------------------------------------------------------------------------------{ +class Node(object): + def __init__(self, ID, x, y, z=0, **kwargs): + self.ID = int(ID) + self.x = x + self.y = y + self.z = z + self.data = kwargs + + def setData(self, data_dict): + """ set or add data""" + for k,v in data_dict.items(): + #if k in self.data.keys(): + # print('Warning overriding key {} for node {}'.format(k,self.ID)) + self.data[k]=v + + def __repr__(self): + s=' x:{:7.2f} y:{:7.2f} z:{:7.2f} {:}'.format(self.ID, self.x, self.y, self.z, self.data) + return s + +# --------------------------------------------------------------------------------} +# --- Properties +# --------------------------------------------------------------------------------{ +class Property(dict): + def __init__(self, ID, data=None, **kwargs): + """ + data is a dictionary + """ + dict.__init__(self) + self.ID= int(ID) + self.update(kwargs) + if data is not None: + self.update(data) + + @property + def data(self): + return {k:v for k,v in self.items() if k!='ID'} + + def __repr__(self): + s=' {:}'.format(self.ID, self.data) + return s + +class NodeProperty(Property): + def __init__(self, ID, data=None, **kwargs): + Property.__init__(self, ID, data, **kwargs) + def __repr__(self): + s=' {:}'.format(self.ID, self.data) + return s + +class ElemProperty(Property): + def __init__(self, ID, data=None, **kwargs): + Property.__init__(self, ID, data, **kwargs) + def __repr__(self): + s=' {:}'.format(self.ID, self.data) + return s + + +# --------------------------------------------------------------------------------} +# --- Elements +# --------------------------------------------------------------------------------{ +class Element(dict): + def __init__(self, ID, nodeIDs, nodes=None, propset=None, propIDs=None, properties=None, **kwargs): + """ + + """ + self.ID = int(ID) + self.nodeIDs = nodeIDs + self.propset = propset + self.propIDs = propIDs + self.data = kwargs # Nodal data + self.nodes = nodes # Typically a trigger based on nodeIDs + self.nodeProps= properties # Typically a trigger based on propIDs. Otherwise list of dictionaries + if (self.propIDs is not None) and (self.propset is None): + raise Exception('`propset` should be provided if `propIDs` are provided') + if (self.propIDs is not None) and (self.propset is not None) and properties is not None: + raise Exception('When providing `propset` & `propIDs`, properties should not be provided') + if nodes is not None: + if len(nodes)!=len(nodeIDs): + raise Exception('List of nodes has different length than list of nodeIDs') + for i, (ID,n) in enumerate(zip(nodeIDs,nodes)): + if n.ID!=ID: + raise Exception('Node ID do not match {}/={} for node index {}'.format(n.ID,ID,i)) + + @property + def length(self): + n1=self.nodes[0] + n2=self.nodes[1] + return np.sqrt((n1.x-n2.x)**2+(n1.y-n2.y)**2+(n1.z-n2.z)**2) + + def __repr__(self): + s=' NodeIDs: {} {}'.format(self.ID, self.nodeIDs, self.data) + if self.propIDs is not None: + s+=' {'+'propIDs:{} propset:{}'.format(self.propIDs, self.propset)+'}' + if self.nodes is not None: + s+=' l={:.2f}'.format(self.length) + return s + + +# --------------------------------------------------------------------------------} +# --- Mode +# --------------------------------------------------------------------------------{ +class Mode(dict): + def __init__(self, data, name, freq=1, **kwargs): + dict.__init__(self) + + self['name']=name + self['freq']=freq + self['data']=data # displacements nNodes x 3 assuming a given sorting of nodes + + def __repr__(self): + s=' name:{:4s} freq:{:} '.format(self['name'], self['freq']) + return s + + def reSort(self,I): + self['data']=self['data'][I,:] + +# --------------------------------------------------------------------------------} +# --- Graph +# --------------------------------------------------------------------------------{ +class GraphModel(object): + def __init__(self, Elements=None, Nodes=None, NodePropertySets=None, ElemPropertySets=None, MiscPropertySets=None ): + self.Elements = Elements if Elements is not None else [] + self.Nodes = Nodes if Nodes is not None else [] + self.NodePropertySets = NodePropertySets if NodePropertySets is not None else {} + self.ElemPropertySets = ElemPropertySets if ElemPropertySets is not None else {} + self.MiscPropertySets = MiscPropertySets if MiscPropertySets is not None else {} + # Dynamics + self.Modes = [] + self.Motions = [] + # Optimization variables + self._nodeIDs2Elements = {} # dictionary with key NodeID and value list of ElementID + self._nodeIDs2Elements = {} # dictionary with key NodeID and value list of elements + self._elementIDs2NodeIDs = {} # dictionary with key ElemID and value list of nodes IDs + self._connectivity =[]# + + def addNode(self,node): + self.Nodes.append(node) + + def addElement(self,elem): + # Giving nodes to element if these were not provided + elem.nodes=[self.getNode(i) for i in elem.nodeIDs] + # Giving props to element if these were not provided + if elem.propIDs is not None: + elem.nodeProps=[self.getNodeProperty(elem.propset, i) for i in elem.propIDs] + self.Elements.append(elem) + + # --- Getters + def getNode(self, nodeID): + for n in self.Nodes: + if n.ID==nodeID: + return n + raise KeyError('NodeID {} not found in Nodes'.format(nodeID)) + + def getElement(self, elemID): + for e in self.Elements: + if e.ID==elemID: + return e + raise KeyError('ElemID {} not found in Elements'.format(elemID)) + + def getNodeProperty(self, setname, propID): + for p in self.NodePropertySets[setname]: + if p.ID==propID: + return p + raise KeyError('PropID {} not found for Node propset {}'.format(propID,setname)) + + def getElementProperty(self, setname, propID): + for p in self.ElemPropertySets[setname]: + if p.ID==propID: + return p + raise KeyError('PropID {} not found for Element propset {}'.format(propID,setname)) + + def getMiscProperty(self, setname, propID): + for p in self.MiscPropertySets[setname]: + if p.ID==propID: + return p + raise KeyError('PropID {} not found for Misc propset {}'.format(propID,setname)) + + # --- + @property + def nodeIDs2ElementIDs(self): + """ Return list of elements IDs connected to each node""" + if len(self._nodeIDs2ElementIDs) == 0: + # Compute list of connected elements for each node + self._nodeIDs2ElementIDs=dict() + for i,n in enumerate(self.Nodes): + self._nodeIDs2ElementIDs[n.ID] = [e.ID for e in self.Elements if n.ID in e.nodeIDs] + return self._nodeIDs2ElementIDs + + @property + def nodeIDs2Elements(self): + """ Return list of elements connected to each node""" + if len(self._nodeIDs2Elements) == 0: + # Compute list of connected elements for each node + self._nodeIDs2Elements + for i,n in enumerate(self.Nodes): + self._nodeIDs2Elements[n.ID] = [e for e in self.Elements if n.ID in e.nodeIDs] + return self._nodeIDs2Elements + + + @property + def elementIDs2NodeIDs(self): + """ returns """ + if len(self._elementIDs2NodeIDs) ==0: + self._elementIDs2NodeIDs =dict() + for e in self.Elements: + self._elementIDs2NodeIDs[e.ID] = [n.ID for n in e.nodes] + return self._elementIDs2NodeIDs + + + @property + def connectivity(self): + """ returns connectivity, assuming points are indexed starting at 0 + NOTE: this is basically element2Nodes but reindexed + """ + if len(self._connectivity) ==0: + self._connectivity = [[self.Nodes.index(n) for n in e.nodes] for e in self.Elements] + return self._connectivity + + + # --- Handling of (element/material) Properties + def addElementPropertySet(self, setname): + self.ElemPropertySets[setname]= [] + + def addNodePropertySet(self, setname): + self.NodePropertySets[setname]= [] + + def addMiscPropertySet(self, setname): + self.MiscPropertySets[setname]= [] + + def addNodeProperty(self, setname, prop): + if not isinstance(prop, NodeProperty): + print(type(prop)) + raise Exception('Property needs to inherit from NodeProperty') + self.PropertySets[setname].append(prop) + + def addNodeProperty(self, setname, prop): + if not isinstance(prop, NodeProperty): + print(type(prop)) + raise Exception('Property needs to inherit from NodeProperty') + self.NodePropertySets[setname].append(prop) + + def addElementProperty(self, setname, prop): + if not isinstance(prop, ElemProperty): + print(type(prop)) + raise Exception('Property needs to inherit from ElementProperty') + self.ElemPropertySets[setname].append(prop) + + def addMiscProperty(self, setname, prop): + if not isinstance(prop, ElemProperty): + print(type(prop)) + raise Exception('Property needs to inherit from Property') + self.MiscPropertySets[setname].append(prop) + + # --- Data and node and element prop setters + def setElementNodalProp(self, elem, propset, propIDs): + """ + Set Nodal Properties to each node of an element + """ + for node, pID in zip(elem.nodes, propIDs): + node.setData(self.getNodeProperty(propset, pID).data) + + def setNodeNodalProp(self, node, propset, propID): + """ + Set Nodal Properties to a node + """ + node.setData(self.getNodeProperty(propset, propID).data) + + def setNodalData(self, nodeID, **data_dict): + self.getNode(nodeID).setData(data_dict) + + def __repr__(self): + s='<{} object> with keys:\n'.format(type(self).__name__) + s+='- Nodes ({}):\n'.format(len(self.Nodes)) + s+='\n'.join(str(n) for n in self.Nodes) + s+='\n- Elements ({}):\n'.format(len(self.Elements)) + s+='\n'.join(str(n) for n in self.Elements) + s+='\n- NodePropertySets ({}):'.format(len(self.NodePropertySets)) + for k,v in self.NodePropertySets.items(): + s+='\n> {} ({}):\n'.format(k, len(v)) + s+='\n'.join(str(p) for p in v) + s+='\n- ElementPropertySets ({}):'.format(len(self.ElemPropertySets)) + for k,v in self.ElemPropertySets.items(): + s+='\n> {} ({}):\n'.format(k, len(v)) + s+='\n'.join(str(p) for p in v) + s+='\n- MiscPropertySets ({}):'.format(len(self.MiscPropertySets)) + for k,v in self.MiscPropertySets.items(): + s+='\n> {} ({}):\n'.format(k, len(v)) + s+='\n'.join(str(p) for p in v) + s+='\n- Modes ({}):\n'.format(len(self.Modes)) + s+='\n'.join(str(m) for m in self.Modes) + s+='\n- Motions ({}):'.format(len(self.Motions)) + for m in self.Motions: + s+='\n> {}\n'.format({k:v for k,v in m.items() if not isintance(v,np.ndarray)}) + return s + + # --------------------------------------------------------------------------------} + # --- Geometrical properties + # --------------------------------------------------------------------------------{ + @property + def extent(self): + xmax=np.max([node.x for node in self.Nodes]) + ymax=np.max([node.y for node in self.Nodes]) + zmax=np.max([node.z for node in self.Nodes]) + xmin=np.min([node.x for node in self.Nodes]) + ymin=np.min([node.y for node in self.Nodes]) + zmin=np.min([node.z for node in self.Nodes]) + return [xmin,ymin,zmin],[xmax,ymax,zmax],[xmax-xmin,ymax-ymin,zmax-zmin] + + @property + def maxDimension(self): + _,_,D=self.extent + return np.max(D) + + @property + def points(self): + nNodes = len(self.Nodes) + Points = np.zeros((nNodes,3)) + for i,n in enumerate(self.Nodes): + Points[i,:]=(n.x, n.y, n.z) + return Points + + def toLines(self, output='coord'): + if output=='coord': + lines = np.zeros((len(self.Elements), 2, 3)) # + for ie, e in enumerate(self.Elements): + n1=e.nodes[0] + n2=e.nodes[-1] + lines[ie, 0, : ] = (n1.x, n1.y, n1.z) + lines[ie, 1, : ] = (n2.x, n2.y, n2.z) + elif output=='lines3d': + import mpl_toolkits.mplot3d as plt3d + lines=[] + for ie, e in enumerate(self.Elements): + n1=e.nodes[0] + n2=e.nodes[-1] + line = plt3d.art3d.Line3D((n1.x,n2.x), (n1.y,n2.y), (n1.z,n2.z)) + lines.append(line) + else: + raise NotImplementedError() + + return lines + + # --------------------------------------------------------------------------------} + # --- Change of connectivity + # --------------------------------------------------------------------------------{ + def connecticityHasChanged(self): + self._nodeIDs2ElementIDs = dict() + self._nodeIDs2Elements = dict() + self._elementIDs2NodeIDs = dict() + self._connectivity=[] + + def updateConnectivity(self): + for e in self.Elements: + e.nodes=[self.getNode(i) for i in e.nodeIDs] + + for e in self.Elements: + e.nodeProps = [self.getNodeProperty(e.propset, ID) for ID in e.propIDs] + + # Potentially call nodeIDs2ElementIDs etc + + + def _divideElement(self, elemID, nPerElement, maxElemId, keysNotToCopy=[]): + """ divide a given element by nPerElement (add nodes and elements to graph) """ + if len(self.Modes)>0: + raise Exception('Cannot divide graph when mode data is present') + if len(self.Motions)>0: + raise Exception('Cannot divide graph when motion data is present') + + + maxNodeId=np.max([n.ID for n in self.Nodes]) + e = self.getElement(elemID) + newElems = [] + if len(e.nodes)==2: + n1=e.nodes[0] + n2=e.nodes[1] + subNodes=[n1] + for iSub in range(1,nPerElement): + maxNodeId += 1 + #data_dict = n1.data.copy() + data_dict = dict() + fact = float(iSub)/nPerElement + # Interpolating position + x = n1.x*(1-fact)+n2.x*fact + y = n1.y*(1-fact)+n2.y*fact + z = n1.z*(1-fact)+n2.z*fact + # Interpolating data (only if floats) + for k,v in n1.data.items(): + if k not in keysNotToCopy: + try: + data_dict[k] = n1.data[k]*(1-fact) + n2.data[k]*fact + except: + data_dict[k] = n1.data[k] + ni = Node(maxNodeId, x, y, z, **data_dict) + subNodes.append(ni) + self.addNode(ni) + subNodes+=[n2] + e.nodes =subNodes[0:2] + e.nodeIDs=[e.ID for e in e.nodes] + for i in range(1,nPerElement): + maxElemId+=1 + elem_dict = e.data.copy() + # Creating extra properties if necessary + if e.propIDs is not None: + if all(e.propIDs==e.propIDs[0]): + # No need to create a new property + propIDs=e.propIDs + propset=e.propset + else: + raise NotImplementedError('Division of element with different properties on both ends. TODO add new property.') + elem= Element(maxElemId, [subNodes[i].ID, subNodes[i+1].ID], propset=propset, propIDs=propIDs, **elem_dict ) + newElems.append(elem) + return newElems + + + def sortNodesBy(self,key): + """ Sort nodes, will affect the connectivity, but node IDs remain the same""" + + # TODO, that's quite doable + if len(self.Modes)>0: + raise Exception('Cannot sort nodes when mode data is present') + if len(self.Motions)>0: + raise Exception('Cannot sort nodes when motion data is present') + + nNodes = len(self.Nodes) + if key=='x': + values=[n.x for n in self.Nodes] + elif key=='y': + values=[n.y for n in self.Nodes] + elif key=='z': + values=[n.z for n in self.Nodes] + elif key=='ID': + values=[n.ID for n in self.Nodes] + else: + values=[n[key] for n in self.Nodes] + I= np.argsort(values) + self.Nodes=[self.Nodes[i] for i in I] + + # Trigger, remove precomputed values related to connectivity: + self.connecticityHasChanged() + + return self + + def divideElements(self, nPerElement, excludeDataKey='', excludeDataList=[], method='append', keysNotToCopy=[]): + """ divide all elements by nPerElement (add nodes and elements to graph) + + - excludeDataKey: is provided, will exclude elements such that e.data[key] in `excludeDataList` + + - method: append or insert + + - keysNotToCopy: when duplicating node and element data, make sure not to duplicate data with these keys + For instance if a node that has a boundary condition, it should not be passed to the + node that is created when dividing an element. + + Example: + to avoid dividing elements of `Type` 'Cable' or `Rigid`, call as follows: + self.divideElements(3, excludeDataKey='Type', excludeDataList=['Cable','Rigid'] ) + + """ + maxNodeId=np.max([n.ID for n in self.Nodes]) + maxElemId=np.max([e.ID for e in self.Elements]) + + if nPerElement<=0: + raise Exception('nPerElement should be more than 0') + + newElements=[] + for ie in np.arange(len(self.Elements)): # cannot enumerate since length increases + elemID = self.Elements[ie].ID + if method=='insert': + newElements+=[self.getElement(elemID)] # newElements contains + if (len(excludeDataKey)>0 and self.Elements[ie].data[excludeDataKey] not in excludeDataList) or len(excludeDataKey)==0: + elems = self._divideElement(elemID, nPerElement, maxElemId, keysNotToCopy) + maxElemId+=len(elems) + newElements+=elems + else: + print('Not dividing element with ID {}, based on key `{}` with value `{}`'.format(elemID, excludeDataKey,self.Elements[ie].data[excludeDataKey])) + # Adding elements at the end + if method=='append': + pass + elif method=='insert': + self.Elements=[] # We clear all elements + else: + raise NotImplementedError('Element Insertions') + + for e in newElements: + self.addElement(e) + + # Trigger, remove precomputed values related to connectivity: + self.connecticityHasChanged() + + return self + + # --------------------------------------------------------------------------------} + # --- Dynamics + # --------------------------------------------------------------------------------{ + def addMode(self,displ,name=None,freq=1): + if name is None: + name='Mode '+str(len(self.Modes)) + mode = Mode(data=displ, name=name, freq=freq) + self.Modes.append(mode) + + + # --------------------------------------------------------------------------------} + # --- Ouputs / converters + # --------------------------------------------------------------------------------{ + def nodalDataFrame(self, sortBy=None): + """ return a DataFrame of all the nodal data """ + data=dict() + nNodes=len(self.Nodes) + for i,n in enumerate(self.Nodes): + if i==0: + data['ID'] = np.zeros(nNodes).astype(int) + data['x'] = np.zeros(nNodes) + data['y'] = np.zeros(nNodes) + data['z'] = np.zeros(nNodes) + + data['ID'][i] = n.ID + data['x'][i] = n.x + data['y'][i] = n.y + data['z'][i] = n.z + for k,v in n.data.items(): + if k not in data: + data[k] = np.zeros(nNodes) + try: + data[k][i]=v + except: + pass + df = pd.DataFrame(data) + # Sorting + if sortBy is not None: + df.sort_values([sortBy],inplace=True,ascending=True) + df.reset_index(drop=True,inplace=True) + return df + + + def toJSON(self,outfile=None): + d=dict(); + Points=self.points + d['Connectivity'] = self.connectivity + d['Nodes'] = Points.tolist() + + d['ElemProps']=list() + for iElem,elem in enumerate(self.Elements): + Shape = elem.data['shape'] if 'shape' in elem.data.keys() else 'cylinder' + Type = elem.data['Type'] if 'Type' in elem.data.keys() else 1 + try: + Diam = elem.D + except: + Diam = elem.data['D'] if 'D' in elem.data.keys() else 1 + if Shape=='cylinder': + d['ElemProps'].append({'shape':'cylinder','type':Type, 'Diam':Diam}) + else: + raise NotImplementedError() + + + d['Modes']=[ + { + 'name': self.Modes[iMode]['name'], + 'omega':self.Modes[iMode]['freq']*2*np.pi, #in [rad/s] + 'Displ':self.Modes[iMode]['data'].tolist() + } for iMode,mode in enumerate(self.Modes)] + d['groundLevel']=np.min(Points[:,2]) # TODO + + if outfile is not None: + import json + from io import open + jsonFile=outfile + with open(jsonFile, 'w', encoding='utf-8') as f: + #f.write(to_json(d)) + try: + #f.write(unicode(json.dumps(d, ensure_ascii=False))) #, indent=2) + #f.write(json.dumps(d, ensure_ascii=False)) #, indent=2) + f.write(json.dumps(d, ensure_ascii=False)) + except: + print('>>> FAILED') + json.dump(d, f, indent=0) + return d + +# + + + +INDENT = 3 +SPACE = " " +NEWLINE = "\n" +# Changed basestring to str, and dict uses items() instead of iteritems(). + +def to_json(o, level=0): + ret = "" + if isinstance(o, dict): + if level==0: + ret += "{" + NEWLINE + comma = "" + for k, v in o.items(): + ret += comma + comma = ",\n" + ret += SPACE * INDENT * (level + 1) + ret += '"' + str(k) + '":' + SPACE + ret += to_json(v, level + 1) + ret += NEWLINE + SPACE * INDENT * level + "}" + else: + ret += "{" + comma = "" + for k, v in o.items(): + ret += comma + comma = ",\n" + ret += SPACE + ret += '"' + str(k) + '":' + SPACE + ret += to_json(v, level + 1) + ret += "}" + + elif isinstance(o, str): + ret += '"' + o + '"' + elif isinstance(o, list): + ret += "[" + ",".join([to_json(e, level + 1) for e in o]) + "]" + # Tuples are interpreted as lists + elif isinstance(o, tuple): + ret += "[" + ",".join(to_json(e, level + 1) for e in o) + "]" + elif isinstance(o, bool): + ret += "true" if o else "false" + elif isinstance(o, int): + ret += str(o) + elif isinstance(o, float): + ret += '%.7g' % o + elif isinstance(o, numpy.ndarray) and numpy.issubdtype(o.dtype, numpy.integer): + ret += "[" + ','.join(map(str, o.flatten().tolist())) + "]" + elif isinstance(o, numpy.ndarray) and numpy.issubdtype(o.dtype, numpy.inexact): + ret += "[" + ','.join(map(lambda x: '%.7g' % x, o.flatten().tolist())) + "]" + elif o is None: + ret += 'null' + else: + raise TypeError("Unknown type '%s' for json serialization" % str(type(o))) + return ret diff --git a/pydatview/io/turbsim_file.py b/pydatview/io/turbsim_file.py new file mode 100644 index 0000000..ef9d8a0 --- /dev/null +++ b/pydatview/io/turbsim_file.py @@ -0,0 +1,770 @@ +"""Read/Write TurbSim File + +Part of weio library: https://github.com/ebranlard/weio + +""" +import pandas as pd +import numpy as np +import os +import struct +import time + +try: + from .file import File, EmptyFileError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + File=dict + +class TurbSimFile(File): + """ + Read/write a TurbSim turbulence file (.bts). The object behaves as a dictionary. + + Main keys + --------- + - 'u': velocity field, shape (3 x nt x ny x nz) + - 'y', 'z', 't': space and time coordinates + - 'dt', 'ID', 'info' + - 'zTwr', 'uTwr': tower coordinates and field if present (3 x nt x nTwr) + - 'zRef', 'uRef': height and velocity at a reference point (usually not hub) + + Main methods + ------------ + - read, write, toDataFrame, keys, valuesAt, makePeriodic, checkPeriodic, closestPoint + + Examples + -------- + + ts = TurbSimFile('Turb.bts') + print(ts.keys()) + print(ts['u'].shape) + u,v,w = ts.valuesAt(y=10.5, z=90) + + + """ + + @staticmethod + def defaultExtensions(): + return ['.bts'] + + @staticmethod + def formatName(): + return 'TurbSim binary' + + def __init__(self,filename=None, **kwargs): + self.filename = None + if filename: + self.read(filename, **kwargs) + + def read(self, filename=None, header_only=False): + """ read BTS file, with field: + u (3 x nt x ny x nz) + uTwr (3 x nt x nTwr) + """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + + scl = np.zeros(3, np.float32); off = np.zeros(3, np.float32) + with open(self.filename, mode='rb') as f: + # Reading header info + ID, nz, ny, nTwr, nt = struct.unpack('0) + for it in range(nt): + Buffer = np.frombuffer(f.read(2*3*ny*nz), dtype=np.int16).astype(np.float32).reshape([3, ny, nz], order='F') + u[:,it,:,:]=Buffer + Buffer = np.frombuffer(f.read(2*3*nTwr), dtype=np.int16).astype(np.float32).reshape([3, nTwr], order='F') + uTwr[:,it,:]=Buffer + u -= off[:, None, None, None] + u /= scl[:, None, None, None] + self['u'] = u + uTwr -= off[:, None, None] + uTwr /= scl[:, None, None] + self['uTwr'] = uTwr + self['info'] = info + self['ID'] = ID + self['dt'] = dt + self['y'] = np.arange(ny)*dy + self['y'] -= np.mean(self['y']) # y always centered on 0 + self['z'] = np.arange(nz)*dz +zBottom + self['t'] = np.arange(nt)*dt + self['zTwr'] =-np.arange(nTwr)*dz + zBottom + self['zRef'] = zHub + self['uRef'] = uHub + + def write(self, filename=None): + """ + write a BTS file, using the following keys: 'u','z','y','t','uTwr' + u (3 x nt x ny x nz) + uTwr (3 x nt x nTwr) + """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + + nDim, nt, ny, nz = self['u'].shape + if 'uTwr' not in self.keys() : + self['uTwr']=np.zeros((3,nt,0)) + if 'ID' not in self.keys() : + self['ID']=7 + + _, _, nTwr = self['uTwr'].shape + tsTwr = self['uTwr'] + ts = self['u'] + intmin = -32768 + intrng = 65535 + off = np.empty((3), dtype = np.float32) + scl = np.empty((3), dtype = np.float32) + info = 'Generated by TurbSimFile on {:s}.'.format(time.strftime('%d-%b-%Y at %H:%M:%S', time.localtime())) + # Calculate scaling, offsets and scaling data + out = np.empty(ts.shape, dtype=np.int16) + outTwr = np.empty(tsTwr.shape, dtype=np.int16) + for k in range(3): + all_min, all_max = ts[k].min(), ts[k].max() + if nTwr>0: + all_min=min(all_min, tsTwr[k].min()) + all_max=max(all_max, tsTwr[k].max()) + if all_min == all_max: + scl[k] = 1 + else: + scl[k] = intrng / (all_max-all_min) + off[k] = intmin - scl[k] * all_min + out[k] = (ts[k] * scl[k] + off[k]).astype(np.int16) + outTwr[k] = (tsTwr[k] * scl[k] + off[k]).astype(np.int16) + z0 = self['z'][0] + dz = self['z'][1]- self['z'][0] + dy = self['y'][1]- self['y'][0] + dt = self['t'][1]- self['t'][0] + + # Providing estimates of uHub and zHub even if these fields are not used + zHub,uHub, bHub = self.hubValues() + + with open(self.filename, mode='wb') as f: + f.write(struct.pack('0: + s+=' - zTwr: [{} ... {}], dz: {}, n: {} \n'.format(self['zTwr'][0],self['zTwr'][-1],self['zTwr'][1]-self['zTwr'][0],len(self['zTwr'])) + if 'uTwr' in self.keys() and self['uTwr'].shape[2]>0: + s+=' - uTwr: ({} x {} x {} ) \n'.format(*(self['uTwr'].shape)) + ux,uy,uz=self['uTwr'][0], self['uTwr'][1], self['uTwr'][2] + s+=' ux: min: {}, max: {}, mean: {} \n'.format(np.min(ux), np.max(ux), np.mean(ux)) + s+=' uy: min: {}, max: {}, mean: {} \n'.format(np.min(uy), np.max(uy), np.mean(uy)) + s+=' uz: min: {}, max: {}, mean: {} \n'.format(np.min(uz), np.max(uz), np.mean(uz)) + + return s + + def toDataFrame(self): + dfs={} + + ny = len(self['y']) + nz = len(self['y']) + # Index at mid box + iy,iz = self.iMid + + # Mean vertical profile + z, m, s = self.vertProfile + ti = s/m*100 + cols=['z_[m]','u_[m/s]','v_[m/s]','w_[m/s]','sigma_u_[m/s]','sigma_v_[m/s]','sigma_w_[m/s]','TI_[%]'] + data = np.column_stack((z, m[0,:],m[1,:],m[2,:],s[0,:],s[1,:],s[2,:],ti[0,:])) + dfs['VertProfile'] = pd.DataFrame(data = data ,columns = cols) + + # Mid time series + u = self['u'][:,:,iy,iz] + cols=['t_[s]','u_[m/s]','v_[m/s]','w_[m/s]'] + data = np.column_stack((self['t'],u[0,:],u[1,:],u[2,:])) + dfs['ZMidLine'] = pd.DataFrame(data = data ,columns = cols) + + + # ZMid YStart time series + u = self['u'][:,:,0,iz] + cols=['t_[s]','u_[m/s]','v_[m/s]','w_[m/s]'] + data = np.column_stack((self['t'],u[0,:],u[1,:],u[2,:])) + dfs['ZMidYStartLine'] = pd.DataFrame(data = data ,columns = cols) + + # ZMid YEnd time series + u = self['u'][:,:,-1,iz] + cols=['t_[s]','u_[m/s]','v_[m/s]','w_[m/s]'] + data = np.column_stack((self['t'],u[0,:],u[1,:],u[2,:])) + dfs['ZMidYEndLine'] = pd.DataFrame(data = data ,columns = cols) + + # Mid crosscorr y + y, rho_uu_y, rho_vv_y, rho_ww_y = self.crosscorr_y() + cols = ['y_[m]', 'rho_uu_[-]','rho_vv_[-]','rho_ww_[-]'] + data = np.column_stack((y, rho_uu_y, rho_vv_y, rho_ww_y)) + dfs['Mid_xcorr_y'] = pd.DataFrame(data = data ,columns = cols) + + # Mid crosscorr z + z, rho_uu_z, rho_vv_z, rho_ww_z = self.crosscorr_z() + cols = ['z_[m]', 'rho_uu_[-]','rho_vv_[-]','rho_ww_[-]'] + data = np.column_stack((z, rho_uu_z, rho_vv_z, rho_ww_z)) + dfs['Mid_xcorr_z'] = pd.DataFrame(data = data ,columns = cols) + + # Mid csd + try: + fc, chi_uu, chi_vv, chi_ww = self.csd_longi() + cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] + data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) + dfs['Mid_csd_longi'] = pd.DataFrame(data = data ,columns = cols) + + # Mid csd + fc, chi_uu, chi_vv, chi_ww = self.csd_lat() + cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] + data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) + dfs['Mid_csd_lat'] = pd.DataFrame(data = data ,columns = cols) + + # Mid csd + fc, chi_uu, chi_vv, chi_ww = self.csd_vert() + cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] + data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) + dfs['Mid_csd_vert'] = pd.DataFrame(data = data ,columns = cols) + except ModuleNotFoundError: + print('Module scipy.signal not available') + except ImportError: + print('Likely issue with fftpack') + + + # Hub time series + #try: + # zHub = self['zHub'] + # iz = np.argmin(np.abs(self['z']-zHub)) + # u = self['u'][:,:,iy,iz] + # Cols=['t_[s]','u_[m/s]','v_[m/s]','w_[m/s]'] + # data = np.column_stack((self['t'],u[0,:],u[1,:],u[2,:])) + # dfs['TSHubLine'] = pd.DataFrame(data = data ,columns = Cols) + #except: + # pass + return dfs + + + # Useful converters + def fromMannBox(self, u, v, w, dx, U, y, z, addU=None): + """ + Convert current TurbSim file into one generated from MannBox + Assumes: + u, v, w (nt x ny x nz) + + y: goes from -ly/2 to ly/2 this is an IMPORTANT subtlety + The field u needs to respect this convention! + (fields from weio.mannbox_file do respect this convention + but when exported to binary files, the y axis is flipped again) + + INPUTS: + - u, v, w : mann box fields + - dx: axial spacing of mann box (to compute time) + - U: reference speed of mann box (to compute time) + - y: y coords of mann box + - z: z coords of mann box + """ + nt,ny,nz = u.shape + dt = dx/U + t = np.arange(0, dt*(nt-0.5), dt) + nt = len(t) + if y[0]>y[-1]: + raise Exception('y is assumed to go from - to +') + + self['u']=np.zeros((3, nt, ny, nz)) + self['u'][0,:,:,:] = u + self['u'][1,:,:,:] = v + self['u'][2,:,:,:] = w + if addU is not None: + self['u'][0,:,:,:] += addU + self['t'] = t + self['y'] = y + self['z'] = z + self['dt'] = dt + # TODO + self['ID'] = 7 # ... + self['info'] = 'Converted from MannBox fields {:s}.'.format(time.strftime('%d-%b-%Y at %H:%M:%S', time.localtime())) +# self['zTwr'] = np.array([]) +# self['uTwr'] = np.array([]) + self['zRef'] = None + self['uRef'] = None + self['zRef'], self['uRef'], bHub = self.hubValues() + + def toMannBox(self, base=None, removeUConstant=None, removeAllUMean=False): + """ + removeUConstant: float, will be removed from all values of the U box + removeAllUMean: If true, the time-average of each y-z points will be substracted + """ + try: + from weio.mannbox_file import MannBoxFile + except: + try: + from .mannbox_file import MannBoxFile + except: + from mannbox_file import MannBoxFile + # filename + if base is None: + base = os.path.splitext(self.filename)[0] + base = base+'_{}x{}x{}'.format(*self['u'].shape[1:]) + + mn = MannBoxFile() + mn.fromTurbSim(self['u'], 0, removeConstant=removeUConstant, removeAllMean=removeAllUMean) + mn.write(base+'.u') + + mn.fromTurbSim(self['u'], 1) + mn.write(base+'.v') + + mn.fromTurbSim(self['u'], 2) + mn.write(base+'.w') + + # --- Useful IO + def writeInfo(ts, filename): + """ Write info to txt """ + import scipy.optimize as so + def fit_powerlaw_u_alpha(x, y, z_ref=100, p0=(10,0.1)): + """ + p[0] : u_ref + p[1] : alpha + """ + pfit, _ = so.curve_fit(lambda x, *p : p[0] * (x / z_ref) ** p[1], x, y, p0=p0) + y_fit = pfit[0] * (x / z_ref) ** pfit[1] + coeffs_dict={'u_ref':pfit[0],'alpha':pfit[1]} + formula = '{u_ref} * (z / {z_ref}) ** {alpha}' + fitted_fun = lambda xx: pfit[0] * (xx / z_ref) ** pfit[1] + return y_fit, pfit, {'coeffs':coeffs_dict,'formula':formula,'fitted_function':fitted_fun} + infofile = filename + with open(filename,'w') as f: + f.write(str(ts)) + zMid =(ts['z'][0]+ts['z'][-1])/2 + f.write('Middle height of box: {:.3f}\n'.format(zMid)) + + iy,_ = ts.iMid + u = np.mean(ts['u'][0,:,iy,:], axis=0) + z=ts['z'] + f.write('\n') + y_fit, pfit, model = fit_powerlaw_u_alpha(z, u, z_ref=zMid, p0=(10,0.1)) + f.write('Power law: alpha={:.5f} - u={:.5f} at z={:.5f}\n'.format(pfit[1],pfit[0],zMid)) + f.write('Periodic: {}\n'.format(ts.checkPeriodic(sigmaTol=1.5, aTol=0.5))) + + + + def writeProbes(ts, probefile, yProbe, zProbe): + # Creating csv file with data at some probe locations + Columns=['Time_[s]'] + Data = ts['t'] + for y in yProbe: + for z in zProbe: + iy = np.argmin(np.abs(ts['y']-y)) + iz = np.argmin(np.abs(ts['z']-z)) + lbl = '_y{:.0f}_z{:.0f}'.format(ts['y'][iy], ts['z'][iz]) + Columns+=['{}{}_[m/s]'.format(c,lbl) for c in['u','v','w']] + DataSub = np.column_stack((ts['u'][0,:,iy,iz],ts['u'][1,:,iy,iz],ts['u'][2,:,iy,iz])) + Data = np.column_stack((Data, DataSub)) + np.savetxt(probefile, Data, header=','.join(Columns), delimiter=',') + + + + +if __name__=='__main__': + ts = TurbSimFile('../_tests/TurbSim.bts') diff --git a/pydatview/io/turbsim_ts_file.py b/pydatview/io/turbsim_ts_file.py new file mode 100644 index 0000000..6d573a7 --- /dev/null +++ b/pydatview/io/turbsim_ts_file.py @@ -0,0 +1,101 @@ +from __future__ import division +from __future__ import print_function +from __future__ import absolute_import +from io import open +from .file import File, isBinary, WrongFormatError, BrokenFormatError +import pandas as pd +import numpy as np +from itertools import takewhile + + +class TurbSimTSFile(File): + + @staticmethod + def defaultExtensions(): + return ['.txt'] + + @staticmethod + def formatName(): + return 'TurbSim time series' + + def _read(self, *args, **kwargs): + self['header']=[] + nHeaderMax=10 + # Reading + iFirstData=-1 + with open(self.filename, 'r', errors="surrogateescape") as f: + for i, line in enumerate(f): + if i>nHeaderMax: + raise BrokenFormatError('`nComp` not found in file') + if line.lower().find('ncomp')>=0: + iFirstData=i + break + self['header'].append(line.strip()) + self['nComp'] = int(line.split()[0]) + line = f.readline().strip() + nPoints = int(line.split()[0]) + line = f.readline().strip() + self['ID'] = int(line.split()[0]) + f.readline() + f.readline() + self['Points']=np.zeros((nPoints,2)) + for i in np.arange(nPoints): + line = f.readline().strip() + self['Points'][i,:]= np.array(line.split()).astype(float) + f.readline() + f.readline() + f.readline() + lines=[] + # reading full data + self['data'] = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(float) + + def columns(self): + Comp=['u','v','w'] + return ['Time']+['Point{}{}'.format(ip+1,Comp[ic]) for ic in np.arange(self['nComp']) for ip in np.arange(len(self['Points']))] + + def units(self): + nPoints = self['Points'].shape[0] + return ['(s)'] + ['(m/s)']*nPoints*self['nComp'] + + def toString(self): + + def toStringVLD(val,lab,descr): + val='{}'.format(val) + lab='{}'.format(lab) + if len(val)<13: + val='{:13s}'.format(val) + if len(lab)<13: + lab='{:13s}'.format(lab) + return val+' '+lab+' - '+descr.strip().strip('-')+'\n' + + s='\n'.join(self['header'])+'\n' + nPoints = self['Points'].shape[0] + s+=toStringVLD(self['nComp'],'nComp' ,'Number of velocity components in the file' ) + s+=toStringVLD(nPoints ,'nPoints','Number of time series points contained in this file(-)') + s+=toStringVLD(self['ID'] ,'RefPtID','Index of the reference point (1-nPoints)') + s+='{:^16s}{:^16s} {}\n'.format('Pointyi','Pointzi','! nPoints listed in order of increasing height') + s+='{:^16s}{:^16s}\n'.format('(m)','(m)') + for row in self['Points']: + s+=''.join(['{:16.8e}'.format(v) for v in row])+'\n' + + s+='--------Time Series-------------------------------------------------------------\n' + s+=''.join(['{:^16s}'.format(c) for c in self.columns()])+'\n' + s+=''.join(['{:^16s}'.format(c) for c in self.units()])+'\n' + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['data']) + return s + + def _write(self): + with open(self.filename,'w') as f: + f.write(self.toString()) + + + + def _toDataFrame(self): + Cols = ['{}_{}'.format(c.replace(' ','_'), u.replace('(','[').replace(')',']')) for c,u in zip(self.columns(),self.units())] + dfs={} + dfs['Points'] = pd.DataFrame(data = self['Points'],columns = ['PointYi','PointZi']) + dfs['TimeSeries'] = pd.DataFrame(data = self['data'] ,columns = Cols) + + return dfs + + diff --git a/pydatview/io/user.py b/pydatview/io/user.py new file mode 100644 index 0000000..272f545 --- /dev/null +++ b/pydatview/io/user.py @@ -0,0 +1,8 @@ + +from weio import userFileClasses + +UserClasses, UserPaths, UserModules, UserModuleNames, errors = userFileClasses() +UserClassNames = [cls.__name__ for cls in UserClasses] + +for mod_name, mod, cls, cls_name in zip(UserModuleNames, UserModules, UserClasses, UserClassNames): + globals()[cls_name] = cls diff --git a/pydatview/io/vtk_file.py b/pydatview/io/vtk_file.py new file mode 100644 index 0000000..e3ed2f7 --- /dev/null +++ b/pydatview/io/vtk_file.py @@ -0,0 +1,1349 @@ +""" +Read/Write VTK files + +Part of weio library: https://github.com/ebranlard/weio + +""" +import pandas as pd +import numpy as np +import numpy +import os +from functools import reduce +import collections + +try: + from .file import File, EmptyFileError, WrongFormatError, BrokenFormatError +except ImportError: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + File=dict + +class VTKFile(File): + """ + Read/write a VTK file (.vtk). + + Main attributes for grids: + --------- + - xp_grid, yp_grid, zp_grid: vectors of points locations + - point_data_grid: dictionary containing data at the grid points + + Main attributes for mesh: + --------- + - points + - point_data + - cells + - cell_data + + Main methods + ------------ + - read, write + + Examples + -------- + vtk = VTKFile('DisXZ1.vtk') + x = vtk.x_grid + z = vtk.z_grid + Ux = vtk.point_data_grid['DisXZ'][:,0,:,0] + + """ + @staticmethod + def defaultExtensions(): + return ['.vtk','.vtp'] + + @staticmethod + def formatName(): + return 'VTK file' + + def __init__(self,filename=None,**kwargs): + self.filename = None + # For regular grid + self.xp_grid=None # location of points + self.yp_grid=None + self.zp_grid=None + self.point_data_grid = None + + # Main Data + self.points = None + self.field_data = {} + self.point_data = {} + self.dataset = {} + + + + # Data for reading only + self.cell_data_raw = {} + self.c = None # Cell + self.ct = None # CellTypes + self.active = None + self.is_ascii = False + self.split = [] + self.num_items = 0 + self.section = None + + # Propagate read + if filename: + self.read(filename=filename,**kwargs) + + + def read(self, filename=None): + """ read a VTK file """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + + with open(filename, "rb") as f: + # initialize output data + # skip header and title + f.readline() + f.readline() + + data_type = f.readline().decode("utf-8").strip().upper() + if data_type not in ["ASCII", "BINARY"]: + raise ReadError('Unknown VTK data type ',data_type) + self.is_ascii = data_type == "ASCII" + + while True: + line = f.readline().decode("utf-8") + if not line: + # EOF + break + + line = line.strip() + if len(line) == 0: + continue + + self.split = line.split() + self.section = self.split[0].upper() + + if self.section in vtk_sections: + _read_section(f, self) + else: + _read_subsection(f, self) + + # --- Postpro + _check_mesh(self) # generate points if needed + cells, cell_data = translate_cells(self.c, self.ct, self.cell_data_raw) + self.cells = cells + self.cell_data = cell_data + + if self.dataset['type']=='STRUCTURED_POINTS': + self.point_data_grid = {} + # We provide point_data_grid, corresponds to point_data but reshaped + for k,PD in self.point_data.items(): + # NOTE: tested foe len(y)=1, len(z)=1 + self.point_data_grid[k]=PD.reshape(len(self.xp_grid), len(self.yp_grid), len(self.zp_grid),PD.shape[1], order='F') + + + def write(self, filename=None, binary=True): + """ + Write to unstructured grid + TODO structured grid + """ + + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + + + def pad(array): + return np.pad(array, ((0, 0), (0, 1)), "constant") + + if self.points.shape[1] == 2: + points = pad(self.points) + else: + points = self.points + + if self.point_data: + for name, values in self.point_data.items(): + if len(values.shape) == 2 and values.shape[1] == 2: + self.point_data[name] = pad(values) + + for name, data in self.cell_data.items(): + for k, values in enumerate(data): + if len(values.shape) == 2 and values.shape[1] == 2: + data[k] = pad(data[k]) + + with open(filename, "wb") as f: + f.write(b"# vtk DataFile Version 4.2\n") + f.write("written \n".encode("utf-8")) + f.write(("BINARY\n" if binary else "ASCII\n").encode("utf-8")) + f.write(b"DATASET UNSTRUCTURED_GRID\n") + + # write points and cells + _write_points(f, points, binary) + _write_cells(f, self.cells, binary) + + # write point data + if self.point_data: + num_points = self.points.shape[0] + f.write("POINT_DATA {}\n".format(num_points).encode("utf-8")) + _write_field_data(f, self.point_data, binary) + + # write cell data + if self.cell_data: + total_num_cells = sum(len(c.data) for c in self.cells) + f.write("CELL_DATA {}\n".format(total_num_cells).encode("utf-8")) + _write_field_data(f, self.cell_data, binary) + + + def __repr__(self): + s='<{} object> with keys:\n'.format(type(self).__name__) + for k,v in self.items(): + s+=' - {}: {}\n'.format(k,v) + return s + + + def __repr__(self): + """ print function """ + def show_grid(v,s): + if v is None: + return + if len(v)==0: + return + if len(v)==1: + lines.append('- {}: [{}], n: {}'.format(s,v[0],len(v))) + else: + lines.append('- {}: [{} ... {}], dx: {}, n: {}'.format(s,v[0],v[-1],v[1]-v[0],len(v))) + + lines = ['<{} object> with attributes:'.format(type(self).__name__)] + show_grid(self.xp_grid, 'xp_grid') + show_grid(self.yp_grid, 'yp_grid') + show_grid(self.zp_grid, 'zp_grid') + + if self.point_data_grid: + lines.append('- point_data_grid:') + for k,v in self.point_data_grid.items(): + lines.append(' "{}" : {}'.format(k,v.shape)) + + lines.append('- points {}'.format(len(self.points))) + if len(self.cells) > 0: + lines.append("- cells:") + for tpe, elems in self.cells: + lines.append(" {}: {}".format(tpe,len(elems))) + else: + lines.append(" No cells.") + + if self.point_data: + lines.append('- point_data:') + for k,v in self.point_data.items(): + lines.append(' "{}" : {}'.format(k,v.shape)) + + if self.cell_data: + names = ", ".join(self.cell_data.keys()) + lines.append(" Cell data: {}".format(names)) + + return "\n".join(lines) + + + def toDataFrame(self): + return None + + + +# Save FlowData Object to vtk +# """ +# n_points = self.dimensions.x1 * self.dimensions.x2 * self.dimensions.x3 +# vtk_file = Output(filename) + #self.file = open(self.filename, "w") + #self.ln = "\n" +# vtk_file.write_line('# vtk DataFile Version 3.0') +# vtk_file.write_line('array.mean0D') +# vtk_file.write_line('ASCII') +# vtk_file.write_line('DATASET STRUCTURED_POINTS') +# vtk_file.write_line('DIMENSIONS {}'.format(self.dimensions)) +# vtk_file.write_line('ORIGIN {}'.format(self.origin)) +# vtk_file.write_line('SPACING {}'.format(self.spacing)) +# vtk_file.write_line('POINT_DATA {}'.format(n_points)) +# vtk_file.write_line('FIELD attributes 1') +# vtk_file.write_line('UAvg 3 {} float'.format(n_points)) +# for u, v, w in zip(self.u, self.v, self.w): +# vtk_file.write_line('{}'.format(Vec3(u, v, w))) + +# --- Paraview +# except: from paraview.simple import * +# sliceFile = sliceDir + '/' + tStartStr + '/' + sliceName +# print ' Slice file 1a: ' + sliceFile +# slice_1a_vtk = LegacyVTKReader( FileNames=[sliceFile] ) +# sliceFile = sliceDir + '/' + tEndStr + '/' + sliceName +# DataRepresentation3 = GetDisplayProperties(slice_1a_vtk) +# DataRepresentation3.Visibility = 0 +# SetActiveSource(slice_1a_vtk) + +# --- VTK +# import np +# from vtk import vtkStructuredPointsReader +# from vtk.util import np as VN +# +# reader = vtkStructuredPointsReader() +# reader.SetFileName(filename) +# reader.ReadAllVectorsOn() +# reader.ReadAllScalarsOn() +# reader.Update() +# +# data = reader.GetOutput() +# +# dim = data.GetDimensions() +# vec = list(dim) +# vec = [i-1 for i in dim] +# vec.append(3) +# +# u = VN.vtk_to_np(data.GetCellData().GetArray('velocity')) +# b = VN.vtk_to_numpy(data.GetCellData().GetArray('cell_centered_B')) +# +# u = u.reshape(vec,order='F') +# b = b.reshape(vec,order='F') +# +# x = zeros(data.GetNumberOfPoints()) +# y = zeros(data.GetNumberOfPoints()) +# z = zeros(data.GetNumberOfPoints()) +# +# for i in range(data.GetNumberOfPoints()): +# x[i],y[i],z[i] = data.GetPoint(i) +# +# x = x.reshape(dim,order='F') +# y = y.reshape(dim,order='F') +# z = z.reshape(dim,order='F') + +# --- vtk +# import vtk +# import numpy +# import vtk_demo.version as version +# +# +# def main(): +# """ +# :return: The render window interactor. +# """ +# +# chessboard_resolution = 5 +# n_lut_colors = 256 +# data_max_value = 1 +# +# # Provide some geometry +# chessboard = vtk.vtkPlaneSource() +# chessboard.SetXResolution(chessboard_resolution) +# chessboard.SetYResolution(chessboard_resolution) +# num_squares = chessboard_resolution * chessboard_resolution +# # Force an update so we can set cell data +# chessboard.Update() +# +# # Make some arbitrary data to show on the chessboard geometry +# data = vtk.vtkFloatArray() +# for i in range(num_squares): +# if i == 4: +# # This square should in principle light up with color given by SetNanColor below +# data.InsertNextTuple1(numpy.nan) +# else: +# thing = (i * data_max_value) / (num_squares - 1) +# data.InsertNextTuple1(thing) +# +# # Make a LookupTable +# lut = vtk.vtkLookupTable() +# lut.SetNumberOfColors(n_lut_colors) +# lut.Build() +# lut.SetTableRange(0, data_max_value) +# lut.SetNanColor(.1, .5, .99, 1.0) # <------ This color gets used +# for i in range(n_lut_colors): +# # Fill it with arbitrary colors, e.g. grayscale +# x = data_max_value*i/(n_lut_colors-1) +# lut.SetTableValue(i, x, x, x, 1.0) +# lut.SetNanColor(.99, .99, .1, 1.0) # <----- This color gets ignored! ...except by GetNanColor +# +# print(lut.GetNanColor()) # <-- Prints the color set by the last SetNanColor call above! +# +# chessboard.GetOutput().GetCellData().SetScalars(data) +# +# mapper = vtk.vtkPolyDataMapper() +# mapper.SetInputConnection(chessboard.GetOutputPort()) +# mapper.SetScalarRange(0, data_max_value) +# mapper.SetLookupTable(lut) +# mapper.Update() +# +# actor = vtk.vtkActor() +# actor.SetMapper(mapper) +# +# renderer = vtk.vtkRenderer() +# ren_win = vtk.vtkRenderWindow() +# ren_win.AddRenderer(renderer) +# renderer.SetBackground(vtk.vtkNamedColors().GetColor3d('MidnightBlue')) +# renderer.AddActor(actor) +# +# iren = vtk.vtkRenderWindowInteractor() + +# --- pyvtk +# """Read vtk-file stored previously with tovtk.""" +# p = pyvtk.VtkData(filename) +# xn = array(p.structure.points) +# dims = p.structure.dimensions +# try: +# N = eval(p.header.split(" ")[-1]) + + # --- Extract + # # Convert the center of the turbine coordinate in m to High-Resolution domains left most corner in (i,j) + # xe_index = int(origin_at_precusr[0]/10) + # ye_index = int(origin_at_precusr[1]/10) + # + # # Read the full domain from VTK + # reader = vtk.vtkStructuredPointsReader() + # reader.SetFileName(in_vtk) + # reader.Update() + # + # # Extract the High Resolution domain at same spacial spacing by specifying the (i,i+14),(j,j+14),(k,k+20) tuples + # extract = vtk.vtkExtractVOI() + # extract.SetInputConnection(reader.GetOutputPort()) + # extract.SetVOI(xe_index, xe_index+14, ye_index, ye_index+14, 0, 26) + # extract.SetSampleRate(1, 1, 1) + # extract.Update() + # + # # Write the extract as VTK + # points = extract.GetOutput() + # vec = points.GetPointData().GetVectors('Amb') + # + # with open(out_vtk, 'a') as the_file: + # the_file.write('# vtk DataFile Version 3.0\n') + # the_file.write('High\n') + # the_file.write('ASCII\n') + # the_file.write('DATASET STRUCTURED_POINTS\n') + # the_file.write('DIMENSIONS %d %d %d\n' % points.GetDimensions()) + # the_file.write('ORIGIN %f %f %f\n' % origin_at_stitch) + # the_file.write('SPACING %f %f %f\n' % points.GetSpacing()) + # the_file.write('POINT_DATA %d\n' % points.GetNumberOfPoints()) + # the_file.write('VECTORS Amb float\n') + # for i in range(points.GetNumberOfPoints()): + # the_file.write('%f %f %f\n' % vec.GetTuple(i) ) + + # --- Stitch + # reader = vtk.vtkStructuredPointsReader() + # reader.SetFileName(in_vtk) + # reader.Update() + # + # hAppend = vtk.vtkImageAppend() + # hAppend.SetAppendAxis(0) + # for i in range(nx): + # hAppend.AddInputData(reader.GetOutput()) + # hAppend.Update() + # + # vAppend = vtk.vtkImageAppend() + # vAppend.SetAppendAxis(1) + # for i in range(ny): + # vAppend.AddInputData(hAppend.GetOutput()) + # vAppend.Update() + # + # points = vAppend.GetOutput() + # vec = points.GetPointData().GetVectors('Amb') + # + # with open(out_vtk, 'a') as the_file: + # the_file.write('# vtk DataFile Version 3.0\n') + # the_file.write('Low\n') + # the_file.write('ASCII\n') + # the_file.write('DATASET STRUCTURED_POINTS\n') + # the_file.write('DIMENSIONS %d %d %d\n' % points.GetDimensions()) + # the_file.write('ORIGIN %f %f %f\n' % points.GetOrigin()) + # the_file.write('SPACING %f %f %f\n' % points.GetSpacing()) + # the_file.write('POINT_DATA %d\n' % points.GetNumberOfPoints()) + # the_file.write('VECTORS Amb float\n') + # for i in range(points.GetNumberOfPoints()): + # the_file.write('%f %f %f\n' % vec.GetTuple(i) ) + + + # + + +# --------------------------------------------------------------------------------} +# --- The code below is taken from meshio +# https://github.com/nschloe/meshio +# The MIT License (MIT) +# Copyright (c) 2015-2020 meshio developers +# --------------------------------------------------------------------------------{ +ReadError = BrokenFormatError +WriteError = BrokenFormatError + +def _vtk_to_meshio_order(vtk_type, numnodes, dtype=int): + # meshio uses the same node ordering as VTK for most cell types. However, for the + # linear wedge, the ordering of the gmsh Prism [1] is adopted since this is found in + # most codes (Abaqus, Ansys, Nastran,...). In the vtkWedge [2], the normal of the + # (0,1,2) triangle points outwards, while in gmsh this normal points inwards. + # [1] http://gmsh.info/doc/texinfo/gmsh.html#Node-ordering + # [2] https://vtk.org/doc/nightly/html/classvtkWedge.html + if vtk_type == 13: + return numpy.array([0, 2, 1, 3, 5, 4], dtype=dtype) + else: + return numpy.arange(0, numnodes, dtype=dtype) + +def _meshio_to_vtk_order(meshio_type, numnodes, dtype=int): + if meshio_type == "wedge": + return numpy.array([0, 2, 1, 3, 5, 4], dtype=dtype) + else: + return numpy.arange(0, numnodes, dtype=dtype) + +vtk_to_meshio_type = { + 0: "empty", + 1: "vertex", + # 2: 'poly_vertex', + 3: "line", + # 4: 'poly_line', + 5: "triangle", + # 6: 'triangle_strip', + 7: "polygon", + 8: "pixel", + 9: "quad", + 10: "tetra", + # 11: 'voxel', + 12: "hexahedron", + 13: "wedge", + 14: "pyramid", + 15: "penta_prism", + 16: "hexa_prism", + 21: "line3", + 22: "triangle6", + 23: "quad8", + 24: "tetra10", + 25: "hexahedron20", + 26: "wedge15", + 27: "pyramid13", + 28: "quad9", + 29: "hexahedron27", + 30: "quad6", + 31: "wedge12", + 32: "wedge18", + 33: "hexahedron24", + 34: "triangle7", + 35: "line4", + 42: "polyhedron", + # + # 60: VTK_HIGHER_ORDER_EDGE, + # 61: VTK_HIGHER_ORDER_TRIANGLE, + # 62: VTK_HIGHER_ORDER_QUAD, + # 63: VTK_HIGHER_ORDER_POLYGON, + # 64: VTK_HIGHER_ORDER_TETRAHEDRON, + # 65: VTK_HIGHER_ORDER_WEDGE, + # 66: VTK_HIGHER_ORDER_PYRAMID, + # 67: VTK_HIGHER_ORDER_HEXAHEDRON, + # Arbitrary order Lagrange elements + 68: "VTK_LAGRANGE_CURVE", + 69: "VTK_LAGRANGE_TRIANGLE", + 70: "VTK_LAGRANGE_QUADRILATERAL", + 71: "VTK_LAGRANGE_TETRAHEDRON", + 72: "VTK_LAGRANGE_HEXAHEDRON", + 73: "VTK_LAGRANGE_WEDGE", + 74: "VTK_LAGRANGE_PYRAMID", + # Arbitrary order Bezier elements + 75: "VTK_BEZIER_CURVE", + 76: "VTK_BEZIER_TRIANGLE", + 77: "VTK_BEZIER_QUADRILATERAL", + 78: "VTK_BEZIER_TETRAHEDRON", + 79: "VTK_BEZIER_HEXAHEDRON", + 80: "VTK_BEZIER_WEDGE", + 81: "VTK_BEZIER_PYRAMID", +} +meshio_to_vtk_type = {v: k for k, v in vtk_to_meshio_type.items()} + + +# --------------------------------------------------------------------------------} +# --- Mesh +# --------------------------------------------------------------------------------{ +class CellBlock(collections.namedtuple("CellBlock", ["type", "data"])): + def __repr__(self): + return "".format(self.type,len(self.data)) +# --------------------------------------------------------------------------------} +# --- File _vtk.py from meshio +# --------------------------------------------------------------------------------{ +vtk_type_to_numnodes = numpy.array( + [ + 0, # empty + 1, # vertex + -1, # poly_vertex + 2, # line + -1, # poly_line + 3, # triangle + -1, # triangle_strip + -1, # polygon + -1, # pixel + 4, # quad + 4, # tetra + -1, # voxel + 8, # hexahedron + 6, # wedge + 5, # pyramid + 10, # penta_prism + 12, # hexa_prism + -1, + -1, + -1, + -1, + 3, # line3 + 6, # triangle6 + 8, # quad8 + 10, # tetra10 + 20, # hexahedron20 + 15, # wedge15 + 13, # pyramid13 + 9, # quad9 + 27, # hexahedron27 + 6, # quad6 + 12, # wedge12 + 18, # wedge18 + 24, # hexahedron24 + 7, # triangle7 + 4, # line4 + ] +) + + +# These are all VTK data types. +# One sometimes finds 'vtktypeint64', but this is ill-formed. +vtk_to_numpy_dtype_name = { + "bit": "bool", + "unsigned_char": "uint8", + "char": "int8", + "unsigned_short": "uint16", + "short": "int16", + "unsigned_int": "uint32", + "int": "int32", + "unsigned_long": "uint64", + "long": "int64", + "float": "float32", + "double": "float64", + "vtktypeint32": "int32", # vtk DataFile Version 5.1 + "vtktypeint64": "int64", # vtk DataFile Version 5.1 + "vtkidtype": "int32", # may be either 32-bit or 64-bit (VTK_USE_64BIT_IDS) +} + +numpy_to_vtk_dtype = { + v: k for k, v in vtk_to_numpy_dtype_name.items() if "vtk" not in k +} + +# supported vtk dataset types +vtk_dataset_types = [ + "UNSTRUCTURED_GRID", + "STRUCTURED_POINTS", + "STRUCTURED_GRID", + "RECTILINEAR_GRID", +] +# additional infos per dataset type +vtk_dataset_infos = { + "UNSTRUCTURED_GRID": [], + "STRUCTURED_POINTS": [ + "DIMENSIONS", + "ORIGIN", + "SPACING", + "ASPECT_RATIO", # alternative for SPACING in version 1.0 and 2.0 + ], + "STRUCTURED_GRID": ["DIMENSIONS"], + "RECTILINEAR_GRID": [ + "DIMENSIONS", + "X_COORDINATES", + "Y_COORDINATES", + "Z_COORDINATES", + ], +} + +# all main sections in vtk +vtk_sections = [ + "METADATA", + "DATASET", + "POINTS", + "CELLS", + "CELL_TYPES", + "POINT_DATA", + "CELL_DATA", + "LOOKUP_TABLE", + "COLOR_SCALARS", +] + + + + + +def read(filename): + """Reads a VTK vtk file.""" + with open(filename, "rb") as f: + out = read_buffer(f) + return out + + +def read_buffer(f): + # initialize output data + info = VTKFile() + + # skip header and title + f.readline() + f.readline() + + data_type = f.readline().decode("utf-8").strip().upper() + if data_type not in ["ASCII", "BINARY"]: + raise WrongFormatError("Unknown VTK data type ",data_type) + info.is_ascii = data_type == "ASCII" + + while True: + line = f.readline().decode("utf-8") + if not line: + # EOF + break + + line = line.strip() + if len(line) == 0: + continue + + info.split = line.split() + info.section = info.split[0].upper() + + if info.section in vtk_sections: + _read_section(f, info) + else: + _read_subsection(f, info) + + _check_mesh(info) + cells, cell_data = translate_cells(info.c, info.ct, info.cell_data_raw) + + info.cells = cells + info.cell_data = cell_data + + return info + + +def _read_section(f, info): + if info.section == "METADATA": + _skip_meta(f) + + elif info.section == "DATASET": + info.active = "DATASET" + info.dataset["type"] = info.split[1].upper() + if info.dataset["type"] not in vtk_dataset_types: + raise BrokenFormatError( + "Only VTK '{}' supported (not {}).".format( + "', '".join(vtk_dataset_types), info.dataset["type"] + ) + ) + + elif info.section == "POINTS": + info.active = "POINTS" + info.num_points = int(info.split[1]) + data_type = info.split[2].lower() + info.points = _read_points(f, data_type, info.is_ascii, info.num_points) + + elif info.section == "CELLS": + info.active = "CELLS" + last_pos = f.tell() + try: + line = f.readline().decode("utf-8") + except UnicodeDecodeError: + line = "" + if "OFFSETS" in line: + # vtk DataFile Version 5.1 - appearing in Paraview 5.8.1 outputs + # No specification found for this file format. + # See the question on ParaView Discourse Forum: + # . + info.num_offsets = int(info.split[1]) + info.num_items = int(info.split[2]) + dtype = numpy.dtype(vtk_to_numpy_dtype_name[line.split()[1]]) + offsets = _read_cells(f, info.is_ascii, info.num_offsets, dtype) + line = f.readline().decode("utf-8") + assert "CONNECTIVITY" in line + dtype = numpy.dtype(vtk_to_numpy_dtype_name[line.split()[1]]) + connectivity = _read_cells(f, info.is_ascii, info.num_items, dtype) + info.c = (offsets, connectivity) + else: + f.seek(last_pos) + info.num_items = int(info.split[2]) + info.c = _read_cells(f, info.is_ascii, info.num_items) + + elif info.section == "CELL_TYPES": + info.active = "CELL_TYPES" + info.num_items = int(info.split[1]) + info.ct = _read_cell_types(f, info.is_ascii, info.num_items) + + elif info.section == "POINT_DATA": + info.active = "POINT_DATA" + info.num_items = int(info.split[1]) + + elif info.section == "CELL_DATA": + info.active = "CELL_DATA" + info.num_items = int(info.split[1]) + + elif info.section == "LOOKUP_TABLE": + info.num_items = int(info.split[2]) + numpy.fromfile(f, count=info.num_items * 4, sep=" ", dtype=float) + # rgba = data.reshape((info.num_items, 4)) + + elif info.section == "COLOR_SCALARS": + nValues = int(info.split[2]) + # re-use num_items from active POINT/CELL_DATA + num_items = info.num_items + dtype = numpy.ubyte + if info.is_ascii: + dtype = float + numpy.fromfile(f, count=num_items * nValues, dtype=dtype) + + +def _read_subsection(f, info): + if info.active == "POINT_DATA": + d = info.point_data + elif info.active == "CELL_DATA": + d = info.cell_data_raw + elif info.active == "DATASET": + d = info.dataset + else: + d = info.field_data + + if info.section in vtk_dataset_infos[info.dataset["type"]]: + if info.section[1:] == "_COORDINATES": + info.num_points = int(info.split[1]) + data_type = info.split[2].lower() + d[info.section] = _read_coords(f, data_type, info.is_ascii, info.num_points) + else: + if info.section == "DIMENSIONS": + d[info.section] = list(map(int, info.split[1:])) + else: + d[info.section] = list(map(float, info.split[1:])) + if len(d[info.section]) != 3: + raise BrokenFormatError( + "Wrong number of info in section '{}'. Need 3, got {}.".format( + info.section, len(d[info.section]) + ) + ) + elif info.section == "SCALARS": + d.update(_read_scalar_field(f, info.num_items, info.split, info.is_ascii)) + elif info.section == "VECTORS": + d.update(_read_field(f, info.num_items, info.split, [3], info.is_ascii)) + elif info.section == "TENSORS": + d.update(_read_field(f, info.num_items, info.split, [3, 3], info.is_ascii)) + elif info.section == "FIELD": + d.update(_read_fields(f, int(info.split[2]), info.is_ascii)) + else: + raise WrongFormatError("Unknown section ",info.section) + + +def _check_mesh(info): + if info.dataset["type"] == "UNSTRUCTURED_GRID": + if info.c is None: + raise ReadError("Required section CELLS not found.") + if info.ct is None: + raise ReadError("Required section CELL_TYPES not found.") + + elif info.dataset["type"] == "STRUCTURED_POINTS": + dim = info.dataset["DIMENSIONS"] + ori = info.dataset["ORIGIN"] + spa = ( + info.dataset["SPACING"] + if "SPACING" in info.dataset + else info.dataset["ASPECT_RATIO"] + ) + axis = [ + numpy.linspace(ori[i], ori[i] + (dim[i] - 1.0) * spa[i], dim[i]) + for i in range(3) + ] + info.xp_grid=axis[0] + info.yp_grid=axis[1] + info.zp_grid=axis[2] + + info.points = _generate_points(axis) + info.c, info.ct = _generate_cells(dim=info.dataset["DIMENSIONS"]) + + elif info.dataset["type"] == "RECTILINEAR_GRID": + axis = [ + info.dataset["X_COORDINATES"], + info.dataset["Y_COORDINATES"], + info.dataset["Z_COORDINATES"], + ] + info.xp_grid=axis[0] + info.yp_grid=axis[1] + info.zp_grid=axis[2] + + info.points = _generate_points(axis) + info.c, info.ct = _generate_cells(dim=info.dataset["DIMENSIONS"]) + + elif info.dataset["type"] == "STRUCTURED_GRID": + info.c, info.ct = _generate_cells(dim=info.dataset["DIMENSIONS"]) + # TODO x_grid, y_grid, z_grid points + + + +def _generate_cells(dim): + ele_dim = [d - 1 for d in dim if d > 1] + ele_no = numpy.prod(ele_dim, dtype=int) + spatial_dim = len(ele_dim) + + if spatial_dim == 1: + # cells are lines in 1D + cells = numpy.empty((ele_no, 3), dtype=int) + cells[:, 0] = 2 + cells[:, 1] = numpy.arange(ele_no, dtype=int) + cells[:, 2] = cells[:, 1] + 1 + cell_types = numpy.full(ele_no, 3, dtype=int) + + elif spatial_dim == 2: + # cells are quad in 2D + cells = numpy.empty((ele_no, 5), dtype=int) + cells[:, 0] = 4 + cells[:, 1] = numpy.arange(0, ele_no, dtype=int) + cells[:, 1] += numpy.arange(0, ele_no, dtype=int) // ele_dim[0] + cells[:, 2] = cells[:, 1] + 1 + cells[:, 3] = cells[:, 1] + 2 + ele_dim[0] + cells[:, 4] = cells[:, 3] - 1 + cell_types = numpy.full(ele_no, 9, dtype=int) + else: + # cells are hex in 3D + cells = numpy.empty((ele_no, 9), dtype=int) + cells[:, 0] = 8 + cells[:, 1] = numpy.arange(ele_no) + cells[:, 1] += (ele_dim[0] + ele_dim[1] + 1) * ( + numpy.arange(ele_no) // (ele_dim[0] * ele_dim[1]) + ) + cells[:, 1] += (numpy.arange(ele_no) % (ele_dim[0] * ele_dim[1])) // ele_dim[0] + cells[:, 2] = cells[:, 1] + 1 + cells[:, 3] = cells[:, 1] + 2 + ele_dim[0] + cells[:, 4] = cells[:, 3] - 1 + cells[:, 5] = cells[:, 1] + (1 + ele_dim[0]) * (1 + ele_dim[1]) + cells[:, 6] = cells[:, 5] + 1 + cells[:, 7] = cells[:, 5] + 2 + ele_dim[0] + cells[:, 8] = cells[:, 7] - 1 + cell_types = numpy.full(ele_no, 12, dtype=int) + + return cells.reshape(-1), cell_types + +def _generate_points(axis): + x_dim = len(axis[0]) + y_dim = len(axis[1]) + z_dim = len(axis[2]) + pnt_no = x_dim * y_dim * z_dim + x_id, y_id, z_id = numpy.mgrid[0:x_dim, 0:y_dim, 0:z_dim] + points = numpy.empty((pnt_no, 3), dtype=axis[0].dtype) + # VTK sorts points and cells in Fortran order + points[:, 0] = axis[0][x_id.reshape(-1, order="F")] + points[:, 1] = axis[1][y_id.reshape(-1, order="F")] + points[:, 2] = axis[2][z_id.reshape(-1, order="F")] + return points + + +def _read_coords(f, data_type, is_ascii, num_points): + dtype = numpy.dtype(vtk_to_numpy_dtype_name[data_type]) + if is_ascii: + coords = numpy.fromfile(f, count=num_points, sep=" ", dtype=dtype) + else: + # Binary data is big endian, see + # . + dtype = dtype.newbyteorder(">") + coords = numpy.fromfile(f, count=num_points, dtype=dtype) + line = f.readline().decode("utf-8") + if line != "\n": + raise ReadError() + return coords + + +def _read_points(f, data_type, is_ascii, num_points): + dtype = numpy.dtype(vtk_to_numpy_dtype_name[data_type]) + if is_ascii: + points = numpy.fromfile(f, count=num_points * 3, sep=" ", dtype=dtype) + else: + # Binary data is big endian, see + # . + dtype = dtype.newbyteorder(">") + points = numpy.fromfile(f, count=num_points * 3, dtype=dtype) + line = f.readline().decode("utf-8") + if line != "\n": + raise ReadError() + return points.reshape((num_points, 3)) + + +def _read_cells(f, is_ascii, num_items, dtype=numpy.dtype("int32")): + if is_ascii: + c = numpy.fromfile(f, count=num_items, sep=" ", dtype=dtype) + else: + dtype = dtype.newbyteorder(">") + c = numpy.fromfile(f, count=num_items, dtype=dtype) + line = f.readline().decode("utf-8") + if line != "\n": + raise ReadError() + return c + + +def _read_cell_types(f, is_ascii, num_items): + if is_ascii: + ct = numpy.fromfile(f, count=int(num_items), sep=" ", dtype=int) + else: + # binary + ct = numpy.fromfile(f, count=int(num_items), dtype=">i4") + line = f.readline().decode("utf-8") + # Sometimes, there's no newline at the end + if line.strip() != "": + raise ReadError() + return ct + + +def _read_scalar_field(f, num_data, split, is_ascii): + data_name = split[1] + data_type = split[2].lower() + try: + num_comp = int(split[3]) + except IndexError: + num_comp = 1 + + # The standard says: + # > The parameter numComp must range between (1,4) inclusive; [...] + if not (0 < num_comp < 5): + raise ReadError("The parameter numComp must range between (1,4) inclusive") + + dtype = numpy.dtype(vtk_to_numpy_dtype_name[data_type]) + lt, _ = f.readline().decode("utf-8").split() + if lt.upper() != "LOOKUP_TABLE": + raise ReadError() + + if is_ascii: + data = numpy.fromfile(f, count=num_data * num_comp, sep=" ", dtype=dtype) + else: + # Binary data is big endian, see + # . + dtype = dtype.newbyteorder(">") + data = numpy.fromfile(f, count=num_data * num_comp, dtype=dtype) + line = f.readline().decode("utf-8") + if line != "\n": + raise ReadError() + + data = data.reshape(-1, num_comp) + return {data_name: data} + + +def _read_field(f, num_data, split, shape, is_ascii): + data_name = split[1] + data_type = split[2].lower() + + dtype = numpy.dtype(vtk_to_numpy_dtype_name[data_type]) + # prod() + # + k = reduce((lambda x, y: x * y), shape) + + if is_ascii: + data = numpy.fromfile(f, count=k * num_data, sep=" ", dtype=dtype) + else: + # Binary data is big endian, see + # . + dtype = dtype.newbyteorder(">") + data = numpy.fromfile(f, count=k * num_data, dtype=dtype) + line = f.readline().decode("utf-8") + if line != "\n": + raise ReadError() + + data = data.reshape(-1, *shape) + return {data_name: data} + + +def _read_fields(f, num_fields, is_ascii): + data = {} + for _ in range(num_fields): + line = f.readline().decode("utf-8").split() + if line[0] == "METADATA": + _skip_meta(f) + name, shape0, shape1, data_type = f.readline().decode("utf-8").split() + else: + name, shape0, shape1, data_type = line + + shape0 = int(shape0) + shape1 = int(shape1) + dtype = numpy.dtype(vtk_to_numpy_dtype_name[data_type.lower()]) + + if is_ascii: + dat = numpy.fromfile(f, count=shape0 * shape1, sep=" ", dtype=dtype) + else: + # Binary data is big endian, see + # . + dtype = dtype.newbyteorder(">") + dat = numpy.fromfile(f, count=shape0 * shape1, dtype=dtype) + line = f.readline().decode("utf-8") + if line != "\n": + raise ReadError() + + if shape0 != 1: + dat = dat.reshape((shape1, shape0)) + + data[name] = dat + + return data + + +def _skip_meta(f): + # skip possible metadata + # https://vtk.org/doc/nightly/html/IOLegacyInformationFormat.html + while True: + line = f.readline().decode("utf-8").strip() + if not line: + # end of metadata is a blank line + break + + +def translate_cells(data, types, cell_data_raw): + # https://www.vtk.org/doc/nightly/html/vtkCellType_8h_source.html + # Translate it into the cells array. + # `data` is a one-dimensional vector with + # (num_points0, p0, p1, ... ,pk, numpoints1, p10, p11, ..., p1k, ... + # or a tuple with (offsets, connectivity) + has_polygon = numpy.any(types == meshio_to_vtk_type["polygon"]) + + cells = [] + cell_data = {} + if has_polygon: + numnodes = numpy.empty(len(types), dtype=int) + # If some polygons are in the VTK file, loop over the cells + numcells = len(types) + offsets = numpy.empty(len(types), dtype=int) + offsets[0] = 0 + for idx in range(numcells - 1): + numnodes[idx] = data[offsets[idx]] + offsets[idx + 1] = offsets[idx] + numnodes[idx] + 1 + + idx = numcells - 1 + numnodes[idx] = data[offsets[idx]] + if not numpy.all(numnodes == data[offsets]): + raise ReadError() + + # TODO: cell_data + for idx, vtk_cell_type in enumerate(types): + start = offsets[idx] + 1 + cell_idx = start + _vtk_to_meshio_order( + vtk_cell_type, numnodes[idx], offsets.dtype + ) + cell = data[cell_idx] + + cell_type = vtk_to_meshio_type[vtk_cell_type] + if cell_type == "polygon": + cell_type += str(data[offsets[idx]]) + + if len(cells) > 0 and cells[-1].type == cell_type: + cells[-1].data.append(cell) + else: + cells.append(CellBlock(cell_type, [cell])) + + # convert data to numpy arrays + for k, c in enumerate(cells): + cells[k] = CellBlock(c.type, numpy.array(c.data)) + else: + # Deduct offsets from the cell types. This is much faster than manually going + # through the data array. Slight disadvantage: This doesn't work for cells with + # a custom number of points. + numnodes = vtk_type_to_numnodes[types] + if not numpy.all(numnodes > 0): + raise ReadError("File contains cells that meshio cannot handle.") + if isinstance(data, tuple): + offsets, conn = data + if not numpy.all(numnodes == numpy.diff(offsets)): + raise ReadError() + idx0 = 0 + else: + offsets = numpy.cumsum(numnodes + 1) - (numnodes + 1) + + if not numpy.all(numnodes == data[offsets]): + raise ReadError() + idx0 = 1 + conn = data + + b = numpy.concatenate( + [[0], numpy.where(types[:-1] != types[1:])[0] + 1, [len(types)]] + ) + for start, end in zip(b[:-1], b[1:]): + meshio_type = vtk_to_meshio_type[types[start]] + n = numnodes[start] + cell_idx = idx0 + _vtk_to_meshio_order(types[start], n, dtype=offsets.dtype) + indices = numpy.add.outer(offsets[start:end], cell_idx) + cells.append(CellBlock(meshio_type, conn[indices])) + for name, d in cell_data_raw.items(): + if name not in cell_data: + cell_data[name] = [] + cell_data[name].append(d[start:end]) + + return cells, cell_data + + +def write(filename, mesh, binary=True): + def pad(array): + return numpy.pad(array, ((0, 0), (0, 1)), "constant") + + if mesh.points.shape[1] == 2: + points = pad(mesh.points) + else: + points = mesh.points + + if mesh.point_data: + for name, values in mesh.point_data.items(): + if len(values.shape) == 2 and values.shape[1] == 2: + mesh.point_data[name] = pad(values) + + for name, data in mesh.cell_data.items(): + for k, values in enumerate(data): + if len(values.shape) == 2 and values.shape[1] == 2: + data[k] = pad(data[k]) + + with open(filename, "wb") as f: + f.write(b"# vtk DataFile Version 4.2\n") + f.write("written \n".encode("utf-8")) + f.write(("BINARY\n" if binary else "ASCII\n").encode("utf-8")) + f.write(b"DATASET UNSTRUCTURED_GRID\n") + + # write points and cells + _write_points(f, points, binary) + _write_cells(f, mesh.cells, binary) + + # write point data + if mesh.point_data: + num_points = mesh.points.shape[0] + f.write("POINT_DATA {}\n".format(num_points).encode("utf-8")) + _write_field_data(f, mesh.point_data, binary) + + # write cell data + if mesh.cell_data: + total_num_cells = sum(len(c.data) for c in mesh.cells) + f.write("CELL_DATA {}\n".format(total_num_cells).encode("utf-8")) + _write_field_data(f, mesh.cell_data, binary) + + +def _write_points(f, points, binary): + f.write( + "POINTS {} {}\n".format( + len(points), numpy_to_vtk_dtype[points.dtype.name] + ).encode("utf-8") + ) + + if binary: + # Binary data must be big endian, see + # . + # if points.dtype.byteorder == "<" or ( + # points.dtype.byteorder == "=" and sys.byteorder == "little" + # ): + points.astype(points.dtype.newbyteorder(">")).tofile(f, sep="") + else: + # ascii + points.tofile(f, sep=" ") + f.write(b"\n") + + +def _write_cells(f, cells, binary): + total_num_cells = sum([len(c.data) for c in cells]) + total_num_idx = sum([c.data.size for c in cells]) + # For each cell, the number of nodes is stored + total_num_idx += total_num_cells + f.write("CELLS {} {}\n".format(total_num_cells,total_num_idx).encode("utf-8")) + if binary: + for c in cells: + n = c.data.shape[1] + cell_idx = _meshio_to_vtk_order(c.type, n) + dtype = numpy.dtype(">i4") + # One must force endianness here: + # + numpy.column_stack( + [ + numpy.full(c.data.shape[0], n, dtype=dtype), + c.data[:, cell_idx].astype(dtype), + ], + ).astype(dtype).tofile(f, sep="") + f.write(b"\n") + else: + # ascii + for c in cells: + n = c.data.shape[1] + cell_idx = _meshio_to_vtk_order(c.type, n) + # prepend a column with the value n + numpy.column_stack( + [ + numpy.full(c.data.shape[0], n, dtype=c.data.dtype), + c.data[:, cell_idx], + ] + ).tofile(f, sep="\n") + f.write(b"\n") + + # write cell types + f.write("CELL_TYPES {}\n".format(total_num_cells).encode("utf-8")) + if binary: + for c in cells: + key_ = c.type[:7] if c.type[:7] == "polygon" else c.type + vtk_type = meshio_to_vtk_type[key_] + numpy.full(len(c.data), vtk_type, dtype=numpy.dtype(">i4")).tofile( + f, sep="" + ) + f.write(b"\n") + else: + # ascii + for c in cells: + key_ = c.type[:7] if c.type[:7] == "polygon" else c.type + numpy.full(len(c.data), meshio_to_vtk_type[key_]).tofile(f, sep="\n") + f.write(b"\n") + + +def _write_field_data(f, data, binary): + f.write(("FIELD FieldData {}\n".format(len(data))).encode("utf-8")) + for name, values in data.items(): + if isinstance(values, list): + values = numpy.concatenate(values) + if len(values.shape) == 1: + num_tuples = values.shape[0] + num_components = 1 + else: + num_tuples = values.shape[0] + num_components = values.shape[1] + + if " " in name: + raise WriteError("VTK doesn't support spaces in field names", name) + + f.write( + ( + "{} {} {} {}\n".format( + name, + num_components, + num_tuples, + numpy_to_vtk_dtype[values.dtype.name], + ) + ).encode("utf-8") + ) + if binary: + values.astype(values.dtype.newbyteorder(">")).tofile(f, sep="") + else: + # ascii + values.tofile(f, sep=" ") + # numpy.savetxt(f, points) + f.write(b"\n") + + + +if __name__=='__main__': + #plane=VTKFile('tests/_TODO/FastFarm.Low.DisXY1.t1200.vtk') + #plane=VTKFile('tests/_TODO/FastFarm.Low.DisXZ1.t1200.vtk') + plane=VTKFile('tests/_TODO/FastFarm.Low.DisXY1.t0_fake.vtk') + print(plane.points) + #plane=VTKFile('tests/_TODO/Main_NM80_OF24_vc.FVW_Hub.AllSeg.000000130.vtk') + print(plane) +# print(plane.points) +# print(plane.cells) +# print(plane.cell_data_raw) +# print(plane.cell_data) +# print('x_grid',plane.x_grid) +# print('PointData',plane.point_data.keys()) +# print('PointData',plane.point_data_grid.keys()) +# print('PointData',plane.points.shape) +# print(plane.dataset) +# if len(plane.z_grid)==1: +# print('PointData',plane.point_data['DisXY'].shape) +# D=plane.point_data['DisXY'] +# print(len(plane.x_grid), len(plane.y_grid), len(plane.z_grid),D.shape[1]) +# +# DD= D.reshape(len(plane.x_grid), len(plane.y_grid), len(plane.z_grid),D.shape[1], order='F') +# print(DD.shape) +# import matplotlib.pyplot as plt +# plt.contourf(plane.x_grid, plane.y_grid, DD[:,:,0,0].T) +# plt.show() +# elif len(plane.y_grid)==1: +# +# print('PointData',plane.point_data['DisXZ'].shape) +# D=plane.point_data['DisXZ'] +# print(len(plane.x_grid), len(plane.y_grid), len(plane.z_grid),D.shape[1]) +# +# DD= D.reshape(len(plane.x_grid), len(plane.y_grid), len(plane.z_grid),D.shape[1], order='F') +# print(DD.shape) +# import matplotlib.pyplot as plt +# #plt.contourf(plane.x_grid, plane.z_grid, DD[:,0,:,1].T, antialiased=False) +# plt.pcolor(plane.x_grid, plane.z_grid, DD[:,0,:,1].T) +# plt.show() diff --git a/pydatview/io/wetb/__init__.py b/pydatview/io/wetb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydatview/io/wetb/hawc2/Hawc2io.py b/pydatview/io/wetb/hawc2/Hawc2io.py new file mode 100644 index 0000000..4bfce05 --- /dev/null +++ b/pydatview/io/wetb/hawc2/Hawc2io.py @@ -0,0 +1,351 @@ +# -*- coding: utf-8 -*- +""" +Author: + Bjarne S. Kallesoee + + +Description: + Reads all HAWC2 output data formats, HAWC2 ascii, HAWC2 binary and FLEX + +call ex.: + # creat data file object, call without extension, but with parth + file = ReadHawc2("HAWC2ex/test") + # if called with ReadOnly = 1 as + file = ReadHawc2("HAWC2ex/test",ReadOnly=1) + # no channels a stored in memory, otherwise read channels are stored for reuse + + # channels are called by a list + file([0,2,1,1]) => channels 1,3,2,2 + # if empty all channels are returned + file() => all channels as 1,2,3,... + file.t => time vector + +1. version: 19/4-2011 +2. version: 5/11-2015 fixed columns to get description right, fixed time vector (mmpe@dtu.dk) + +Need to be done: + * add error handling for allmost every thing + +""" +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import absolute_import +from builtins import int +from builtins import range +from io import open as opent +from builtins import str +from future import standard_library +standard_library.install_aliases() +from builtins import object +import numpy as np +import os + +#from wetb import gtsdf + +# FIXME: numpy doesn't like io.open binary fid in PY27, why is that? As a hack +# workaround, use opent for PY23 compatibility when handling text files, +# and default open for binary + +################################################################################ +################################################################################ +################################################################################ +# Read HAWC2 class +################################################################################ +class ReadHawc2(object): + """ + """ +################################################################################ +# read *.sel file + def _ReadSelFile(self): + """ + Some title + ========== + + Using docstrings formatted according to the reStructuredText specs + can be used for automated documentation generation with for instance + Sphinx: http://sphinx.pocoo.org/. + + Parameters + ---------- + signal : ndarray + some description + + Returns + ------- + output : int + describe variable + """ + + # read *.sel hawc2 output file for result info + if self.FileName.lower().endswith('.sel'): + self.FileName = self.FileName[:-4] + fid = opent(self.FileName + '.sel', 'r') + Lines = fid.readlines() + fid.close() + if Lines[0].lower().find('bhawc')>=0: + # --- Find line with scan info + iLine=0 + for i in np.arange(5,10): + if Lines[i].lower().find('scans')>=0: + iLine=i+1 + if iLine==0: + raise Exception('Cannot find the keyword "scans"') + temp = Lines[iLine].split() + self.NrSc = int(temp[0]) + self.NrCh = int(temp[1]) + self.Time = float(temp[2]) + self.Freq = self.NrSc / self.Time + self.t = np.linspace(0, self.Time, self.NrSc + 1)[1:] + # --- Find line with channel info + iLine=0 + for i in np.arange(5,13): + if Lines[i].lower().find('channel')>=0: + iLine=i+1 + if iLine==0: + raise Exception('Cannot find the keyword "Channel"') + + # reads channel info (name, unit and description) + Name = []; Unit = []; Description = []; + for i in range(0, self.NrCh+1): + if (i+iLine)>=len(Lines): + break + line = Lines[i + iLine].strip() + if len(line)==0: + continue + # --- removing number and unit + sp=[sp.strip() for sp in line.split() if len(sp.strip())>0] + num = sp[0] + iNum = line.find(num) + line = line[iNum+len(num)+1:] + unit = sp[-1] + iUnit = line.find(unit) + line = line[:iUnit] + # --- Splitting to find label and description + sp=[sp.strip() for sp in line.split('\t') if len(sp.strip())>0] + if len(sp)!=2: + for nSpaces in np.arange(2,15): + sp=[sp.strip() for sp in line.split(' '*nSpaces) if len(sp.strip())>0] + if len(sp)==2: + break + if len(sp)!=2: + raise Exception('Dont know how to split the input of the sel file into 4 columns') + + Unit.append(unit) + Description.append(sp[0]) + Name.append(sp[1]) + + self.ChInfo = [Name, Unit, Description] + self.FileFormat = 'BHAWC_ASCII' + else: + + # findes general result info (number of scans, number of channels, + # simulation time and file format) + temp = Lines[8].split() + self.NrSc = int(temp[0]) + self.NrCh = int(temp[1]) + self.Time = float(temp[2]) + self.Freq = self.NrSc / self.Time + self.t = np.linspace(0, self.Time, self.NrSc + 1)[1:] + Format = temp[3] + # reads channel info (name, unit and description) + Name = []; Unit = []; Description = []; + for i in range(0, self.NrCh): + temp = str(Lines[i + 12][12:43]); Name.append(temp.strip()) + temp = str(Lines[i + 12][43:54]); Unit.append(temp.strip()) + temp = str(Lines[i + 12][54:-1]); Description.append(temp.strip()) + self.ChInfo = [Name, Unit, Description] + # if binary file format, scaling factors are read + if Format.lower() == 'binary': + self.ScaleFactor = np.zeros(self.NrCh) + self.FileFormat = 'HAWC2_BINARY' + for i in range(0, self.NrCh): + self.ScaleFactor[i] = float(Lines[i + 12 + self.NrCh + 2]) + else: + self.FileFormat = 'HAWC2_ASCII' +################################################################################ +# read sensor file for FLEX format + def _ReadSensorFile(self): + # read sensor file used if results are saved in FLEX format + DirName = os.path.dirname(self.FileName) + try: + fid = opent(DirName + r"\sensor ", 'r') + except IOError: + print ("can't finde sensor file for FLEX format") + return + Lines = fid.readlines() + fid.close() + # reads channel info (name, unit and description) + self.NrCh = 0 + Name = [] + Unit = [] + Description = [] + for i in range(2, len(Lines)): + temp = Lines[i] + if not temp.strip(): + break + self.NrCh += 1 + temp = str(Lines[i][38:45]); Unit.append(temp.strip()) + temp = str(Lines[i][45:53]); Name.append(temp.strip()) + temp = str(Lines[i][53:]); Description.append(temp.strip()) + self.ChInfo = [Name, Unit, Description] + # read general info from *.int file + fid = open(self.FileName, 'rb') + fid.seek(4 * 19) + if not np.fromfile(fid, 'int32', 1) == self.NrCh: + print ("number of sensors in sensor file and data file are not consisten") + fid.seek(4 * (self.NrCh) + 4, 1) + self.Version = np.fromfile(fid, 'int32',1)[0] + temp = np.fromfile(fid, 'f', 2) + self.Freq = 1 / temp[1]; + self.ScaleFactor = np.fromfile(fid, 'f', self.NrCh) + fid.seek(2 * 4 * self.NrCh + 48 * 2) + self.NrSc = int(len(np.fromfile(fid, 'int16')) / self.NrCh) + self.Time = self.NrSc * temp[1] + self.t = np.arange(0, self.Time, temp[1]) + fid.close() +################################################################################ +# init function, load channel and other general result file info + def __init__(self, FileName, ReadOnly=0): + self.FileName = FileName + self.ReadOnly = ReadOnly + self.Iknown = [] # to keep track of what has been read all ready + self.Data = np.zeros(0) + if FileName.lower().endswith('.sel') or os.path.isfile(FileName + ".sel"): + self._ReadSelFile() + elif FileName.lower().endswith('.dat') and os.path.isfile(os.path.splitext(FileName)[0] + ".sel"): + self.FileName = os.path.splitext(FileName)[0] + self._ReadSelFile() + elif FileName.lower().endswith('.int') or FileName.lower().endswith('.res'): + self.FileFormat = 'FLEX' + self._ReadSensorFile() + elif os.path.isfile(self.FileName + ".int"): + self.FileName = self.FileName + ".int" + self.FileFormat = 'FLEX' + self._ReadSensorFile() + elif os.path.isfile(self.FileName + ".res"): + self.FileName = self.FileName + ".res" + self.FileFormat = 'FLEX' + self._ReadSensorFile() + elif FileName.lower().endswith('.hdf5') or os.path.isfile(self.FileName + ".hdf5"): + self.FileFormat = 'GTSDF' + self.ReadGtsdf() + else: + raise Exception("unknown file: " + FileName) +################################################################################ +# Read results in binary format + def ReadBinary(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec: + ChVec = range(0, self.NrCh) + with open(self.FileName + '.dat', 'rb') as fid: + data = np.zeros((self.NrSc, len(ChVec))) + j = 0 + for i in ChVec: + fid.seek(i * self.NrSc * 2, 0) + data[:, j] = np.fromfile(fid, 'int16', self.NrSc) * self.ScaleFactor[i] + j += 1 + return data +################################################################################ +# Read results in ASCII format + def ReadAscii(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec: + ChVec = range(0, self.NrCh) + temp = np.loadtxt(self.FileName + '.dat', usecols=ChVec) + return temp.reshape((self.NrSc, len(ChVec))) +################################################################################ +# Read results in FLEX format + def ReadFLEX(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec: + ChVec = range(1, self.NrCh) + fid = open(self.FileName, 'rb') + fid.seek(2 * 4 * self.NrCh + 48 * 2) + temp = np.fromfile(fid, 'int16') + if self.Version==3: + temp = temp.reshape(self.NrCh, self.NrSc).transpose() + else: + temp = temp.reshape(self.NrSc, self.NrCh) + fid.close() + return np.dot(temp[:, ChVec], np.diag(self.ScaleFactor[ChVec])) +################################################################################ +# Read results in GTSD format + def ReadGtsdf(self): + raise NotImplementedError + #self.t, data, info = gtsdf.load(self.FileName + '.hdf5') + #self.Time = self.t + #self.ChInfo = [['Time'] + info['attribute_names'], + # ['s'] + info['attribute_units'], + # ['Time'] + info['attribute_descriptions']] + #self.NrCh = data.shape[1] + 1 + #self.NrSc = data.shape[0] + #self.Freq = self.NrSc / self.Time + #self.FileFormat = 'GTSDF' + #self.gtsdf_description = info['description'] + #data = np.hstack([self.Time[:,np.newaxis], data]) + #return data +################################################################################ +# One stop call for reading all data formats + def ReadAll(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec and not self.FileFormat == 'GTSDF': + ChVec = range(0, self.NrCh) + if self.FileFormat == 'HAWC2_BINARY': + return self.ReadBinary(ChVec) + elif self.FileFormat == 'HAWC2_ASCII' or self.FileFormat == 'BHAWC_ASCII': + return self.ReadAscii(ChVec) + elif self.FileFormat == 'GTSDF': + return self.ReadGtsdf() + elif self.FileFormat == 'FLEX': + return self.ReadFLEX(ChVec) + else: + raise Exception('Unknown file format {} for hawc2 out file'.format(self.FileFormat)) + +################################################################################ +# Main read data call, read, save and sort data + def __call__(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec: + ChVec = range(0, self.NrCh) + elif max(ChVec) >= self.NrCh: + print("to high channel number") + return + # if ReadOnly, read data but no storeing in memory + if self.ReadOnly: + return self.ReadAll(ChVec) + # if not ReadOnly, sort in known and new channels, read new channels + # and return all requested channels + else: + # sort into known channels and channels to be read + I1 = [] + I2 = [] # I1=Channel mapping, I2=Channels to be read + for i in ChVec: + try: + I1.append(self.Iknown.index(i)) + except: + self.Iknown.append(i) + I2.append(i) + I1.append(len(I1)) + # read new channels + if I2: + temp = self.ReadAll(I2) + # add new channels to Data + if self.Data.any(): + self.Data = np.append(self.Data, temp, axis=1) + # if first call, so Daata is empty + else: + self.Data = temp + return self.Data[:, tuple(I1)] + + +################################################################################ +################################################################################ +################################################################################ +# write HAWC2 class, to be implemented +################################################################################ + +if __name__ == '__main__': + res_file = ReadHawc2('structure_wind') + results = res_file.ReadAscii() + channelinfo = res_file.ChInfo diff --git a/pydatview/io/wetb/hawc2/__init__.py b/pydatview/io/wetb/hawc2/__init__.py new file mode 100644 index 0000000..1cd84f9 --- /dev/null +++ b/pydatview/io/wetb/hawc2/__init__.py @@ -0,0 +1,18 @@ +# from __future__ import unicode_literals +# from __future__ import print_function +# from __future__ import division +# from __future__ import absolute_import +# from future import standard_library +# standard_library.install_aliases() +# d = None +# d = dir() +# +# from .htc_file import HTCFile +# from .log_file import LogFile +# from .ae_file import AEFile +# from .at_time_file import AtTimeFile +# from .pc_file import PCFile +# from . import shear_file +# from .st_file import StFile +# +# __all__ = sorted([m for m in set(dir()) - set(d)]) diff --git a/pydatview/io/wetb/hawc2/ae_file.py b/pydatview/io/wetb/hawc2/ae_file.py new file mode 100644 index 0000000..36c7364 --- /dev/null +++ b/pydatview/io/wetb/hawc2/ae_file.py @@ -0,0 +1,132 @@ +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from io import open +from builtins import range +from builtins import int +from future import standard_library +import os +standard_library.install_aliases() + +import numpy as np + + +class AEFile(object): + + """Read and write the HAWC2 AE (aerodynamic blade layout) file + + examples + -------- + >>> aefile = AEFile(r"tests/test_files/NREL_5MW_ae.txt") + >>> aefile.thickness(36) # Interpolated thickness at radius 36 + 23.78048780487805 + >>> aefile.chord(36) # Interpolated chord at radius 36 + 3.673 + >>> aefile.pc_set_nr(36) # pc set number at radius 36 + 1 + >>> ae= AEFile() + ae.add_set(radius=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], + chord=[1.1, 1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1], + thickness=[100.0, 100.0, 90.0, 80.0, 70.0, 60.0, 50.0, 40.0, 30.0, 20.0, 10.0], + pc_set_id=[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]) + >>> str(ae) + 1 r[m] Chord[m] T/C[%] Set no. + 1 11 + 0.00000000000000000e+00 1.10000000000000009e+00 1.00000000000000000e+02 1 + 1.00000000000000006e-01 1.00000000000000000e+00 1.00000000000000000e+02 1 + 2.00000000000000011e-01 9.00000000000000022e-01 9.00000000000000000e+01 1 + 2.99999999999999989e-01 8.00000000000000044e-01 8.00000000000000000e+01 1 + 4.00000000000000022e-01 6.99999999999999956e-01 7.00000000000000000e+01 1 + 5.00000000000000000e-01 5.99999999999999978e-01 6.00000000000000000e+01 1 + 5.99999999999999978e-01 5.00000000000000000e-01 5.00000000000000000e+01 1 + 6.99999999999999956e-01 4.00000000000000022e-01 4.00000000000000000e+01 1 + 8.00000000000000044e-01 2.99999999999999989e-01 3.00000000000000000e+01 1 + 9.00000000000000022e-01 2.00000000000000011e-01 2.00000000000000000e+01 1 + 1.00000000000000000e+00 1.00000000000000006e-01 1.00000000000000000e+01 1 + """ + + cols = ['radius', 'chord', 'relative_thickness', 'setnr'] + + def __init__(self, filename=None): + self.ae_sets = {} + if filename is not None: + self._read_file(filename) + + def _value(self, radius, column, set_nr=1): + ae_data = self.ae_sets[set_nr] + if radius is None: + return ae_data[:, column] + else: + return np.interp(radius, ae_data[:, 0], ae_data[:, column]) + + def chord(self, radius=None, set_nr=1): + return self._value(radius, 1, set_nr) + + def thickness(self, radius=None, set_nr=1): + return self._value(radius, 2, set_nr) + + def radius_ae(self, radius=None, set_nr=1): + radii = self.ae_sets[set_nr][:, 0] + if radius: + return radii[np.argmin(np.abs(radii - radius))] + else: + return radii + + def pc_set_nr(self, radius, set_nr=1): + ae_data = self.ae_sets[set_nr] + index = np.searchsorted(ae_data[:, 0], radius) + index = max(1, index) + setnrs = ae_data[index - 1:index + 1, 3] + if setnrs[0] != setnrs[-1]: + raise NotImplementedError + return setnrs[0] + + def add_set(self, radius, chord, thickness, pc_set_id, set_id=None): + '''This method will add another set to the ae data''' + if set_id is None: + set_id = 1 + while set_id in self.ae_sets.keys(): + set_id += 1 + self.ae_sets[set_id] = np.array([radius, chord, thickness, pc_set_id]).T + return set_id + + def __str__(self): + '''This method will create a string that is formatted like an ae file with the data in this class''' + n_sets = len(self.ae_sets) + retval = str(n_sets) + ' r[m] Chord[m] T/C[%] Set no.\n' + for st_idx, st in self.ae_sets.items(): + retval += str(st_idx) + ' ' + str(len(st)) + '\n' + for line in st: + retval += '%25.17e %25.17e %25.17e %5d\n' % (line[0], line[1], line[2], line[3]) + return retval + + def save(self, filename): + if not os.path.isdir(os.path.dirname(filename)): + os.makedirs(os.path.dirname(filename)) + with open(filename, 'w') as fid: + fid.write(str(self)) + + def _read_file(self, filename): + ''' This method will read in the ae data from a HAWC2 ae file''' + with open(filename) as fid: + lines = fid.readlines() + nsets = int(lines[0].split()[0]) + lptr = 1 + self.ae_sets = {} + for _ in range(1, nsets + 1): + set_nr, n_rows = [int(v) for v in lines[lptr].split()[:2]] + lptr += 1 + data = np.array([[float(v) for v in l.split()[:4]] for l in lines[lptr:lptr + n_rows]]) + self.ae_sets[set_nr] = data + lptr += n_rows + + + + +if __name__ == "__main__": + ae = AEFile(r"tests/test_files/NREL_5MW_ae.txt") + print (ae.radius_ae(36)) + print (ae.thickness()) + print (ae.chord(36)) + print (ae.pc_set_nr(36)) diff --git a/pydatview/io/wetb/hawc2/htc_contents.py b/pydatview/io/wetb/hawc2/htc_contents.py new file mode 100644 index 0000000..7805eb3 --- /dev/null +++ b/pydatview/io/wetb/hawc2/htc_contents.py @@ -0,0 +1,489 @@ +''' +Created on 20/01/2014 + +@author: MMPE + +See documentation of HTCFile below + +''' +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from builtins import zip +from builtins import int +from builtins import str +from future import standard_library +import os +standard_library.install_aliases() +from collections import OrderedDict +import collections + + +class OrderedDict(collections.OrderedDict): + pass + + def __str__(self): + return "\n".join(["%-30s\t %s" % ((str(k) + ":"), str(v)) for k, v in self.items()]) + + +def parse_next_line(lines): + _3to2list = list(lines.pop(0).split(";")) + line, comments, = _3to2list[:1] + [_3to2list[1:]] + comments = ";".join(comments).rstrip() + while lines and lines[0].lstrip().startswith(";"): + comments += "\n%s" % lines.pop(0).rstrip() + return line.strip(), comments + + +def fmt_value(v): + try: + if int(float(v)) == float(v): + return int(float(v)) + return float(v) + except ValueError: + return v.replace("\\", "/") + + +c = 0 + + +class HTCContents(object): + lines = [] + contents = None + name_ = "" + parent = None + + def __getitem__(self, key): + if isinstance(key, str): + key = key.replace(".", "/") + if "/" in key: + keys = key.split('/') + val = self.contents[keys[0]] + for k in keys[1:]: + val = val[k] + return val + return self.contents[key] + else: + return self.values[key] + + def __getattr__(self, *args, **kwargs): + if args[0] in ['__members__', '__methods__']: + # fix python2 related issue. In py2, dir(self) calls + # __getattr__(('__members__',)), and this call must fail unhandled to work + return object.__getattribute__(self, *args, **kwargs) + try: + return object.__getattribute__(self, *args, **kwargs) + except Exception: + k = args[0] + if k.endswith("__1"): + k = k[:-3] + return self.contents[k] + + def __setattr__(self, *args, **kwargs): + _3to2list1 = list(args) + k, v, = _3to2list1[:1] + _3to2list1[1:] + if k in dir(self): # in ['section', 'filename', 'lines']: + if isinstance(self, HTCLine) and k == 'values': + args = k, list(v) + return object.__setattr__(self, *args, **kwargs) + if isinstance(v, str): + v = [fmt_value(v) for v in v.split()] + if not isinstance(v, HTCContents): + if not isinstance(v, (list, tuple)): + v = [v] + if k in self.contents: + self.contents[k].values = v + return + + v = HTCLine(k, v, "") + self.contents[k] = v + v.parent = self + + def __delattr__(self, *args, **kwargs): + k, = args + if k in self: + del self.contents[k] + + def __iter__(self): + # mainbodies must preceed constraints + values = ([v for v in self.contents.values() if v.name_ == 'main_body'] + + [v for v in self.contents.values() if v.name_ != 'main_body']) + return iter(values) + + def __contains__(self, key): + if self.contents is None: + return False + return key in self.contents + + def get(self, section, default=None): + try: + return self[section] + except KeyError: + return default + + def __call__(self, **kwargs): + """Allow accesing one of multiple subsections with same name, e.g. the main body where name=='shaft' + > htc.new_htc_structure.main_body(name='shaft') + + or one of multiple lines with same name, e.g. the section in c2_def where value[0]==3 + > htc.new_htc_structure.main_body.c2_def.sec(v0=3) + """ + lst = [s for s in self.parent if s.name_ == self.name_ and ( + all([k in s and s[k][0] == v for k, v in kwargs.items()]) or + (all([k[0] == 'v' for k in kwargs]) and all([s[int(k[1:])] == v for k, v in kwargs.items()])) + )] + assert len(lst) == 1 + return lst[0] + + def keys(self): + return list(self.contents.keys()) + + def _add_contents(self, contents): + if contents.name_ not in self: + self[contents.name_] = contents + else: + ending = "__2" + while contents.name_ + ending in self: + ending = "__%d" % (1 + float("0%s" % ending.replace("__", ""))) + self[contents.name_ + ending] = contents + contents.parent = self + + def add_section(self, section_name, members={}, section=None, allow_duplicate=False, **kwargs): + if isinstance(section_name, HTCSection): + section = section_name + section_name = section.name_ + if section_name in self and allow_duplicate is False: + return self[section_name] + if section_name == "output": + section = HTCOutputSection(section_name) + elif section_name.startswith("output_at_time"): + section = HTCOutputAtTimeSection(section_name) + elif section is None: + section = HTCSection(section_name) + self._add_contents(section) + kwargs.update(members) + for k, v in kwargs.items(): + section[k] = v + return section + + def delete(self): + keys = [k for (k, v) in self.parent.contents.items() if v == self] + for k in keys: + del self.parent.contents[k] + + def location(self): + if self.parent is None: + return os.path.basename(self.filename) + else: + name = [k for k in self.parent.keys() if self.parent[k] == self][0] + return self.parent.location() + "/" + name + + def compare(self, other, compare_order=False): + my_keys = self.keys() + other_keys = other.keys() + s = "" + while my_keys or other_keys: + if my_keys: + if (my_keys[0] in other_keys): + if compare_order: + other_i = 0 + else: + other_i = other_keys.index(my_keys[0]) + while other_keys[other_i] != my_keys[0]: + s += "\n".join(["+ %s" % l for l in str(other[other_keys.pop(other_i)] + ).strip().split("\n")]) + "\n\n" + + s += self[my_keys.pop(0)].compare(other[other_keys.pop(other_i)]) + + else: + s += "\n".join(["- %s" % l for l in str(self[my_keys.pop(0)]).strip().split("\n")]) + "\n\n" + else: + s += "\n".join(["+ %s" % l for l in str(other[other_keys.pop(0)]).strip().split("\n")]) + "\n\n" + return s + + +class HTCSection(HTCContents): + end_comments = "" + begin_comments = "" + + def __init__(self, name, begin_comments="", end_comments=""): + self.name_ = name + self.begin_comments = begin_comments.strip(" \t") + self.end_comments = end_comments.strip(" \t") + self.contents = OrderedDict() + + @property + def section_name(self): + return self.name_ + + @section_name.setter + def section_name(self, value): + self.name_ = value + + def add_line(self, name, values, comments=""): + line = HTCLine(name, values, comments) + self._add_contents(line) + return line + + def __setitem__(self, key, value): + if isinstance(value, HTCContents): + self.contents[key] = value + value.parent = self + elif isinstance(value, (str, int, float)): + self.add_line(key, [value]) + else: + self.add_line(key, value) + + @staticmethod + def from_lines(lines): + line, begin_comments = parse_next_line(lines) + name = line[6:].lower() + if name == "output": + section = HTCOutputSection(name, begin_comments) + elif name.startswith("output_at_time"): + section = HTCOutputAtTimeSection(name, begin_comments) + else: + section = HTCSection(name, begin_comments) + while lines: + if lines[0].lower().startswith("begin"): + section._add_contents(HTCSection.from_lines(lines)) + elif lines[0].lower().startswith("end"): + line, section.end_comments = parse_next_line(lines) + break + else: + section._add_contents(section.line_from_line(lines)) + else: + raise Exception("Section '%s' has not end" % section.name_) + return section + + def line_from_line(self, lines): + return HTCLine.from_lines(lines) + + def __str__(self, level=0): + s = "%sbegin %s;%s\n" % (" " * level, self.name_, (("", "\t" + self.begin_comments) + [bool(self.begin_comments.strip())]).replace("\t\n", "\n")) + s += "".join([c.__str__(level + 1) for c in self]) + s += "%send %s;%s\n" % (" " * level, self.name_, (("", "\t" + self.end_comments) + [self.end_comments.strip() != ""]).replace("\t\n", "\n")) + return s + + def get_subsection_by_name(self, name, field='name'): + return self.get_section(name, field) + + def get_section(self, name, field='name'): + lst = [s for s in self if field in s and s[field][0] == name] + if len(lst) == 1: + return lst[0] + elif len(lst) == 0: + raise ValueError("subsection with %s='%s' not found" % (field, name)) + else: + raise ValueError("Multiple subsection with %s='%s' not found" % (field, name)) + + def get_element(self, key, value): + """Return subsection where subsection.==value or line where line.values[key]==value""" + if isinstance(key, int): + lst = [s for s in self if s.values[key] == value] + elif isinstance(key, str): + lst = [s for s in self if key in s and s[key][0] == name] + else: + raise ValueError("Key argument must be int or str") + if len(lst) == 1: + return lst[0] + elif len(lst) == 0: + raise ValueError("contents with '%s=%s' not found" % (key, value)) + else: + raise ValueError("Multiple contents with '%s=%s' not found" % (key, value)) + + def copy(self): + copy = HTCSection(name=self.name_, begin_comments=self.begin_comments, end_comments=self.end_comments) + for k, v in self.contents.items(): + copy.contents[k] = v.copy() + return copy + + +class HTCLine(HTCContents): + values = None + comments = "" + + def __init__(self, name, values, comments): + if "__" in name: + name = name[:name.index("__")] + self.name_ = name + self.values = list(values) + self.comments = comments.strip(" \t") + + def __repr__(self): + return str(self) + + def __str__(self, level=0): + if self.name_ == "": + return "" + return "%s%s%s;%s\n" % (" " * (level), self.name_, + ("", "\t" + self.str_values())[bool(self.values)], + ("", "\t" + self.comments)[bool(self.comments.strip())]) + + def str_values(self): + return " ".join([str(v) for v in self.values]) + + def __getitem__(self, key): + try: + return self.values[key] + except Exception: + raise IndexError("Parameter %s does not exists for %s" % (key + 1, self.location())) + + def __setitem__(self, key, value): + if isinstance(key, int): + self.values[key] = value + else: + raise NotImplementedError + + @staticmethod + def from_lines(lines): + line, end_comments = parse_next_line(lines) + if len(line.split()) > 0: + _3to2list3 = list(line.split()) + name, values, = _3to2list3[:1] + [_3to2list3[1:]] + else: + name = line + values = [] + + values = [fmt_value(v) for v in values] + return HTCLine(name, values, end_comments) + + def compare(self, other): + s = "" + if self.values != other.values: + s += "\n".join(["+ %s" % l for l in str(self).strip().split("\n")]) + "\n" + s += "\n".join(["- %s" % l for l in str(other).strip().split("\n")]) + "\n" + s += "\n" + return s + + def copy(self): + return HTCLine(name=self.name_, values=self.values, comments=self.comments) + + +class HTCOutputSection(HTCSection): + sensors = None + + def __init__(self, name, begin_comments="", end_comments=""): + HTCSection.__init__(self, name, begin_comments=begin_comments, end_comments=end_comments) + self.sensors = [] + + def add_sensor(self, type, sensor, values=None, comment="", nr=None): + values = [] if values is None else values + self._add_sensor(HTCSensor(type, sensor, values, comment), nr) + + def _add_sensor(self, htcSensor, nr=None): + if nr is None: + nr = len(self.sensors) + self.sensors.insert(nr, htcSensor) + htcSensor.parent = self + + def line_from_line(self, lines): + while len(lines) and lines[0].strip() == "": + lines.pop(0) + name = lines[0].split()[0].strip() + if name in ['filename', 'data_format', 'buffer', 'time']: + return HTCLine.from_lines(lines) + else: + return HTCSensor.from_lines(lines) + + def _add_contents(self, contents): + if isinstance(contents, HTCSensor): + self._add_sensor(contents) + else: + return HTCSection._add_contents(self, contents) + + def __str__(self, level=0): + s = "%sbegin %s;%s\n" % (" " * level, self.name_, ("", "\t" + self.begin_comments) + [len(self.begin_comments.strip()) > 0]) + s += "".join([c.__str__(level + 1) for c in self]) + s += "".join([s.__str__(level + 1) for s in self.sensors]) + s += "%send %s;%s\n" % (" " * level, self.name_, ("", "\t" + self.end_comments) + [self.end_comments.strip() != ""]) + return s + + def compare(self, other): + s = HTCContents.compare(self, other) + for s1, s2 in zip(self.sensors, other.sensors): + s += s1.compare(s2) + for s1 in self.sensors[len(other.sensors):]: + s += "\n".join(["- %s" % l for l in str(s1).strip().split("\n")]) + "\n" + for s2 in self.sensors[len(self.sensors):]: + s += "\n".join(["- %s" % l for l in str(s2).strip().split("\n")]) + "\n" + + return s + + +class HTCOutputAtTimeSection(HTCOutputSection): + type = None + time = None + + def __init__(self, name, begin_comments="", end_comments=""): + if len(name.split()) < 3: + raise ValueError('"keyword" and "time" arguments required for output_at_time command:\n%s' % name) + name, self.type, time = name.split() + self.time = float(time) + HTCOutputSection.__init__(self, name, begin_comments=begin_comments, end_comments=end_comments) + + def __str__(self, level=0): + s = "%sbegin %s %s %s;%s\n" % (" " * level, self.name_, self.type, self.time, + ("", "\t" + self.begin_comments)[len(self.begin_comments.strip())]) + s += "".join([c.__str__(level + 1) for c in self]) + s += "".join([s.__str__(level + 1) for s in self.sensors]) + s += "%send %s;%s\n" % (" " * level, self.name_, ("", "\t" + self.end_comments) + [self.end_comments.strip() != ""]) + return s + + +class HTCSensor(HTCLine): + type = "" + sensor = "" + values = [] + + def __init__(self, type, sensor, values, comments): + self.type = type + self.sensor = sensor + self.values = list(values) + self.comments = comments.strip(" \t") + + @staticmethod + def from_lines(lines): + line, comments = parse_next_line(lines) + if len(line.split()) > 2: + _3to2list5 = list(line.split()) + type, sensor, values, = _3to2list5[:2] + [_3to2list5[2:]] + elif len(line.split()) == 2: + type, sensor = line.split() + values = [] + else: + type, sensor, values = "", "", [] + + def fmt(v): + try: + if int(float(v)) == float(v): + return int(float(v)) + return float(v) + except ValueError: + return v + values = [fmt(v) for v in values] + return HTCSensor(type, sensor, values, comments) + + def __str__(self, level=0): + return "%s%s %s%s;%s\n" % (" " * (level), + self.type, + self.sensor, + ("", "\t" + self.str_values())[bool(self.values)], + ("", "\t" + self.comments)[bool(self.comments.strip())]) + + def delete(self): + self.parent.sensors.remove(self) + + def compare(self, other): + s = "" + if self.sensor != other.sensor or self.values != other.values: + s += "\n".join(["+ %s" % l for l in str(self).strip().split("\n")]) + "\n" + s += "\n".join(["- %s" % l for l in str(other).strip().split("\n")]) + "\n" + s += "\n" + return s diff --git a/pydatview/io/wetb/hawc2/htc_extensions.py b/pydatview/io/wetb/hawc2/htc_extensions.py new file mode 100644 index 0000000..4e109c6 --- /dev/null +++ b/pydatview/io/wetb/hawc2/htc_extensions.py @@ -0,0 +1,152 @@ +''' +Created on 20/01/2014 + +@author: MMPE + +See documentation of HTCFile below + +''' +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from builtins import zip +from builtins import int +from builtins import str +from future import standard_library +import os + + +standard_library.install_aliases() + + +class HTCDefaults(object): + + empty_htc = """begin simulation; + time_stop 600; + solvertype 2; (newmark) + begin newmark; + deltat 0.02; + end newmark; + end simulation; + ; + ;---------------------------------------------------------------------------------------------------------------------------------------------------------------- + ; + begin wind ; + density 1.225 ; + wsp 10 ; + tint 1; + horizontal_input 1 ; 0=false, 1=true + windfield_rotations 0 0.0 0.0 ; yaw, tilt, rotation + center_pos0 0 0 -30 ; hub heigth + shear_format 1 0;0=none,1=constant,2=log,3=power,4=linear + turb_format 0 ; 0=none, 1=mann,2=flex + tower_shadow_method 0 ; 0=none, 1=potential flow, 2=jet + end wind; + ; + ;---------------------------------------------------------------------------------------------------------------------------------------------------------------- + ; + ; + begin output; + filename ./tmp; + general time; + end output; + exit;""" + + def add_mann_turbulence(self, L=29.4, ae23=1, Gamma=3.9, seed=1001, high_frq_compensation=True, + filenames=None, + no_grid_points=(16384, 32, 32), box_dimension=(6000, 100, 100), + dont_scale=False, + std_scaling=None): + wind = self.add_section('wind') + wind.turb_format = 1 + mann = wind.add_section('mann') + if 'create_turb_parameters' in mann: + mann.create_turb_parameters.values = [L, ae23, Gamma, seed, int(high_frq_compensation)] + else: + mann.add_line('create_turb_parameters', [L, ae23, Gamma, seed, int(high_frq_compensation)], + "L, alfaeps, gamma, seed, highfrq compensation") + if filenames is None: + + import numpy as np + dxyz = tuple(np.array(box_dimension) / no_grid_points) + from wetb.wind.turbulence import mann_turbulence + filenames = ["./turb/" + mann_turbulence.name_format % + ((L, ae23, Gamma, high_frq_compensation) + no_grid_points + dxyz + (seed, uvw)) + for uvw in ['u', 'v', 'w']] + if isinstance(filenames, str): + filenames = ["./turb/%s_s%04d%s.bin" % (filenames, seed, c) for c in ['u', 'v', 'w']] + for filename, c in zip(filenames, ['u', 'v', 'w']): + setattr(mann, 'filename_%s' % c, filename) + for c, n, dim in zip(['u', 'v', 'w'], no_grid_points, box_dimension): + setattr(mann, 'box_dim_%s' % c, "%d %.4f" % (n, dim / (n))) + if dont_scale: + mann.dont_scale = 1 + else: + try: + del mann.dont_scale + except KeyError: + pass + if std_scaling is not None: + mann.std_scaling = "%f %f %f" % std_scaling + else: + try: + del mann.std_scaling + except KeyError: + pass + + def add_turb_export(self, filename="export_%s.turb", samplefrq=None): + exp = self.wind.add_section('turb_export', allow_duplicate=True) + for uvw in 'uvw': + exp.add_line('filename_%s' % uvw, [filename % uvw]) + sf = samplefrq or max(1, int(self.wind.mann.box_dim_u[1] / (self.wind.wsp[0] * self.deltat()))) + exp.samplefrq = sf + if "time" in self.output: + exp.time_start = self.output.time[0] + else: + exp.time_start = 0 + exp.nsteps = (self.simulation.time_stop[0] - exp.time_start[0]) / self.deltat() + for vw in 'vw': + exp.add_line('box_dim_%s' % vw, self.wind.mann['box_dim_%s' % vw].values) + + def import_dtu_we_controller_input(self, filename): + dtu_we_controller = [dll for dll in self.dll if dll.name[0] == 'dtu_we_controller'][0] + with open(filename) as fid: + lines = fid.readlines() + K_r1 = float(lines[1].replace("K = ", '').replace("[Nm/(rad/s)^2]", '')) + Kp_r2 = float(lines[4].replace("Kp = ", '').replace("[Nm/(rad/s)]", '')) + Ki_r2 = float(lines[5].replace("Ki = ", '').replace("[Nm/rad]", '')) + Kp_r3 = float(lines[7].replace("Kp = ", '').replace("[rad/(rad/s)]", '')) + Ki_r3 = float(lines[8].replace("Ki = ", '').replace("[rad/rad]", '')) + KK = lines[9].split("]") + KK1 = float(KK[0].replace("K1 = ", '').replace("[deg", '')) + KK2 = float(KK[1].replace(", K2 = ", '').replace("[deg^2", '')) + cs = dtu_we_controller.init + cs.constant__11.values[1] = "%.6E" % K_r1 + cs.constant__12.values[1] = "%.6E" % Kp_r2 + cs.constant__13.values[1] = "%.6E" % Ki_r2 + cs.constant__16.values[1] = "%.6E" % Kp_r3 + cs.constant__17.values[1] = "%.6E" % Ki_r3 + cs.constant__21.values[1] = "%.6E" % KK1 + cs.constant__22.values[1] = "%.6E" % KK2 + + def add_hydro(self, mudlevel, mwl, gravity=9.81, rho=1027): + wp = self.add_section("hydro").add_section('water_properties') + wp.mudlevel = mudlevel + wp.mwl = mwl + wp.gravity = gravity + wp.rho = rho + + +class HTCExtensions(object): + def get_shear(self): + shear_type, parameter = self.wind.shear_format.values + z0 = -self.wind.center_pos0[2] + wsp = self.wind.wsp[0] + if shear_type == 1: # constant + return lambda z: wsp + elif shear_type == 3: + from wetb.wind.shear import power_shear + return power_shear(parameter, z0, wsp) + else: + raise NotImplementedError diff --git a/pydatview/io/wetb/hawc2/htc_file.py b/pydatview/io/wetb/hawc2/htc_file.py new file mode 100644 index 0000000..5d1a6d7 --- /dev/null +++ b/pydatview/io/wetb/hawc2/htc_file.py @@ -0,0 +1,585 @@ +''' +Created on 20/01/2014 + +@author: MMPE + +See documentation of HTCFile below + +''' +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import str +from future import standard_library +# from wetb.utils.process_exec import pexec +# from wetb.hawc2.hawc2_pbs_file import HAWC2PBSFile +# from wetb.utils.cluster_tools.os_path import fixcase, abspath, pjoin +# import jinja2 +standard_library.install_aliases() +from collections import OrderedDict +from .htc_contents import HTCContents, HTCSection, HTCLine +from .htc_extensions import HTCDefaults, HTCExtensions +import os + +# --- cluster_tools/os_path +def repl(path): + return path.replace("\\", "/") + +def abspath(path): + return repl(os.path.abspath(path)) + +def relpath(path, start=None): + return repl(os.path.relpath(path, start)) + +def realpath(path): + return repl(os.path.realpath(path)) + +def pjoin(*path): + return repl(os.path.join(*path)) + +def fixcase(path): + path = realpath(str(path)).replace("\\", "/") + p, rest = os.path.splitdrive(path) + p += "/" + for f in rest[1:].split("/"): + f_lst = [f_ for f_ in os.listdir(p) if f_.lower() == f.lower()] + if len(f_lst) > 1: + # use the case sensitive match + f_lst = [f_ for f_ in f_lst if f_ == f] + if len(f_lst) == 0: + raise IOError("'%s' not found in '%s'" % (f, p)) + # Use matched folder + p = pjoin(p, f_lst[0]) + return p +# --- end os_path + +class HTCFile(HTCContents, HTCDefaults, HTCExtensions): + """Wrapper for HTC files + + Examples: + --------- + >>> htcfile = HTCFile('htc/test.htc') + >>> htcfile.wind.wsp = 10 + >>> htcfile.save() + + #--------------------------------------------- + >>> htc = HTCFile(filename=None, modelpath=None) # create minimal htcfile + + #Add section + >>> htc.add_section('hydro') + + #Add subsection + >>> htc.hydro.add_section("hydro_element") + + #Set values + >>> htc.hydro.hydro_element.wave_breaking = [2, 6.28, 1] # or + >>> htc.hydro.hydro_element.wave_breaking = 2, 6.28, 1 + + #Set comments + >>> htc.hydro.hydro_element.wave_breaking.comments = "This is a comment" + + #Access section + >>> hydro_element = htc.hydro.hydro_element #or + >>> hydro_element = htc['hydro.hydro_element'] # or + >>> hydro_element = htc['hydro/hydro_element'] # or + >>> print (hydro_element.wave_breaking) #string represenation + wave_breaking 2 6.28 1; This is a comment + >>> print (hydro_element.wave_breaking.name_) # command + wave_breaking + >>> print (hydro_element.wave_breaking.values) # values + [2, 6.28, 1 + >>> print (hydro_element.wave_breaking.comments) # comments + This is a comment + >>> print (hydro_element.wave_breaking[0]) # first value + 2 + + #Delete element + htc.simulation.logfile.delete() + #or + del htc.simulation.logfile #Delete logfile line. Raise keyerror if not exists + + """ + + filename = None + jinja_tags = {} + htc_inputfiles = [] + level = 0 + modelpath = "../" + initial_comments = None + _contents = None + + def __init__(self, filename=None, modelpath=None, jinja_tags={}): + """ + Parameters + --------- + filename : str + Absolute filename of htc file + modelpath : str + Model path relative to htc file + """ + + if filename is not None: + try: + filename = fixcase(abspath(filename)) + with self.open(str(filename)): + pass + except Exception: + pass + + self.filename = filename + + self.jinja_tags = jinja_tags + self.modelpath = modelpath or self.auto_detect_modelpath() + + if filename and self.modelpath != "unknown" and not os.path.isabs(self.modelpath): + drive, p = os.path.splitdrive(os.path.join(os.path.dirname(str(self.filename)), self.modelpath)) + self.modelpath = os.path.join(drive, os.path.splitdrive(os.path.realpath(p))[1]).replace("\\", "/") + if self.modelpath != 'unknown' and self.modelpath[-1] != '/': + self.modelpath += "/" + + #assert 'simulation' in self.contents, "%s could not be loaded. 'simulation' section missing" % filename + + def auto_detect_modelpath(self): + if self.filename is None: + return "../" + + #print (["../"*i for i in range(3)]) + import numpy as np + input_files = HTCFile(self.filename, 'unknown').input_files() + if len(input_files) == 1: # only input file is the htc file + return "../" + rel_input_files = [f for f in input_files if not os.path.isabs(f)] + + def isfile_case_insensitive(f): + try: + f = fixcase(f) # raises exception if not existing + return os.path.isfile(f) + except IOError: + return False + found = ([np.sum([isfile_case_insensitive(os.path.join(os.path.dirname(self.filename), "../" * i, f)) + for f in rel_input_files]) for i in range(4)]) + + if max(found) > 0: + relpath = "../" * np.argmax(found) + return abspath(pjoin(os.path.dirname(self.filename), relpath)) + else: + raise ValueError( + "Modelpath cannot be autodetected for '%s'.\nInput files not found near htc file" % self.filename) + + def _load(self): + self.reset() + self.initial_comments = [] + self.htc_inputfiles = [] + self.contents = OrderedDict() + if self.filename is None: + lines = self.empty_htc.split("\n") + else: + lines = self.readlines(self.filename) + + lines = [l.strip() for l in lines] + + #lines = copy(self.lines) + while lines: + if lines[0].startswith(";"): + self.initial_comments.append(lines.pop(0).strip() + "\n") + elif lines[0].lower().startswith("begin"): + self._add_contents(HTCSection.from_lines(lines)) + else: + line = HTCLine.from_lines(lines) + if line.name_ == "exit": + break + self._add_contents(line) + + def reset(self): + self._contents = None + + @property + def contents(self): + if self._contents is None: + self._load() + return self._contents + + @contents.setter + def contents(self, value): + self._contents = value + + def readfilelines(self, filename): + with self.open(self.unix_path(filename), encoding='cp1252') as fid: + txt = fid.read() + if txt[:10].encode().startswith(b'\xc3\xaf\xc2\xbb\xc2\xbf'): + txt = txt[3:] + if self.jinja_tags: + template = jinja2.Template(txt) + txt = template.render(**self.jinja_tags) + return txt.replace("\r", "").split("\n") + + def readlines(self, filename): + if filename != self.filename: # self.filename may be changed by set_name/save. Added it when needed instead + self.htc_inputfiles.append(filename) + htc_lines = [] + lines = self.readfilelines(filename) + for l in lines: + if l.lower().lstrip().startswith('continue_in_file'): + filename = l.lstrip().split(";")[0][len("continue_in_file"):].strip().lower() + + if self.modelpath == 'unknown': + p = os.path.dirname(self.filename) + lu = [os.path.isfile(os.path.join(p, "../" * i, filename)) for i in range(4)].index(True) + filename = os.path.join(p, "../" * lu, filename) + else: + filename = os.path.join(self.modelpath, filename) + for line in self.readlines(filename): + if line.lstrip().lower().startswith('exit'): + break + htc_lines.append(line) + else: + htc_lines.append(l) + return htc_lines + + def __setitem__(self, key, value): + self.contents[key] = value + + def __str__(self): + self.contents # load + return "".join(self.initial_comments + [c.__str__(1) for c in self] + ["exit;"]) + + def save(self, filename=None): + self.contents # load if not loaded + if filename is None: + filename = self.filename + else: + self.filename = filename + # exist_ok does not exist in Python27 + if not os.path.exists(os.path.dirname(filename)) and os.path.dirname(filename) != "": + os.makedirs(os.path.dirname(filename)) # , exist_ok=True) + with self.open(filename, 'w', encoding='cp1252') as fid: + fid.write(str(self)) + + def set_name(self, name, subfolder=''): + # if os.path.isabs(folder) is False and os.path.relpath(folder).startswith("htc" + os.path.sep): + self.contents # load if not loaded + + def fmt_folder(folder, subfolder): return "./" + \ + os.path.relpath(os.path.join(folder, subfolder)).replace("\\", "/") + + self.filename = os.path.abspath(os.path.join(self.modelpath, fmt_folder( + 'htc', subfolder), "%s.htc" % name)).replace("\\", "/") + if 'simulation' in self and 'logfile' in self.simulation: + self.simulation.logfile = os.path.join(fmt_folder('log', subfolder), "%s.log" % name).replace("\\", "/") + if 'animation' in self.simulation: + self.simulation.animation = os.path.join(fmt_folder( + 'animation', subfolder), "%s.dat" % name).replace("\\", "/") + if 'visualization' in self.simulation: + self.simulation.visualization = os.path.join(fmt_folder( + 'visualization', subfolder), "%s.hdf5" % name).replace("\\", "/") + elif 'test_structure' in self and 'logfile' in self.test_structure: # hawc2aero + self.test_structure.logfile = os.path.join(fmt_folder('log', subfolder), "%s.log" % name).replace("\\", "/") + self.output.filename = os.path.join(fmt_folder('res', subfolder), "%s" % name).replace("\\", "/") + + def set_time(self, start=None, stop=None, step=None): + self.contents # load if not loaded + if stop is not None: + self.simulation.time_stop = stop + else: + stop = self.simulation.time_stop[0] + if step is not None: + self.simulation.newmark.deltat = step + if start is not None: + self.output.time = start, stop + if "wind" in self: # and self.wind.turb_format[0] > 0: + self.wind.scale_time_start = start + + def expected_simulation_time(self): + return 600 + + def pbs_file(self, hawc2_path, hawc2_cmd, queue='workq', walltime=None, + input_files=None, output_files=None, copy_turb=(True, True)): + walltime = walltime or self.expected_simulation_time() * 2 + if len(copy_turb) == 1: + copy_turb_fwd, copy_turb_back = copy_turb, copy_turb + else: + copy_turb_fwd, copy_turb_back = copy_turb + + input_files = input_files or self.input_files() + if copy_turb_fwd: + input_files += [f for f in self.turbulence_files() if os.path.isfile(f)] + + output_files = output_files or self.output_files() + if copy_turb_back: + output_files += self.turbulence_files() + + return HAWC2PBSFile(hawc2_path, hawc2_cmd, self.filename, self.modelpath, + input_files, output_files, + queue, walltime) + + def input_files(self): + self.contents # load if not loaded + if self.modelpath == "unknown": + files = [str(f).replace("\\", "/") for f in [self.filename] + self.htc_inputfiles] + else: + files = [os.path.abspath(str(f)).replace("\\", "/") for f in [self.filename] + self.htc_inputfiles] + if 'new_htc_structure' in self: + for mb in [self.new_htc_structure[mb] + for mb in self.new_htc_structure.keys() if mb.startswith('main_body')]: + if "timoschenko_input" in mb: + files.append(mb.timoschenko_input.filename[0]) + files.append(mb.get('external_bladedata_dll', [None, None, None])[2]) + if 'aero' in self: + files.append(self.aero.ae_filename[0]) + files.append(self.aero.pc_filename[0]) + files.append(self.aero.get('external_bladedata_dll', [None, None, None])[2]) + files.append(self.aero.get('output_profile_coef_filename', [None])[0]) + if 'dynstall_ateflap' in self.aero: + files.append(self.aero.dynstall_ateflap.get('flap', [None] * 3)[2]) + if 'bemwake_method' in self.aero: + files.append(self.aero.bemwake_method.get('a-ct-filename', [None] * 3)[0]) + for dll in [self.dll[dll] for dll in self.get('dll', {}).keys() if 'filename' in self.dll[dll]]: + files.append(dll.filename[0]) + f, ext = os.path.splitext(dll.filename[0]) + files.append(f + "_64" + ext) + if 'wind' in self: + files.append(self.wind.get('user_defined_shear', [None])[0]) + files.append(self.wind.get('user_defined_shear_turbulence', [None])[0]) + files.append(self.wind.get('met_mast_wind', [None])[0]) + if 'wakes' in self: + files.append(self.wind.get('use_specific_deficit_file', [None])[0]) + files.append(self.wind.get('write_ct_cq_file', [None])[0]) + files.append(self.wind.get('write_final_deficits', [None])[0]) + if 'hydro' in self: + if 'water_properties' in self.hydro: + files.append(self.hydro.water_properties.get('water_kinematics_dll', [None])[0]) + files.append(self.hydro.water_properties.get('water_kinematics_dll', [None, None])[1]) + if 'soil' in self: + if 'soil_element' in self.soil: + files.append(self.soil.soil_element.get('datafile', [None])[0]) + try: + dtu_we_controller = self.dll.get_subsection_by_name('dtu_we_controller') + theta_min = dtu_we_controller.init.constant__5[1] + if theta_min >= 90: + files.append(os.path.join(os.path.dirname( + dtu_we_controller.filename[0]), "wpdata.%d" % theta_min).replace("\\", "/")) + except Exception: + pass + + try: + files.append(self.force.dll.dll[0]) + except Exception: + pass + + def fix_path_case(f): + if os.path.isabs(f): + return self.unix_path(f) + elif self.modelpath != "unknown": + try: + return "./" + os.path.relpath(self.unix_path(os.path.join(self.modelpath, f)), + self.modelpath).replace("\\", "/") + except IOError: + return f + else: + return f + return [fix_path_case(f) for f in set(files) if f] + + def output_files(self): + self.contents # load if not loaded + files = [] + for k, index in [('simulation/logfile', 0), + ('simulation/animation', 0), + ('simulation/visualization', 0), + ('new_htc_structure/beam_output_file_name', 0), + ('new_htc_structure/body_output_file_name', 0), + ('new_htc_structure/struct_inertia_output_file_name', 0), + ('new_htc_structure/body_eigenanalysis_file_name', 0), + ('new_htc_structure/constraint_output_file_name', 0), + ('wind/turb_export/filename_u', 0), + ('wind/turb_export/filename_v', 0), + ('wind/turb_export/filename_w', 0)]: + line = self.get(k) + if line: + files.append(line[index]) + if 'new_htc_structure' in self: + if 'system_eigenanalysis' in self.new_htc_structure: + f = self.new_htc_structure.system_eigenanalysis[0] + files.append(f) + files.append(os.path.join(os.path.dirname(f), 'mode*.dat').replace("\\", "/")) + if 'structure_eigenanalysis_file_name' in self.new_htc_structure: + f = self.new_htc_structure.structure_eigenanalysis_file_name[0] + files.append(f) + files.append(os.path.join(os.path.dirname(f), 'mode*.dat').replace("\\", "/")) + files.extend(self.res_file_lst()) + + for key in [k for k in self.contents.keys() if k.startswith("output_at_time")]: + files.append(self[key]['filename'][0] + ".dat") + return [f.lower() for f in files if f] + + def turbulence_files(self): + self.contents # load if not loaded + if 'wind' not in self.contents.keys() or self.wind.turb_format[0] == 0: + return [] + elif self.wind.turb_format[0] == 1: + files = [self.get('wind.mann.filename_%s' % comp, [None])[0] for comp in ['u', 'v', 'w']] + elif self.wind.turb_format[0] == 2: + files = [self.get('wind.flex.filename_%s' % comp, [None])[0] for comp in ['u', 'v', 'w']] + return [f for f in files if f] + + def res_file_lst(self): + self.contents # load if not loaded + res = [] + for output in [self[k] for k in self.keys() + if self[k].name_.startswith("output") and not self[k].name_.startswith("output_at_time")]: + dataformat = output.get('data_format', 'hawc_ascii') + res_filename = output.filename[0] + if dataformat[0] == "gtsdf" or dataformat[0] == "gtsdf64": + res.append(res_filename + ".hdf5") + elif dataformat[0] == "flex_int": + res.extend([res_filename + ".int", os.path.join(os.path.dirname(res_filename), 'sensor')]) + else: + res.extend([res_filename + ".sel", res_filename + ".dat"]) + return res + + def _simulate(self, exe, skip_if_up_to_date=False): + self.contents # load if not loaded + if skip_if_up_to_date: + from os.path import isfile, getmtime, isabs + res_file = os.path.join(self.modelpath, self.res_file_lst()[0]) + htc_file = os.path.join(self.modelpath, self.filename) + if isabs(exe): + exe_file = exe + else: + exe_file = os.path.join(self.modelpath, exe) + #print (from_unix(getmtime(res_file)), from_unix(getmtime(htc_file))) + if (isfile(htc_file) and isfile(res_file) and isfile(exe_file) and + str(HTCFile(htc_file)) == str(self) and + getmtime(res_file) > getmtime(htc_file) and getmtime(res_file) > getmtime(exe_file)): + if "".join(self.readfilelines(htc_file)) == str(self): + return + + self.save() + htcfile = os.path.relpath(self.filename, self.modelpath) + assert any([os.path.isfile(os.path.join(f, exe)) for f in [''] + os.environ['PATH'].split(";")]), exe + return pexec([exe, htcfile], self.modelpath) + + def simulate(self, exe, skip_if_up_to_date=False): + errorcode, stdout, stderr, cmd = self._simulate(exe, skip_if_up_to_date) + if ('simulation' in self.keys() and "logfile" in self.simulation and + os.path.isfile(os.path.join(self.modelpath, self.simulation.logfile[0]))): + with self.open(os.path.join(self.modelpath, self.simulation.logfile[0])) as fid: + log = fid.read() + else: + log = stderr + + if errorcode or 'Elapsed time' not in log: + log_lines = log.split("\n") + error_lines = [i for i, l in enumerate(log_lines) if 'error' in l.lower()] + if error_lines: + i = error_lines[0] + error_log = "\n".join(log_lines[i - 3:i + 3]) + else: + error_log = log + raise Exception("\nstdout:\n%s\n--------------\nstderr:\n%s\n--------------\nlog:\n%s\n--------------\ncmd:\n%s" % + (str(stdout), str(stderr), error_log, cmd)) + return str(stdout) + str(stderr), log + + def simulate_hawc2stab2(self, exe): + errorcode, stdout, stderr, cmd = self._simulate(exe, skip_if_up_to_date=False) + + if errorcode: + raise Exception("\nstdout:\n%s\n--------------\nstderr:\n%s\n--------------\ncmd:\n%s" % + (str(stdout), str(stderr), cmd)) + return str(stdout) + str(stderr) + + def deltat(self): + return self.simulation.newmark.deltat[0] + + def compare(self, other): + if isinstance(other, str): + other = HTCFile(other) + return HTCContents.compare(self, other) + + @property + def open(self): + return open + + def unix_path(self, filename): + filename = os.path.realpath(str(filename)).replace("\\", "/") + ufn, rest = os.path.splitdrive(filename) + ufn += "/" + for f in rest[1:].split("/"): + f_lst = [f_ for f_ in os.listdir(ufn) if f_.lower() == f.lower()] + if len(f_lst) > 1: + # use the case sensitive match + f_lst = [f_ for f_ in f_lst if f_ == f] + if len(f_lst) == 0: + raise IOError("'%s' not found in '%s'" % (f, ufn)) + else: # one match found + ufn = os.path.join(ufn, f_lst[0]) + return ufn.replace("\\", "/") + + +# +# def get_body(self, name): +# lst = [b for b in self.new_htc_structure if b.name_=="main_body" and b.name[0]==name] +# if len(lst)==1: +# return lst[0] +# else: +# if len(lst)==0: +# raise ValueError("Body '%s' not found"%name) +# else: +# raise NotImplementedError() +# + +class H2aeroHTCFile(HTCFile): + def __init__(self, filename=None, modelpath=None): + HTCFile.__init__(self, filename=filename, modelpath=modelpath) + + @property + def simulation(self): + return self.test_structure + + def set_time(self, start=None, stop=None, step=None): + if stop is not None: + self.test_structure.time_stop = stop + else: + stop = self.simulation.time_stop[0] + if step is not None: + self.test_structure.deltat = step + if start is not None: + self.output.time = start, stop + if "wind" in self and self.wind.turb_format[0] > 0: + self.wind.scale_time_start = start + + +class SSH_HTCFile(HTCFile): + def __init__(self, ssh, filename=None, modelpath=None): + object.__setattr__(self, 'ssh', ssh) + HTCFile.__init__(self, filename=filename, modelpath=modelpath) + + @property + def open(self): + return self.ssh.open + + def unix_path(self, filename): + rel_filename = os.path.relpath(filename, self.modelpath).replace("\\", "/") + _, out, _ = self.ssh.execute("find -ipath ./%s" % rel_filename, cwd=self.modelpath) + out = out.strip() + if out == "": + raise IOError("'%s' not found in '%s'" % (rel_filename, self.modelpath)) + elif "\n" in out: + raise IOError("Multiple '%s' found in '%s' (due to case senitivity)" % (rel_filename, self.modelpath)) + else: + drive, path = os.path.splitdrive(os.path.join(self.modelpath, out)) + path = os.path.realpath(path).replace("\\", "/") + return os.path.join(drive, os.path.splitdrive(path)[1]) + + +if "__main__" == __name__: + f = HTCFile(r"C:/Work/BAR-Local/Hawc2ToBeamDyn/sim.htc", ".") + print(f.input_files()) + import pdb; pdb.set_trace() +# f.save(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT_power_curve.htc") +# +# f = HTCFile(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT.htc", "../") +# f.set_time = 0, 1, .1 +# print(f.simulate(r"C:\mmpe\HAWC2\bin\HAWC2_12.8\hawc2mb.exe")) +# +# f.save(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT.htc") diff --git a/pydatview/io/wetb/hawc2/htc_file_set.py b/pydatview/io/wetb/hawc2/htc_file_set.py new file mode 100644 index 0000000..990f52b --- /dev/null +++ b/pydatview/io/wetb/hawc2/htc_file_set.py @@ -0,0 +1,55 @@ +import glob +import os +import copy +from wetb.hawc2.hawc2_pbs_file import JESS_WINE32_HAWC2MB +from wetb.hawc2.htc_file import HTCFile +from wetb.utils.cluster_tools.pbsfile import PBSMultiRunner + + +class HTCFileSet(): + def __init__(self, model_path, htc_lst="**/*.htc"): + self.model_path = model_path + + if not isinstance(htc_lst, list): + htc_lst = [htc_lst] + + self.htc_files = [] + for htc_path in htc_lst: + if os.path.isfile(htc_path): + self.htc_files.append(htc_path) + else: + if not os.path.isabs(htc_path): + htc_path = os.path.join(model_path, htc_path) + for filename in glob.iglob(htc_path, recursive=True): + self.htc_files.append(filename) + + def pbs_files(self, hawc2_path, hawc2_cmd, queue='workq', walltime=None, + input_files=None, output_files=None, copy_turb=(True, True)): + + return (HTCFile(htc).pbs_file(hawc2_path, hawc2_cmd, queue=queue, walltime=walltime, + input_files=copy.copy(input_files), + output_files=copy.copy(output_files), + copy_turb=copy_turb) for htc in self.htc_files) + + def save_pbs_files(self, hawc2_path=None, hawc2_cmd=JESS_WINE32_HAWC2MB, queue='workq', walltime=None, + input_files=None, output_files=None, copy_turb=(True, True)): + for pbs in self.pbs_files(hawc2_path, hawc2_cmd, queue=queue, walltime=walltime, + input_files=input_files, output_files=output_files, + copy_turb=copy_turb): + pbs.save(self.model_path) + + +if __name__ == '__main__': + #model_path = r'R:\HAWC2_tests\v12.6_mmpe3\win32\simple1' + model_path = "w:/simple1" + pbs_files = HTCFileSet(model_path).pbs_files( + hawc2_path=r"R:\HAWC2_tests\v12.6_mmpe3\hawc2\win32", hawc2_cmd=JESS_WINE32_HAWC2MB, input_files=['data/*']) + import pandas as pd + time_overview = pd.read_excel( + r'C:\mmpe\programming\Fortran\HAWC2_git\HAWC2\pytest_hawc2\release_tests\Time_overview.xlsx') + for pbs in pbs_files: + f = pbs.filename + + pbs.walltime = time_overview.loc[f[:-3].replace("pbs_in/", 'simple1/')]['mean'] * 24 * 3600 + pbs.save(model_path) + PBSMultiRunner(model_path, nodes=1, ppn=10).save() diff --git a/pydatview/io/wetb/hawc2/pc_file.py b/pydatview/io/wetb/hawc2/pc_file.py new file mode 100644 index 0000000..b91655d --- /dev/null +++ b/pydatview/io/wetb/hawc2/pc_file.py @@ -0,0 +1,170 @@ +''' +Created on 24/04/2014 + +@author: MMPE +''' +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import absolute_import +from io import open +from builtins import range +from builtins import int +from future import standard_library +standard_library.install_aliases() + +import os +import numpy as np + +class PCFile(object): + """Read HAWC2 PC (profile coefficients) file + + examples + -------- + >>> pcfile = PCFile("tests/test_files/NREL_5MW_pc.txt") + >>> pcfile.CL(21,10) # CL for thickness 21% and AOA=10deg + 1.358 + >>> pcfile.CD(21,10) # CD for thickness 21% and AOA=10deg + 0.0255 + >>> pcfile.CM(21,10) # CM for thickness 21% and AOA=10deg + -0.1103 + """ + def __init__(self, filename=None): + self.pc_sets = {} + if filename is not None: + with open (filename) as fid: + lines = fid.readlines() + self._parse_lines(lines) + self.filename = filename + self.fmt = ' 19.015e' + + def _parse_lines(self, lines): + """Read HAWC2 PC file (profile coefficient file). + """ + nsets = int(lines[0].split()[0]) + lptr = 1 + for nset in range(1, nsets + 1): + nprofiles = int(lines[lptr].split()[0]) + lptr += 1 + #assert nprofiles >= 2 + thicknesses = [] + profiles = [] + for profile_nr in range(nprofiles): + profile_nr, n_rows, thickness = lines[lptr ].split()[:3] + profile_nr, n_rows, thickness = int(profile_nr), int(n_rows), float(thickness) + lptr += 1 + data = np.array([[float(v) for v in l.split()[:4]] for l in lines[lptr:lptr + n_rows]]) + thicknesses.append(thickness) + profiles.append(data) + lptr += n_rows + self.pc_sets[nset] = (np.array(thicknesses), profiles) + + def _Cxxx(self, thickness, alpha, column, pc_set_nr=1): + thicknesses, profiles = self.pc_sets[pc_set_nr] + index = np.searchsorted(thicknesses, thickness) + if index == 0: + index = 1 + + Cx0, Cx1 = profiles[index - 1:index + 1] + Cx0 = np.interp(alpha, Cx0[:, 0], Cx0[:, column]) + Cx1 = np.interp(alpha, Cx1[:, 0], Cx1[:, column]) + th0, th1 = thicknesses[index - 1:index + 1] + return Cx0 + (Cx1 - Cx0) * (thickness - th0) / (th1 - th0) + + def _CxxxH2(self, thickness, alpha, column, pc_set_nr=1): + thicknesses, profiles = self.pc_sets[pc_set_nr] + index = np.searchsorted(thicknesses, thickness) + if index == 0: + index = 1 + + Cx0, Cx1 = profiles[index - 1:index + 1] + + Cx0 = np.interp(np.arange(360), Cx0[:,0]+180, Cx0[:,column]) + Cx1 = np.interp(np.arange(360), Cx1[:,0]+180, Cx1[:,column]) + #Cx0 = np.interp(alpha, Cx0[:, 0], Cx0[:, column]) + #Cx1 = np.interp(alpha, Cx1[:, 0], Cx1[:, column]) + th0, th1 = thicknesses[index - 1:index + 1] + cx = Cx0 + (Cx1 - Cx0) * (thickness - th0) / (th1 - th0) + return np.interp(alpha+180, np.arange(360), cx) + + def CL(self, thickness, alpha, pc_set_nr=1): + """Lift coefficient + + Parameters + --------- + thickness : float + thickness [5] + alpha : float + Angle of attack [deg] + pc_set_nr : int optional + pc set number, default is 1, normally obtained from ae-file + + Returns + ------- + Lift coefficient : float + """ + return self._Cxxx(thickness, alpha, 1, pc_set_nr) + + def CL_H2(self, thickness, alpha, pc_set_nr=1): + return self._CxxxH2(thickness, alpha, 1, pc_set_nr) + + def CD(self, thickness, alpha, pc_set_nr=1): + """Drag coefficient + + Parameters + --------- + radius : float + radius [m] + alpha : float + Angle of attack [deg] + pc_set_nr : int optional + pc set number, default is 1, normally obtained from ae-file + + Returns + ------- + Drag coefficient : float + """ + return self._Cxxx(thickness, alpha, 2, pc_set_nr) + + def CM(self, thickness, alpha, pc_set_nr=1): + return self._Cxxx(thickness, alpha, 3, pc_set_nr) + + def __str__(self, comments=None): + """This method will create a string that is formatted like a pc file + with the data in this class. + """ + + if comments is None: + comments = {} + + cols = ['Angle of Attac', 'cl', 'cd', 'cm'] + linefmt = ' '.join(['{%i:%s}' % (i, self.fmt) for i in range(len(cols))]) + + n_sets = len(self.pc_sets) + retval = str(n_sets) + '\n' + for idx_pc, (set_tcs, set_pcs) in self.pc_sets.items(): + retval += str(len(set_tcs)) + '\n' + for i, (tc, pc) in enumerate(zip(set_tcs, set_pcs)): + nr = pc.shape[0] + retval += '%i %i %1.08f\n' % (i+1, nr, tc) + for line in pc: + retval += linefmt.format(*line) + '\n' + return retval + + def save(self, filename): + if not os.path.isdir(os.path.dirname(filename)): + os.makedirs(os.path.dirname(filename)) + with open(filename, 'w') as fid: + fid.write(str(self)) + self.filename = filename + + +if __name__ == "__main__": + pcfile = PCFile("tests/test_files/NREL_5MW_pc.txt") + + print (pcfile.CL(21,10)) # CL for thickness 21% and AOA=10deg + #1.358 + print (pcfile.CD(21,10)) # CD for thickness 21% and AOA=10deg + #0.0255 + print (pcfile.CM(21,10)) # CM for thickness 21% and AOA=10deg + #-0.1103 diff --git a/pydatview/io/wetb/hawc2/st_file.py b/pydatview/io/wetb/hawc2/st_file.py new file mode 100644 index 0000000..8fce4b5 --- /dev/null +++ b/pydatview/io/wetb/hawc2/st_file.py @@ -0,0 +1,274 @@ +''' +Created on 24/04/2014 + +@author: MMPE +''' +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from io import open +from builtins import range +from builtins import int +from future import standard_library +import types +standard_library.install_aliases() +import os +import numpy as np + + +stc = "r m x_cg y_cg ri_x ri_y x_sh y_sh E G I_x I_y I_p k_x k_y A pitch x_e y_e" + + +class StFile(object): + """Read HAWC2 St (beam element structural data) file + + Methods are autogenerated for: + + - r : curved length distance from main_body node 1 [m] + - m : mass per unit length [kg/m] + - x_cg : xc2-coordinate from C1/2 to mass center [m] + - y_cg : yc2-coordinate from C1/2 to mass center [m] + - ri_x : radius of gyration related to elastic center. Corresponds to rotation about principal bending xe axis [m] + - ri_y : radius of gyration related to elastic center. Corresponds to rotation about principal bending ye axis [m] + - xs : xc2-coordinate from C1/2 to shear center [m]. The shear center is the point where external forces only contributes to pure bending and no torsion. + - ys : yc2-coordinate from C1/2 to shear center [m]. The shear center is the point where external forces only contributes to pure bending and no torsion. + - E : modulus of elasticity [N/m2] + - G : shear modulus of elasticity [N/m2] + - Ix : area moment of inertia with respect to principal bending xe axis [m4]. This is the principal bending axis most parallel to the xc2 axis + - Iy : area moment of inertia with respect to principal bending ye axis [m4] + - K : torsional stiffness constant with respect to ze axis at the shear center [m4/rad]. For a circular section only this is identical to the polar moment of inertia. + - k_x : shear factor for force in principal bending xe direction [-] + - k_y : shear factor for force in principal bending ye direction [-] + - A : cross sectional area [m2] + - pitch : structural pitch about z_c2 axis. This is the angle between the xc2 -axis defined with the c2_def command and the main principal bending axis xe. + - xe : xc2-coordinate from C1/2 to center of elasticity [m]. The elastic center is the point where radial force (in the z-direction) does not contribute to bending around the x or y directions. + - ye : yc2-coordinate from C1/2 to center of elasticity [m]. The elastic center is the point where radial force (in the + + The autogenerated methods have the following structure + + def xxx(radius=None, mset=1, set=1): + Parameters: + ----------- + radius : int, float, array_like or None, optional + Radius/radii of interest\n + If int, float or array_like: values are interpolated to requested radius/radii + If None (default): Values of all radii specified in st file returned + mset : int, optional + Main set number + set : int, optional + Sub set number + + + Examples + -------- + >>> stfile = StFile(r"tests/test_files/DTU_10MW_RWT_Blade_st.dat") + >>> print (stfile.m()) # Mass at nodes + [ 1189.51054664 1191.64291781 1202.76694262 ... 15.42438683] + >>> print (st.E(radius=36, mset=1, set=1)) # Elasticity interpolated to radius 36m + 8722924514.652649 + >>> print (st.E(radius=36, mset=1, set=2)) # Same for stiff blade set + 8.722924514652648e+17 + """ + + cols = stc.split() + + def __init__(self, filename=None): + + # in case the user wants to create a new non-existing st file + if filename is None: + self.main_data_sets = {} + return + + with open(filename) as fid: + txt = fid.read() +# Some files starts with first set ("#1...") with out specifying number of sets +# no_maindata_sets = int(txt.strip()[0]) +# assert no_maindata_sets == txt.count("#") + self.main_data_sets = {} + for mset in txt.split("#")[1:]: + mset_nr = int(mset.strip().split()[0]) + set_data_dict = {} + + for set_txt in mset.split("$")[1:]: + set_lines = set_txt.split("\n") + set_nr, no_rows = map(int, set_lines[0].split()[:2]) + assert set_nr not in set_data_dict + set_data_dict[set_nr] = np.array([set_lines[i].split() for i in range(1, no_rows + 1)], dtype=float) + self.main_data_sets[mset_nr] = set_data_dict + + for i, name in enumerate(self.cols): + setattr(self, name, lambda radius=None, mset=1, set=1, + column=i: self._value(radius, column, mset, set)) + + def _value(self, radius, column, mset_nr=1, set_nr=1): + st_data = self.main_data_sets[mset_nr][set_nr] + if radius is None: + radius = self.radius_st(None, mset_nr, set_nr) + return np.interp(radius, st_data[:, 0], st_data[:, column]) + + def radius_st(self, radius=None, mset=1, set=1): + r = self.main_data_sets[mset][set][:, 0] + if radius is None: + return r + return r[np.argmin(np.abs(r - radius))] + + def to_str(self, mset=1, set=1, precision='%12.5e '): + d = self.main_data_sets[mset][set] + return '\n'.join([(precision * d.shape[1]) % tuple(row) for row in d]) + + def set_value(self, mset_nr, set_nr, **kwargs): + for k, v in kwargs.items(): + column = self.cols.index(k) + self.main_data_sets[mset_nr][set_nr][:, column] = v + + def save(self, filename, precision='%15.07e', encoding='utf-8'): + """Save all data defined in main_data_sets to st file. + """ + colwidth = len(precision % 1) + sep = '=' * colwidth * len(self.cols) + '\n' + colhead = ''.join([k.center(colwidth) for k in self.cols]) + '\n' + + nsets = len(self.main_data_sets) + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, 'w', encoding=encoding) as fid: + fid.write('%i ; number of sets, Nset\n' % nsets) + for mset, set_data_dict in self.main_data_sets.items(): + fid.write('#%i ; set number\n' % mset) + for set, set_array in set_data_dict.items(): + dstr = self.to_str(mset=mset, set=set, precision=precision) + npoints = self.main_data_sets[mset][set].shape[0] + fid.write(sep + colhead + sep) + fid.write('$%i %i\n' % (set, npoints)) + fid.write(dstr + '\n') + + def element_stiffnessmatrix(self, radius, mset_nr, set_nr, length): + """Compute the element stiffness matrix + + Parameters + ---------- + radius : float + radius of element (used of obtain element properties + length : float + eleement length + """ + K = np.zeros((13, 13)) + "r m x_cg y_cg ri_x ri_y x_sh y_sh E G I_x I_y I_p k_x k_y A pitch x_e y_e" + ES1, ES2, EMOD, GMOD, IX, IY, IZ, KX, KY, A = [getattr(self, n)(radius, mset_nr, set_nr) + for n in "x_sh,y_sh,E,G,I_x,I_y,I_p,k_x,k_y,A".split(",")] + ELLGTH = length + + ETAX = EMOD * IX / (KY * GMOD * A * ELLGTH**2) + ETAY = EMOD * IY / (KX * GMOD * A * ELLGTH**2) + ROX = 1 / (1 + 12 * ETAX) + ROY = 1 / (1 + 12 * ETAY) + ROY = 1 / (1 + 12 * ETAX) + K[1, 1] = 12 * EMOD * IY * ROY / ELLGTH**3 + K[1, 5] = 6 * EMOD * IY * ROY / ELLGTH**2 + K[1, 6] = -K[1, 1] * ES2 + K[1, 7] = -K[1, 1] + K[1, 11] = K[1, 5] + K[1, 12] = -K[1, 6] + K[2, 2] = 12 * EMOD * IX * ROX / ELLGTH**3 + K[2, 4] = -6 * EMOD * IX * ROX / ELLGTH**2 + K[2, 6] = K[2, 2] * ES1 + K[2, 8] = -K[2, 2] + K[2, 10] = K[2, 4] + K[2, 12] = -K[2, 6] + K[3, 3] = A * EMOD / ELLGTH + K[3, 9] = -K[3, 3] + K[4, 4] = 4 * EMOD * IX * (1 + 3 * ETAX) * ROX / ELLGTH + K[4, 6] = K[2, 4] * ES1 + K[4, 8] = -K[2, 4] + K[4, 10] = 2 * EMOD * IX * (1 - 6 * ETAX) * ROX / ELLGTH + K[4, 12] = -K[4, 6] + K[5, 5] = 4 * EMOD * IY * (1 + 3 * ETAY) * ROY / ELLGTH + K[5, 6] = -K[1, 5] * ES2 + K[5, 7] = -K[1, 5] + K[5, 11] = 2 * EMOD * IY * (1 - 6 * ETAY) * ROY / ELLGTH + K[5, 12] = -K[5, 6] + K[6, 6] = GMOD * IZ / ELLGTH + 12 * EMOD * (IX * ES1**2 * ROX + IY * ES2**2 * ROY) / ELLGTH**3 + K[6, 7] = K[1, 12] + K[6, 8] = K[2, 12] + K[6, 10] = -K[4, 12] + K[6, 11] = -K[5, 12] + K[6, 12] = -K[6, 6] + K[7, 7] = K[1, 1] + K[7, 11] = -K[1, 5] + K[7, 12] = K[1, 6] + K[8, 8] = K[2, 2] + K[8, 10] = -K[2, 4] + K[8, 12] = K[2, 6] + K[9, 9] = K[3, 3] + K[10, 10] = K[4, 4] + K[10, 12] = -K[4, 6] + K[11, 11] = K[5, 5] + K[11, 12] = -K[5, 6] + K[12, 12] = K[6, 6] + K = K[1:, 1:] + K = K + K.T - np.eye(12) * K + return K + + def shape_function_ori(self, radius, mset_nr, set_nr, length, z): + XSC, YSC, EMOD, GMOD, IX, IY, IZ, KX, KY, AREA = [getattr(self, n)(radius, mset_nr, set_nr) + for n in "x_sh,y_sh,E,G,I_x,I_y,I_p,k_x,k_y,A".split(",")] + + etax = EMOD * IX / KY / GMOD / AREA / (length**2) + etay = EMOD * IY / KX / GMOD / AREA / (length**2) + rhox = 1 / (1 + 12 * etax) + rhoy = 1 / (1 + 12 * etay) + + f1 = z / length + f2 = 1 - f1 + f3x = f1 * (3 * f1 - 2 * f1**2 + 12 * etax) * rhox + f4x = f2 * (3 * f2 - 2 * f2**2 + 12 * etax) * rhox + f5x = length * (f1**2 - (1 - 6 * etax) * f1 - 6 * etax) * f1 * rhox + f6x = length * (f2**2 - (1 - 6 * etax) * f2 - 6 * etax) * f2 * rhox + f7x = 6 / length * f1 * f2 * rhox + f8x = f1 * (3 * f1 - 2 * (1 - 6 * etax)) * rhox + f9x = f2 * (3 * f2 - 2 * (1 - 6 * etax)) * rhox + + f3y = f1 * (3 * f1 - 2 * f1**2 + 12 * etay) * rhoy + f4y = f2 * (3 * f2 - 2 * f2**2 + 12 * etay) * rhoy + f5y = length * (f1**2 - (1 - 6 * etay) * f1 - 6 * etay) * f1 * rhoy + f6y = length * (f2**2 - (1 - 6 * etay) * f2 - 6 * etay) * f2 * rhoy + f7y = 6 / length * f1 * f2 * rhoy + f8y = f1 * (3 * f1 - 2 * (1 - 6 * etay)) * rhoy + f9y = f2 * (3 * f2 - 2 * (1 - 6 * etay)) * rhoy + + return np.array([[f4y, 0, 0, 0, -f7y, 0], + [0, f4x, 0, f7x, 0, 0], + [0, 0, f2, 0, 0, 0], + [0, f6x, 0, f9x, 0, 0], + [-f6y, 0, 0, 0, f9y, 0], + [(f2 - f4y) * YSC, -(f2 - f4x) * XSC, 0, f7x * XSC, f7y * YSC, f2], + [f3y, 0, 0, 0, f7y, 0], + [0, f3x, 0, -f7x, 0, 0], + [0, 0, f1, 0, 0, 0], + [0, -f5x, 0, f8x, 0, 0], + [f5y, 0, 0, 0, f8y, 0], + [(f1 - f3y) * YSC, -(f1 - f3x) * XSC, 0, -f7x * XSC, -f7y * YSC, f1]]).T + + +if __name__ == "__main__": + import os + cwd = os.path.dirname(__file__) + st = StFile(os.path.join(cwd, r'tests/test_files/DTU_10MW_RWT_Blade_st.dat')) + print(st.m()) + print(st.E(radius=36, mset=1, set=1)) # Elastic blade + print(st.E(radius=36, mset=1, set=2)) # stiff blade + #print (st.radius()) + xyz = np.array([st.x_e(), st.y_e(), st.r()]).T[:40] + n = 2 + xyz = np.array([st.x_e(None, 1, n), st.y_e(None, 1, n), st.r(None, 1, n)]).T[:40] + #print (xyz) + print(np.sqrt(np.sum((xyz[1:] - xyz[:-1]) ** 2, 1)).sum()) + print(xyz[-1, 2]) + print(np.sqrt(np.sum((xyz[1:] - xyz[:-1]) ** 2, 1)).sum() - xyz[-1, 2]) + print(st.x_e(67.8883), st.y_e(67.8883)) + #print (np.sqrt(np.sum(np.diff(xyz, 0) ** 2, 1))) + print(st.pitch(67.8883 - 0.01687)) + print(st.pitch(23.2446)) + + # print (st.) + # print (st.) diff --git a/pydatview/main.py b/pydatview/main.py index 483b880..44628e1 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -35,21 +35,9 @@ # Helper from .common import * from .GUICommon import * +import pydatview.io as weio # File Formats and File Readers # Pluggins from .plugins import dataPlugins - -try: - import weio.weio as weio# File Formats and File Readers -except: - print('') - print('Error: the python package `weio` was not imported successfully.\n') - print('Most likely the submodule `weio` was not cloned with `pyDatView`') - print('Type the following command to retrieve it:\n') - print(' git submodule update --init --recursive\n') - print('Alternatively re-clone this repository into a separate folder:\n') - print(' git clone --recurse-submodules https://github.com/ebranlard/pyDatView\n') - sys.exit(-1) - from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData # --------------------------------------------------------------------------------} @@ -165,7 +153,8 @@ def __init__(self, data=None): self.Bind(wx.EVT_MENU,self.onReset, resetMenuItem) - self.FILE_FORMATS, errors= weio.fileFormats(ignoreErrors=True, verbose=False) + io_userpath = os.path.join(weio.defaultUserDataDir(), 'pydatview_io') + self.FILE_FORMATS, errors= weio.fileFormats(userpath=io_userpath, ignoreErrors=True, verbose=False) if len(errors)>0: for e in errors: Warn(self, e) @@ -533,10 +522,10 @@ def onSave(self, event=None): self.plotPanel.navTB.save_figure() def onAbout(self, event=None): - defaultDir = weio.defaultUserDataDir() # TODO input file options + io_userpath = os.path.join(weio.defaultUserDataDir(), 'pydatview_io') About(self,PROG_NAME+' '+PROG_VERSION+'\n\n' 'pyDatView config file:\n {}\n'.format(configFilePath())+ - 'weio data directory: \n {}\n'.format(os.path.join(defaultDir,'weio'))+ + 'pyDatView io data directory:\n {}\n'.format(io_userpath)+ '\n\nVisit http://github.com/ebranlard/pyDatView for documentation.') def onReset (self, event=None): diff --git a/requirements.txt b/requirements.txt index bf32bf6..aad0a7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,5 @@ pyarrow # for parquet files matplotlib future chardet -wxpython scipy +wxpython diff --git a/setup.py b/setup.py index 41c2544..f8dd4f5 100644 --- a/setup.py +++ b/setup.py @@ -12,14 +12,3 @@ zip_safe=False ) -setup( - name='weio', - version='1.0', - description='Library to read and write files', - url='http://github.com/elmanuelito/weio/', - author='Emmanuel Branlard', - author_email='lastname@gmail.com', - license='MIT', - packages=['weio'], - zip_safe=False -) diff --git a/tests/prof_all.py b/tests/prof_all.py index 3fcce30..66d187a 100644 --- a/tests/prof_all.py +++ b/tests/prof_all.py @@ -74,7 +74,7 @@ def test_heavy(): sys.path.append(root_dir) # print(root_dir) #filenames=['../_TODO/DLC120_ws13_yeNEG_s2_r3_PIT.SFunc.outb','../_TODO/DLC120_ws13_ye000_s1_r1.SFunc.outb'] -# filenames=['../weio/_tests/CSVComma.csv'] +# filenames=['../example_files/CSVComma.csv'] # filenames =[os.path.join(script_dir,f) for f in filenames] #pydatview.test(filenames=filenames) diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 0b11459..0e66e60 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -44,8 +44,8 @@ def test_table_name(self): def test_load_files_misc_formats(self): tablist = TableList() files =[ - os.path.join(self.scriptdir,'../weio/weio/tests/example_files/CSVComma.csv'), - os.path.join(self.scriptdir,'../weio/weio/tests/example_files/HAWCStab2.pwr') + os.path.join(self.scriptdir,'../example_files/CSVComma.csv'), + os.path.join(self.scriptdir,'../example_files/HAWCStab2.pwr') ] # --- First read without fileformats tablist.load_tables_from_files(filenames=files, fileformats=None, bAdd=False) diff --git a/weio b/weio deleted file mode 160000 index 8f5b2b8..0000000 --- a/weio +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8f5b2b8a387789910f6fc59b0f9466c3a160b4b6 From bc21b20bf6c496d836bc0eb513d29cac5d9e5289 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 22 Jun 2022 16:08:42 -0600 Subject: [PATCH 006/178] Switch to python 3.9 for setup exe (#115) --- installer.cfg | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/installer.cfg b/installer.cfg index 6f49f55..a3f4d78 100644 --- a/installer.cfg +++ b/installer.cfg @@ -8,7 +8,7 @@ icon=ressources/pyDatView.ico #entry_point=pydatview:cmdline [Python] -version=3.6.0 +version=3.9.9 bitness=64 [Include] @@ -17,19 +17,35 @@ files=_tools/pyDatView.exe > $INSTDIR LICENSE.TXT > $INSTDIR pypi_wheels = - numpy==1.19.3 - wxPython==4.0.3 - matplotlib==3.0.0 - pyparsing==2.2.2 - cycler==0.10.0 - six==1.11.0 - python-dateutil==2.7.3 - kiwisolver==1.0.1 - pandas==0.23.4 - pytz==2018.5 - chardet==3.0.4 - scipy==1.1.0 - pyarrow==4.0.1 + numpy==1.22.4 + wxPython==4.1.1 + matplotlib==3.5.2 + pyparsing==2.4.7 + cycler==0.11.0 + six==1.16.0 + python-dateutil==2.8.2 + kiwisolver==1.3.2 + pandas==1.4.2 + pytz==2021.3 + chardet==4.0.0 + scipy==1.8.1 + pyarrow==8.0.0 + Pillow==9.1.1 + packaging==21.2 + +# numpy==1.19.3 +# wxPython==4.0.3 +# matplotlib==3.0.0 +# pyparsing==2.2.2 +# cycler==0.10.0 +# six==1.11.0 +# python-dateutil==2.7.3 +# kiwisolver==1.0.1 +# pandas==0.23.4 +# pytz==2018.5 +# chardet==3.0.4 +# scipy==1.1.0 +# pyarrow==4.0.1 # numpy==1.20.3 # wxPython==4.0.7 From f1e9c507aec32d0db3cc92f254934c5ea3bff831 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 22 Jun 2022 16:18:13 -0600 Subject: [PATCH 007/178] Adding one more path to exe lookup --- _tools/Makefile | 2 +- _tools/pyDatView.c | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/_tools/Makefile b/_tools/Makefile index b813bfe..44a5348 100644 --- a/_tools/Makefile +++ b/_tools/Makefile @@ -11,7 +11,7 @@ compile: test: @echo -------------------------------------------------------------------------- - ./pyDatView.exe C:\Bin\test.outb BB CC + ./pyDatView.exe ../example_files/HAWCStab2.pwr ../example_files/CSVComma.csv clean: rm *.obj *.res pyDatView.exe diff --git a/_tools/pyDatView.c b/_tools/pyDatView.c index c66b963..60a8672 100644 --- a/_tools/pyDatView.c +++ b/_tools/pyDatView.c @@ -91,6 +91,7 @@ int main (int argc, char** argv) { char pythonwpath2[MAX_PATH]=""; char pythonwpath3[MAX_PATH]=""; char pythonwpath4[MAX_PATH]=""; + char pythonwpath5[MAX_PATH]=""; char* pfullCommand ; bool useImport = true; int index=0; @@ -146,6 +147,12 @@ int main (int argc, char** argv) { concatenate(pythonwpath3,"\\AppData\\Local\\pyDatView\\Python\\pythonw.exe"); printf("Pythonw3 : %s\n", pythonwpath3); + // --- Pythonw path (assuming user installed in AppData) + concatenate(pythonwpath5,"C:\\Users\\"); + concatenate(pythonwpath5,user); + concatenate(pythonwpath5,"\\AppData\\Local\\Programs\\pyDatView\\Python\\pythonw.exe"); + printf("Pythonw5 : %s\n", pythonwpath5); + // --- Pythonw path (using PYDATPATH env variable) if (pydatpath) { concatenate(pythonwpath4, pydatpath); @@ -153,8 +160,6 @@ int main (int argc, char** argv) { printf("Pythonw4 : %s\n", pythonwpath4); } - - // --- Selecting pythonw path that exist if (strcmp(parentdir,"_tools")==0) { exedir[strlen(exedir) - 7] = '\0'; // remove pyDatView.exe from path @@ -175,6 +180,10 @@ int main (int argc, char** argv) { concatenate(pythonwpath0, pythonwpath4); printf(">>> Using Pythonw4\n"); + } else if (file_exists(pythonwpath5)) { + concatenate(pythonwpath0, pythonwpath5); + printf(">>> Using Pythonw5\n"); + } else { ShowWindow( hWnd, SW_RESTORE); printf("\n"); From 7e047040c79fe039841ac26c68fcfb06695373f6 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 22 Jun 2022 16:20:02 -0600 Subject: [PATCH 008/178] Adding openpyxl to installer (#115) --- installer.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/installer.cfg b/installer.cfg index a3f4d78..1995c75 100644 --- a/installer.cfg +++ b/installer.cfg @@ -29,6 +29,8 @@ pypi_wheels = pytz==2021.3 chardet==4.0.0 scipy==1.8.1 + openpyxl==3.0.10 + et-xmlfile==1.1.0 pyarrow==8.0.0 Pillow==9.1.1 packaging==21.2 From c8ce1d38aebfcf9408b4224e34dfc3de9b920cd0 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 22 Jun 2022 16:32:12 -0600 Subject: [PATCH 009/178] GH: deploy using python 3.9 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8425cb1..22685b3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.8] # 2.7, + python-version: [3.6, 3.8, 3.9] steps: # --- Install steps @@ -81,7 +81,7 @@ jobs: echo "PY_VERSION : $PY_VERSION" export OK=0 # Only deploy for push events - if [[ $PY_VERSION == "3.6" ]]; then + if [[ $PY_VERSION == "3.9" ]]; then if [[ $GH_EVENT == "push" ]]; then export OK=1 ; fi From 56b73b97811e9af67b85b50492bef6a4c121dd93 Mon Sep 17 00:00:00 2001 From: SimonHH <38310787+SimonHH@users.noreply.github.com> Date: Thu, 23 Jun 2022 08:11:37 +0200 Subject: [PATCH 010/178] Add MAX_X_COLUMNS --- pydatview/GUISelectionPanel.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 82bd112..d5bbaa9 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -16,6 +16,7 @@ SEL_MODES = ['auto','Same tables' ,'Sim. tables' ,'2 tables','3 tables (exp.)' ] SEL_MODES_ID = ['auto','sameColumnsMode','simColumnsMode','twoColumnsMode' ,'threeColumnsMode' ] +MAX_X_COLUMNS=300 # Maximum number of columns used in combo box of the x-axis (for performance) def ireplace(text, old, new): """ Replace case insensitive """ @@ -755,11 +756,11 @@ def setGUIColumns(self, xSel=-1, ySel=[]): # NOTE: limiting to 300 for now.. I'm not sure anywant would want to scroll more than that # Consider adding a "more button" # see e.g. https://comp.soft-sys.wxwindows.narkive.com/gDfA1Ds5/long-load-time-in-wxpython - if self.comboX.GetCurrentSelection()==300: + if self.comboX.GetCurrentSelection()==MAX_X_COLUMNS: self.comboX.Set(columnsX) else: - if len(columnsX)>300: - columnsX_show=np.append(columnsX[:300],'[...]') + if len(columnsX)>MAX_X_COLUMNS: + columnsX_show=np.append(columnsX[:MAX_X_COLUMNS],'[...]') else: columnsX_show=columnsX self.comboX.Set(columnsX_show) # non filtered @@ -820,7 +821,7 @@ def getColumnSelection(self): iXFull = iX # NOTE: x is always in full IYFull = [self.Filt2Full[iY] for iY in IY] - if self.comboX.GetCurrentSelection()==300: + if self.comboX.GetCurrentSelection()==MAX_X_COLUMNS: self.setGUIColumns(xSel=iXFull, ySel=IYFull) return iXFull,IYFull,sX,SY From b74c94eccf6819d68913ffc3423be4e3eaf12480 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 23 Jun 2022 08:34:48 -0600 Subject: [PATCH 011/178] Fix: adding files using open and add buttons (Fixes #119) --- pydatview/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pydatview/main.py b/pydatview/main.py index 44628e1..70709fa 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -590,8 +590,8 @@ def selectFile(self,bAdd=False): wildcard='|'.join([n+'|*'+';*'.join(e) for n,e in zip(self.FILE_FORMATS_NAMEXT,self.FILE_FORMATS_EXTENSIONS)]) #wildcard = sFormat + extensions+'|all (*.*)|*.*' else: - Format = FILE_FORMATS[iFormat-1] - extensions = '|*'+';*'.join(FILE_FORMATS[iFormat-1].extensions) + Format = self.FILE_FORMATS[iFormat-1] + extensions = '|*'+';*'.join(self.FILE_FORMATS[iFormat-1].extensions) wildcard = sFormat + extensions+'|all (*.*)|*.*' with wx.FileDialog(self, "Open file", wildcard=wildcard, @@ -601,7 +601,8 @@ def selectFile(self,bAdd=False): #dlg.Center() if dlg.ShowModal() == wx.ID_CANCEL: return # the user changed their mind - self.load_files(dlg.GetPaths(),fileformat=Format,bAdd=bAdd) + filenames = dlg.GetPaths() + self.load_files(filenames,fileformats=[Format]*len(filenames),bAdd=bAdd) def onModeChange(self, event=None): if hasattr(self,'selPanel'): @@ -693,7 +694,7 @@ def test(filenames=None): if filenames is not None: app = wx.App(False) frame = MainFrame() - frame.load_files(filenames,fileformat=None) + frame.load_files(filenames,fileformats=None) return # --------------------------------------------------------------------------------} From e49189e471c6e6e972854fafa2f3d209b33871e0 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 23 Jun 2022 10:06:49 -0600 Subject: [PATCH 012/178] GH: different counts and release push on main branch --- .github/workflows/tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 22685b3..7707017 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,8 @@ jobs: id: versioning run: | git fetch --unshallow - export CURRENT_DEV_TAG="v0.3-dev" + export CURRENT_TAG="v0.3" + export CURRENT_DEV_TAG="$CURRENT_TAG-dev" echo "GIT DESCRIBE: `git describe`" echo "GITHUB_REF: $GITHUB_REF" # Check if current version corresponds to a tagged commit @@ -35,6 +36,11 @@ jobs: export VERSION_TAG=${GITHUB_REF/refs\/tags\//} export VERSION_NAME=$VERSION_TAG export FULL_VERSION_NAME="version $VERSION_NAME" + elif [[ $GITHUB_REF == *"main"* ]]; then + echo ">>> This is not a tagged version, but on the main branch" + export VERSION_TAG=$CURRENT_TAG + export VERSION_NAME="$CURRENT_TAG-`git rev-list $CURRENT_TAG.. --count`" + export FULL_VERSION_NAME="version $VERSION_NAME" else echo ">>> This is not a tagged version" export VERSION_TAG=$CURRENT_DEV_TAG ; From 267c5034ca7883e04d81d04721f555c34f688859 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 23 Jun 2022 10:16:32 -0600 Subject: [PATCH 013/178] GH: figuring out number of commits --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7707017..b86c37f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,6 +30,8 @@ jobs: export CURRENT_DEV_TAG="$CURRENT_TAG-dev" echo "GIT DESCRIBE: `git describe`" echo "GITHUB_REF: $GITHUB_REF" + echo "Commits: `git rev-list $CURRENT_TAG.. --count`" + echo "Commits-dev: `git rev-list $CURRENT_DEV_TAG.. --count`" # Check if current version corresponds to a tagged commit if [[ $GITHUB_REF == *"tags"* ]]; then echo ">>> This is a tagged version" From 6ffa5dd3ef93f128d0d8fd777c6f57f4adc8acdd Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 23 Jun 2022 10:35:12 -0600 Subject: [PATCH 014/178] GH: fetching all tags --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b86c37f..b921e10 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,6 +26,7 @@ jobs: id: versioning run: | git fetch --unshallow + git fetch --tags export CURRENT_TAG="v0.3" export CURRENT_DEV_TAG="$CURRENT_TAG-dev" echo "GIT DESCRIBE: `git describe`" From 173c4ab9a1f084505bf081667657b3ef1f800d92 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 29 Jun 2022 14:20:54 -0600 Subject: [PATCH 015/178] Renaming signal to signal_analysis --- pydatview/GUITools.py | 8 ++++---- pydatview/Tables.py | 4 ++-- pydatview/plotdata.py | 6 +++--- pydatview/tools/{signal.py => signal_analysis.py} | 10 +++++++++- 4 files changed, 18 insertions(+), 10 deletions(-) rename pydatview/tools/{signal.py => signal_analysis.py} (96%) diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 3ff9d27..c70f2bb 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -188,7 +188,7 @@ class FilterToolPanel(GUIToolPanel): I need to think of a better way to do that """ def __init__(self, parent): - from pydatview.tools.signal import FILTERS + from pydatview.tools.signal_analysis import FILTERS super(FilterToolPanel,self).__init__(parent) self.parent = parent # parent is GUIPlotPanel @@ -346,7 +346,7 @@ def onPlot(self, event=None): """ Overlay on current axis the filter """ - from pydatview.tools.signal import applyFilter + from pydatview.tools.signal_analysis import applyFilter if len(self.parent.plotData)!=1: Error(self,'Plotting only works for a single plot. Plot less data.') return @@ -436,7 +436,7 @@ def __init__(self, parent): super(ResampleToolPanel,self).__init__(parent) # --- Data from other modules - from pydatview.tools.signal import SAMPLERS + from pydatview.tools.signal_analysis import SAMPLERS self.parent = parent # parent is GUIPlotPanel self._SAMPLERS=SAMPLERS # Setting default states to parent @@ -598,7 +598,7 @@ def onAdd(self,event=None): raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) def onPlot(self,event=None): - from pydatview.tools.signal import applySampler + from pydatview.tools.signal_analysis import applySampler if len(self.parent.plotData)!=1: Error(self,'Plotting only works for a single plot. Plot less data.') return diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 3e940e1..c866761 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -436,7 +436,7 @@ def applyMaskString(self,maskString,bAdd=True): # --- Important manipulation TODO MOVE THIS OUT OF HERE OR UNIFY def applyResampling(self, iCol, sampDict, bAdd=True): - from pydatview.tools.signal import applySamplerDF + from pydatview.tools.signal_analysis import applySamplerDF if iCol==0: raise Exception('Cannot resample based on index') colName=self.data.columns[iCol-1] @@ -450,7 +450,7 @@ def applyResampling(self, iCol, sampDict, bAdd=True): return df_new, name_new def applyFiltering(self, iCol, options, bAdd=True): - from pydatview.tools.signal import applyFilterDF + from pydatview.tools.signal_analysis import applyFilterDF if iCol==0: raise Exception('Cannot filter based on index') colName=self.data.columns[iCol-1] diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index ac147da..0c2b37b 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -75,19 +75,19 @@ def _post_init(PD, Options={}): # TODO setup an "Order" if 'RemoveOutliers' in keys: if Options['RemoveOutliers']: - from pydatview.tools.signal import reject_outliers + from pydatview.tools.signal_analysis import reject_outliers try: PD.x, PD.y = reject_outliers(PD.y, PD.x, m=Options['OutliersMedianDeviation']) except: raise Exception('Warn: Outlier removal failed. Desactivate it or use a different signal. ') if 'Filter' in keys: if Options['Filter']: - from pydatview.tools.signal import applyFilter + from pydatview.tools.signal_analysis import applyFilter PD.y = applyFilter(PD.x, PD.y, Options['Filter']) if 'Sampler' in keys: if Options['Sampler']: - from pydatview.tools.signal import applySampler + from pydatview.tools.signal_analysis import applySampler PD.x, PD.y = applySampler(PD.x, PD.y, Options['Sampler']) if 'Binning' in keys: diff --git a/pydatview/tools/signal.py b/pydatview/tools/signal_analysis.py similarity index 96% rename from pydatview/tools/signal.py rename to pydatview/tools/signal_analysis.py index b8371a7..54fdab1 100644 --- a/pydatview/tools/signal.py +++ b/pydatview/tools/signal_analysis.py @@ -1,3 +1,8 @@ +""" +Signal analysis tools. +NOTE: naming this module "signal.py" can sometimes create conflict with numpy + +""" from __future__ import division import numpy as np from numpy.random import rand @@ -102,6 +107,9 @@ def interpArray(x, xp, fp, extrap='bounded'): xp = np.asarray(xp) assert fp.shape[1]==len(xp), 'Second dimension of fp should have the same length as xp' + if fp.shape[1]==0: + raise Exception('Second dimension of fp should be >0') + j = np.searchsorted(xp, x) - 1 if j<0: # Before bounds @@ -444,7 +452,7 @@ def sine_approx(t, x, method='least_square'): # --- Convolution # --------------------------------------------------------------------------------{ def convolution_integral(time, f, g, method='auto'): - """ + r""" Compute convolution integral: f * g = \int 0^t f(tau) g(t-tau) dtau = g * f For now, only works for uniform time vector, an exception is raised otherwise From 5ca1b30b03fd9664eee74f7776876ef9e263ef55 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 29 Jun 2022 14:23:03 -0600 Subject: [PATCH 016/178] --amend --- tests/test_signal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_signal.py b/tests/test_signal.py index 5d5f043..f70f6db 100644 --- a/tests/test_signal.py +++ b/tests/test_signal.py @@ -1,7 +1,7 @@ import unittest import numpy as np import pandas as pd -from pydatview.tools.signal import * +from pydatview.tools.signal_analysis import * # --------------------------------------------------------------------------------} # --- From 63d8ddfbfcca67573d35f4adc43f8d3835a963ea Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 14 Jul 2022 20:13:46 -0600 Subject: [PATCH 017/178] Update of weio --- pydatview/fast/case_gen.py | 9 +- pydatview/fast/fastfarm.py | 1 + pydatview/fast/postpro.py | 39 +- pydatview/fast/runner.py | 2 - pydatview/io/fast_input_file.py | 3116 ++++++++++++--------- pydatview/io/fast_linearization_file.py | 702 ++--- pydatview/io/fast_output_file.py | 996 +++---- pydatview/io/hawc2_ae_file.py | 162 +- pydatview/io/hawc2_htc_file.py | 249 +- pydatview/io/hawc2_pc_file.py | 174 +- pydatview/io/wetb/hawc2/Hawc2io.py | 705 ++--- pydatview/io/wetb/hawc2/ae_file.py | 42 +- pydatview/io/wetb/hawc2/htc_contents.py | 24 +- pydatview/io/wetb/hawc2/htc_extensions.py | 9 - pydatview/io/wetb/hawc2/htc_file.py | 1177 ++++---- pydatview/io/wetb/hawc2/pc_file.py | 13 +- pydatview/io/wetb/hawc2/st_file.py | 56 +- pydatview/tools/curve_fitting.py | 2 +- pydatview/tools/stats.py | 19 +- 19 files changed, 4062 insertions(+), 3435 deletions(-) diff --git a/pydatview/fast/case_gen.py b/pydatview/fast/case_gen.py index 4b7076e..ca32f18 100644 --- a/pydatview/fast/case_gen.py +++ b/pydatview/fast/case_gen.py @@ -12,10 +12,7 @@ import pydatview.io.fast_input_file as fi import pydatview.fast.runner as runner import pydatview.fast.postpro as postpro -#import pyFAST.input_output.fast_input_file as fi -#import pyFAST.case_generation.runner as runner -#import pyFAST.input_output.postpro as postpro - +from pydatview.io.fast_wind_file import FASTWndFile # --------------------------------------------------------------------------------} # --- Template replace @@ -445,7 +442,7 @@ def paramsLinearTrim(p=None): # --- # --------------------------------------------------------------------------------{ def createStepWind(filename,WSstep=1,WSmin=3,WSmax=25,tstep=100,dt=0.5,tmin=0,tmax=999): - f = weio.FASTWndFile() + f = FASTWndFile() Steps= np.arange(WSmin,WSmax+WSstep,WSstep) print(Steps) nCol = len(f.colNames) @@ -536,7 +533,7 @@ def CPCT_LambdaPitch(refdir,main_fastfile,Lambda=None,Pitch=np.linspace(-10,40,5 # --- Creating set of parameters to be changed # TODO: verify that RtAeroCp and RtAeroCt are present in AeroDyn outlist - PARAMS = paramsWS_RPM_Pitch(WS_flat,RPM_flat,Pitch_flat,baseDict=baseDict, FlatInputs=True) + PARAMS = paramsWS_RPM_Pitch(WS_flat,RPM_flat,Pitch_flat,baseDict=baseDict, flatInputs=True) # --- Generating all files in a workDir workDir = refdir.strip('/').strip('\\')+'_CPLambdaPitch' diff --git a/pydatview/fast/fastfarm.py b/pydatview/fast/fastfarm.py index 072f10e..1a05fa3 100644 --- a/pydatview/fast/fastfarm.py +++ b/pydatview/fast/fastfarm.py @@ -2,6 +2,7 @@ import glob import numpy as np import pandas as pd + from pydatview.io.fast_input_file import FASTInputFile from pydatview.io.fast_output_file import FASTOutputFile from pydatview.io.turbsim_file import TurbSimFile diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 3eb7f89..7ba88ce 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -6,12 +6,11 @@ import re # --- fast libraries +import pydatview.io as weio from pydatview.io.fast_input_file import FASTInputFile from pydatview.io.fast_output_file import FASTOutputFile from pydatview.io.fast_input_deck import FASTInputDeck -# from pyFAST.input_output.fast_input_file import FASTInputFile -# from pyFAST.input_output.fast_output_file import FASTOutputFile -# from pyFAST.input_output.fast_input_deck import FASTInputDeck + # --------------------------------------------------------------------------------} # --- Tools for IO @@ -1035,35 +1034,42 @@ def _zero_crossings(y,x=None,direction=None): raise Exception('Direction should be either `up` or `down`') return xzc, iBef, sign -def find_matching_pattern(List, pattern, sort=False): - """ Return elements of a list of strings that match a pattern +def find_matching_pattern(List, pattern, sort=False, integers=True): + r""" Return elements of a list of strings that match a pattern and return the first matching group + + Example: + + find_matching_pattern(['Misc','TxN1_[m]', 'TxN20_[m]'], 'TxN(\d+)_\[m\]') + returns: Matches = 1,20 """ reg_pattern=re.compile(pattern) MatchedElements=[] - MatchedStrings=[] + Matches=[] for l in List: match=reg_pattern.search(l) if match: MatchedElements.append(l) if len(match.groups(1))>0: - MatchedStrings.append(match.groups(1)[0]) + Matches.append(match.groups(1)[0]) else: - MatchedStrings.append('') + Matches.append('') + + MatchedElements = np.asarray(MatchedElements) + Matches = np.asarray(Matches) + + if integers: + Matches = Matches.astype(int) if sort: # Sorting by Matched string, NOTE: assumes that MatchedStrings are int. # that's probably not necessary since alphabetical/integer sorting should be the same # but it might be useful if number of leading zero differs, which would skew the sorting.. - MatchedElements = np.asarray(MatchedElements) - MatchedStrings = np.asarray(MatchedStrings) - Idx = np.array([int(s) for s in MatchedStrings]) - Isort = np.argsort(Idx) - Idx = Idx[Isort] + Isort = np.argsort(Matches) MatchedElements = MatchedElements[Isort] - MatchedStrings = MatchedStrings[Isort] + Matches = Matches[Isort] - return MatchedElements, MatchedStrings + return MatchedElements, Matches def extractSpanTS(df, pattern): @@ -1367,7 +1373,8 @@ def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKee # Loop trough files and populate result for i,f in enumerate(outFiles): try: - df=FASTOutputFile(f).toDataFrame() + df=weio.read(f).toDataFrame() + #df=FASTOutputFile(f).toDataFrame()A # For pyFAST except: invalidFiles.append(f) continue diff --git a/pydatview/fast/runner.py b/pydatview/fast/runner.py index eaddce1..385962b 100644 --- a/pydatview/fast/runner.py +++ b/pydatview/fast/runner.py @@ -15,8 +15,6 @@ # --- Fast libraries from pydatview.io.fast_input_file import FASTInputFile from pydatview.io.fast_output_file import FASTOutputFile -# from pyFAST.input_output.fast_input_file import FASTInputFile -# from pyFAST.input_output.fast_output_file import FASTOutputFile FAST_EXE='openfast' diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index a975148..9f57e5e 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -1,1288 +1,1828 @@ -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from io import open -from builtins import range -from builtins import str -from future import standard_library -standard_library.install_aliases() -try: - from .file import File, WrongFormatError, BrokenFormatError -except: - # --- Allowing this file to be standalone.. - class WrongFormatError(Exception): - pass - class BrokenFormatError(Exception): - pass - File = dict -import os -import numpy as np -import re -import pandas as pd - -__all__ = ['FASTInputFile'] - -TABTYPE_NOT_A_TAB = 0 -TABTYPE_NUM_WITH_HEADER = 1 -TABTYPE_NUM_WITH_HEADERCOM = 2 -TABTYPE_NUM_NO_HEADER = 4 -TABTYPE_NUM_BEAMDYN = 5 -TABTYPE_MIX_WITH_HEADER = 6 -TABTYPE_FIL = 3 -TABTYPE_FMT = 9999 # TODO - -# --------------------------------------------------------------------------------} -# --- INPUT FILE -# --------------------------------------------------------------------------------{ -class FASTInputFile(File): - """ - Read/write an OpenFAST input file. The object behaves like a dictionary. - - Main methods - ------------ - - read, write, toDataFrame, keys - - Main keys - --------- - The keys correspond to the keys used in the file. For instance for a .fst file: 'DT','TMax' - - Examples - -------- - - filename = 'AeroDyn.dat' - f = FASTInputFile(filename) - f['TwrAero'] = True - f['AirDens'] = 1.225 - f.write('AeroDyn_Changed.dat') - - """ - - @staticmethod - def defaultExtensions(): - return ['.dat','.fst','.txt','.fstf'] - - @staticmethod - def formatName(): - return 'FAST input file' - - def __init__(self, filename=None, **kwargs): - self._size=None - self._encoding=None - if filename: - self.filename = filename - self.read() - else: - self.filename = None - - def keys(self): - self.labels = [ d['label'] for d in self.data if not d['isComment'] ] - return self.labels - - def getID(self,label): - i=self.getIDSafe(label) - if i<0: - raise KeyError('Variable `'+ label+'` not found in FAST file:'+self.filename) - else: - return i - - def getIDs(self,label): - I=[] - # brute force search - for i in range(len(self.data)): - d = self.data[i] - if d['label'].lower()==label.lower(): - I.append(i) - if len(I)<0: - raise KeyError('Variable `'+ label+'` not found in FAST file:'+self.filename) - else: - return I - - def getIDSafe(self,label): - # brute force search - for i in range(len(self.data)): - d = self.data[i] - if d['label'].lower()==label.lower(): - return i - return -1 - - # Making object an iterator - def __iter__(self): - self.iCurrent=-1 - self.iMax=len(self.data)-1 - return self - - def __next__(self): # Python 2: def next(self) - if self.iCurrent > self.iMax: - raise StopIteration - else: - self.iCurrent += 1 - return self.data[self.iCurrent] - - # Making it behave like a dictionary - def __setitem__(self,key,item): - I = self.getIDs(key) - for i in I: - self.data[i]['value'] = item - - def __getitem__(self,key): - i = self.getID(key) - return self.data[i]['value'] - - def __repr__(self): - s ='Fast input file: {}\n'.format(self.filename) - return s+'\n'.join(['{:15s}: {}'.format(d['label'],d['value']) for i,d in enumerate(self.data)]) - - def addKeyVal(self,key,val,descr=None): - d=getDict() - d['label']=key - d['value']=val - if descr is not None: - d['descr']=descr - self.data.append(d) - - def read(self, filename=None): - if filename: - self.filename = filename - if self.filename: - if not os.path.isfile(self.filename): - raise OSError(2,'File not found:',self.filename) - if os.stat(self.filename).st_size == 0: - raise EmptyFileError('File is empty:',self.filename) - self._read() - else: - raise Exception('No filename provided') - - def _read(self): - - # --- Tables that can be detected based on the "Value" (first entry on line) - # TODO members for BeamDyn with mutliple key point ####### TODO PropSetID is Duplicate SubDyn and used in HydroDyn - NUMTAB_FROM_VAL_DETECT = ['HtFract' , 'TwrElev' , 'BlFract' , 'Genspd_TLU' , 'BlSpn' , 'WndSpeed' , 'HvCoefID' , 'AxCoefID' , 'JointID' , 'Dpth' , 'FillNumM' , 'MGDpth' , 'SimplCd' , 'RNodes' , 'kp_xr' , 'mu1' , 'TwrHtFr' , 'TwrRe' , 'WT_X'] - NUMTAB_FROM_VAL_DIM_VAR = ['NTwInpSt' , 'NumTwrNds' , 'NBlInpSt' , 'DLL_NumTrq' , 'NumBlNds' , 'NumCases' , 'NHvCoef' , 'NAxCoef' , 'NJoints' , 'NCoefDpth' , 'NFillGroups' , 'NMGDepths' , 1 , 'BldNodes' , 'kp_total' , 1 , 'NTwrHt' , 'NTwrRe' , 'NumTurbines'] - NUMTAB_FROM_VAL_VARNAME = ['TowProp' , 'TowProp' , 'BldProp' , 'DLLProp' , 'BldAeroNodes' , 'Cases' , 'HvCoefs' , 'AxCoefs' , 'Joints' , 'DpthProp' , 'FillGroups' , 'MGProp' , 'SmplProp' , 'BldAeroNodes' , 'MemberGeom' , 'DampingCoeffs' , 'TowerProp' , 'TowerRe', 'WindTurbines'] - NUMTAB_FROM_VAL_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 1 , 2 , 2 , 1 , 1 , 2 ] - NUMTAB_FROM_VAL_TYPE = ['num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' , 'num' , 'mix'] - # SubDyn - NUMTAB_FROM_VAL_DETECT += [ 'RJointID' , 'IJointID' , 'COSMID' , 'CMJointID' ] - NUMTAB_FROM_VAL_DIM_VAR += [ 'NReact' , 'NInterf' , 'NCOSMs' , 'NCmass' ] - NUMTAB_FROM_VAL_VARNAME += [ 'BaseJoints' , 'InterfaceJoints' , 'MemberCosineMatrix' , 'ConcentratedMasses'] - NUMTAB_FROM_VAL_NHEADER += [ 2 , 2 , 2 , 2 ] - NUMTAB_FROM_VAL_TYPE += [ 'mix' , 'num' , 'num' , 'num' ] - - - # --- Tables that can be detected based on the "Label" (second entry on line) - # NOTE: MJointID1, used by SubDyn and HydroDyn - NUMTAB_FROM_LAB_DETECT = ['NumAlf' , 'F_X' , 'MemberCd1' , 'MJointID1' , 'NOutLoc' , 'NOutCnt' , 'PropD' ,'Diam' ,'Type' ,'LineType' ] - NUMTAB_FROM_LAB_DIM_VAR = ['NumAlf' , 'NKInpSt' , 'NCoefMembers' , 'NMembers' , 'NMOutputs' , 'NMOutputs' , 'NPropSets' ,'NTypes' ,'NConnects' ,'NLines' ] - NUMTAB_FROM_LAB_VARNAME = ['AFCoeff' , 'TMDspProp' , 'MemberProp' , 'Members' , 'MemberOuts' , 'MemberOuts' , 'SectionProp' ,'LineTypes' ,'ConnectionProp' ,'LineProp' ] - NUMTAB_FROM_LAB_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 ] - NUMTAB_FROM_LAB_NOFFSET = [0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ] - NUMTAB_FROM_LAB_TYPE = ['num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' ,'mix' ,'mix' ,'mix' ] - # SubDyn - NUMTAB_FROM_LAB_DETECT += ['GuyanDampSize' , 'YoungE' , 'YoungE' , 'EA' , 'MatDens' ] - NUMTAB_FROM_LAB_DIM_VAR += [6 , 'NPropSets', 'NXPropSets', 'NCablePropSets' , 'NRigidPropSets'] - NUMTAB_FROM_LAB_VARNAME += ['GuyanDampMatrix' , 'BeamProp' , 'BeamPropX' , 'CableProp' , 'RigidProp' ] - NUMTAB_FROM_LAB_NHEADER += [0 , 2 , 2 , 2 , 2 ] - NUMTAB_FROM_LAB_NOFFSET += [1 , 0 , 0 , 0 , 0 ] - NUMTAB_FROM_LAB_TYPE += ['num' , 'num' , 'num' , 'num' , 'num' ] - # OLAF - NUMTAB_FROM_LAB_DETECT += ['GridName' ] - NUMTAB_FROM_LAB_DIM_VAR += ['nGridOut' ] - NUMTAB_FROM_LAB_VARNAME += ['GridOutputs'] - NUMTAB_FROM_LAB_NHEADER += [0 ] - NUMTAB_FROM_LAB_NOFFSET += [2 ] - NUMTAB_FROM_LAB_TYPE += ['mix' ] - - FILTAB_FROM_LAB_DETECT = ['FoilNm' ,'AFNames'] - FILTAB_FROM_LAB_DIM_VAR = ['NumFoil','NumAFfiles'] - FILTAB_FROM_LAB_VARNAME = ['FoilNm' ,'AFNames'] - - # Using lower case to be more tolerant.. - NUMTAB_FROM_VAL_DETECT_L = [s.lower() for s in NUMTAB_FROM_VAL_DETECT] - NUMTAB_FROM_LAB_DETECT_L = [s.lower() for s in NUMTAB_FROM_LAB_DETECT] - FILTAB_FROM_LAB_DETECT_L = [s.lower() for s in FILTAB_FROM_LAB_DETECT] - - self.data = [] - self.hasNodal=False - self.module = None - #with open(self.filename, 'r', errors="surrogateescape") as f: - with open(self.filename, 'r', errors="surrogateescape") as f: - lines=f.read().splitlines() - # IF NEEDED> DO THE FOLLOWING FORMATTING: - #lines = [str(l).encode('utf-8').decode('ascii','ignore') for l in f.read().splitlines()] - - # Fast files start with ! or - - #if lines[0][0]!='!' and lines[0][0]!='-': - # raise Exception('Fast file do not start with ! or -, is it the right format?') - - # Special filetypes - if self.detectAndReadExtPtfmSE(lines): - return - if self.detectAndReadAirfoil(lines): - return - - # Parsing line by line, storing each line into a dictionary - i=0 - nComments = 0 - nWrongLabels = 0 - allowSpaceSeparatedList=False - while i0 \ - or line.upper().find('MESH-BASED OUTPUTS')>0 \ - or line.upper().find('OUTPUT CHANNELS' )>0: # "OutList - The next line(s) contains a list of output parameters. See OutListParameters.xlsx for a listing of available output channels, (-)'" - # TODO, lazy implementation so far, MAKE SUB FUNCTION - parts = re.match(r'^\W*\w+', line) - if parts: - firstword = parts.group(0).strip() - else: - raise NotImplementedError - remainer = re.sub(r'^\W*\w+\W*', '', line) - # Parsing outlist, and then we continue at a new "i" (to read END etc.) - OutList,i = parseFASTOutList(lines,i+1) - d = getDict() - if self.hasNodal: - d['label'] = firstword+'_Nodal' - else: - d['label'] = firstword - d['descr'] = remainer - d['tabType'] = TABTYPE_FIL # TODO - d['value'] = ['']+OutList - self.data.append(d) - if i>=len(lines): - break - # --- Here we cheat and force an exit of the input file - # The reason for this is that some files have a lot of things after the END, which will result in the file being intepreted as a wrong format due to too many comments - if i+20 or lines[i+2].lower().find('bldnd_bloutnd')>0): - self.hasNodal=True - else: - self.data.append(parseFASTInputLine('END of input file (the word "END" must appear in the first 3 columns of this last OutList line)',i+1)) - self.data.append(parseFASTInputLine('---------------------------------------------------------------------------------------',i+2)) - break - elif line.upper().find('SSOUTLIST' )>0 or line.upper().find('SDOUTLIST' )>0: - # SUBDYN Outlist doesn not follow regular format - self.data.append(parseFASTInputLine(line,i)) - # OUTLIST Exception for BeamDyn - OutList,i = parseFASTOutList(lines,i+1) - # TODO - for o in OutList: - d = getDict() - d['isComment'] = True - d['value']=o - self.data.append(d) - # --- Here we cheat and force an exit of the input file - self.data.append(parseFASTInputLine('END of input file (the word "END" must appear in the first 3 columns of this last OutList line)',i+1)) - self.data.append(parseFASTInputLine('---------------------------------------------------------------------------------------',i+2)) - break - elif line.upper().find('ADDITIONAL STIFFNESS')>0: - # TODO, lazy implementation so far, MAKE SUB FUNCTION - self.data.append(parseFASTInputLine(line,i)) - i +=1 - KDAdd = [] - for _ in range(19): - KDAdd.append(lines[i]) - i +=1 - d = getDict() - d['label'] = 'KDAdd' # TODO - d['tabType'] = TABTYPE_FIL # TODO - d['value'] = KDAdd - self.data.append(d) - if i>=len(lines): - break - elif line.upper().find('DISTRIBUTED PROPERTIES')>0: - self.data.append(parseFASTInputLine(line,i)); - i+=1; - self.readBeamDynProps(lines,i) - return - - # --- Parsing of standard lines: value(s) key comment - line = lines[i] - d = parseFASTInputLine(line,i,allowSpaceSeparatedList) - - # --- Handling of special files - if d['label'].lower()=='kp_total': - # BeamDyn has weird space speparated list around keypoint definition - allowSpaceSeparatedList=True - elif d['label'].lower()=='numcoords': - # TODO, lazy implementation so far, MAKE SUB FUNCTION - if isStr(d['value']): - if d['value'][0]=='@': - # it's a ref to the airfoil coord file - pass - else: - if not strIsInt(d['value']): - raise WrongFormatError('Wrong value of NumCoords') - if int(d['value'])<=0: - pass - else: - self.data.append(d); i+=1; - # 3 comment lines - self.data.append(parseFASTInputLine(lines[i],i)); i+=1; - self.data.append(parseFASTInputLine(lines[i],i)); i+=1; - self.data.append(parseFASTInputLine(lines[i],i)); i+=1; - splits=cleanAfterChar(cleanLine(lines[i]),'!').split() - # Airfoil ref point - try: - pos=[float(splits[0]), float(splits[1])] - except: - raise WrongFormatError('Wrong format while reading coordinates of airfoil reference') - i+=1 - d = getDict() - d['label'] = 'AirfoilRefPoint' - d['value'] = pos - self.data.append(d) - # 2 comment lines - self.data.append(parseFASTInputLine(lines[i],i)); i+=1; - self.data.append(parseFASTInputLine(lines[i],i)); i+=1; - # Table of coordinats itself - d = getDict() - d['label'] = 'AirfoilCoord' - d['tabDimVar'] = 'NumCoords' - d['tabType'] = TABTYPE_NUM_WITH_HEADERCOM - nTabLines = self[d['tabDimVar']]-1 # SOMEHOW ONE DATA POINT LESS - d['value'], d['tabColumnNames'],_ = parseFASTNumTable(self.filename,lines[i:i+nTabLines+1],nTabLines,i,1) - d['tabUnits'] = ['(-)','(-)'] - self.data.append(d) - break - - - - #print('label>',d['label'],'<',type(d['label'])); - #print('value>',d['value'],'<',type(d['value'])); - #print(isStr(d['value'])) - #if isStr(d['value']): - # print(d['value'].lower() in NUMTAB_FROM_VAL_DETECT_L) - - - # --- Handling of tables - if isStr(d['value']) and d['value'].lower() in NUMTAB_FROM_VAL_DETECT_L: - # Table with numerical values, - ii = NUMTAB_FROM_VAL_DETECT_L.index(d['value'].lower()) - tab_type = NUMTAB_FROM_VAL_TYPE[ii] - if tab_type=='num': - d['tabType'] = TABTYPE_NUM_WITH_HEADER - else: - d['tabType'] = TABTYPE_MIX_WITH_HEADER - d['label'] = NUMTAB_FROM_VAL_VARNAME[ii] - d['tabDimVar'] = NUMTAB_FROM_VAL_DIM_VAR[ii] - nHeaders = NUMTAB_FROM_VAL_NHEADER[ii] - nTabLines=0 - if isinstance(d['tabDimVar'],int): - nTabLines = d['tabDimVar'] - else: - nTabLines = self[d['tabDimVar']] - #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); - d['value'], d['tabColumnNames'], d['tabUnits'] = parseFASTNumTable(self.filename,lines[i:i+nTabLines+nHeaders], nTabLines, i, nHeaders, tableType=tab_type, varNumLines=d['tabDimVar']) - i += nTabLines+nHeaders-1 - - # --- Temporary hack for e.g. SubDyn, that has duplicate table, impossible to detect in the current way... - # So we remove the element form the list one read - del NUMTAB_FROM_VAL_DETECT[ii] - del NUMTAB_FROM_VAL_DIM_VAR[ii] - del NUMTAB_FROM_VAL_VARNAME[ii] - del NUMTAB_FROM_VAL_NHEADER[ii] - del NUMTAB_FROM_VAL_TYPE [ii] - del NUMTAB_FROM_VAL_DETECT_L[ii] - - elif isStr(d['label']) and d['label'].lower() in NUMTAB_FROM_LAB_DETECT_L: - ii = NUMTAB_FROM_LAB_DETECT_L.index(d['label'].lower()) - tab_type = NUMTAB_FROM_LAB_TYPE[ii] - # Special case for airfoil data, the table follows NumAlf, so we add d first - if d['label'].lower()=='numalf': - d['tabType']=TABTYPE_NOT_A_TAB - self.data.append(d) - # Creating a new dictionary for the table - d = {'value':None, 'label':'NumAlf', 'isComment':False, 'descr':'', 'tabType':None} - i += 1 - nHeaders = NUMTAB_FROM_LAB_NHEADER[ii] - nOffset = NUMTAB_FROM_LAB_NOFFSET[ii] - if nOffset>0: - # Creating a dictionary for that entry - dd = {'value':d['value'], 'label':d['label'], 'isComment':False, 'descr':d['descr'], 'tabType':TABTYPE_NOT_A_TAB} - self.data.append(dd) - - d['label'] = NUMTAB_FROM_LAB_VARNAME[ii] - d['tabDimVar'] = NUMTAB_FROM_LAB_DIM_VAR[ii] - if d['label'].lower()=='afcoeff' : - d['tabType'] = TABTYPE_NUM_WITH_HEADERCOM - else: - if tab_type=='num': - d['tabType'] = TABTYPE_NUM_WITH_HEADER - else: - d['tabType'] = TABTYPE_MIX_WITH_HEADER - if isinstance(d['tabDimVar'],int): - nTabLines = d['tabDimVar'] - else: - nTabLines = self[d['tabDimVar']] - #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); - d['value'], d['tabColumnNames'], d['tabUnits'] = parseFASTNumTable(self.filename,lines[i:i+nTabLines+nHeaders+nOffset],nTabLines,i, nHeaders, tableType=tab_type, nOffset=nOffset, varNumLines=d['tabDimVar']) - i += nTabLines+1-nOffset - - # --- Temporary hack for e.g. SubDyn, that has duplicate table, impossible to detect in the current way... - # So we remove the element form the list one read - del NUMTAB_FROM_LAB_DETECT[ii] - del NUMTAB_FROM_LAB_DIM_VAR[ii] - del NUMTAB_FROM_LAB_VARNAME[ii] - del NUMTAB_FROM_LAB_NHEADER[ii] - del NUMTAB_FROM_LAB_NOFFSET[ii] - del NUMTAB_FROM_LAB_TYPE [ii] - del NUMTAB_FROM_LAB_DETECT_L[ii] - - elif isStr(d['label']) and d['label'].lower() in FILTAB_FROM_LAB_DETECT_L: - ii = FILTAB_FROM_LAB_DETECT_L.index(d['label'].lower()) - d['label'] = FILTAB_FROM_LAB_VARNAME[ii] - d['tabDimVar'] = FILTAB_FROM_LAB_DIM_VAR[ii] - d['tabType'] = TABTYPE_FIL - nTabLines = self[d['tabDimVar']] - #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); - d['value'] = parseFASTFilTable(lines[i:i+nTabLines],nTabLines,i) - i += nTabLines-1 - - - - self.data.append(d) - i += 1 - # --- Safety checks - if d['isComment']: - #print(line) - nComments +=1 - else: - if hasSpecialChars(d['label']): - nWrongLabels +=1 - #print('label>',d['label'],'<',type(d['label']),line); - if i>3: # first few lines may be comments, we allow it - #print('Line',i,'Label:',d['label']) - raise WrongFormatError('Special Character found in Label: `{}`, for line: `{}`'.format(d['label'],line)) - if len(d['label'])==0: - nWrongLabels +=1 - if nComments>len(lines)*0.35: - #print('Comment fail',nComments,len(lines),self.filename) - raise WrongFormatError('Most lines were read as comments, probably not a FAST Input File') - if nWrongLabels>len(lines)*0.10: - #print('Label fail',nWrongLabels,len(lines),self.filename) - raise WrongFormatError('Too many lines with wrong labels, probably not a FAST Input File') - - # --- PostReading checks - labels = self.keys() - duplicates = set([x for x in labels if (labels.count(x) > 1) and x!='OutList' and x.strip()!='-']) - if len(duplicates)>0: - print('[WARN] Duplicate labels found in file: '+self.filename) - print(' Duplicates: '+', '.join(duplicates)) - print(' It\'s strongly recommended to make them unique! ') -# except WrongFormatError as e: -# raise WrongFormatError('Fast File {}: '.format(self.filename)+'\n'+e.args[0]) -# except Exception as e: -# raise e -# # print(e) -# raise Exception('Fast File {}: '.format(self.filename)+'\n'+e.args[0]) - - - def toString(self): - s='' - # Special file formats, TODO subclass - if self.module=='ExtPtfm': - s+='!Comment\n' - s+='!Comment Flex 5 Format\n' - s+='!Dimension: {}\n'.format(self['nDOF']) - s+='!Time increment in simulation: {}\n'.format(self['dt']) - s+='!Total simulation time in file: {}\n'.format(self['T']) - - s+='\n!Mass Matrix\n' - s+='!Dimension: {}\n'.format(self['nDOF']) - s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['MassMatrix']) - - s+='\n\n!Stiffness Matrix\n' - s+='!Dimension: {}\n'.format(self['nDOF']) - s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['StiffnessMatrix']) - - s+='\n\n!Damping Matrix\n' - s+='!Dimension: {}\n'.format(self['nDOF']) - s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['DampingMatrix']) - - s+='\n\n!Loading and Wave Elevation\n' - s+='!Dimension: 1 time column - {} force columns\n'.format(self['nDOF']) - s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['Loading']) - return s - - def toStringVLD(val,lab,descr): - val='{}'.format(val) - lab='{}'.format(lab) - if len(val)<13: - val='{:13s}'.format(val) - if len(lab)<13: - lab='{:13s}'.format(lab) - return val+' '+lab+' - '+descr.strip().strip('-').strip()+'\n' - - def beamdyn_section_mat_tostring(x,K,M): - def mat_tostring(M,fmt='24.16e'): - return '\n'.join([' '+' '.join(['{:24.16E}'.format(m) for m in M[i,:]]) for i in range(np.size(M,1))]) - s='' - s+='{:.6f}\n'.format(x) - s+=mat_tostring(K) - #s+=np.array2string(K) - s+='\n' - s+='\n' - s+=mat_tostring(M) - #s+=np.array2string(M) - s+='\n' - s+='\n' - return s - - - for i in range(len(self.data)): - d=self.data[i] - if d['isComment']: - s+='{}'.format(d['value']) - elif d['tabType']==TABTYPE_NOT_A_TAB: - if isinstance(d['value'], list): - sList=', '.join([str(x) for x in d['value']]) - s+='{} {} {}'.format(sList,d['label'],d['descr']) - else: - s+=toStringVLD(d['value'],d['label'],d['descr']).strip() - elif d['tabType']==TABTYPE_NUM_WITH_HEADER: - if d['tabColumnNames'] is not None: - s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) - #s+=d['descr'] # Not ready for that - if d['tabUnits'] is not None: - s+='\n' - s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) - newline='\n' - else: - newline='' - if np.size(d['value'],0) > 0 : - s+=newline - s+='\n'.join('\t'.join( ('{:15.0f}'.format(x) if int(x)==x else '{:15.8e}'.format(x) ) for x in y) for y in d['value']) - elif d['tabType']==TABTYPE_MIX_WITH_HEADER: - s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) - if d['tabUnits'] is not None: - s+='\n' - s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) - if np.size(d['value'],0) > 0 : - s+='\n' - s+='\n'.join('\t'.join('{}'.format(x) for x in y) for y in d['value']) - elif d['tabType']==TABTYPE_NUM_WITH_HEADERCOM: - s+='! {}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) - s+='! {}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) - s+='\n'.join('\t'.join('{:15.8e}'.format(x) for x in y) for y in d['value']) - elif d['tabType']==TABTYPE_FIL: - #f.write('{} {} {}\n'.format(d['value'][0],d['tabDetect'],d['descr'])) - s+='{} {} {}\n'.format(d['value'][0],d['label'],d['descr']) # TODO? - s+='\n'.join(fil for fil in d['value'][1:]) - elif d['tabType']==TABTYPE_NUM_BEAMDYN: - data = d['value'] - Cols =['Span'] - Cols+=['K{}{}'.format(i+1,j+1) for i in range(6) for j in range(6)] - Cols+=['M{}{}'.format(i+1,j+1) for i in range(6) for j in range(6)] - for i in np.arange(len(data['span'])): - x = data['span'][i] - K = data['K'][i] - M = data['M'][i] - s += beamdyn_section_mat_tostring(x,K,M) - else: - raise Exception('Unknown table type for variable {}'.format(d)) - if i0: - # Hack for blade files, we add the modes - x=Val[:,0] - Modes=np.zeros((x.shape[0],3)) - Modes[:,0] = x**2 * self['BldFl1Sh(2)'] \ - + x**3 * self['BldFl1Sh(3)'] \ - + x**4 * self['BldFl1Sh(4)'] \ - + x**5 * self['BldFl1Sh(5)'] \ - + x**6 * self['BldFl1Sh(6)'] - Modes[:,1] = x**2 * self['BldFl2Sh(2)'] \ - + x**3 * self['BldFl2Sh(3)'] \ - + x**4 * self['BldFl2Sh(4)'] \ - + x**5 * self['BldFl2Sh(5)'] \ - + x**6 * self['BldFl2Sh(6)'] - Modes[:,2] = x**2 * self['BldEdgSh(2)'] \ - + x**3 * self['BldEdgSh(3)'] \ - + x**4 * self['BldEdgSh(4)'] \ - + x**5 * self['BldEdgSh(5)'] \ - + x**6 * self['BldEdgSh(6)'] - Val = np.hstack((Val,Modes)) - Cols = Cols + ['ShapeFlap1_[-]','ShapeFlap2_[-]','ShapeEdge1_[-]'] - - elif self.getIDSafe('TwFAM1Sh(2)')>0: - # Hack for tower files, we add the modes - x=Val[:,0] - Modes=np.zeros((x.shape[0],4)) - Modes[:,0] = x**2 * self['TwFAM1Sh(2)'] \ - + x**3 * self['TwFAM1Sh(3)'] \ - + x**4 * self['TwFAM1Sh(4)'] \ - + x**5 * self['TwFAM1Sh(5)'] \ - + x**6 * self['TwFAM1Sh(6)'] - Modes[:,1] = x**2 * self['TwFAM2Sh(2)'] \ - + x**3 * self['TwFAM2Sh(3)'] \ - + x**4 * self['TwFAM2Sh(4)'] \ - + x**5 * self['TwFAM2Sh(5)'] \ - + x**6 * self['TwFAM2Sh(6)'] - Modes[:,2] = x**2 * self['TwSSM1Sh(2)'] \ - + x**3 * self['TwSSM1Sh(3)'] \ - + x**4 * self['TwSSM1Sh(4)'] \ - + x**5 * self['TwSSM1Sh(5)'] \ - + x**6 * self['TwSSM1Sh(6)'] - Modes[:,3] = x**2 * self['TwSSM2Sh(2)'] \ - + x**3 * self['TwSSM2Sh(3)'] \ - + x**4 * self['TwSSM2Sh(4)'] \ - + x**5 * self['TwSSM2Sh(5)'] \ - + x**6 * self['TwSSM2Sh(6)'] - Val = np.hstack((Val,Modes)) - Cols = Cols + ['ShapeForeAft1_[-]','ShapeForeAft2_[-]','ShapeSideSide1_[-]','ShapeSideSide2_[-]'] - elif d['label']=='AFCoeff': - try: - pol = d['value'] - alpha = pol[:,0]*np.pi/180. - Cl = pol[:,1] - Cd = pol[:,2] - Cd0 = self['Cd0'] - # Cn (with or without Cd0) - Cn1 = Cl*np.cos(alpha)+ (Cd-Cd0)*np.sin(alpha) - Cn = Cl*np.cos(alpha)+ Cd*np.sin(alpha) - Val=np.column_stack((Val,Cn)); Cols+=['Cn_[-]'] - Val=np.column_stack((Val,Cn1)); Cols+=['Cn_Cd0off_[-]'] - - CnLin = self['C_nalpha']*(alpha-self['alpha0']*np.pi/180.) - CnLin[alpha<-20*np.pi/180]=np.nan - CnLin[alpha> 30*np.pi/180]=np.nan - Val=np.column_stack((Val,CnLin)); Cols+=['Cn_pot_[-]'] - - # Highlighting points surrounding 0 1 2 Cn points - CnPoints = Cn*np.nan - iBef2 = np.where(alpha0: - if l[0]=='!': - if l.find('!dimension')==0: - self.addKeyVal('nDOF',int(l.split(':')[1])) - nDOFCommon=self['nDOF'] - elif l.find('!time increment')==0: - self.addKeyVal('dt',float(l.split(':')[1])) - elif l.find('!total simulation time')==0: - self.addKeyVal('T',float(l.split(':')[1])) - elif len(l.strip())==0: - pass - else: - raise BrokenFormatError('Unexcepted content found on line {}'.format(i)) - i+=1 - except BrokenFormatError as e: - raise e - except: - raise - - - return True - - - - def detectAndReadAirfoil(self,lines): - if len(lines)<14: - return False - # Reading number of tables - L3 = lines[2].strip().split() - if len(L3)<=0: - return False - if not strIsInt(L3[0]): - return False - nTables=int(L3[0]) - # Reading table ID - L4 = lines[3].strip().split() - if len(L4)<=nTables: - return False - TableID=L4[:nTables] - if nTables==1: - TableID=[''] - # Keywords for file format - KW1=lines[12].strip().split() - KW2=lines[13].strip().split() - if len(KW1)>nTables and len(KW2)>nTables: - if KW1[nTables].lower()=='angle' and KW2[nTables].lower()=='minimum': - d = getDict(); d['isComment'] = True; d['value'] = lines[0]; self.data.append(d); - d = getDict(); d['isComment'] = True; d['value'] = lines[1]; self.data.append(d); - for i in range(2,14): - splits = lines[i].split() - #print(splits) - d = getDict() - d['label'] = ' '.join(splits[1:]) # TODO - d['descr'] = ' '.join(splits[1:]) # TODO - d['value'] = float(splits[0]) - self.data.append(d) - #pass - #for i in range(2,14): - nTabLines=0 - while 14+nTabLines0 : - nTabLines +=1 - #data = np.array([lines[i].strip().split() for i in range(14,len(lines)) if len(lines[i])>0]).astype(float) - #data = np.array([lines[i].strip().split() for i in takewhile(lambda x: len(lines[i].strip())>0, range(14,len(lines)-1))]).astype(float) - data = np.array([lines[i].strip().split() for i in range(14,nTabLines+14)]).astype(float) - #print(data) - d = getDict() - d['label'] = 'Polar' - d['tabDimVar'] = nTabLines - d['tabType'] = TABTYPE_NUM_NO_HEADER - d['value'] = data - if np.size(data,1)==1+nTables*3: - d['tabColumnNames'] = ['Alpha']+[n+l for l in TableID for n in ['Cl','Cd','Cm']] - d['tabUnits'] = ['(deg)']+['(-)' , '(-)' , '(-)']*nTables - elif np.size(data,1)==1+nTables*2: - d['tabColumnNames'] = ['Alpha']+[n+l for l in TableID for n in ['Cl','Cd']] - d['tabUnits'] = ['(deg)']+['(-)' , '(-)']*nTables - else: - d['tabColumnNames'] = ['col{}'.format(j) for j in range(np.size(data,1))] - self.data.append(d) - return True - - def readBeamDynProps(self,lines,iStart): - nStations=self['station_total'] - #M=np.zeros((nStations,1+36+36)) - M = np.zeros((nStations,6,6)) - K = np.zeros((nStations,6,6)) - span = np.zeros(nStations) - i=iStart; - try: - for j in range(nStations): - # Read span location - span[j]=float(lines[i]); i+=1; - # Read stiffness matrix - K[j,:,:]=np.array((' '.join(lines[i:i+6])).split()).astype(float).reshape(6,6) - i+=7 - # Read mass matrix - M[j,:,:]=np.array((' '.join(lines[i:i+6])).split()).astype(float).reshape(6,6) - i+=7 - except: - raise WrongFormatError('An error occured while reading section {}/{}'.format(j+1,nStations)) - - d = getDict() - d['label'] = 'BeamProperties' - d['descr'] = '' - d['tabType'] = TABTYPE_NUM_BEAMDYN - d['value'] = {'span':span, 'K':K, 'M':M} - self.data.append(d) - -# --------------------------------------------------------------------------------} -# --- Helper functions -# --------------------------------------------------------------------------------{ -def isStr(s): - # Python 2 and 3 compatible - # Two options below - # NOTE: all this avoided since we import str from builtins - # --- Version 2 - # isString = False; - # if(isinstance(s, str)): - # isString = True; - # try: - # if(isinstance(s, basestring)): # todo unicode as well - # isString = True; - # except NameError: - # pass; - # return isString - # --- Version 1 - # try: - # basestring # python 2 - # return isinstance(s, basestring) or isinstance(s,unicode) - # except NameError: - # basestring=str #python 3 - # return isinstance(s, str) - return isinstance(s, str) - -def strIsFloat(s): - #return s.replace('.',',1').isdigit() - try: - float(s) - return True - except: - return False - -def strIsBool(s): - return s.lower() in ['true','false','t','f'] - -def strIsInt(s): - s = str(s) - if s[0] in ('-', '+'): - return s[1:].isdigit() - return s.isdigit() - -def strToBool(s): - return s.lower() in ['true','t'] - -def hasSpecialChars(s): - # fast allows for parenthesis - # For now we allow for - but that's because of BeamDyn geometry members - return not re.match("^[\"\'a-zA-Z0-9_()-]*$", s) - -def cleanLine(l): - # makes a string single space separated - l = l.replace('\t',' ') - l = ' '.join(l.split()) - l = l.strip() - return l - -def cleanAfterChar(l,c): - # remove whats after a character - n = l.find(c); - if n>0: - return l[:n] - else: - return l - -def getDict(): - return {'value':None, 'label':'', 'isComment':False, 'descr':'', 'tabType':TABTYPE_NOT_A_TAB} - -def _merge_value(splits): - - merged = splits.pop(0) - if merged[0] == '"': - while merged[-1] != '"': - merged += " "+splits.pop(0) - splits.insert(0, merged) - - - - -def parseFASTInputLine(line_raw,i,allowSpaceSeparatedList=False): - d = getDict() - #print(line_raw) - try: - # preliminary cleaning (Note: loss of formatting) - line = cleanLine(line_raw) - # Comment - if any(line.startswith(c) for c in ['#','!','--','==']) or len(line)==0: - d['isComment']=True - d['value']=line_raw - return d - if line.lower().startswith('end'): - sp =line.split() - if len(sp)>2 and sp[1]=='of': - d['isComment']=True - d['value']=line_raw - - # Detecting lists - List=[]; - iComma=line.find(',') - if iComma>0 and iComma<30: - fakeline=line.replace(' ',',') - fakeline=re.sub(',+',',',fakeline) - csplits=fakeline.split(',') - # Splitting based on comma and looping while it's numbers of booleans - ii=0 - s=csplits[ii] - #print(csplits) - while strIsFloat(s) or strIsBool(s) and ii=len(csplits): - raise WrongFormatError('Wrong number of list values') - s = csplits[ii] - #print('[INFO] Line {}: Found list: '.format(i),List) - # Defining value and remaining splits - if len(List)>=2: - d['value']=List - line_remaining=line - # eating line, removing each values - for iii in range(ii): - sValue=csplits[iii] - ipos=line_remaining.find(sValue) - line_remaining = line_remaining[ipos+len(sValue):] - splits=line_remaining.split() - iNext=0 - else: - # It's not a list, we just use space as separators - splits=line.split(' ') - _merge_value(splits) - s=splits[0] - - if strIsInt(s): - d['value']=int(s) - if allowSpaceSeparatedList and len(splits)>1: - if strIsInt(splits[1]): - d['value']=splits[0]+ ' '+splits[1] - elif strIsFloat(s): - d['value']=float(s) - elif strIsBool(s): - d['value']=strToBool(s) - else: - d['value']=s - iNext=1 - #import pdb ; pdb.set_trace(); - - # Extracting label (TODO, for now only second split) - bOK=False - while (not bOK) and iNext comment assumed'.format(i+1)) - d['isComment']=True - d['value']=line_raw - iNext = len(splits)+1 - - # Recombining description - if len(splits)>=iNext+1: - d['descr']=' '.join(splits[iNext:]) - except WrongFormatError as e: - raise WrongFormatError('Line {}: '.format(i+1)+e.args[0]) - except Exception as e: - raise Exception('Line {}: '.format(i+1)+e.args[0]) - - return d - -def parseFASTOutList(lines,iStart): - OutList=[] - i = iStart - MAX=200 - while iMAX : - raise Exception('More that 200 lines found in outlist') - if i>=len(lines): - print('[WARN] End of file reached while reading Outlist') - #i=min(i+1,len(lines)) - return OutList,iStart+len(OutList) - - -def extractWithinParenthesis(s): - mo = re.search(r'\((.*)\)', s) - if mo: - return mo.group(1) - return '' - -def extractWithinBrackets(s): - mo = re.search(r'\((.*)\)', s) - if mo: - return mo.group(1) - return '' - -def detectUnits(s,nRef): - nPOpen=s.count('(') - nPClos=s.count(')') - nBOpen=s.count('[') - nBClos=s.count(']') - - sep='!#@#!' - if (nPOpen == nPClos) and (nPOpen>=nRef): - #that's pretty good - Units=s.replace('(','').replace(')',sep).split(sep)[:-1] - elif (nBOpen == nBClos) and (nBOpen>=nRef): - Units=s.replace('[','').replace(']',sep).split(sep)[:-1] - else: - Units=s.split() - return Units - - -def parseFASTNumTable(filename,lines,n,iStart,nHeaders=2,tableType='num',nOffset=0, varNumLines=''): - """ - First lines of data starts at: nHeaders+nOffset - - """ - Tab = None - ColNames = None - Units = None - - - if len(lines)!=n+nHeaders+nOffset: - raise BrokenFormatError('Not enough lines in table: {} lines instead of {}\nFile:{}'.format(len(lines)-nHeaders,n,filename)) - try: - if nHeaders==0: - # Extract number of values from number of numerical values on first line - numeric_const_pattern = r'[-+]? (?: (?: \d* \. \d+ ) | (?: \d+ \.? ) )(?: [Ee] [+-]? \d+ ) ?' - rx = re.compile(numeric_const_pattern, re.VERBOSE) - header = cleanAfterChar(lines[nOffset], '!') - if tableType=='num': - dat= np.array(rx.findall(header)).astype(float) - ColNames=['C{}'.format(j) for j in range(len(dat))] - else: - raise NotImplementedError('Reading FAST tables with no headers for type different than num not implemented yet') - - elif nHeaders>=1: - # Extract column names - i = 0 - sTmp = cleanLine(lines[i]) - sTmp = cleanAfterChar(sTmp,'[') - sTmp = cleanAfterChar(sTmp,'(') - sTmp = cleanAfterChar(sTmp,'!') - sTmp = cleanAfterChar(sTmp,'#') - if sTmp.startswith('!'): - sTmp=sTmp[1:].strip() - ColNames=sTmp.split() - if nHeaders>=2: - # Extract units - i = 1 - sTmp = cleanLine(lines[i]) - sTmp = cleanAfterChar(sTmp,'!') - sTmp = cleanAfterChar(sTmp,'#') - if sTmp.startswith('!'): - sTmp=sTmp[1:].strip() - - Units = detectUnits(sTmp,len(ColNames)) - Units = ['({})'.format(u.strip()) for u in Units] - # Forcing user to match number of units and column names - if len(ColNames) != len(Units): - print(ColNames) - print(Units) - print('[WARN] {}: Line {}: Number of column names different from number of units in table'.format(filename, iStart+i+1)) - - nCols=len(ColNames) - - if tableType=='num': - if n==0: - Tab = np.zeros((n, nCols)) - for i in range(nHeaders+nOffset,n+nHeaders+nOffset): - l = cleanAfterChar(lines[i].lower(),'!') - l = cleanAfterChar(l,'#') - v = l.split() - if len(v) != nCols: - # Discarding SubDyn special cases - if ColNames[-1].lower() not in ['nodecnt']: - print('[WARN] {}: Line {}: number of data different from number of column names. ColumnNames: {}'.format(filename, iStart+i+1, ColNames)) - if i==nHeaders+nOffset: - # Node Cnt - if len(v) != nCols: - if ColNames[-1].lower()== 'nodecnt': - ColNames = ColNames+['Col']*(len(v)-nCols) - Units = Units+['Col']*(len(v)-nCols) - - nCols=len(v) - Tab = np.zeros((n, nCols)) - # Accounting for TRUE FALSE and converting to float - v = [s.replace('true','1').replace('false','0').replace('noprint','0').replace('print','1') for s in v] - v = [float(s) for s in v[0:nCols]] - if len(v) < nCols: - raise Exception('Number of data is lower than number of column names') - Tab[i-nHeaders-nOffset,:] = v - elif tableType=='mix': - # a mix table contains a mixed of strings and floats - # For now, we are being a bit more relaxed about the number of columns - if n==0: - Tab = np.zeros((n, nCols)).astype(object) - for i in range(nHeaders+nOffset,n+nHeaders+nOffset): - l = lines[i] - l = cleanAfterChar(l,'!') - l = cleanAfterChar(l,'#') - v = l.split() - if l.startswith('---'): - raise BrokenFormatError('Error reading line {} while reading table. Is the variable `{}` set correctly?'.format(iStart+i+1, varNumLines)) - if len(v) != nCols: - # Discarding SubDyn special cases - if ColNames[-1].lower() not in ['cosmid', 'ssifile']: - print('[WARN] {}: Line {}: Number of data is different than number of column names. Column Names: {}'.format(filename,iStart+1+i, ColNames)) - if i==nHeaders+nOffset: - if len(v)>nCols: - ColNames = ColNames+['Col']*(len(v)-nCols) - Units = Units+['Col']*(len(v)-nCols) - nCols=len(v) - Tab = np.zeros((n, nCols)).astype(object) - v=v[0:min(len(v),nCols)] - Tab[i-nHeaders-nOffset,0:len(v)] = v - # If all values are float, we convert to float - if all([strIsFloat(x) for x in Tab.ravel()]): - Tab=Tab.astype(float) - else: - raise Exception('Unknown table type') - - ColNames = ColNames[0:nCols] - if Units is not None: - Units = Units[0:nCols] - Units = ['('+u.replace('(','').replace(')','')+')' for u in Units] - if nHeaders==0: - ColNames=None - - except Exception as e: - raise BrokenFormatError('Line {}: {}'.format(iStart+i+1,e.args[0])) - return Tab, ColNames, Units - - -def parseFASTFilTable(lines,n,iStart): - Tab = [] - try: - i=0 - if len(lines)!=n: - raise WrongFormatError('Not enough lines in table: {} lines instead of {}'.format(len(lines),n)) - for i in range(n): - l = lines[i].split() - #print(l[0].strip()) - Tab.append(l[0].strip()) - - except Exception as e: - raise Exception('Line {}: '.format(iStart+i+1)+e.args[0]) - return Tab - - -if __name__ == "__main__": - pass - #B=FASTIn('Turbine.outb') - - - +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import absolute_import +from io import open +from builtins import range +from builtins import str +from future import standard_library +standard_library.install_aliases() +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + # --- Allowing this file to be standalone.. + class WrongFormatError(Exception): + pass + class BrokenFormatError(Exception): + pass + File = dict +import os +import numpy as np +import re +import pandas as pd + +__all__ = ['FASTInputFile'] + +TABTYPE_NOT_A_TAB = 0 +TABTYPE_NUM_WITH_HEADER = 1 +TABTYPE_NUM_WITH_HEADERCOM = 2 +TABTYPE_NUM_NO_HEADER = 4 +TABTYPE_NUM_BEAMDYN = 5 +TABTYPE_MIX_WITH_HEADER = 6 +TABTYPE_FIL = 3 +TABTYPE_FMT = 9999 # TODO + + + +class FASTInputFile(File): + """ + Read/write an OpenFAST input file. The object behaves like a dictionary. + A generic reader/writer is used at first. + If a dedicated OpenFAST input file is detected, additional functionalities are added. + See at the end of this file for dedicated class that can be used instead of this generic reader. + + Main methods + ------------ + - read, write, toDataFrame, keys, toGraph + + + Return an object which inherits from FASTInputFileBase + - The generic file reader is run first + - If a specific file format/module is detected, a fixed file format object is returned + The fixed file format have additional outputs, sanity checks and methods + """ + + @staticmethod + def defaultExtensions(): + return ['.dat','.fst','.txt','.fstf'] + + @staticmethod + def formatName(): + return 'FAST input file' + + def __init__(self, filename=None, **kwargs): + self._fixedfile = None + self.basefile = FASTInputFileBase(filename, **kwargs) # Generic fileformat + + @property + def fixedfile(self): + if self._fixedfile is not None: + return self._fixedfile + elif len(self.basefile.data)>0: + self._fixedfile=self.fixedFormat() + return self._fixedfile + else: + return self.basefile + @property + def module(self): + if self._fixedfile is None: + return self.basefile.module + else: + return self._fixedfile.module + + def fixedFormat(self): + # --- Creating a dedicated Child + KEYS = list(self.basefile.keys()) + if 'NumBlNds' in KEYS: + return ADBladeFile.from_fast_input_file(self.basefile) + elif 'NBlInpSt' in KEYS: + return EDBladeFile.from_fast_input_file(self.basefile) + elif 'MassMatrix' in KEYS and self.module =='ExtPtfm': + return ExtPtfmFile.from_fast_input_file(self.basefile) + elif 'NumCoords' in KEYS and 'InterpOrd' in KEYS: + return ADPolarFile.from_fast_input_file(self.basefile) + else: + # TODO: HD, SD, SvD, ED, AD, EDbld, BD, + #print('>>>>>>>>>>>> NO FILEFORMAT', KEYS) + return self.basefile + + + def read(self, filename=None): + return self.fixedfile.read(filename) + + def write(self, filename=None): + return self.fixedfile.write(filename) + + def toDataFrame(self): + return self.fixedfile.toDataFrame() + + def toString(self): + return self.fixedfile.toString() + + def keys(self): + return self.fixedfile.keys() + + def toGraph(self): + return self.fixedfile.toGraph() + + @property + def filename(self): + return self.fixedfile.filename + + @property + def comment(self): + return self.fixedfile.comment + + @comment.setter + def comment(self,comment): + self.fixedfile.comment(comment) + + def __iter__(self): + return self.fixedfile.__iter__() + + def __next__(self): + return self.fixedfile.__next__() + + def __setitem__(self,key,item): + return self.fixedfile.__setitem__(key,item) + + def __getitem__(self,key): + return self.fixedfile.__getitem__(key) + + def __repr__(self): + return self.fixedfile.__repr__() + #s ='Fast input file: {}\n'.format(self.filename) + #return s+'\n'.join(['{:15s}: {}'.format(d['label'],d['value']) for i,d in enumerate(self.data)]) + + +# --------------------------------------------------------------------------------} +# --- BASE INPUT FILE +# --------------------------------------------------------------------------------{ +class FASTInputFileBase(File): + """ + Read/write an OpenFAST input file. The object behaves like a dictionary. + + Main methods + ------------ + - read, write, toDataFrame, keys + + Main keys + --------- + The keys correspond to the keys used in the file. For instance for a .fst file: 'DT','TMax' + + Examples + -------- + + filename = 'AeroDyn.dat' + f = FASTInputFile(filename) + f['TwrAero'] = True + f['AirDens'] = 1.225 + f.write('AeroDyn_Changed.dat') + + """ + + def __init__(self, filename=None, **kwargs): + self._size=None + self._encoding=None + self.setData() # Init data + if filename: + self.filename = filename + self.read() + + def setData(self, filename=None, data=None, hasNodal=False, module=None): + """ Set the data of this object. This object shouldn't store anything else. """ + if data is None: + self.data = [] + else: + self.data = data + self.hasNodal = hasNodal + self.module = module + self.filename = filename + + def keys(self): + self.labels = [ d['label'] for i,d in enumerate(self.data) if (not d['isComment']) and (i not in self._IComment)] + return self.labels + + def getID(self,label): + i=self.getIDSafe(label) + if i<0: + raise KeyError('Variable `'+ label+'` not found in FAST file:'+self.filename) + else: + return i + + def getIDs(self,label): + I=[] + # brute force search + for i in range(len(self.data)): + d = self.data[i] + if d['label'].lower()==label.lower(): + I.append(i) + if len(I)<0: + raise KeyError('Variable `'+ label+'` not found in FAST file:'+self.filename) + else: + return I + + def getIDSafe(self,label): + # brute force search + for i in range(len(self.data)): + d = self.data[i] + if d['label'].lower()==label.lower(): + return i + return -1 + + # Making object an iterator + def __iter__(self): + self.iCurrent=-1 + self.iMax=len(self.data)-1 + return self + + def __next__(self): # Python 2: def next(self) + if self.iCurrent > self.iMax: + raise StopIteration + else: + self.iCurrent += 1 + return self.data[self.iCurrent] + + # Making it behave like a dictionary + def __setitem__(self,key,item): + I = self.getIDs(key) + for i in I: + self.data[i]['value'] = item + + def __getitem__(self,key): + i = self.getID(key) + return self.data[i]['value'] + + def __repr__(self): + s ='Fast input file base: {}\n'.format(self.filename) + return s+'\n'.join(['{:15s}: {}'.format(d['label'],d['value']) for i,d in enumerate(self.data)]) + + def addKeyVal(self, key, val, descr=None): + i=self.getIDSafe(key) + if i<0: + d = getDict() + else: + d = self.data[i] + d['label']=key + d['value']=val + if descr is not None: + d['descr']=descr + if i<0: + self.data.append(d) + + def addValKey(self,val,key,descr=None): + self.addKeyVal(key, val, descr) + + def addComment(self, comment='!'): + d=getDict() + d['isComment'] = True + d['value'] = comment + self.data.append(d) + + def addTable(self, label, tab, cols=None, units=None, tabType=1, tabDimVar=None): + d=getDict() + d['label'] = label + d['value'] = tab + d['tabType'] = tabType + d['tabDimVar'] = tabDimVar + d['tabColumnNames'] = cols + d['tabUnits'] = units + self.data.append(d) + + @property + def comment(self): + return '\n'.join([self.data[i]['value'] for i in self._IComment]) + + @comment.setter + def comment(self, comment): + splits = comment.split('\n') + for i,com in zip(self._IComment, splits): + self.data[i]['value'] = com + + @property + def _IComment(self): + """ return indices of comment line""" + return [] # Typical OpenFAST files have comment on second line [1] + + + def read(self, filename=None): + if filename: + self.filename = filename + if self.filename: + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + self._read() + else: + raise Exception('No filename provided') + + def _read(self): + + # --- Tables that can be detected based on the "Value" (first entry on line) + # TODO members for BeamDyn with mutliple key point ####### TODO PropSetID is Duplicate SubDyn and used in HydroDyn + NUMTAB_FROM_VAL_DETECT = ['HtFract' , 'TwrElev' , 'BlFract' , 'Genspd_TLU' , 'BlSpn' , 'WndSpeed' , 'HvCoefID' , 'AxCoefID' , 'JointID' , 'Dpth' , 'FillNumM' , 'MGDpth' , 'SimplCd' , 'RNodes' , 'kp_xr' , 'mu1' , 'TwrHtFr' , 'TwrRe' , 'WT_X'] + NUMTAB_FROM_VAL_DIM_VAR = ['NTwInpSt' , 'NumTwrNds' , 'NBlInpSt' , 'DLL_NumTrq' , 'NumBlNds' , 'NumCases' , 'NHvCoef' , 'NAxCoef' , 'NJoints' , 'NCoefDpth' , 'NFillGroups' , 'NMGDepths' , 1 , 'BldNodes' , 'kp_total' , 1 , 'NTwrHt' , 'NTwrRe' , 'NumTurbines'] + NUMTAB_FROM_VAL_VARNAME = ['TowProp' , 'TowProp' , 'BldProp' , 'DLLProp' , 'BldAeroNodes' , 'Cases' , 'HvCoefs' , 'AxCoefs' , 'Joints' , 'DpthProp' , 'FillGroups' , 'MGProp' , 'SmplProp' , 'BldAeroNodes' , 'MemberGeom' , 'DampingCoeffs' , 'TowerProp' , 'TowerRe', 'WindTurbines'] + NUMTAB_FROM_VAL_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 1 , 2 , 2 , 1 , 1 , 2 ] + NUMTAB_FROM_VAL_TYPE = ['num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' , 'num' , 'mix'] + # SubDyn + NUMTAB_FROM_VAL_DETECT += [ 'RJointID' , 'IJointID' , 'COSMID' , 'CMJointID' ] + NUMTAB_FROM_VAL_DIM_VAR += [ 'NReact' , 'NInterf' , 'NCOSMs' , 'NCmass' ] + NUMTAB_FROM_VAL_VARNAME += [ 'BaseJoints' , 'InterfaceJoints' , 'MemberCosineMatrix' , 'ConcentratedMasses'] + NUMTAB_FROM_VAL_NHEADER += [ 2 , 2 , 2 , 2 ] + NUMTAB_FROM_VAL_TYPE += [ 'mix' , 'num' , 'num' , 'num' ] + + + # --- Tables that can be detected based on the "Label" (second entry on line) + # NOTE: MJointID1, used by SubDyn and HydroDyn + NUMTAB_FROM_LAB_DETECT = ['NumAlf' , 'F_X' , 'MemberCd1' , 'MJointID1' , 'NOutLoc' , 'NOutCnt' , 'PropD' ,'Diam' ,'Type' ,'LineType' ] + NUMTAB_FROM_LAB_DIM_VAR = ['NumAlf' , 'NKInpSt' , 'NCoefMembers' , 'NMembers' , 'NMOutputs' , 'NMOutputs' , 'NPropSets' ,'NTypes' ,'NConnects' ,'NLines' ] + NUMTAB_FROM_LAB_VARNAME = ['AFCoeff' , 'TMDspProp' , 'MemberProp' , 'Members' , 'MemberOuts' , 'MemberOuts' , 'SectionProp' ,'LineTypes' ,'ConnectionProp' ,'LineProp' ] + NUMTAB_FROM_LAB_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 ] + NUMTAB_FROM_LAB_NOFFSET = [0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ] + NUMTAB_FROM_LAB_TYPE = ['num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' ,'mix' ,'mix' ,'mix' ] + # SubDyn + NUMTAB_FROM_LAB_DETECT += ['GuyanDampSize' , 'YoungE' , 'YoungE' , 'EA' , 'MatDens' ] + NUMTAB_FROM_LAB_DIM_VAR += [6 , 'NPropSets', 'NXPropSets', 'NCablePropSets' , 'NRigidPropSets'] + NUMTAB_FROM_LAB_VARNAME += ['GuyanDampMatrix' , 'BeamProp' , 'BeamPropX' , 'CableProp' , 'RigidProp' ] + NUMTAB_FROM_LAB_NHEADER += [0 , 2 , 2 , 2 , 2 ] + NUMTAB_FROM_LAB_NOFFSET += [1 , 0 , 0 , 0 , 0 ] + NUMTAB_FROM_LAB_TYPE += ['num' , 'num' , 'num' , 'num' , 'num' ] + # OLAF + NUMTAB_FROM_LAB_DETECT += ['GridName' ] + NUMTAB_FROM_LAB_DIM_VAR += ['nGridOut' ] + NUMTAB_FROM_LAB_VARNAME += ['GridOutputs'] + NUMTAB_FROM_LAB_NHEADER += [0 ] + NUMTAB_FROM_LAB_NOFFSET += [2 ] + NUMTAB_FROM_LAB_TYPE += ['mix' ] + + FILTAB_FROM_LAB_DETECT = ['FoilNm' ,'AFNames'] + FILTAB_FROM_LAB_DIM_VAR = ['NumFoil','NumAFfiles'] + FILTAB_FROM_LAB_VARNAME = ['FoilNm' ,'AFNames'] + + # Using lower case to be more tolerant.. + NUMTAB_FROM_VAL_DETECT_L = [s.lower() for s in NUMTAB_FROM_VAL_DETECT] + NUMTAB_FROM_LAB_DETECT_L = [s.lower() for s in NUMTAB_FROM_LAB_DETECT] + FILTAB_FROM_LAB_DETECT_L = [s.lower() for s in FILTAB_FROM_LAB_DETECT] + + # Reset data + self.data = [] + self.hasNodal=False + self.module = None + #with open(self.filename, 'r', errors="surrogateescape") as f: + with open(self.filename, 'r', errors="surrogateescape") as f: + lines=f.read().splitlines() + # IF NEEDED> DO THE FOLLOWING FORMATTING: + #lines = [str(l).encode('utf-8').decode('ascii','ignore') for l in f.read().splitlines()] + + # Fast files start with ! or - + #if lines[0][0]!='!' and lines[0][0]!='-': + # raise Exception('Fast file do not start with ! or -, is it the right format?') + + # Special filetypes + if detectAndReadExtPtfmSE(self, lines): + return + if self.detectAndReadAirfoilAD14(lines): + return + + # Parsing line by line, storing each line into a dictionary + i=0 + nComments = 0 + nWrongLabels = 0 + allowSpaceSeparatedList=False + iTab = 0 + + labOffset='' + while i0 \ + or line.upper().find('MESH-BASED OUTPUTS')>0 \ + or line.upper().find('OUTPUT CHANNELS' )>0: # "OutList - The next line(s) contains a list of output parameters. See OutListParameters.xlsx for a listing of available output channels, (-)'" + # TODO, lazy implementation so far, MAKE SUB FUNCTION + parts = re.match(r'^\W*\w+', line) + if parts: + firstword = parts.group(0).strip() + else: + raise NotImplementedError + remainer = re.sub(r'^\W*\w+\W*', '', line) + # Parsing outlist, and then we continue at a new "i" (to read END etc.) + OutList,i = parseFASTOutList(lines,i+1) + d = getDict() + if self.hasNodal: + d['label'] = firstword+'_Nodal' + else: + d['label'] = firstword + d['descr'] = remainer + d['tabType'] = TABTYPE_FIL # TODO + d['value'] = ['']+OutList + self.data.append(d) + if i>=len(lines): + break + # --- Here we cheat and force an exit of the input file + # The reason for this is that some files have a lot of things after the END, which will result in the file being intepreted as a wrong format due to too many comments + if i+20 or lines[i+2].lower().find('bldnd_bloutnd')>0): + self.hasNodal=True + else: + self.data.append(parseFASTInputLine('END of input file (the word "END" must appear in the first 3 columns of this last OutList line)',i+1)) + self.data.append(parseFASTInputLine('---------------------------------------------------------------------------------------',i+2)) + break + elif line.upper().find('SSOUTLIST' )>0 or line.upper().find('SDOUTLIST' )>0: + # SUBDYN Outlist doesn not follow regular format + self.data.append(parseFASTInputLine(line,i)) + # OUTLIST Exception for BeamDyn + OutList,i = parseFASTOutList(lines,i+1) + # TODO + for o in OutList: + d = getDict() + d['isComment'] = True + d['value']=o + self.data.append(d) + # --- Here we cheat and force an exit of the input file + self.data.append(parseFASTInputLine('END of input file (the word "END" must appear in the first 3 columns of this last OutList line)',i+1)) + self.data.append(parseFASTInputLine('---------------------------------------------------------------------------------------',i+2)) + break + elif line.upper().find('ADDITIONAL STIFFNESS')>0: + # TODO, lazy implementation so far, MAKE SUB FUNCTION + self.data.append(parseFASTInputLine(line,i)) + i +=1 + KDAdd = [] + for _ in range(19): + KDAdd.append(lines[i]) + i +=1 + d = getDict() + d['label'] = 'KDAdd' # TODO + d['tabType'] = TABTYPE_FIL # TODO + d['value'] = KDAdd + self.data.append(d) + if i>=len(lines): + break + elif line.upper().find('DISTRIBUTED PROPERTIES')>0: + self.data.append(parseFASTInputLine(line,i)); + i+=1; + self.readBeamDynProps(lines,i) + return + + # --- Parsing of standard lines: value(s) key comment + line = lines[i] + d = parseFASTInputLine(line,i,allowSpaceSeparatedList) + labelRaw =d['label'].lower() + d['label']+=labOffset + + # --- Handling of special files + if labelRaw=='kp_total': + # BeamDyn has weird space speparated list around keypoint definition + allowSpaceSeparatedList=True + elif labelRaw=='numcoords': + # TODO, lazy implementation so far, MAKE SUB FUNCTION + if isStr(d['value']): + if d['value'][0]=='@': + # it's a ref to the airfoil coord file + pass + else: + if not strIsInt(d['value']): + raise WrongFormatError('Wrong value of NumCoords') + if int(d['value'])<=0: + pass + else: + self.data.append(d); i+=1; + # 3 comment lines + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + splits=cleanAfterChar(cleanLine(lines[i]),'!').split() + # Airfoil ref point + try: + pos=[float(splits[0]), float(splits[1])] + except: + raise WrongFormatError('Wrong format while reading coordinates of airfoil reference') + i+=1 + d = getDict() + d['label'] = 'AirfoilRefPoint' + d['value'] = pos + self.data.append(d) + # 2 comment lines + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + self.data.append(parseFASTInputLine(lines[i],i)); i+=1; + # Table of coordinats itself + d = getDict() + d['label'] = 'AirfoilCoord' + d['tabDimVar'] = 'NumCoords' + d['tabType'] = TABTYPE_NUM_WITH_HEADERCOM + nTabLines = self[d['tabDimVar']]-1 # SOMEHOW ONE DATA POINT LESS + d['value'], d['tabColumnNames'],_ = parseFASTNumTable(self.filename,lines[i:i+nTabLines+1],nTabLines,i,1) + d['tabUnits'] = ['(-)','(-)'] + self.data.append(d) + break + + elif labelRaw=='re': + nAirfoilTab = self['NumTabs'] + iTab +=1 + if nAirfoilTab>1: + labOffset ='_'+str(iTab) + d['label']=labelRaw+labOffset + + #print('label>',d['label'],'<',type(d['label'])); + #print('value>',d['value'],'<',type(d['value'])); + #print(isStr(d['value'])) + #if isStr(d['value']): + # print(d['value'].lower() in NUMTAB_FROM_VAL_DETECT_L) + + + # --- Handling of tables + if isStr(d['value']) and d['value'].lower() in NUMTAB_FROM_VAL_DETECT_L: + # Table with numerical values, + ii = NUMTAB_FROM_VAL_DETECT_L.index(d['value'].lower()) + tab_type = NUMTAB_FROM_VAL_TYPE[ii] + if tab_type=='num': + d['tabType'] = TABTYPE_NUM_WITH_HEADER + else: + d['tabType'] = TABTYPE_MIX_WITH_HEADER + d['label'] = NUMTAB_FROM_VAL_VARNAME[ii]+labOffset + d['tabDimVar'] = NUMTAB_FROM_VAL_DIM_VAR[ii] + nHeaders = NUMTAB_FROM_VAL_NHEADER[ii] + nTabLines=0 + if isinstance(d['tabDimVar'],int): + nTabLines = d['tabDimVar'] + else: + nTabLines = self[d['tabDimVar']] + #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); + d['value'], d['tabColumnNames'], d['tabUnits'] = parseFASTNumTable(self.filename,lines[i:i+nTabLines+nHeaders], nTabLines, i, nHeaders, tableType=tab_type, varNumLines=d['tabDimVar']) + i += nTabLines+nHeaders-1 + + # --- Temporary hack for e.g. SubDyn, that has duplicate table, impossible to detect in the current way... + # So we remove the element form the list one read + del NUMTAB_FROM_VAL_DETECT[ii] + del NUMTAB_FROM_VAL_DIM_VAR[ii] + del NUMTAB_FROM_VAL_VARNAME[ii] + del NUMTAB_FROM_VAL_NHEADER[ii] + del NUMTAB_FROM_VAL_TYPE [ii] + del NUMTAB_FROM_VAL_DETECT_L[ii] + + elif isStr(labelRaw) and labelRaw in NUMTAB_FROM_LAB_DETECT_L: + ii = NUMTAB_FROM_LAB_DETECT_L.index(labelRaw) + tab_type = NUMTAB_FROM_LAB_TYPE[ii] + # Special case for airfoil data, the table follows NumAlf, so we add d first + doDelete =True + if labelRaw=='numalf': + doDelete =False + d['tabType']=TABTYPE_NOT_A_TAB + self.data.append(d) + # Creating a new dictionary for the table + d = {'value':None, 'label':'NumAlf'+labOffset, 'isComment':False, 'descr':'', 'tabType':None} + i += 1 + nHeaders = NUMTAB_FROM_LAB_NHEADER[ii] + nOffset = NUMTAB_FROM_LAB_NOFFSET[ii] + if nOffset>0: + # Creating a dictionary for that entry + dd = {'value':d['value'], 'label':d['label']+labOffset, 'isComment':False, 'descr':d['descr'], 'tabType':TABTYPE_NOT_A_TAB} + self.data.append(dd) + + d['label'] = NUMTAB_FROM_LAB_VARNAME[ii] + d['tabDimVar'] = NUMTAB_FROM_LAB_DIM_VAR[ii] + if d['label'].lower()=='afcoeff' : + d['tabType'] = TABTYPE_NUM_WITH_HEADERCOM + else: + if tab_type=='num': + d['tabType'] = TABTYPE_NUM_WITH_HEADER + else: + d['tabType'] = TABTYPE_MIX_WITH_HEADER + if isinstance(d['tabDimVar'],int): + nTabLines = d['tabDimVar'] + else: + nTabLines = self[d['tabDimVar']+labOffset] + + d['label'] += labOffset + #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); + d['value'], d['tabColumnNames'], d['tabUnits'] = parseFASTNumTable(self.filename,lines[i:i+nTabLines+nHeaders+nOffset],nTabLines,i, nHeaders, tableType=tab_type, nOffset=nOffset, varNumLines=d['tabDimVar']) + i += nTabLines+1-nOffset + + # --- Temporary hack for e.g. SubDyn, that has duplicate table, impossible to detect in the current way... + # So we remove the element form the list one read + if doDelete: + del NUMTAB_FROM_LAB_DETECT[ii] + del NUMTAB_FROM_LAB_DIM_VAR[ii] + del NUMTAB_FROM_LAB_VARNAME[ii] + del NUMTAB_FROM_LAB_NHEADER[ii] + del NUMTAB_FROM_LAB_NOFFSET[ii] + del NUMTAB_FROM_LAB_TYPE [ii] + del NUMTAB_FROM_LAB_DETECT_L[ii] + + elif isStr(d['label']) and d['label'].lower() in FILTAB_FROM_LAB_DETECT_L: + ii = FILTAB_FROM_LAB_DETECT_L.index(d['label'].lower()) + d['label'] = FILTAB_FROM_LAB_VARNAME[ii]+labOffset + d['tabDimVar'] = FILTAB_FROM_LAB_DIM_VAR[ii] + d['tabType'] = TABTYPE_FIL + nTabLines = self[d['tabDimVar']] + #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); + d['value'] = parseFASTFilTable(lines[i:i+nTabLines],nTabLines,i) + i += nTabLines-1 + + + + self.data.append(d) + i += 1 + # --- Safety checks + if d['isComment']: + #print(line) + nComments +=1 + else: + if hasSpecialChars(d['label']): + nWrongLabels +=1 + #print('label>',d['label'],'<',type(d['label']),line); + if i>3: # first few lines may be comments, we allow it + #print('Line',i,'Label:',d['label']) + raise WrongFormatError('Special Character found in Label: `{}`, for line: `{}`'.format(d['label'],line)) + if len(d['label'])==0: + nWrongLabels +=1 + if nComments>len(lines)*0.35: + #print('Comment fail',nComments,len(lines),self.filename) + raise WrongFormatError('Most lines were read as comments, probably not a FAST Input File: {}'.format(self.filename)) + if nWrongLabels>len(lines)*0.10: + #print('Label fail',nWrongLabels,len(lines),self.filename) + raise WrongFormatError('Too many lines with wrong labels, probably not a FAST Input File {}:'.format(self.filename)) + + # --- END OF FOR LOOP ON LINES + + # --- PostReading checks + labels = self.keys() + duplicates = set([x for x in labels if (labels.count(x) > 1) and x!='OutList' and x.strip()!='-']) + if len(duplicates)>0: + print('[WARN] Duplicate labels found in file: '+self.filename) + print(' Duplicates: '+', '.join(duplicates)) + print(' It\'s strongly recommended to make them unique! ') +# except WrongFormatError as e: +# raise WrongFormatError('Fast File {}: '.format(self.filename)+'\n'+e.args[0]) +# except Exception as e: +# raise e +# # print(e) +# raise Exception('Fast File {}: '.format(self.filename)+'\n'+e.args[0]) + self._lines = lines + + + def toString(self): + s='' + # Special file formats, TODO subclass + def toStringVLD(val,lab,descr): + val='{}'.format(val) + lab='{}'.format(lab) + if len(val)<13: + val='{:13s}'.format(val) + if len(lab)<13: + lab='{:13s}'.format(lab) + return val+' '+lab+' - '+descr.strip().strip('-').strip()+'\n' + + def beamdyn_section_mat_tostring(x,K,M): + def mat_tostring(M,fmt='24.16e'): + return '\n'.join([' '+' '.join(['{:24.16E}'.format(m) for m in M[i,:]]) for i in range(np.size(M,1))]) + s='' + s+='{:.6f}\n'.format(x) + s+=mat_tostring(K) + #s+=np.array2string(K) + s+='\n' + s+='\n' + s+=mat_tostring(M) + #s+=np.array2string(M) + s+='\n' + s+='\n' + return s + + + for i in range(len(self.data)): + d=self.data[i] + if d['isComment']: + s+='{}'.format(d['value']) + elif d['tabType']==TABTYPE_NOT_A_TAB: + if isinstance(d['value'], list): + sList=', '.join([str(x) for x in d['value']]) + s+='{} {} {}'.format(sList,d['label'],d['descr']) + else: + s+=toStringVLD(d['value'],d['label'],d['descr']).strip() + elif d['tabType']==TABTYPE_NUM_WITH_HEADER: + if d['tabColumnNames'] is not None: + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) + #s+=d['descr'] # Not ready for that + if d['tabUnits'] is not None: + s+='\n' + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) + newline='\n' + else: + newline='' + if np.size(d['value'],0) > 0 : + s+=newline + s+='\n'.join('\t'.join( ('{:15.0f}'.format(x) if int(x)==x else '{:15.8e}'.format(x) ) for x in y) for y in d['value']) + elif d['tabType']==TABTYPE_MIX_WITH_HEADER: + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) + if d['tabUnits'] is not None: + s+='\n' + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) + if np.size(d['value'],0) > 0 : + s+='\n' + s+='\n'.join('\t'.join('{}'.format(x) for x in y) for y in d['value']) + elif d['tabType']==TABTYPE_NUM_WITH_HEADERCOM: + s+='! {}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) + s+='! {}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) + s+='\n'.join('\t'.join('{:15.8e}'.format(x) for x in y) for y in d['value']) + elif d['tabType']==TABTYPE_FIL: + #f.write('{} {} {}\n'.format(d['value'][0],d['tabDetect'],d['descr'])) + s+='{} {} {}\n'.format(d['value'][0],d['label'],d['descr']) # TODO? + s+='\n'.join(fil for fil in d['value'][1:]) + elif d['tabType']==TABTYPE_NUM_BEAMDYN: + # TODO use dedicated sub-class + data = d['value'] + Cols =['Span'] + Cols+=['K{}{}'.format(i+1,j+1) for i in range(6) for j in range(6)] + Cols+=['M{}{}'.format(i+1,j+1) for i in range(6) for j in range(6)] + for i in np.arange(len(data['span'])): + x = data['span'][i] + K = data['K'][i] + M = data['M'][i] + s += beamdyn_section_mat_tostring(x,K,M) + else: + raise Exception('Unknown table type for variable {}'.format(d)) + if i0: + # Hack for tower files, we add the modes + x=Val[:,0] + Modes=np.zeros((x.shape[0],4)) + Modes[:,0] = x**2 * self['TwFAM1Sh(2)'] \ + + x**3 * self['TwFAM1Sh(3)'] \ + + x**4 * self['TwFAM1Sh(4)'] \ + + x**5 * self['TwFAM1Sh(5)'] \ + + x**6 * self['TwFAM1Sh(6)'] + Modes[:,1] = x**2 * self['TwFAM2Sh(2)'] \ + + x**3 * self['TwFAM2Sh(3)'] \ + + x**4 * self['TwFAM2Sh(4)'] \ + + x**5 * self['TwFAM2Sh(5)'] \ + + x**6 * self['TwFAM2Sh(6)'] + Modes[:,2] = x**2 * self['TwSSM1Sh(2)'] \ + + x**3 * self['TwSSM1Sh(3)'] \ + + x**4 * self['TwSSM1Sh(4)'] \ + + x**5 * self['TwSSM1Sh(5)'] \ + + x**6 * self['TwSSM1Sh(6)'] + Modes[:,3] = x**2 * self['TwSSM2Sh(2)'] \ + + x**3 * self['TwSSM2Sh(3)'] \ + + x**4 * self['TwSSM2Sh(4)'] \ + + x**5 * self['TwSSM2Sh(5)'] \ + + x**6 * self['TwSSM2Sh(6)'] + Val = np.hstack((Val,Modes)) + Cols = Cols + ['ShapeForeAft1_[-]','ShapeForeAft2_[-]','ShapeSideSide1_[-]','ShapeSideSide2_[-]'] + + name=d['label'] + + if name=='DampingCoeffs': + pass + else: + dfs[name]=pd.DataFrame(data=Val,columns=Cols) + elif d['tabType'] in [TABTYPE_NUM_BEAMDYN]: + span = d['value']['span'] + M = d['value']['M'] + K = d['value']['K'] + nSpan=len(span) + MM=np.zeros((nSpan,1+36+36)) + MM[:,0] = span + MM[:,1:37] = K.reshape(nSpan,36) + MM[:,37:] = M.reshape(nSpan,36) + Cols =['Span'] + Cols+=['K{}{}'.format(i+1,j+1) for i in range(6) for j in range(6)] + Cols+=['M{}{}'.format(i+1,j+1) for i in range(6) for j in range(6)] + # Putting the main terms first + IAll = range(1+36+36) + IMain= [0] + [i*6+i+1 for i in range(6)] + [i*6+i+37 for i in range(6)] + IOrg = IMain + [i for i in range(1+36+36) if i not in IMain] + Cols = [Cols[i] for i in IOrg] + data = MM[:,IOrg] + name=d['label'] + dfs[name]=pd.DataFrame(data=data,columns=Cols) + if len(dfs)==1: + dfs=dfs[list(dfs.keys())[0]] + return dfs + + def toGraph(self): + from .fast_input_file_graph import fastToGraph + return fastToGraph(self) + + + +# --------------------------------------------------------------------------------} +# --- SubReaders /detectors +# --------------------------------------------------------------------------------{ + + + def detectAndReadAirfoilAD14(self,lines): + if len(lines)<14: + return False + # Reading number of tables + L3 = lines[2].strip().split() + if len(L3)<=0: + return False + if not strIsInt(L3[0]): + return False + nTables=int(L3[0]) + # Reading table ID + L4 = lines[3].strip().split() + if len(L4)<=nTables: + return False + TableID=L4[:nTables] + if nTables==1: + TableID=[''] + # Keywords for file format + KW1=lines[12].strip().split() + KW2=lines[13].strip().split() + if len(KW1)>nTables and len(KW2)>nTables: + if KW1[nTables].lower()=='angle' and KW2[nTables].lower()=='minimum': + d = getDict(); d['isComment'] = True; d['value'] = lines[0]; self.data.append(d); + d = getDict(); d['isComment'] = True; d['value'] = lines[1]; self.data.append(d); + for i in range(2,14): + splits = lines[i].split() + #print(splits) + d = getDict() + d['label'] = ' '.join(splits[1:]) # TODO + d['descr'] = ' '.join(splits[1:]) # TODO + d['value'] = float(splits[0]) + self.data.append(d) + #pass + #for i in range(2,14): + nTabLines=0 + while 14+nTabLines0 : + nTabLines +=1 + #data = np.array([lines[i].strip().split() for i in range(14,len(lines)) if len(lines[i])>0]).astype(float) + #data = np.array([lines[i].strip().split() for i in takewhile(lambda x: len(lines[i].strip())>0, range(14,len(lines)-1))]).astype(float) + data = np.array([lines[i].strip().split() for i in range(14,nTabLines+14)]).astype(float) + #print(data) + d = getDict() + d['label'] = 'Polar' + d['tabDimVar'] = nTabLines + d['tabType'] = TABTYPE_NUM_NO_HEADER + d['value'] = data + if np.size(data,1)==1+nTables*3: + d['tabColumnNames'] = ['Alpha']+[n+l for l in TableID for n in ['Cl','Cd','Cm']] + d['tabUnits'] = ['(deg)']+['(-)' , '(-)' , '(-)']*nTables + elif np.size(data,1)==1+nTables*2: + d['tabColumnNames'] = ['Alpha']+[n+l for l in TableID for n in ['Cl','Cd']] + d['tabUnits'] = ['(deg)']+['(-)' , '(-)']*nTables + else: + d['tabColumnNames'] = ['col{}'.format(j) for j in range(np.size(data,1))] + self.data.append(d) + return True + + def readBeamDynProps(self,lines,iStart): + nStations=self['station_total'] + #M=np.zeros((nStations,1+36+36)) + M = np.zeros((nStations,6,6)) + K = np.zeros((nStations,6,6)) + span = np.zeros(nStations) + i=iStart; + try: + for j in range(nStations): + # Read span location + span[j]=float(lines[i]); i+=1; + # Read stiffness matrix + K[j,:,:]=np.array((' '.join(lines[i:i+6])).split()).astype(float).reshape(6,6) + i+=7 + # Read mass matrix + M[j,:,:]=np.array((' '.join(lines[i:i+6])).split()).astype(float).reshape(6,6) + i+=7 + except: + raise WrongFormatError('An error occured while reading section {}/{}'.format(j+1,nStations)) + + d = getDict() + d['label'] = 'BeamProperties' + d['descr'] = '' + d['tabType'] = TABTYPE_NUM_BEAMDYN + d['value'] = {'span':span, 'K':K, 'M':M} + self.data.append(d) + +# --------------------------------------------------------------------------------} +# --- Helper functions +# --------------------------------------------------------------------------------{ +def isStr(s): + # Python 2 and 3 compatible + # Two options below + # NOTE: all this avoided since we import str from builtins + # --- Version 2 + # isString = False; + # if(isinstance(s, str)): + # isString = True; + # try: + # if(isinstance(s, basestring)): # todo unicode as well + # isString = True; + # except NameError: + # pass; + # return isString + # --- Version 1 + # try: + # basestring # python 2 + # return isinstance(s, basestring) or isinstance(s,unicode) + # except NameError: + # basestring=str #python 3 + # return isinstance(s, str) + return isinstance(s, str) + +def strIsFloat(s): + #return s.replace('.',',1').isdigit() + try: + float(s) + return True + except: + return False + +def strIsBool(s): + return s.lower() in ['true','false','t','f'] + +def strIsInt(s): + s = str(s) + if s[0] in ('-', '+'): + return s[1:].isdigit() + return s.isdigit() + +def strToBool(s): + return s.lower() in ['true','t'] + +def hasSpecialChars(s): + # fast allows for parenthesis + # For now we allow for - but that's because of BeamDyn geometry members + return not re.match("^[\"\'a-zA-Z0-9_()-]*$", s) + +def cleanLine(l): + # makes a string single space separated + l = l.replace('\t',' ') + l = ' '.join(l.split()) + l = l.strip() + return l + +def cleanAfterChar(l,c): + # remove whats after a character + n = l.find(c); + if n>0: + return l[:n] + else: + return l + +def getDict(): + return {'value':None, 'label':'', 'isComment':False, 'descr':'', 'tabType':TABTYPE_NOT_A_TAB} + +def _merge_value(splits): + + merged = splits.pop(0) + if merged[0] == '"': + while merged[-1] != '"': + merged += " "+splits.pop(0) + splits.insert(0, merged) + + + + +def parseFASTInputLine(line_raw,i,allowSpaceSeparatedList=False): + d = getDict() + #print(line_raw) + try: + # preliminary cleaning (Note: loss of formatting) + line = cleanLine(line_raw) + # Comment + if any(line.startswith(c) for c in ['#','!','--','==']) or len(line)==0: + d['isComment']=True + d['value']=line_raw + return d + if line.lower().startswith('end'): + sp =line.split() + if len(sp)>2 and sp[1]=='of': + d['isComment']=True + d['value']=line_raw + + # Detecting lists + List=[]; + iComma=line.find(',') + if iComma>0 and iComma<30: + fakeline=line.replace(' ',',') + fakeline=re.sub(',+',',',fakeline) + csplits=fakeline.split(',') + # Splitting based on comma and looping while it's numbers of booleans + ii=0 + s=csplits[ii] + #print(csplits) + while strIsFloat(s) or strIsBool(s) and ii=len(csplits): + raise WrongFormatError('Wrong number of list values') + s = csplits[ii] + #print('[INFO] Line {}: Found list: '.format(i),List) + # Defining value and remaining splits + if len(List)>=2: + d['value']=List + line_remaining=line + # eating line, removing each values + for iii in range(ii): + sValue=csplits[iii] + ipos=line_remaining.find(sValue) + line_remaining = line_remaining[ipos+len(sValue):] + splits=line_remaining.split() + iNext=0 + else: + # It's not a list, we just use space as separators + splits=line.split(' ') + _merge_value(splits) + s=splits[0] + + if strIsInt(s): + d['value']=int(s) + if allowSpaceSeparatedList and len(splits)>1: + if strIsInt(splits[1]): + d['value']=splits[0]+ ' '+splits[1] + elif strIsFloat(s): + d['value']=float(s) + elif strIsBool(s): + d['value']=strToBool(s) + else: + d['value']=s + iNext=1 + + # Extracting label (TODO, for now only second split) + bOK=False + while (not bOK) and iNext comment assumed'.format(i+1)) + d['isComment']=True + d['value']=line_raw + iNext = len(splits)+1 + + # Recombining description + if len(splits)>=iNext+1: + d['descr']=' '.join(splits[iNext:]) + except WrongFormatError as e: + raise WrongFormatError('Line {}: '.format(i+1)+e.args[0]) + except Exception as e: + raise Exception('Line {}: '.format(i+1)+e.args[0]) + + return d + +def parseFASTOutList(lines,iStart): + OutList=[] + i = iStart + MAX=200 + while iMAX : + raise Exception('More that 200 lines found in outlist') + if i>=len(lines): + print('[WARN] End of file reached while reading Outlist') + #i=min(i+1,len(lines)) + return OutList,iStart+len(OutList) + + +def extractWithinParenthesis(s): + mo = re.search(r'\((.*)\)', s) + if mo: + return mo.group(1) + return '' + +def extractWithinBrackets(s): + mo = re.search(r'\((.*)\)', s) + if mo: + return mo.group(1) + return '' + +def detectUnits(s,nRef): + nPOpen=s.count('(') + nPClos=s.count(')') + nBOpen=s.count('[') + nBClos=s.count(']') + + sep='!#@#!' + if (nPOpen == nPClos) and (nPOpen>=nRef): + #that's pretty good + Units=s.replace('(','').replace(')',sep).split(sep)[:-1] + elif (nBOpen == nBClos) and (nBOpen>=nRef): + Units=s.replace('[','').replace(']',sep).split(sep)[:-1] + else: + Units=s.split() + return Units + + +def parseFASTNumTable(filename,lines,n,iStart,nHeaders=2,tableType='num',nOffset=0, varNumLines=''): + """ + First lines of data starts at: nHeaders+nOffset + + """ + Tab = None + ColNames = None + Units = None + + + if len(lines)!=n+nHeaders+nOffset: + raise BrokenFormatError('Not enough lines in table: {} lines instead of {}\nFile:{}'.format(len(lines)-nHeaders,n,filename)) + try: + if nHeaders==0: + # Extract number of values from number of numerical values on first line + numeric_const_pattern = r'[-+]? (?: (?: \d* \. \d+ ) | (?: \d+ \.? ) )(?: [Ee] [+-]? \d+ ) ?' + rx = re.compile(numeric_const_pattern, re.VERBOSE) + header = cleanAfterChar(lines[nOffset], '!') + if tableType=='num': + dat= np.array(rx.findall(header)).astype(float) + ColNames=['C{}'.format(j) for j in range(len(dat))] + else: + raise NotImplementedError('Reading FAST tables with no headers for type different than num not implemented yet') + + elif nHeaders>=1: + # Extract column names + i = 0 + sTmp = cleanLine(lines[i]) + sTmp = cleanAfterChar(sTmp,'[') + sTmp = cleanAfterChar(sTmp,'(') + sTmp = cleanAfterChar(sTmp,'!') + sTmp = cleanAfterChar(sTmp,'#') + if sTmp.startswith('!'): + sTmp=sTmp[1:].strip() + ColNames=sTmp.split() + if nHeaders>=2: + # Extract units + i = 1 + sTmp = cleanLine(lines[i]) + sTmp = cleanAfterChar(sTmp,'!') + sTmp = cleanAfterChar(sTmp,'#') + if sTmp.startswith('!'): + sTmp=sTmp[1:].strip() + + Units = detectUnits(sTmp,len(ColNames)) + Units = ['({})'.format(u.strip()) for u in Units] + # Forcing user to match number of units and column names + if len(ColNames) != len(Units): + print(ColNames) + print(Units) + print('[WARN] {}: Line {}: Number of column names different from number of units in table'.format(filename, iStart+i+1)) + + nCols=len(ColNames) + + if tableType=='num': + if n==0: + Tab = np.zeros((n, nCols)) + for i in range(nHeaders+nOffset,n+nHeaders+nOffset): + l = cleanAfterChar(lines[i].lower(),'!') + l = cleanAfterChar(l,'#') + v = l.split() + if len(v) != nCols: + # Discarding SubDyn special cases + if ColNames[-1].lower() not in ['nodecnt']: + print('[WARN] {}: Line {}: number of data different from number of column names. ColumnNames: {}'.format(filename, iStart+i+1, ColNames)) + if i==nHeaders+nOffset: + # Node Cnt + if len(v) != nCols: + if ColNames[-1].lower()== 'nodecnt': + ColNames = ColNames+['Col']*(len(v)-nCols) + Units = Units+['Col']*(len(v)-nCols) + + nCols=len(v) + Tab = np.zeros((n, nCols)) + # Accounting for TRUE FALSE and converting to float + v = [s.replace('true','1').replace('false','0').replace('noprint','0').replace('print','1') for s in v] + v = [float(s) for s in v[0:nCols]] + if len(v) < nCols: + raise Exception('Number of data is lower than number of column names') + Tab[i-nHeaders-nOffset,:] = v + elif tableType=='mix': + # a mix table contains a mixed of strings and floats + # For now, we are being a bit more relaxed about the number of columns + if n==0: + Tab = np.zeros((n, nCols)).astype(object) + for i in range(nHeaders+nOffset,n+nHeaders+nOffset): + l = lines[i] + l = cleanAfterChar(l,'!') + l = cleanAfterChar(l,'#') + v = l.split() + if l.startswith('---'): + raise BrokenFormatError('Error reading line {} while reading table. Is the variable `{}` set correctly?'.format(iStart+i+1, varNumLines)) + if len(v) != nCols: + # Discarding SubDyn special cases + if ColNames[-1].lower() not in ['cosmid', 'ssifile']: + print('[WARN] {}: Line {}: Number of data is different than number of column names. Column Names: {}'.format(filename,iStart+1+i, ColNames)) + if i==nHeaders+nOffset: + if len(v)>nCols: + ColNames = ColNames+['Col']*(len(v)-nCols) + Units = Units+['Col']*(len(v)-nCols) + nCols=len(v) + Tab = np.zeros((n, nCols)).astype(object) + v=v[0:min(len(v),nCols)] + Tab[i-nHeaders-nOffset,0:len(v)] = v + # If all values are float, we convert to float + if all([strIsFloat(x) for x in Tab.ravel()]): + Tab=Tab.astype(float) + else: + raise Exception('Unknown table type') + + ColNames = ColNames[0:nCols] + if Units is not None: + Units = Units[0:nCols] + Units = ['('+u.replace('(','').replace(')','')+')' for u in Units] + if nHeaders==0: + ColNames=None + + except Exception as e: + raise BrokenFormatError('Line {}: {}'.format(iStart+i+1,e.args[0])) + return Tab, ColNames, Units + + +def parseFASTFilTable(lines,n,iStart): + Tab = [] + try: + i=0 + if len(lines)!=n: + raise WrongFormatError('Not enough lines in table: {} lines instead of {}'.format(len(lines),n)) + for i in range(n): + l = lines[i].split() + #print(l[0].strip()) + Tab.append(l[0].strip()) + + except Exception as e: + raise Exception('Line {}: '.format(iStart+i+1)+e.args[0]) + return Tab + + + +# --------------------------------------------------------------------------------} +# --------------------------------------------------------------------------------} +# --------------------------------------------------------------------------------} +# --- Predefined types (may change with OpenFAST version..) +# --------------------------------------------------------------------------------{ +# --------------------------------------------------------------------------------{ +# --------------------------------------------------------------------------------{ + +# --------------------------------------------------------------------------------} +# --- AeroDyn Blade +# --------------------------------------------------------------------------------{ +class EDBladeFile(FASTInputFileBase): + @classmethod + def from_fast_input_file(cls, parent): + self = cls() + self.setData(filename=parent.filename, data=parent.data, hasNodal=parent.hasNodal, module='EDBlade') + return self + + def __init__(self, filename=None, **kwargs): + FASTInputFileBase.__init__(self, filename, **kwargs) + if filename is None: + # Define a prototype for this file format + self.addComment('------- ELASTODYN V1.00.* INDIVIDUAL BLADE INPUT FILE --------------------------') + self.addComment('ElastoDyn blade definition, written by EDBladeFile.') + self.addComment('---------------------- BLADE PARAMETERS ----------------------------------------') + self.addValKey( 0 , 'NBlInpSt' , 'Number of blade input stations (-)') + self.addValKey( 1. , 'BldFlDmp(1)', 'Blade flap mode #1 structural damping in percent of critical (%)') + self.addValKey( 1. , 'BldFlDmp(2)', 'Blade flap mode #2 structural damping in percent of critical (%)') + self.addValKey( 1. , 'BldEdDmp(1)', 'Blade edge mode #1 structural damping in percent of critical (%)') + self.addComment('---------------------- BLADE ADJUSTMENT FACTORS --------------------------------') + self.addValKey( 1. , 'FlStTunr(1)', 'Blade flapwise modal stiffness tuner, 1st mode (-)') + self.addValKey( 1. , 'FlStTunr(2)', 'Blade flapwise modal stiffness tuner, 2nd mode (-)') + self.addValKey( 1. , 'AdjBlMs' , 'Factor to adjust blade mass density (-)') + self.addValKey( 1. , 'AdjFlSt' , 'Factor to adjust blade flap stiffness (-)') + self.addValKey( 1. , 'AdjEdSt' , 'Factor to adjust blade edge stiffness (-)') + self.addComment('---------------------- DISTRIBUTED BLADE PROPERTIES ----------------------------') + self.addTable('BldProp', np.zeros((0,6)), tabType=1, tabDimVar='NBlInpSt', cols=['BlFract', 'PitchAxis', 'StrcTwst', 'BMassDen', 'FlpStff', 'EdgStff'], units=['(-)', '(-)', '(deg)', '(kg/m)', '(Nm^2)', '(Nm^2)']) + self.addComment('---------------------- BLADE MODE SHAPES ---------------------------------------') + self.addValKey( 1.0 , 'BldFl1Sh(2)', 'Flap mode 1, coeff of x^2') + self.addValKey( 0.0 , 'BldFl1Sh(3)', ' , coeff of x^3') + self.addValKey( 0.0 , 'BldFl1Sh(4)', ' , coeff of x^4') + self.addValKey( 0.0 , 'BldFl1Sh(5)', ' , coeff of x^5') + self.addValKey( 0.0 , 'BldFl1Sh(6)', ' , coeff of x^6') + self.addValKey( 1.0 , 'BldFl2Sh(2)', 'Flap mode 2, coeff of x^2') + self.addValKey( 0.0 , 'BldFl2Sh(3)', ' , coeff of x^3') + self.addValKey( 0.0 , 'BldFl2Sh(4)', ' , coeff of x^4') + self.addValKey( 0.0 , 'BldFl2Sh(5)', ' , coeff of x^5') + self.addValKey( 0.0 , 'BldFl2Sh(6)', ' , coeff of x^6') + self.addValKey( 1.0 , 'BldEdgSh(2)', 'Edge mode 1, coeff of x^2') + self.addValKey( 0.0 , 'BldEdgSh(3)', ' , coeff of x^3') + self.addValKey( 0.0 , 'BldEdgSh(4)', ' , coeff of x^4') + self.addValKey( 0.0 , 'BldEdgSh(5)', ' , coeff of x^5') + self.addValKey( 0.0 , 'BldEdgSh(6)', ' , coeff of x^6') + else: + # fix some stuff that generic reader fail at + self.data[1] = self._lines[1] + self.module='EDBlade' + + def _writeSanityChecks(self): + """ Sanity checks before write """ + self['NBlInpSt']=self['BldProp'].shape[0] + # Sum of Coeffs should be 1 + for s in ['BldFl1Sh','BldFl2Sh','BldEdgSh']: + sumcoeff=np.sum([self[s+'('+str(i)+')'] for i in [2,3,4,5,6] ]) + if np.abs(sumcoeff-1)>1e-4: + print('[WARN] Sum of coefficients for polynomial {} not equal to 1 ({}). File: {}'.format(s, sumcoeff, self.filename)) + + def _toDataFrame(self): + df = FASTInputFileBase._toDataFrame(self) + # We add the shape functions for EDBladeFile + x=df['BlFract_[-]'].values + Modes=np.zeros((x.shape[0],3)) + Modes[:,0] = x**2 * self['BldFl1Sh(2)'] \ + + x**3 * self['BldFl1Sh(3)'] \ + + x**4 * self['BldFl1Sh(4)'] \ + + x**5 * self['BldFl1Sh(5)'] \ + + x**6 * self['BldFl1Sh(6)'] + Modes[:,1] = x**2 * self['BldFl2Sh(2)'] \ + + x**3 * self['BldFl2Sh(3)'] \ + + x**4 * self['BldFl2Sh(4)'] \ + + x**5 * self['BldFl2Sh(5)'] \ + + x**6 * self['BldFl2Sh(6)'] + Modes[:,2] = x**2 * self['BldEdgSh(2)'] \ + + x**3 * self['BldEdgSh(3)'] \ + + x**4 * self['BldEdgSh(4)'] \ + + x**5 * self['BldEdgSh(5)'] \ + + x**6 * self['BldEdgSh(6)'] + df[['ShapeFlap1_[-]','ShapeFlap2_[-]','ShapeEdge1_[-]']]=Modes + return df + + @property + def _IComment(self): return [1] + + +# --------------------------------------------------------------------------------} +# --- AeroDyn Blade +# --------------------------------------------------------------------------------{ +class ADBladeFile(FASTInputFileBase): + @classmethod + def from_fast_input_file(cls, parent): + self = cls() + self.setData(filename=parent.filename, data=parent.data, hasNodal=parent.hasNodal, module='ADBlade') + return self + + def __init__(self, filename=None, **kwargs): + FASTInputFileBase.__init__(self, filename, **kwargs) + if filename is None: + # Define a prototype for this file format + self.addComment('------- AERODYN BLADE DEFINITION INPUT FILE ----------------------------------------------') + self.addComment('Aerodynamic blade definition, written by ADBladeFile') + self.addComment('====== Blade Properties =================================================================') + self.addKeyVal('NumBlNds', 0, 'Number of blade nodes used in the analysis (-)') + self.addTable('BldAeroNodes', np.zeros((0,7)), tabType=1, tabDimVar='NumBlNds', cols=['BlSpn', 'BlCrvAC', 'BlSwpAC', 'BlCrvAng', 'BlTwist', 'BlChord', 'BlAFID'], units=['(m)', '(m)', '(m)', '(deg)', '(deg)', '(m)', '(-)']) + self.module='ADBlade' + + def _writeSanityChecks(self): + """ Sanity checks before write""" + self['NumBlNds']=self['BldAeroNodes'].shape[0] + aeroNodes = self['BldAeroNodes'] + # TODO double check this calculation with gradient + dr = np.gradient(aeroNodes[:,0]) + dx = np.gradient(aeroNodes[:,1]) + crvAng = np.degrees(np.arctan2(dx,dr))*np.pi/180 + if np.mean(np.abs(crvAng-aeroNodes[:,3]))>0.1: + print('[WARN] BlCrvAng might not be computed correctly') + + def _toDataFrame(self): + df = FASTInputFileBase._toDataFrame(self) + aeroNodes = self['BldAeroNodes'] + r = aeroNodes[:,0] + chord = aeroNodes[:,5] + twist = aeroNodes[:,4]*np.pi/180 + prebendAC = aeroNodes[:,1] + sweepAC = aeroNodes[:,2] + + # --- IEA 15 + ##'le_location: 'Leading-edge positions from a reference blade axis (usually blade pitch axis). Locations are normalized by the local chord length. Positive in -x direction for airfoil-aligned coordinate system') + ## pitch_axis + ##'1D array of the chordwise position of the pitch axis (0-LE, 1-TE), defined along blade span.') + #grid = [0.0, 0.02040816326530612, 0.04081632653061224, 0.061224489795918366, 0.08163265306122448, 0.1020408163265306, 0.12244897959183673, 0.14285714285714285, 0.16326530612244897, 0.18367346938775508, 0.2040816326530612, 0.22448979591836732, 0.24489795918367346, 0.26530612244897955, 0.2857142857142857, 0.3061224489795918, 0.32653061224489793, 0.3469387755102041, 0.36734693877551017, 0.3877551020408163, 0.4081632653061224, 0.42857142857142855, 0.44897959183673464, 0.4693877551020408, 0.4897959183673469, 0.5102040816326531, 0.5306122448979591, 0.5510204081632653, 0.5714285714285714, 0.5918367346938775, 0.6122448979591836, 0.6326530612244897, 0.6530612244897959, 0.673469387755102, 0.6938775510204082, 0.7142857142857142, 0.7346938775510203, 0.7551020408163265, 0.7755102040816326, 0.7959183673469387, 0.8163265306122448, 0.836734693877551, 0.8571428571428571, 0.8775510204081632, 0.8979591836734693, 0.9183673469387754, 0.9387755102040816, 0.9591836734693877, 0.9795918367346939, 1.0] + #values = [0.5045454545454545, 0.4900186808012221, 0.47270018284548393, 0.4540147730610375, 0.434647782591965, 0.4156278851950606, 0.3979378721273935, 0.38129960745617403, 0.3654920515699109, 0.35160780834472827, 0.34008443128769117, 0.3310670675965599, 0.3241031342163746, 0.3188472934612394, 0.3146895762675238, 0.311488897995355, 0.3088429219529899, 0.3066054031112312, 0.3043613335231313, 0.3018756624023877, 0.2992017656131912, 0.29648581499532917, 0.29397119399704474, 0.2918571873240831, 0.2901098902886204, 0.28880659979944606, 0.28802634398115073, 0.28784151044623507, 0.28794253614539367, 0.28852264941156663, 0.28957685074559625, 0.2911108045758606, 0.2930139151081327, 0.2952412111444283, 0.2977841397364215, 0.300565286724993, 0.3035753776130124, 0.30670446458784534, 0.30988253764299156, 0.3130107259708016, 0.31639042766652853, 0.32021109189825026, 0.32462311714967124, 0.329454188784972, 0.33463306413024474, 0.3401190402144396, 0.3460555975714659, 0.3527211856428439, 0.3600890296396286, 0.36818181818181805] + ##'ref_axis_blade' desc='2D array of the coordinates (x,y,z) of the blade reference axis, defined along blade span. The coordinate system is the one of BeamDyn: it is placed at blade root with x pointing the suction side of the blade, y pointing the trailing edge and z along the blade span. A standard configuration will have negative x values (prebend), if swept positive y values, and positive z values.') + #x_grid = [0.0, 0.02040816326530612, 0.04081632653061224, 0.061224489795918366, 0.08163265306122448, 0.1020408163265306, 0.12244897959183673, 0.14285714285714285, 0.16326530612244897, 0.18367346938775508, 0.2040816326530612, 0.22448979591836732, 0.24489795918367346, 0.26530612244897955, 0.2857142857142857, 0.3061224489795918, 0.32653061224489793, 0.3469387755102041, 0.36734693877551017, 0.3877551020408163, 0.4081632653061224, 0.42857142857142855, 0.44897959183673464, 0.4693877551020408, 0.4897959183673469, 0.5102040816326531, 0.5306122448979591, 0.5510204081632653, 0.5714285714285714, 0.5918367346938775, 0.6122448979591836, 0.6326530612244897, 0.6530612244897959, 0.673469387755102, 0.6938775510204082, 0.7142857142857142, 0.7346938775510203, 0.7551020408163265, 0.7755102040816326, 0.7959183673469387, 0.8163265306122448, 0.836734693877551, 0.8571428571428571, 0.8775510204081632, 0.8979591836734693, 0.9183673469387754, 0.9387755102040816, 0.9591836734693877, 0.9795918367346939, 1.0] + #x_values = [0.0, 0.018400065266506227, 0.04225083661157623, 0.0713435070518306, 0.1036164118664373, 0.13698065932882636, 0.16947761902506267, 0.19850810716711273, 0.22314347791028566, 0.24053558565655847, 0.24886598803245524, 0.2502470372487695, 0.24941257744761433, 0.24756615214432298, 0.24481686563607896, 0.24130290560673967, 0.23698965095246982, 0.23242285078249267, 0.22531163517427788, 0.2110134548882222, 0.18623119147117725, 0.1479307251853749, 0.09847131457569316, 0.04111540547132665, -0.02233952894219675, -0.08884150619038655, -0.15891966620096387, -0.2407441175807782, -0.3366430472730907, -0.44693576549987823, -0.5680658106768092, -0.6975208703059096, -0.8321262196998409, -0.9699653368698024, -1.1090930486685822, -1.255144506570033, -1.4103667735456449, -1.5733007007462756, -1.7434963771088456, -1.9194542609028804, -2.1000907378795275, -2.285501961499942, -2.4756894577736315, -2.6734165188032692, -2.8782701025304545, -3.090085737186208, -3.308459127246535, -3.533712868740941, -3.7641269864926348, -4.0] + #y_grid = [0.0, 1.0] + #y_values = [0.0, 0.0] + #z_grid = [0.0, 0.02040816326530612, 0.04081632653061224, 0.061224489795918366, 0.08163265306122448, 0.1020408163265306, 0.12244897959183673, 0.14285714285714285, 0.16326530612244897, 0.18367346938775508, 0.2040816326530612, 0.22448979591836732, 0.24489795918367346, 0.26530612244897955, 0.2857142857142857, 0.3061224489795918, 0.32653061224489793, 0.3469387755102041, 0.36734693877551017, 0.3877551020408163, 0.4081632653061224, 0.42857142857142855, 0.44897959183673464, 0.4693877551020408, 0.4897959183673469, 0.5102040816326531, 0.5306122448979591, 0.5510204081632653, 0.5714285714285714, 0.5918367346938775, 0.6122448979591836, 0.6326530612244897, 0.6530612244897959, 0.673469387755102, 0.6938775510204082, 0.7142857142857142, 0.7346938775510203, 0.7551020408163265, 0.7755102040816326, 0.7959183673469387, 0.8163265306122448, 0.836734693877551, 0.8571428571428571, 0.8775510204081632, 0.8979591836734693, 0.9183673469387754, 0.9387755102040816, 0.9591836734693877, 0.9795918367346939, 1.0] + #z_values = [0.0, 2.387755102040816, 4.775510204081632, 7.163265306122448, 9.551020408163264, 11.938775510204081, 14.326530612244898, 16.714285714285715, 19.10204081632653, 21.489795918367346, 23.877551020408163, 26.265306122448976, 28.653061224489797, 31.04081632653061, 33.42857142857143, 35.81632653061224, 38.20408163265306, 40.59183673469388, 42.979591836734684, 45.36734693877551, 47.75510204081632, 50.14285714285714, 52.53061224489795, 54.91836734693877, 57.30612244897959, 59.69387755102041, 62.08163265306122, 64.46938775510203, 66.85714285714285, 69.24489795918367, 71.63265306122447, 74.0204081632653, 76.40816326530611, 78.79591836734693, 81.18367346938776, 83.57142857142857, 85.95918367346938, 88.3469387755102, 90.73469387755102, 93.12244897959182, 95.51020408163265, 97.89795918367345, 100.28571428571428, 102.6734693877551, 105.0612244897959, 107.44897959183673, 109.83673469387753, 112.22448979591836, 114.61224489795919, 117.0] + #r_ = [0.0, 0.02, 0.15, 0.245170, 1.0] + #ac = [0.5, 0.5, 0.316, 0.25, 0.25] + #r0 = r/r[-1] + #z = np.interp(r0, z_grid, z_values) + #x = np.interp(r0, x_grid, x_values) + #y = np.interp(r0, y_grid, y_values) + #xp = np.interp(r0, grid, values) + #df['z'] = z + #df['x'] = x + #df['y'] = y + #df['xp'] = xp + #ACloc = np.interp(r0, r_,ac) + + ## Get the absolute offset between pitch axis (rotation center) and aerodynamic center + #ch_offset = inputs['chord'] * (inputs['ac'] - inputs['le_location']) + ## Rotate it by the twist using the AD15 coordinate system + #x , y = util.rotate(0., 0., 0., ch_offset, -np.deg2rad(inputs['theta'])) + ## Apply offset to determine the AC axis + #BlCrvAC = inputs['ref_axis_blade'][:,0] + x + #BlSwpAC = inputs['ref_axis_blade'][:,1] + y + + # --- Adding C2 axis + ACloc = r*0 + 0.25 # distance (in chord) from leading edge to aero center + n=int(len(r)*0.15) # 15% span + ACloc[:n]=np.linspace(0.5,0.25, n) # Root is at 0 + + dx = chord*(0.5-ACloc) * np.sin(twist) # Should be mostly >0 + dy = chord*(0.5-ACloc) * np.cos(twist) # Should be mostly >0 + prebend = prebendAC + dx + sweep = sweepAC + dy + df['c2_Crv_Approx_[m]'] = prebend + df['c2_Swp_Approx_[m]'] = sweep + df['AC_Approx_[-]'] = ACloc + return df + + @property + def _IComment(self): return [1] + + +# --------------------------------------------------------------------------------} +# --- AeroDyn Polar +# --------------------------------------------------------------------------------{ +class ADPolarFile(FASTInputFileBase): + @classmethod + def from_fast_input_file(cls, parent): + self = cls() + self.setData(filename=parent.filename, data=parent.data, hasNodal=parent.hasNodal, module='ADPolar') + return self + + def __init__(self, filename=None, hasUA=True, numTabs=1, **kwargs): + FASTInputFileBase.__init__(self, filename, **kwargs) + if filename is None: + # Define a prototype for this file format + self.addComment('! ------------ AirfoilInfo Input File ------------------------------------------') + self.addComment('! Airfoil definition, written by ADPolarFile') + self.addComment('! ') + self.addComment('! ') + self.addComment('! ------------------------------------------------------------------------------') + self.addValKey("DEFAULT", 'InterpOrd' , 'Interpolation order to use for quasi-steady table lookup {1=linear; 3=cubic spline; "default"} [default=3]') + self.addValKey( 1, 'NonDimArea', 'The non-dimensional area of the airfoil (area/chord^2) (set to 1.0 if unsure or unneeded)') + self.addValKey( 0, 'NumCoords' , 'The number of coordinates in the airfoil shape file. Set to zero if coordinates not included.') + self.addValKey( numTabs , 'NumTabs' , 'Number of airfoil tables in this file. Each table must have lines for Re and Ctrl.') + # TODO multiple tables + for iTab in range(numTabs): + if numTabs==1: + labOffset ='' + else: + labOffset ='_'+str(iTab+1) + self.addComment('! ------------------------------------------------------------------------------') + self.addComment('! data for table {}'.format(iTab+1)) + self.addComment('! ------------------------------------------------------------------------------') + self.addValKey( 1.0 , 'Re' +labOffset , 'Reynolds number in millions') + self.addValKey( 0 , 'Ctrl'+labOffset , 'Control setting') + if hasUA: + self.addValKey(True , 'InclUAdata', 'Is unsteady aerodynamics data included in this table? If TRUE, then include 30 UA coefficients below this line') + self.addComment('!........................................') + self.addValKey( np.nan , 'alpha0' + labOffset, r"0-lift angle of attack, depends on airfoil.") + self.addValKey( np.nan , 'alpha1' + labOffset, r"Angle of attack at f=0.7, (approximately the stall angle) for AOA>alpha0. (deg)") + self.addValKey( np.nan , 'alpha2' + labOffset, r"Angle of attack at f=0.7, (approximately the stall angle) for AOA1]") + self.addValKey( 0 , 'S2' + labOffset, r"Constant in the f curve best-fit for AOA> alpha1; by definition it depends on the airfoil. [ignored if UAMod<>1]") + self.addValKey( 0 , 'S3' + labOffset, r"Constant in the f curve best-fit for alpha2<=AOA< alpha0; by definition it depends on the airfoil. [ignored if UAMod<>1]") + self.addValKey( 0 , 'S4' + labOffset, r"Constant in the f curve best-fit for AOA< alpha2; by definition it depends on the airfoil. [ignored if UAMod<>1]") + self.addValKey( np.nan , 'Cn1' + labOffset, r"Critical value of C0n at leading edge separation. It should be extracted from airfoil data at a given Mach and Reynolds number. It can be calculated from the static value of Cn at either the break in the pitching moment or the loss of chord force at the onset of stall. It is close to the condition of maximum lift of the airfoil at low Mach numbers.") + self.addValKey( np.nan , 'Cn2' + labOffset, r"As Cn1 for negative AOAs.") + self.addValKey( "DEFAULT" , 'St_sh' + labOffset, r"Strouhal's shedding frequency constant. [default = 0.19]") + self.addValKey( np.nan , 'Cd0' + labOffset, r"2D drag coefficient value at 0-lift.") + self.addValKey( np.nan , 'Cm0' + labOffset, r"2D pitching moment coefficient about 1/4-chord location, at 0-lift, positive if nose up. [If the aerodynamics coefficients table does not include a column for Cm, this needs to be set to 0.0]") + self.addValKey( 0 , 'k0' + labOffset, r"Constant in the \hat(x)_cp curve best-fit; = (\hat(x)_AC-0.25). [ignored if UAMod<>1]") + self.addValKey( 0 , 'k1' + labOffset, r"Constant in the \hat(x)_cp curve best-fit. [ignored if UAMod<>1]") + self.addValKey( 0 , 'k2' + labOffset, r"Constant in the \hat(x)_cp curve best-fit. [ignored if UAMod<>1]") + self.addValKey( 0 , 'k3' + labOffset, r"Constant in the \hat(x)_cp curve best-fit. [ignored if UAMod<>1]") + self.addValKey( 0 , 'k1_hat' + labOffset, r"Constant in the expression of Cc due to leading edge vortex effects. [ignored if UAMod<>1]") + self.addValKey( "DEFAULT" , 'x_cp_bar' + labOffset, r"Constant in the expression of \hat(x)_cp^v. [ignored if UAMod<>1, default = 0.2]") + self.addValKey( "DEFAULT" , 'UACutout' + labOffset, r"Angle of attack above which unsteady aerodynamics are disabled (deg). [Specifying the string 'Default' sets UACutout to 45 degrees]") + self.addValKey( "DEFAULT" , 'filtCutOff'+ labOffset, r"Reduced frequency cut-off for low-pass filtering the AoA input to UA, as well as the 1st and 2nd derivatives (-) [default = 0.5]") + self.addComment('!........................................') + else: + self.addValKey(False , 'InclUAdata'+labOffset, 'Is unsteady aerodynamics data included in this table? If TRUE, then include 30 UA coefficients below this line') + self.addComment('! Table of aerodynamics coefficients') + self.addValKey(0 , 'NumAlf'+labOffset, '! Number of data lines in the following table') + self.addTable('AFCoeff'+labOffset, np.zeros((0,4)), tabType=2, tabDimVar='NumAlf', cols=['Alpha', 'Cl', 'Cd', 'Cm'], units=['(deg)', '(-)', '(-)', '(-)']) + self.module='ADPolar' + + def _writeSanityChecks(self): + """ Sanity checks before write""" + nTabs = self['NumTabs'] + if nTabs==1: + self['NumAlf']=self['AFCoeff'].shape[0] + else: + for iTab in range(nTabs): + labOffset='_{}'.format(iTab+1) + self['NumAlf'+labOffset] = self['AFCoeff'+labOffset].shape[0] + # Potentially compute unsteady params here + + def _write(self): + nTabs = self['NumTabs'] + if nTabs==1: + FASTInputFileBase._write(self) + else: + self._writeSanityChecks() + Labs=['Re','Ctrl','UserProp','alpha0','alpha1','alpha2','eta_e','C_nalpha','T_f0','T_V0','T_p','T_VL','b1','b2','b5','A1','A2','A5','S1','S2','S3','S4','Cn1','Cn2','St_sh','Cd0','Cm0','k0','k1','k2','k3','k1_hat','x_cp_bar','UACutout','filtCutOff','InclUAdata','NumAlf','AFCoeff'] + # Store all labels + AllLabels=[self.data[i]['label'] for i in range(len(self.data))] + # Removing lab Offset - TODO TEMPORARY HACK + for iTab in range(nTabs): + labOffset='_{}'.format(iTab+1) + for labRaw in Labs: + i = self.getIDSafe(labRaw+labOffset) + if i>0: + self.data[i]['label'] = labRaw + # Write + with open(self.filename,'w') as f: + f.write(self.toString()) + # Restore labels + for i,labFull in enumerate(AllLabels): + self.data[i]['label'] = labFull + + def _toDataFrame(self): + dfs = FASTInputFileBase._toDataFrame(self) + if not isinstance(dfs, dict): + dfs={'AFCoeff':dfs} + + for k,df in dfs.items(): + sp = k.split('_') + if len(sp)==2: + labOffset='_'+sp[1] + else: + labOffset='' + alpha = df['Alpha_[deg]'].values*np.pi/180. + Cl = df['Cl_[-]'].values + Cd = df['Cd_[-]'].values + Cd0 = self['Cd0'+labOffset] + # Cn (with or without Cd0) + Cn1 = Cl*np.cos(alpha)+ (Cd-Cd0)*np.sin(alpha) + Cn = Cl*np.cos(alpha)+ Cd*np.sin(alpha) + df['Cn_[-]'] = Cn + df['Cn_Cd0off_[-]'] = Cn1 + + CnLin = self['C_nalpha'+labOffset]*(alpha-self['alpha0'+labOffset]*np.pi/180.) + CnLin[alpha<-20*np.pi/180]=np.nan + CnLin[alpha> 30*np.pi/180]=np.nan + df['Cn_pot_[-]'] = CnLin + + # Highlighting points surrounding 0 1 2 Cn points + CnPoints = Cn*np.nan + try: + iBef2 = np.where(alpha0: + if l[0]=='!': + if l.find('!dimension')==0: + self.addKeyVal('nDOF',int(l.split(':')[1])) + nDOFCommon=self['nDOF'] + elif l.find('!time increment')==0: + self.addKeyVal('dt',float(l.split(':')[1])) + elif l.find('!total simulation time')==0: + self.addKeyVal('T',float(l.split(':')[1])) + elif len(l.strip())==0: + pass + else: + raise BrokenFormatError('Unexcepted content found on line {}'.format(i)) + i+=1 + except BrokenFormatError as e: + raise e + except: + raise + + return True + +class ExtPtfmFile(FASTInputFileBase): + @classmethod + def from_fast_input_file(cls, parent): + self = cls() + self.setData(filename=parent.filename, data=parent.data, hasNodal=parent.hasNodal, module='ExtPtfm') + return self + + def __init__(self, filename=None, **kwargs): + FASTInputFileBase.__init__(self, filename, **kwargs) + if filename is None: + # Define a prototype for this file format + self.addValKey(0 , 'nDOF', '') + self.addValKey(1 , 'dt' , '') + self.addValKey(0 , 'T' , '') + self.addTable('MassMatrix' , np.zeros((0,0)), tabType=0) + self.addTable('StiffnessMatrix', np.zeros((0,0)), tabType=0) + self.addTable('DampingMatrix' , np.zeros((0,0)), tabType=0) + self.addTable('Loading' , np.zeros((0,0)), tabType=0) + self.comment='' + self.module='ExtPtfm' + + + def _read(self): + with open(self.filename, 'r', errors="surrogateescape") as f: + lines=f.read().splitlines() + detectAndReadExtPtfmSE(self, lines) + + def toString(self): + s='' + s+='!Comment\n' + s+='!Comment Flex 5 Format\n' + s+='!Dimension: {}\n'.format(self['nDOF']) + s+='!Time increment in simulation: {}\n'.format(self['dt']) + s+='!Total simulation time in file: {}\n'.format(self['T']) + + s+='\n!Mass Matrix\n' + s+='!Dimension: {}\n'.format(self['nDOF']) + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['MassMatrix']) + + s+='\n\n!Stiffness Matrix\n' + s+='!Dimension: {}\n'.format(self['nDOF']) + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['StiffnessMatrix']) + + s+='\n\n!Damping Matrix\n' + s+='!Dimension: {}\n'.format(self['nDOF']) + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['DampingMatrix']) + + s+='\n\n!Loading and Wave Elevation\n' + s+='!Dimension: 1 time column - {} force columns\n'.format(self['nDOF']) + s+='\n'.join(''.join('{:16.8e}'.format(x) for x in y) for y in self['Loading']) + return s + + def _writeSanityChecks(self): + """ Sanity checks before write""" + assert self['MassMatrix'].shape[0] == self['nDOF'] + assert self['StiffnessMatrix'].shape[0] == self['nDOF'] + assert self['DampingMatrix'].shape[0] == self['nDOF'] + assert self['MassMatrix'].shape[0] == self['MassMatrix'].shape[1] + assert self['StiffnessMatrix'].shape[0] == self['StiffnessMatrix'].shape[1] + assert self['DampingMatrix'].shape[0] == self['DampingMatrix'].shape[1] + # if self['T']>0: + # assert self['Loading'].shape[0] == (int(self['T']/self['dT'])+1 + + def _toDataFrame(self): + # Special types, TODO Subclass + nDOF=self['nDOF'] + Cols=['Time_[s]','InpF_Fx_[N]', 'InpF_Fy_[N]', 'InpF_Fz_[N]', 'InpF_Mx_[Nm]', 'InpF_My_[Nm]', 'InpF_Mz_[Nm]'] + Cols+=['CBF_{:03d}_[-]'.format(iDOF+1) for iDOF in np.arange(nDOF)] + Cols=Cols[:nDOF+1] + #dfs['Loading'] = pd.DataFrame(data = self['Loading'],columns = Cols) + dfs = pd.DataFrame(data = self['Loading'],columns = Cols) + + #Cols=['SurgeAcc_[m/s]', 'SwayAcc_[m/s]', 'HeaveAcc_[m/s]', 'RollAcc_[rad/s]', 'PitchAcc_[rad/s]', 'YawAcc_[rad/s]'] + #Cols+=['CBQD_{:03d}_[-]'.format(iDOF+1) for iDOF in np.arange(nDOF)] + #Cols=Cols[:nDOF] + #dfs['MassMatrix'] = pd.DataFrame(data = self['MassMatrix'], columns=Cols) + + #Cols=['SurgeVel_[m/s]', 'SwayVel_[m/s]', 'HeaveVel_[m/s]', 'RollVel_[rad/s]', 'PitchVel_[rad/s]', 'YawVel_[rad/s]'] + #Cols+=['CBQD_{:03d}_[-]'.format(iDOF+1) for iDOF in np.arange(nDOF)] + #Cols=Cols[:nDOF] + #dfs['DampingMatrix'] = pd.DataFrame(data = self['DampingMatrix'], columns=Cols) + + #Cols=['Surge_[m]', 'Sway_[m]', 'Heave_[m]', 'Roll_[rad]', 'Pitch_[rad]', 'Yaw_[rad]'] + #Cols+=['CBQ_{:03d}_[-]'.format(iDOF+1) for iDOF in np.arange(nDOF)] + #Cols=Cols[:nDOF] + #dfs['StiffnessMatrix'] = pd.DataFrame(data = self['StiffnessMatrix'], columns=Cols) + return dfs + + + +if __name__ == "__main__": + f = FASTInputFile() + pass + #B=FASTIn('Turbine.outb') + + + diff --git a/pydatview/io/fast_linearization_file.py b/pydatview/io/fast_linearization_file.py index 3409201..e604d41 100644 --- a/pydatview/io/fast_linearization_file.py +++ b/pydatview/io/fast_linearization_file.py @@ -1,348 +1,354 @@ -from __future__ import division -from __future__ import print_function -from __future__ import absolute_import -from io import open -from .file import File, isBinary, WrongFormatError, BrokenFormatError -import pandas as pd -import numpy as np -import re - -class FASTLinearizationFile(File): - """ - Read/write an OpenFAST linearization file. The object behaves like a dictionary. - - Main keys - --------- - - 'x', 'xdot' 'u', 'y', 'A', 'B', 'C', 'D' - - Main methods - ------------ - - read, write, toDataFrame, keys, xdescr, ydescr, udescr - - Examples - -------- - - f = FASTLinearizationFile('5MW.1.lin') - print(f.keys()) - print(f['u']) # input operating point - print(f.udescr()) # description of inputs - - # use a dataframe with "named" columns and rows - df = f.toDataFrame() - print(df['A'].columns) - print(df['A']) - - """ - @staticmethod - def defaultExtensions(): - return ['.lin'] - - @staticmethod - def formatName(): - return 'FAST linearization output' - - def _read(self, *args, **kwargs): - self['header']=[] - - def extractVal(lines, key): - for l in lines: - if l.find(key)>=0: - return l.split(key)[1].split()[0] - return None - - def readToMarker(fid, marker, nMax): - lines=[] - for i, line in enumerate(fid): - if i>nMax: - raise BrokenFormatError('`{}` not found in file'.format(marker)) - if line.find(marker)>=0: - break - lines.append(line.strip()) - return lines, line - - def readOP(fid, n): - OP=[] - Var = {'RotatingFrame': [], 'DerivativeOrder': [], 'Description': []} - colNames=fid.readline().strip() - dummy= fid.readline().strip() - bHasDeriv= colNames.find('Derivative Order')>=0 - for i, line in enumerate(fid): - sp=line.strip().split() - if sp[1].find(',')>=0: - # Most likely this OP has three values (e.g. orientation angles) - # For now we discard the two other values - OP.append(float(sp[1][:-1])) - iRot=4 - else: - OP.append(float(sp[1])) - iRot=2 - Var['RotatingFrame'].append(sp[iRot]) - if bHasDeriv: - Var['DerivativeOrder'].append(int(sp[iRot+1])) - Var['Description'].append(' '.join(sp[iRot+2:]).strip()) - else: - Var['DerivativeOrder'].append(-1) - Var['Description'].append(' '.join(sp[iRot+1:]).strip()) - if i>=n-1: - break - return OP, Var - - def readMat(fid, n, m): - vals=[f.readline().strip().split() for i in np.arange(n)] -# try: - return np.array(vals).astype(float) -# except ValueError: -# import pdb; pdb.set_trace() - - # Reading - with open(self.filename, 'r', errors="surrogateescape") as f: - # --- Reader header - self['header'], lastLine=readToMarker(f, 'Jacobians included', 30) - self['header'].append(lastLine) - nx = int(extractVal(self['header'],'Number of continuous states:')) - nxd = int(extractVal(self['header'],'Number of discrete states:' )) - nz = int(extractVal(self['header'],'Number of constraint states:')) - nu = int(extractVal(self['header'],'Number of inputs:' )) - ny = int(extractVal(self['header'],'Number of outputs:' )) - bJac = extractVal(self['header'],'Jacobians included in this file?') - try: - self['Azimuth'] = float(extractVal(self['header'],'Azimuth:')) - except: - self['Azimuth'] = None - try: - self['RotSpeed'] = float(extractVal(self['header'],'Rotor Speed:')) # rad/s - except: - self['RotSpeed'] = None - try: - self['WindSpeed'] = float(extractVal(self['header'],'Wind Speed:')) - except: - self['WindSpeed'] = None - - KEYS=['Order of','A:','B:','C:','D:','ED M:', 'dUdu','dUdy'] - - for i, line in enumerate(f): - line = line.strip() - KeyFound=any([line.find(k)>=0 for k in KEYS]) - if KeyFound: - if line.find('Order of continuous states:')>=0: - self['x'], self['x_info'] = readOP(f, nx) - elif line.find('Order of continuous state derivatives:')>=0: - self['xdot'], self['xdot_info'] = readOP(f, nx) - elif line.find('Order of inputs')>=0: - self['u'], self['u_info'] = readOP(f, nu) - elif line.find('Order of outputs')>=0: - self['y'], self['y_info'] = readOP(f, ny) - elif line.find('A:')>=0: - self['A'] = readMat(f, nx, nx) - elif line.find('B:')>=0: - self['B'] = readMat(f, nx, nu) - elif line.find('C:')>=0: - self['C'] = readMat(f, ny, nx) - elif line.find('D:')>=0: - self['D'] = readMat(f, ny, nu) - elif line.find('dUdu:')>=0: - self['dUdu'] = readMat(f, nu, nu) - elif line.find('dUdy:')>=0: - self['dUdy'] = readMat(f, nu, ny) - elif line.find('ED M:')>=0: - self['EDDOF'] = line[5:].split() - self['M'] = readMat(f, 24, 24) - - def toString(self): - s='' - return s - - def _write(self): - with open(self.filename,'w') as f: - f.write(self.toString()) - - def short_descr(self,slist): - def shortname(s): - s=s.strip() - s = s.replace('(m/s)' , '_[m/s]' ); - s = s.replace('(kW)' , '_[kW]' ); - s = s.replace('(deg)' , '_[deg]' ); - s = s.replace('(N)' , '_[N]' ); - s = s.replace('(kN-m)' , '_[kNm]' ); - s = s.replace('(N-m)' , '_[Nm]' ); - s = s.replace('(kN)' , '_[kN]' ); - s = s.replace('(rpm)' , '_[rpm]' ); - s = s.replace('(rad)' , '_[rad]' ); - s = s.replace('(rad/s)' , '_[rad/s]' ); - s = s.replace('(rad/s^2)', '_[rad/s^2]' ); - s = s.replace('(m/s^2)' , '_[m/s^2]'); - s = s.replace('(deg/s^2)','_[deg/s^2]'); - s = s.replace('(m)' , '_[m]' ); - s = s.replace(', m/s/s','_[m/s^2]'); - s = s.replace(', m/s^2','_[m/s^2]'); - s = s.replace(', m/s','_[m/s]'); - s = s.replace(', m','_[m]'); - s = s.replace(', rad/s/s','_[rad/s^2]'); - s = s.replace(', rad/s^2','_[rad/s^2]'); - s = s.replace(', rad/s','_[rad/s]'); - s = s.replace(', rad','_[rad]'); - s = s.replace(', -','_[-]'); - s = s.replace(', Nm/m','_[Nm/m]'); - s = s.replace(', Nm','_[Nm]'); - s = s.replace(', N/m','_[N/m]'); - s = s.replace(', N','_[N]'); - s = s.replace('(1)','1') - s = s.replace('(2)','2') - s = s.replace('(3)','3') - s= re.sub(r'\([^)]*\)','', s) # remove parenthesis - s = s.replace('ED ',''); - s = s.replace('BD_','BD_B'); - s = s.replace('IfW ',''); - s = s.replace('Extended input: ','') - s = s.replace('1st tower ','qt1'); - s = s.replace('2nd tower ','qt2'); - nd = s.count('First time derivative of ') - if nd>=0: - s = s.replace('First time derivative of ' ,''); - if nd==1: - s = 'd_'+s.strip() - elif nd==2: - s = 'dd_'+s.strip() - s = s.replace('Variable speed generator DOF ','psi_rot'); # NOTE: internally in FAST this is the azimuth of the rotor - s = s.replace('fore-aft bending mode DOF ' ,'FA' ); - s = s.replace('side-to-side bending mode DOF','SS' ); - s = s.replace('bending-mode DOF of blade ' ,'' ); - s = s.replace(' rotational-flexibility DOF, rad','-ROT' ); - s = s.replace('rotational displacement in ','rot' ); - s = s.replace('Drivetrain','DT' ); - s = s.replace('translational displacement in ','trans' ); - s = s.replace('finite element node ','N' ); - s = s.replace('-component position of node ','posN') - s = s.replace('-component inflow on tower node','TwrN') - s = s.replace('-component inflow on blade 1, node','Bld1N') - s = s.replace('-component inflow on blade 2, node','Bld2N') - s = s.replace('-component inflow on blade 3, node','Bld3N') - s = s.replace('-component inflow velocity at node','N') - s = s.replace('X translation displacement, node','TxN') - s = s.replace('Y translation displacement, node','TyN') - s = s.replace('Z translation displacement, node','TzN') - s = s.replace('X translation velocity, node','TVxN') - s = s.replace('Y translation velocity, node','TVyN') - s = s.replace('Z translation velocity, node','TVzN') - s = s.replace('X translation acceleration, node','TAxN') - s = s.replace('Y translation acceleration, node','TAyN') - s = s.replace('Z translation acceleration, node','TAzN') - s = s.replace('X orientation angle, node' ,'RxN') - s = s.replace('Y orientation angle, node' ,'RyN') - s = s.replace('Z orientation angle, node' ,'RzN') - s = s.replace('X rotation velocity, node' ,'RVxN') - s = s.replace('Y rotation velocity, node' ,'RVyN') - s = s.replace('Z rotation velocity, node' ,'RVzN') - s = s.replace('X rotation acceleration, node' ,'RAxN') - s = s.replace('Y rotation acceleration, node' ,'RAyN') - s = s.replace('Z rotation acceleration, node' ,'RAzN') - s = s.replace('X force, node','FxN') - s = s.replace('Y force, node','FyN') - s = s.replace('Z force, node','FzN') - s = s.replace('X moment, node','MxN') - s = s.replace('Y moment, node','MyN') - s = s.replace('Z moment, node','MzN') - s = s.replace('FX', 'Fx') - s = s.replace('FY', 'Fy') - s = s.replace('FZ', 'Fz') - s = s.replace('MX', 'Mx') - s = s.replace('MY', 'My') - s = s.replace('MZ', 'Mz') - s = s.replace('FKX', 'FKx') - s = s.replace('FKY', 'FKy') - s = s.replace('FKZ', 'FKz') - s = s.replace('MKX', 'MKx') - s = s.replace('MKY', 'MKy') - s = s.replace('MKZ', 'MKz') - s = s.replace('Nodes motion','') - s = s.replace('cosine','cos' ); - s = s.replace('sine','sin' ); - s = s.replace('collective','coll.'); - s = s.replace('Blade','Bld'); - s = s.replace('rotZ','TORS-R'); - s = s.replace('transX','FLAP-D'); - s = s.replace('transY','EDGE-D'); - s = s.replace('rotX','EDGE-R'); - s = s.replace('rotY','FLAP-R'); - s = s.replace('flapwise','FLAP'); - s = s.replace('edgewise','EDGE'); - s = s.replace('horizontal surge translation DOF','Surge'); - s = s.replace('horizontal sway translation DOF','Sway'); - s = s.replace('vertical heave translation DOF','Heave'); - s = s.replace('roll tilt rotation DOF','Roll'); - s = s.replace('pitch tilt rotation DOF','Pitch'); - s = s.replace('yaw rotation DOF','Yaw'); - s = s.replace('vertical power-law shear exponent','alpha') - s = s.replace('horizontal wind speed ','WS') - s = s.replace('propagation direction','WD') - s = s.replace(' pitch command','pitch') - s = s.replace('HSS_','HSS') - s = s.replace('Bld','B') - s = s.replace('tower','Twr') - s = s.replace('Tower','Twr') - s = s.replace('Nacelle','Nac') - s = s.replace('Platform','Ptfm') - s = s.replace('SrvD','SvD') - s = s.replace('Generator torque','Qgen') - s = s.replace('coll. blade-pitch command','PitchColl') - s = s.replace('wave elevation at platform ref point','WaveElevRefPoint') - s = s.replace('1)','1'); - s = s.replace('2)','2'); - s = s.replace('3)','3'); - s = s.replace(',',''); - s = s.replace(' ',''); - s=s.strip() - return s - return [shortname(s) for s in slist] - - def xdescr(self): - return self.short_descr(self['x_info']['Description']) - - def xdotdescr(self): - return self.short_descr(self['xdot_info']['Description']) - - def ydescr(self): - if 'y_info' in self.keys(): - return self.short_descr(self['y_info']['Description']) - else: - return [] - def udescr(self): - if 'u_info' in self.keys(): - return self.short_descr(self['u_info']['Description']) - else: - return [] - - def _toDataFrame(self): - dfs={} - - xdescr_short = self.xdescr() - xdotdescr_short = self.xdotdescr() - ydescr_short = self.ydescr() - udescr_short = self.udescr() - - if 'A' in self.keys(): - dfs['A'] = pd.DataFrame(data = self['A'], index=xdescr_short, columns=xdescr_short) - if 'B' in self.keys(): - dfs['B'] = pd.DataFrame(data = self['B'], index=xdescr_short, columns=udescr_short) - if 'C' in self.keys(): - dfs['C'] = pd.DataFrame(data = self['C'], index=ydescr_short, columns=xdescr_short) - if 'D' in self.keys(): - dfs['D'] = pd.DataFrame(data = self['D'], index=ydescr_short, columns=udescr_short) - if 'x' in self.keys(): - dfs['x'] = pd.DataFrame(data = np.asarray(self['x']).reshape((1,-1)), columns=xdescr_short) - if 'xdot' in self.keys(): - dfs['xdot'] = pd.DataFrame(data = np.asarray(self['xdot']).reshape((1,-1)), columns=xdotdescr_short) - if 'u' in self.keys(): - dfs['u'] = pd.DataFrame(data = np.asarray(self['u']).reshape((1,-1)), columns=udescr_short) - if 'y' in self.keys(): - dfs['y'] = pd.DataFrame(data = np.asarray(self['y']).reshape((1,-1)), columns=ydescr_short) - if 'M' in self.keys(): - dfs['M'] = pd.DataFrame(data = self['M'], index=self['EDDOF'], columns=self['EDDOF']) - if 'dUdu' in self.keys(): - dfs['dUdu'] = pd.DataFrame(data = self['dUdu'], index=udescr_short, columns=udescr_short) - if 'dUdy' in self.keys(): - dfs['dUdy'] = pd.DataFrame(data = self['dUdy'], index=udescr_short, columns=ydescr_short) - - return dfs - - +from __future__ import division +from __future__ import print_function +from __future__ import absolute_import +from io import open +from .file import File, isBinary, WrongFormatError, BrokenFormatError +import pandas as pd +import numpy as np +import re + +class FASTLinearizationFile(File): + """ + Read/write an OpenFAST linearization file. The object behaves like a dictionary. + + Main keys + --------- + - 'x', 'xdot' 'u', 'y', 'A', 'B', 'C', 'D' + + Main methods + ------------ + - read, write, toDataFrame, keys, xdescr, ydescr, udescr + + Examples + -------- + + f = FASTLinearizationFile('5MW.1.lin') + print(f.keys()) + print(f['u']) # input operating point + print(f.udescr()) # description of inputs + + # use a dataframe with "named" columns and rows + df = f.toDataFrame() + print(df['A'].columns) + print(df['A']) + + """ + @staticmethod + def defaultExtensions(): + return ['.lin'] + + @staticmethod + def formatName(): + return 'FAST linearization output' + + def _read(self, *args, **kwargs): + self['header']=[] + + def extractVal(lines, key): + for l in lines: + if l.find(key)>=0: + return l.split(key)[1].split()[0] + return None + + def readToMarker(fid, marker, nMax): + lines=[] + for i, line in enumerate(fid): + if i>nMax: + raise BrokenFormatError('`{}` not found in file'.format(marker)) + if line.find(marker)>=0: + break + lines.append(line.strip()) + return lines, line + + def readOP(fid, n): + OP=[] + Var = {'RotatingFrame': [], 'DerivativeOrder': [], 'Description': []} + colNames=fid.readline().strip() + dummy= fid.readline().strip() + bHasDeriv= colNames.find('Derivative Order')>=0 + for i, line in enumerate(fid): + sp=line.strip().split() + if sp[1].find(',')>=0: + # Most likely this OP has three values (e.g. orientation angles) + # For now we discard the two other values + OP.append(float(sp[1][:-1])) + iRot=4 + else: + OP.append(float(sp[1])) + iRot=2 + Var['RotatingFrame'].append(sp[iRot]) + if bHasDeriv: + Var['DerivativeOrder'].append(int(sp[iRot+1])) + Var['Description'].append(' '.join(sp[iRot+2:]).strip()) + else: + Var['DerivativeOrder'].append(-1) + Var['Description'].append(' '.join(sp[iRot+1:]).strip()) + if i>=n-1: + break + return OP, Var + + def readMat(fid, n, m): + vals=[f.readline().strip().split() for i in np.arange(n)] +# try: + return np.array(vals).astype(float) +# except ValueError: +# import pdb; pdb.set_trace() + + # Reading + with open(self.filename, 'r', errors="surrogateescape") as f: + # --- Reader header + self['header'], lastLine=readToMarker(f, 'Jacobians included', 30) + self['header'].append(lastLine) + nx = int(extractVal(self['header'],'Number of continuous states:')) + nxd = int(extractVal(self['header'],'Number of discrete states:' )) + nz = int(extractVal(self['header'],'Number of constraint states:')) + nu = int(extractVal(self['header'],'Number of inputs:' )) + ny = int(extractVal(self['header'],'Number of outputs:' )) + bJac = extractVal(self['header'],'Jacobians included in this file?') + try: + self['Azimuth'] = float(extractVal(self['header'],'Azimuth:')) + except: + self['Azimuth'] = None + try: + self['RotSpeed'] = float(extractVal(self['header'],'Rotor Speed:')) # rad/s + except: + self['RotSpeed'] = None + try: + self['WindSpeed'] = float(extractVal(self['header'],'Wind Speed:')) + except: + self['WindSpeed'] = None + + KEYS=['Order of','A:','B:','C:','D:','ED M:', 'dUdu','dUdy'] + + for i, line in enumerate(f): + line = line.strip() + KeyFound=any([line.find(k)>=0 for k in KEYS]) + if KeyFound: + if line.find('Order of continuous states:')>=0: + self['x'], self['x_info'] = readOP(f, nx) + elif line.find('Order of continuous state derivatives:')>=0: + self['xdot'], self['xdot_info'] = readOP(f, nx) + elif line.find('Order of inputs')>=0: + self['u'], self['u_info'] = readOP(f, nu) + elif line.find('Order of outputs')>=0: + self['y'], self['y_info'] = readOP(f, ny) + elif line.find('A:')>=0: + self['A'] = readMat(f, nx, nx) + elif line.find('B:')>=0: + self['B'] = readMat(f, nx, nu) + elif line.find('C:')>=0: + self['C'] = readMat(f, ny, nx) + elif line.find('D:')>=0: + self['D'] = readMat(f, ny, nu) + elif line.find('dUdu:')>=0: + self['dUdu'] = readMat(f, nu, nu) + elif line.find('dUdy:')>=0: + self['dUdy'] = readMat(f, nu, ny) + elif line.find('ED M:')>=0: + self['EDDOF'] = line[5:].split() + self['M'] = readMat(f, 24, 24) + + def toString(self): + s='' + return s + + def _write(self): + with open(self.filename,'w') as f: + f.write(self.toString()) + + def short_descr(self,slist): + def shortname(s): + s=s.strip() + s = s.replace('(m/s)' , '_[m/s]' ); + s = s.replace('(kW)' , '_[kW]' ); + s = s.replace('(deg)' , '_[deg]' ); + s = s.replace('(N)' , '_[N]' ); + s = s.replace('(kN-m)' , '_[kNm]' ); + s = s.replace('(N-m)' , '_[Nm]' ); + s = s.replace('(kN)' , '_[kN]' ); + s = s.replace('(rpm)' , '_[rpm]' ); + s = s.replace('(rad)' , '_[rad]' ); + s = s.replace('(rad/s)' , '_[rad/s]' ); + s = s.replace('(rad/s^2)', '_[rad/s^2]' ); + s = s.replace('(m/s^2)' , '_[m/s^2]'); + s = s.replace('(deg/s^2)','_[deg/s^2]'); + s = s.replace('(m)' , '_[m]' ); + s = s.replace(', m/s/s','_[m/s^2]'); + s = s.replace(', m/s^2','_[m/s^2]'); + s = s.replace(', m/s','_[m/s]'); + s = s.replace(', m','_[m]'); + s = s.replace(', rad/s/s','_[rad/s^2]'); + s = s.replace(', rad/s^2','_[rad/s^2]'); + s = s.replace(', rad/s','_[rad/s]'); + s = s.replace(', rad','_[rad]'); + s = s.replace(', -','_[-]'); + s = s.replace(', Nm/m','_[Nm/m]'); + s = s.replace(', Nm','_[Nm]'); + s = s.replace(', N/m','_[N/m]'); + s = s.replace(', N','_[N]'); + s = s.replace('(1)','1') + s = s.replace('(2)','2') + s = s.replace('(3)','3') + s= re.sub(r'\([^)]*\)','', s) # remove parenthesis + s = s.replace('ED ',''); + s = s.replace('BD_','BD_B'); + s = s.replace('IfW ',''); + s = s.replace('Extended input: ','') + s = s.replace('1st tower ','qt1'); + s = s.replace('2nd tower ','qt2'); + nd = s.count('First time derivative of ') + if nd>=0: + s = s.replace('First time derivative of ' ,''); + if nd==1: + s = 'd_'+s.strip() + elif nd==2: + s = 'dd_'+s.strip() + s = s.replace('Variable speed generator DOF ','psi_rot'); # NOTE: internally in FAST this is the azimuth of the rotor + s = s.replace('fore-aft bending mode DOF ' ,'FA' ); + s = s.replace('side-to-side bending mode DOF','SS' ); + s = s.replace('bending-mode DOF of blade ' ,'' ); + s = s.replace(' rotational-flexibility DOF, rad','-ROT' ); + s = s.replace('rotational displacement in ','rot' ); + s = s.replace('Drivetrain','DT' ); + s = s.replace('translational displacement in ','trans' ); + s = s.replace('finite element node ','N' ); + s = s.replace('-component position of node ','posN') + s = s.replace('-component inflow on tower node','TwrN') + s = s.replace('-component inflow on blade 1, node','Bld1N') + s = s.replace('-component inflow on blade 2, node','Bld2N') + s = s.replace('-component inflow on blade 3, node','Bld3N') + s = s.replace('-component inflow velocity at node','N') + s = s.replace('X translation displacement, node','TxN') + s = s.replace('Y translation displacement, node','TyN') + s = s.replace('Z translation displacement, node','TzN') + s = s.replace('X translation velocity, node','TVxN') + s = s.replace('Y translation velocity, node','TVyN') + s = s.replace('Z translation velocity, node','TVzN') + s = s.replace('X translation acceleration, node','TAxN') + s = s.replace('Y translation acceleration, node','TAyN') + s = s.replace('Z translation acceleration, node','TAzN') + s = s.replace('X orientation angle, node' ,'RxN') + s = s.replace('Y orientation angle, node' ,'RyN') + s = s.replace('Z orientation angle, node' ,'RzN') + s = s.replace('X rotation velocity, node' ,'RVxN') + s = s.replace('Y rotation velocity, node' ,'RVyN') + s = s.replace('Z rotation velocity, node' ,'RVzN') + s = s.replace('X rotation acceleration, node' ,'RAxN') + s = s.replace('Y rotation acceleration, node' ,'RAyN') + s = s.replace('Z rotation acceleration, node' ,'RAzN') + s = s.replace('X force, node','FxN') + s = s.replace('Y force, node','FyN') + s = s.replace('Z force, node','FzN') + s = s.replace('X moment, node','MxN') + s = s.replace('Y moment, node','MyN') + s = s.replace('Z moment, node','MzN') + s = s.replace('FX', 'Fx') + s = s.replace('FY', 'Fy') + s = s.replace('FZ', 'Fz') + s = s.replace('MX', 'Mx') + s = s.replace('MY', 'My') + s = s.replace('MZ', 'Mz') + s = s.replace('FKX', 'FKx') + s = s.replace('FKY', 'FKy') + s = s.replace('FKZ', 'FKz') + s = s.replace('MKX', 'MKx') + s = s.replace('MKY', 'MKy') + s = s.replace('MKZ', 'MKz') + s = s.replace('Nodes motion','') + s = s.replace('cosine','cos' ); + s = s.replace('sine','sin' ); + s = s.replace('collective','coll.'); + s = s.replace('Blade','Bld'); + s = s.replace('rotZ','TORS-R'); + s = s.replace('transX','FLAP-D'); + s = s.replace('transY','EDGE-D'); + s = s.replace('rotX','EDGE-R'); + s = s.replace('rotY','FLAP-R'); + s = s.replace('flapwise','FLAP'); + s = s.replace('edgewise','EDGE'); + s = s.replace('horizontal surge translation DOF','Surge'); + s = s.replace('horizontal sway translation DOF','Sway'); + s = s.replace('vertical heave translation DOF','Heave'); + s = s.replace('roll tilt rotation DOF','Roll'); + s = s.replace('pitch tilt rotation DOF','Pitch'); + s = s.replace('yaw rotation DOF','Yaw'); + s = s.replace('vertical power-law shear exponent','alpha') + s = s.replace('horizontal wind speed ','WS') + s = s.replace('propagation direction','WD') + s = s.replace(' pitch command','pitch') + s = s.replace('HSS_','HSS') + s = s.replace('Bld','B') + s = s.replace('tower','Twr') + s = s.replace('Tower','Twr') + s = s.replace('Nacelle','Nac') + s = s.replace('Platform','Ptfm') + s = s.replace('SrvD','SvD') + s = s.replace('Generator torque','Qgen') + s = s.replace('coll. blade-pitch command','PitchColl') + s = s.replace('wave elevation at platform ref point','WaveElevRefPoint') + s = s.replace('1)','1'); + s = s.replace('2)','2'); + s = s.replace('3)','3'); + s = s.replace(',',''); + s = s.replace(' ',''); + s=s.strip() + return s + return [shortname(s) for s in slist] + + def xdescr(self): + if 'x_info' in self.keys(): + return self.short_descr(self['x_info']['Description']) + else: + return [] + + def xdotdescr(self): + if 'xdot_info' in self.keys(): + return self.short_descr(self['xdot_info']['Description']) + else: + return [] + + def ydescr(self): + if 'y_info' in self.keys(): + return self.short_descr(self['y_info']['Description']) + else: + return [] + def udescr(self): + if 'u_info' in self.keys(): + return self.short_descr(self['u_info']['Description']) + else: + return [] + + def _toDataFrame(self): + dfs={} + + xdescr_short = self.xdescr() + xdotdescr_short = self.xdotdescr() + ydescr_short = self.ydescr() + udescr_short = self.udescr() + + if 'A' in self.keys(): + dfs['A'] = pd.DataFrame(data = self['A'], index=xdescr_short, columns=xdescr_short) + if 'B' in self.keys(): + dfs['B'] = pd.DataFrame(data = self['B'], index=xdescr_short, columns=udescr_short) + if 'C' in self.keys(): + dfs['C'] = pd.DataFrame(data = self['C'], index=ydescr_short, columns=xdescr_short) + if 'D' in self.keys(): + dfs['D'] = pd.DataFrame(data = self['D'], index=ydescr_short, columns=udescr_short) + if 'x' in self.keys(): + dfs['x'] = pd.DataFrame(data = np.asarray(self['x']).reshape((1,-1)), columns=xdescr_short) + if 'xdot' in self.keys(): + dfs['xdot'] = pd.DataFrame(data = np.asarray(self['xdot']).reshape((1,-1)), columns=xdotdescr_short) + if 'u' in self.keys(): + dfs['u'] = pd.DataFrame(data = np.asarray(self['u']).reshape((1,-1)), columns=udescr_short) + if 'y' in self.keys(): + dfs['y'] = pd.DataFrame(data = np.asarray(self['y']).reshape((1,-1)), columns=ydescr_short) + if 'M' in self.keys(): + dfs['M'] = pd.DataFrame(data = self['M'], index=self['EDDOF'], columns=self['EDDOF']) + if 'dUdu' in self.keys(): + dfs['dUdu'] = pd.DataFrame(data = self['dUdu'], index=udescr_short, columns=udescr_short) + if 'dUdy' in self.keys(): + dfs['dUdy'] = pd.DataFrame(data = self['dUdy'], index=udescr_short, columns=ydescr_short) + + return dfs + + diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index 8f1d982..95947c2 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -1,498 +1,498 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from io import open -from builtins import map -from builtins import range -from builtins import chr -from builtins import str -from future import standard_library -standard_library.install_aliases() - -from itertools import takewhile - -# try: -from .file import File, WrongFormatError, BrokenReaderError, EmptyFileError -# except: -# # --- Allowing this file to be standalone.. -# class WrongFormatError(Exception): -# pass -# class WrongReaderError(Exception): -# pass -# class EmptyFileError(Exception): -# pass -# File = dict -try: - from .csv_file import CSVFile -except: - print('CSVFile not available') -import numpy as np -import pandas as pd -import struct -import os -import re - - -# --------------------------------------------------------------------------------} -# --- OUT FILE -# --------------------------------------------------------------------------------{ -class FASTOutputFile(File): - """ - Read an OpenFAST ouput file (.out, .outb, .elev). - - Main methods - ------------ - - read, write, toDataFrame - - Examples - -------- - - # read an output file, convert it to pandas dataframe, modify it, write it back - f = FASTOutputFile('5MW.outb') - df=f.toDataFrame() - time = df['Time_[s]'] - Omega = df['RotSpeed_[rpm]'] - df['Time_[s]'] -=100 - f.writeDataFrame(df, '5MW_TimeShifted.outb') - - """ - - @staticmethod - def defaultExtensions(): - return ['.out','.outb','.elm','.elev'] - - @staticmethod - def formatName(): - return 'FAST output file' - - def _read(self): - def readline(iLine): - with open(self.filename) as f: - for i, line in enumerate(f): - if i==iLine-1: - return line.strip() - elif i>=iLine: - break - - ext = os.path.splitext(self.filename.lower())[1] - self.info={} - self['binary']=False - try: - if ext in ['.out','.elev']: - self.data, self.info = load_ascii_output(self.filename) - elif ext=='.outb': - self.data, self.info = load_binary_output(self.filename) - self['binary']=True - elif ext=='.elm': - F=CSVFile(filename=self.filename, sep=' ', commentLines=[0,2],colNamesLine=1) - self.data = F.data - del F - self.info['attribute_units']=readline(3).replace('sec','s').split() - self.info['attribute_names']=self.data.columns.values - else: - self.data, self.info = load_output(self.filename) - except MemoryError as e: - raise BrokenReaderError('FAST Out File {}: Memory error encountered\n{}'.format(self.filename,e)) - except Exception as e: - raise WrongFormatError('FAST Out File {}: {}'.format(self.filename,e.args)) - if self.data.shape[0]==0: - raise EmptyFileError('This FAST output file contains no data: {}'.format(self.filename)) - - if self.info['attribute_units'] is not None: - self.info['attribute_units'] = [re.sub(r'[()\[\]]','',u) for u in self.info['attribute_units']] - - - def _write(self): - if self['binary']: - channels = self.data - chanNames = self.info['attribute_names'] - chanUnits = self.info['attribute_units'] - descStr = self.info['description'] - writeBinary(self.filename, channels, chanNames, chanUnits, fileID=2, descStr=descStr) - else: - # ascii output - with open(self.filename,'w') as f: - f.write('\t'.join(['{:>10s}'.format(c) for c in self.info['attribute_names']])+'\n') - f.write('\t'.join(['{:>10s}'.format('('+u+')') for u in self.info['attribute_units']])+'\n') - # TODO better.. - f.write('\n'.join(['\t'.join(['{:10.4f}'.format(y[0])]+['{:10.3e}'.format(x) for x in y[1:]]) for y in self.data])) - - def _toDataFrame(self): - if self.info['attribute_units'] is not None: - cols=[n+'_['+u.replace('sec','s')+']' for n,u in zip(self.info['attribute_names'],self.info['attribute_units'])] - else: - cols=self.info['attribute_names'] - if isinstance(self.data, pd.DataFrame): - df= self.data - df.columns=cols - else: - df = pd.DataFrame(data=self.data,columns=cols) - - return df - - def writeDataFrame(self, df, filename, binary=True): - writeDataFrame(df, filename, binary=binary) - -# -------------------------------------------------------------------------------- -# --- Helper low level functions -# -------------------------------------------------------------------------------- -def load_output(filename): - """Load a FAST binary or ascii output file - - Parameters - ---------- - filename : str - filename - - Returns - ------- - data : ndarray - data values - info : dict - info containing: - - name: filename - - description: description of dataset - - attribute_names: list of attribute names - - attribute_units: list of attribute units - """ - - assert os.path.isfile(filename), "File, %s, does not exists" % filename - with open(filename, 'r') as f: - try: - f.readline() - except UnicodeDecodeError: - return load_binary_output(filename) - return load_ascii_output(filename) - -def load_ascii_output(filename): - with open(filename) as f: - info = {} - info['name'] = os.path.splitext(os.path.basename(filename))[0] - # Header is whatever is before the keyword `time` - in_header = True - header = [] - while in_header: - l = f.readline() - if not l: - raise Exception('Error finding the end of FAST out file header. Keyword Time missing.') - in_header= (l+' dummy').lower().split()[0] != 'time' - if in_header: - header.append(l) - else: - info['description'] = header - info['attribute_names'] = l.split() - info['attribute_units'] = [unit[1:-1] for unit in f.readline().split()] - # --- - # Data, up to end of file or empty line (potential comment line at the end) -# data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(np.float) - # --- - data = np.loadtxt(f, comments=('This')) # Adding "This" for the Hydro Out files.. - return data, info - - -def load_binary_output(filename, use_buffer=True): - """ - 03/09/15: Ported from ReadFASTbinary.m by Mads M Pedersen, DTU Wind - 24/10/18: Low memory/buffered version by E. Branlard, NREL - 18/01/19: New file format for exctended channels, by E. Branlard, NREL - - Info about ReadFASTbinary.m: - % Author: Bonnie Jonkman, National Renewable Energy Laboratory - % (c) 2012, National Renewable Energy Laboratory - % - % Edited for FAST v7.02.00b-bjj 22-Oct-2012 - """ - def fread(fid, n, type): - fmt, nbytes = {'uint8': ('B', 1), 'int16':('h', 2), 'int32':('i', 4), 'float32':('f', 4), 'float64':('d', 8)}[type] - return struct.unpack(fmt * n, fid.read(nbytes * n)) - - def freadRowOrderTableBuffered(fid, n, type_in, nCols, nOff=0, type_out='float64'): - """ - Reads of row-ordered table from a binary file. - - Read `n` data of type `type_in`, assumed to be a row ordered table of `nCols` columns. - Memory usage is optimized by allocating the data only once. - Buffered reading is done for improved performances (in particular for 32bit python) - - `nOff` allows for additional column space at the begining of the storage table. - Typically, `nOff=1`, provides a column at the beginning to store the time vector. - - @author E.Branlard, NREL - - """ - fmt, nbytes = {'uint8': ('B', 1), 'int16':('h', 2), 'int32':('i', 4), 'float32':('f', 4), 'float64':('d', 8)}[type_in] - nLines = int(n/nCols) - GoodBufferSize = 4096*40 - nLinesPerBuffer = int(GoodBufferSize/nCols) - BufferSize = nCols * nLinesPerBuffer - nBuffer = int(n/BufferSize) - # Allocation of data - data = np.zeros((nLines,nCols+nOff), dtype = type_out) - # Reading - try: - nIntRead = 0 - nLinesRead = 0 - while nIntRead0: - op,cl = chars - iu=c.rfind(op) - if iu>1: - name = c[:iu] - unit = c[iu+1:].replace(cl,'') - if name[-1]=='_': - name=name[:-1] - - chanNames.append(name) - chanUnits.append(unit) - - if binary: - writeBinary(filename, channels, chanNames, chanUnits) - else: - NotImplementedError() - - -def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr=''): - """ - Write an OpenFAST binary file. - - Based on contributions from - Hugo Castro, David Schlipf, Hochschule Flensburg - - Input: - FileName - string: contains file name to open - Channels - 2-D array: dimension 1 is time, dimension 2 is channel - ChanName - cell array containing names of output channels - ChanUnit - cell array containing unit names of output channels, preferably surrounded by parenthesis - FileID - constant that determines if the time is stored in the - output, indicating possible non-constant time step - DescStr - String describing the file - """ - # Data sanitization - chanNames = list(chanNames) - channels = np.asarray(channels) - if chanUnits[0][0]!='(': - chanUnits = ['('+u+')' for u in chanUnits] # units surrounded by parenthesis to match OpenFAST convention - - nT, nChannelsWithTime = np.shape(channels) - nChannels = nChannelsWithTime - 1 - - # For FileID =2, time needs to be present and at the first column - try: - iTime = chanNames.index('Time') - except ValueError: - raise Exception('`Time` needs to be present in channel names' ) - if iTime!=0: - raise Exception('`Time` needs to be the first column of `chanName`' ) - - time = channels[:,iTime] - timeStart = time[0] - timeIncr = time[1]-time[0] - dataWithoutTime = channels[:,1:] - - # Compute data range, scaling and offsets to convert to int16 - # To use the int16 range to its fullest, the max float is matched to 2^15-1 and the - # the min float is matched to -2^15. Thus, we have the to equations we need - # to solve to get scaling and offset, see line 120 of ReadFASTbinary: - # Int16Max = FloatMax * Scaling + Offset - # Int16Min = FloatMin * Scaling + Offset - int16Max = np.single( 32767.0) # Largest integer represented in 2 bytes, 2**15 - 1 - int16Min = np.single(-32768.0) # Smallest integer represented in 2 bytes -2**15 - int16Rng = np.single(int16Max - int16Min) # Max Range of 2 byte integer - mins = np.min(dataWithoutTime, axis=0) - ranges = np.single(np.max(dataWithoutTime, axis=0) - mins) - ranges[ranges==0]=1 # range set to 1 for constant channel. In OpenFAST: /sqrt(epsilon(1.0_SiKi)) - ColScl = np.single(int16Rng/ranges) - ColOff = np.single(int16Min - np.single(mins)*ColScl) - - #Just available for fileID - if fileID != 2: - print("current version just works with FileID = 2") - - else: - with open(fileName,'wb') as fid: - # Notes on struct: - # @ is used for packing in native byte order - # B - unsigned integer 8 bits - # h - integer 16 bits - # i - integer 32 bits - # f - float 32 bits - # d - float 64 bits - - # Write header informations - fid.write(struct.pack('@h',fileID)) - fid.write(struct.pack('@i',nChannels)) - fid.write(struct.pack('@i',nT)) - fid.write(struct.pack('@d',timeStart)) - fid.write(struct.pack('@d',timeIncr)) - fid.write(struct.pack('@{}f'.format(nChannels), *ColScl)) - fid.write(struct.pack('@{}f'.format(nChannels), *ColOff)) - descStrASCII = [ord(char) for char in descStr] - fid.write(struct.pack('@i',len(descStrASCII))) - fid.write(struct.pack('@{}B'.format(len((descStrASCII))), *descStrASCII)) - - # Write channel names - for chan in chanNames: - ordchan = [ord(char) for char in chan]+ [32]*(10-len(chan)) - fid.write(struct.pack('@10B', *ordchan)) - - # Write channel units - for unit in chanUnits: - ordunit = [ord(char) for char in unit]+ [32]*(10-len(unit)) - fid.write(struct.pack('@10B', *ordunit)) - - # Pack data - packedData=np.zeros((nT, nChannels), dtype=np.int16) - for iChan in range(nChannels): - packedData[:,iChan] = np.clip( ColScl[iChan]*dataWithoutTime[:,iChan]+ColOff[iChan], int16Min, int16Max) - - # Write data - fid.write(struct.pack('@{}h'.format(packedData.size), *packedData.flatten())) - fid.close() - - -if __name__ == "__main__": - B=FASTOutputFile('tests/example_files/FASTOutBin.outb') - df=B.toDataFrame() - B.writeDataFrame(df, 'tests/example_files/FASTOutBin_OUT.outb') - - +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +from io import open +from builtins import map +from builtins import range +from builtins import chr +from builtins import str +from future import standard_library +standard_library.install_aliases() + +from itertools import takewhile + +try: + from .file import File, WrongFormatError, BrokenReaderError, EmptyFileError +except: + # --- Allowing this file to be standalone.. + class WrongFormatError(Exception): + pass + class WrongReaderError(Exception): + pass + class EmptyFileError(Exception): + pass + File = dict +try: + from .csv_file import CSVFile +except: + print('CSVFile not available') +import numpy as np +import pandas as pd +import struct +import os +import re + + +# --------------------------------------------------------------------------------} +# --- OUT FILE +# --------------------------------------------------------------------------------{ +class FASTOutputFile(File): + """ + Read an OpenFAST ouput file (.out, .outb, .elev). + + Main methods + ------------ + - read, write, toDataFrame + + Examples + -------- + + # read an output file, convert it to pandas dataframe, modify it, write it back + f = FASTOutputFile('5MW.outb') + df=f.toDataFrame() + time = df['Time_[s]'] + Omega = df['RotSpeed_[rpm]'] + df['Time_[s]'] -=100 + f.writeDataFrame(df, '5MW_TimeShifted.outb') + + """ + + @staticmethod + def defaultExtensions(): + return ['.out','.outb','.elm','.elev'] + + @staticmethod + def formatName(): + return 'FAST output file' + + def _read(self): + def readline(iLine): + with open(self.filename) as f: + for i, line in enumerate(f): + if i==iLine-1: + return line.strip() + elif i>=iLine: + break + + ext = os.path.splitext(self.filename.lower())[1] + self.info={} + self['binary']=False + try: + if ext in ['.out','.elev']: + self.data, self.info = load_ascii_output(self.filename) + elif ext=='.outb': + self.data, self.info = load_binary_output(self.filename) + self['binary']=True + elif ext=='.elm': + F=CSVFile(filename=self.filename, sep=' ', commentLines=[0,2],colNamesLine=1) + self.data = F.data + del F + self.info['attribute_units']=readline(3).replace('sec','s').split() + self.info['attribute_names']=self.data.columns.values + else: + self.data, self.info = load_output(self.filename) + except MemoryError as e: + raise BrokenReaderError('FAST Out File {}: Memory error encountered\n{}'.format(self.filename,e)) + except Exception as e: + raise WrongFormatError('FAST Out File {}: {}'.format(self.filename,e.args)) + if self.data.shape[0]==0: + raise EmptyFileError('This FAST output file contains no data: {}'.format(self.filename)) + + if self.info['attribute_units'] is not None: + self.info['attribute_units'] = [re.sub(r'[()\[\]]','',u) for u in self.info['attribute_units']] + + + def _write(self): + if self['binary']: + channels = self.data + chanNames = self.info['attribute_names'] + chanUnits = self.info['attribute_units'] + descStr = self.info['description'] + writeBinary(self.filename, channels, chanNames, chanUnits, fileID=2, descStr=descStr) + else: + # ascii output + with open(self.filename,'w') as f: + f.write('\t'.join(['{:>10s}'.format(c) for c in self.info['attribute_names']])+'\n') + f.write('\t'.join(['{:>10s}'.format('('+u+')') for u in self.info['attribute_units']])+'\n') + # TODO better.. + f.write('\n'.join(['\t'.join(['{:10.4f}'.format(y[0])]+['{:10.3e}'.format(x) for x in y[1:]]) for y in self.data])) + + def _toDataFrame(self): + if self.info['attribute_units'] is not None: + cols=[n+'_['+u.replace('sec','s')+']' for n,u in zip(self.info['attribute_names'],self.info['attribute_units'])] + else: + cols=self.info['attribute_names'] + if isinstance(self.data, pd.DataFrame): + df= self.data + df.columns=cols + else: + df = pd.DataFrame(data=self.data,columns=cols) + + return df + + def writeDataFrame(self, df, filename, binary=True): + writeDataFrame(df, filename, binary=binary) + +# -------------------------------------------------------------------------------- +# --- Helper low level functions +# -------------------------------------------------------------------------------- +def load_output(filename): + """Load a FAST binary or ascii output file + + Parameters + ---------- + filename : str + filename + + Returns + ------- + data : ndarray + data values + info : dict + info containing: + - name: filename + - description: description of dataset + - attribute_names: list of attribute names + - attribute_units: list of attribute units + """ + + assert os.path.isfile(filename), "File, %s, does not exists" % filename + with open(filename, 'r') as f: + try: + f.readline() + except UnicodeDecodeError: + return load_binary_output(filename) + return load_ascii_output(filename) + +def load_ascii_output(filename): + with open(filename) as f: + info = {} + info['name'] = os.path.splitext(os.path.basename(filename))[0] + # Header is whatever is before the keyword `time` + in_header = True + header = [] + while in_header: + l = f.readline() + if not l: + raise Exception('Error finding the end of FAST out file header. Keyword Time missing.') + in_header= (l+' dummy').lower().split()[0] != 'time' + if in_header: + header.append(l) + else: + info['description'] = header + info['attribute_names'] = l.split() + info['attribute_units'] = [unit[1:-1] for unit in f.readline().split()] + # --- + # Data, up to end of file or empty line (potential comment line at the end) +# data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(np.float) + # --- + data = np.loadtxt(f, comments=('This')) # Adding "This" for the Hydro Out files.. + return data, info + + +def load_binary_output(filename, use_buffer=True): + """ + 03/09/15: Ported from ReadFASTbinary.m by Mads M Pedersen, DTU Wind + 24/10/18: Low memory/buffered version by E. Branlard, NREL + 18/01/19: New file format for exctended channels, by E. Branlard, NREL + + Info about ReadFASTbinary.m: + % Author: Bonnie Jonkman, National Renewable Energy Laboratory + % (c) 2012, National Renewable Energy Laboratory + % + % Edited for FAST v7.02.00b-bjj 22-Oct-2012 + """ + def fread(fid, n, type): + fmt, nbytes = {'uint8': ('B', 1), 'int16':('h', 2), 'int32':('i', 4), 'float32':('f', 4), 'float64':('d', 8)}[type] + return struct.unpack(fmt * n, fid.read(nbytes * n)) + + def freadRowOrderTableBuffered(fid, n, type_in, nCols, nOff=0, type_out='float64'): + """ + Reads of row-ordered table from a binary file. + + Read `n` data of type `type_in`, assumed to be a row ordered table of `nCols` columns. + Memory usage is optimized by allocating the data only once. + Buffered reading is done for improved performances (in particular for 32bit python) + + `nOff` allows for additional column space at the begining of the storage table. + Typically, `nOff=1`, provides a column at the beginning to store the time vector. + + @author E.Branlard, NREL + + """ + fmt, nbytes = {'uint8': ('B', 1), 'int16':('h', 2), 'int32':('i', 4), 'float32':('f', 4), 'float64':('d', 8)}[type_in] + nLines = int(n/nCols) + GoodBufferSize = 4096*40 + nLinesPerBuffer = int(GoodBufferSize/nCols) + BufferSize = nCols * nLinesPerBuffer + nBuffer = int(n/BufferSize) + # Allocation of data + data = np.zeros((nLines,nCols+nOff), dtype = type_out) + # Reading + try: + nIntRead = 0 + nLinesRead = 0 + while nIntRead0: + op,cl = chars + iu=c.rfind(op) + if iu>1: + name = c[:iu] + unit = c[iu+1:].replace(cl,'') + if name[-1]=='_': + name=name[:-1] + + chanNames.append(name) + chanUnits.append(unit) + + if binary: + writeBinary(filename, channels, chanNames, chanUnits) + else: + NotImplementedError() + + +def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr=''): + """ + Write an OpenFAST binary file. + + Based on contributions from + Hugo Castro, David Schlipf, Hochschule Flensburg + + Input: + FileName - string: contains file name to open + Channels - 2-D array: dimension 1 is time, dimension 2 is channel + ChanName - cell array containing names of output channels + ChanUnit - cell array containing unit names of output channels, preferably surrounded by parenthesis + FileID - constant that determines if the time is stored in the + output, indicating possible non-constant time step + DescStr - String describing the file + """ + # Data sanitization + chanNames = list(chanNames) + channels = np.asarray(channels) + if chanUnits[0][0]!='(': + chanUnits = ['('+u+')' for u in chanUnits] # units surrounded by parenthesis to match OpenFAST convention + + nT, nChannelsWithTime = np.shape(channels) + nChannels = nChannelsWithTime - 1 + + # For FileID =2, time needs to be present and at the first column + try: + iTime = chanNames.index('Time') + except ValueError: + raise Exception('`Time` needs to be present in channel names' ) + if iTime!=0: + raise Exception('`Time` needs to be the first column of `chanName`' ) + + time = channels[:,iTime] + timeStart = time[0] + timeIncr = time[1]-time[0] + dataWithoutTime = channels[:,1:] + + # Compute data range, scaling and offsets to convert to int16 + # To use the int16 range to its fullest, the max float is matched to 2^15-1 and the + # the min float is matched to -2^15. Thus, we have the to equations we need + # to solve to get scaling and offset, see line 120 of ReadFASTbinary: + # Int16Max = FloatMax * Scaling + Offset + # Int16Min = FloatMin * Scaling + Offset + int16Max = np.single( 32767.0) # Largest integer represented in 2 bytes, 2**15 - 1 + int16Min = np.single(-32768.0) # Smallest integer represented in 2 bytes -2**15 + int16Rng = np.single(int16Max - int16Min) # Max Range of 2 byte integer + mins = np.min(dataWithoutTime, axis=0) + ranges = np.single(np.max(dataWithoutTime, axis=0) - mins) + ranges[ranges==0]=1 # range set to 1 for constant channel. In OpenFAST: /sqrt(epsilon(1.0_SiKi)) + ColScl = np.single(int16Rng/ranges) + ColOff = np.single(int16Min - np.single(mins)*ColScl) + + #Just available for fileID + if fileID != 2: + print("current version just works with FileID = 2") + + else: + with open(fileName,'wb') as fid: + # Notes on struct: + # @ is used for packing in native byte order + # B - unsigned integer 8 bits + # h - integer 16 bits + # i - integer 32 bits + # f - float 32 bits + # d - float 64 bits + + # Write header informations + fid.write(struct.pack('@h',fileID)) + fid.write(struct.pack('@i',nChannels)) + fid.write(struct.pack('@i',nT)) + fid.write(struct.pack('@d',timeStart)) + fid.write(struct.pack('@d',timeIncr)) + fid.write(struct.pack('@{}f'.format(nChannels), *ColScl)) + fid.write(struct.pack('@{}f'.format(nChannels), *ColOff)) + descStrASCII = [ord(char) for char in descStr] + fid.write(struct.pack('@i',len(descStrASCII))) + fid.write(struct.pack('@{}B'.format(len((descStrASCII))), *descStrASCII)) + + # Write channel names + for chan in chanNames: + ordchan = [ord(char) for char in chan]+ [32]*(10-len(chan)) + fid.write(struct.pack('@10B', *ordchan)) + + # Write channel units + for unit in chanUnits: + ordunit = [ord(char) for char in unit]+ [32]*(10-len(unit)) + fid.write(struct.pack('@10B', *ordunit)) + + # Pack data + packedData=np.zeros((nT, nChannels), dtype=np.int16) + for iChan in range(nChannels): + packedData[:,iChan] = np.clip( ColScl[iChan]*dataWithoutTime[:,iChan]+ColOff[iChan], int16Min, int16Max) + + # Write data + fid.write(struct.pack('@{}h'.format(packedData.size), *packedData.flatten())) + fid.close() + + +if __name__ == "__main__": + B=FASTOutputFile('tests/example_files/FASTOutBin.outb') + df=B.toDataFrame() + B.writeDataFrame(df, 'tests/example_files/FASTOutBin_OUT.outb') + + diff --git a/pydatview/io/hawc2_ae_file.py b/pydatview/io/hawc2_ae_file.py index 46e7f13..58c1cc0 100644 --- a/pydatview/io/hawc2_ae_file.py +++ b/pydatview/io/hawc2_ae_file.py @@ -1,71 +1,91 @@ -""" -Hawc2 AE file -""" -import os -import pandas as pd - -try: - from .file import File, WrongFormatError, EmptyFileError -except: - EmptyFileError = type('EmptyFileError', (Exception,),{}) - WrongFormatError = type('WrongFormatError', (Exception,),{}) - -from .wetb.hawc2.ae_file import AEFile - -class HAWC2AEFile(File): - - @staticmethod - def defaultExtensions(): - return ['.dat','.ae','.txt'] - - @staticmethod - def formatName(): - return 'HAWC2 AE file' - - def __init__(self,filename=None,**kwargs): - if filename: - self.filename = filename - self.read(**kwargs) - else: - self.filename = None - self.data = AEFile() - - def read(self, filename=None, **kwargs): - if filename: - self.filename = filename - if not self.filename: - raise Exception('No filename provided') - if not os.path.isfile(self.filename): - raise OSError(2,'File not found:',self.filename) - if os.stat(self.filename).st_size == 0: - raise EmptyFileError('File is empty:',self.filename) - # --- - try: - self.data = AEFile(self.filename) - except Exception as e: - raise WrongFormatError('AE File {}: '.format(self.filename)+e.args[0]) - - def write(self, filename=None): - if filename: - self.filename = filename - if not self.filename: - raise Exception('No filename provided') - # --- - self.data.save(self.filename) - - def toDataFrame(self): - cols=['radius_[m]','chord_[m]','thickness_[%]','pc_set_[#]'] - nset = len(self.data.ae_sets) - if nset == 1: - return pd.DataFrame(data=self.data.ae_sets[1], columns=cols) - else: - dfs = {} - for iset,aeset in enumerate(self.data.ae_sets): - name='ae_set_{}'.format(iset+1) - dfs[name] = pd.DataFrame(data=self.data.ae_sets[iset+1], columns=cols) - return dfs - - # --- Convenient utils - def add_set(self, **kwargs): - self.data.add_set(**kwargs) - +""" +Hawc2 AE file +""" +import os +import numpy as np +import pandas as pd + +try: + from .file import File, WrongFormatError, EmptyFileError +except: + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + +from .wetb.hawc2.ae_file import AEFile + +class HAWC2AEFile(File): + + @staticmethod + def defaultExtensions(): + return ['.dat','.ae','.txt'] + + @staticmethod + def formatName(): + return 'HAWC2 AE file' + + def __init__(self,filename=None,**kwargs): + if filename: + self.filename = filename + self.read(**kwargs) + else: + self.filename = None + self.data = AEFile() + + def read(self, filename=None, **kwargs): + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # --- + try: + self.data = AEFile(self.filename) + except Exception as e: + raise WrongFormatError('AE File {}: '.format(self.filename)+e.args[0]) + + def write(self, filename=None): + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + # --- + self.data.save(self.filename) + + def toDataFrame(self): + cols=['radius_[m]','chord_[m]','thickness_[%]','pc_set_[#]'] + nset = len(self.data.ae_sets) + if nset == 1: + return pd.DataFrame(data=self.data.ae_sets[1], columns=cols) + else: + dfs = {} + for iset,aeset in enumerate(self.data.ae_sets): + name='ae_set_{}'.format(iset+1) + dfs[name] = pd.DataFrame(data=self.data.ae_sets[iset+1], columns=cols) + return dfs + + @property + def sets(self): + # Returns a list of ae_sets, otherwise not easy to iterate + sets=[] + for iset,aeset in enumerate(self.data.ae_sets): + sets.append(self.data.ae_sets[iset+1]) + return sets + + # --- Convenient utils + def add_set(self, **kwargs): + self.data.add_set(**kwargs) + + def __repr__(self): + cols=['radius_[m]','chord_[m]','thickness_[%]','pc_set_[#]'] + nRows = [np.asarray(s).shape[0] for s in self.sets] + s='<{} object>\n'.format(type(self).__name__) + s+='| Attributes:\n' + s+='| - filename: {}\n'.format(self.filename) + s+='| - data: AEFile, with attributes `ae_sets`\n' + s+='| Derived attributes:\n' + s+='| * sets: list of {} arrays, length: {}, 4 columns: {}\n'.format(len(self.data.ae_sets), nRows, cols) + s+='| Methods: add_set, toDataFrame\n' + return s diff --git a/pydatview/io/hawc2_htc_file.py b/pydatview/io/hawc2_htc_file.py index 51fb1ce..7a95c5d 100644 --- a/pydatview/io/hawc2_htc_file.py +++ b/pydatview/io/hawc2_htc_file.py @@ -1,117 +1,132 @@ -""" -Wrapper around wetb to read/write htc files. -TODO: rewrite of c2_def might not be obvious -""" -from .file import File - -import numpy as np -import pandas as pd -import os - -from .wetb.hawc2.htc_file import HTCFile -from .hawc2_st_file import HAWC2StFile - -class HAWC2HTCFile(File): - - @staticmethod - def defaultExtensions(): - return ['.htc'] - - @staticmethod - def formatName(): - return 'HAWC2 htc file' - - def _read(self): - self.data = HTCFile(self.filename) - self.data.contents # trigger read - - def _write(self): - self.data.save(self.filename) - - def __repr__(self): - s='<{} object> with attribute `data`\n'.format(type(self).__name__) - return s - - def bodyByName(self, bodyname): - """ return body inputs given a body name""" - struct = self.data.new_htc_structure - bodyKeys = [k for k in struct.keys() if k.startswith('main_body')] - bdies = [struct[k] for k in bodyKeys if struct[k].name[0]==bodyname] - if len(bdies)==1: - return bdies[0] - elif len(bdies)==0: - raise Exception('No body found with name {} in file {}'.format(bodyname,self.filename)) - else: - raise Exception('Several bodies found with name {} in file {}'.format(bodyname,self.filename)) - - def bodyC2(self, bdy): - """ return body C2_def given body inputs""" - try: - nsec = bdy.c2_def.nsec[0] - except: - raise Exception('body has no c2_def section') - val = np.array([bdy.c2_def[k].values[0:] for k in bdy.c2_def.keys() if k.startswith('sec')]) - val = val.reshape((-1,5)).astype(float) - val = val[np.argsort(val[:,0]),:] - return val - - def setBodyC2(self, bdy, val): - """ set body C2_def given body inputs and new c2def""" - # TODO different number of section - nsec = bdy.c2_def.nsec[0] - nsec_new = val.shape[0] - sec_keys = [k for k in bdy.c2_def.keys() if k.startswith('sec')] - bdy.c2_def.nsec = nsec_new - if nsec != nsec_new: - if nsec_new>> keys', struct.keys()) + bdDict={} + for k in bodyKeys: + bodyName = struct[k].name[0] + bdDict[bodyName] = struct[k] + return bdDict + + def bodyByName(self, bodyname): + """ return body inputs given a body name""" + bodyDict= self.bodyDict + if bodyname not in bodyDict.keys(): + raise Exception('No body found with name {} in file {}'.format(bodyname,self.filename)) + return bodyDict[bodyname] + + def bodyC2(self, bdy): + """ return body C2_def given body inputs""" + try: + nsec = bdy.c2_def.nsec[0] + except: + raise Exception('body has no c2_def section') + val = np.array([bdy.c2_def[k].values[0:] for k in bdy.c2_def.keys() if k.startswith('sec')]) + val = val.reshape((-1,5)).astype(float) + val = val[np.argsort(val[:,0]),:] + return val + + def setBodyC2(self, bdy, val): + """ set body C2_def given body inputs and new c2def""" + # TODO different number of section + nsec = bdy.c2_def.nsec[0] + nsec_new = val.shape[0] + sec_keys = [k for k in bdy.c2_def.keys() if k.startswith('sec')] + bdy.c2_def.nsec = nsec_new + if nsec != nsec_new: + if nsec_new channels 1,3,2,2 - # if empty all channels are returned - file() => all channels as 1,2,3,... - file.t => time vector - -1. version: 19/4-2011 -2. version: 5/11-2015 fixed columns to get description right, fixed time vector (mmpe@dtu.dk) - -Need to be done: - * add error handling for allmost every thing - -""" -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import absolute_import -from builtins import int -from builtins import range -from io import open as opent -from builtins import str -from future import standard_library -standard_library.install_aliases() -from builtins import object -import numpy as np -import os - -#from wetb import gtsdf - -# FIXME: numpy doesn't like io.open binary fid in PY27, why is that? As a hack -# workaround, use opent for PY23 compatibility when handling text files, -# and default open for binary - -################################################################################ -################################################################################ -################################################################################ -# Read HAWC2 class -################################################################################ -class ReadHawc2(object): - """ - """ -################################################################################ -# read *.sel file - def _ReadSelFile(self): - """ - Some title - ========== - - Using docstrings formatted according to the reStructuredText specs - can be used for automated documentation generation with for instance - Sphinx: http://sphinx.pocoo.org/. - - Parameters - ---------- - signal : ndarray - some description - - Returns - ------- - output : int - describe variable - """ - - # read *.sel hawc2 output file for result info - if self.FileName.lower().endswith('.sel'): - self.FileName = self.FileName[:-4] - fid = opent(self.FileName + '.sel', 'r') - Lines = fid.readlines() - fid.close() - if Lines[0].lower().find('bhawc')>=0: - # --- Find line with scan info - iLine=0 - for i in np.arange(5,10): - if Lines[i].lower().find('scans')>=0: - iLine=i+1 - if iLine==0: - raise Exception('Cannot find the keyword "scans"') - temp = Lines[iLine].split() - self.NrSc = int(temp[0]) - self.NrCh = int(temp[1]) - self.Time = float(temp[2]) - self.Freq = self.NrSc / self.Time - self.t = np.linspace(0, self.Time, self.NrSc + 1)[1:] - # --- Find line with channel info - iLine=0 - for i in np.arange(5,13): - if Lines[i].lower().find('channel')>=0: - iLine=i+1 - if iLine==0: - raise Exception('Cannot find the keyword "Channel"') - - # reads channel info (name, unit and description) - Name = []; Unit = []; Description = []; - for i in range(0, self.NrCh+1): - if (i+iLine)>=len(Lines): - break - line = Lines[i + iLine].strip() - if len(line)==0: - continue - # --- removing number and unit - sp=[sp.strip() for sp in line.split() if len(sp.strip())>0] - num = sp[0] - iNum = line.find(num) - line = line[iNum+len(num)+1:] - unit = sp[-1] - iUnit = line.find(unit) - line = line[:iUnit] - # --- Splitting to find label and description - sp=[sp.strip() for sp in line.split('\t') if len(sp.strip())>0] - if len(sp)!=2: - for nSpaces in np.arange(2,15): - sp=[sp.strip() for sp in line.split(' '*nSpaces) if len(sp.strip())>0] - if len(sp)==2: - break - if len(sp)!=2: - raise Exception('Dont know how to split the input of the sel file into 4 columns') - - Unit.append(unit) - Description.append(sp[0]) - Name.append(sp[1]) - - self.ChInfo = [Name, Unit, Description] - self.FileFormat = 'BHAWC_ASCII' - else: - - # findes general result info (number of scans, number of channels, - # simulation time and file format) - temp = Lines[8].split() - self.NrSc = int(temp[0]) - self.NrCh = int(temp[1]) - self.Time = float(temp[2]) - self.Freq = self.NrSc / self.Time - self.t = np.linspace(0, self.Time, self.NrSc + 1)[1:] - Format = temp[3] - # reads channel info (name, unit and description) - Name = []; Unit = []; Description = []; - for i in range(0, self.NrCh): - temp = str(Lines[i + 12][12:43]); Name.append(temp.strip()) - temp = str(Lines[i + 12][43:54]); Unit.append(temp.strip()) - temp = str(Lines[i + 12][54:-1]); Description.append(temp.strip()) - self.ChInfo = [Name, Unit, Description] - # if binary file format, scaling factors are read - if Format.lower() == 'binary': - self.ScaleFactor = np.zeros(self.NrCh) - self.FileFormat = 'HAWC2_BINARY' - for i in range(0, self.NrCh): - self.ScaleFactor[i] = float(Lines[i + 12 + self.NrCh + 2]) - else: - self.FileFormat = 'HAWC2_ASCII' -################################################################################ -# read sensor file for FLEX format - def _ReadSensorFile(self): - # read sensor file used if results are saved in FLEX format - DirName = os.path.dirname(self.FileName) - try: - fid = opent(DirName + r"\sensor ", 'r') - except IOError: - print ("can't finde sensor file for FLEX format") - return - Lines = fid.readlines() - fid.close() - # reads channel info (name, unit and description) - self.NrCh = 0 - Name = [] - Unit = [] - Description = [] - for i in range(2, len(Lines)): - temp = Lines[i] - if not temp.strip(): - break - self.NrCh += 1 - temp = str(Lines[i][38:45]); Unit.append(temp.strip()) - temp = str(Lines[i][45:53]); Name.append(temp.strip()) - temp = str(Lines[i][53:]); Description.append(temp.strip()) - self.ChInfo = [Name, Unit, Description] - # read general info from *.int file - fid = open(self.FileName, 'rb') - fid.seek(4 * 19) - if not np.fromfile(fid, 'int32', 1) == self.NrCh: - print ("number of sensors in sensor file and data file are not consisten") - fid.seek(4 * (self.NrCh) + 4, 1) - self.Version = np.fromfile(fid, 'int32',1)[0] - temp = np.fromfile(fid, 'f', 2) - self.Freq = 1 / temp[1]; - self.ScaleFactor = np.fromfile(fid, 'f', self.NrCh) - fid.seek(2 * 4 * self.NrCh + 48 * 2) - self.NrSc = int(len(np.fromfile(fid, 'int16')) / self.NrCh) - self.Time = self.NrSc * temp[1] - self.t = np.arange(0, self.Time, temp[1]) - fid.close() -################################################################################ -# init function, load channel and other general result file info - def __init__(self, FileName, ReadOnly=0): - self.FileName = FileName - self.ReadOnly = ReadOnly - self.Iknown = [] # to keep track of what has been read all ready - self.Data = np.zeros(0) - if FileName.lower().endswith('.sel') or os.path.isfile(FileName + ".sel"): - self._ReadSelFile() - elif FileName.lower().endswith('.dat') and os.path.isfile(os.path.splitext(FileName)[0] + ".sel"): - self.FileName = os.path.splitext(FileName)[0] - self._ReadSelFile() - elif FileName.lower().endswith('.int') or FileName.lower().endswith('.res'): - self.FileFormat = 'FLEX' - self._ReadSensorFile() - elif os.path.isfile(self.FileName + ".int"): - self.FileName = self.FileName + ".int" - self.FileFormat = 'FLEX' - self._ReadSensorFile() - elif os.path.isfile(self.FileName + ".res"): - self.FileName = self.FileName + ".res" - self.FileFormat = 'FLEX' - self._ReadSensorFile() - elif FileName.lower().endswith('.hdf5') or os.path.isfile(self.FileName + ".hdf5"): - self.FileFormat = 'GTSDF' - self.ReadGtsdf() - else: - raise Exception("unknown file: " + FileName) -################################################################################ -# Read results in binary format - def ReadBinary(self, ChVec=None): - ChVec = [] if ChVec is None else ChVec - if not ChVec: - ChVec = range(0, self.NrCh) - with open(self.FileName + '.dat', 'rb') as fid: - data = np.zeros((self.NrSc, len(ChVec))) - j = 0 - for i in ChVec: - fid.seek(i * self.NrSc * 2, 0) - data[:, j] = np.fromfile(fid, 'int16', self.NrSc) * self.ScaleFactor[i] - j += 1 - return data -################################################################################ -# Read results in ASCII format - def ReadAscii(self, ChVec=None): - ChVec = [] if ChVec is None else ChVec - if not ChVec: - ChVec = range(0, self.NrCh) - temp = np.loadtxt(self.FileName + '.dat', usecols=ChVec) - return temp.reshape((self.NrSc, len(ChVec))) -################################################################################ -# Read results in FLEX format - def ReadFLEX(self, ChVec=None): - ChVec = [] if ChVec is None else ChVec - if not ChVec: - ChVec = range(1, self.NrCh) - fid = open(self.FileName, 'rb') - fid.seek(2 * 4 * self.NrCh + 48 * 2) - temp = np.fromfile(fid, 'int16') - if self.Version==3: - temp = temp.reshape(self.NrCh, self.NrSc).transpose() - else: - temp = temp.reshape(self.NrSc, self.NrCh) - fid.close() - return np.dot(temp[:, ChVec], np.diag(self.ScaleFactor[ChVec])) -################################################################################ -# Read results in GTSD format - def ReadGtsdf(self): - raise NotImplementedError - #self.t, data, info = gtsdf.load(self.FileName + '.hdf5') - #self.Time = self.t - #self.ChInfo = [['Time'] + info['attribute_names'], - # ['s'] + info['attribute_units'], - # ['Time'] + info['attribute_descriptions']] - #self.NrCh = data.shape[1] + 1 - #self.NrSc = data.shape[0] - #self.Freq = self.NrSc / self.Time - #self.FileFormat = 'GTSDF' - #self.gtsdf_description = info['description'] - #data = np.hstack([self.Time[:,np.newaxis], data]) - #return data -################################################################################ -# One stop call for reading all data formats - def ReadAll(self, ChVec=None): - ChVec = [] if ChVec is None else ChVec - if not ChVec and not self.FileFormat == 'GTSDF': - ChVec = range(0, self.NrCh) - if self.FileFormat == 'HAWC2_BINARY': - return self.ReadBinary(ChVec) - elif self.FileFormat == 'HAWC2_ASCII' or self.FileFormat == 'BHAWC_ASCII': - return self.ReadAscii(ChVec) - elif self.FileFormat == 'GTSDF': - return self.ReadGtsdf() - elif self.FileFormat == 'FLEX': - return self.ReadFLEX(ChVec) - else: - raise Exception('Unknown file format {} for hawc2 out file'.format(self.FileFormat)) - -################################################################################ -# Main read data call, read, save and sort data - def __call__(self, ChVec=None): - ChVec = [] if ChVec is None else ChVec - if not ChVec: - ChVec = range(0, self.NrCh) - elif max(ChVec) >= self.NrCh: - print("to high channel number") - return - # if ReadOnly, read data but no storeing in memory - if self.ReadOnly: - return self.ReadAll(ChVec) - # if not ReadOnly, sort in known and new channels, read new channels - # and return all requested channels - else: - # sort into known channels and channels to be read - I1 = [] - I2 = [] # I1=Channel mapping, I2=Channels to be read - for i in ChVec: - try: - I1.append(self.Iknown.index(i)) - except: - self.Iknown.append(i) - I2.append(i) - I1.append(len(I1)) - # read new channels - if I2: - temp = self.ReadAll(I2) - # add new channels to Data - if self.Data.any(): - self.Data = np.append(self.Data, temp, axis=1) - # if first call, so Daata is empty - else: - self.Data = temp - return self.Data[:, tuple(I1)] - - -################################################################################ -################################################################################ -################################################################################ -# write HAWC2 class, to be implemented -################################################################################ - -if __name__ == '__main__': - res_file = ReadHawc2('structure_wind') - results = res_file.ReadAscii() - channelinfo = res_file.ChInfo +# -*- coding: utf-8 -*- +""" +Author: + Bjarne S. Kallesoee + + +Description: + Reads all HAWC2 output data formats, HAWC2 ascii, HAWC2 binary and FLEX + +call ex.: + # creat data file object, call without extension, but with parth + file = ReadHawc2("HAWC2ex/tests") + # if called with ReadOnly = 1 as + file = ReadHawc2("HAWC2ex/tests",ReadOnly=1) + # no channels a stored in memory, otherwise read channels are stored for reuse + + # channels are called by a list + file([0,2,1,1]) => channels 1,3,2,2 + # if empty all channels are returned + file() => all channels as 1,2,3,... + file.t => time vector + +1. version: 19/4-2011 +2. version: 5/11-2015 fixed columns to get description right, fixed time vector (mmpe@dtu.dk) + +Need to be done: + * add error handling for allmost every thing + +""" +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import absolute_import +from builtins import int +from builtins import range +from io import open as opent +from builtins import str +from future import standard_library +standard_library.install_aliases() +from builtins import object +import numpy as np +import os + +#from wetb import gtsdf + +# FIXME: numpy doesn't like io.open binary fid in PY27, why is that? As a hack +# workaround, use opent for PY23 compatibility when handling text files, +# and default open for binary + +################################################################################ +################################################################################ +################################################################################ +# Read HAWC2 class +################################################################################ +class ReadHawc2(object): + """ + """ +################################################################################ +# read *.sel file + def _ReadSelFile(self): + """ + Some title + ========== + + Using docstrings formatted according to the reStructuredText specs + can be used for automated documentation generation with for instance + Sphinx: http://sphinx.pocoo.org/. + + Parameters + ---------- + signal : ndarray + some description + + Returns + ------- + output : int + describe variable + """ + + # read *.sel hawc2 output file for result info + if self.FileName.lower().endswith('.sel'): + self.FileName = self.FileName[:-4] + fid = opent(self.FileName + '.sel', 'r') + Lines = fid.readlines() + fid.close() + if Lines[0].lower().find('bhawc')>=0: + # --- Find line with scan info + iLine=0 + for i in np.arange(5,10): + if Lines[i].lower().find('scans')>=0: + iLine=i+1 + if iLine==0: + raise Exception('Cannot find the keyword "scans"') + temp = Lines[iLine].split() + self.NrSc = int(temp[0]) + self.NrCh = int(temp[1]) + self.Time = float(temp[2]) + self.Freq = self.NrSc / self.Time + self.t = np.linspace(0, self.Time, self.NrSc + 1)[1:] + # --- Find line with channel info + iLine=0 + for i in np.arange(5,13): + if Lines[i].lower().find('channel')>=0: + iLine=i+1 + if iLine==0: + raise Exception('Cannot find the keyword "Channel"') + + # reads channel info (name, unit and description) + Name = []; Unit = []; Description = []; + for i in range(0, self.NrCh+1): + if (i+iLine)>=len(Lines): + break + line = Lines[i + iLine].strip() + if len(line)==0: + continue + # --- removing number and unit + sp=[sp.strip() for sp in line.split() if len(sp.strip())>0] + num = sp[0] + iNum = line.find(num) + line = line[iNum+len(num)+1:] + unit = sp[-1] + iUnit = line.find(unit) + line = line[:iUnit] + # --- Splitting to find label and description + sp=[sp.strip() for sp in line.split('\t') if len(sp.strip())>0] + if len(sp)!=2: + for nSpaces in np.arange(2,15): + sp=[sp.strip() for sp in line.split(' '*nSpaces) if len(sp.strip())>0] + if len(sp)==2: + break + if len(sp)!=2: + raise Exception('Dont know how to split the input of the sel file into 4 columns') + + Unit.append(unit) + Description.append(sp[0]) + Name.append(sp[1]) + + self.ChInfo = [Name, Unit, Description] + self.FileFormat = 'BHAWC_ASCII' + else: + + # findes general result info (number of scans, number of channels, + # simulation time and file format) + temp = Lines[8].split() + self.NrSc = int(temp[0]) + self.NrCh = int(temp[1]) + self.Time = float(temp[2]) + self.Freq = self.NrSc / self.Time + self.t = np.linspace(0, self.Time, self.NrSc + 1)[1:] + Format = temp[3] + # reads channel info (name, unit and description) + Name = []; Unit = []; Description = []; + for i in range(0, self.NrCh): + temp = str(Lines[i + 12][12:43]); Name.append(temp.strip()) + temp = str(Lines[i + 12][43:54]); Unit.append(temp.strip()) + temp = str(Lines[i + 12][54:-1]); Description.append(temp.strip()) + self.ChInfo = [Name, Unit, Description] + # if binary file format, scaling factors are read + if Format.lower() == 'binary': + self.ScaleFactor = np.zeros(self.NrCh) + self.FileFormat = 'HAWC2_BINARY' + for i in range(0, self.NrCh): + self.ScaleFactor[i] = float(Lines[i + 12 + self.NrCh + 2]) + else: + self.FileFormat = 'HAWC2_ASCII' +################################################################################ +# read sensor file for FLEX format + def _ReadSensorFile(self): + # read sensor file used if results are saved in FLEX format + DirName = os.path.dirname(self.FileName) + try: + fid = opent(DirName + r"\sensor ", 'r') + except IOError: + print("can't finde sensor file for FLEX format") + return + Lines = fid.readlines() + fid.close() + # reads channel info (name, unit and description) + self.NrCh = 0 + Name = [] + Unit = [] + Description = [] + for i in range(2, len(Lines)): + temp = Lines[i] + if not temp.strip(): + break + self.NrCh += 1 + temp = str(Lines[i][38:45]) + Unit.append(temp.strip()) + temp = str(Lines[i][45:53]) + Name.append(temp.strip()) + temp = str(Lines[i][53:]) + Description.append(temp.strip()) + self.ChInfo = [Name, Unit, Description] + # read general info from *.int file + fid = open(self.FileName, 'rb') + fid.seek(4 * 19) + if not np.fromfile(fid, 'int32', 1) == self.NrCh: + print("number of sensors in sensor file and data file are not consisten") + fid.seek(4 * (self.NrCh) + 4, 1) + self.Version = np.fromfile(fid, 'int32',1)[0] + time_start, time_step = np.fromfile(fid, 'f', 2) + self.Freq = 1 / time_step + self.ScaleFactor = np.fromfile(fid, 'f', self.NrCh) + fid.seek(2 * 4 * self.NrCh + 48 * 2) + self.NrSc = int(len(np.fromfile(fid, 'int16')) / self.NrCh) + self.Time = self.NrSc * time_step + self.t = np.arange(0, self.Time, time_step) + time_start + fid.close() +################################################################################ +# init function, load channel and other general result file info + def __init__(self, FileName, ReadOnly=0): + self.FileName = FileName + self.ReadOnly = ReadOnly + self.Iknown = [] # to keep track of what has been read all ready + self.Data = np.zeros(0) + if FileName.lower().endswith('.sel') or os.path.isfile(FileName + ".sel"): + self._ReadSelFile() + elif FileName.lower().endswith('.dat') and os.path.isfile(os.path.splitext(FileName)[0] + ".sel"): + self.FileName = os.path.splitext(FileName)[0] + self._ReadSelFile() + elif FileName.lower().endswith('.int') or FileName.lower().endswith('.res'): + self.FileFormat = 'FLEX' + self._ReadSensorFile() + elif os.path.isfile(self.FileName + ".int"): + self.FileName = self.FileName + ".int" + self.FileFormat = 'FLEX' + self._ReadSensorFile() + elif os.path.isfile(self.FileName + ".res"): + self.FileName = self.FileName + ".res" + self.FileFormat = 'FLEX' + self._ReadSensorFile() + elif FileName.lower().endswith('.hdf5') or os.path.isfile(self.FileName + ".hdf5"): + self.FileFormat = 'GTSDF' + self.ReadGtsdf() + else: + raise Exception("unknown file: " + FileName) +################################################################################ +# Read results in binary format + def ReadBinary(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec: + ChVec = range(0, self.NrCh) + with open(self.FileName + '.dat', 'rb') as fid: + data = np.zeros((self.NrSc, len(ChVec))) + j = 0 + for i in ChVec: + fid.seek(i * self.NrSc * 2, 0) + data[:, j] = np.fromfile(fid, 'int16', self.NrSc) * self.ScaleFactor[i] + j += 1 + return data +################################################################################ +# Read results in ASCII format + def ReadAscii(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec: + ChVec = range(0, self.NrCh) + temp = np.loadtxt(self.FileName + '.dat', usecols=ChVec) + return temp.reshape((self.NrSc, len(ChVec))) +################################################################################ +# Read results in FLEX format + def ReadFLEX(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec: + ChVec = range(1, self.NrCh) + fid = open(self.FileName, 'rb') + fid.seek(2 * 4 * self.NrCh + 48 * 2) + temp = np.fromfile(fid, 'int16') + if self.Version==3: + temp = temp.reshape(self.NrCh, self.NrSc).transpose() + else: + temp = temp.reshape(self.NrSc, self.NrCh) + fid.close() + return np.dot(temp[:, ChVec], np.diag(self.ScaleFactor[ChVec])) +################################################################################ +# Read results in GTSD format + def ReadGtsdf(self): + raise NotImplementedError + #self.t, data, info = gtsdf.load(self.FileName + '.hdf5') + #self.Time = self.t + #self.ChInfo = [['Time'] + info['attribute_names'], + # ['s'] + info['attribute_units'], + # ['Time'] + info['attribute_descriptions']] + #self.NrCh = data.shape[1] + 1 + #self.NrSc = data.shape[0] + #self.Freq = self.NrSc / self.Time + #self.FileFormat = 'GTSDF' + #self.gtsdf_description = info['description'] + #data = np.hstack([self.Time[:,np.newaxis], data]) + #return data +################################################################################ +# One stop call for reading all data formats + def ReadAll(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec and not self.FileFormat == 'GTSDF': + ChVec = range(0, self.NrCh) + if self.FileFormat == 'HAWC2_BINARY': + return self.ReadBinary(ChVec) + elif self.FileFormat == 'HAWC2_ASCII' or self.FileFormat == 'BHAWC_ASCII': + return self.ReadAscii(ChVec) + elif self.FileFormat == 'GTSDF': + return self.ReadGtsdf() + elif self.FileFormat == 'FLEX': + return self.ReadFLEX(ChVec) + else: + raise Exception('Unknown file format {} for hawc2 out file'.format(self.FileFormat)) + +################################################################################ +# Main read data call, read, save and sort data + def __call__(self, ChVec=None): + ChVec = [] if ChVec is None else ChVec + if not ChVec: + ChVec = range(0, self.NrCh) + elif max(ChVec) >= self.NrCh: + print("to high channel number") + return + # if ReadOnly, read data but no storeing in memory + if self.ReadOnly: + return self.ReadAll(ChVec) + # if not ReadOnly, sort in known and new channels, read new channels + # and return all requested channels + else: + # sort into known channels and channels to be read + I1 = [] + I2 = [] # I1=Channel mapping, I2=Channels to be read + for i in ChVec: + try: + I1.append(self.Iknown.index(i)) + except: + self.Iknown.append(i) + I2.append(i) + I1.append(len(I1)) + # read new channels + if I2: + temp = self.ReadAll(I2) + # add new channels to Data + if self.Data.any(): + self.Data = np.append(self.Data, temp, axis=1) + # if first call, so Daata is empty + else: + self.Data = temp + return self.Data[:, tuple(I1)] + + +################################################################################ +################################################################################ +################################################################################ +# write HAWC2 class, to be implemented +################################################################################ + +if __name__ == '__main__': + res_file = ReadHawc2('structure_wind') + results = res_file.ReadAscii() + channelinfo = res_file.ChInfo diff --git a/pydatview/io/wetb/hawc2/ae_file.py b/pydatview/io/wetb/hawc2/ae_file.py index 36c7364..2521d47 100644 --- a/pydatview/io/wetb/hawc2/ae_file.py +++ b/pydatview/io/wetb/hawc2/ae_file.py @@ -1,14 +1,4 @@ -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import -from io import open -from builtins import range -from builtins import int -from future import standard_library import os -standard_library.install_aliases() - import numpy as np @@ -77,9 +67,16 @@ def pc_set_nr(self, radius, set_nr=1): ae_data = self.ae_sets[set_nr] index = np.searchsorted(ae_data[:, 0], radius) index = max(1, index) + # --- Emmanuel's addition + maxRad = np.max(ae_data[:,0]) + index2 = np.argmin(np.abs(ae_data[:, 0]-radius)) + if abs(ae_data[index2,0]-radius)<1e-4*maxRad: + # We are very close to an ae location, we use this set + return ae_data[index2, 3] + # Otherwise we look at index before or after setnrs = ae_data[index - 1:index + 1, 3] if setnrs[0] != setnrs[-1]: - raise NotImplementedError + print('[WARN] AE file, at radius {}, should return a set between {}. Using first one.'.format(radius,setnrs)) return setnrs[0] def add_set(self, radius, chord, thickness, pc_set_id, set_id=None): @@ -103,7 +100,9 @@ def __str__(self): def save(self, filename): if not os.path.isdir(os.path.dirname(filename)): - os.makedirs(os.path.dirname(filename)) + # fails if dirname is empty string + if len(os.path.dirname(filename)) > 0: + os.makedirs(os.path.dirname(filename)) with open(filename, 'w') as fid: fid.write(str(self)) @@ -122,11 +121,18 @@ def _read_file(self, filename): lptr += n_rows +def main(): + if __name__ == "__main__": + ae = AEFile(os.path.dirname(__file__) + "/tests/test_files/NREL_5MW_ae.txt") + print(ae.radius_ae(36)) + print(ae.thickness()) + print(ae.chord(36)) + print(ae.pc_set_nr(36)) + ae.add_set(radius=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], + chord=[1.1, 1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1], + thickness=[100.0, 100.0, 90.0, 80.0, 70.0, 60.0, 50.0, 40.0, 30.0, 20.0, 10.0], + pc_set_id=[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]) + print(str(ae)) -if __name__ == "__main__": - ae = AEFile(r"tests/test_files/NREL_5MW_ae.txt") - print (ae.radius_ae(36)) - print (ae.thickness()) - print (ae.chord(36)) - print (ae.pc_set_nr(36)) +main() diff --git a/pydatview/io/wetb/hawc2/htc_contents.py b/pydatview/io/wetb/hawc2/htc_contents.py index 7805eb3..22aa3e4 100644 --- a/pydatview/io/wetb/hawc2/htc_contents.py +++ b/pydatview/io/wetb/hawc2/htc_contents.py @@ -6,16 +6,7 @@ See documentation of HTCFile below ''' -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from builtins import zip -from builtins import int -from builtins import str -from future import standard_library import os -standard_library.install_aliases() from collections import OrderedDict import collections @@ -45,9 +36,6 @@ def fmt_value(v): return v.replace("\\", "/") -c = 0 - - class HTCContents(object): lines = [] contents = None @@ -95,7 +83,6 @@ def __setattr__(self, *args, **kwargs): if k in self.contents: self.contents[k].values = v return - v = HTCLine(k, v, "") self.contents[k] = v v.parent = self @@ -208,10 +195,11 @@ class HTCSection(HTCContents): begin_comments = "" def __init__(self, name, begin_comments="", end_comments=""): - self.name_ = name + self.name_ = name.strip() # strip if tabs in name somehow self.begin_comments = begin_comments.strip(" \t") self.end_comments = end_comments.strip(" \t") self.contents = OrderedDict() + self.parent = None @property def section_name(self): @@ -246,12 +234,14 @@ def from_lines(lines): else: section = HTCSection(name, begin_comments) while lines: + if lines[0].strip() == "": + lines.pop(0) if lines[0].lower().startswith("begin"): section._add_contents(HTCSection.from_lines(lines)) elif lines[0].lower().startswith("end"): line, section.end_comments = parse_next_line(lines) break - else: + elif lines: section._add_contents(section.line_from_line(lines)) else: raise Exception("Section '%s' has not end" % section.name_) @@ -312,6 +302,7 @@ def __init__(self, name, values, comments): self.name_ = name self.values = list(values) self.comments = comments.strip(" \t") + self.parent = None def __repr__(self): return str(self) @@ -381,9 +372,8 @@ def _add_sensor(self, htcSensor, nr=None): htcSensor.parent = self def line_from_line(self, lines): - while len(lines) and lines[0].strip() == "": - lines.pop(0) name = lines[0].split()[0].strip() + if name in ['filename', 'data_format', 'buffer', 'time']: return HTCLine.from_lines(lines) else: diff --git a/pydatview/io/wetb/hawc2/htc_extensions.py b/pydatview/io/wetb/hawc2/htc_extensions.py index 4e109c6..7275bba 100644 --- a/pydatview/io/wetb/hawc2/htc_extensions.py +++ b/pydatview/io/wetb/hawc2/htc_extensions.py @@ -6,18 +6,9 @@ See documentation of HTCFile below ''' -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from builtins import zip -from builtins import int -from builtins import str -from future import standard_library import os -standard_library.install_aliases() class HTCDefaults(object): diff --git a/pydatview/io/wetb/hawc2/htc_file.py b/pydatview/io/wetb/hawc2/htc_file.py index 5d1a6d7..169e015 100644 --- a/pydatview/io/wetb/hawc2/htc_file.py +++ b/pydatview/io/wetb/hawc2/htc_file.py @@ -1,585 +1,592 @@ -''' -Created on 20/01/2014 - -@author: MMPE - -See documentation of HTCFile below - -''' -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import -from builtins import str -from future import standard_library -# from wetb.utils.process_exec import pexec -# from wetb.hawc2.hawc2_pbs_file import HAWC2PBSFile -# from wetb.utils.cluster_tools.os_path import fixcase, abspath, pjoin -# import jinja2 -standard_library.install_aliases() -from collections import OrderedDict -from .htc_contents import HTCContents, HTCSection, HTCLine -from .htc_extensions import HTCDefaults, HTCExtensions -import os - -# --- cluster_tools/os_path -def repl(path): - return path.replace("\\", "/") - -def abspath(path): - return repl(os.path.abspath(path)) - -def relpath(path, start=None): - return repl(os.path.relpath(path, start)) - -def realpath(path): - return repl(os.path.realpath(path)) - -def pjoin(*path): - return repl(os.path.join(*path)) - -def fixcase(path): - path = realpath(str(path)).replace("\\", "/") - p, rest = os.path.splitdrive(path) - p += "/" - for f in rest[1:].split("/"): - f_lst = [f_ for f_ in os.listdir(p) if f_.lower() == f.lower()] - if len(f_lst) > 1: - # use the case sensitive match - f_lst = [f_ for f_ in f_lst if f_ == f] - if len(f_lst) == 0: - raise IOError("'%s' not found in '%s'" % (f, p)) - # Use matched folder - p = pjoin(p, f_lst[0]) - return p -# --- end os_path - -class HTCFile(HTCContents, HTCDefaults, HTCExtensions): - """Wrapper for HTC files - - Examples: - --------- - >>> htcfile = HTCFile('htc/test.htc') - >>> htcfile.wind.wsp = 10 - >>> htcfile.save() - - #--------------------------------------------- - >>> htc = HTCFile(filename=None, modelpath=None) # create minimal htcfile - - #Add section - >>> htc.add_section('hydro') - - #Add subsection - >>> htc.hydro.add_section("hydro_element") - - #Set values - >>> htc.hydro.hydro_element.wave_breaking = [2, 6.28, 1] # or - >>> htc.hydro.hydro_element.wave_breaking = 2, 6.28, 1 - - #Set comments - >>> htc.hydro.hydro_element.wave_breaking.comments = "This is a comment" - - #Access section - >>> hydro_element = htc.hydro.hydro_element #or - >>> hydro_element = htc['hydro.hydro_element'] # or - >>> hydro_element = htc['hydro/hydro_element'] # or - >>> print (hydro_element.wave_breaking) #string represenation - wave_breaking 2 6.28 1; This is a comment - >>> print (hydro_element.wave_breaking.name_) # command - wave_breaking - >>> print (hydro_element.wave_breaking.values) # values - [2, 6.28, 1 - >>> print (hydro_element.wave_breaking.comments) # comments - This is a comment - >>> print (hydro_element.wave_breaking[0]) # first value - 2 - - #Delete element - htc.simulation.logfile.delete() - #or - del htc.simulation.logfile #Delete logfile line. Raise keyerror if not exists - - """ - - filename = None - jinja_tags = {} - htc_inputfiles = [] - level = 0 - modelpath = "../" - initial_comments = None - _contents = None - - def __init__(self, filename=None, modelpath=None, jinja_tags={}): - """ - Parameters - --------- - filename : str - Absolute filename of htc file - modelpath : str - Model path relative to htc file - """ - - if filename is not None: - try: - filename = fixcase(abspath(filename)) - with self.open(str(filename)): - pass - except Exception: - pass - - self.filename = filename - - self.jinja_tags = jinja_tags - self.modelpath = modelpath or self.auto_detect_modelpath() - - if filename and self.modelpath != "unknown" and not os.path.isabs(self.modelpath): - drive, p = os.path.splitdrive(os.path.join(os.path.dirname(str(self.filename)), self.modelpath)) - self.modelpath = os.path.join(drive, os.path.splitdrive(os.path.realpath(p))[1]).replace("\\", "/") - if self.modelpath != 'unknown' and self.modelpath[-1] != '/': - self.modelpath += "/" - - #assert 'simulation' in self.contents, "%s could not be loaded. 'simulation' section missing" % filename - - def auto_detect_modelpath(self): - if self.filename is None: - return "../" - - #print (["../"*i for i in range(3)]) - import numpy as np - input_files = HTCFile(self.filename, 'unknown').input_files() - if len(input_files) == 1: # only input file is the htc file - return "../" - rel_input_files = [f for f in input_files if not os.path.isabs(f)] - - def isfile_case_insensitive(f): - try: - f = fixcase(f) # raises exception if not existing - return os.path.isfile(f) - except IOError: - return False - found = ([np.sum([isfile_case_insensitive(os.path.join(os.path.dirname(self.filename), "../" * i, f)) - for f in rel_input_files]) for i in range(4)]) - - if max(found) > 0: - relpath = "../" * np.argmax(found) - return abspath(pjoin(os.path.dirname(self.filename), relpath)) - else: - raise ValueError( - "Modelpath cannot be autodetected for '%s'.\nInput files not found near htc file" % self.filename) - - def _load(self): - self.reset() - self.initial_comments = [] - self.htc_inputfiles = [] - self.contents = OrderedDict() - if self.filename is None: - lines = self.empty_htc.split("\n") - else: - lines = self.readlines(self.filename) - - lines = [l.strip() for l in lines] - - #lines = copy(self.lines) - while lines: - if lines[0].startswith(";"): - self.initial_comments.append(lines.pop(0).strip() + "\n") - elif lines[0].lower().startswith("begin"): - self._add_contents(HTCSection.from_lines(lines)) - else: - line = HTCLine.from_lines(lines) - if line.name_ == "exit": - break - self._add_contents(line) - - def reset(self): - self._contents = None - - @property - def contents(self): - if self._contents is None: - self._load() - return self._contents - - @contents.setter - def contents(self, value): - self._contents = value - - def readfilelines(self, filename): - with self.open(self.unix_path(filename), encoding='cp1252') as fid: - txt = fid.read() - if txt[:10].encode().startswith(b'\xc3\xaf\xc2\xbb\xc2\xbf'): - txt = txt[3:] - if self.jinja_tags: - template = jinja2.Template(txt) - txt = template.render(**self.jinja_tags) - return txt.replace("\r", "").split("\n") - - def readlines(self, filename): - if filename != self.filename: # self.filename may be changed by set_name/save. Added it when needed instead - self.htc_inputfiles.append(filename) - htc_lines = [] - lines = self.readfilelines(filename) - for l in lines: - if l.lower().lstrip().startswith('continue_in_file'): - filename = l.lstrip().split(";")[0][len("continue_in_file"):].strip().lower() - - if self.modelpath == 'unknown': - p = os.path.dirname(self.filename) - lu = [os.path.isfile(os.path.join(p, "../" * i, filename)) for i in range(4)].index(True) - filename = os.path.join(p, "../" * lu, filename) - else: - filename = os.path.join(self.modelpath, filename) - for line in self.readlines(filename): - if line.lstrip().lower().startswith('exit'): - break - htc_lines.append(line) - else: - htc_lines.append(l) - return htc_lines - - def __setitem__(self, key, value): - self.contents[key] = value - - def __str__(self): - self.contents # load - return "".join(self.initial_comments + [c.__str__(1) for c in self] + ["exit;"]) - - def save(self, filename=None): - self.contents # load if not loaded - if filename is None: - filename = self.filename - else: - self.filename = filename - # exist_ok does not exist in Python27 - if not os.path.exists(os.path.dirname(filename)) and os.path.dirname(filename) != "": - os.makedirs(os.path.dirname(filename)) # , exist_ok=True) - with self.open(filename, 'w', encoding='cp1252') as fid: - fid.write(str(self)) - - def set_name(self, name, subfolder=''): - # if os.path.isabs(folder) is False and os.path.relpath(folder).startswith("htc" + os.path.sep): - self.contents # load if not loaded - - def fmt_folder(folder, subfolder): return "./" + \ - os.path.relpath(os.path.join(folder, subfolder)).replace("\\", "/") - - self.filename = os.path.abspath(os.path.join(self.modelpath, fmt_folder( - 'htc', subfolder), "%s.htc" % name)).replace("\\", "/") - if 'simulation' in self and 'logfile' in self.simulation: - self.simulation.logfile = os.path.join(fmt_folder('log', subfolder), "%s.log" % name).replace("\\", "/") - if 'animation' in self.simulation: - self.simulation.animation = os.path.join(fmt_folder( - 'animation', subfolder), "%s.dat" % name).replace("\\", "/") - if 'visualization' in self.simulation: - self.simulation.visualization = os.path.join(fmt_folder( - 'visualization', subfolder), "%s.hdf5" % name).replace("\\", "/") - elif 'test_structure' in self and 'logfile' in self.test_structure: # hawc2aero - self.test_structure.logfile = os.path.join(fmt_folder('log', subfolder), "%s.log" % name).replace("\\", "/") - self.output.filename = os.path.join(fmt_folder('res', subfolder), "%s" % name).replace("\\", "/") - - def set_time(self, start=None, stop=None, step=None): - self.contents # load if not loaded - if stop is not None: - self.simulation.time_stop = stop - else: - stop = self.simulation.time_stop[0] - if step is not None: - self.simulation.newmark.deltat = step - if start is not None: - self.output.time = start, stop - if "wind" in self: # and self.wind.turb_format[0] > 0: - self.wind.scale_time_start = start - - def expected_simulation_time(self): - return 600 - - def pbs_file(self, hawc2_path, hawc2_cmd, queue='workq', walltime=None, - input_files=None, output_files=None, copy_turb=(True, True)): - walltime = walltime or self.expected_simulation_time() * 2 - if len(copy_turb) == 1: - copy_turb_fwd, copy_turb_back = copy_turb, copy_turb - else: - copy_turb_fwd, copy_turb_back = copy_turb - - input_files = input_files or self.input_files() - if copy_turb_fwd: - input_files += [f for f in self.turbulence_files() if os.path.isfile(f)] - - output_files = output_files or self.output_files() - if copy_turb_back: - output_files += self.turbulence_files() - - return HAWC2PBSFile(hawc2_path, hawc2_cmd, self.filename, self.modelpath, - input_files, output_files, - queue, walltime) - - def input_files(self): - self.contents # load if not loaded - if self.modelpath == "unknown": - files = [str(f).replace("\\", "/") for f in [self.filename] + self.htc_inputfiles] - else: - files = [os.path.abspath(str(f)).replace("\\", "/") for f in [self.filename] + self.htc_inputfiles] - if 'new_htc_structure' in self: - for mb in [self.new_htc_structure[mb] - for mb in self.new_htc_structure.keys() if mb.startswith('main_body')]: - if "timoschenko_input" in mb: - files.append(mb.timoschenko_input.filename[0]) - files.append(mb.get('external_bladedata_dll', [None, None, None])[2]) - if 'aero' in self: - files.append(self.aero.ae_filename[0]) - files.append(self.aero.pc_filename[0]) - files.append(self.aero.get('external_bladedata_dll', [None, None, None])[2]) - files.append(self.aero.get('output_profile_coef_filename', [None])[0]) - if 'dynstall_ateflap' in self.aero: - files.append(self.aero.dynstall_ateflap.get('flap', [None] * 3)[2]) - if 'bemwake_method' in self.aero: - files.append(self.aero.bemwake_method.get('a-ct-filename', [None] * 3)[0]) - for dll in [self.dll[dll] for dll in self.get('dll', {}).keys() if 'filename' in self.dll[dll]]: - files.append(dll.filename[0]) - f, ext = os.path.splitext(dll.filename[0]) - files.append(f + "_64" + ext) - if 'wind' in self: - files.append(self.wind.get('user_defined_shear', [None])[0]) - files.append(self.wind.get('user_defined_shear_turbulence', [None])[0]) - files.append(self.wind.get('met_mast_wind', [None])[0]) - if 'wakes' in self: - files.append(self.wind.get('use_specific_deficit_file', [None])[0]) - files.append(self.wind.get('write_ct_cq_file', [None])[0]) - files.append(self.wind.get('write_final_deficits', [None])[0]) - if 'hydro' in self: - if 'water_properties' in self.hydro: - files.append(self.hydro.water_properties.get('water_kinematics_dll', [None])[0]) - files.append(self.hydro.water_properties.get('water_kinematics_dll', [None, None])[1]) - if 'soil' in self: - if 'soil_element' in self.soil: - files.append(self.soil.soil_element.get('datafile', [None])[0]) - try: - dtu_we_controller = self.dll.get_subsection_by_name('dtu_we_controller') - theta_min = dtu_we_controller.init.constant__5[1] - if theta_min >= 90: - files.append(os.path.join(os.path.dirname( - dtu_we_controller.filename[0]), "wpdata.%d" % theta_min).replace("\\", "/")) - except Exception: - pass - - try: - files.append(self.force.dll.dll[0]) - except Exception: - pass - - def fix_path_case(f): - if os.path.isabs(f): - return self.unix_path(f) - elif self.modelpath != "unknown": - try: - return "./" + os.path.relpath(self.unix_path(os.path.join(self.modelpath, f)), - self.modelpath).replace("\\", "/") - except IOError: - return f - else: - return f - return [fix_path_case(f) for f in set(files) if f] - - def output_files(self): - self.contents # load if not loaded - files = [] - for k, index in [('simulation/logfile', 0), - ('simulation/animation', 0), - ('simulation/visualization', 0), - ('new_htc_structure/beam_output_file_name', 0), - ('new_htc_structure/body_output_file_name', 0), - ('new_htc_structure/struct_inertia_output_file_name', 0), - ('new_htc_structure/body_eigenanalysis_file_name', 0), - ('new_htc_structure/constraint_output_file_name', 0), - ('wind/turb_export/filename_u', 0), - ('wind/turb_export/filename_v', 0), - ('wind/turb_export/filename_w', 0)]: - line = self.get(k) - if line: - files.append(line[index]) - if 'new_htc_structure' in self: - if 'system_eigenanalysis' in self.new_htc_structure: - f = self.new_htc_structure.system_eigenanalysis[0] - files.append(f) - files.append(os.path.join(os.path.dirname(f), 'mode*.dat').replace("\\", "/")) - if 'structure_eigenanalysis_file_name' in self.new_htc_structure: - f = self.new_htc_structure.structure_eigenanalysis_file_name[0] - files.append(f) - files.append(os.path.join(os.path.dirname(f), 'mode*.dat').replace("\\", "/")) - files.extend(self.res_file_lst()) - - for key in [k for k in self.contents.keys() if k.startswith("output_at_time")]: - files.append(self[key]['filename'][0] + ".dat") - return [f.lower() for f in files if f] - - def turbulence_files(self): - self.contents # load if not loaded - if 'wind' not in self.contents.keys() or self.wind.turb_format[0] == 0: - return [] - elif self.wind.turb_format[0] == 1: - files = [self.get('wind.mann.filename_%s' % comp, [None])[0] for comp in ['u', 'v', 'w']] - elif self.wind.turb_format[0] == 2: - files = [self.get('wind.flex.filename_%s' % comp, [None])[0] for comp in ['u', 'v', 'w']] - return [f for f in files if f] - - def res_file_lst(self): - self.contents # load if not loaded - res = [] - for output in [self[k] for k in self.keys() - if self[k].name_.startswith("output") and not self[k].name_.startswith("output_at_time")]: - dataformat = output.get('data_format', 'hawc_ascii') - res_filename = output.filename[0] - if dataformat[0] == "gtsdf" or dataformat[0] == "gtsdf64": - res.append(res_filename + ".hdf5") - elif dataformat[0] == "flex_int": - res.extend([res_filename + ".int", os.path.join(os.path.dirname(res_filename), 'sensor')]) - else: - res.extend([res_filename + ".sel", res_filename + ".dat"]) - return res - - def _simulate(self, exe, skip_if_up_to_date=False): - self.contents # load if not loaded - if skip_if_up_to_date: - from os.path import isfile, getmtime, isabs - res_file = os.path.join(self.modelpath, self.res_file_lst()[0]) - htc_file = os.path.join(self.modelpath, self.filename) - if isabs(exe): - exe_file = exe - else: - exe_file = os.path.join(self.modelpath, exe) - #print (from_unix(getmtime(res_file)), from_unix(getmtime(htc_file))) - if (isfile(htc_file) and isfile(res_file) and isfile(exe_file) and - str(HTCFile(htc_file)) == str(self) and - getmtime(res_file) > getmtime(htc_file) and getmtime(res_file) > getmtime(exe_file)): - if "".join(self.readfilelines(htc_file)) == str(self): - return - - self.save() - htcfile = os.path.relpath(self.filename, self.modelpath) - assert any([os.path.isfile(os.path.join(f, exe)) for f in [''] + os.environ['PATH'].split(";")]), exe - return pexec([exe, htcfile], self.modelpath) - - def simulate(self, exe, skip_if_up_to_date=False): - errorcode, stdout, stderr, cmd = self._simulate(exe, skip_if_up_to_date) - if ('simulation' in self.keys() and "logfile" in self.simulation and - os.path.isfile(os.path.join(self.modelpath, self.simulation.logfile[0]))): - with self.open(os.path.join(self.modelpath, self.simulation.logfile[0])) as fid: - log = fid.read() - else: - log = stderr - - if errorcode or 'Elapsed time' not in log: - log_lines = log.split("\n") - error_lines = [i for i, l in enumerate(log_lines) if 'error' in l.lower()] - if error_lines: - i = error_lines[0] - error_log = "\n".join(log_lines[i - 3:i + 3]) - else: - error_log = log - raise Exception("\nstdout:\n%s\n--------------\nstderr:\n%s\n--------------\nlog:\n%s\n--------------\ncmd:\n%s" % - (str(stdout), str(stderr), error_log, cmd)) - return str(stdout) + str(stderr), log - - def simulate_hawc2stab2(self, exe): - errorcode, stdout, stderr, cmd = self._simulate(exe, skip_if_up_to_date=False) - - if errorcode: - raise Exception("\nstdout:\n%s\n--------------\nstderr:\n%s\n--------------\ncmd:\n%s" % - (str(stdout), str(stderr), cmd)) - return str(stdout) + str(stderr) - - def deltat(self): - return self.simulation.newmark.deltat[0] - - def compare(self, other): - if isinstance(other, str): - other = HTCFile(other) - return HTCContents.compare(self, other) - - @property - def open(self): - return open - - def unix_path(self, filename): - filename = os.path.realpath(str(filename)).replace("\\", "/") - ufn, rest = os.path.splitdrive(filename) - ufn += "/" - for f in rest[1:].split("/"): - f_lst = [f_ for f_ in os.listdir(ufn) if f_.lower() == f.lower()] - if len(f_lst) > 1: - # use the case sensitive match - f_lst = [f_ for f_ in f_lst if f_ == f] - if len(f_lst) == 0: - raise IOError("'%s' not found in '%s'" % (f, ufn)) - else: # one match found - ufn = os.path.join(ufn, f_lst[0]) - return ufn.replace("\\", "/") - - -# -# def get_body(self, name): -# lst = [b for b in self.new_htc_structure if b.name_=="main_body" and b.name[0]==name] -# if len(lst)==1: -# return lst[0] -# else: -# if len(lst)==0: -# raise ValueError("Body '%s' not found"%name) -# else: -# raise NotImplementedError() -# - -class H2aeroHTCFile(HTCFile): - def __init__(self, filename=None, modelpath=None): - HTCFile.__init__(self, filename=filename, modelpath=modelpath) - - @property - def simulation(self): - return self.test_structure - - def set_time(self, start=None, stop=None, step=None): - if stop is not None: - self.test_structure.time_stop = stop - else: - stop = self.simulation.time_stop[0] - if step is not None: - self.test_structure.deltat = step - if start is not None: - self.output.time = start, stop - if "wind" in self and self.wind.turb_format[0] > 0: - self.wind.scale_time_start = start - - -class SSH_HTCFile(HTCFile): - def __init__(self, ssh, filename=None, modelpath=None): - object.__setattr__(self, 'ssh', ssh) - HTCFile.__init__(self, filename=filename, modelpath=modelpath) - - @property - def open(self): - return self.ssh.open - - def unix_path(self, filename): - rel_filename = os.path.relpath(filename, self.modelpath).replace("\\", "/") - _, out, _ = self.ssh.execute("find -ipath ./%s" % rel_filename, cwd=self.modelpath) - out = out.strip() - if out == "": - raise IOError("'%s' not found in '%s'" % (rel_filename, self.modelpath)) - elif "\n" in out: - raise IOError("Multiple '%s' found in '%s' (due to case senitivity)" % (rel_filename, self.modelpath)) - else: - drive, path = os.path.splitdrive(os.path.join(self.modelpath, out)) - path = os.path.realpath(path).replace("\\", "/") - return os.path.join(drive, os.path.splitdrive(path)[1]) - - -if "__main__" == __name__: - f = HTCFile(r"C:/Work/BAR-Local/Hawc2ToBeamDyn/sim.htc", ".") - print(f.input_files()) - import pdb; pdb.set_trace() -# f.save(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT_power_curve.htc") -# -# f = HTCFile(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT.htc", "../") -# f.set_time = 0, 1, .1 -# print(f.simulate(r"C:\mmpe\HAWC2\bin\HAWC2_12.8\hawc2mb.exe")) -# -# f.save(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT.htc") +''' +Created on 20/01/2014 + +See documentation of HTCFile below + +''' +# from wetb.utils.process_exec import pexec +# from wetb.hawc2.hawc2_pbs_file import HAWC2PBSFile +# import jinja2 +# from wetb.utils.cluster_tools.os_path import fixcase, abspath, pjoin + +from collections import OrderedDict +from .htc_contents import HTCContents, HTCSection, HTCLine +from .htc_extensions import HTCDefaults, HTCExtensions +import os + +# --- cluster_tools/os_path +def fmt_path(path): + return path.lower().replace("\\", "/") + +def repl(path): + return path.replace("\\", "/") + +def abspath(path): + return repl(os.path.abspath(path)) + +def relpath(path, start=None): + return repl(os.path.relpath(path, start)) + +def realpath(path): + return repl(os.path.realpath(path)) + +def pjoin(*path): + return repl(os.path.join(*path)) + +def fixcase(path): + path = realpath(str(path)).replace("\\", "/") + p, rest = os.path.splitdrive(path) + p += "/" + for f in rest[1:].split("/"): + f_lst = [f_ for f_ in os.listdir(p) if f_.lower() == f.lower()] + if len(f_lst) > 1: + # use the case sensitive match + f_lst = [f_ for f_ in f_lst if f_ == f] + if len(f_lst) == 0: + raise IOError("'%s' not found in '%s'" % (f, p)) + # Use matched folder + p = pjoin(p, f_lst[0]) + return p +# --- end os_path + +class HTCFile(HTCContents, HTCDefaults, HTCExtensions): + """Wrapper for HTC files + + Examples: + --------- + >>> htcfile = HTCFile('htc/test.htc') + >>> htcfile.wind.wsp = 10 + >>> htcfile.save() + + #--------------------------------------------- + >>> htc = HTCFile(filename=None, modelpath=None) # create minimal htcfile + + #Add section + >>> htc.add_section('hydro') + + #Add subsection + >>> htc.hydro.add_section("hydro_element") + + #Set values + >>> htc.hydro.hydro_element.wave_breaking = [2, 6.28, 1] # or + >>> htc.hydro.hydro_element.wave_breaking = 2, 6.28, 1 + + #Set comments + >>> htc.hydro.hydro_element.wave_breaking.comments = "This is a comment" + + #Access section + >>> hydro_element = htc.hydro.hydro_element #or + >>> hydro_element = htc['hydro.hydro_element'] # or + >>> hydro_element = htc['hydro/hydro_element'] # or + >>> print (hydro_element.wave_breaking) #string represenation + wave_breaking 2 6.28 1; This is a comment + >>> print (hydro_element.wave_breaking.name_) # command + wave_breaking + >>> print (hydro_element.wave_breaking.values) # values + [2, 6.28, 1 + >>> print (hydro_element.wave_breaking.comments) # comments + This is a comment + >>> print (hydro_element.wave_breaking[0]) # first value + 2 + + #Delete element + htc.simulation.logfile.delete() + #or + del htc.simulation.logfile #Delete logfile line. Raise keyerror if not exists + + """ + + filename = None + jinja_tags = {} + htc_inputfiles = [] + level = 0 + modelpath = "../" + initial_comments = None + + def __init__(self, filename=None, modelpath=None, jinja_tags={}): + """ + Parameters + --------- + filename : str + Absolute filename of htc file + modelpath : str + Model path relative to htc file + """ + if filename is not None: + try: + filename = fixcase(abspath(filename)) + with self.open(str(filename)): + pass + except Exception: + pass + + self.filename = filename + + self.jinja_tags = jinja_tags + self.modelpath = modelpath or self.auto_detect_modelpath() + + if filename and self.modelpath != "unknown" and not os.path.isabs(self.modelpath): + drive, p = os.path.splitdrive(os.path.join(os.path.dirname(str(self.filename)), self.modelpath)) + self.modelpath = os.path.join(drive, os.path.splitdrive(os.path.realpath(p))[1]).replace("\\", "/") + if self.modelpath != 'unknown' and self.modelpath[-1] != '/': + self.modelpath += "/" + + self.load() + + def auto_detect_modelpath(self): + if self.filename is None: + return "../" + + #print (["../"*i for i in range(3)]) + import numpy as np + input_files = HTCFile(self.filename, 'unknown').input_files() + if len(input_files) == 1: # only input file is the htc file + return "../" + rel_input_files = [f for f in input_files if not os.path.isabs(f)] + + def isfile_case_insensitive(f): + try: + f = fixcase(f) # raises exception if not existing + return os.path.isfile(f) + except IOError: + return False + found = ([np.sum([isfile_case_insensitive(os.path.join(os.path.dirname(self.filename), "../" * i, f)) + for f in rel_input_files]) for i in range(4)]) + + if max(found) > 0: + relpath = "../" * np.argmax(found) + return abspath(pjoin(os.path.dirname(self.filename), relpath)) + else: + raise ValueError( + "Modelpath cannot be autodetected for '%s'.\nInput files not found near htc file" % self.filename) + + def load(self): + self.contents = OrderedDict() + self.initial_comments = [] + self.htc_inputfiles = [] + if self.filename is None: + lines = self.empty_htc.split("\n") + else: + lines = self.readlines(self.filename) + + lines = [l.strip() for l in lines] + + #lines = copy(self.lines) + while lines: + if lines[0].startswith(";"): + self.initial_comments.append(lines.pop(0).strip() + "\n") + elif lines[0].lower().startswith("begin"): + self._add_contents(HTCSection.from_lines(lines)) + else: + line = HTCLine.from_lines(lines) + if line.name_ == "exit": + break + self._add_contents(line) + + def readfilelines(self, filename): + with self.open(self.unix_path(os.path.abspath(filename.replace('\\', '/'))), encoding='cp1252') as fid: + txt = fid.read() + if txt[:10].encode().startswith(b'\xc3\xaf\xc2\xbb\xc2\xbf'): + txt = txt[3:] + if self.jinja_tags: + template = jinja2.Template(txt) + txt = template.render(**self.jinja_tags) + return txt.replace("\r", "").split("\n") + + def readlines(self, filename): + if filename != self.filename: # self.filename may be changed by set_name/save. Added it when needed instead + self.htc_inputfiles.append(filename) + htc_lines = [] + lines = self.readfilelines(filename) + for l in lines: + if l.lower().lstrip().startswith('continue_in_file'): + filename = l.lstrip().split(";")[0][len("continue_in_file"):].strip().lower() + + if self.modelpath == 'unknown': + p = os.path.dirname(self.filename) + lu = [os.path.isfile(os.path.abspath(os.path.join(p, "../" * i, filename.replace("\\", "/")))) + for i in range(4)].index(True) + filename = os.path.join(p, "../" * lu, filename) + else: + filename = os.path.join(self.modelpath, filename) + for line in self.readlines(filename): + if line.lstrip().lower().startswith('exit'): + break + htc_lines.append(line) + else: + htc_lines.append(l) + return htc_lines + + def __setitem__(self, key, value): + self.contents[key] = value + + def __str__(self): + self.contents # load + return "".join(self.initial_comments + [c.__str__(1) for c in self] + ["exit;"]) + + def save(self, filename=None): + """Saves the htc object to an htc file. + + Args: + filename (str, optional): Specifies the filename of the htc file to be saved. + If the value is none, the filename attribute of the object will be used as the filename. + Defaults to None. + """ + self.contents # load if not loaded + if filename is None: + filename = self.filename + else: + self.filename = filename + # exist_ok does not exist in Python27 + if not os.path.exists(os.path.dirname(filename)) and os.path.dirname(filename) != "": + os.makedirs(os.path.dirname(filename)) # , exist_ok=True) + with self.open(filename, 'w', encoding='cp1252') as fid: + fid.write(str(self)) + + def set_name(self, name, subfolder=''): + """Sets the base filename of the simulation files. + + Args: + name (str): Specifies name of the log file, dat file (for animation), hdf5 file (for visualization) and htc file. + subfolder (str, optional): Specifies the name of a subfolder to place the files in. + If the value is an empty string, no subfolders will be created. + Defaults to ''. + + Returns: + None + """ + # if os.path.isabs(folder) is False and os.path.relpath(folder).startswith("htc" + os.path.sep): + self.contents # load if not loaded + + def fmt_folder(folder, subfolder): return "./" + \ + os.path.relpath(os.path.join(folder, subfolder)).replace("\\", "/") + + self.filename = os.path.abspath(os.path.join(self.modelpath, fmt_folder( + 'htc', subfolder), "%s.htc" % name)).replace("\\", "/") + if 'simulation' in self and 'logfile' in self.simulation: + self.simulation.logfile = os.path.join(fmt_folder('log', subfolder), "%s.log" % name).replace("\\", "/") + if 'animation' in self.simulation: + self.simulation.animation = os.path.join(fmt_folder( + 'animation', subfolder), "%s.dat" % name).replace("\\", "/") + if 'visualization' in self.simulation: + f = os.path.join(fmt_folder('visualization', subfolder), "%s.hdf5" % name).replace("\\", "/") + self.simulation.visualization[0] = f + elif 'test_structure' in self and 'logfile' in self.test_structure: # hawc2aero + self.test_structure.logfile = os.path.join(fmt_folder('log', subfolder), "%s.log" % name).replace("\\", "/") + if 'output' in self: + self.output.filename = os.path.join(fmt_folder('res', subfolder), "%s" % name).replace("\\", "/") + + def set_time(self, start=None, stop=None, step=None): + self.contents # load if not loaded + if stop is not None: + self.simulation.time_stop = stop + else: + stop = self.simulation.time_stop[0] + if step is not None: + self.simulation.newmark.deltat = step + if start is not None: + self.output.time = start, stop + if "wind" in self: # and self.wind.turb_format[0] > 0: + self.wind.scale_time_start = start + + def expected_simulation_time(self): + return 600 + + def pbs_file(self, hawc2_path, hawc2_cmd, queue='workq', walltime=None, + input_files=None, output_files=None, copy_turb=(True, True)): + walltime = walltime or self.expected_simulation_time() * 2 + if len(copy_turb) == 1: + copy_turb_fwd, copy_turb_back = copy_turb, copy_turb + else: + copy_turb_fwd, copy_turb_back = copy_turb + + input_files = input_files or self.input_files() + if copy_turb_fwd: + input_files += [f for f in self.turbulence_files() if os.path.isfile(f)] + + output_files = output_files or self.output_files() + if copy_turb_back: + output_files += self.turbulence_files() + + return HAWC2PBSFile(hawc2_path, hawc2_cmd, self.filename, self.modelpath, + input_files, output_files, + queue, walltime) + + def input_files(self): + self.contents # load if not loaded + if self.modelpath == "unknown": + files = [str(f).replace("\\", "/") for f in [self.filename] + self.htc_inputfiles] + else: + files = [os.path.abspath(str(f)).replace("\\", "/") for f in [self.filename] + self.htc_inputfiles] + if 'new_htc_structure' in self: + for mb in [self.new_htc_structure[mb] + for mb in self.new_htc_structure.keys() if mb.startswith('main_body')]: + if "timoschenko_input" in mb: + files.append(mb.timoschenko_input.filename[0]) + files.append(mb.get('external_bladedata_dll', [None, None, None])[2]) + if 'aero' in self: + files.append(self.aero.ae_filename[0]) + files.append(self.aero.pc_filename[0]) + files.append(self.aero.get('external_bladedata_dll', [None, None, None])[2]) + files.append(self.aero.get('output_profile_coef_filename', [None])[0]) + if 'dynstall_ateflap' in self.aero: + files.append(self.aero.dynstall_ateflap.get('flap', [None] * 3)[2]) + if 'bemwake_method' in self.aero: + files.append(self.aero.bemwake_method.get('a-ct-filename', [None] * 3)[0]) + for dll in [self.dll[dll] for dll in self.get('dll', {}).keys() if 'filename' in self.dll[dll]]: + files.append(dll.filename[0]) + f, ext = os.path.splitext(dll.filename[0]) + files.append(f + "_64" + ext) + if 'wind' in self: + files.append(self.wind.get('user_defined_shear', [None])[0]) + files.append(self.wind.get('user_defined_shear_turbulence', [None])[0]) + files.append(self.wind.get('met_mast_wind', [None])[0]) + if 'wakes' in self: + files.append(self.wind.get('use_specific_deficit_file', [None])[0]) + files.append(self.wind.get('write_ct_cq_file', [None])[0]) + files.append(self.wind.get('write_final_deficits', [None])[0]) + if 'hydro' in self: + if 'water_properties' in self.hydro: + files.append(self.hydro.water_properties.get('water_kinematics_dll', [None])[0]) + files.append(self.hydro.water_properties.get('water_kinematics_dll', [None, None])[1]) + if 'soil' in self: + if 'soil_element' in self.soil: + files.append(self.soil.soil_element.get('datafile', [None])[0]) + try: + dtu_we_controller = self.dll.get_subsection_by_name('dtu_we_controller') + theta_min = dtu_we_controller.init.constant__5[1] + if theta_min >= 90: + files.append(os.path.join(os.path.dirname( + dtu_we_controller.filename[0]), "wpdata.%d" % theta_min).replace("\\", "/")) + except Exception: + pass + + try: + files.append(self.force.dll.dll[0]) + except Exception: + pass + + def fix_path_case(f): + if os.path.isabs(f): + return self.unix_path(f) + elif self.modelpath != "unknown": + try: + return "./" + os.path.relpath(self.unix_path(os.path.join(self.modelpath, f)), + self.modelpath).replace("\\", "/") + except IOError: + return f + else: + return f + return [fix_path_case(f) for f in set(files) if f] + + def output_files(self): + self.contents # load if not loaded + files = [] + for k, index in [('simulation/logfile', 0), + ('simulation/animation', 0), + ('simulation/visualization', 0), + ('new_htc_structure/beam_output_file_name', 0), + ('new_htc_structure/body_output_file_name', 0), + ('new_htc_structure/struct_inertia_output_file_name', 0), + ('new_htc_structure/body_eigenanalysis_file_name', 0), + ('new_htc_structure/constraint_output_file_name', 0), + ('wind/turb_export/filename_u', 0), + ('wind/turb_export/filename_v', 0), + ('wind/turb_export/filename_w', 0)]: + line = self.get(k) + if line: + files.append(line[index]) + if 'new_htc_structure' in self: + if 'system_eigenanalysis' in self.new_htc_structure: + f = self.new_htc_structure.system_eigenanalysis[0] + files.append(f) + files.append(os.path.join(os.path.dirname(f), 'mode*.dat').replace("\\", "/")) + if 'structure_eigenanalysis_file_name' in self.new_htc_structure: + f = self.new_htc_structure.structure_eigenanalysis_file_name[0] + files.append(f) + files.append(os.path.join(os.path.dirname(f), 'mode*.dat').replace("\\", "/")) + files.extend(self.res_file_lst()) + + for key in [k for k in self.contents.keys() if k.startswith("output_at_time")]: + files.append(self[key]['filename'][0] + ".dat") + return [f.lower() for f in files if f] + + def turbulence_files(self): + self.contents # load if not loaded + if 'wind' not in self.contents.keys() or self.wind.turb_format[0] == 0: + return [] + elif self.wind.turb_format[0] == 1: + files = [self.get('wind.mann.filename_%s' % comp, [None])[0] for comp in ['u', 'v', 'w']] + elif self.wind.turb_format[0] == 2: + files = [self.get('wind.flex.filename_%s' % comp, [None])[0] for comp in ['u', 'v', 'w']] + return [f for f in files if f] + + def res_file_lst(self): + self.contents # load if not loaded + res = [] + for output in [self[k] for k in self.keys() + if self[k].name_.startswith("output") and not self[k].name_.startswith("output_at_time")]: + dataformat = output.get('data_format', 'hawc_ascii') + res_filename = output.filename[0] + if dataformat[0] == "gtsdf" or dataformat[0] == "gtsdf64": + res.append(res_filename + ".hdf5") + elif dataformat[0] == "flex_int": + res.extend([res_filename + ".int", os.path.join(os.path.dirname(res_filename), 'sensor')]) + else: + res.extend([res_filename + ".sel", res_filename + ".dat"]) + return res + + def _simulate(self, exe, skip_if_up_to_date=False): + self.contents # load if not loaded + if skip_if_up_to_date: + from os.path import isfile, getmtime, isabs + res_file = os.path.join(self.modelpath, self.res_file_lst()[0]) + htc_file = os.path.join(self.modelpath, self.filename) + if isabs(exe): + exe_file = exe + else: + exe_file = os.path.join(self.modelpath, exe) + #print (from_unix(getmtime(res_file)), from_unix(getmtime(htc_file))) + if (isfile(htc_file) and isfile(res_file) and isfile(exe_file) and + str(HTCFile(htc_file)) == str(self) and + getmtime(res_file) > getmtime(htc_file) and getmtime(res_file) > getmtime(exe_file)): + if "".join(self.readfilelines(htc_file)) == str(self): + return + + self.save() + htcfile = os.path.relpath(self.filename, self.modelpath) + assert any([os.path.isfile(os.path.join(f, exe)) for f in [''] + os.environ['PATH'].split(";")]), exe + return pexec([exe, htcfile], self.modelpath) + + def simulate(self, exe, skip_if_up_to_date=False): + errorcode, stdout, stderr, cmd = self._simulate(exe, skip_if_up_to_date) + if ('simulation' in self.keys() and "logfile" in self.simulation and + os.path.isfile(os.path.join(self.modelpath, self.simulation.logfile[0]))): + with self.open(os.path.join(self.modelpath, self.simulation.logfile[0])) as fid: + log = fid.read() + else: + log = "%s\n%s" % (str(stdout), str(stderr)) + + if errorcode or 'Elapsed time' not in log: + log_lines = log.split("\n") + error_lines = [i for i, l in enumerate(log_lines) if 'error' in l.lower()] + if error_lines: + import numpy as np + line_i = np.r_[np.array([error_lines + i for i in np.arange(-3, 4)]).flatten(), + np.arange(-5, 0) + len(log_lines)] + line_i = sorted(np.unique(np.maximum(np.minimum(line_i, len(log_lines) - 1), 0))) + + lines = ["%04d %s" % (i, log_lines[i]) for i in line_i] + for jump in np.where(np.diff(line_i) > 1)[0]: + lines.insert(jump, "...") + + error_log = "\n".join(lines) + else: + error_log = log + raise Exception("\nError code: %s\nstdout:\n%s\n--------------\nstderr:\n%s\n--------------\nlog:\n%s\n--------------\ncmd:\n%s" % + (errorcode, str(stdout), str(stderr), error_log, cmd)) + return str(stdout) + str(stderr), log + + def simulate_hawc2stab2(self, exe): + errorcode, stdout, stderr, cmd = self._simulate(exe, skip_if_up_to_date=False) + + if errorcode: + raise Exception("\nstdout:\n%s\n--------------\nstderr:\n%s\n--------------\ncmd:\n%s" % + (str(stdout), str(stderr), cmd)) + return str(stdout) + str(stderr) + + def deltat(self): + return self.simulation.newmark.deltat[0] + + def compare(self, other): + if isinstance(other, str): + other = HTCFile(other) + return HTCContents.compare(self, other) + + @property + def open(self): + return open + + def unix_path(self, filename): + filename = os.path.realpath(str(filename)).replace("\\", "/") + ufn, rest = os.path.splitdrive(filename) + ufn += "/" + for f in rest[1:].split("/"): + f_lst = [f_ for f_ in os.listdir(ufn) if f_.lower() == f.lower()] + if len(f_lst) > 1: + # use the case sensitive match + f_lst = [f_ for f_ in f_lst if f_ == f] + if len(f_lst) == 0: + raise IOError("'%s' not found in '%s'" % (f, ufn)) + else: # one match found + ufn = os.path.join(ufn, f_lst[0]) + return ufn.replace("\\", "/") + + +# +# def get_body(self, name): +# lst = [b for b in self.new_htc_structure if b.name_=="main_body" and b.name[0]==name] +# if len(lst)==1: +# return lst[0] +# else: +# if len(lst)==0: +# raise ValueError("Body '%s' not found"%name) +# else: +# raise NotImplementedError() +# + +class H2aeroHTCFile(HTCFile): + def __init__(self, filename=None, modelpath=None): + HTCFile.__init__(self, filename=filename, modelpath=modelpath) + + @property + def simulation(self): + return self.test_structure + + def set_time(self, start=None, stop=None, step=None): + if stop is not None: + self.test_structure.time_stop = stop + else: + stop = self.simulation.time_stop[0] + if step is not None: + self.test_structure.deltat = step + if start is not None: + self.output.time = start, stop + if "wind" in self and self.wind.turb_format[0] > 0: + self.wind.scale_time_start = start + + +class SSH_HTCFile(HTCFile): + def __init__(self, ssh, filename=None, modelpath=None): + object.__setattr__(self, 'ssh', ssh) + HTCFile.__init__(self, filename=filename, modelpath=modelpath) + + @property + def open(self): + return self.ssh.open + + def unix_path(self, filename): + rel_filename = os.path.relpath(filename, self.modelpath).replace("\\", "/") + _, out, _ = self.ssh.execute("find -ipath ./%s" % rel_filename, cwd=self.modelpath) + out = out.strip() + if out == "": + raise IOError("'%s' not found in '%s'" % (rel_filename, self.modelpath)) + elif "\n" in out: + raise IOError("Multiple '%s' found in '%s' (due to case senitivity)" % (rel_filename, self.modelpath)) + else: + drive, path = os.path.splitdrive(os.path.join(self.modelpath, out)) + path = os.path.realpath(path).replace("\\", "/") + return os.path.join(drive, os.path.splitdrive(path)[1]) + + +if "__main__" == __name__: + f = HTCFile(r"C:/Work/BAR-Local/Hawc2ToBeamDyn/sim.htc", ".") + print(f.input_files()) + import pdb; pdb.set_trace() +# f.save(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT_power_curve.htc") +# +# f = HTCFile(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT.htc", "../") +# f.set_time = 0, 1, .1 +# print(f.simulate(r"C:\mmpe\HAWC2\bin\HAWC2_12.8\hawc2mb.exe")) +# +# f.save(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT.htc") diff --git a/pydatview/io/wetb/hawc2/pc_file.py b/pydatview/io/wetb/hawc2/pc_file.py index b91655d..83e0a1b 100644 --- a/pydatview/io/wetb/hawc2/pc_file.py +++ b/pydatview/io/wetb/hawc2/pc_file.py @@ -3,15 +3,6 @@ @author: MMPE ''' -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import absolute_import -from io import open -from builtins import range -from builtins import int -from future import standard_library -standard_library.install_aliases() import os import numpy as np @@ -153,7 +144,9 @@ def __str__(self, comments=None): def save(self, filename): if not os.path.isdir(os.path.dirname(filename)): - os.makedirs(os.path.dirname(filename)) + # fails if dirname is empty string + if len(os.path.dirname(filename)) > 0: + os.makedirs(os.path.dirname(filename)) with open(filename, 'w') as fid: fid.write(str(self)) self.filename = filename diff --git a/pydatview/io/wetb/hawc2/st_file.py b/pydatview/io/wetb/hawc2/st_file.py index 8fce4b5..91497a0 100644 --- a/pydatview/io/wetb/hawc2/st_file.py +++ b/pydatview/io/wetb/hawc2/st_file.py @@ -3,22 +3,14 @@ @author: MMPE ''' -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import -from io import open -from builtins import range -from builtins import int -from future import standard_library import types -standard_library.install_aliases() import os import numpy as np stc = "r m x_cg y_cg ri_x ri_y x_sh y_sh E G I_x I_y I_p k_x k_y A pitch x_e y_e" - +fpm = 'r m x_cg y_cg ri_x ri_y pitch x_e y_e K_11 K_12 K_13 K_14 K_15 K_16 K_22' +fpm += ' K_23 K_24 K_25 K_26 K_33 K_34 K_35 K_36 K_44 K_45 K_46 K_55 K_56 K_66' class StFile(object): """Read HAWC2 St (beam element structural data) file @@ -71,8 +63,6 @@ def xxx(radius=None, mset=1, set=1): 8.722924514652648e+17 """ - cols = stc.split() - def __init__(self, filename=None): # in case the user wants to create a new non-existing st file @@ -94,9 +84,24 @@ def __init__(self, filename=None): set_lines = set_txt.split("\n") set_nr, no_rows = map(int, set_lines[0].split()[:2]) assert set_nr not in set_data_dict - set_data_dict[set_nr] = np.array([set_lines[i].split() for i in range(1, no_rows + 1)], dtype=float) + try: + # HAWC2 will ignore everything after the 19th element, + # some users have placed comments here after a ; + linelst = [set_lines[i].split(';')[0].split() for i in range(1, no_rows + 1)] + except Exception as e: + print('it went wrong at (set/subset):', mset_nr, set_nr, + 'with', no_rows, 'rows') + raise e + set_data_dict[set_nr] = np.array(linelst, dtype=float) self.main_data_sets[mset_nr] = set_data_dict + if len(linelst[0])==len(stc.split()): + self.cols = stc.split() + elif len(linelst[0])==len(fpm.split()): + self.cols = fpm.split() + else: + raise TypeError('wrong number of columns in st file') + for i, name in enumerate(self.cols): setattr(self, name, lambda radius=None, mset=1, set=1, column=i: self._value(radius, column, mset, set)) @@ -125,12 +130,25 @@ def set_value(self, mset_nr, set_nr, **kwargs): def save(self, filename, precision='%15.07e', encoding='utf-8'): """Save all data defined in main_data_sets to st file. """ + + # when creating empty st object, cols is not set yet + if not hasattr(self, 'cols'): + if self.main_data_sets[1][1].shape[1]==19: + self.cols = stc.split() + elif self.main_data_sets[1][1].shape[1]==30: + self.cols = fpm.split() + else: + c = self.main_data_sets[1][1].shape[1] + raise TypeError(f'St input needs 19 (iso) or 30 (aniso/fpm) cols, not {c}') + colwidth = len(precision % 1) sep = '=' * colwidth * len(self.cols) + '\n' colhead = ''.join([k.center(colwidth) for k in self.cols]) + '\n' nsets = len(self.main_data_sets) - os.makedirs(os.path.dirname(filename), exist_ok=True) + # fails if dirname is empty string + if len(os.path.dirname(filename)) > 0: + os.makedirs(os.path.dirname(filename), exist_ok=True) with open(filename, 'w', encoding=encoding) as fid: fid.write('%i ; number of sets, Nset\n' % nsets) for mset, set_data_dict in self.main_data_sets.items(): @@ -152,6 +170,11 @@ def element_stiffnessmatrix(self, radius, mset_nr, set_nr, length): length : float eleement length """ + + # not supported for FPM format + if len(self.cols)==30: + return + K = np.zeros((13, 13)) "r m x_cg y_cg ri_x ri_y x_sh y_sh E G I_x I_y I_p k_x k_y A pitch x_e y_e" ES1, ES2, EMOD, GMOD, IX, IY, IZ, KX, KY, A = [getattr(self, n)(radius, mset_nr, set_nr) @@ -210,6 +233,11 @@ def element_stiffnessmatrix(self, radius, mset_nr, set_nr, length): return K def shape_function_ori(self, radius, mset_nr, set_nr, length, z): + + # not supported for FPM format + if len(self.cols)==30: + return + XSC, YSC, EMOD, GMOD, IX, IY, IZ, KX, KY, AREA = [getattr(self, n)(radius, mset_nr, set_nr) for n in "x_sh,y_sh,E,G,I_x,I_y,I_p,k_x,k_y,A".split(",")] diff --git a/pydatview/tools/curve_fitting.py b/pydatview/tools/curve_fitting.py index c03f158..cf99636 100644 --- a/pydatview/tools/curve_fitting.py +++ b/pydatview/tools/curve_fitting.py @@ -1304,7 +1304,7 @@ def _clean_formula(s, latex=False): def main_frequency(t,y): """ Returns main frequency of a signal - NOTE: this tool below to welib.tools.signal, but put here for convenience + NOTE: this tool below to welib.tools.signal_analysis, but put here for convenience """ dt = t[1]-t[0] # assume uniform spacing of time and frequency om = np.fft.fftfreq(len(t), (dt))*2*np.pi diff --git a/pydatview/tools/stats.py b/pydatview/tools/stats.py index aaa66dc..c5492d7 100644 --- a/pydatview/tools/stats.py +++ b/pydatview/tools/stats.py @@ -53,7 +53,7 @@ def rsquare(y,f, c = True): rmse = np.sqrt(np.mean((y - f) ** 2)) return r2,rmse -def mean_rel_err(t1=None, y1=None, t2=None, y2=None, method='mean', verbose=False): +def mean_rel_err(t1=None, y1=None, t2=None, y2=None, method='mean', verbose=False, varname=''): """ return mean relative error in % @@ -69,32 +69,37 @@ def mean_rel_err(t1=None, y1=None, t2=None, y2=None, method='mean', verbose=Fals else: if len(y1)!=len(y2): y2=np.interp(t1,t2,y2) - # Method 1 relative to mean if method=='mean': + # Method 1 relative to mean ref_val = np.mean(y1) - meanrelerr = np.mean(np.abs(y1-y2)/ref_val)*100 + meanrelerr = np.mean(np.abs(y2-y1)/ref_val)*100 elif method=='meanabs': ref_val = np.mean(np.abs(y1)) - meanrelerr = np.mean(np.abs(y1-y2)/ref_val)*100 + meanrelerr = np.mean(np.abs(y2-y1)/ref_val)*100 + elif method=='loc': + meanrelerr = np.mean(np.abs(y2-y1)/abs(y1))*100 elif method=='minmax': # Method 2 scaling signals Min=min(np.min(y1), np.min(y2)) Max=max(np.max(y1), np.max(y2)) y1=(y1-Min)/(Max-Min)+0.5 y2=(y2-Min)/(Max-Min)+0.5 - meanrelerr = np.mean(np.abs(y1-y2)/np.abs(y1))*100 + meanrelerr = np.mean(np.abs(y2-y1)/np.abs(y1))*100 elif method=='1-2': # transform values from 1 to 2 Min=min(np.min(y1), np.min(y2)) Max=max(np.max(y1), np.max(y2)) y1 = (y1-Min)/(Max-Min)+1 y2 = (y2-Min)/(Max-Min)+1 - meanrelerr = np.mean(np.abs(y1-y2)/np.abs(y1))*100 + meanrelerr = np.mean(np.abs(y2-y1)/np.abs(y1))*100 else: raise Exception('Unknown method',method) if verbose: - print('Mean rel error {:7.2f} %'.format( meanrelerr)) + if len(varname)>0: + print('Mean rel error {:15s} {:7.2f} %'.format(varname, meanrelerr)) + else: + print('Mean rel error {:7.2f} %'.format( meanrelerr)) return meanrelerr From 5b8291fb6f0f0f2b449be3b3400dd8dcf5796a27 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 25 Jul 2022 21:28:16 -0600 Subject: [PATCH 018/178] Passing multiple dataframes at once (#120) --- README.md | 12 +++++++++--- pydatview/main.py | 28 +++++++++++++++++----------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 22cb984..294a0c6 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,18 @@ If you cloned this repository, the main script at the root (`pyDatView.py`) is e ```bash pyDatView file.csv ``` -The python package can also be used directly from python/jupyter to display a dataframe or show the data in a file +The python package can also be used directly from python/jupyter to display one or multiple dataframe(s) (called `df1` and `df2` in the example below) or show the data present in one or several file(s). The interface is forgiving for the first argument, and can accept a list or a single value: ```python import pydatview -pydatview.show(dataframe=df) +pydatview.show(dataframes=[df1,df2], names=['data1','data2']) # OR -pydatview.show(filenames=['file.csv']) +pydatview.show([df1,df2], names=['data1','data2']) +# OR +pydatview.show(df1) +# OR +pydatview.show(filenames=['file.csv','file2.csv']) +# OR +pydatview.show(['file.csv','file2.csv']) # OR pydatview.show('file.csv') ``` diff --git a/pydatview/main.py b/pydatview/main.py index 70709fa..10cfc21 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -753,27 +753,33 @@ def __init__(self, redirect=False, filename=None): # --------------------------------------------------------------------------------} # --- Mains # --------------------------------------------------------------------------------{ -def showApp(firstArg=None,dataframe=None,filenames=[]): +def showApp(firstArg=None, dataframes=None, filenames=[], names=None): """ - The main function to start the data frame GUI. + The main function to start the pyDatView GUI and loads + Call this function with: + - filenames : list of filenames or a single filename (string) + OR + - dataframes: list of dataframes or a single dataframe + - names: list of names to be used for the multiple dataframes """ app = MyWxApp(False) frame = MainFrame() # Optional first argument if firstArg is not None: if isinstance(firstArg,list): - filenames=firstArg + if isinstance(firstArg[0],str): + filenames=firstArg + else: + dataframes=firstArg elif isinstance(firstArg,str): filenames=[firstArg] elif isinstance(firstArg, pd.DataFrame): - dataframe=firstArg - # - if (dataframe is not None) and (len(dataframe)>0): - #import time - #tstart = time.time() - frame.load_df(dataframe) - #tend = time.time() - #print('PydatView time: ',tend-tstart) + dataframes=[firstArg] + # Load files or dataframe depending on interface + if (dataframes is not None) and (len(dataframes)>0): + if names is None: + names=['df{}'.format(i+1) for i in range(len(dataframes))] + frame.load_dfs(dataframes, names) elif len(filenames)>0: frame.load_files(filenames, fileformats=None) app.MainLoop() From afef5f0622c521eba10fe06936f34a87af1a1ac5 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 27 Jul 2022 10:53:03 -0600 Subject: [PATCH 019/178] Change of dependencies for executable (#122) --- installer.cfg | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/installer.cfg b/installer.cfg index 1995c75..c672653 100644 --- a/installer.cfg +++ b/installer.cfg @@ -82,7 +82,6 @@ exclude= pkgs/matplotlib/sphinxext pkgs/matplotlib/testing pkgs/matplotlib/mpl-data/sample_data - pkgs/matplotlib/mpl-data/fonts pkgs/matplotlib/mpl-data/images/*.pdf pkgs/matplotlib/mpl-data/images/*.svg pkgs/matplotlib/mpl-data/images/*.ppm @@ -105,12 +104,12 @@ exclude= pkgs/wx/lib/pubsub pkgs/wx/lib/wxcairo pkgs/scipy/cluster - pkgs/scipy/constants pkgs/scipy/io/tests pkgs/scipy/io/arff pkgs/scipy/io/matlab/tests pkgs/scipy/io/harwell_boieng/tests - pkgs/scipy/ndimage + pkgs/scipy/ndimage/_ctest*.pyd + pkgs/scipy/ndimage/_cytest*.pyd pkgs/scipy/odr pkgs/scipy/extra-dll/libbanded* pkgs/scipy/extra-dll/libd_odr* @@ -120,6 +119,9 @@ exclude= pkgs/scipy/fftpack pkgs/scipy/signal +# pkgs/matplotlib/mpl-data/fonts +# pkgs/scipy/constants + # pkgs\matplotlib\mpl-data ##Click==7.0 From a965afc3612648154782bc4636a83c627f454259 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 29 Jul 2022 08:15:31 -0600 Subject: [PATCH 020/178] Changing matplotlib backend to 'WX' (#122) --- installer.cfg | 2 +- pydatview/GUIPlotPanel.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/installer.cfg b/installer.cfg index c672653..1e9a23e 100644 --- a/installer.cfg +++ b/installer.cfg @@ -81,6 +81,7 @@ exclude= pkgs/pandas/tests pkgs/matplotlib/sphinxext pkgs/matplotlib/testing + pkgs/matplotlib/mpl-data/fonts pkgs/matplotlib/mpl-data/sample_data pkgs/matplotlib/mpl-data/images/*.pdf pkgs/matplotlib/mpl-data/images/*.svg @@ -119,7 +120,6 @@ exclude= pkgs/scipy/fftpack pkgs/scipy/signal -# pkgs/matplotlib/mpl-data/fonts # pkgs/scipy/constants # pkgs\matplotlib\mpl-data diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index d80e08c..81fff1e 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -5,7 +5,9 @@ import dateutil # required by matplotlib #from matplotlib import pyplot as plt import matplotlib -matplotlib.use('wxAgg') # Important for Windows version of installer. NOTE: changed from Agg to wxAgg +# Backends: +# ['GTK3Agg', 'GTK3Cairo', 'GTK4Agg', 'GTK4Cairo', 'MacOSX', 'nbAgg', 'QtAgg', 'QtCairo', 'Qt5Agg', 'Qt5Cairo', 'TkAgg', 'TkCairo', 'WebAgg', 'WX', 'WXAgg', 'WXCairo', 'agg', 'cairo', 'pdf', 'pgf', 'ps', 'svg', 'template'] +matplotlib.use('WX') # Important for Windows version of installer. NOTE: changed from Agg to wxAgg, then to WX from matplotlib import rc as matplotlib_rc try: from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas From 9b4180d030caa932779bc40e2a97161400d74c29 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 29 Jul 2022 15:42:59 -0600 Subject: [PATCH 021/178] Attempt updates for latest MacOS --- README.md | 53 ++++-- pydatview/GUIToolBox.py | 40 ++++- pydatview/main.py | 9 +- pythonmac | 377 ++++++++++++++++++++-------------------- 4 files changed, 267 insertions(+), 212 deletions(-) diff --git a/README.md b/README.md index 294a0c6..e57a510 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,27 @@ Additional file formats can easily be added. ## QuickStart For **Windows** users, an installer executable is available [here](https://github.com/ebranlard/pyDatView/releases) (look for the latest pyDatView\*.exe) -**Linux** and **MacOS** users can use the command lines below. **Linux** users may need to install the package python-wxgtk\* (e.g. `python-gtk3.0`) from their distribution. **MacOS** users can use a `brew`, `anaconda` or `virtualenv` version of python and pip, but the final version of python that calls the script needs to have access to the screen (see [details for MacOS](#macos-installation)). The main commands for **Linux** and **MacOS** users are: +**Linux** users can use the command lines below, but first they'll need to install the package python-wxgtk\* (e.g. `python-gtk3.0`) from their distribution: ```bash git clone https://github.com/ebranlard/pyDatView cd pyDatView python -m pip install --user -r requirements.txt -make # executes: 'python pyDatView.py' (on linux) or './pythonmac pyDatView.py' (on Mac) +make # will run python pyDatView.py echo "alias pydat='make -C `pwd`'" >> ~/.bashrc ``` + +**MacOS** users can use a `brew`, `anaconda` or `virtualenv` version of python and pip, but the final version of python that calls the script needs to have access to the screen (for instance using `pythonw`) (see [details for MacOS](#macos-installation)). We recommend using conda, for which the following commands should work: +```bash +conda install -c conda-forge wxpython # install wxpython +git clone https://github.com/ebranlard/pyDatView +cd pyDatView +python -m pip install --user -r requirements.txt +make # will run ./pythonmac pyDatView.py +# OR try +#pythonw pyDatView.py # NOTE: using pythonw not python +echo "alias pydat='make -C `pwd`'" >> ~/.bashrc +``` + More information about the download, requirements and installation is provided [further down this page](#installation) @@ -31,7 +44,7 @@ Windows users that used a `setup.exe` file should be able to look for `pyDatView If you cloned this repository, the main script at the root (`pyDatView.py`) is executable and will open the GUI directly. A command line interface is provided, e.g.: ```bash -pyDatView file.csv +python pyDatView.py file.csv # or pythonw pyDatView.py file.csv ``` The python package can also be used directly from python/jupyter to display one or multiple dataframe(s) (called `df1` and `df2` in the example below) or show the data present in one or several file(s). The interface is forgiving for the first argument, and can accept a list or a single value: ```python @@ -168,7 +181,7 @@ For further troubleshooting you can check the [wxPython wiki page](https://wiki. If the requirements are successfully installed you can run pyDatView by typing: ```bash -python pyDatView.py +python pyDatView.py # or pythonw pyDatView.py ``` To easily access it later, you can add an alias to your `.bashrc` or install the pydatview module: ```bash @@ -179,7 +192,7 @@ python setup.py install ## MacOS installation -The installation works with python2 and python3, with `brew` (with or without a `virtualenv`) or `anaconda`. +The installation should work with python2 and python3, with `brew` (with or without a `virtualenv`) or `anaconda`. First, download the source code: ```bash git clone https://github.com/ebranlard/pyDatView @@ -191,6 +204,25 @@ Before installing the requirements, you need to be aware of the two following is The script `pythonmac` provided in this repository attempt to find the correct python program depending if you are in a virtual environment, in a conda environment, a system-python or a python from brew or conda. Different solutions are provided below depending on your preferred way of working. +For the latest Mac version, we recommend using anaconda. + +### Anaconda-python version (outside a virtualenv) +The installation of anaconda sometimes replaces the system python with the anaconda version of python. You can see that by typing `which python`. Use the following: +``` +python -m pip install --user -r requirements.txt # install requirements +conda install -c conda-forge wxpython # install wxpython +pythonw pyDatView.py # NOTE: using pythonw not python +``` +If the `pythonw` command above fails, try the few next options, and post an issue. You can try the `./pythonmac` provided in this repository +```bash +./pythonmac pyDatView.py +``` +If that still doesn't work, you can try using the `python.app` from anaconda: +```bash +/anaconda3/bin/python.app +``` +where `/anaconda3/bin/` is the path that would be returned by the command `which conda`. Note the `.app` at the end. If you don't have `python.app`, try installing it with `conda install -c anaconda python.app` + ### Brew-python version (outside of a virtualenv) If you have `brew` installed, and you installed python with `brew install python`, then the easiest is to use your `python3` version: @@ -212,17 +244,6 @@ $(brew --prefix)/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/b where the result from `brew --prefix` is usually `/usr/loca/` and the `XXX` above corresponds to the version of python you are using in your virtual environment. -### Anaconda-python version (outside a virtualenv) -The installation of anaconda sometimes replaces the system python with the anaconda version of python. You can see that by typing `which python`. Use the following: -``` -python -m pip install --user -r requirements.txt -./pythonmac pyDatView.py -``` -If the `pythonmac` commands fails, contact the developer, and in the meantime try to replace it with a path similar to -```bash -/anaconda3/bin/python.app -``` -where `/anaconda3/bin/` is the path that would be returned by the command `which conda`. Note the `.app` at the end. If you don't have `python.app`, try installing it with `conda install -c anaconda python.app` ### Easy access diff --git a/pydatview/GUIToolBox.py b/pydatview/GUIToolBox.py index a32141e..8bca906 100644 --- a/pydatview/GUIToolBox.py +++ b/pydatview/GUIToolBox.py @@ -50,10 +50,43 @@ def TBAddCheckTool(tb,label,bitmap,callback=None,bitmap2=None): tb.Bind(wx.EVT_TOOL, callback, tl) return tl -def TBAddTool(tb,label,bitmap,callback=None,Type=None): +def TBAddTool(tb, label, defaultBitmap=None, callback=None, Type=None): """ Adding a toolbar tool, safe depending on interface and compatibility see also wx_compat AddTool in wx backends """ + try: + wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN) + hasBitMap=True + except: + # Somehow fails on recent Mac OS + hasBitMap = False + bitmap = None + defaultBitmap = None + + if defaultBitmap is None: + # Last resort, we add a button only + bt=wx.Button(tb,wx.ID_ANY, label) + tl=tb.AddControl(bt) + if callback is not None: + tb.Bind(wx.EVT_BUTTON, callback, bt) + return tl + else: + # --- TODO this is not pretty.. Use wx.INDEX directly? + if defaultBitmap=='ART_REDO': + bitmap = wx.ArtProvider.GetBitmap(wx.ART_REDO) + elif defaultBitmap=='ART_FILE_OPEN': + bitmap = wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN) + elif defaultBitmap=='ART_PLUS': + try: + bitmap = wx.ArtProvider.GetBitmap(wx.ART_PLUS) + except: + bitmap = wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN) + elif defaultBitmap=='ART_ERROR': + bitmap = wx.ArtProvider.GetBitmap(wx.ART_ERROR) + else: + raise NotImplementedError(defaultBitmap) + + # Modern API if Type is None or Type==0: try: @@ -85,11 +118,6 @@ def TBAddTool(tb,label,bitmap,callback=None,Type=None): return tl except: Type=None - # Last resort, we add a button only - bt=wx.Button(tb,wx.ID_ANY, label) - tl=tb.AddControl(bt) - if callback is not None: - tb.Bind(wx.EVT_BUTTON, callback, bt) return tl diff --git a/pydatview/main.py b/pydatview/main.py index 10cfc21..1dbef42 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -178,13 +178,10 @@ def __init__(self, data=None): tb.AddControl( wx.StaticText(tb, -1, 'Format: ' ) ) tb.AddControl(self.comboFormats ) tb.AddSeparator() + TBAddTool(tb, "Open" , 'ART_FILE_OPEN', self.onLoad) + TBAddTool(tb, "Reload", 'ART_REDO' , self.onReload) + TBAddTool(tb, "Add" , 'ART_PLUS' , self.onAdd) #bmp = wx.Bitmap('help.png') #wx.Bitmap("NEW.BMP", wx.BITMAP_TYPE_BMP) - TBAddTool(tb,"Open" ,wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN),self.onLoad) - TBAddTool(tb,"Reload",wx.ArtProvider.GetBitmap(wx.ART_REDO),self.onReload) - try: - TBAddTool(tb,"Add" ,wx.ArtProvider.GetBitmap(wx.ART_PLUS),self.onAdd) - except: - TBAddTool(tb,"Add" ,wx.ArtProvider.GetBitmap(wx.FILE_OPEN),self.onAdd) #self.AddTBBitmapTool(tb,"Debug" ,wx.ArtProvider.GetBitmap(wx.ART_ERROR),self.onDEBUG) tb.AddStretchableSpace() tb.Realize() diff --git a/pythonmac b/pythonmac index 6a60205..c4f7014 100755 --- a/pythonmac +++ b/pythonmac @@ -1,184 +1,193 @@ -#!/bin/bash -# pythonmac: detects a python executable which has access to the screen on MacOS -# -# usage: -# ./pythonmac File.py [Arguments] -# -# background: -# On MacOS, wxPython cannot access the screen with a python version which is not a -# "framework" version. -# -# The following script: `import wx. wx.App();` will fail with the error: -# " This program needs access to the screen. Please run with a -# Framework build of python, and only when you are logged in -# on the main display of your Mac." -# -# The "Framework build of python" are in different locations: -# - for the system python: -# /Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX -# or -# /System/Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX -# -# - for python installed with `brew` (likely in `/usr/local`): -# $(brew --suffix)/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX"); -# - for python installed with anaconda, typically: -# /anaconda3/bin/python.app (NOTE: the '.app'!") -# -# The following attempts to detect which version of python is used, whether the user -# is currently in a virtual environment or a conda environment, and tries to find the -# proper "python" to use that has access to the screen. -# -# created: January 2019 -# author : E.Branlard -# website: https://github.com/ebranlard/pyDatView -# - -DEBUG=1 - -# --- Detecting some user settings -if [ -x "$(command -v brew)" ]; then - HAS_BREW=1 -else - HAS_BREW=0 -fi -if [ -x "$(command -v python3)" ]; then - HAS_PY3=1 -else - HAS_PY3=0 -fi -if [ -z "$VIRTUAL_ENV" ] ; then - IS_VIRTUAL_ENV=0 -else - IS_VIRTUAL_ENV=1 -fi -CONDA_IN_PYTHON=`which python | grep conda |wc -l|xargs` -if [ -z "$CONDA_PROMPT_MODIFIER" ] ; then - CONDA_ACTIVE=0 -else - CONDA_ACTIVE=1 -fi - -CURR_PYVER="$(python --version 2>&1 | cut -d ' ' -f2)" # e.g., 2.7.10 -CURR_PYXpY="$(python --version 2>&1 | cut -d ' ' -f2|cut -c 1-3)" # e.g., 2.7 or 3.7 -CURR_PYN="$(python --version 2>&1 | cut -d ' ' -f2|cut -c 1)" # e.g., 2 or 3 - -if [ "$DEBUG" == "1" ] ; then - echo "[INFO] HAS BREW : $HAS_BREW" - echo "[INFO] HAS PY3 : $HAS_PY3" - echo "[INFO] PYTHON_VER N : $CURR_PYN" - echo "[INFO] PYTHON_VER N.X : $CURR_PYXpY" - echo "[INFO] PYTHON_VER : $CURR_PYVER" - echo "[INFO] IS_VIRTUAL_ENV : $IS_VIRTUAL_ENV" - echo "[INFO] CONDA_IN_PYT : $CONDA_IN_PYTHON" - echo "[INFO] CONDA_ACTIVE : $CONDA_ACTIVE" -fi - - -# --- Finding Framework python -FRAMEWORK_PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions" -#echo $FRAMEWORK_PYTHON_ROOT -FRAMEWORK_FOUND=1 -if [ ! -d $FRAMEWORK_PYTHON_ROOT ]; then - #echo "Framework not found in /Library/. Trying in /System" - FRAMEWORK_PYTHON_ROOT="/System/Library/Frameworks/Python.framework/Versions" - #echo $FRAMEWORK_PYTHON_ROOT - if [ ! -d $FRAMEWORK_PYTHON_ROOT ]; then - FRAMEWORK_FOUND=0 - echo "[WARN] Framework python directory not found, things may not work" - fi -fi -if [ "$DEBUG" == "1" ] ; then - echo "[INFO] FRAMEWORK_ROOT : $FRAMEWORK_PYTHON_ROOT" -fi - - -# --- Python exe - TODO, try python3 first -#if [ "$FRAMEWORK_FOUND" == "1" ]; then -# PYVER=2.7 -# FRAMEWORK_PYTHON="$FRAMEWORK_PYTHON_ROOT/$PYVER/bin/python$PYVER" -#else -# # Try /usr/bin/python -# echo "[FAIL] Cannot find a system python installation" -# echo " Find the path to your system python." -# echo " Use the system python to launch pyDatView.py" -#fi -#if [ ! -f $FRAMEWORK_PYTHON ]; then -#fi -#PYTHON_EXE=$FRAMEWORK_PYTHON -# --- Setting up PYTHONHOME and PYTHON_EXE -if [ "$IS_VIRTUAL_ENV" == "0" ] ; then - if [ "$CONDA_ACTIVE" == "1" ] ; then - echo "[INFO] It seems you are in a conda environment, trying with conda python.app" - # - PYTHON_EXE=`which conda|rev|cut -c 6-|rev`"python.app" - HELP="[HELP] If a weird error appears(e.g. loading pandas), try 'conda update python.app'" - elif [ "$CONDA_IN_PYTHON" == "1" ] ; then - PYTHON_EXE=`which python`".app" - HELP="[HELP] If a weird error appears (e.g. loading pandas), try 'conda update python.app'" - else - echo "[WARN] You are not in a virtualenv. Things might not work." - if [ "$HAS_PY3" == "1" ] && [ "$HAS_BREW" == "1" ] ; then - echo "[INFO] Continuing assuming that you used pip3 to install the requirements and a brew python3" - PYTHON_EXE=python3 - HELP="[HELP] If you have a module import error, use 'pip3 install --user -r requirements.txt'" - else - echo "[WARN] don't know which python to use, using default" - PYTHON_EXE=python - # find the root of the virtualenv, it should be the parent of the dir this script is in - #ENV_FROM_DIR=`dirname "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"` - #ENV_FROM_PY=`$FRAMEWORK_PYTHON -c "import os; print os.path.abspath(os.path.join(os.path.dirname(\"$0\"), '..'))"` - #ENV_HOME=$HOME/Library/Python/2.7/lib/python/site-packages/ - #ENV_HOME=$HOME/Library/Python/2.7/ - #echo "ENV from python: $ENV_FROM_PY" - #echo "ENV from dir: $ENV_FROM_DIR" - #echo "ENV from home: $ENV_HOME" - #export PYTHONHOME=$ENV_HOME - fi - fi - - -else - # --------------------------------------------------------------------------------} - # --- VIRTUAL ENV - # --------------------------------------------------------------------------------{ - echo "[INFO] VIRTUAL_ENV: $VIRTUAL_ENV" - if [ "$CURR_PYN" == "2" ] ; then - echo "[INFO] You are using a version 2 of python. Using framework python" - PYTHON_EXE="$FRAMEWORK_PYTHON_ROOT/$CURR_PYXpY/bin/python$CURR_PYXpY" - if [ ! -f $PYTHON_EXE ]; then - echo "[FAIL] Framework python exe not found : $PYTHON_EXE" - exit 1 - fi - - elif [ "$HAS_BREW" == "1" ] ; then - echo "[INFO] Continuing assuming that the virtual env has a brew python" - BREW_VER=`ls -1 $(brew --prefix)/Cellar/python/ |grep $CURR_PYVER | head -1 ` - FRAMEWORK_PYTHON_ROOT="$(brew --prefix)/Cellar/python/$BREW_VER/Frameworks/Python.framework/Versions" - echo $FRAMEWORK_PYTHON_ROOT - if [ ! -d $FRAMEWORK_PYTHON_ROOT ]; then - echo "[FAIL] Brew framework python not found: $FRAMEWORK_PYTHON_ROOT" - exit 1 - fi - PYTHON_EXE="$FRAMEWORK_PYTHON_ROOT/$CURR_PYXpY/bin/python$CURR_PYXpY" - - else - echo "[FAIL] This script does not support your configuration. " - echo " Try running 'python pyDatView.py' and figure how to make it work" - echo " Contact the developer to account for your configuration." - exit 1 - fi - #VENV_SITE_PACKAGES="$VIRTUAL_ENV/lib/python$PYVER/site-packages" - # Ensure wx.pth is set up in the virtualenv - #cp "/Library/Python/$PYVER/site-packages/wxredirect.pth" "$VENV_SITE_PACKAGES/wx.pth" - # Use the Framework Python to run the app - export PYTHONHOME=$VIRTUAL_ENV - echo "[INFO] PYTHONHOME: $PYTHONHOME" -fi - -# --- Launching python -echo "[INFO] Using: $PYTHON_EXE" "$@" -echo "" -echo $HELP -echo "" -exec $PYTHON_EXE "$@" - +#!/bin/bash +# pythonmac: detects a python executable which has access to the screen on MacOS +# +# usage: +# ./pythonmac File.py [Arguments] +# +# background: +# On MacOS, wxPython cannot access the screen with a python version which is not a +# "framework" version. +# +# The following script: `import wx. wx.App();` will fail with the error: +# " This program needs access to the screen. Please run with a +# Framework build of python, and only when you are logged in +# on the main display of your Mac." +# +# The "Framework build of python" are in different locations: +# - for the system python: +# /Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX +# or +# /System/Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX +# +# - for python installed with `brew` (likely in `/usr/local`): +# $(brew --suffix)/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX"); +# - for python installed with anaconda, typically: +# /anaconda3/bin/python.app (NOTE: the '.app'!") +# +# The following attempts to detect which version of python is used, whether the user +# is currently in a virtual environment or a conda environment, and tries to find the +# proper "python" to use that has access to the screen. +# +# created: January 2019 +# author : E.Branlard +# website: https://github.com/ebranlard/pyDatView +# + +DEBUG=1 + +# --- Detecting some user settings +if [ -x "$(command -v brew)" ]; then + HAS_BREW=1 +else + HAS_BREW=0 +fi +if [ -x "$(command -v python3)" ]; then + HAS_PY3=1 +else + HAS_PY3=0 +fi +if [ -z "$VIRTUAL_ENV" ] ; then + IS_VIRTUAL_ENV=0 +else + IS_VIRTUAL_ENV=1 +fi +CONDA_IN_PYTHON=`which python | grep conda |wc -l|xargs` +if [ -z "$CONDA_PROMPT_MODIFIER" ] ; then + CONDA_ACTIVE=0 +else + CONDA_ACTIVE=1 +fi + +CURR_PYVER="$(python --version 2>&1 | cut -d ' ' -f2)" # e.g., 2.7.10 +CURR_PYXpY="$(python --version 2>&1 | cut -d ' ' -f2|cut -c 1-3)" # e.g., 2.7 or 3.7 +CURR_PYN="$(python --version 2>&1 | cut -d ' ' -f2|cut -c 1)" # e.g., 2 or 3 + +if [ "$DEBUG" == "1" ] ; then + echo "[INFO] HAS BREW : $HAS_BREW" + echo "[INFO] HAS PY3 : $HAS_PY3" + echo "[INFO] PYTHON_VER N : $CURR_PYN" + echo "[INFO] PYTHON_VER N.X : $CURR_PYXpY" + echo "[INFO] PYTHON_VER : $CURR_PYVER" + echo "[INFO] IS_VIRTUAL_ENV : $IS_VIRTUAL_ENV" + echo "[INFO] CONDA_IN_PYT : $CONDA_IN_PYTHON" + echo "[INFO] CONDA_ACTIVE : $CONDA_ACTIVE" +fi + + +# --- Finding Framework python +FRAMEWORK_PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions" +#echo $FRAMEWORK_PYTHON_ROOT +FRAMEWORK_FOUND=1 +if [ ! -d $FRAMEWORK_PYTHON_ROOT ]; then + #echo "Framework not found in /Library/. Trying in /System" + FRAMEWORK_PYTHON_ROOT="/System/Library/Frameworks/Python.framework/Versions" + #echo $FRAMEWORK_PYTHON_ROOT + if [ ! -d $FRAMEWORK_PYTHON_ROOT ]; then + FRAMEWORK_FOUND=0 + echo "[WARN] Framework python directory not found, things may not work" + fi +fi +if [ "$DEBUG" == "1" ] ; then + echo "[INFO] FRAMEWORK_ROOT : $FRAMEWORK_PYTHON_ROOT" +fi + + +# --- Python exe - TODO, try python3 first +#if [ "$FRAMEWORK_FOUND" == "1" ]; then +# PYVER=2.7 +# FRAMEWORK_PYTHON="$FRAMEWORK_PYTHON_ROOT/$PYVER/bin/python$PYVER" +#else +# # Try /usr/bin/python +# echo "[FAIL] Cannot find a system python installation" +# echo " Find the path to your system python." +# echo " Use the system python to launch pyDatView.py" +#fi +#if [ ! -f $FRAMEWORK_PYTHON ]; then +#fi +#PYTHON_EXE=$FRAMEWORK_PYTHON +# --- Setting up PYTHONHOME and PYTHON_EXE +if [ "$CONDA_ACTIVE" == "1" ] ; then + # TODO: NEW we use pythonw + echo "[INFO] It seems you are in a conda environment, trying with conda python.app" + # + #PYTHON_EXE=`which conda|rev|cut -c 6-|rev`"python.app" + PYTHON_EXE=pythonw + # NOTE: error message needs updating + HELP="[HELP] If a weird error appears(e.g. loading pandas), try 'conda update python.app'" + +elif [ "$IS_VIRTUAL_ENV" == "0" ] ; then + if [ "$CONDA_ACTIVE" == "1" ] ; then + echo "[INFO] It seems you are in a conda environment, trying with conda python.app" + # + PYTHON_EXE=`which conda|rev|cut -c 6-|rev`"python.app" + HELP="[HELP] If a weird error appears(e.g. loading pandas), try 'conda update python.app'" + elif [ "$CONDA_IN_PYTHON" == "1" ] ; then + PYTHON_EXE=`which python`".app" + HELP="[HELP] If a weird error appears (e.g. loading pandas), try 'conda update python.app'" + else + echo "[WARN] You are not in a virtualenv. Things might not work." + if [ "$HAS_PY3" == "1" ] && [ "$HAS_BREW" == "1" ] ; then + echo "[INFO] Continuing assuming that you used pip3 to install the requirements and a brew python3" + PYTHON_EXE=python3 + HELP="[HELP] If you have a module import error, use 'pip3 install --user -r requirements.txt'" + else + echo "[WARN] don't know which python to use, using default" + PYTHON_EXE=python + # find the root of the virtualenv, it should be the parent of the dir this script is in + #ENV_FROM_DIR=`dirname "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"` + #ENV_FROM_PY=`$FRAMEWORK_PYTHON -c "import os; print os.path.abspath(os.path.join(os.path.dirname(\"$0\"), '..'))"` + #ENV_HOME=$HOME/Library/Python/2.7/lib/python/site-packages/ + #ENV_HOME=$HOME/Library/Python/2.7/ + #echo "ENV from python: $ENV_FROM_PY" + #echo "ENV from dir: $ENV_FROM_DIR" + #echo "ENV from home: $ENV_HOME" + #export PYTHONHOME=$ENV_HOME + fi + fi + + +else + # --------------------------------------------------------------------------------} + # --- VIRTUAL ENV + # --------------------------------------------------------------------------------{ + echo "[INFO] VIRTUAL_ENV: $VIRTUAL_ENV" + if [ "$CURR_PYN" == "2" ] ; then + echo "[INFO] You are using a version 2 of python. Using framework python" + PYTHON_EXE="$FRAMEWORK_PYTHON_ROOT/$CURR_PYXpY/bin/python$CURR_PYXpY" + if [ ! -f $PYTHON_EXE ]; then + echo "[FAIL] Framework python exe not found : $PYTHON_EXE" + exit 1 + fi + + elif [ "$HAS_BREW" == "1" ] ; then + echo "[INFO] Continuing assuming that the virtual env has a brew python" + BREW_VER=`ls -1 $(brew --prefix)/Cellar/python/ |grep $CURR_PYVER | head -1 ` + FRAMEWORK_PYTHON_ROOT="$(brew --prefix)/Cellar/python/$BREW_VER/Frameworks/Python.framework/Versions" + echo $FRAMEWORK_PYTHON_ROOT + if [ ! -d $FRAMEWORK_PYTHON_ROOT ]; then + echo "[FAIL] Brew framework python not found: $FRAMEWORK_PYTHON_ROOT" + exit 1 + fi + PYTHON_EXE="$FRAMEWORK_PYTHON_ROOT/$CURR_PYXpY/bin/python$CURR_PYXpY" + + else + echo "[FAIL] This script does not support your configuration. " + echo " Try running 'python pyDatView.py' and figure how to make it work" + echo " Contact the developer to account for your configuration." + exit 1 + fi + #VENV_SITE_PACKAGES="$VIRTUAL_ENV/lib/python$PYVER/site-packages" + # Ensure wx.pth is set up in the virtualenv + #cp "/Library/Python/$PYVER/site-packages/wxredirect.pth" "$VENV_SITE_PACKAGES/wx.pth" + # Use the Framework Python to run the app + export PYTHONHOME=$VIRTUAL_ENV + echo "[INFO] PYTHONHOME: $PYTHONHOME" +fi + +# --- Launching python +echo "[INFO] Using: $PYTHON_EXE" "$@" +echo "" +echo $HELP +echo "" +exec $PYTHON_EXE "$@" + From d93a077d3d87a0c05eb468e9d014c198fe39d2ba Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sat, 30 Jul 2022 14:54:54 -0600 Subject: [PATCH 022/178] IO: passing hasNodal property to new FASTInputFile class --- pydatview/io/fast_input_file.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index 9f57e5e..dc944ba 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -79,6 +79,9 @@ def module(self): return self.basefile.module else: return self._fixedfile.module + @property + def hasNodal(self): + return self.fixedfile.hasNodal def fixedFormat(self): # --- Creating a dedicated Child From b0897fcc400960069f5e6d10af08dd3b55316ddf Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 31 Jul 2022 12:00:08 -0600 Subject: [PATCH 023/178] Depreciating python 2 (#114) --- README.md | 8 +- _tools/travis_requirements.txt | 10 +-- installer.cfg | 1 - pyDatView.py | 1 - pydatview/GUIPlotPanel.py | 4 +- pydatview/GUISelectionPanel.py | 10 +-- pydatview/GUITools.py | 1 - pydatview/Tables.py | 5 +- pydatview/__init__.py | 2 - pydatview/fast/case_gen.py | 1 - pydatview/fast/postpro.py | 62 ++++++++++---- pydatview/fast/runner.py | 1 - pydatview/io/bladed_out_file.py | 29 +++++-- pydatview/io/bmodes_out_file.py | 2 +- pydatview/io/cactus_file.py | 2 +- pydatview/io/csv_file.py | 5 -- pydatview/io/excel_file.py | 6 -- pydatview/io/fast_input_deck.py | 9 -- pydatview/io/fast_input_file.py | 48 ++--------- pydatview/io/fast_linearization_file.py | 13 +-- pydatview/io/fast_output_file.py | 33 ++------ pydatview/io/fast_summary_file.py | 1 - pydatview/io/fast_wind_file.py | 16 +--- pydatview/io/file.py | 1 - pydatview/io/flex_blade_file.py | 15 ++-- pydatview/io/flex_doc_file.py | 13 ++- pydatview/io/flex_out_file.py | 23 ++--- pydatview/io/flex_profile_file.py | 15 ++-- pydatview/io/flex_wavekin_file.py | 17 ++-- pydatview/io/hawc2_dat_file.py | 16 +--- pydatview/io/hawcstab2_ind_file.py | 22 ++--- pydatview/io/hawcstab2_pwr_file.py | 19 ++--- pydatview/io/mini_yaml.py | 3 - pydatview/io/netcdf_file.py | 20 ++--- pydatview/io/tools/graph.py | 1 - pydatview/io/turbsim_file.py | 91 +++++++++++++------- pydatview/io/turbsim_ts_file.py | 13 +-- pydatview/io/wetb/hawc2/Hawc2io.py | 13 +-- pydatview/io/wetb/hawc2/__init__.py | 6 -- pydatview/main.py | 9 -- pydatview/perfmon.py | 2 - pydatview/plotdata.py | 1 - pydatview/tools/damping.py | 1 - pydatview/tools/fatigue.py | 6 -- pydatview/tools/signal_analysis.py | 106 +++++++++++++++++++++++- pydatview/tools/spectral.py | 2 - requirements.txt | 10 +-- tests/prof_all.py | 3 - tests/test_common.py | 2 - 49 files changed, 333 insertions(+), 367 deletions(-) diff --git a/README.md b/README.md index e57a510..8f8ca4c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ # pyDatView -A crossplatform GUI to display tabulated data from files or python pandas dataframes. It's compatible Windows, Linux and MacOS, python 2 and python 3. Some of its features are: multiples plots, FFT plots, probability plots, export of figures... +A crossplatform GUI to display tabulated data from files or python pandas dataframes. It's compatible Windows, Linux and MacOS, with python 3. Some of its features are: multiples plots, FFT plots, probability plots, export of figures... The file formats supported, are: CSV files and other formats present in the [weio](http://github.com/ebranlard/weio/) library. Additional file formats can easily be added. @@ -168,7 +168,7 @@ Scaling all plots between 0 and 1 (by selecting `MinMax`) For Windows users, installer executables are available [here](https://github.com/ebranlard/pyDatView/releases) (look for the latest pyDatView\*.exe) ### Linux installation -The script is compatible python 2.7 and python 3 and relies on the following python packages: `numpy` `matplotlib`, `pandas`, `wxpython`. +The script is compatible with python 3 and relies on the following python packages: `numpy` `matplotlib`, `pandas`, `wxpython`. To download the code and install the dependencies (with pip) run the following: ```bash git clone https://github.com/ebranlard/pyDatView @@ -192,7 +192,7 @@ python setup.py install ## MacOS installation -The installation should work with python2 and python3, with `brew` (with or without a `virtualenv`) or `anaconda`. +The installation should work with python3, with `brew` (with or without a `virtualenv`) or `anaconda`. First, download the source code: ```bash git clone https://github.com/ebranlard/pyDatView @@ -232,7 +232,7 @@ python3 pyDatView.py ``` ### Brew-python version (inside a virtualenv) -If you are inside a virtualenv, with python 2 or 3, use: +If you are inside a virtualenv, with python 3, use: ``` pip install -r requirements.txt ./pythonmac pyDatView.py diff --git a/_tools/travis_requirements.txt b/_tools/travis_requirements.txt index 6aba4ee..77dfc8e 100644 --- a/_tools/travis_requirements.txt +++ b/_tools/travis_requirements.txt @@ -1,11 +1,7 @@ -openpyxl ; python_version>"3.0" -numpy>=1.16.5; python_version>"3.0" -numpy ; python_version<"3.0" -pandas>=1.0.1; python_version>"3.0" -pandas; python_version<="3.0" -xlrd==1.2.0; python_version<"3.0" +openpyxl +numpy +pandas pyarrow # for parquet files matplotlib -future chardet scipy diff --git a/installer.cfg b/installer.cfg index 1e9a23e..d1b0d3c 100644 --- a/installer.cfg +++ b/installer.cfg @@ -65,7 +65,6 @@ pypi_wheels = # PyYAML==5.1.2 packages= - future exclude= pkgs/numpy/core/include diff --git a/pyDatView.py b/pyDatView.py index 2c2e19a..13b9a25 100644 --- a/pyDatView.py +++ b/pyDatView.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import absolute_import import sys #import click diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 81fff1e..c73e6f5 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -24,8 +24,8 @@ print('') print(' You can solve this by either:') print(' - using python3, and pip3 e.g. installing it with brew') - print(' - using a virtual environment with python 2 or 3') - print(' - using anaconda with python 2 or 3'); + print(' - using a virtual environment with python 3') + print(' - using anaconda with python 3'); print('') import sys sys.exit(1) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index d5bbaa9..c3e13ac 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -296,10 +296,7 @@ def __init__(self, mainframe, parent, fullmenu=False): self.Bind(wx.EVT_MENU, self.OnExportTab, item) def MyAppend(self, item): - try: - self.Append(item) # python3 - except: - self.AppendItem(item) # python2 + self.Append(item) # python3 def OnNaming(self, event=None): tabPanel=self.parent.GetParent() @@ -358,10 +355,7 @@ def __init__(self, parent, fullmenu=False): self.Bind(wx.EVT_MENU, self.OnDeleteColumn, item) def MyAppend(self, item): - try: - self.Append(item) # python3 - except: - self.AppendItem(item) # python2 + self.Append(item) # python3 def OnShowID(self, event=None): self.parent.bShowID=self.itShowID.IsChecked() diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index c70f2bb..4d47ae4 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import wx import numpy as np import pandas as pd diff --git a/pydatview/Tables.py b/pydatview/Tables.py index c866761..f2512c8 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -660,10 +660,7 @@ def setColumnByFormula(self,sNewName,sFormula,i=-1): def export(self,path): if isinstance(self.data, pd.DataFrame): - try: - self.data.to_csv(path,sep=',',index=False) #python3 - except: - self.data.to_csv(path,sep=str(u',').encode('utf-8'),index=False) #python 2. + self.data.to_csv(path,sep=',',index=False) #python3 else: raise NotImplementedError('Export of data that is not a dataframe') diff --git a/pydatview/__init__.py b/pydatview/__init__.py index 904c32f..b5dc28b 100644 --- a/pydatview/__init__.py +++ b/pydatview/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - __all__ = ['show'] # defining main function here, to avoid import of pydatview and wx of some unittests diff --git a/pydatview/fast/case_gen.py b/pydatview/fast/case_gen.py index ca32f18..e3823b2 100644 --- a/pydatview/fast/case_gen.py +++ b/pydatview/fast/case_gen.py @@ -1,4 +1,3 @@ -from __future__ import division, print_function import os import collections import glob diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 7ba88ce..a658b05 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -1,5 +1,4 @@ # --- For cmd.py -from __future__ import division, print_function import os import pandas as pd import numpy as np @@ -140,7 +139,6 @@ def AD_BldGag(AD,AD_bld,chordOut=False): if hasattr(AD_bld,'startswith'): # if string AD_bld = FASTInputFile(AD_bld) #print(AD_bld.keys()) - nOuts=AD['NBlOuts'] if nOuts<=0: if chordOut: @@ -156,31 +154,65 @@ def AD_BldGag(AD,AD_bld,chordOut=False): return r_gag def BD_BldStations(BD, BDBld): - """ Returns BeamDyn Blade Station positions, useful to know where the outputs are. + + """ Returns BeamDyn Blade Quadrature Points positions: + - Defines where BeamDyn outputs are provided. + - Used by BeamDyn for the Input Mesh u%DistrLoad + and the Output Mesh y%BldMotion + NOTE: This should match the quadrature points in the summary file of BeamDyn for a straight beam + This will NOT match the "Initial Nodes" reported in the summary file. INPUTS: - BD: either: - a filename of a ElastoDyn input file - an instance of FileCl, as returned by reading the file, BD = weio.read(BD_filename) - OUTUPTS: + OUTPUTS: - r_nodes: spanwise position from the balde root of the Blade stations """ + GAUSS_QUADRATURE = 1 + TRAP_QUADRATURE = 2 if hasattr(BD,'startswith'): # if string BD = FASTInputFile(BD) - if hasattr(BD,'startswith'): # if string + if hasattr(BDBld,'startswith'): # if string BDBld = FASTInputFile(BDBld) # BD['BldFile'].replace('"','')) + + + # --- Extract relevant info from BD files z_kp = BD['MemberGeom'][:,2] R = z_kp[-1]-z_kp[0] - r = BDBld['BeamProperties']['span']*R + + nStations = BDBld['station_total'] + rStations = BDBld['BeamProperties']['span']*R quad = BD['quadrature'] - ref = BD['refine'] - if 'default' in str(ref).lower(): - ref = 1 - dr = np.diff(r)/ref - rmid = np.concatenate( [r[:-1]+dr*(iref+1) for iref in np.arange(ref-1) ]) - r = np.concatenate( (r, rmid)) - r = np.unique(np.sort(r)) + + refine = BD['refine'] + nodes_per_elem = BD['order_elem'] + 1 + if 'default' in str(refine).lower(): + refine = 1 + + # --- Distribution of points + if quad==GAUSS_QUADRATURE: + # See BD_GaussPointWeight + # Number of Gauss points + nqp = nodes_per_elem #- 1 + # qp_indx_offset = 1 ! we skip the first node on the input mesh (AD needs values at the end points, but BD doesn't use them) + x, _ = np.polynomial.legendre.leggauss(nqp) + r= R*(1+x)/2 + + elif quad==TRAP_QUADRATURE: + # See BD_TrapezoidalPointWeight + nqp = (nStations - 1)*refine + 1 + # qp_indx_offset = 0 + # BldMotionNodeLoc = BD_MESH_QP ! we want to output y%BldMotion at the blade input property stations, and this will be a short-cut + dr = np.diff(rStations)/refine + rmid = np.concatenate( [rStations[:-1]+dr*(iref+1) for iref in np.arange(refine-1) ]) + r = np.concatenate( (rStations, rmid)) + r = np.unique(np.sort(r)) + else: + + raise NotImplementedError('BeamDyn with Gaussian quadrature points') + return r def BD_BldGag(BD): @@ -1401,7 +1433,7 @@ def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKee def integrateMoment(r, F): - """ + r""" Integrate moment from force and radial station M_j = \int_{r_j}^(r_n) f(r) * (r-r_j) dr for j=1,nr TODO: integrate analytically the "r" part @@ -1412,7 +1444,7 @@ def integrateMoment(r, F): return M def integrateMomentTS(r, F): - """ + r""" Integrate moment from time series of forces at nr radial stations Compute diff --git a/pydatview/fast/runner.py b/pydatview/fast/runner.py index 385962b..c8faf9f 100644 --- a/pydatview/fast/runner.py +++ b/pydatview/fast/runner.py @@ -1,5 +1,4 @@ # --- For cmd.py -from __future__ import division, print_function import os import subprocess import multiprocessing diff --git a/pydatview/io/bladed_out_file.py b/pydatview/io/bladed_out_file.py index 1f7a6c1..f08e2d1 100644 --- a/pydatview/io/bladed_out_file.py +++ b/pydatview/io/bladed_out_file.py @@ -1,17 +1,15 @@ -# -*- coding: utf-8 -*- import os import numpy as np import re import pandas as pd import glob import shlex -# try: -from .file import File, WrongFormatError, BrokenFormatError, isBinary -# except: -# EmptyFileError = type('EmptyFileError', (Exception,),{}) -# WrongFormatError = type('WrongFormatError', (Exception,),{}) -# BrokenFormatError = type('BrokenFormatError', (Exception,),{}) -# File=dict +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass # --------------------------------------------------------------------------------} @@ -371,6 +369,21 @@ def toDataFrame(self): else: return dfs + +def isBinary(filename): + with open(filename, 'r') as f: + try: + # first try to read as string + l = f.readline() + # then look for weird characters + for c in l: + code = ord(c) + if code<10 or (code>14 and code<31): + return True + return False + except UnicodeDecodeError: + return True + if __name__ == '__main__': pass #filename = r'E:\Work_Google Drive\Bladed_Sims\Bladed_out_binary.$41' diff --git a/pydatview/io/bmodes_out_file.py b/pydatview/io/bmodes_out_file.py index 337d284..7d99546 100644 --- a/pydatview/io/bmodes_out_file.py +++ b/pydatview/io/bmodes_out_file.py @@ -8,10 +8,10 @@ try: from .file import File, WrongFormatError, BrokenFormatError except: + File=dict EmptyFileError = type('EmptyFileError', (Exception,),{}) WrongFormatError = type('WrongFormatError', (Exception,),{}) BrokenFormatError = type('BrokenFormatError', (Exception,),{}) - File=dict class BModesOutFile(File): """ diff --git a/pydatview/io/cactus_file.py b/pydatview/io/cactus_file.py index 272e575..265d0d8 100644 --- a/pydatview/io/cactus_file.py +++ b/pydatview/io/cactus_file.py @@ -5,10 +5,10 @@ try: from .file import File, WrongFormatError, BrokenFormatError, EmptyFileError except: + File=dict EmptyFileError = type('EmptyFileError', (Exception,),{}) WrongFormatError = type('WrongFormatError', (Exception,),{}) BrokenFormatError = type('BrokenFormatError', (Exception,),{}) - File=dict class CactusFile(File): diff --git a/pydatview/io/csv_file.py b/pydatview/io/csv_file.py index 1b45578..18cdcbf 100644 --- a/pydatview/io/csv_file.py +++ b/pydatview/io/csv_file.py @@ -1,8 +1,3 @@ -from __future__ import division,unicode_literals,print_function,absolute_import -from builtins import map, range, chr, str -from io import open -from future import standard_library -standard_library.install_aliases() import os from .file import File, WrongFormatError diff --git a/pydatview/io/excel_file.py b/pydatview/io/excel_file.py index df28931..0b9c9ab 100644 --- a/pydatview/io/excel_file.py +++ b/pydatview/io/excel_file.py @@ -1,9 +1,3 @@ -from __future__ import division,unicode_literals,print_function,absolute_import -from builtins import map, range, chr, str -from io import open -from future import standard_library -standard_library.install_aliases() - from .file import File, WrongFormatError, BrokenFormatError import numpy as np import pandas as pd diff --git a/pydatview/io/fast_input_deck.py b/pydatview/io/fast_input_deck.py index 48531e7..fe39428 100644 --- a/pydatview/io/fast_input_deck.py +++ b/pydatview/io/fast_input_deck.py @@ -1,12 +1,3 @@ -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from io import open -from builtins import range -from builtins import str -from future import standard_library -standard_library.install_aliases() import os import numpy as np import re diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index dc944ba..5881846 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -1,25 +1,13 @@ -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from io import open -from builtins import range -from builtins import str -from future import standard_library -standard_library.install_aliases() +import numpy as np +import os +import pandas as pd +import re try: from .file import File, WrongFormatError, BrokenFormatError except: - # --- Allowing this file to be standalone.. - class WrongFormatError(Exception): - pass - class BrokenFormatError(Exception): - pass File = dict -import os -import numpy as np -import re -import pandas as pd + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass __all__ = ['FASTInputFile'] @@ -79,9 +67,6 @@ def module(self): return self.basefile.module else: return self._fixedfile.module - @property - def hasNodal(self): - return self.fixedfile.hasNodal def fixedFormat(self): # --- Creating a dedicated Child @@ -176,7 +161,6 @@ class FASTInputFileBase(File): def __init__(self, filename=None, **kwargs): self._size=None - self._encoding=None self.setData() # Init data if filename: self.filename = filename @@ -931,26 +915,6 @@ def readBeamDynProps(self,lines,iStart): # --- Helper functions # --------------------------------------------------------------------------------{ def isStr(s): - # Python 2 and 3 compatible - # Two options below - # NOTE: all this avoided since we import str from builtins - # --- Version 2 - # isString = False; - # if(isinstance(s, str)): - # isString = True; - # try: - # if(isinstance(s, basestring)): # todo unicode as well - # isString = True; - # except NameError: - # pass; - # return isString - # --- Version 1 - # try: - # basestring # python 2 - # return isinstance(s, basestring) or isinstance(s,unicode) - # except NameError: - # basestring=str #python 3 - # return isinstance(s, str) return isinstance(s, str) def strIsFloat(s): diff --git a/pydatview/io/fast_linearization_file.py b/pydatview/io/fast_linearization_file.py index e604d41..af23860 100644 --- a/pydatview/io/fast_linearization_file.py +++ b/pydatview/io/fast_linearization_file.py @@ -1,11 +1,12 @@ -from __future__ import division -from __future__ import print_function -from __future__ import absolute_import -from io import open -from .file import File, isBinary, WrongFormatError, BrokenFormatError -import pandas as pd import numpy as np +import pandas as pd import re +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass class FASTLinearizationFile(File): """ diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index 95947c2..e3b3b1a 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -1,37 +1,20 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from io import open -from builtins import map -from builtins import range -from builtins import chr -from builtins import str -from future import standard_library -standard_library.install_aliases() - from itertools import takewhile - +import numpy as np +import pandas as pd +import struct +import os +import re try: from .file import File, WrongFormatError, BrokenReaderError, EmptyFileError except: - # --- Allowing this file to be standalone.. - class WrongFormatError(Exception): - pass - class WrongReaderError(Exception): - pass - class EmptyFileError(Exception): - pass File = dict + class WrongFormatError(Exception): pass + class WrongReaderError(Exception): pass + class EmptyFileError(Exception): pass try: from .csv_file import CSVFile except: print('CSVFile not available') -import numpy as np -import pandas as pd -import struct -import os -import re # --------------------------------------------------------------------------------} diff --git a/pydatview/io/fast_summary_file.py b/pydatview/io/fast_summary_file.py index d3004d5..735c702 100644 --- a/pydatview/io/fast_summary_file.py +++ b/pydatview/io/fast_summary_file.py @@ -1,6 +1,5 @@ import numpy as np import pandas as pd -from io import open import os # Local from .mini_yaml import yaml_read diff --git a/pydatview/io/fast_wind_file.py b/pydatview/io/fast_wind_file.py index b642d89..e2a75c5 100644 --- a/pydatview/io/fast_wind_file.py +++ b/pydatview/io/fast_wind_file.py @@ -1,19 +1,7 @@ -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from io import open -from builtins import map -from builtins import range -from builtins import chr -from builtins import str -from future import standard_library -standard_library.install_aliases() - -from .csv_file import CSVFile -from .file import isBinary, WrongFormatError import numpy as np import pandas as pd +from .csv_file import CSVFile +from .file import isBinary, WrongFormatError class FASTWndFile(CSVFile): diff --git a/pydatview/io/file.py b/pydatview/io/file.py index aa432a3..9c00f99 100644 --- a/pydatview/io/file.py +++ b/pydatview/io/file.py @@ -155,7 +155,6 @@ def test_ascii(self,bCompareWritesOnly=False,bDelete=True): # --- Helper functions # --------------------------------------------------------------------------------{ def isBinary(filename): - from io import open with open(filename, 'r') as f: try: # first try to read as string diff --git a/pydatview/io/flex_blade_file.py b/pydatview/io/flex_blade_file.py index 4442ffa..5ce5a0e 100644 --- a/pydatview/io/flex_blade_file.py +++ b/pydatview/io/flex_blade_file.py @@ -1,16 +1,13 @@ -from __future__ import division,unicode_literals,print_function,absolute_import -from builtins import map, range, chr, str -from io import open -from future import standard_library -standard_library.install_aliases() - -from .file import File, WrongFormatError, BrokenFormatError import numpy as np import pandas as pd import os -#from .wetb.fast import fast_io - +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass class FLEXBladeFile(File): diff --git a/pydatview/io/flex_doc_file.py b/pydatview/io/flex_doc_file.py index d3e656d..46eef7b 100644 --- a/pydatview/io/flex_doc_file.py +++ b/pydatview/io/flex_doc_file.py @@ -1,14 +1,13 @@ -from __future__ import division,unicode_literals,print_function,absolute_import -from builtins import map, range, chr, str -from io import open -from future import standard_library -standard_library.install_aliases() - -from .file import File, WrongFormatError, BrokenFormatError import numpy as np import pandas as pd import os import re +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass class FLEXDocFile(File): diff --git a/pydatview/io/flex_out_file.py b/pydatview/io/flex_out_file.py index d9c95e5..5352351 100644 --- a/pydatview/io/flex_out_file.py +++ b/pydatview/io/flex_out_file.py @@ -1,23 +1,12 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from io import open -from builtins import map -from builtins import range -from builtins import chr -from builtins import str -from future import standard_library -standard_library.install_aliases() - -from .file import File, WrongFormatError, BrokenFormatError import numpy as np import pandas as pd import os - -#from .wetb.fast import fast_io - - +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass # --------------------------------------------------------------------------------} # --- OUT FILE diff --git a/pydatview/io/flex_profile_file.py b/pydatview/io/flex_profile_file.py index 9a29f17..90c4fb8 100644 --- a/pydatview/io/flex_profile_file.py +++ b/pydatview/io/flex_profile_file.py @@ -1,15 +1,12 @@ -from __future__ import division,unicode_literals,print_function,absolute_import -from builtins import map, range, chr, str -from io import open -from future import standard_library -standard_library.install_aliases() - -from .file import File, WrongFormatError, BrokenFormatError import numpy as np import pandas as pd import os - -#from .wetb.fast import fast_io +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass class ProfileSet(): def __init__(self,header,thickness,polars,polar_headers): diff --git a/pydatview/io/flex_wavekin_file.py b/pydatview/io/flex_wavekin_file.py index 684e8f8..d6cdf72 100644 --- a/pydatview/io/flex_wavekin_file.py +++ b/pydatview/io/flex_wavekin_file.py @@ -1,17 +1,14 @@ -from __future__ import division,unicode_literals,print_function,absolute_import -from builtins import map, range, chr, str -from io import open -from future import standard_library -standard_library.install_aliases() - -from .file import File, WrongFormatError, BrokenFormatError -from .csv_file import CSVFile import numpy as np import pandas as pd import os import re - -#from .wetb.fast import fast_io +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass +from .csv_file import CSVFile class FLEXWaveKinFile(File): diff --git a/pydatview/io/hawc2_dat_file.py b/pydatview/io/hawc2_dat_file.py index 3a3bab4..7ee37f6 100644 --- a/pydatview/io/hawc2_dat_file.py +++ b/pydatview/io/hawc2_dat_file.py @@ -1,20 +1,8 @@ -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from io import open -from builtins import map -from builtins import range -from builtins import chr -from builtins import str -from future import standard_library -standard_library.install_aliases() -import os import numpy as np - -from .file import File, WrongFormatError, FileNotFoundError import pandas as pd +import os +from .file import File, WrongFormatError, FileNotFoundError from .wetb.hawc2.Hawc2io import ReadHawc2 diff --git a/pydatview/io/hawcstab2_ind_file.py b/pydatview/io/hawcstab2_ind_file.py index 3d41940..ea15db3 100644 --- a/pydatview/io/hawcstab2_ind_file.py +++ b/pydatview/io/hawcstab2_ind_file.py @@ -1,20 +1,12 @@ -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from io import open -from builtins import map -from builtins import range -from builtins import chr -from builtins import str -from future import standard_library -standard_library.install_aliases() -import os -import re - -from .file import File, WrongFormatError import numpy as np import pandas as pd +import os +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass class HAWCStab2IndFile(File): diff --git a/pydatview/io/hawcstab2_pwr_file.py b/pydatview/io/hawcstab2_pwr_file.py index add7a22..8fc1376 100644 --- a/pydatview/io/hawcstab2_pwr_file.py +++ b/pydatview/io/hawcstab2_pwr_file.py @@ -1,18 +1,11 @@ -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from io import open -from builtins import map -from builtins import range -from builtins import chr -from builtins import str -from future import standard_library -standard_library.install_aliases() - -from .file import File, WrongFormatError import numpy as np import pandas as pd +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass class HAWCStab2PwrFile(File): diff --git a/pydatview/io/mini_yaml.py b/pydatview/io/mini_yaml.py index 04269b9..2768f02 100644 --- a/pydatview/io/mini_yaml.py +++ b/pydatview/io/mini_yaml.py @@ -1,6 +1,3 @@ -from __future__ import unicode_literals -from __future__ import print_function -from io import open import numpy as np def yaml_read(filename,dictIn=None): diff --git a/pydatview/io/netcdf_file.py b/pydatview/io/netcdf_file.py index e54e412..5fe5c1a 100644 --- a/pydatview/io/netcdf_file.py +++ b/pydatview/io/netcdf_file.py @@ -1,19 +1,11 @@ -from __future__ import division -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import absolute_import -from io import open -from builtins import map -from builtins import range -from builtins import chr -from builtins import str -from future import standard_library -standard_library.install_aliases() - -from .file import File, WrongFormatError import pandas as pd -#import xarray as xr # +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass class NetCDFFile(File): diff --git a/pydatview/io/tools/graph.py b/pydatview/io/tools/graph.py index e924ef1..04d7eb3 100644 --- a/pydatview/io/tools/graph.py +++ b/pydatview/io/tools/graph.py @@ -602,7 +602,6 @@ def toJSON(self,outfile=None): if outfile is not None: import json - from io import open jsonFile=outfile with open(jsonFile, 'w', encoding='utf-8') as f: #f.write(to_json(d)) diff --git a/pydatview/io/turbsim_file.py b/pydatview/io/turbsim_file.py index ef9d8a0..0b421c5 100644 --- a/pydatview/io/turbsim_file.py +++ b/pydatview/io/turbsim_file.py @@ -29,7 +29,10 @@ class TurbSimFile(File): Main methods ------------ - - read, write, toDataFrame, keys, valuesAt, makePeriodic, checkPeriodic, closestPoint + - read, write, toDataFrame, keys + - valuesAt, vertProfile, horizontalPlane, verticalPlane, closestPoint + - fitPowerLaw + - makePeriodic, checkPeriodic Examples -------- @@ -187,8 +190,8 @@ def valuesAt(self, y, z, method='nearest'): if method == 'nearest': iy, iz = self.closestPoint(y, z) u = self['u'][0,:,iy,iz] - v = self['u'][0,:,iy,iz] - w = self['u'][0,:,iy,iz] + v = self['u'][1,:,iy,iz] + w = self['u'][2,:,iy,iz] else: raise NotImplementedError() return u, v, w @@ -324,11 +327,21 @@ def verticalPlane(ts, y=None, iy0=None, removeMean=False): # --------------------------------------------------------------------------------} # --- Extracting average data # --------------------------------------------------------------------------------{ - @property - def vertProfile(self): - iy, iz = self.iMid - m = np.mean(self['u'][:,:,iy,:], axis=1) - s = np.std( self['u'][:,:,iy,:], axis=1) + def vertProfile(self, y_span='full'): + """ Vertical profile of the box + INPUTS: + - y_span: if 'full', average the vertical profile accross all y-values + if 'mid', average the vertical profile at the middle y value + """ + if y_span=='full': + m = np.mean(np.mean(self['u'][:,:,:,:], axis=1), axis=1) + s = np.std( np.std( self['u'][:,:,:,:], axis=1), axis=1) + elif y_span=='mid': + iy, iz = self.iMid + m = np.mean(self['u'][:,:,iy,:], axis=1) + s = np.std( self['u'][:,:,iy,:], axis=1) + else: + raise NotImplementedError() return self.z, m, s @@ -493,7 +506,7 @@ def scale(self, new_mean=None, new_std=None, component=0, reference='mid', y_ref print('New std : {:7.3f} (target: {:7.3f}, old: {:7.3f})'.format(new_std2 , new_std , old_std)) def makePeriodic(self): - """ Make the box periodic by mirroring it """ + """ Make the box periodic in the streamwise direction by mirroring it """ nDim, nt0, ny, nz = self['u'].shape u = self['u'].copy() del self['u'] @@ -572,7 +585,7 @@ def toDataFrame(self): iy,iz = self.iMid # Mean vertical profile - z, m, s = self.vertProfile + z, m, s = self.vertProfile() ti = s/m*100 cols=['z_[m]','u_[m/s]','v_[m/s]','w_[m/s]','sigma_u_[m/s]','sigma_v_[m/s]','sigma_w_[m/s]','TI_[%]'] data = np.column_stack((z, m[0,:],m[1,:],m[2,:],s[0,:],s[1,:],s[2,:],ti[0,:])) @@ -721,36 +734,25 @@ def toMannBox(self, base=None, removeUConstant=None, removeAllUMean=False): # --- Useful IO def writeInfo(ts, filename): """ Write info to txt """ - import scipy.optimize as so - def fit_powerlaw_u_alpha(x, y, z_ref=100, p0=(10,0.1)): - """ - p[0] : u_ref - p[1] : alpha - """ - pfit, _ = so.curve_fit(lambda x, *p : p[0] * (x / z_ref) ** p[1], x, y, p0=p0) - y_fit = pfit[0] * (x / z_ref) ** pfit[1] - coeffs_dict={'u_ref':pfit[0],'alpha':pfit[1]} - formula = '{u_ref} * (z / {z_ref}) ** {alpha}' - fitted_fun = lambda xx: pfit[0] * (xx / z_ref) ** pfit[1] - return y_fit, pfit, {'coeffs':coeffs_dict,'formula':formula,'fitted_function':fitted_fun} infofile = filename with open(filename,'w') as f: f.write(str(ts)) zMid =(ts['z'][0]+ts['z'][-1])/2 f.write('Middle height of box: {:.3f}\n'.format(zMid)) - - iy,_ = ts.iMid - u = np.mean(ts['u'][0,:,iy,:], axis=0) - z=ts['z'] - f.write('\n') - y_fit, pfit, model = fit_powerlaw_u_alpha(z, u, z_ref=zMid, p0=(10,0.1)) + y_fit, pfit, model, _ = ts.fitPowerLaw(z_ref=zMid, y_span='mid', U_guess=10, alpha_guess=0.1) f.write('Power law: alpha={:.5f} - u={:.5f} at z={:.5f}\n'.format(pfit[1],pfit[0],zMid)) f.write('Periodic: {}\n'.format(ts.checkPeriodic(sigmaTol=1.5, aTol=0.5))) def writeProbes(ts, probefile, yProbe, zProbe): - # Creating csv file with data at some probe locations + """ Create a CSV file with wind speed data at given probe locations + defined by the vectors yProbe and zProbe. All combinations of y and z are extracted. + INPUTS: + - probefile: filename of CSV file to be written + - yProbe: array like of y locations + - zProbe: array like of z locations + """ Columns=['Time_[s]'] Data = ts['t'] for y in yProbe: @@ -763,8 +765,35 @@ def writeProbes(ts, probefile, yProbe, zProbe): Data = np.column_stack((Data, DataSub)) np.savetxt(probefile, Data, header=','.join(Columns), delimiter=',') - - + def fitPowerLaw(ts, z_ref=None, y_span='full', U_guess=10, alpha_guess=0.1): + """ + Fit power law to vertical profile + INPUTS: + - z_ref: reference height used to define the "U_ref" + - y_span: if 'full', average the vertical profile accross all y-values + if 'mid', average the vertical profile at the middle y value + """ + if z_ref is None: + # use mid height for z_ref + z_ref =(ts['z'][0]+ts['z'][-1])/2 + # Average time series + z, u, _ = ts.vertProfile(y_span=y_span) + u = u[0,:] + u_fit, pfit, model = fit_powerlaw_u_alpha(z, u, z_ref=z_ref, p0=(U_guess, alpha_guess)) + return u_fit, pfit, model, z_ref + +def fit_powerlaw_u_alpha(x, y, z_ref=100, p0=(10,0.1)): + """ + p[0] : u_ref + p[1] : alpha + """ + import scipy.optimize as so + pfit, _ = so.curve_fit(lambda x, *p : p[0] * (x / z_ref) ** p[1], x, y, p0=p0) + y_fit = pfit[0] * (x / z_ref) ** pfit[1] + coeffs_dict={'u_ref':pfit[0],'alpha':pfit[1]} + formula = '{u_ref} * (z / {z_ref}) ** {alpha}' + fitted_fun = lambda xx: pfit[0] * (xx / z_ref) ** pfit[1] + return y_fit, pfit, {'coeffs':coeffs_dict,'formula':formula,'fitted_function':fitted_fun} if __name__=='__main__': ts = TurbSimFile('../_tests/TurbSim.bts') diff --git a/pydatview/io/turbsim_ts_file.py b/pydatview/io/turbsim_ts_file.py index 6d573a7..85be93b 100644 --- a/pydatview/io/turbsim_ts_file.py +++ b/pydatview/io/turbsim_ts_file.py @@ -1,11 +1,12 @@ -from __future__ import division -from __future__ import print_function -from __future__ import absolute_import -from io import open -from .file import File, isBinary, WrongFormatError, BrokenFormatError +from itertools import takewhile import pandas as pd import numpy as np -from itertools import takewhile +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass class TurbSimTSFile(File): diff --git a/pydatview/io/wetb/hawc2/Hawc2io.py b/pydatview/io/wetb/hawc2/Hawc2io.py index 192ab50..4484bae 100644 --- a/pydatview/io/wetb/hawc2/Hawc2io.py +++ b/pydatview/io/wetb/hawc2/Hawc2io.py @@ -27,17 +27,6 @@ * add error handling for allmost every thing """ -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import absolute_import -from builtins import int -from builtins import range -from io import open as opent -from builtins import str -from future import standard_library -standard_library.install_aliases() -from builtins import object import numpy as np import os @@ -80,7 +69,7 @@ def _ReadSelFile(self): # read *.sel hawc2 output file for result info if self.FileName.lower().endswith('.sel'): self.FileName = self.FileName[:-4] - fid = opent(self.FileName + '.sel', 'r') + fid = open(self.FileName + '.sel', 'r') Lines = fid.readlines() fid.close() if Lines[0].lower().find('bhawc')>=0: diff --git a/pydatview/io/wetb/hawc2/__init__.py b/pydatview/io/wetb/hawc2/__init__.py index 1cd84f9..f58b7bc 100644 --- a/pydatview/io/wetb/hawc2/__init__.py +++ b/pydatview/io/wetb/hawc2/__init__.py @@ -1,9 +1,3 @@ -# from __future__ import unicode_literals -# from __future__ import print_function -# from __future__ import division -# from __future__ import absolute_import -# from future import standard_library -# standard_library.install_aliases() # d = None # d = dir() # diff --git a/pydatview/main.py b/pydatview/main.py index 1dbef42..8a54a26 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -1,9 +1,3 @@ -from __future__ import division, unicode_literals, print_function, absolute_import -from builtins import map, range, chr, str -from io import open -from future import standard_library -standard_library.install_aliases() - import numpy as np import os.path import sys @@ -728,9 +722,6 @@ def __init__(self, redirect=False, filename=None): /usr/lib/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX; - Your python is an anaconda python, use something like:; /anaconda3/bin/python.app (NOTE: the '.app'! - - You are using a python 2 version, you can use the system one: - /Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX - /System/Library/Frameworks/Python.framework/Versions/XXX/bin/pythonXXX """ elif wx.Platform == '__WXGTK__': diff --git a/pydatview/perfmon.py b/pydatview/perfmon.py index 879e243..762b83d 100644 --- a/pydatview/perfmon.py +++ b/pydatview/perfmon.py @@ -1,5 +1,3 @@ - -from __future__ import print_function import numpy as np import time import os diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 0c2b37b..edfe5c2 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import os import numpy as np from .common import no_unit, unit, inverse_unit, has_chinese_char diff --git a/pydatview/tools/damping.py b/pydatview/tools/damping.py index 0264935..f1340f1 100644 --- a/pydatview/tools/damping.py +++ b/pydatview/tools/damping.py @@ -1,4 +1,3 @@ -from __future__ import division, print_function import numpy as np __all__ = ['logDecFromDecay'] diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index 3c53d9d..2b70be2 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -24,13 +24,7 @@ - 'rainflow_astm' (based on the c-implementation by Adam Nieslony found at the MATLAB Central File Exchange http://www.mathworks.com/matlabcentral/fileexchange/3026) ''' -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import absolute_import -from future import standard_library import warnings -standard_library.install_aliases() import numpy as np diff --git a/pydatview/tools/signal_analysis.py b/pydatview/tools/signal_analysis.py index 54fdab1..6babc09 100644 --- a/pydatview/tools/signal_analysis.py +++ b/pydatview/tools/signal_analysis.py @@ -3,7 +3,6 @@ NOTE: naming this module "signal.py" can sometimes create conflict with numpy """ -from __future__ import division import numpy as np from numpy.random import rand import pandas as pd @@ -430,6 +429,111 @@ def find_time_offset(t, f, g, outputAll=False): else: return t_offset +def amplitude(x, t=None, T = None, mask=None, debug=False): + """ + Compute signal amplitude (max-min)/2. + If a frequency is provided, the calculation is the average on each period + + x: signal time series + mask - time at which transient starts + """ + if mask is not None: + x = x[mask] + if t is not None: + t = t[mask] + # + if T is not None and t is not None: + t -= t[0] + if t[-1]<=T: + return (np.max(x)-np.min(x))/2 + n = int(t[-1]/T) + A = 0 + for i in range(n): + b = np.logical_and(t<=(i+1)*T , t>=i*T) + A+=(np.max(x[b])-np.min(x[b]))/2 + A/=n + + if debug: + import matplotlib.pyplot as plt + from welib.tools.colors import python_colors + fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) + fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) + ax.plot(t, x-np.mean(x) ,'k-', lw=3, label='Original') + for i in range(n): + b = np.logical_and(t<=(i+1)*T , t>=i*T) + A=(np.max(x[b])-np.min(x[b]))/2 + ax.plot(t[b]- i*T, x[b]-np.mean(x[b]), c=python_colors(i), label='A={}'.format(A)) + ax.plot([0,0,T,T,0],[-A,A,A,-A,-A] ,'--', c=python_colors(i) ) + ax.set_xlabel('time') + ax.set_ylabel('x') + ax.legend() + return A + + # split signals into subsets + import pdb; pdb.set_trace() + else: + return (np.max(x)-np.min(x))/2 + +def phase_shift(A, B, t, omega, tStart=0, deg=True, debug=False): + """ + A: reference signal + omega: expected frequency of reference signal + """ + b =t>=tStart + t = t[b] + A = A[b] + B = B[b] + t_offset, lag, xcorr = find_time_offset(t, A, B, outputAll=True) + phi0 = t_offset*omega # Phase offset in radians + phi = np.mod(phi0, 2*np.pi) + if phi > 0.8 * np.pi: + phi = phi-2*np.pi + if deg: + phi *=180/np.pi + phi0*=180/np.pi + if phi<-190: + phi+=360 +# if debug: +# raise NotImplementedError() + return phi + +def input_output_amplitude_phase(t, u, y, omega_u=None, A_u=None, deg=True, mask=None, debug=False): + """ + Return amplitude ratio and phase shift between a reference input `u` and output `y` + Typically used when the input is a sinusoidal signal and the output is "similar". + + INPUTS: + - t: time vector, length nt + - u: input time series, length nt + - y: output time series, length nt + - omega_u: cyclic frequency, required to convert time offset to phase + when provided, amplitude ratio is computed for each possible periods, and averaged + - A_u : amplitude of input signal (typically known if y is a sinusoid) + - deg: phase is returned in degrees + - mask: mask to be applied to t, u, y + """ + if mask is not None: + t = t[mask] + u = u[mask] + y = y[mask] + if omega_u is None: + raise NotImplementedError() + T=None + else: + T = 2*np.pi/omega_u + + # --- Amplitude ratio + A_y = amplitude(y, t, T=T, debug=debug) + if A_u is None: + A_u = amplitude(u, t, T=T) + G = A_y/A_u + + # --- Phase shift + phi = phase_shift(u, y, t, omega_u, deg=deg, debug=debug) + + return G, phi + + def sine_approx(t, x, method='least_square'): """ Sinusoidal approximation of input signal x diff --git a/pydatview/tools/spectral.py b/pydatview/tools/spectral.py index fa71395..245cb7a 100644 --- a/pydatview/tools/spectral.py +++ b/pydatview/tools/spectral.py @@ -17,8 +17,6 @@ # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import division, print_function, absolute_import - import numpy as np import pandas as pd from six import string_types diff --git a/requirements.txt b/requirements.txt index aad0a7c..fb898b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,8 @@ -openpyxl ; python_version>"3.0" -numpy>=1.16.5; python_version>"3.0" -numpy ; python_version<"3.0" -pandas>=1.0.1; python_version>"3.0" -pandas; python_version<="3.0" -xlrd==1.2.0; python_version<"3.0" +openpyxl +numpy +pandas pyarrow # for parquet files matplotlib -future chardet scipy wxpython diff --git a/tests/prof_all.py b/tests/prof_all.py index 66d187a..5f0188c 100644 --- a/tests/prof_all.py +++ b/tests/prof_all.py @@ -1,7 +1,4 @@ -from __future__ import absolute_import - - def test_heavy(): import time import sys diff --git a/tests/test_common.py b/tests/test_common.py index 1e5ab9b..17f61ff 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,5 +1,3 @@ -# - *- coding: utf- 8 - *- -from __future__ import unicode_literals,print_function import unittest import numpy as np import pandas as pd From 7ee999c8a6154ea0b97a8aaf875f44af9f355b59 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sat, 6 Aug 2022 13:42:09 -0600 Subject: [PATCH 024/178] FASTInputFile: reading of polars without airfoil coefficients --- pydatview/io/fast_input_file.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index 5881846..fa5642f 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -1571,17 +1571,28 @@ def _toDataFrame(self): alpha = df['Alpha_[deg]'].values*np.pi/180. Cl = df['Cl_[-]'].values Cd = df['Cd_[-]'].values - Cd0 = self['Cd0'+labOffset] - # Cn (with or without Cd0) - Cn1 = Cl*np.cos(alpha)+ (Cd-Cd0)*np.sin(alpha) + + # Cn with Cd0 + try: + Cd0 = self['Cd0'+labOffset] + # Cn (with or without Cd0) + Cn1 = Cl*np.cos(alpha)+ (Cd-Cd0)*np.sin(alpha) + df['Cn_Cd0off_[-]'] = Cn1 + except: + pass + + # Regular Cn Cn = Cl*np.cos(alpha)+ Cd*np.sin(alpha) df['Cn_[-]'] = Cn - df['Cn_Cd0off_[-]'] = Cn1 - CnLin = self['C_nalpha'+labOffset]*(alpha-self['alpha0'+labOffset]*np.pi/180.) - CnLin[alpha<-20*np.pi/180]=np.nan - CnLin[alpha> 30*np.pi/180]=np.nan - df['Cn_pot_[-]'] = CnLin + # Linear Cn + try: + CnLin = self['C_nalpha'+labOffset]*(alpha-self['alpha0'+labOffset]*np.pi/180.) + CnLin[alpha<-20*np.pi/180]=np.nan + CnLin[alpha> 30*np.pi/180]=np.nan + df['Cn_pot_[-]'] = CnLin + except: + pass # Highlighting points surrounding 0 1 2 Cn points CnPoints = Cn*np.nan From dc60669c4418995a85432830cb64fe62ee4041ab Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 8 Aug 2022 17:53:13 -0600 Subject: [PATCH 025/178] Fix: hasNodal missing from ED file --- pydatview/io/fast_input_file.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index fa5642f..fbf645b 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -61,6 +61,7 @@ def fixedfile(self): return self._fixedfile else: return self.basefile + @property def module(self): if self._fixedfile is None: @@ -68,6 +69,20 @@ def module(self): else: return self._fixedfile.module + @property + def hasNodal(self): + if self._fixedfile is None: + return self.basefile.hasNodal + else: + return self._fixedfile.hasNodal + + def getID(self, label): + return self.basefile.getID(label) + + @property + def data(self): + return self.basefile.data + def fixedFormat(self): # --- Creating a dedicated Child KEYS = list(self.basefile.keys()) From ccd9c17eb9b565467fc9ec1756c62a92f6b2bf04 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 8 Aug 2022 19:25:43 -0600 Subject: [PATCH 026/178] Bug: failed to find font, attempt with rollback --- installer.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer.cfg b/installer.cfg index d1b0d3c..30f73c2 100644 --- a/installer.cfg +++ b/installer.cfg @@ -80,7 +80,7 @@ exclude= pkgs/pandas/tests pkgs/matplotlib/sphinxext pkgs/matplotlib/testing - pkgs/matplotlib/mpl-data/fonts +# pkgs/matplotlib/mpl-data/fonts pkgs/matplotlib/mpl-data/sample_data pkgs/matplotlib/mpl-data/images/*.pdf pkgs/matplotlib/mpl-data/images/*.svg From 7364f93706af3383133cd5885c1d4b0ee8449dd2 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 11 Aug 2022 11:04:11 -0600 Subject: [PATCH 027/178] Update of TDMS file (see #21) --- pydatview/io/fast_input_file.py | 2 +- pydatview/io/tdms_file.py | 182 ++++++++++++++++++++------------ 2 files changed, 118 insertions(+), 66 deletions(-) diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index fbf645b..8e9a190 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -128,7 +128,7 @@ def comment(self): @comment.setter def comment(self,comment): - self.fixedfile.comment(comment) + self.fixedfile.comment = comment def __iter__(self): return self.fixedfile.__iter__() diff --git a/pydatview/io/tdms_file.py b/pydatview/io/tdms_file.py index d05e1a9..2d21bdb 100644 --- a/pydatview/io/tdms_file.py +++ b/pydatview/io/tdms_file.py @@ -1,65 +1,117 @@ -from .file import File, WrongFormatError, BrokenFormatError -import numpy as np -import pandas as pd - -class TDMSFile(File): - - @staticmethod - def defaultExtensions(): - return ['.tdms'] - - @staticmethod - def formatName(): - return 'TDMS file' - - def _read(self): - try: - from nptdms import TdmsFile - except: - raise Exception('Install the library nptdms to read this file') - - fh = TdmsFile(self.filename, read_metadata_only=False) - channels_address = list(fh.objects.keys()) - channels_address = [ s.replace("'",'') for s in channels_address] - channel_keys= [ s.split('/')[1:] for s in channels_address if len(s.split('/'))==3] - # --- Setting up list of signals and times - signals=[] - times=[] - for i,ck in enumerate(channel_keys): - channel = fh.object(ck[0],ck[1]) - signals.append(channel.data) - times.append (channel.time_track()) - - lenTimes = [len(time) for time in times] - minTimes = [np.min(time) for time in times] - maxTimes = [np.max(time) for time in times] - if len(np.unique(lenTimes))>1: - print(lenTimes) - raise NotImplementedError('Different time length') - # NOTE: could use fh.as_dataframe - if len(np.unique(minTimes))>1: - print(minTimes) - raise NotImplementedError('Different time span') - if len(np.unique(maxTimes))>1: - print(maxTimes) - raise NotImplementedError('Different time span') - # --- Gathering into a data frame with time - time =times[0] - signals = [time]+signals - M = np.column_stack(signals) - colnames = ['Time_[s]'] + [ck[1] for ck in channel_keys] - self['data'] = pd.DataFrame(data = M, columns=colnames) - -# def toString(self): -# s='' -# return s -# def _write(self): -# pass - - def __repr__(self): - s ='Class TDMS (key: data)\n' - return s - - def _toDataFrame(self): - return self['data'] - +import numpy as np +import pandas as pd +import os + +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File = dict + class WrongFormatError(Exception): pass + class BrokenFormatError(Exception): pass + +class TDMSFile(File): + + @staticmethod + def defaultExtensions(): + return ['.tdms'] + + @staticmethod + def formatName(): + return 'TDMS file' + + def __init__(self, filename=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + try: + from nptdms import TdmsFile + except: + raise Exception('Install the library nptdms to read this file') + + fh = TdmsFile(self.filename, read_metadata_only=False) + # --- OLD, using some kind of old version of tdms and probably specific to one file + # channels_address = list(fh.objects.keys()) + # channels_address = [ s.replace("'",'') for s in channels_address] + # channel_keys= [ s.split('/')[1:] for s in channels_address if len(s.split('/'))==3] + # # --- Setting up list of signals and times + # signals=[] + # times=[] + # for i,ck in enumerate(channel_keys): + # channel = fh.object(ck[0],ck[1]) + # signals.append(channel.data) + # times.append (channel.time_track()) + + # lenTimes = [len(time) for time in times] + # minTimes = [np.min(time) for time in times] + # maxTimes = [np.max(time) for time in times] + # if len(np.unique(lenTimes))>1: + # print(lenTimes) + # raise NotImplementedError('Different time length') + # # NOTE: could use fh.as_dataframe + # if len(np.unique(minTimes))>1: + # print(minTimes) + # raise NotImplementedError('Different time span') + # if len(np.unique(maxTimes))>1: + # print(maxTimes) + # raise NotImplementedError('Different time span') + # # --- Gathering into a data frame with time + # time =times[0] + # signals = [time]+signals + # M = np.column_stack(signals) + # colnames = ['Time_[s]'] + [ck[1] for ck in channel_keys] + # self['data'] = pd.DataFrame(data = M, columns=colnames) + # --- NEW + self['data'] = fh + + @property + def groupNames(self): + return [group.name for group in self['data'].groups()] + + def __repr__(self): + s ='Class TDMS (key: data)\n' + s +=' - data: TdmsFile\n' + s +=' * groupNames: {}\n'.format(self.groupNames) + #for group in fh.groups(): + # for channel in group.channels(): + # print(group.name) + # print(channel.name) + return s + + def toDataFrame(self): + + df = self['data'].as_dataframe(time_index=True) + + # Cleanup columns + colnames = df.columns + colnames=[c.replace('\'','') for c in colnames] + colnames=[c[1:] if c.startswith('/') else c for c in colnames] + # If there is only one group, we remove the group key + groupNames = self.groupNames + if len(groupNames)==1: + nChar = len(groupNames[0]) + colnames=[c[nChar+1:] for c in colnames] # +1 for the "/" + + df.columns = colnames + + df.insert(0,'Time_[s]', df.index.values) + df.index=np.arange(0,len(df)) + + return df + +if __name__ == '__main__': + df = TDMSFile('DOE15_FastData_2019_11_19_13_51_35_50Hz.tdms').toDataFrame() + print(df) From b350987b691895a56fdb1335b9ea081398347413 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 15 Aug 2022 16:50:32 -0600 Subject: [PATCH 028/178] Simple option to merging tables, based on first common column (see #124) --- pydatview/GUISelectionPanel.py | 138 +++++++++++++++++++---------- pydatview/Tables.py | 57 ++++++++++-- pydatview/main.py | 15 ++-- pydatview/tools/signal_analysis.py | 31 +++++-- tests/test_Tables.py | 13 ++- tests/test_signal.py | 136 +++++++++++++++++++--------- 6 files changed, 280 insertions(+), 110 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index c3e13ac..55e1d54 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -1,12 +1,9 @@ import wx import platform -try: - from .common import * - from .GUICommon import * - from .GUIMultiSplit import MultiSplit - from .GUIToolBox import GetKeyString -except: - raise +from pydatview.common import * +from pydatview.GUICommon import * +from pydatview.GUIMultiSplit import MultiSplit +from pydatview.GUIToolBox import GetKeyString # from common import * # from GUICommon import * # from GUIMultiSplit import MultiSplit @@ -258,54 +255,88 @@ def onCancel(self, event): # --- Popup menus # --------------------------------------------------------------------------------{ class TablePopup(wx.Menu): - def __init__(self, mainframe, parent, fullmenu=False): + """ Popup Menu when right clicking on the table list """ + def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=False): wx.Menu.__init__(self) - self.parent = parent # parent is listbox + self.parent = parent # parent is listbox + self.tabPanel = tabPanel + self.tabList = tabPanel.tabList + self.selPanel = selPanel self.mainframe = mainframe self.ISel = self.parent.GetSelections() if fullmenu: self.itNameFile = wx.MenuItem(self, -1, "Naming: by file names", kind=wx.ITEM_CHECK) - self.MyAppend(self.itNameFile) + self.Append(self.itNameFile) self.Bind(wx.EVT_MENU, self.OnNaming, self.itNameFile) self.Check(self.itNameFile.GetId(), self.parent.GetParent().tabList.Naming=='FileNames') # Checking the menu box item = wx.MenuItem(self, -1, "Sort by name") - self.MyAppend(item) + self.Append(item) self.Bind(wx.EVT_MENU, self.OnSort, item) - item = wx.MenuItem(self, -1, "Add") - self.MyAppend(item) - self.Bind(wx.EVT_MENU, self.mainframe.onAdd, item) + if self.mainframe is not None: + item = wx.MenuItem(self, -1, "Add") + self.Append(item) + self.Bind(wx.EVT_MENU, self.mainframe.onAdd, item) + + if len(self.ISel)>1: + item = wx.MenuItem(self, -1, "Merge") + self.Append(item) + self.Bind(wx.EVT_MENU, self.OnMergeTabs, item) if len(self.ISel)>0: item = wx.MenuItem(self, -1, "Delete") - self.MyAppend(item) + self.Append(item) self.Bind(wx.EVT_MENU, self.OnDeleteTabs, item) if len(self.ISel)==1: - tabPanel=self.parent.GetParent() - if tabPanel.tabList.Naming!='FileNames': + if self.tabPanel.tabList.Naming!='FileNames': item = wx.MenuItem(self, -1, "Rename") - self.MyAppend(item) + self.Append(item) self.Bind(wx.EVT_MENU, self.OnRenameTab, item) if len(self.ISel)==1: item = wx.MenuItem(self, -1, "Export") - self.MyAppend(item) + self.Append(item) self.Bind(wx.EVT_MENU, self.OnExportTab, item) - def MyAppend(self, item): - self.Append(item) # python3 - def OnNaming(self, event=None): - tabPanel=self.parent.GetParent() if self.itNameFile.IsChecked(): - tabPanel.tabList.setNaming('FileNames') + self.tabPanel.tabList.setNaming('FileNames') else: - tabPanel.tabList.setNaming('Ellude') + self.tabPanel.tabList.setNaming('Ellude') + + self.tabPanel.updateTabNames() + + def OnMergeTabs(self, event): + # --- Figure out the common columns + tabs = [self.tabList.get(i) for i in self.ISel] + IKeepPerTab, IMissPerTab, IDuplPerTab, _ = getTabCommonColIndices(tabs) + nCommonCols = len(IKeepPerTab[0]) + commonCol = None + ICommonColPerTab = None + samplDict = None + if nCommonCols>=1: + # We use the first one + # TODO Menu to figure out which column to chose and how to merge (interp?) + keepAllX = True + #samplDict ={'name':'Replace', 'param':[], 'paramName':'New x'} + # Index of common column for each table + ICommonColPerTab = [I[0] for I in IKeepPerTab] + else: + # we'll merge based on index.. + pass - tabPanel.updateTabNames() + + + # Merge tables and add it to the list + self.tabList.mergeTabs(self.ISel, ICommonColPerTab, samplDict=samplDict) + # Updating tables + self.selPanel.update_tabs(self.tabList) + # TODO select latest + if self.mainframe: + self.mainframe.mergeTabsTrigger() def OnDeleteTabs(self, event): self.mainframe.deleteTabs(self.ISel) @@ -325,38 +356,36 @@ def OnSort(self, event): self.mainframe.sortTabs() class ColumnPopup(wx.Menu): + """ Popup Menu when right clicking on the column list """ def __init__(self, parent, fullmenu=False): wx.Menu.__init__(self) - self.parent = parent + self.parent = parent # parent is ColumnPanel self.ISel = self.parent.lbColumns.GetSelections() self.itShowID = wx.MenuItem(self, -1, "Show ID", kind=wx.ITEM_CHECK) - self.MyAppend(self.itShowID) + self.Append(self.itShowID) self.Bind(wx.EVT_MENU, self.OnShowID, self.itShowID) self.Check(self.itShowID.GetId(), self.parent.bShowID) if self.parent.tab is not None: # TODO otherwise item = wx.MenuItem(self, -1, "Add") - self.MyAppend(item) + self.Append(item) self.Bind(wx.EVT_MENU, self.OnAddColumn, item) if len(self.ISel)==1 and self.ISel[0]>=0: item = wx.MenuItem(self, -1, "Rename") - self.MyAppend(item) + self.Append(item) self.Bind(wx.EVT_MENU, self.OnRenameColumn, item) if len(self.ISel) == 1 and any( f['pos'] == self.ISel[0] for f in self.parent.tab.formulas): item = wx.MenuItem(self, -1, "Edit") - self.MyAppend(item) + self.Append(item) self.Bind(wx.EVT_MENU, self.OnEditColumn, item) if len(self.ISel)>=1 and self.ISel[0]>=0: item = wx.MenuItem(self, -1, "Delete") - self.MyAppend(item) + self.Append(item) self.Bind(wx.EVT_MENU, self.OnDeleteColumn, item) - def MyAppend(self, item): - self.Append(item) # python3 - def OnShowID(self, event=None): self.parent.bShowID=self.itShowID.IsChecked() xSel,ySel,_,_ = self.parent.getColumnSelection() @@ -490,13 +519,14 @@ def showFormulaDialog(self, title, name='', formula=''): # --------------------------------------------------------------------------------{ class TablePanel(wx.Panel): """ Display list of tables """ - def __init__(self, parent, mainframe,tabList): + def __init__(self, parent, selPanel, mainframe, tabList): # Superclass constructor super(TablePanel,self).__init__(parent) # DATA - self.parent=parent - self.mainframe=mainframe - self.tabList=tabList + self.parent = parent # splitter + self.selPanel = selPanel + self.mainframe = mainframe + self.tabList = tabList # GUI tb = wx.ToolBar(self,wx.ID_ANY,style=wx.TB_HORIZONTAL|wx.TB_TEXT|wx.TB_HORZ_LAYOUT|wx.TB_NODIVIDER) self.bt=wx.Button(tb,wx.ID_ANY,CHAR['menu'], style=wx.BU_EXACTFIT) @@ -513,14 +543,21 @@ def __init__(self, parent, mainframe,tabList): #sizer.Add(label, 0, border=5) sizer.Add(self.lbTab, 2, flag=wx.EXPAND, border=5) self.SetSizer(sizer) + # Bind + self.lbTab.Bind(wx.EVT_RIGHT_DOWN, self.onTabPopup) + + def onTabPopup(self, event=None): + menu = TablePopup(self.lbTab, self, self.selPanel, self.mainframe, fullmenu=False) + self.PopupMenu(menu, event.GetPosition()) + menu.Destroy() def showTableMenu(self,event=None): + """ Table Menu is Table Popup but at button position, and with "full" menu options """ pos = (self.bt.GetPosition()[0], self.bt.GetPosition()[1] + self.bt.GetSize()[1]) - menu = TablePopup(self.mainframe,self.lbTab,fullmenu=True) + menu = TablePopup(self.lbTab, self, self.selPanel, self.mainframe, fullmenu=True) self.PopupMenu(menu, pos) menu.Destroy() - def updateTabNames(self): tabnames_display=self.tabList.getDisplayTabNames() # Storing selection @@ -861,7 +898,7 @@ def __init__(self, parent, tabList, mode='auto',mainframe=None): # GUI DATA self.splitter = MultiSplit(self, style=wx.SP_LIVE_UPDATE) self.splitter.SetMinimumPaneSize(70) - self.tabPanel = TablePanel (self.splitter,mainframe, tabList) + self.tabPanel = TablePanel (self.splitter, self, mainframe, tabList) self.colPanel1 = ColumnPanel(self.splitter, self, mainframe); self.colPanel2 = ColumnPanel(self.splitter, self, mainframe); self.colPanel3 = ColumnPanel(self.splitter, self, mainframe); @@ -1309,21 +1346,26 @@ def xCol(self): if __name__ == '__main__': import pandas as pd; - from Tables import Table + from Tables import Table, TableList import numpy as np - def OnTabPopup(event): - self.PopupMenu(TablePopup(self,selPanel.tabPanel.lbTab), event.GetPosition()) app = wx.App(False) self=wx.Frame(None,-1,"Title") - tab=Table(data=pd.DataFrame(data={'ColA': np.random.normal(0,1,100)+1,'ColB':np.random.normal(0,1,100)+2})) - selPanel=SelectionPanel(self,[tab],mode='twoColumnsMode') + tab1=Table(data=pd.DataFrame(data={'ID': np.arange(0,100),'ColA': np.random.normal(0,1,100)+1,'ColB':np.random.normal(0,1,100)+2})) + tab2=Table(data=pd.DataFrame(data={'ID': np.arange(50,150),'ColA': np.random.normal(0,1,100)+1,'ColB':np.random.normal(0,1,100)+2})) + tabs = TableList([tab1,tab2]) + + selPanel=SelectionPanel(self, tabs, mode='twoColumnsMode') self.SetSize((800, 600)) self.Center() self.Show() - selPanel.tabPanel.lbTab.Bind(wx.EVT_RIGHT_DOWN, OnTabPopup) + selPanel.tabPanel.lbTab.SetSelection(0) + selPanel.tabPanel.lbTab.SetSelection(1) + menu = TablePopup(selPanel.tabPanel.lbTab, selPanel.tabPanel, selPanel) + menu.OnMergeTabs(None) + app.MainLoop() diff --git a/pydatview/Tables.py b/pydatview/Tables.py index f2512c8..5e58085 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -14,9 +14,11 @@ # --- TabList # --------------------------------------------------------------------------------{ class TableList(object): # todo inherit list - def __init__(self,tabs=[]): - self._tabs=tabs - self.Naming='Ellude' + def __init__(self, tabs=None): + if tabs is None: + tabs =[] + self._tabs = tabs + self.Naming = 'Ellude' # --- behaves like a list... def __iter__(self): @@ -30,7 +32,10 @@ def __next__(self): else: raise StopIteration - def append(self,t): + def __len__(self): + return len(self._tabs) + + def append(self, t): if isinstance(t,list): self._tabs += t else: @@ -157,6 +162,48 @@ def sort(self, method='byName'): else: raise Exception('Sorting method unknown: `{}`'.format(method)) + def mergeTabs(self, I=None, ICommonColPerTab=None, samplDict=None, extrap='nan'): + """ + Merge table together. + TODO: add options for how interpolation/merging is done + + I: index of tables to merge, if None: all tables are merged + """ + from pydatview.tools.signal_analysis import interpDF + #from pydatview.tools.signal_analysis import applySampler + #df_new, name_new = t.applyResampling(iCol,sampDict, bAdd=bAdd) + if I is None: + I = range(len(self._tabs)) + + dfs = [self._tabs[i].data for i in I] + if ICommonColPerTab is None: + # --- Option 0 - Index concatenation + df = pd.concat(dfs, axis=1) + # Remove duplicated columns + #df = df.loc[:,~df.columns.duplicated()].copy() + else: + # --- Option 1 - We combine all the x from the common column together + # NOTE: We use unique and sort, which will distrupt the user data (e.g. Airfoil Coords) + # The user should then use other methods (when implemented) + x_new=[] + cols = [] + for it, icol in zip(I, ICommonColPerTab): + xtab = self._tabs[it].data.iloc[:, icol].values + cols.append(self._tabs[it].data.columns[icol]) + x_new = np.concatenate( (x_new, xtab) ) + x_new = np.unique(np.sort(x_new)) + # Create interpolated dataframes based on x_new + dfs_new = [] + for i, (col, df_old) in enumerate(zip(cols, dfs)): + df = interpDF(x_new, col, df_old, extrap=extrap) + if i>0: + df = df.loc[:, df.columns!=col] # We remove the common columns + dfs_new.append(df) + df = pd.concat(dfs_new, axis=1) + newName = self._tabs[I[0]].name+'_merged' + self.append(Table(data=df, name=newName)) + return newName, df + def deleteTabs(self, I): self._tabs = [t for i,t in enumerate(self._tabs) if i not in I] @@ -714,5 +761,3 @@ def nRows(self): from Tables import Table import numpy as np - def OnTabPopup(event): - self.PopupMenu(TablePopup(self,selPanel.tabPanel.lbTab), event.GetPosition()) diff --git a/pydatview/main.py b/pydatview/main.py index 8a54a26..c6a7094 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -343,8 +343,6 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): self.Bind(wx.EVT_LISTBOX , self.onTabSelectionChange, self.selPanel.tabPanel.lbTab) self.Bind(wx.EVT_SPLITTER_SASH_POS_CHANGED, self.onSashChangeMain, self.vSplitter) - self.selPanel.tabPanel.lbTab.Bind(wx.EVT_RIGHT_DOWN, self.OnTabPopup) - # plot trigger if bPlot: self.mainFrameUpdateLayout() @@ -379,6 +377,7 @@ def setStatusBar(self, ISel=None): self.statusbar.SetStatusText(", ".join(list(set([self.tabList.filenames[i] for i in ISel]))),1) self.statusbar.SetStatusText('',2) + # --- Table Actions - TODO consider a table handler, or doing only the triggers def renameTable(self, iTab, newName): oldName = self.tabList.renameTable(iTab, newName) self.selPanel.renameTable(iTab, oldName, newName) @@ -390,9 +389,15 @@ def sortTabs(self, method='byName'): # Trigger a replot self.onTabSelectionChange() + def mergeTabsTrigger(self): + # Trigger a replot + self.onTabSelectionChange() def deleteTabs(self, I): self.tabList.deleteTabs(I) + if len(self.tabList)==0: + self.cleanGUI() + return # Invalidating selections self.selPanel.tabPanel.lbTab.SetSelection(-1) @@ -405,6 +410,7 @@ def deleteTabs(self, I): # Trigger a replot self.onTabSelectionChange() + def exportTab(self, iTab): tab=self.tabList.get(iTab) default_filename=tab.basename +'.csv' @@ -455,11 +461,6 @@ def onSashChangeMain(self,event=None): # print('ON SASH') # self.selPanel.setEquiSash(event) - def OnTabPopup(self,event): - menu = TablePopup(self,self.selPanel.tabPanel.lbTab) - self.PopupMenu(menu, event.GetPosition()) - menu.Destroy() - def onTabSelectionChange(self,event=None): # TODO This can be cleaned-up ISel=self.selPanel.tabPanel.lbTab.GetSelections() diff --git a/pydatview/tools/signal_analysis.py b/pydatview/tools/signal_analysis.py index 6babc09..d9cb0a2 100644 --- a/pydatview/tools/signal_analysis.py +++ b/pydatview/tools/signal_analysis.py @@ -69,26 +69,28 @@ def multiInterp(x, xp, fp, extrap='bounded'): xp = np.asarray(xp) assert fp.shape[1]==len(xp), 'Second dimension of fp should have the same length as xp' - j = np.searchsorted(xp, x) - 1 - dd = np.zeros(len(x)) + j = np.searchsorted(xp, x, 'left') - 1 + dd = np.zeros(len(x)) #*np.nan bOK = np.logical_and(j>=0, j< len(xp)-1) - bLower =j<0 - bUpper =j>=len(xp)-1 jOK = j[bOK] dd[bOK] = (x[bOK] - xp[jOK]) / (xp[jOK + 1] - xp[jOK]) jBef=j jAft=j+1 # + bLower =j<0 + bUpper =j>=len(xp)-1 # Use first and last values for anything beyond xp jAft[bUpper] = len(xp)-1 jBef[bUpper] = len(xp)-1 jAft[bLower] = 0 jBef[bLower] = 0 if extrap=='bounded': + #OK pass - # OK elif extrap=='nan': - dd[~bOK] = np.nan + # Set values to nan if out of bounds + bBeyond= np.logical_or(xnp.max(xp)) + dd[bBeyond] = np.nan else: raise NotImplementedError() @@ -133,6 +135,23 @@ def interpArray(x, xp, fp, extrap='bounded'): return (1 - dd) * fp[:,j] + fp[:,j+1] * dd +def interpDF(x_new, xLabel, df, extrap='bounded'): + """ Resample a dataframe using linear interpolation""" + x_old = df[xLabel].values + #x_new=np.sort(x_new) + # --- Method 1 (pandas) + #df_new = df_old.copy() + #df_new = df_new.set_index(x_old) + #df_new = df_new.reindex(df_new.index | x_new) + #df_new = df_new.interpolate().loc[x_new] + #df_new = df_new.reset_index() + # --- Method 2 interp storing dx + data_new=multiInterp(x_new, x_old, df.values.T, extrap=extrap) + df_new = pd.DataFrame(data=data_new.T, columns=df.columns.values) + df_new[xLabel] = x_new # Just in case this value was replaced by nan.. + return df_new + + def resample_interp(x_old, x_new, y_old=None, df_old=None): #x_new=np.sort(x_new) if df_old is not None: diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 0e66e60..fcf8e77 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -40,6 +40,16 @@ def test_table_name(self): # self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) # + def test_merge(self): + + tab1=Table(data=pd.DataFrame(data={'ID': np.arange(0,3,0.5),'ColA': [10,10.5,11,11.5,12,12.5]})) + tab2=Table(data=pd.DataFrame(data={'ID': np.arange(1,4),'ColB': [11,12,13]})) + tablist = TableList([tab1,tab2]) + # + name, df = tablist.mergeTabs(ICommonColPerTab=[0,0], extrap='nan') + np.testing.assert_almost_equal(df['ID'] , [0 , 0.5 , 1.0 , 1.5 , 2.0 , 2.5 , 3.0]) + np.testing.assert_almost_equal(df['ColA'] , [10 , 10.5 , 11 , 11.5 , 12 , 12.5 , np.nan] ) + np.testing.assert_almost_equal(df['ColB'] , [np.nan , np.nan , 11 , 11.5 , 12 , 12.5 , 13.0] ) def test_load_files_misc_formats(self): tablist = TableList() @@ -78,6 +88,7 @@ def test_change_units(self): if __name__ == '__main__': # TestTable.setUpClass() + TestTable().test_merge() # tt= TestTable() # tt.test_load_files_misc_formats() - unittest.main() +# unittest.main() diff --git a/tests/test_signal.py b/tests/test_signal.py index f70f6db..c3b17a9 100644 --- a/tests/test_signal.py +++ b/tests/test_signal.py @@ -1,42 +1,94 @@ -import unittest -import numpy as np -import pandas as pd -from pydatview.tools.signal_analysis import * - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -class TestSignal(unittest.TestCase): - - def test_zero_crossings(self): - self.assertEqual(zero_crossings(np.array([0 ]))[0].size,0 ) - self.assertEqual(zero_crossings(np.array([0 ,0]))[0].size,0) - self.assertEqual(zero_crossings(np.array([0 ,1]))[0].size,0) - self.assertEqual(zero_crossings(np.array([-1,0,0, 1]))[0].size,0) - self.assertEqual(zero_crossings(np.array([-1 ,1])), (0.5, 0, 1)) - self.assertEqual(zero_crossings(np.array([ 1, -1])), (0.5, 0,-1)) - self.assertEqual(zero_crossings(np.array([-1,0, 1])), (1.0, 1, 1)) - xz,iz,sz=zero_crossings(np.array([-1,1,-1])) - self.assertTrue(np.all(xz==[0.5,1.5])) - self.assertTrue(np.all(iz==[0,1])) - self.assertTrue(np.all(sz==[1,-1])) - self.assertEqual(zero_crossings(np.array([ 1,-1]),direction='up' )[0].size,0) - self.assertEqual(zero_crossings(np.array([-1, 1]),direction='down')[0].size,0) - - def test_up_down_sample(self): - name = 'Time-based' - x, y = applySampler(range(0, 4), [5, 0, 5, 0], {'name': name, 'param': [2]}) - self.assertTrue(np.all(x==[0.5, 2.5])) - self.assertTrue(np.all(y==[2.5, 2.5])) - x, y = applySampler(range(0, 3), [5, 0, 5], {'name': name, 'param': [0.5]}) - self.assertTrue(np.all(x==[0, 0.5, 1, 1.5, 2])) - self.assertTrue(np.all(y==[5, 2.5, 0, 2.5, 5])) - x, df = applySampler(range(0, 6), None, {'name': name, 'param': [3]}, pd.DataFrame({"y": [0, 6, 6, 2, -4, -4]})) - self.assertTrue(np.all(x==[1, 4])) - self.assertTrue(np.all(df["y"]==[4, -2])) - x, df = applySampler(range(0, 3), None, {'name': name, 'param': [0.5]}, pd.DataFrame({"y": [0, 6, -6]})) - self.assertTrue(np.all(x==[0, 0.5, 1, 1.5, 2])) - self.assertTrue(np.all(df["y"]==[0, 3, 6, 0, -6])) - -if __name__ == '__main__': - unittest.main() +import unittest +import numpy as np +import pandas as pd +from pydatview.tools.signal_analysis import * + +# --------------------------------------------------------------------------------} +# --- +# --------------------------------------------------------------------------------{ +class TestSignal(unittest.TestCase): + + def test_zero_crossings(self): + self.assertEqual(zero_crossings(np.array([0 ]))[0].size,0 ) + self.assertEqual(zero_crossings(np.array([0 ,0]))[0].size,0) + self.assertEqual(zero_crossings(np.array([0 ,1]))[0].size,0) + self.assertEqual(zero_crossings(np.array([-1,0,0, 1]))[0].size,0) + self.assertEqual(zero_crossings(np.array([-1 ,1])), (0.5, 0, 1)) + self.assertEqual(zero_crossings(np.array([ 1, -1])), (0.5, 0,-1)) + self.assertEqual(zero_crossings(np.array([-1,0, 1])), (1.0, 1, 1)) + xz,iz,sz=zero_crossings(np.array([-1,1,-1])) + self.assertTrue(np.all(xz==[0.5,1.5])) + self.assertTrue(np.all(iz==[0,1])) + self.assertTrue(np.all(sz==[1,-1])) + self.assertEqual(zero_crossings(np.array([ 1,-1]),direction='up' )[0].size,0) + self.assertEqual(zero_crossings(np.array([-1, 1]),direction='down')[0].size,0) + + def test_up_down_sample(self): + name = 'Time-based' + x, y = applySampler(range(0, 4), [5, 0, 5, 0], {'name': name, 'param': [2]}) + self.assertTrue(np.all(x==[0.5, 2.5])) + self.assertTrue(np.all(y==[2.5, 2.5])) + x, y = applySampler(range(0, 3), [5, 0, 5], {'name': name, 'param': [0.5]}) + self.assertTrue(np.all(x==[0, 0.5, 1, 1.5, 2])) + self.assertTrue(np.all(y==[5, 2.5, 0, 2.5, 5])) + x, df = applySampler(range(0, 6), None, {'name': name, 'param': [3]}, pd.DataFrame({"y": [0, 6, 6, 2, -4, -4]})) + self.assertTrue(np.all(x==[1, 4])) + self.assertTrue(np.all(df["y"]==[4, -2])) + x, df = applySampler(range(0, 3), None, {'name': name, 'param': [0.5]}, pd.DataFrame({"y": [0, 6, -6]})) + self.assertTrue(np.all(x==[0, 0.5, 1, 1.5, 2])) + self.assertTrue(np.all(df["y"]==[0, 3, 6, 0, -6])) + + def test_interpDF(self): + + df = data=pd.DataFrame(data={'ID': np.arange(0,3),'ColA': [10,11,12]}) + x_new = [-1,0,0.5,1,20,1.5,2,-3] + + # Interpolation with nan out of bounds + df_new = interpDF(x_new, 'ID', df, extrap='nan') + x_ref = [np.nan, 10, 10.5, 11, np.nan, 11.5, 12, np.nan] + np.testing.assert_almost_equal(df_new['ColA'], x_ref) + np.testing.assert_almost_equal(df_new['ID'], x_new) # make sure ID is not replaced by nan + + # Interpolation with bounded values outside + df_new = interpDF(x_new, 'ID', df) #, extrap='bounded') + x_ref = [10, 10, 10.5, 11, 12, 11.5, 12, 10] + np.testing.assert_almost_equal(df_new['ColA'], x_ref) + + def test_interp(self, plot=False): + x = np.linspace(0,1,10) + y1= x**2 + y2= x**3 + Y = np.stack((y1,y2)) + + # --- Check that we retrieve proper value on nodes + x_new = x + Y_new = multiInterp(x_new, x, Y) + np.testing.assert_almost_equal(Y_new, Y) + + # using interpArray + Y_new2 = np.zeros(Y_new.shape) + for i,x0 in enumerate(x_new): + Y_new2[:,i] = interpArray(x0, x, Y) + np.testing.assert_almost_equal(Y_new2, Y) + + # --- Check that we retrieve proper value on misc nodes + x_new = np.linspace(-0.8,1.5,20) + Y_new = multiInterp(x_new, x, Y) + y1_new = np.interp(x_new, x, Y[0,:]) + y2_new = np.interp(x_new, x, Y[1,:]) + Y_ref = np.stack((y1_new, y2_new)) + np.testing.assert_almost_equal(Y_new, Y_ref) + + # using interpArray + Y_new2 = np.zeros(Y_new.shape) + for i,x0 in enumerate(x_new): + Y_new2[:,i] = interpArray(x0, x, Y) + np.testing.assert_almost_equal(Y_new2, Y_ref) + + + + +if __name__ == '__main__': + TestSignal().test_interpDF() + TestSignal().test_interp() +# unittest.main() From 00aa6c0fe4ea03af5cc713e58bcb314c7ff4fa6c Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 15 Aug 2022 17:25:30 -0600 Subject: [PATCH 029/178] Adding x_min and x_max for resampling with deltax (see #125) --- pydatview/tools/signal_analysis.py | 35 +++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/pydatview/tools/signal_analysis.py b/pydatview/tools/signal_analysis.py index d9cb0a2..bb372d6 100644 --- a/pydatview/tools/signal_analysis.py +++ b/pydatview/tools/signal_analysis.py @@ -21,7 +21,7 @@ {'name':'Remove', 'param':[], 'paramName':'Remove list'}, {'name':'Every n', 'param':2 , 'paramName':'n'}, {'name':'Time-based', 'param':0.01 , 'paramName':'Sample time (s)'}, - {'name':'Delta x', 'param':0.1, 'paramName':'dx'}, + {'name':'Delta x', 'param':[0.1,np.nan,np.nan], 'paramName':'dx, xmin, xmax'}, ] @@ -164,10 +164,10 @@ def resample_interp(x_old, x_new, y_old=None, df_old=None): # --- Method 2 interp storing dx data_new=multiInterp(x_new, x_old, df_old.values.T) df_new = pd.DataFrame(data=data_new.T, columns=df_old.columns.values) - return x_new, df_new + return df_new if y_old is not None: - return x_new, np.interp(x_new, x_old, y_old) + return np.interp(x_new, x_old, y_old) def applySamplerDF(df_old, x_col, sampDict): @@ -185,13 +185,13 @@ def applySampler(x_old, y_old, sampDict, df_old=None): if len(param)==0: raise Exception('Error: At least one value is required to resample the x values with') x_new = param - return resample_interp(x_old, x_new, y_old, df_old) + return x_new, resample_interp(x_old, x_new, y_old, df_old) elif sampDict['name']=='Insert': if len(param)==0: raise Exception('Error: provide a list of values to insert') x_new = np.sort(np.concatenate((x_old.ravel(),param))) - return resample_interp(x_old, x_new, y_old, df_old) + return x_new, resample_interp(x_old, x_new, y_old, df_old) elif sampDict['name']=='Remove': I=[] @@ -202,14 +202,33 @@ def applySampler(x_old, y_old, sampDict, df_old=None): if len(Ifound)>0: I+=list(Ifound.ravel()) x_new=np.delete(x_old,I) - return resample_interp(x_old, x_new, y_old, df_old) + return x_new, resample_interp(x_old, x_new, y_old, df_old) elif sampDict['name']=='Delta x': if len(param)==0: raise Exception('Error: provide value for dx') dx = param[0] - x_new = np.arange(x_old[0], x_old[-1]+dx/2, dx) - return resample_interp(x_old, x_new, y_old, df_old) + if dx==0: + raise Exception('Error: `dx` cannot be 0') + if len(param)==1: + # NOTE: not using min/max if data loops (like airfoil) + xmin = np.min(x_old) + xmax = np.max(x_old) + dx/2 + elif len(param)==3: + xmin = param[1] + xmax = param[2] + if xmin is np.nan: + xmin = np.min(x_old) + if xmax is np.nan: + xmax = np.max(x_old) + dx/2 + else: + raise Exception('Error: the sampling parameters should be a list of three values `dx, xmin, xmax`') + x_new = np.arange(xmin, xmax, dx) + if len(x_new)==0: + xmax = xmin+dx*1.1 # NOTE: we do it like this to account for negative dx + x_new = np.arange(xmin, xmax, dx) + param = [dx, xmin, xmax] + return x_new, resample_interp(x_old, x_new, y_old, df_old) elif sampDict['name']=='Every n': if len(param)==0: From 63496a44ac1f4d6ebba7aa9fca552a623134e43e Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 15 Aug 2022 18:47:13 -0600 Subject: [PATCH 030/178] Fix issue when mode becomes sim on reload --- pydatview/GUISelectionPanel.py | 14 +++++++++----- pydatview/main.py | 22 +++++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 55e1d54..c181a26 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -881,7 +881,7 @@ def triggerPlot(self): # --------------------------------------------------------------------------------{ class SelectionPanel(wx.Panel): """ Display options for the user to select data """ - def __init__(self, parent, tabList, mode='auto',mainframe=None): + def __init__(self, parent, tabList, mode='auto', mainframe=None): # Superclass constructor super(SelectionPanel,self).__init__(parent) # DATA @@ -894,6 +894,7 @@ def __init__(self, parent, tabList, mode='auto',mainframe=None): self.modeRequested = mode self.currentMode = None self.nSplits = -1 + self.IKeepPerTab=None # GUI DATA self.splitter = MultiSplit(self, style=wx.SP_LIVE_UPDATE) @@ -916,7 +917,7 @@ def __init__(self, parent, tabList, mode='auto',mainframe=None): # TRIGGERS self.setTables(tabList) - def updateLayout(self,mode=None): + def updateLayout(self, mode=None): self.Freeze() if mode is None: mode=self.modeRequested @@ -937,7 +938,6 @@ def updateLayout(self,mode=None): raise Exception('Wrong mode for selection layout: {}'.format(mode)) self.Thaw() - def autoMode(self): ISel=self.tabPanel.lbTab.GetSelections() if self.tabList is not None: @@ -955,6 +955,7 @@ def autoMode(self): IKeepPerTab, IMissPerTab, IDuplPerTab, nCols = getTabCommonColIndices([self.tabList.get(i) for i in ISel]) if np.all(np.array([len(I) for I in IMissPerTab]))=2): self.simColumnsMode() + # >>>self.IKeepPerTab = IKeepPerTab elif len(ISel)==2: self.twoColumnsMode() elif len(ISel)==3: @@ -1057,6 +1058,9 @@ def setTables(self,tabList,update=False): # Trigger - updating columns and layout ISel=self.tabPanel.lbTab.GetSelections() self.tabSelected=ISel + # Mode might have changed if tables changed + if self.modeRequested=='auto': + self.autoMode() if self.currentMode=='simColumnsMode': self.setColForSimTab(ISel) else: @@ -1167,7 +1171,6 @@ def selectDefaultTable(self): def tabSelectionChanged(self): # TODO This can be cleaned-up and merged with updateLayout - #print('Tab selection change') # Storing the previous selection #self.printSelection() self.saveSelection() # @@ -1265,7 +1268,6 @@ def saveSelection(self): self.tabSelected = self.tabPanel.lbTab.GetSelections(); def printSelection(self): - print('Number of tabSelections stored:',len(self.tabSelections)) TS=self.tabSelections for i,tn in enumerate(self.tabList.tabNames): if tn not in TS.keys(): @@ -1281,6 +1283,8 @@ def getPlotDataSelection(self): if self.currentMode=='simColumnsMode' and len(ITab)>1: iiX1,IY1,ssX1,SY1 = self.colPanel1.getColumnSelection() SameCol=False + if self.IKeepPerTab is None: + raise Exception('>>>TODO') for i,(itab,stab) in enumerate(zip(ITab,STab)): IKeep=self.IKeepPerTab[i] for j,(iiy,ssy) in enumerate(zip(IY1,SY1)): diff --git a/pydatview/main.py b/pydatview/main.py index c6a7094..5ed0fd7 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -72,7 +72,7 @@ def OnDropFiles(self, x, y, filenames): Format = None else: Format = self.parent.FILE_FORMATS[iFormat-1] - self.parent.load_files(filenames, fileformats=[Format]*len(filenames), bAdd=bAdd) + self.parent.load_files(filenames, fileformats=[Format]*len(filenames), bAdd=bAdd, bPlot=True) return True @@ -241,7 +241,7 @@ def clean_memory(self,bReload=False): self.plotPanel.cleanPlot() gc.collect() - def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False): + def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, bPlot=True): """ load multiple files, only trigger the plot at the end """ if bReload: if hasattr(self,'selPanel'): @@ -277,7 +277,7 @@ def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False): Warn(self,warn) # Load tables into the GUI if self.tabList.len()>0: - self.load_tabs_into_GUI(bReload=bReload, bAdd=bAdd, bPlot=True) + self.load_tabs_into_GUI(bReload=bReload, bAdd=bAdd, bPlot=bPlot) def load_df(self, df, name=None, bAdd=False, bPlot=True): if bAdd: @@ -308,6 +308,7 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): if bReload or bAdd: self.selPanel.update_tabs(self.tabList) else: + # --- Create a selPanel, plotPanel and infoPanel mode = SEL_MODES_ID[self.comboMode.GetSelection()] #self.vSplitter = wx.SplitterWindow(self.nb) self.vSplitter = wx.SplitterWindow(self.MainPanel) @@ -390,8 +391,11 @@ def sortTabs(self, method='byName'): self.onTabSelectionChange() def mergeTabsTrigger(self): - # Trigger a replot - self.onTabSelectionChange() + if hasattr(self,'selPanel'): + # Select the newly created table + self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)) + # Trigger a replot + self.onTabSelectionChange() def deleteTabs(self, I): self.tabList.deleteTabs(I) @@ -546,7 +550,7 @@ def onReload(self, event=None): f = sorted(f, key=lambda k: k['pos']) # Sort formulae by position in list of formua self.restore_formulas[tab.raw_name]=f # we use raw_name as key # Actually load files (read and add in GUI) - self.load_files(filenames, fileformats=fileformats, bReload=True,bAdd=False) + self.load_files(filenames, fileformats=fileformats, bReload=True, bAdd=False, bPlot=True) else: Error(self,'Open one or more file first.') @@ -594,7 +598,7 @@ def selectFile(self,bAdd=False): if dlg.ShowModal() == wx.ID_CANCEL: return # the user changed their mind filenames = dlg.GetPaths() - self.load_files(filenames,fileformats=[Format]*len(filenames),bAdd=bAdd) + self.load_files(filenames,fileformats=[Format]*len(filenames),bAdd=bAdd, bPlot=True) def onModeChange(self, event=None): if hasattr(self,'selPanel'): @@ -686,7 +690,7 @@ def test(filenames=None): if filenames is not None: app = wx.App(False) frame = MainFrame() - frame.load_files(filenames,fileformats=None) + frame.load_files(filenames,fileformats=None, bPlot=True) return # --------------------------------------------------------------------------------} @@ -770,7 +774,7 @@ def showApp(firstArg=None, dataframes=None, filenames=[], names=None): names=['df{}'.format(i+1) for i in range(len(dataframes))] frame.load_dfs(dataframes, names) elif len(filenames)>0: - frame.load_files(filenames, fileformats=None) + frame.load_files(filenames, fileformats=None, bPlot=True) app.MainLoop() def cmdline(): From 9c8a5a16ab20a87515e278673eb199f22d5b07b3 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 15 Aug 2022 20:47:37 -0600 Subject: [PATCH 031/178] Remember filters and selections on reload (see #104, #126, #110) --- pydatview/GUISelectionPanel.py | 154 ++++++++++++++++++++++++--------- pydatview/Tables.py | 83 ++++++++++-------- pydatview/main.py | 17 +++- 3 files changed, 176 insertions(+), 78 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index c181a26..94d846d 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -711,8 +711,10 @@ def setReadOnly(self, tabLabel='', cols=[]): self.tFilter.Enable(False) self.tFilter.SetValue('') - def setTab(self,tab=None,xSel=-1,ySel=[],colNames=None, tabLabel=''): - """ Set the table used for the columns, update the GUI """ + def setTab(self, tab=None, xSel=-1, ySel=[], colNames=None, tabLabel='', sFilter=None): + """ Set the table used for the columns, update the GUI + tab is None, when in simColumnsMode + """ self.tab=tab; self.lbColumns.Enable(True) self.comboX.Enable(True) @@ -721,17 +723,23 @@ def setTab(self,tab=None,xSel=-1,ySel=[],colNames=None, tabLabel=''): self.btFilter.Enable(True) self.tFilter.Enable(True) self.bt.Enable(True) + + selInFull = True + if sFilter is not None and len(sFilter.strip())>0: + self.tFilter.SetValue(sFilter) + selInFull = False + if tab is not None: - self.Filt2Full=None # TODO + # For a single tab if tab.active_name!='default': self.lb.SetLabel(' '+tab.active_name) + # Setting raw columns from raw table (self.tab) self.setColumns() - self.setGUIColumns(xSel=xSel, ySel=ySel) + self.setGUIColumns(xSel=xSel, ySel=ySel, selInFull=selInFull) # Filt2Full will be created if a filter is present else: - self.Filt2Full=None # TODO Decide whether filter should be applied... self.lb.SetLabel(tabLabel) self.setColumns(columnNames=colNames) - self.setGUIColumns(xSel=xSel, ySel=ySel) + self.setGUIColumns(xSel=xSel, ySel=ySel, selInFull=selInFull) # Filt2Full will be created if a filter is present def updateColumn(self,i,newName): """ Update of one column name @@ -751,6 +759,10 @@ def Full2Filt(self,iFull): return -1 def setColumns(self, columnNames=None): + """ + For a regular table, sets "full columns" from the tab. + In simColumnsMode, tab is None, and the columns are given by the user. + """ # Get columns from user inputs, or table, or stored. if columnNames is not None: # Populating based on user inputs.. @@ -763,13 +775,30 @@ def setColumns(self, columnNames=None): # Storing columns, considered as "Full" self.columns=np.array(columns) - def setGUIColumns(self, xSel=-1, ySel=[]): - """ Set GUI columns based on self.columns and potential filter """ + def setGUIColumns(self, xSel=-1, ySel=[], selInFull=True): + """ Set columns actually shown on the GUI based on self.columns and potential filter + if selInFull is True, the selection is assumed to be in the full/raw columns + Otherwise, the selection is assumed to be in the filtered column + """ # Filtering columns if neeed sFilt = self.tFilter.GetLineText(0).strip() if len(sFilt)>0: Lf, If = filter_list(self.columns, sFilt) self.Filt2Full = If + + if len(If)==0: + # No results + if not selInFull: + # Then it's likely a reload, we cancel the filter + self.tFilter.SetValue('') + self.Filt2Full = list(np.arange(len(self.columns))) + selInFull=False + elif len(If)==1: + # Only one result, we select first value + selInFull=False + ySel=[0] + + else: self.Filt2Full = list(np.arange(len(self.columns))) columns=self.columns[self.Filt2Full] @@ -797,16 +826,23 @@ def setGUIColumns(self, xSel=-1, ySel=[]): self.comboX.Set(columnsX_show) # non filtered # Set selection for y, if any, and considering filtering - for iFull in ySel: - if iFull=0: - iFilt = self.Full2Filt(iFull) - if iFilt>0: + if selInFull: + for iFull in ySel: + if iFull=0: + iFilt = self.Full2Filt(iFull) + if iFilt>0: + self.lbColumns.SetSelection(iFilt) + self.lbColumns.EnsureVisible(iFilt) + else: + for iFilt in ySel: + if iFilt>=0 and iFilt<=len(columnsY): self.lbColumns.SetSelection(iFilt) self.lbColumns.EnsureVisible(iFilt) + if len(self.lbColumns.GetSelections())<=0: self.lbColumns.SetSelection(self.getDefaultColumnY(self.tab,len(columnsY)-1)) - # Set selection for x, if any, NOTE x is not filtered! + # Set selection for x, if any, NOTE x is not filtered, alwasy in full! if (xSel<0) or xSel>len(columnsX): self.comboX.SetSelection(self.getDefaultColumnX(self.tab,len(columnsX)-1)) else: @@ -839,6 +875,14 @@ def empty(self): self.tFilter.SetValue('') def getColumnSelection(self): + """ return the indices selected for the given table so that the plotData can be extracted + The indices will be in "orignal/full" table, removing the account for a potential filter. + + iX - index in table corresponding to selected x column + sX - selected x column (in table) + IY - indices in table corresponding to selected y columns + SY - selected Y columns (in table) + """ iX = self.comboX.GetSelection() if self.bShowID: sX = self.comboX.GetStringSelection()[4:] @@ -889,7 +933,9 @@ def __init__(self, parent, tabList, mode='auto', mainframe=None): self.tabList = None self.itabForCol = None self.parent = parent - self.tabSelections = {} + self.tabSelections = {} # x-Y-Columns selected for each table + self.simTabSelection = {} # selection for simTable case + self.filterSelection = ['','',''] # filters self.tabSelected = [] # NOTE only used to remember a selection after a reload self.modeRequested = mode self.currentMode = None @@ -955,7 +1001,6 @@ def autoMode(self): IKeepPerTab, IMissPerTab, IDuplPerTab, nCols = getTabCommonColIndices([self.tabList.get(i) for i in ISel]) if np.all(np.array([len(I) for I in IMissPerTab]))=2): self.simColumnsMode() - # >>>self.IKeepPerTab = IKeepPerTab elif len(ISel)==2: self.twoColumnsMode() elif len(ISel)==3: @@ -1081,11 +1126,11 @@ def setTabForCol(self,iTabSel,iPanel): t = self.tabList.get(iTabSel) ts = self.tabSelections[t.name] if iPanel==1: - self.colPanel1.setTab(t,ts['xSel'],ts['ySel']) + self.colPanel1.setTab(t,ts['xSel'],ts['ySel'], sFilter=self.filterSelection[0]) elif iPanel==2: - self.colPanel2.setTab(t,ts['xSel'],ts['ySel']) + self.colPanel2.setTab(t,ts['xSel'],ts['ySel'], sFilter=self.filterSelection[1]) elif iPanel==3: - self.colPanel3.setTab(t,ts['xSel'],ts['ySel']) + self.colPanel3.setTab(t,ts['xSel'],ts['ySel'], sFilter=self.filterSelection[2]) else: raise Exception('Wrong ipanel') @@ -1154,8 +1199,18 @@ def setColForSimTab(self,ISel): ColInfo.append('----------------------------------') + colNames = ['Index'] + [tabs[0].columns[i] for i in IKeepPerTab[0]] - self.colPanel1.setTab(tab=None, colNames=colNames, tabLabel=' Tab. Intersection') + + # restore selection + xSel = -1 + ySel = [] + sFilter = self.filterSelection[0] + if 'xSel' in self.simTabSelection: + xSel = self.simTabSelection['xSel'] + ySel = self.simTabSelection['ySel'] + # Set the colPanels + self.colPanel1.setTab(tab=None, colNames=colNames, tabLabel=' Tab. Intersection', xSel=xSel, ySel=ySel, sFilter=sFilter) self.colPanel2.setReadOnly(' Tab. Difference', ColInfo) self.IKeepPerTab=IKeepPerTab @@ -1172,9 +1227,7 @@ def selectDefaultTable(self): def tabSelectionChanged(self): # TODO This can be cleaned-up and merged with updateLayout # Storing the previous selection - #self.printSelection() self.saveSelection() # - #self.printSelection() ISel=self.tabPanel.lbTab.GetSelections() if len(ISel)>0: if self.modeRequested=='auto': @@ -1215,7 +1268,7 @@ def tabSelectionChanged(self): self.tabPanel.lbTab.SetSelection(ISel[0]) for iPanel,iTab in enumerate(ISel): self.setTabForCol(iTab,iPanel+1) - #print('>>>Updating tabSelected, from',self.tabSelected,'to',self.tabPanel.lbTab.GetSelections()) + #print('>>Updating tabSelected, from',self.tabSelected,'to',self.tabPanel.lbTab.GetSelections()) self.tabSelected=self.tabPanel.lbTab.GetSelections() def colSelectionChanged(self): @@ -1247,35 +1300,52 @@ def renameTable(self,iTab, oldName, newName): def saveSelection(self): #self.ISel=self.tabPanel.lbTab.GetSelections() ISel=self.tabSelected # - if self.tabList.haveSameColumns(ISel): - for ii in ISel: - t=self.tabList.get(ii) - self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() - self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() + + # --- Save filters + self.filterSelection = [self.colPanel1.tFilter.GetLineText(0).strip()] + self.filterSelection += [self.colPanel2.tFilter.GetLineText(0).strip()] + self.filterSelection += [self.colPanel3.tFilter.GetLineText(0).strip()] + + # --- Save simTab is needed + if self.currentMode=='simColumnsMode': + self.simTabSelection['xSel'] = self.colPanel1.comboX.GetSelection() + self.simTabSelection['ySel'] = self.colPanel1.lbColumns.GetSelections() else: - if len(ISel)>=1: - t=self.tabList.get(ISel[0]) - self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() - self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() - if len(ISel)>=2: - t=self.tabList.get(ISel[1]) - self.tabSelections[t.name]['xSel'] = self.colPanel2.comboX.GetSelection() - self.tabSelections[t.name]['ySel'] = self.colPanel2.lbColumns.GetSelections() - if len(ISel)>=3: - t=self.tabList.get(ISel[2]) - self.tabSelections[t.name]['xSel'] = self.colPanel3.comboX.GetSelection() - self.tabSelections[t.name]['ySel'] = self.colPanel3.lbColumns.GetSelections() - self.tabSelected = self.tabPanel.lbTab.GetSelections(); + #self.simTabSelection = {} # We do not erase it + # --- Save selected columns for each tab + if self.tabList.haveSameColumns(ISel): + for ii in ISel: + t=self.tabList.get(ii) + self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() + self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() + else: + if len(ISel)>=1: + t=self.tabList.get(ISel[0]) + self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() + self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() + if len(ISel)>=2: + t=self.tabList.get(ISel[1]) + self.tabSelections[t.name]['xSel'] = self.colPanel2.comboX.GetSelection() + self.tabSelections[t.name]['ySel'] = self.colPanel2.lbColumns.GetSelections() + if len(ISel)>=3: + t=self.tabList.get(ISel[2]) + self.tabSelections[t.name]['xSel'] = self.colPanel3.comboX.GetSelection() + self.tabSelections[t.name]['ySel'] = self.colPanel3.lbColumns.GetSelections() + self.tabSelected = self.tabPanel.lbTab.GetSelections(); + #self.printSelection() def printSelection(self): TS=self.tabSelections for i,tn in enumerate(self.tabList.tabNames): if tn not in TS.keys(): - print('Tab',i,'>>> Name {} not found in selection'.format(tn)) + print('Tab',i,' Name {} not found in selection'.format(tn)) else: print('Tab',i,'xSel:',TS[tn]['xSel'],'ySel:',TS[tn]['ySel'],'Name:',tn) + print('simTab ', self.simTabSelection) + print('filters', self.filterSelection) def getPlotDataSelection(self): + """ Returns the table/columns indices to be plotted""" ID = [] SameCol=False if self.tabList is not None and self.tabList.len()>0: @@ -1283,8 +1353,6 @@ def getPlotDataSelection(self): if self.currentMode=='simColumnsMode' and len(ITab)>1: iiX1,IY1,ssX1,SY1 = self.colPanel1.getColumnSelection() SameCol=False - if self.IKeepPerTab is None: - raise Exception('>>>TODO') for i,(itab,stab) in enumerate(zip(ITab,STab)): IKeep=self.IKeepPerTab[i] for j,(iiy,ssy) in enumerate(zip(IY1,SY1)): diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 5e58085..573b0a8 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -50,7 +50,7 @@ def from_dataframes(self, dataframes=[], names=[], bAdd=False): if df is not None: self.append(Table(data=df, name=name)) - def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False): + def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bReload=False): """ load multiple files into table list""" if not bAdd: self.clean() # TODO figure it out @@ -68,14 +68,14 @@ def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False): pass # warn+= 'Warn: an empty filename was skipped' +'\n' else: - tabs, warnloc = self._load_file_tabs(f,fileformat=ff) + tabs, warnloc = self._load_file_tabs(f,fileformat=ff, bReload=bReload) if len(warnloc)>0: warnList.append(warnloc) self.append(tabs) return warnList - def _load_file_tabs(self, filename, fileformat=None): + def _load_file_tabs(self, filename, fileformat=None, bReload=False): """ load a single file, adds table """ # Returning a list of tables tabs=[] @@ -83,37 +83,52 @@ def _load_file_tabs(self, filename, fileformat=None): if not os.path.isfile(filename): warn = 'Error: File not found: `'+filename+'`\n' return tabs, warn - try: - #F = weio.read(filename, fileformat = fileformat) - # --- Expanded version of weio.read - F = None - if fileformat is None: - fileformat, F = weio.detectFormat(filename) - # Reading the file with the appropriate class if necessary - if not isinstance(F, fileformat.constructor): - F=fileformat.constructor(filename=filename) - dfs = F.toDataFrame() - except weio.FileNotFoundError as e: - warn = 'Error: A file was not found!\n\n While opening:\n\n {}\n\n the following file was not found:\n\n {}\n'.format(filename, e.filename) - except IOError: - warn = 'Error: IO Error thrown while opening file: '+filename+'\n' - except MemoryError: - warn='Error: Insufficient memory!\n\nFile: '+filename+'\n\nTry closing and reopening the program, or use a 64 bit version of this program (i.e. of python).\n' - except weio.EmptyFileError: - warn='Error: File empty!\n\nFile is empty: '+filename+'\n\nOpen a different file.\n' - except weio.FormatNotDetectedError: - warn='Error: File format not detected!\n\nFile: '+filename+'\n\nUse an explicit file-format from the list\n' - except weio.WrongFormatError as e: - warn='Error: Wrong file format!\n\nFile: '+filename+'\n\n' \ - 'The file parser for the selected format failed to open the file.\n\n'+ \ - 'The reported error was:\n'+e.args[0]+'\n\n' + \ - 'Double-check your file format and report this error if you think it''s a bug.\n' - except weio.BrokenFormatError as e: - warn = 'Error: Inconsistency in the file format!\n\nFile: '+filename+'\n\n' \ - 'The reported error was:\n\n'+e.args[0]+'\n\n' + \ - 'Double-check your file format and report this error if you think it''s a bug.' - except: - raise + + fileformatAllowedToFailOnReload = (fileformat is not None) and bReload + if fileformatAllowedToFailOnReload: + try: + F = None + if not isinstance(F, fileformat.constructor): + F=fileformat.constructor(filename=filename) + dfs = F.toDataFrame() + except: + warnLoc = 'Failed to read file:\n\n {}\n\nwith fileformat: {}\n\nIf you see this message, the reader tried again and succeeded with "auto"-fileformat.\n\n'.format(filename, fileformat.name) + tabs,warn = self._load_file_tabs(filename, fileformat=None, bReload=False) + return tabs, warnLoc+warn + + else: + + try: + #F = weio.read(filename, fileformat = fileformat) + # --- Expanded version of weio.read + F = None + if fileformat is None: + fileformat, F = weio.detectFormat(filename) + # Reading the file with the appropriate class if necessary + if not isinstance(F, fileformat.constructor): + F=fileformat.constructor(filename=filename) + dfs = F.toDataFrame() + except weio.FileNotFoundError as e: + warn = 'Error: A file was not found!\n\n While opening:\n\n {}\n\n the following file was not found:\n\n {}\n'.format(filename, e.filename) + except IOError: + warn = 'Error: IO Error thrown while opening file: '+filename+'\n' + except MemoryError: + warn='Error: Insufficient memory!\n\nFile: '+filename+'\n\nTry closing and reopening the program, or use a 64 bit version of this program (i.e. of python).\n' + except weio.EmptyFileError: + warn='Error: File empty!\n\nFile is empty: '+filename+'\n\nOpen a different file.\n' + except weio.FormatNotDetectedError: + warn='Error: File format not detected!\n\nFile: '+filename+'\n\nUse an explicit file-format from the list\n' + except weio.WrongFormatError as e: + warn='Error: Wrong file format!\n\nFile: '+filename+'\n\n' \ + 'The file parser for the selected format failed to open the file.\n\n'+ \ + 'The reported error was:\n'+e.args[0]+'\n\n' + \ + 'Double-check your file format and report this error if you think it''s a bug.\n' + except weio.BrokenFormatError as e: + warn = 'Error: Inconsistency in the file format!\n\nFile: '+filename+'\n\n' \ + 'The reported error was:\n\n'+e.args[0]+'\n\n' + \ + 'Double-check your file format and report this error if you think it''s a bug.' + except: + raise if len(warn)>0: return tabs, warn diff --git a/pydatview/main.py b/pydatview/main.py index 5ed0fd7..9114a49 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -165,6 +165,7 @@ def __init__(self, data=None): self.comboMode = wx.ComboBox(tb, choices = SEL_MODES, style=wx.CB_READONLY) self.comboMode.SetSelection(0) self.Bind(wx.EVT_COMBOBOX, self.onModeChange, self.comboMode ) + self.Bind(wx.EVT_COMBOBOX, self.onFormatChange, self.comboFormats ) tb.AddSeparator() tb.AddControl( wx.StaticText(tb, -1, 'Mode: ' ) ) tb.AddControl( self.comboMode ) @@ -264,7 +265,7 @@ def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, #filenames = [f for __, f in sorted(zip(base_filenames, filenames))] # Load the tables - warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformats=fileformats, bAdd=bAdd) + warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformats=fileformats, bAdd=bAdd, bReload=bReload) if bReload: # Restore formulas that were previously added for tab in self.tabList: @@ -543,6 +544,14 @@ def onReset (self, event=None): def onReload(self, event=None): filenames, fileformats = self.tabList.filenames_and_formats if len(filenames)>0: + # If only one file, use the comboBox to decide which fileformat to use + if len(filenames)==1: + iFormat=self.comboFormats.GetSelection() + if iFormat==0: # auto-format + fileformats = [None] + else: + fileformats = [self.FILE_FORMATS[iFormat-1]] + # Save formulas to restore them after reload with sorted tabs self.restore_formulas = {} for tab in self.tabList._tabs: @@ -607,6 +616,12 @@ def onModeChange(self, event=None): # --- Trigger to check number of columns self.onTabSelectionChange() + def onFormatChange(self, event=None): + """ The user changed the format """ + #if hasattr(self,'selPanel'): + # ISel=self.selPanel.tabPanel.lbTab.GetSelections() + pass + def mainFrameUpdateLayout(self, event=None): if hasattr(self,'selPanel'): nWind=self.selPanel.splitter.nWindows From dd71389b6a182fbdfb4ee95956333edaf56ff483 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 15 Aug 2022 21:28:13 -0600 Subject: [PATCH 032/178] Update status bar when loading, and dipslay when no filtered results found --- pydatview/GUISelectionPanel.py | 26 +++++++++++++++++++----- pydatview/Tables.py | 6 ++++-- pydatview/main.py | 37 +++++++++++++++++++++------------- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 94d846d..ef588e2 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -694,15 +694,26 @@ def getGUIcolumns(self): raise Exception('Error in Filt2Full') return self.columns[self.Filt2Full] - def setReadOnly(self, tabLabel='', cols=[]): + + def _setReadOnly(self): + self.bReadOnly=True + self.comboX.Enable(False) + self.lbColumns.Enable(False) + + def _unsetReadOnly(self): + self.bReadOnly=False + self.comboX.Enable(True) + self.lbColumns.Enable(True) + + def setReadOnly(self, tabLabel=None, cols=[]): """ Set this list of columns as readonly and non selectable """ self.tab=None - self.bReadOnly=True - self.lb.SetLabel(tabLabel) + if tabLabel is not None: + self.lb.SetLabel(tabLabel) + self._setReadOnly() + self.lbColumns.Enable(True) self.setColumns(columnNames=cols) self.setGUIColumns() - self.lbColumns.Enable(True) - self.comboX.Enable(False) self.lbColumns.SetSelection(-1) self.bt.Enable(False) self.bShowID=False @@ -811,6 +822,11 @@ def setGUIColumns(self, xSel=-1, ySel=[], selInFull=True): else: columnsY= columns columnsX= self.columns + if len(columnsY)==0: + columnsY=['No results'] + self._setReadOnly() + else: + self._unsetReadOnly() self.lbColumns.Set(columnsY) # potentially filterd # Slow line for many columns # NOTE: limiting to 300 for now.. I'm not sure anywant would want to scroll more than that diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 573b0a8..3b75ad0 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -50,7 +50,7 @@ def from_dataframes(self, dataframes=[], names=[], bAdd=False): if df is not None: self.append(Table(data=df, name=name)) - def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bReload=False): + def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bReload=False, statusFunction=None): """ load multiple files into table list""" if not bAdd: self.clean() # TODO figure it out @@ -61,7 +61,9 @@ def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bRe # Loop through files, appending tables within files warnList=[] - for f,ff in zip(filenames, fileformats): + for i, (f,ff) in enumerate(zip(filenames, fileformats)): + if statusFunction is not None: + statusFunction(i) if f in self.unique_filenames: warnList.append('Warn: Cannot add a file already opened ' + f) elif len(f)==0: diff --git a/pydatview/main.py b/pydatview/main.py index 9114a49..4be5cec 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -42,6 +42,7 @@ SIDE_COL = [160,160,300,420,530] SIDE_COL_LARGE = [200,200,360,480,600] BOT_PANL =85 +ISTAT = 0 # Index of Status bar where main status info is provided #matplotlib.rcParams['text.usetex'] = False # matplotlib.rcParams['font.sans-serif'] = 'DejaVu Sans' @@ -183,7 +184,7 @@ def __init__(self, data=None): # --- Status bar self.statusbar=self.CreateStatusBar(3, style=0) - self.statusbar.SetStatusWidths([200, -1, 70]) + self.statusbar.SetStatusWidths([150, -1, 70]) # --- Main Panel and Notebook self.MainPanel = wx.Panel(self) @@ -247,6 +248,11 @@ def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, if bReload: if hasattr(self,'selPanel'): self.selPanel.saveSelection() # TODO move to tables + else: + self.statusbar.SetStatusText('Loading files...', ISTAT) + + # A function to update the status bar while we load files + statusFunction = lambda i: self.statusbar.SetStatusText('Loading files {}/{}'.format(i+1,len(filenames)), ISTAT) if not bAdd: self.clean_memory(bReload=bReload) @@ -265,7 +271,7 @@ def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, #filenames = [f for __, f in sorted(zip(base_filenames, filenames))] # Load the tables - warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformats=fileformats, bAdd=bAdd, bReload=bReload) + warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformats=fileformats, bAdd=bAdd, bReload=bReload, statusFunction=statusFunction) if bReload: # Restore formulas that were previously added for tab in self.tabList: @@ -302,6 +308,8 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): if (not bReload) and (not bAdd): self.cleanGUI() + if (bReload): + self.statusbar.SetStatusText('Done reloading.', ISTAT) self.Freeze() # Setting status bar self.setStatusBar() @@ -363,21 +371,21 @@ def setStatusBar(self, ISel=None): if ISel is None: ISel = list(np.arange(nTabs)) if nTabs<0: - self.statusbar.SetStatusText('', 0) # Format - self.statusbar.SetStatusText('', 1) # Filenames - self.statusbar.SetStatusText('', 2) # Shape + self.statusbar.SetStatusText('', ISTAT) # Format + self.statusbar.SetStatusText('', ISTAT+1) # Filenames + self.statusbar.SetStatusText('', ISTAT+2) # Shape elif nTabs==1: - self.statusbar.SetStatusText(self.tabList.get(0).fileformat_name, 0) - self.statusbar.SetStatusText(self.tabList.get(0).filename , 1) - self.statusbar.SetStatusText(self.tabList.get(0).shapestring, 2) + self.statusbar.SetStatusText(self.tabList.get(0).fileformat_name, ISTAT+0) + self.statusbar.SetStatusText(self.tabList.get(0).filename , ISTAT+1) + self.statusbar.SetStatusText(self.tabList.get(0).shapestring , ISTAT+2) elif len(ISel)==1: - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).fileformat_name , 0) - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).filename , 1) - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).shapestring, 2) + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).fileformat_name , ISTAT+0) + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).filename , ISTAT+1) + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).shapestring , ISTAT+2) else: - self.statusbar.SetStatusText('' ,0) - self.statusbar.SetStatusText(", ".join(list(set([self.tabList.filenames[i] for i in ISel]))),1) - self.statusbar.SetStatusText('',2) + self.statusbar.SetStatusText('{} tables loaded'.format(nTabs) ,ISTAT+0) + self.statusbar.SetStatusText(", ".join(list(set([self.tabList.filenames[i] for i in ISel]))),ISTAT+1) + self.statusbar.SetStatusText('' ,ISTAT+2) # --- Table Actions - TODO consider a table handler, or doing only the triggers def renameTable(self, iTab, newName): @@ -543,6 +551,7 @@ def onReset (self, event=None): def onReload(self, event=None): filenames, fileformats = self.tabList.filenames_and_formats + self.statusbar.SetStatusText('Reloading...', ISTAT) if len(filenames)>0: # If only one file, use the comboBox to decide which fileformat to use if len(filenames)==1: From 5e303a3b9e5c7c348d47a07754efae2a1054ecc8 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 15 Aug 2022 21:47:53 -0600 Subject: [PATCH 033/178] Adding example for calling pyDatView from python (#120) --- example_files/Example_FromPython.py | 22 ++ pydatview/io/__init__.py | 532 ++++++++++++++-------------- 2 files changed, 290 insertions(+), 264 deletions(-) create mode 100644 example_files/Example_FromPython.py diff --git a/example_files/Example_FromPython.py b/example_files/Example_FromPython.py new file mode 100644 index 0000000..45e86a1 --- /dev/null +++ b/example_files/Example_FromPython.py @@ -0,0 +1,22 @@ +""" +Example for calling pydatview to visuzalize a dataframe while scripting from python + +""" +import numpy as np +import pandas as pd +import pydatview + +df1 = pd.DataFrame(data={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0}) +df2 = pd.DataFrame(data={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0}) + +# --- Opening two dataframes +pydatview.show(dataframes=[df1,df2], names=['WT1','WT2']) +# pydatview.show([df1,df2]) + +# --- Opening one dataframe +# pydatview.show(df1) + +# --- Opening files: +#pydatview.show(filenames=['file.csv','file2.csv']) +#pydatview.show(['file.csv','file2.csv']) +#pydatview.show('file.csv') diff --git a/pydatview/io/__init__.py b/pydatview/io/__init__.py index 73737be..a09ca6c 100644 --- a/pydatview/io/__init__.py +++ b/pydatview/io/__init__.py @@ -1,264 +1,268 @@ -from .file import File, WrongFormatError, BrokenFormatError, FileNotFoundError, EmptyFileError -from .file_formats import FileFormat, isRightFormat -import sys -import os -import numpy as np - -class FormatNotDetectedError(Exception): - pass - -class UserFormatImportError(Exception): - pass - - -_FORMATS=None - -def fileFormats(userpath=None, ignoreErrors=False, verbose=False): - """ return list of fileformats supported by the library - If userpath is provided, - - """ - global _FORMATS - if _FORMATS is not None: - return _FORMATS - # --- Library formats - from .fast_input_file import FASTInputFile - from .fast_output_file import FASTOutputFile - from .csv_file import CSVFile - from .fast_wind_file import FASTWndFile - from .fast_linearization_file import FASTLinearizationFile - from .fast_summary_file import FASTSummaryFile - from .bmodes_out_file import BModesOutFile - from .hawc2_pc_file import HAWC2PCFile - from .hawc2_ae_file import HAWC2AEFile - from .hawc2_dat_file import HAWC2DatFile - from .hawc2_htc_file import HAWC2HTCFile - from .hawc2_st_file import HAWC2StFile - from .hawcstab2_pwr_file import HAWCStab2PwrFile - from .hawcstab2_ind_file import HAWCStab2IndFile - from .hawcstab2_cmb_file import HAWCStab2CmbFile - from .mannbox_file import MannBoxFile - from .flex_blade_file import FLEXBladeFile - from .flex_profile_file import FLEXProfileFile - from .flex_out_file import FLEXOutFile - from .flex_doc_file import FLEXDocFile - from .flex_wavekin_file import FLEXWaveKinFile - from .excel_file import ExcelFile - from .turbsim_ts_file import TurbSimTSFile - from .turbsim_file import TurbSimFile - from .netcdf_file import NetCDFFile - from .tdms_file import TDMSFile - from .tecplot_file import TecplotFile - from .vtk_file import VTKFile - from .bladed_out_file import BladedFile - from .parquet_file import ParquetFile - from .cactus_file import CactusFile - from .raawmat_file import RAAWMatFile - from .rosco_performance_file import ROSCOPerformanceFile - priorities = [] - formats = [] - def addFormat(priority, fmt): - priorities.append(priority) - formats.append(fmt) - addFormat(0, FileFormat(CSVFile)) - addFormat(0, FileFormat(ExcelFile)) - addFormat(10, FileFormat(TecplotFile)) - addFormat(10, FileFormat(BladedFile)) - addFormat(20, FileFormat(FASTInputFile)) - addFormat(20, FileFormat(FASTOutputFile)) - addFormat(20, FileFormat(FASTWndFile)) - addFormat(20, FileFormat(FASTLinearizationFile)) - addFormat(20, FileFormat(FASTSummaryFile)) - addFormat(20, FileFormat(TurbSimTSFile)) - addFormat(20, FileFormat(TurbSimFile)) - addFormat(30, FileFormat(HAWC2DatFile)) - addFormat(30, FileFormat(HAWC2HTCFile)) - addFormat(30, FileFormat(HAWC2StFile)) - addFormat(30, FileFormat(HAWC2PCFile)) - addFormat(30, FileFormat(HAWC2AEFile)) - addFormat(30, FileFormat(HAWCStab2PwrFile)) - addFormat(30, FileFormat(HAWCStab2IndFile)) - addFormat(30, FileFormat(HAWCStab2CmbFile)) - addFormat(30, FileFormat(MannBoxFile)) - addFormat(40, FileFormat(FLEXBladeFile)) - addFormat(40, FileFormat(FLEXProfileFile)) - addFormat(40, FileFormat(FLEXOutFile)) - addFormat(40, FileFormat(FLEXWaveKinFile)) - addFormat(40, FileFormat(FLEXDocFile)) - addFormat(50, FileFormat(BModesOutFile)) - addFormat(50, FileFormat(ROSCOPerformanceFile)) - addFormat(60, FileFormat(NetCDFFile)) - addFormat(60, FileFormat(VTKFile)) - addFormat(60, FileFormat(TDMSFile)) - addFormat(60, FileFormat(ParquetFile)) - addFormat(70, FileFormat(CactusFile)) - addFormat(70, FileFormat(RAAWMatFile)) - - # --- User defined formats from user path - UserClasses, UserPaths, UserModules, UserModuleNames, errors = userFileClasses(userpath, ignoreErrors, verbose=verbose) - for cls, f in zip(UserClasses, UserPaths): - try: - ff = FileFormat(cls) - except Exception as e: - s='Error registering a user fileformat.\n\nThe module location was: {}\n\nThe class name was: {}\n\nMake sure the class has `defaultExtensions` and `formatName` as static methods.\n\nThe exception was:\n{}'.format(f, cls.__name__, e) - if ignoreErrors: - errors.append(s) - continue - else: - raise UserFormatImportError(s) - # Use class.priority - try: - priority = cls.priority() - except: - priority=2 - addFormat(priority, ff) - - # --- Sort fileformats by priorities - formats = np.asarray(formats)[np.argsort(priorities, kind='stable')] - - _FORMATS=formats - if ignoreErrors: - return formats, errors - else: - return formats - - - -def userFileClasses(userpath=None, ignoreErrors=False, verbose=True): - """ return list of user file class in UserData folder""" - if userpath is None: - dataDir = defaultUserDataDir() - userpath = os.path.join(dataDir, 'weio') - errors = [] - UserClasses = [] - UserPaths = [] - UserModules = [] - UserModuleNames = [] - if os.path.exists(userpath): - if verbose: - print('>>> Looking for user modules in folder:',userpath) - import glob - from importlib.machinery import SourceFileLoader - import inspect - pyfiles = glob.glob(os.path.join(userpath,'*.py')) - # Loop through files, look for classes of the form ClassNameFile, - for f in pyfiles: - if f in ['__init__.py']: - continue - mod_name = os.path.basename(os.path.splitext(f)[0]) - try: - if verbose: - print('>>> Trying to load user module:',f) - module = SourceFileLoader(mod_name,f).load_module() - except Exception as e: - s='Error importing a user module.\n\nThe module location was: {}\n\nTry importing this module to debug it.\n\nThe Exception was:\n{}'.format(f, e) - if ignoreErrors: - errors.append(s) - continue - else: - raise UserFormatImportError(s) - found=False - for name, obj in inspect.getmembers(module): - if inspect.isclass(obj): - classname = obj.__name__.lower() - if classname!='file' and classname.find('file')>=0 and classname.find('error')<0: - if verbose: - print(' Found File class with name:',obj.__name__) - UserClasses.append(obj) - UserPaths.append(f) - UserModules.append(module) - UserModuleNames.append(mod_name) - found=True # allowing only one class per file for now.. - break - if not found: - s='Error finding a class named "*File" in the user module.\n\nThe module location was: {}\n\nNo class containing the string "File" in its name was found.'.format(f) - if ignoreErrors: - errors.append(s) - else: - raise UserFormatImportError(s) - return UserClasses, UserPaths, UserModules, UserModuleNames, errors - - -def defaultUserDataDir(): - """ - Returns a parent directory path - where persistent application data can be stored. - # linux: ~/.local/share - # macOS: ~/Library/Application Support - # windows: C:/Users//AppData/Roaming - """ - home = os.path.expanduser('~') - ptfm = sys.platform - if ptfm == "win32": - return os.path.join(home , 'AppData','Roaming') - elif ptfm.startswith("linux"): - return os.path.join(home, '.local', 'share') - elif ptfm == "darwin": - return os.path.join(home, 'Library','Application Support') - else: - print('>>>>>>>>>>>>>>>>> Unknown Platform', sys.platform) - return './UserData' - - - -def detectFormat(filename, **kwargs): - """ Detect the file formats by looping through the known list. - The method may simply try to open the file, if that's the case - the read file is returned. """ - import os - import re - global _FORMATS - if _FORMATS is None: - formats=fileFormats() - else: - formats=_FORMATS - ext = os.path.splitext(filename.lower())[1] - detected = False - i = 0 - while not detected and i0: - extPatMatch = [re.match(pat, ext) is not None for pat in extPatterns] - extMatch = any(extPatMatch) - else: - extMatch = False - if extMatch: # we have a match on the extension - valid, F = isRightFormat(myformat, filename, **kwargs) - if valid: - #print('File detected as :',myformat) - detected=True - return myformat,F - - i += 1 - - if not detected: - raise FormatNotDetectedError('The file format could not be detected for the file: '+filename) - -def read(filename, fileformat=None, **kwargs): - F = None - # Detecting format if necessary - if fileformat is None: - fileformat,F = detectFormat(filename, **kwargs) - # Reading the file with the appropriate class if necessary - if not isinstance(F, fileformat.constructor): - F=fileformat.constructor(filename=filename) - return F - - -# --- For legacy code -def FASTInputFile(*args,**kwargs): - from .fast_input_file import FASTInputFile as fi - return fi(*args,**kwargs) -def FASTOutputFile(*args,**kwargs): - from .fast_output_file import FASTOutputFile as fo - return fo(*args,**kwargs) -def CSVFile(*args,**kwargs): - from .csv_file import CSVFile as csv - return csv(*args,**kwargs) - - +from .file import File, WrongFormatError, BrokenFormatError, FileNotFoundError, EmptyFileError +from .file_formats import FileFormat, isRightFormat +import sys +import os +import numpy as np + +class FormatNotDetectedError(Exception): + pass + +class UserFormatImportError(Exception): + pass + + +_FORMATS=None + +def fileFormats(userpath=None, ignoreErrors=False, verbose=False): + """ return list of fileformats supported by the library + If userpath is provided, + + """ + global _FORMATS + errors=[] + if _FORMATS is not None: + if ignoreErrors: + return _FORMATS, errors + else: + return _FORMATS + # --- Library formats + from .fast_input_file import FASTInputFile + from .fast_output_file import FASTOutputFile + from .csv_file import CSVFile + from .fast_wind_file import FASTWndFile + from .fast_linearization_file import FASTLinearizationFile + from .fast_summary_file import FASTSummaryFile + from .bmodes_out_file import BModesOutFile + from .hawc2_pc_file import HAWC2PCFile + from .hawc2_ae_file import HAWC2AEFile + from .hawc2_dat_file import HAWC2DatFile + from .hawc2_htc_file import HAWC2HTCFile + from .hawc2_st_file import HAWC2StFile + from .hawcstab2_pwr_file import HAWCStab2PwrFile + from .hawcstab2_ind_file import HAWCStab2IndFile + from .hawcstab2_cmb_file import HAWCStab2CmbFile + from .mannbox_file import MannBoxFile + from .flex_blade_file import FLEXBladeFile + from .flex_profile_file import FLEXProfileFile + from .flex_out_file import FLEXOutFile + from .flex_doc_file import FLEXDocFile + from .flex_wavekin_file import FLEXWaveKinFile + from .excel_file import ExcelFile + from .turbsim_ts_file import TurbSimTSFile + from .turbsim_file import TurbSimFile + from .netcdf_file import NetCDFFile + from .tdms_file import TDMSFile + from .tecplot_file import TecplotFile + from .vtk_file import VTKFile + from .bladed_out_file import BladedFile + from .parquet_file import ParquetFile + from .cactus_file import CactusFile + from .raawmat_file import RAAWMatFile + from .rosco_performance_file import ROSCOPerformanceFile + priorities = [] + formats = [] + def addFormat(priority, fmt): + priorities.append(priority) + formats.append(fmt) + addFormat(0, FileFormat(CSVFile)) + addFormat(0, FileFormat(ExcelFile)) + addFormat(10, FileFormat(TecplotFile)) + addFormat(10, FileFormat(BladedFile)) + addFormat(20, FileFormat(FASTInputFile)) + addFormat(20, FileFormat(FASTOutputFile)) + addFormat(20, FileFormat(FASTWndFile)) + addFormat(20, FileFormat(FASTLinearizationFile)) + addFormat(20, FileFormat(FASTSummaryFile)) + addFormat(20, FileFormat(TurbSimTSFile)) + addFormat(20, FileFormat(TurbSimFile)) + addFormat(30, FileFormat(HAWC2DatFile)) + addFormat(30, FileFormat(HAWC2HTCFile)) + addFormat(30, FileFormat(HAWC2StFile)) + addFormat(30, FileFormat(HAWC2PCFile)) + addFormat(30, FileFormat(HAWC2AEFile)) + addFormat(30, FileFormat(HAWCStab2PwrFile)) + addFormat(30, FileFormat(HAWCStab2IndFile)) + addFormat(30, FileFormat(HAWCStab2CmbFile)) + addFormat(30, FileFormat(MannBoxFile)) + addFormat(40, FileFormat(FLEXBladeFile)) + addFormat(40, FileFormat(FLEXProfileFile)) + addFormat(40, FileFormat(FLEXOutFile)) + addFormat(40, FileFormat(FLEXWaveKinFile)) + addFormat(40, FileFormat(FLEXDocFile)) + addFormat(50, FileFormat(BModesOutFile)) + addFormat(50, FileFormat(ROSCOPerformanceFile)) + addFormat(60, FileFormat(NetCDFFile)) + addFormat(60, FileFormat(VTKFile)) + addFormat(60, FileFormat(TDMSFile)) + addFormat(60, FileFormat(ParquetFile)) + addFormat(70, FileFormat(CactusFile)) + addFormat(70, FileFormat(RAAWMatFile)) + + # --- User defined formats from user path + UserClasses, UserPaths, UserModules, UserModuleNames, errors = userFileClasses(userpath, ignoreErrors, verbose=verbose) + for cls, f in zip(UserClasses, UserPaths): + try: + ff = FileFormat(cls) + except Exception as e: + s='Error registering a user fileformat.\n\nThe module location was: {}\n\nThe class name was: {}\n\nMake sure the class has `defaultExtensions` and `formatName` as static methods.\n\nThe exception was:\n{}'.format(f, cls.__name__, e) + if ignoreErrors: + errors.append(s) + continue + else: + raise UserFormatImportError(s) + # Use class.priority + try: + priority = cls.priority() + except: + priority=2 + addFormat(priority, ff) + + # --- Sort fileformats by priorities + formats = np.asarray(formats)[np.argsort(priorities, kind='stable')] + + _FORMATS=formats + if ignoreErrors: + return formats, errors + else: + return formats + + + +def userFileClasses(userpath=None, ignoreErrors=False, verbose=True): + """ return list of user file class in UserData folder""" + if userpath is None: + dataDir = defaultUserDataDir() + userpath = os.path.join(dataDir, 'weio') + errors = [] + UserClasses = [] + UserPaths = [] + UserModules = [] + UserModuleNames = [] + if os.path.exists(userpath): + if verbose: + print('>>> Looking for user modules in folder:',userpath) + import glob + from importlib.machinery import SourceFileLoader + import inspect + pyfiles = glob.glob(os.path.join(userpath,'*.py')) + # Loop through files, look for classes of the form ClassNameFile, + for f in pyfiles: + if f in ['__init__.py']: + continue + mod_name = os.path.basename(os.path.splitext(f)[0]) + try: + if verbose: + print('>>> Trying to load user module:',f) + module = SourceFileLoader(mod_name,f).load_module() + except Exception as e: + s='Error importing a user module.\n\nThe module location was: {}\n\nTry importing this module to debug it.\n\nThe Exception was:\n{}'.format(f, e) + if ignoreErrors: + errors.append(s) + continue + else: + raise UserFormatImportError(s) + found=False + for name, obj in inspect.getmembers(module): + if inspect.isclass(obj): + classname = obj.__name__.lower() + if classname!='file' and classname.find('file')>=0 and classname.find('error')<0: + if verbose: + print(' Found File class with name:',obj.__name__) + UserClasses.append(obj) + UserPaths.append(f) + UserModules.append(module) + UserModuleNames.append(mod_name) + found=True # allowing only one class per file for now.. + break + if not found: + s='Error finding a class named "*File" in the user module.\n\nThe module location was: {}\n\nNo class containing the string "File" in its name was found.'.format(f) + if ignoreErrors: + errors.append(s) + else: + raise UserFormatImportError(s) + return UserClasses, UserPaths, UserModules, UserModuleNames, errors + + +def defaultUserDataDir(): + """ + Returns a parent directory path + where persistent application data can be stored. + # linux: ~/.local/share + # macOS: ~/Library/Application Support + # windows: C:/Users//AppData/Roaming + """ + home = os.path.expanduser('~') + ptfm = sys.platform + if ptfm == "win32": + return os.path.join(home , 'AppData','Roaming') + elif ptfm.startswith("linux"): + return os.path.join(home, '.local', 'share') + elif ptfm == "darwin": + return os.path.join(home, 'Library','Application Support') + else: + print('>>>>>>>>>>>>>>>>> Unknown Platform', sys.platform) + return './UserData' + + + +def detectFormat(filename, **kwargs): + """ Detect the file formats by looping through the known list. + The method may simply try to open the file, if that's the case + the read file is returned. """ + import os + import re + global _FORMATS + if _FORMATS is None: + formats=fileFormats() + else: + formats=_FORMATS + ext = os.path.splitext(filename.lower())[1] + detected = False + i = 0 + while not detected and i0: + extPatMatch = [re.match(pat, ext) is not None for pat in extPatterns] + extMatch = any(extPatMatch) + else: + extMatch = False + if extMatch: # we have a match on the extension + valid, F = isRightFormat(myformat, filename, **kwargs) + if valid: + #print('File detected as :',myformat) + detected=True + return myformat,F + + i += 1 + + if not detected: + raise FormatNotDetectedError('The file format could not be detected for the file: '+filename) + +def read(filename, fileformat=None, **kwargs): + F = None + # Detecting format if necessary + if fileformat is None: + fileformat,F = detectFormat(filename, **kwargs) + # Reading the file with the appropriate class if necessary + if not isinstance(F, fileformat.constructor): + F=fileformat.constructor(filename=filename) + return F + + +# --- For legacy code +def FASTInputFile(*args,**kwargs): + from .fast_input_file import FASTInputFile as fi + return fi(*args,**kwargs) +def FASTOutputFile(*args,**kwargs): + from .fast_output_file import FASTOutputFile as fo + return fo(*args,**kwargs) +def CSVFile(*args,**kwargs): + from .csv_file import CSVFile as csv + return csv(*args,**kwargs) + + From 3d8810d9db0f381a6aca7ee523c7da76e82ff169 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 15 Aug 2022 22:29:20 -0600 Subject: [PATCH 034/178] Small fix to merge trigger --- pydatview/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydatview/main.py b/pydatview/main.py index 4be5cec..1f18be6 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -402,7 +402,8 @@ def sortTabs(self, method='byName'): def mergeTabsTrigger(self): if hasattr(self,'selPanel'): # Select the newly created table - self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)) + self.selPanel.tabPanel.lbTab.SetSelection(-1) # Empty selection + self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)-1) # Select new/last table # Trigger a replot self.onTabSelectionChange() From e2ef17c85c4937ac98cb68eb9be96ae6b659c70d Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 15 Aug 2022 22:33:43 -0600 Subject: [PATCH 035/178] HAWC2: adding computed quantities for st file --- pydatview/io/hawc2_st_file.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/pydatview/io/hawc2_st_file.py b/pydatview/io/hawc2_st_file.py index 0b484f9..571daef 100644 --- a/pydatview/io/hawc2_st_file.py +++ b/pydatview/io/hawc2_st_file.py @@ -47,7 +47,7 @@ def __repr__(self): s='<{} object> with attribute `data`\n'.format(type(self).__name__) return s - def toDataFrame(self): + def toDataFrame(self, extraCols=True): col_reg=['r_[m]','m_[kg/m]','x_cg_[m]','y_cg_[m]','ri_x_[m]','ri_y_[m]', 'x_sh_[m]','y_sh_[m]','E_[N/m^2]','G_[N/m^2]','I_x_[m^4]','I_y_[m^4]','I_p_[m^4]','k_x_[-]','k_y_[-]','A_[m^2]','pitch_[deg]','x_e_[m]','y_e_[m]'] col_fpm=['r_[m]','m_[kg/m]','x_cg_[m]','y_cg_[m]','ri_x_[m]','ri_y_[m]','pitch_[deg]','x_e_[m]','y_e_[m]','K11','K12','K13','K14','K15','K16','K22','K23','K24','K25','K26','K33','K34','K35','K36','K44','K45','K46','K55','K56','K66'] @@ -57,8 +57,25 @@ def toDataFrame(self): for iset in self.data.main_data_sets[mset].keys(): tab = self.data.main_data_sets[mset][iset] if tab.shape[1]==19: - col=col_reg + FPM = False + col = col_reg else: - col=col_fpm - dfs['{}_{}'.format(mset,iset)] = pd.DataFrame(data =tab, columns=col ) + FPM = True + col = col_fpm + df = pd.DataFrame(data =tab, columns=col ) + if extraCols: + df['Ixi_[kg.m]'] = df['ri_x_[m]']**2 * df['m_[kg/m]'] + df['Iyi_[kg.m]'] = df['ri_y_[m]']**2 * df['m_[kg/m]'] + df['StrcTwst_[deg]'] = -df['pitch_[deg]'] + if not FPM: + df['EIx_[Nm^2]'] = df['E_[N/m^2]']*df['I_x_[m^4]'] + df['EIy_[Nm^2]'] = df['E_[N/m^2]']*df['I_y_[m^4]'] + df['GKt_[Nm^2]'] = df['G_[N/m^2]']*df['I_p_[m^4]'] + df['EA_[N]'] = df['E_[N/m^2]']*df['A_[m^2]'] + df['GA_[N]'] = df['G_[N/m^2]']*df['A_[m^2]'] + df['r_bar_[-]'] = df['r_[m]']/df['r_[m]'].values[-1] + + dfs['{}_{}'.format(mset,iset)] = df + + return dfs From 05c85f1fc12750f9d19609a32f24c1892ad624ff Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 16 Aug 2022 13:57:53 -0600 Subject: [PATCH 036/178] Update of io --- pydatview/io/fast_input_file.py | 5 +- pydatview/io/rosco_performance_file.py | 265 ++++++++++++++++++++++--- pydatview/io/tdms_file.py | 151 ++++++++++++-- pydatview/tools/signal_analysis.py | 3 + 4 files changed, 367 insertions(+), 57 deletions(-) diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index 8e9a190..e5f493f 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -42,7 +42,7 @@ class FASTInputFile(File): @staticmethod def defaultExtensions(): - return ['.dat','.fst','.txt','.fstf'] + return ['.dat','.fst','.txt','.fstf','.dvr'] @staticmethod def formatName(): @@ -1813,7 +1813,8 @@ def _toDataFrame(self): if __name__ == "__main__": - f = FASTInputFile() + f = FASTInputFile('tests/example_files/FASTIn_HD_SeaState.dat') + print(f) pass #B=FASTIn('Turbine.outb') diff --git a/pydatview/io/rosco_performance_file.py b/pydatview/io/rosco_performance_file.py index fac98b5..c075ef3 100644 --- a/pydatview/io/rosco_performance_file.py +++ b/pydatview/io/rosco_performance_file.py @@ -6,7 +6,7 @@ import os try: - from .file import File, WrongFormatError, BrokenFormatError + from .file import File, EmptyFileError, WrongFormatError, BrokenFormatError except: EmptyFileError = type('EmptyFileError', (Exception,),{}) WrongFormatError = type('WrongFormatError', (Exception,),{}) @@ -26,6 +26,10 @@ class ROSCOPerformanceFile(File): f = ROSCOPerformanceFile('Cp_Ct_Cq.txt') print(f.keys()) print(f.toDataFrame().columns) + fig = f.plotCP3D() + CPmax, tsr_max, pitch_max = f.CPmax() + CP = fCP([0, 1], [5, 5]) + CT = fCT([0, 1], [5, 5]) """ @@ -39,15 +43,38 @@ def formatName(): """ Short string (~100 char) identifying the file format""" return 'ROSCO Performance file' - def __init__(self, filename=None, **kwargs): - """ Class constructor. If a `filename` is given, the file is read. """ + def __init__(self, filename=None, pitch=None, tsr=None, WS=None, CP=None, CT=None, CQ=None, name='',**kwargs): + """ Class constructor. If a `filename` is given, the file is read. + Otherwise values may be provided directly + + INPUTS: + - filename: input file for ROSCO Performance file + OR + - pitch: pitch angle [deg], array of length nPitch + - tsr: tip-speed ratio [-], array of length nTSR + - CP,CT,CQ: aerodynamic coefficients, arrays of shape nTSR x nPitch + CQ is optional since CP = tsr*CQ + - name: wind turbine name + """ self.filename = filename + self.name = name # Turbine name + self['pitch'] = pitch + self['TSR'] = tsr + self['WS'] = WS + self['CP'] = CP + self['CT'] = CT + self['CQ'] = CQ + if filename: self.read(**kwargs) - def read(self, filename=None, **kwargs): - """ Reads the file self.filename, or `filename` if provided """ + if self['pitch'] is not None: + self.checkConsistency() + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided + stores data into self. + self is (or behaves like) a dictionary""" # --- Standard tests and exceptions (generic code) if filename: self.filename = filename @@ -57,8 +84,14 @@ def read(self, filename=None, **kwargs): raise OSError(2,'File not found:',self.filename) if os.stat(self.filename).st_size == 0: raise EmptyFileError('File is empty:',self.filename) - # --- Calling (children) function to read - self._read(**kwargs) + + pitch, TSR, WS, CP, CT, CQ = load_from_txt(self.filename) + self['pitch'] = pitch + self['TSR'] = TSR + self['WS'] = WS + self['CP'] = CP + self['CT'] = CT + self['CQ'] = CQ def write(self, filename=None): """ Rewrite object to file, or write object to `filename` if provided """ @@ -66,24 +99,41 @@ def write(self, filename=None): self.filename = filename if not self.filename: raise Exception('No filename provided') - # Calling (children) function to write - self._write() + # Sanity + self.checkConsistency() + # Write + write_rotor_performance(self.filename, self['pitch'], self['TSR'], self['CP'],self['CT'], self['CQ'], self['WS'], TurbineName=self.name) - def _read(self): - """ Reads self.filename and stores data into self. Self is (or behaves like) a dictionary""" - # --- Example: - pitch, TSR, WS, Cp, Ct, Cq = load_from_txt(self.filename) - self['pitch'] = pitch - self['TSR'] = TSR - self['WS'] = WS - self['CP'] = Cp - self['CT'] = Ct - self['CQ'] = Cq + def checkConsistency(self): + """ + Check that data makes sense. + in particular, check if CP=lambda CQ + """ + if self['WS'] is not None: + if not hasattr(self['WS'],'__len__' ): + self['WS'] = np.array([self['WS']]).astype(float) - def _write(self): - """ Writes to self.filename""" - # --- Example: - write_rotor_performance(self.filename, self['pitch'], self['TSR'], self['CP'],self['CT'], self['CQ'], self['WS'], TurbineName='') + CQ = self['CQ'] + CP = self['CP'] + tsr = np.asarray(self['TSR']) + TSR = np.tile(tsr.flatten(), (len(self['pitch']),1)).T + if CQ is None and CP is not None: + CQ = CP/TSR + print('[INFO] Computing CQ from CP') + elif CQ is not None and CP is None: + CP = CQ*TSR + print('[INFO] Computing CP from CQ') + elif CQ is not None and CP is not None: + pass + else: + raise Exception('CP and CQ cannot be None') + # Check consistency + CP2 = CQ*TSR + deltaCP = np.abs(CP-CP2)/0.5*100 # relative difference in %, for a mean CP of 0.5 + if np.max(deltaCP)>5: # more than 5% + raise Exception('Inconsitency between power coefficient and torque coefficient') + self['CP'] = CP + self['CQ'] = CQ def toDataFrame(self): """ Returns object into dictionary of DataFrames""" @@ -95,6 +145,131 @@ def toDataFrame(self): return dfs # --- Optional functions + def toAeroDisc(self, filename, R, csv=False, WS=None, omegaRPM=10): + """ Convert to AeroDisc Format + INPUTS: + - filename: filename to be written + - R: rotor radius [m] + - csv: if True write to CSV format, else, use OpenFAST .dat format + either: + - WS: wind speed [m/s] + or + - omegaRPM: rotational speed [rpm] + + Logic to determine wind speed or rotational speed: + - If user provide a wind speed, we use it. Omega is determined from TSR and WS + - If user provide a rotational speed, we use it. WS is determined from TSR and omega + - If ROSCO file contain one wind speed, we use it. + - Otherwise, we don't know what to do so we raise an exception + """ + # --- Logic to determine wind speed or rotational speed + if WS is not None: + WS = WS + elif omegaRPM is not None: + WS = None + omega = omegaRPM*(2*np.pi)/60 + elif self['WS'] is not None and len(self['WS'])==1: + WS = self['WS'][0] + else: + raise Exception('Provide either a wind speed (`WS`) or a rotational speed (`omegaRPM`)') + + with open(filename,'w') as fid: + # Header + if csv: + fid.write("TSR_(-), RtSpd_(rpm) , VRel_(m/s) , Skew_(deg) , Pitch_(deg) , C_Fx_(-) , C_Fy_(-) , C_Fz_(-) , C_Mx_(-) , C_My_(-) , C_Mz_(-)\n") + else: + fid.write(' TSR RtSpd VRel Skew Pitch C_Fx C_Fy C_Fz C_Mx C_My C_Mz\n') + fid.write(' (-) (rpm) (m/s) (deg) (deg) (-) (-) (-) (-) (-) (-)\n') + if csv: + FMT='{:10.4f},{:10.4f},{:10.4f},{:10.4f},{:10.4f},{:10.4f},{:10.4f},{:10.4f},{:10.4f},{:10.4f},{:10.4f}\n' + else: + FMT='{:10.4f} {:10.4f} {:10.4f} {:10.4f} {:10.4f} {:10.4f} {:10.4f} {:10.4f} {:10.4f} {:10.4f} {:10.4f}\n' + # Loop on oper + for j,tsr in enumerate(self['TSR']): + if WS is None: + U0 = omega*R/tsr + else: + U0 = WS + omega = tsr*U0/R + omegaRPM = omega*60/(2*np.pi) + for i,p in enumerate(self['pitch']): + CP=self['CP'][j,i] + CT=self['CT'][j,i] + CQ=self['CQ'][j,i] + skew=0 + cfx=CT + cfy=0 + cfz=0 + cmx=CQ + cmy=0 + cmz=0 + fid.write(FMT.format(tsr, omegaRPM, U0, skew, p, cfx,cfy,cfz,cmx,cmy,cmz)) + + def computeWeights(self): + """ Compute interpolant weights for fast evaluation of CP and CT at intermediate values""" + CP = self['CP'].copy() + CT = self['CT'].copy() + CP = CP[CP<0]=0 + CT = CT[CT<0]=0 + self._fCP = interp2d_pairs(self['pitch'], self['TSR'], CP, kind='cubic') + self._fCT = interp2d_pairs(self['pitch'], self['TSR'], CT, kind='cubic') + + def fCP(self, pitch, tsr): + """ Compute CP for given pitch and tsr, where inputs can be scalar, arrays or matrices""" + if self._fCP is None: + self.computeWeights() + return self.fCP(pitch, tsr) + + def fCT(self, pitch, tsr): + """ Compute CT for given pitch and tsr, where inputs can be scalar, arrays or matrices""" + if self._fCT is None: + self.computeWeights() + return self.fCT(pitch, tsr) + + def CPmax(self): + """ return values at CPmax + TODO: interpolation instead of nearest value.. + """ + CP = self['CP'] + i,j = np.unravel_index(CP.argmax(), CP.shape) + CPmax, tsr_max, pitch_max = CP[i,j], self['TSR'][i], self['pitch'][j] + + return CPmax, tsr_max, pitch_max + + def plotCP3D(self, plotMax=True, trajectory=None): + """ + Plot 3D surface of CP + Optionally plot the maximum and a controller trajectory + """ + import matplotlib.pyplot as plt + from mpl_toolkits.mplot3d import Axes3D + from matplotlib import cm + # Data + LAMBDA, PITCH = np.meshgrid(self['TSR'], self['pitch']) + CP = self['CP'].copy() + CP[CP<0]=0 # + CP_max, tsr_max, pitch_max = self.CPmax() + # plot + fig = plt.figure() + ax = fig.gca(projection='3d') + surf = ax.plot_surface(LAMBDA, PITCH, np.transpose(CP), cmap=cm.coolwarm, linewidth=0, antialiased=True,alpha=0.8, label='$C_p$') + if plotMax: + ax.scatter(tsr_max, pitch_max, CP_max, c='k', marker='o', s=50, label=r'$C_{p,max}$') + if trajectory is not None: + if len(trajectory)==3: + tsr_, pitch_, CP_ = trajectory + else: + tsr_, pitch_ = trajectory + CP_ = self.fCP(tsr_, pitch_) + ax.plot_surface(tsr_, pitch_, CP_, 'k-', linewidth=1 ) + #fig.tight_layout() + #fig.colorbar(surf, shrink=0.5, aspect=15) + ax.view_init(elev=20., azim=26) + ax.set_xlabel('TSR [-]') + ax.set_ylabel('Pitch [deg]') + ax.set_zlabel(r'Power coefficient [-]') + return fig + def __repr__(self): """ String that is written to screen when the user calls `print()` on the object. Provide short and relevant information to save time for the user. @@ -104,12 +279,14 @@ def __repr__(self): s+='| - filename: {}\n'.format(self.filename) # --- Example printing some relevant information for user s+='|Main keys:\n' - s+='| - pitch: {}\n'.format(self['pitch']) - s+='| - TSR: {}\n'.format(self['TSR']) - s+='| - WS: {}\n'.format(self['WS']) - s+='| - CP,CT,CQ : shape {}\n'.format(self['CP'].shape) + s+='| - pitch: {} values: {}\n'.format(len(self['pitch']) if self['pitch'] is not None else 0, self['pitch']) + s+='| - TSR: {} values: {}\n'.format(len(self['TSR'] ) if self['TSR'] is not None else 0, self['TSR'] ) + s+='| - WS: {} values: {}\n'.format(len(self['WS'] ) if self['WS'] is not None else 0, self['WS'] ) + if self['CP'] is not None: + s+='| - CP,CT,CQ : shape {}\n'.format(self['CP'].shape) s+='|Main methods:\n' - s+='| - read, write, toDataFrame, keys' + s+='| - read, write, toDataFrame, keys\n' + s+='| - CPmax, plotCP3d, fCP, fCT, toAeroDisc' return s @@ -117,7 +294,7 @@ def __repr__(self): def load_from_txt(txt_filename): ''' - Taken from ROSCO_toolbox/utitities.py by Nikhar Abbas + Adapted from ROSCO_toolbox/utitities.py by Nikhar Abbas https://github.com/NREL/ROSCO Apache 2.0 License @@ -175,7 +352,7 @@ def load_from_txt(txt_filename): def write_rotor_performance(txt_filename, pitch, TSR, CP, CT, CQ, WS=None, TurbineName=''): ''' - Taken from ROSCO_toolbox/utitities.py by Nikhar Abbas + Adapted from ROSCO_toolbox/utitities.py by Nikhar Abbas https://github.com/NREL/ROSCO Apache 2.0 License @@ -187,8 +364,8 @@ def write_rotor_performance(txt_filename, pitch, TSR, CP, CT, CQ, WS=None, Turbi ''' file = open(txt_filename,'w') # Headerlines - file.write('# ----- Rotor performance tables for the {} wind turbine ----- \n'.format(TurbineName)) - file.write('# ------------ Written on {} using the ROSCO toolbox ------------ \n\n'.format(now.strftime('%b-%d-%y'))) + file.write('# ----- Rotor performance tables for the wind turbine: {} ----- \n'.format(TurbineName)) + file.write('# ------------ Written using weio\n\n') # Pitch angles, TSR, and wind speed file.write('# Pitch angle vector, {} entries - x axis (matrix columns) (deg)\n'.format(len(pitch))) @@ -200,7 +377,7 @@ def write_rotor_performance(txt_filename, pitch, TSR, CP, CT, CQ, WS=None, Turbi if WS is not None: file.write('\n# Wind speed vector - z axis (m/s)\n') for i in range(len(WS)): - file.write('{:0.4} '.format(WS[i])) + file.write('{:0.4f} '.format(WS[i])) file.write('\n') # Cp @@ -229,6 +406,28 @@ def write_rotor_performance(txt_filename, pitch, TSR, CP, CT, CQ, WS=None, Turbi file.close() +def interp2d_pairs(*args,**kwargs): + """ Same interface as interp2d but the returned interpolant will evaluate its inputs as pairs of values. + Inputs can therefore be arrays + + example: + f = interp2d_pairs(vx, vy, M, kind='cubic') + + vx: array of length nx + vy: array of length ny + M : array of shape nx x ny + f : interpolant function + v = f(x,y) : if x,y are array of length n, v is of length n + with v_i = f(x_i, y_i) + author: E. Branlard + """ + import scipy.interpolate as si + # Internal function, that evaluates pairs of values, output has the same shape as input + def interpolant(x,y,f): + x,y = np.asarray(x), np.asarray(y) + return (si.dfitpack.bispeu(f.tck[0], f.tck[1], f.tck[2], f.tck[3], f.tck[4], x.ravel(), y.ravel())[0]).reshape(x.shape) + # Wrapping the scipy interp2 function to call out interpolant instead + return lambda x,y: interpolant(x,y,si.interp2d(*args,**kwargs)) if __name__ == '__main__': diff --git a/pydatview/io/tdms_file.py b/pydatview/io/tdms_file.py index 2d21bdb..1d4cae1 100644 --- a/pydatview/io/tdms_file.py +++ b/pydatview/io/tdms_file.py @@ -77,6 +77,36 @@ def read(self, filename=None, **kwargs): # --- NEW self['data'] = fh + #for group in fh.groups(): + # for channel in group.channels(): + # #channel = group['channel name'] + # print('Group:',group.name , 'Chan:',channel.name) + # channel_data = channel[:] + # if len(channel_data)>0: + # print(' ', type(channel_data)) + # #print(' ', len(channel_data)) + # print(' ', channel_data) + # print(' ', channel_data[0]) + # try: + # print(channel.time_track()) + # except KeyError: + # print('>>> No time track') + + def write(self, filename=None, df=None): + """" + Write to TDMS file. + NOTE: for now only using a conversion from dataframe... + """ + if filename is None: + filename = self.filename + if df is None: + df = self.toDataFrame(split=False) + writeTDMSFromDataFrame(filename, df) + + + def groups(self): + return self['data'].groups() + @property def groupNames(self): return [group.name for group in self['data'].groups()] @@ -91,27 +121,104 @@ def __repr__(self): # print(channel.name) return s - def toDataFrame(self): - - df = self['data'].as_dataframe(time_index=True) - - # Cleanup columns - colnames = df.columns - colnames=[c.replace('\'','') for c in colnames] - colnames=[c[1:] if c.startswith('/') else c for c in colnames] - # If there is only one group, we remove the group key - groupNames = self.groupNames - if len(groupNames)==1: - nChar = len(groupNames[0]) - colnames=[c[nChar+1:] for c in colnames] # +1 for the "/" - - df.columns = colnames - - df.insert(0,'Time_[s]', df.index.values) - df.index=np.arange(0,len(df)) - - return df + def toDataFrame(self, split=True): + """ Export to one (split=False) or several dataframes (split=True) + Splitting on the group + """ + + def cleanColumns(df): + # Cleanup columns + colnames = df.columns + colnames=[c.replace('\'','') for c in colnames] + colnames=[c[1:] if c.startswith('/') else c for c in colnames] + # If there is only one group, we remove the group key + groupNames = self.groupNames + if len(groupNames)==1: + nChar = len(groupNames[0]) + colnames=[c[nChar+1:] for c in colnames] # +1 for the "/" + df.columns = colnames + + fh = self['data'] + if split: + # --- One dataframe per group. We skip group that have empty data + dfs={} + for group in fh.groups(): + try: + df = group.as_dataframe(time_index=True) + df.insert(0,'Time_[s]', df.index.values) + df.index=np.arange(0,len(df)) + except KeyError: + df = group.as_dataframe(time_index=False) + if len(df)>0: + dfs[group.name] = df + if len(dfs)==1: + dfs=dfs[group.name] + return dfs + else: + # --- One dataframe with all data + try: + df = fh.as_dataframe(time_index=True) + cleanColumns(df) + df.insert(0,'Time_[s]', df.index.values) + df.index=np.arange(0,len(df)) + except KeyError: + df = fh.as_dataframe(time_index=False) + return df + +def writeTDMSFromDataFrame(filename, df, defaultGroupName='default'): + """ + Write a TDMS file from a pandas dataframe + + Example: + # --- Create a TDMS file - One group two channels with time track + time = np.linspace(0,1,20) + colA = np.random.normal(0,1,20) + colB = np.random.normal(0,1,20) + df = pd.DataFrame(data={'Time_[s]':time ,'ColA':colA,'ColB':colB}) + writeTDMSFromDataFrame('out12.tdms', df, defaultGroupName = 'myGroup') + + #--- Create a TDMS file - Two groups, two channels without time track but with timestamp + TS = np.arange('2010-02', '2010-02-21', dtype='datetime64[D]') + df = pd.DataFrame(data={'GroupA/ColTime':time,'GroupA/ColA':colA,'GroupB/ColTimestamp': TS,'GroupB/ColA':colB)}) + writeTDMSFromDataFrame('out22.tdms', df) + + """ + from nptdms import TdmsWriter, ChannelObject + + defaultGroupName = 'default' + + columns =df.columns + + # Check if first column is time + if columns[0].lower().find('time')==0: + t = df.iloc[:,0].values + n = len(t) + dt1 = (np.max(t)-np.min(t))/(n-1) + if n>1: + dt2 = t[1]-t[0] + timeProperties = {'wf_increment':dt1, 'wf_start_offset':t[0]} + columns = columns[1:] # We remove the time column + else: + timeProperties = {} + + with TdmsWriter(filename) as tdms_writer: + + channels=[] + for iCol, col in enumerate(columns): + sp = col.split('/') + if len(sp)==2: + groupName = sp[0] + channelName = sp[1] + else: + groupName = defaultGroupName + channelName = col + data_array = df[col].values + channels.append(ChannelObject(groupName, channelName, data_array, timeProperties)) + tdms_writer.write_segment(channels) if __name__ == '__main__': - df = TDMSFile('DOE15_FastData_2019_11_19_13_51_35_50Hz.tdms').toDataFrame() - print(df) + pass +# f = TDMSFile('TDMS_.tdms') +# dfs = f.toDataFrame(split=True) +# print(f) + diff --git a/pydatview/tools/signal_analysis.py b/pydatview/tools/signal_analysis.py index bb372d6..0462258 100644 --- a/pydatview/tools/signal_analysis.py +++ b/pydatview/tools/signal_analysis.py @@ -360,6 +360,9 @@ def zero_crossings(y,x=None,direction=None): """ if x is None: x=np.arange(len(y)) + else: + x = np.asarray(x) + y = np.asarray(y) if np.any((x[1:] - x[0:-1]) <= 0.0): raise Exception('x values need to be in ascending order') From 658f59f223876293bc7ccf0fdeedc6606d60698c Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 15 Sep 2022 15:36:31 -0600 Subject: [PATCH 037/178] Merge branch f/no-index into dev (see #127) --- pydatview/GUISelectionPanel.py | 38 +-- pydatview/Tables.py | 79 ++--- pydatview/fast/case_gen.py | 589 --------------------------------- pydatview/fast/fastfarm.py | 2 +- pydatview/fast/fastlib.py | 6 - pydatview/fast/postpro.py | 9 +- pydatview/fast/runner.py | 190 ----------- 7 files changed, 58 insertions(+), 855 deletions(-) delete mode 100644 pydatview/fast/case_gen.py delete mode 100644 pydatview/fast/fastlib.py delete mode 100644 pydatview/fast/runner.py diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index ef588e2..af03262 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -317,19 +317,17 @@ def OnMergeTabs(self, event): commonCol = None ICommonColPerTab = None samplDict = None - if nCommonCols>=1: + if nCommonCols>=2: # NOTE: index will always be a duplicated... # We use the first one # TODO Menu to figure out which column to chose and how to merge (interp?) keepAllX = True #samplDict ={'name':'Replace', 'param':[], 'paramName':'New x'} # Index of common column for each table - ICommonColPerTab = [I[0] for I in IKeepPerTab] + ICommonColPerTab = [I[1] for I in IKeepPerTab] else: # we'll merge based on index.. pass - - # Merge tables and add it to the list self.tabList.mergeTabs(self.ISel, ICommonColPerTab, samplDict=samplDict) # Updating tables @@ -429,14 +427,14 @@ def OnEditColumn(self, event): break else: raise ValueError('No formula found at {0} for table {1}!'.format(self.ISel[0], sTab)) - self.showFormulaDialog('Edit column', sName, sFormula) + self.showFormulaDialog('Edit column', sName, sFormula, edit=True) def OnDeleteColumn(self, event): main=self.parent.mainframe iX = self.parent.comboX.GetSelection() ITab,STab=main.selPanel.getSelectedTables() # TODO adapt me for Sim. tables mode - IFull = [self.parent.Filt2Full[iFilt]-1 for iFilt in self.ISel] + IFull = [self.parent.Filt2Full[iFilt] for iFilt in self.ISel] IFull = [iFull for iFull in IFull if iFull>=0] if main.tabList.haveSameColumns(ITab): for iTab,sTab in zip(ITab,STab): @@ -451,7 +449,7 @@ def OnAddColumn(self, event): main=self.parent.mainframe self.showFormulaDialog('Add a new column') - def showFormulaDialog(self, title, name='', formula=''): + def showFormulaDialog(self, title, name='', formula='', edit=False): bValid=False bCancelled=False main=self.parent.mainframe @@ -491,10 +489,12 @@ def showFormulaDialog(self, title, name='', formula=''): for iTab,sTab in zip(ITab,STab): if haveSameColumns or self.parent.tab.active_name == sTab: # apply formula to all tables with same columns, otherwise only to active table - if title.startswith('Edit'): + if edit: bValid=main.tabList.get(iTab).setColumnByFormula(sName,sFormula,iFull) + iOffset = 0 # we'll stay on this column that we are editing else: bValid=main.tabList.get(iTab).addColumnByFormula(sName,sFormula,iFull) + iOffset = 1 # we'll select this newly created column if not bValid: sError+='The formula didn''t eval for table {}\n'.format(sTab) nError+=1 @@ -509,7 +509,7 @@ def showFormulaDialog(self, title, name='', formula=''): if bValid: iX = self.parent.comboX.GetSelection() self.parent.setColumns() - self.parent.setGUIColumns(xSel=iX,ySel=[iFull+1]) + self.parent.setGUIColumns(xSel=iX,ySel=[iFull+iOffset]) main.redraw() @@ -782,7 +782,7 @@ def setColumns(self, columnNames=None): columns=self.columns else: # Populating based on table (safest if table was updated) - columns=['Index']+self.tab.columns + columns=self.tab.columns # Storing columns, considered as "Full" self.columns=np.array(columns) @@ -1216,7 +1216,7 @@ def setColForSimTab(self,ISel): - colNames = ['Index'] + [tabs[0].columns[i] for i in IKeepPerTab[0]] + colNames = [tabs[0].columns[i] for i in IKeepPerTab[0]] # restore selection xSel = -1 @@ -1372,18 +1372,10 @@ def getPlotDataSelection(self): for i,(itab,stab) in enumerate(zip(ITab,STab)): IKeep=self.IKeepPerTab[i] for j,(iiy,ssy) in enumerate(zip(IY1,SY1)): - if iiy==0: - iy = 0 - sy = ssy - else: - iy = IKeep[iiy-1]+1 - sy = self.tabList.get(itab).columns[IKeep[iiy-1]] - if iiX1==0: - iX1 = 0 - sX1 = ssX1 - else: - iX1 = IKeep[iiX1-1]+1 - sX1 = self.tabList.get(itab).columns[IKeep[iiX1-1]] + iy = IKeep[iiy] + sy = self.tabList.get(itab).columns[IKeep[iiy]] + iX1 = IKeep[iiX1] + sX1 = self.tabList.get(itab).columns[IKeep[iiX1]] ID.append([itab,iX1,iy,sX1,sy,stab]) else: iX1,IY1,sX1,SY1 = self.colPanel1.getColumnSelection() diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 3b75ad0..13c29d9 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -35,6 +35,9 @@ def __next__(self): def __len__(self): return len(self._tabs) + def len(self): + return len(self._tabs) + def append(self, t): if isinstance(t,list): self._tabs += t @@ -89,9 +92,7 @@ def _load_file_tabs(self, filename, fileformat=None, bReload=False): fileformatAllowedToFailOnReload = (fileformat is not None) and bReload if fileformatAllowedToFailOnReload: try: - F = None - if not isinstance(F, fileformat.constructor): - F=fileformat.constructor(filename=filename) + F = fileformat.constructor(filename=filename) dfs = F.toDataFrame() except: warnLoc = 'Failed to read file:\n\n {}\n\nwith fileformat: {}\n\nIf you see this message, the reader tried again and succeeded with "auto"-fileformat.\n\n'.format(filename, fileformat.name) @@ -151,8 +152,6 @@ def getTabs(self): # TODO remove me later return self._tabs - def len(self): - return len(self._tabs) def haveSameColumns(self,I=None): if I is None: @@ -413,6 +412,11 @@ def __init__(self,data=None,name='',filename='',columns=[], fileformat=None): else: # --- Pandas DataFrame self.data = data + # Adding index + if data.columns[0].lower().find('index')>=0: + pass + else: + data.insert(0, 'Index', np.arange(self.data.shape[0])) self.columns = self.columnsFromDF(data) # --- Trying to figure out how to name this table if name is None or len(str(name))==0: @@ -471,8 +475,6 @@ def clearMask(self): def applyMaskString(self,maskString,bAdd=True): df = self.data - Index = np.array(range(df.shape[0])) - sMask=maskString.replace('{Index}','Index') for i,c in enumerate(self.columns): c_no_unit = no_unit(c).strip() c_in_df = df.columns[i] @@ -529,7 +531,7 @@ def applyFiltering(self, iCol, options, bAdd=True): def radialAvg(self,avgMethod, avgParam): - import pydatview.fast.fastlib as fastlib + import pydatview.fast.postpro as fastlib import pydatview.fast.fastfarm as fastfarm df = self.data base,out_ext = os.path.splitext(self.filename) @@ -616,7 +618,7 @@ def renameColumn(self,iCol,newName): self.columns[iCol]=newName self.data.columns.values[iCol]=newName - def deleteColumns(self,ICol): + def deleteColumns(self, ICol): """ Delete columns by index, not column names which can have duplicates""" IKeep =[i for i in np.arange(self.data.shape[1]) if i not in ICol] self.data = self.data.iloc[:, IKeep] # Drop won't work for duplicates @@ -633,12 +635,13 @@ def deleteColumns(self,ICol): def rename(self,new_name): self.name='>'+new_name - def addColumn(self,sNewName,NewCol,i=-1,sFormula=''): + def addColumn(self, sNewName, NewCol, i=-1, sFormula=''): + print('>>> adding Column') if i<0: i=self.data.shape[1] elif i>self.data.shape[1]+1: i=self.data.shape[1] - self.data.insert(int(i),sNewName,NewCol) + self.data.insert(int(i+1),sNewName,NewCol) self.columns=self.columnsFromDF(self.data) for f in self.formulas: if f['pos'] > i: @@ -648,53 +651,42 @@ def addColumn(self,sNewName,NewCol,i=-1,sFormula=''): def setColumn(self,sNewName,NewCol,i,sFormula=''): if i<1: raise ValueError('Cannot set column at position ' + str(i)) - self.data = self.data.drop(columns=self.data.columns[i-1]) - self.data.insert(int(i-1),sNewName,NewCol) + self.data = self.data.drop(columns=self.data.columns[i]) + self.data.insert(int(i),sNewName,NewCol) self.columns=self.columnsFromDF(self.data) for f in self.formulas: if f['pos'] == i: f['name'] = sNewName f['formula'] = sFormula - def getColumn(self,i): - """ Return column of data, where i=0 is the index column + def getColumn(self, i): + """ Return column of data If a mask exist, the mask is applied TODO TODO TODO get rid of this! """ - if i <= 0 : - x = np.array(range(self.data.shape[0])) - if self.mask is not None: - x=x[self.mask] - - c = None - isString = False - isDate = False + if self.mask is not None: + c = self.data.iloc[self.mask, i] + x = self.data.iloc[self.mask, i].values else: - if self.mask is not None: - c = self.data.iloc[self.mask, i-1] - x = self.data.iloc[self.mask, i-1].values + c = self.data.iloc[:, i] + x = self.data.iloc[:, i].values + + isString = c.dtype == np.object and isinstance(c.values[0], str) + if isString: + x=x.astype(str) + isDate = np.issubdtype(c.dtype, np.datetime64) + if isDate: + dt=getDt(x) + if dt>1: + x=x.astype('datetime64[s]') else: - c = self.data.iloc[:, i-1] - x = self.data.iloc[:, i-1].values - - isString = c.dtype == np.object and isinstance(c.values[0], str) - if isString: - x=x.astype(str) - isDate = np.issubdtype(c.dtype, np.datetime64) - if isDate: - dt=getDt(x) - if dt>1: - x=x.astype('datetime64[s]') - else: - x=x.astype('datetime64') + x=x.astype('datetime64') return x,isString,isDate,c def evalFormula(self,sFormula): df = self.data - Index = np.array(range(df.shape[0])) - sFormula=sFormula.replace('{Index}','Index') for i,c in enumerate(self.columns): c_no_unit = no_unit(c).strip() c_in_df = df.columns[i] @@ -722,9 +714,10 @@ def setColumnByFormula(self,sNewName,sFormula,i=-1): return True - def export(self,path): + def export(self, path): if isinstance(self.data, pd.DataFrame): - self.data.to_csv(path,sep=',',index=False) #python3 + df = self.data.drop('Index', axis=1) + df.to_csv(path, sep=',', index=False) else: raise NotImplementedError('Export of data that is not a dataframe') diff --git a/pydatview/fast/case_gen.py b/pydatview/fast/case_gen.py deleted file mode 100644 index e3823b2..0000000 --- a/pydatview/fast/case_gen.py +++ /dev/null @@ -1,589 +0,0 @@ -import os -import collections -import glob -import pandas as pd -import numpy as np -import shutil -import stat -import re - -# --- Misc fast libraries -import pydatview.io.fast_input_file as fi -import pydatview.fast.runner as runner -import pydatview.fast.postpro as postpro -from pydatview.io.fast_wind_file import FASTWndFile - -# --------------------------------------------------------------------------------} -# --- Template replace -# --------------------------------------------------------------------------------{ -def handleRemoveReadonlyWin(func, path, exc_info): - """ - Error handler for ``shutil.rmtree``. - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - Usage : ``shutil.rmtree(path, onerror=onerror)`` - """ - if not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise - - -def forceCopyFile (sfile, dfile): - # ---- Handling error due to wrong mod - if os.path.isfile(dfile): - if not os.access(dfile, os.W_OK): - os.chmod(dfile, stat.S_IWUSR) - #print(sfile, ' > ', dfile) - shutil.copy2(sfile, dfile) - -def copyTree(src, dst): - """ - Copy a directory to another one, overwritting files if necessary. - copy_tree from distutils and copytree from shutil fail on Windows (in particular on git files) - """ - def forceMergeFlatDir(srcDir, dstDir): - if not os.path.exists(dstDir): - os.makedirs(dstDir) - for item in os.listdir(srcDir): - srcFile = os.path.join(srcDir, item) - dstFile = os.path.join(dstDir, item) - forceCopyFile(srcFile, dstFile) - - def isAFlatDir(sDir): - for item in os.listdir(sDir): - sItem = os.path.join(sDir, item) - if os.path.isdir(sItem): - return False - return True - - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isfile(s): - if not os.path.exists(dst): - os.makedirs(dst) - forceCopyFile(s,d) - if os.path.isdir(s): - isRecursive = not isAFlatDir(s) - if isRecursive: - copyTree(s, d) - else: - forceMergeFlatDir(s, d) - - -def templateReplaceGeneral(PARAMS, templateDir=None, outputDir=None, main_file=None, removeAllowed=False, removeRefSubFiles=False, oneSimPerDir=False): - """ Generate inputs files by replacing different parameters from a template file. - The generated files are placed in the output directory `outputDir` - The files are read and written using the library `weio`. - The template file is read and its content can be changed like a dictionary. - Each item of `PARAMS` correspond to a set of parameters that will be replaced - in the template file to generate one input file. - - For "FAST" input files, parameters can be changed recursively. - - - INPUTS: - PARAMS: list of dictionaries. Each key of the dictionary should be a key present in the - template file when read with `weio` (see: weio.read(main_file).keys() ) - - PARAMS[0]={'DT':0.1, 'EDFile|GBRatio':1, 'ServoFile|GenEff':0.8} - - templateDir: if provided, this directory and its content will be copied to `outputDir` - before doing the parametric substitution - - outputDir : directory where files will be generated. - """ - # --- Helper functions - def rebase_rel(wd,s,sid): - split = os.path.splitext(s) - return os.path.join(wd,split[0]+sid+split[1]) - - def get_strID(p) : - if '__name__' in p.keys(): - strID=p['__name__'] - else: - raise Exception('When calling `templateReplace`, provide the key `__name_` in the parameter dictionaries') - return strID - - def splitAddress(sAddress): - sp = sAddress.split('|') - if len(sp)==1: - return sp[0],[] - else: - return sp[0],sp[1:] - - def rebaseFileName(org_filename, workDir, strID): - new_filename_full = rebase_rel(workDir, org_filename,'_'+strID) - new_filename = os.path.relpath(new_filename_full,workDir).replace('\\','/') - return new_filename, new_filename_full - - def replaceRecurse(templatename_or_newname, FileKey, ParamKey, ParamValue, Files, strID, workDir, TemplateFiles): - """ - FileKey: a single key defining which file we are currently modifying e.g. :'AeroFile', 'EDFile','FVWInputFileName' - ParamKey: the address key of the parameter to be changed, relative to the current FileKey - e.g. 'EDFile|IntMethod' (if FileKey is '') - 'IntMethod' (if FileKey is 'EDFile') - ParamValue: the value to be used - Files: dict of files, as returned by weio, keys are "FileKeys" - """ - # --- Special handling for the root - if FileKey=='': - FileKey='Root' - # --- Open (or get if already open) file where a parameter needs to be changed - if FileKey in Files.keys(): - # The file was already opened, it's stored - f = Files[FileKey] - newfilename_full = f.filename - newfilename = os.path.relpath(newfilename_full,workDir).replace('\\','/') - - else: - templatefilename = templatename_or_newname - templatefilename_full = os.path.join(workDir,templatefilename) - TemplateFiles.append(templatefilename_full) - if FileKey=='Root': - # Root files, we start from strID - ext = os.path.splitext(templatefilename)[-1] - newfilename_full = os.path.join(wd,strID+ext) - newfilename = strID+ext - else: - newfilename, newfilename_full = rebaseFileName(templatefilename, workDir, strID) - #print('--------------------------------------------------------------') - #print('TemplateFile :', templatefilename) - #print('TemplateFileFull:', templatefilename_full) - #print('NewFile :', newfilename) - #print('NewFileFull :', newfilename_full) - shutil.copyfile(templatefilename_full, newfilename_full) - f= fi.FASTInputFile(newfilename_full) # open the template file for that filekey - Files[FileKey]=f # store it - - # --- Changing parameters in that file - NewFileKey_or_Key, ChildrenKeys = splitAddress(ParamKey) - if len(ChildrenKeys)==0: - # A simple parameter is changed - Key = NewFileKey_or_Key - #print('Setting', FileKey, '|',Key, 'to',ParamValue) - if Key=='OutList': - OutList=f[Key] - f[Key]=addToOutlist(OutList, ParamValue) - else: - f[Key] = ParamValue - else: - # Parameters needs to be changed in subfiles (children) - NewFileKey = NewFileKey_or_Key - ChildrenKey = '|'.join(ChildrenKeys) - child_templatefilename = f[NewFileKey].strip('"') # old filename that will be used as a template - baseparent = os.path.dirname(newfilename) - #print('Child templatefilename:',child_templatefilename) - #print('Parent base dir :',baseparent) - workDir = os.path.join(workDir, baseparent) - - # - newchildFilename, Files = replaceRecurse(child_templatefilename, NewFileKey, ChildrenKey, ParamValue, Files, strID, workDir, TemplateFiles) - #print('Setting', FileKey, '|',NewFileKey, 'to',newchildFilename) - f[NewFileKey] = '"'+newchildFilename+'"' - - return newfilename, Files - - - # --- Safety checks - if templateDir is None and outputDir is None: - raise Exception('Provide at least a template directory OR an output directory') - - if templateDir is not None: - if not os.path.exists(templateDir): - raise Exception('Template directory does not exist: '+templateDir) - - # Default value of outputDir if not provided - if templateDir[-1]=='/' or templateDir[-1]=='\\' : - templateDir=templateDir[0:-1] - if outputDir is None: - outputDir=templateDir+'_Parametric' - - # --- Main file use as "master" - if templateDir is not None: - main_file=os.path.join(outputDir, os.path.basename(main_file)) - else: - main_file=main_file - - # Params need to be a list - if not isinstance(PARAMS,list): - PARAMS=[PARAMS] - - if oneSimPerDir: - workDirS=[os.path.join(outputDir,get_strID(p)) for p in PARAMS] - else: - workDirS=[outputDir]*len(PARAMS) - # --- Creating outputDir - Copying template folder to outputDir if necessary - # Copying template folder to workDir - for wd in list(set(workDirS)): - if removeAllowed: - removeFASTOuputs(wd) - if os.path.exists(wd) and removeAllowed: - shutil.rmtree(wd, ignore_errors=False, onerror=handleRemoveReadonlyWin) - copyTree(templateDir, wd) - if removeAllowed: - removeFASTOuputs(wd) - - - TemplateFiles=[] - files=[] - for ip,(wd,p) in enumerate(zip(workDirS,PARAMS)): - if '__index__' not in p.keys(): - p['__index__']=ip - - main_file_base = os.path.basename(main_file) - strID = get_strID(p) - # --- Setting up files for this simulation - Files=dict() - for k,v in p.items(): - if k =='__index__' or k=='__name__': - continue - new_mainFile, Files = replaceRecurse(main_file_base, '', k, v, Files, strID, wd, TemplateFiles) - - # --- Writting files - for k,f in Files.items(): - if k=='Root': - files.append(f.filename) - f.write() - - # --- Remove extra files at the end - if removeRefSubFiles: - TemplateFiles, nCounts = np.unique(TemplateFiles, return_counts=True) - if not oneSimPerDir: - # we can only detele template files that were used by ALL simulations - TemplateFiles=[t for nc,t in zip(nCounts, TemplateFiles) if nc==len(PARAMS)] - for tf in TemplateFiles: - try: - os.remove(tf) - except: - print('[FAIL] Removing '+tf) - pass - return files - -def templateReplace(PARAMS, templateDir, outputDir=None, main_file=None, removeAllowed=False, removeRefSubFiles=False, oneSimPerDir=False): - """ Replace parameters in a fast folder using a list of dictionaries where the keys are for instance: - 'DT', 'EDFile|GBRatio', 'ServoFile|GenEff' - """ - # --- For backward compatibility, remove "FAST|" from the keys - for p in PARAMS: - old_keys=[ k for k,_ in p.items() if k.find('FAST|')==0] - for k_old in old_keys: - k_new=k_old.replace('FAST|','') - p[k_new] = p.pop(k_old) - - return templateReplaceGeneral(PARAMS, templateDir, outputDir=outputDir, main_file=main_file, - removeAllowed=removeAllowed, removeRefSubFiles=removeRefSubFiles, oneSimPerDir=oneSimPerDir) - -def removeFASTOuputs(workDir): - # Cleaning folder - for f in glob.glob(os.path.join(workDir,'*.out')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.outb')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.ech')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.sum')): - os.remove(f) - -# --------------------------------------------------------------------------------} -# --- Tools for template replacement -# --------------------------------------------------------------------------------{ -def paramsSteadyAero(p=None): - p = dict() if p is None else p - p['AeroFile|AFAeroMod']=1 # remove dynamic effects dynamic - p['AeroFile|WakeMod']=1 # remove dynamic inflow dynamic - p['AeroFile|TwrPotent']=0 # remove tower shadow - return p - -def paramsNoGen(p=None): - p = dict() if p is None else p - p['EDFile|GenDOF' ] = 'False' - return p - -def paramsGen(p=None): - p = dict() if p is None else p - p['EDFile|GenDOF' ] = 'True' - return p - -def paramsNoController(p=None): - p = dict() if p is None else p - p['ServoFile|PCMode'] = 0; - p['ServoFile|VSContrl'] = 0; - p['ServoFile|YCMode'] = 0; - return p - -def paramsControllerDLL(p=None): - p = dict() if p is None else p - p['ServoFile|PCMode'] = 5; - p['ServoFile|VSContrl'] = 5; - p['ServoFile|YCMode'] = 5; - p['EDFile|GenDOF'] = 'True'; - return p - - -def paramsStiff(p=None): - p = dict() if p is None else p - p['EDFile|FlapDOF1'] = 'False' - p['EDFile|FlapDOF2'] = 'False' - p['EDFile|EdgeDOF' ] = 'False' - p['EDFile|TeetDOF' ] = 'False' - p['EDFile|DrTrDOF' ] = 'False' - p['EDFile|YawDOF' ] = 'False' - p['EDFile|TwFADOF1'] = 'False' - p['EDFile|TwFADOF2'] = 'False' - p['EDFile|TwSSDOF1'] = 'False' - p['EDFile|TwSSDOF2'] = 'False' - p['EDFile|PtfmSgDOF'] = 'False' - p['EDFile|PtfmSwDOF'] = 'False' - p['EDFile|PtfmHvDOF'] = 'False' - p['EDFile|PtfmRDOF'] = 'False' - p['EDFile|PtfmPDOF'] = 'False' - p['EDFile|PtfmYDOF'] = 'False' - return p - -def paramsWS_RPM_Pitch(WS, RPM, Pitch, baseDict=None, flatInputs=False): - """ - Generate OpenFAST "parameters" (list of dictionaries with "address") - chaing the inputs in ElastoDyn, InflowWind for different wind speed, RPM and Pitch - """ - # --- Ensuring everythin is an iterator - def iterify(x): - if not isinstance(x, collections.Iterable): x = [x] - return x - WS = iterify(WS) - RPM = iterify(RPM) - Pitch = iterify(Pitch) - # --- If inputs are not flat but different vectors to length through, we flatten them (TODO: meshgrid and ravel?) - if not flatInputs : - WS_flat = [] - Pitch_flat = [] - RPM_flat = [] - for pitch in Pitch: - for rpm in RPM: - for ws in WS: - WS_flat.append(ws) - RPM_flat.append(rpm) - Pitch_flat.append(pitch) - else: - WS_flat, Pitch_flat, RPM_flat = WS, Pitch, RPM - - # --- Defining the parametric study - PARAMS=[] - i=0 - for ws,rpm,pitch in zip(WS_flat,RPM_flat,Pitch_flat): - if baseDict is None: - p=dict() - else: - p = baseDict.copy() - p['EDFile|RotSpeed'] = rpm - p['InflowFile|HWindSpeed'] = ws - p['InflowFile|WindType'] = 1 # Setting steady wind - p['EDFile|BlPitch(1)'] = pitch - p['EDFile|BlPitch(2)'] = pitch - p['EDFile|BlPitch(3)'] = pitch - - p['__index__'] = i - p['__name__'] = '{:03d}_ws{:04.1f}_pt{:04.2f}_om{:04.2f}'.format(p['__index__'],p['InflowFile|HWindSpeed'],p['EDFile|BlPitch(1)'],p['EDFile|RotSpeed']) - i=i+1 - PARAMS.append(p) - return PARAMS - -def paramsLinearTrim(p=None): - p = dict() if p is None else p - - # Set a few DOFs, move this to main file - p['Linearize'] = True - p['CalcSteady'] = True - p['TrimGain'] = 1e-4 - p['TrimTol'] = 1e-5 - p['CompMooring'] = 0 - p['CompHydro'] = 0 - p['LinOutJac'] = False - p['LinOutMod'] = False - p['OutFmt'] = '"ES20.12E3"' # Important for decent resolution - - p['AeroFile|AFAeroMod'] = 1 - p['AeroFile|CavitCheck'] = 'False' - p['AeroFile|CompAA'] = 'False' - - p['ServoFile|PCMode'] = 0 - p['ServoFile|VSContrl'] = 1 - - p['ServoFile|CompNTMD'] = 'False' - p['ServoFile|CompTTMD'] = 'False' - - # Set all DOFs off, enable as desired - p['EDFile|FlapDOF1'] = 'False' - p['EDFile|FlapDOF2'] = 'False' - p['EDFile|EdgeDOF'] = 'False' - p['EDFile|TeetDOF'] = 'False' - p['EDFile|DrTrDOF'] = 'False' - p['EDFile|GenDOF'] = 'False' - p['EDFile|YawDOF'] = 'False' - p['EDFile|TwFADOF1'] = 'False' - p['EDFile|TwFADOF2'] = 'False' - p['EDFile|TwSSDOF1'] = 'False' - p['EDFile|TwSSDOF2'] = 'False' - p['EDFile|PtfmSgDOF'] = 'False' - p['EDFile|PtfmSwDOF'] = 'False' - p['EDFile|PtfmHvDOF'] = 'False' - p['EDFile|PtfmRDOF'] = 'False' - p['EDFile|PtfmPDOF'] = 'False' - p['EDFile|PtfmYDOF'] = 'False' - - - return p - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -def createStepWind(filename,WSstep=1,WSmin=3,WSmax=25,tstep=100,dt=0.5,tmin=0,tmax=999): - f = FASTWndFile() - Steps= np.arange(WSmin,WSmax+WSstep,WSstep) - print(Steps) - nCol = len(f.colNames) - nRow = len(Steps)*2 - M = np.zeros((nRow,nCol)); - M[0,0] = tmin - M[0,1] = WSmin - for i,s in enumerate(Steps[:-1]): - M[2*i+1,0] = tmin + (i+1)*tstep-dt - M[2*i+2,0] = tmin + (i+1)*tstep - M[2*i+1,1] = Steps[i] - if i0: - main_fastfile=os.path.basename(main_fastfile) - - # --- Reading main fast file to get rotor radius - fst = fi.FASTInputFile(os.path.join(refdir,main_fastfile)) - ed = fi.FASTInputFile(os.path.join(refdir,fst['EDFile'].replace('"',''))) - R = ed['TipRad'] - - # --- Making sure we have - if (Omega is not None): - if (Lambda is not None): - WS = np.ones(Omega.shape)*WS_default - elif (WS is not None): - if len(WS)!=len(Omega): - raise Exception('When providing Omega and WS, both vectors should have the same dimension') - else: - WS = np.ones(Omega.shape)*WS_default - else: - Omega = WS_default * Lambda/R*60/(2*np.pi) # TODO, use more realistic combinations of WS and Omega - WS = np.ones(Omega.shape)*WS_default - - - # --- Defining flat vectors of operating conditions - WS_flat = [] - RPM_flat = [] - Pitch_flat = [] - for pitch in Pitch: - for (rpm,ws) in zip(Omega,WS): - WS_flat.append(ws) - RPM_flat.append(rpm) - Pitch_flat.append(pitch) - # --- Setting up default options - baseDict={'TMax': TMax, 'DT': 0.01, 'DT_Out': 0.1} # NOTE: Tmax should be at least 2pi/Omega - baseDict = paramsNoController(baseDict) - if bStiff: - baseDict = paramsStiff(baseDict) - if bNoGen: - baseDict = paramsNoGen(baseDict) - if bSteadyAero: - baseDict = paramsSteadyAero(baseDict) - - # --- Creating set of parameters to be changed - # TODO: verify that RtAeroCp and RtAeroCt are present in AeroDyn outlist - PARAMS = paramsWS_RPM_Pitch(WS_flat,RPM_flat,Pitch_flat,baseDict=baseDict, flatInputs=True) - - # --- Generating all files in a workDir - workDir = refdir.strip('/').strip('\\')+'_CPLambdaPitch' - print('>>> Generating inputs files in {}'.format(workDir)) - RemoveAllowed=reRun # If the user want to rerun, we can remove, otherwise we keep existing simulations - fastFiles=templateReplace(PARAMS, refdir, outputDir=workDir,removeRefSubFiles=True,removeAllowed=RemoveAllowed,main_file=main_fastfile) - - # --- Running fast simulations - print('>>> Running {} simulations...'.format(len(fastFiles))) - runner.run_fastfiles(fastFiles, showOutputs=showOutputs, fastExe=fastExe, nCores=nCores, reRun=reRun) - - # --- Postpro - Computing averages at the end of the simluation - print('>>> Postprocessing...') - outFiles = [os.path.splitext(f)[0]+'.outb' for f in fastFiles] - # outFiles = glob.glob(os.path.join(workDir,'*.outb')) - ColKeepStats = ['RotSpeed_[rpm]','BldPitch1_[deg]','RtAeroCp_[-]','RtAeroCt_[-]','Wind1VelX_[m/s]'] - result = postpro.averagePostPro(outFiles,avgMethod='periods',avgParam=1,ColKeep=ColKeepStats,ColSort='RotSpeed_[rpm]') - # print(result) - - # --- Adding lambda, sorting and keeping only few columns - result['lambda_[-]'] = result['RotSpeed_[rpm]']*R*2*np.pi/60/result['Wind1VelX_[m/s]'] - result.sort_values(['lambda_[-]','BldPitch1_[deg]'],ascending=[True,True],inplace=True) - ColKeepFinal=['lambda_[-]','BldPitch1_[deg]','RtAeroCp_[-]','RtAeroCt_[-]'] - result=result[ColKeepFinal] - print('>>> Done') - - # --- Converting to a matrices - CP = result['RtAeroCp_[-]'].values - CT = result['RtAeroCt_[-]'].values - MCP =CP.reshape((len(Lambda),len(Pitch))) - MCT =CT.reshape((len(Lambda),len(Pitch))) - LAMBDA, PITCH = np.meshgrid(Lambda, Pitch) - # --- CP max - i,j = np.unravel_index(MCP.argmax(), MCP.shape) - MaxVal={'CP_max':MCP[i,j],'lambda_opt':LAMBDA[j,i],'pitch_opt':PITCH[j,i]} - - return MCP,MCT,Lambda,Pitch,MaxVal,result - - -if __name__=='__main__': - # --- Test of templateReplace - PARAMS = {} - PARAMS['TMax'] = 10 - PARAMS['__name__'] = 'MyName' - PARAMS['DT'] = 0.01 - PARAMS['DT_Out'] = 0.1 - PARAMS['EDFile|RotSpeed'] = 100 - PARAMS['EDFile|BlPitch(1)'] = 1 - PARAMS['EDFile|GBoxEff'] = 0.92 - PARAMS['ServoFile|VS_Rgn2K'] = 0.00038245 - PARAMS['ServoFile|GenEff'] = 0.95 - PARAMS['InflowFile|HWindSpeed'] = 8 - templateReplace(PARAMS,refDir,RemoveRefSubFiles=True) - diff --git a/pydatview/fast/fastfarm.py b/pydatview/fast/fastfarm.py index 1a05fa3..5e7dd07 100644 --- a/pydatview/fast/fastfarm.py +++ b/pydatview/fast/fastfarm.py @@ -7,7 +7,7 @@ from pydatview.io.fast_output_file import FASTOutputFile from pydatview.io.turbsim_file import TurbSimFile -from . import fastlib +from . import postpro as fastlib # --------------------------------------------------------------------------------} # --- Small helper functions diff --git a/pydatview/fast/fastlib.py b/pydatview/fast/fastlib.py deleted file mode 100644 index bd27917..0000000 --- a/pydatview/fast/fastlib.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -The functions of fastlib were split to match what was done for pyFAST and ease the porting from welib to pyFAST -""" -from pydatview.fast.case_gen import * -from pydatview.fast.postpro import * -from pydatview.fast.runner import * diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index a658b05..cee7cbb 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -1311,7 +1311,7 @@ def renameCol(x): raise Exception('The sensor `Azimuth_[deg]` does not appear to be in the output file. You cannot use the averaging method by `periods`, use `constantwindow` instead.') # NOTE: potentially we could average over each period and then average psi=df['Azimuth_[deg]'].values - _,iBef = _zero_crossings(psi-psi[-10],direction='up') + _,iBef = _zero_crossings(psi-psi[-2],direction='up') if len(iBef)==0: _,iBef = _zero_crossings(psi-180,direction='up') if len(iBef)==0: @@ -1401,6 +1401,9 @@ def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKee Default: None, full simulation length is used """ result=None + if len(outFiles)==0: + raise Exception('No outFiles provided') + invalidFiles =[] # Loop trough files and populate result for i,f in enumerate(outFiles): @@ -1424,9 +1427,9 @@ def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKee result.reset_index(drop=True,inplace=True) if len(invalidFiles)==len(outFiles): - raise Exception('None of the files can be read (or exist)!') + raise Exception('None of the files can be read (or exist)!. For instance, cannot find: {}'.format(invalidFiles[0])) elif len(invalidFiles)>0: - print('[WARN] There were {} missing/invalid files: {}'.format(len(invalidFiles),invalidFiles)) + print('[WARN] There were {} missing/invalid files: \n {}'.format(len(invalidFiles),'\n'.join(invalidFiles))) return result diff --git a/pydatview/fast/runner.py b/pydatview/fast/runner.py deleted file mode 100644 index c8faf9f..0000000 --- a/pydatview/fast/runner.py +++ /dev/null @@ -1,190 +0,0 @@ -# --- For cmd.py -import os -import subprocess -import multiprocessing - -import collections -import glob -import pandas as pd -import numpy as np -import shutil -import stat -import re - -# --- Fast libraries -from pydatview.io.fast_input_file import FASTInputFile -from pydatview.io.fast_output_file import FASTOutputFile - -FAST_EXE='openfast' - -# --------------------------------------------------------------------------------} -# --- Tools for executing FAST -# --------------------------------------------------------------------------------{ -# --- START cmd.py -def run_cmds(inputfiles, exe, parallel=True, showOutputs=True, nCores=None, showCommand=True, verbose=True): - """ Run a set of simple commands of the form `exe input_file` - By default, the commands are run in "parallel" (though the method needs to be improved) - The stdout and stderr may be displayed on screen (`showOutputs`) or hidden. - A better handling is yet required. - """ - Failed=[] - def _report(p): - if p.returncode==0: - if verbose: - print('[ OK ] Input : ',p.input_file) - else: - Failed.append(p) - if verbose: - print('[FAIL] Input : ',p.input_file) - print(' Directory: '+os.getcwd()) - print(' Command : '+p.cmd) - print(' Use `showOutputs=True` to debug, or run the command above.') - #out, err = p.communicate() - #print('StdOut:\n'+out) - #print('StdErr:\n'+err) - ps=[] - iProcess=0 - if nCores is None: - nCores=multiprocessing.cpu_count() - if nCores<0: - nCores=len(inputfiles)+1 - for i,f in enumerate(inputfiles): - #print('Process {}/{}: {}'.format(i+1,len(inputfiles),f)) - ps.append(run_cmd(f, exe, wait=(not parallel), showOutputs=showOutputs, showCommand=showCommand)) - iProcess += 1 - # waiting once we've filled the number of cores - # TODO: smarter method with proper queue, here processes are run by chunks - if parallel: - if iProcess==nCores: - for p in ps: - p.wait() - for p in ps: - _report(p) - ps=[] - iProcess=0 - # Extra process if not multiptle of nCores (TODO, smarter method) - for p in ps: - p.wait() - for p in ps: - _report(p) - # --- Giving a summary - if len(Failed)==0: - if verbose: - print('[ OK ] All simulations run successfully.') - return True, Failed - else: - print('[FAIL] {}/{} simulations failed:'.format(len(Failed),len(inputfiles))) - for p in Failed: - print(' ',p.input_file) - return False, Failed - -def run_cmd(input_file_or_arglist, exe, wait=True, showOutputs=False, showCommand=True): - """ Run a simple command of the form `exe input_file` or `exe arg1 arg2` """ - # TODO Better capture STDOUT - if isinstance(input_file_or_arglist, list): - args= [exe] + input_file_or_arglist - input_file = ' '.join(input_file_or_arglist) - input_file_abs = input_file - else: - input_file=input_file_or_arglist - if not os.path.isabs(input_file): - input_file_abs=os.path.abspath(input_file) - else: - input_file_abs=input_file - if not os.path.exists(exe): - raise Exception('Executable not found: {}'.format(exe)) - args= [exe,input_file] - #args = 'cd '+workDir+' && '+ exe +' '+basename - shell=False - if showOutputs: - STDOut= None - else: - STDOut= open(os.devnull, 'w') - if showCommand: - print('Running: '+' '.join(args)) - if wait: - class Dummy(): - pass - p=Dummy() - p.returncode=subprocess.call(args , stdout=STDOut, stderr=subprocess.STDOUT, shell=shell) - else: - p=subprocess.Popen(args, stdout=STDOut, stderr=subprocess.STDOUT, shell=shell) - # Storing some info into the process - p.cmd = ' '.join(args) - p.args = args - p.input_file = input_file - p.input_file_abs = input_file_abs - p.exe = exe - return p -# --- END cmd.py - -def run_fastfiles(fastfiles, fastExe=None, parallel=True, showOutputs=True, nCores=None, showCommand=True, reRun=True, verbose=True): - if fastExe is None: - fastExe=FAST_EXE - if not reRun: - # Figure out which files exist - newfiles=[] - for f in fastfiles: - base=os.path.splitext(f)[0] - if os.path.exists(base+'.outb') or os.path.exists(base+'.out'): - print('>>> Skipping existing simulation for: ',f) - pass - else: - newfiles.append(f) - fastfiles=newfiles - - return run_cmds(fastfiles, fastExe, parallel=parallel, showOutputs=showOutputs, nCores=nCores, showCommand=showCommand, verbose=verbose) - -def run_fast(input_file, fastExe=None, wait=True, showOutputs=False, showCommand=True): - if fastExe is None: - fastExe=FAST_EXE - return run_cmd(input_file, fastExe, wait=wait, showOutputs=showOutputs, showCommand=showCommand) - - -def writeBatch(batchfile, fastfiles, fastExe=None, nBatches=1, pause=False): - """ Write batch file, everything is written relative to the batch file""" - if fastExe is None: - fastExe=FAST_EXE - fastExe_abs = os.path.abspath(fastExe) - batchfile_abs = os.path.abspath(batchfile) - batchdir = os.path.dirname(batchfile_abs) - fastExe_rel = os.path.relpath(fastExe_abs, batchdir) - def writeb(batchfile, fastfiles): - with open(batchfile,'w') as f: - for ff in fastfiles: - ff_abs = os.path.abspath(ff) - ff_rel = os.path.relpath(ff_abs, batchdir) - l = fastExe_rel + ' '+ ff_rel - f.write("%s\n" % l) - if pause: - f.write("pause\n") # windows only.. - - if nBatches==1: - writeb(batchfile, fastfiles) - else: - splits = np.array_split(fastfiles,nBatches) - base, ext = os.path.splitext(batchfile) - for i in np.arange(nBatches): - writeb(base+'_{:d}'.format(i+1) + ext, splits[i]) - - - - - - -def removeFASTOuputs(workDir): - # Cleaning folder - for f in glob.glob(os.path.join(workDir,'*.out')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.outb')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.ech')): - os.remove(f) - for f in glob.glob(os.path.join(workDir,'*.sum')): - os.remove(f) - -if __name__=='__main__': - run_cmds(['main1.fst','main2.fst'], './Openfast.exe', parallel=True, showOutputs=False, nCores=4, showCommand=True) - pass - # --- Test of templateReplace - From e7d485805f98bf90fcb67d046940c1e1b794cb97 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 15 Sep 2022 15:37:21 -0600 Subject: [PATCH 038/178] Changing exe entry point to include arguments --- installer.cfg | 2 +- pydatview/__init__.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/installer.cfg b/installer.cfg index 30f73c2..96120a1 100644 --- a/installer.cfg +++ b/installer.cfg @@ -1,7 +1,7 @@ [Application] name=pyDatView version=0.3 -entry_point=pydatview:show +entry_point=pydatview:show_sys_args icon=ressources/pyDatView.ico #[Command pydatview] diff --git a/pydatview/__init__.py b/pydatview/__init__.py index b5dc28b..9dd14b4 100644 --- a/pydatview/__init__.py +++ b/pydatview/__init__.py @@ -1,8 +1,14 @@ -__all__ = ['show'] +__all__ = ['show', 'show_sys_args'] # defining main function here, to avoid import of pydatview and wx of some unittests def show(*args,**kwargs): from pydatview.main import showApp showApp(*args,**kwargs) +def show_sys_args(): + import sys + if len(sys.argv)>1: + show(filenames=sys.argv[1:]) + else: + show() From 7f0d6074ba0133448844bb48fb27b06e21fb645c Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 15 Sep 2022 15:46:47 -0600 Subject: [PATCH 039/178] Fix tests now that index is in table (#127) --- pydatview/plugins/tests/test_standardizeUnits.py | 8 ++++---- tests/test_Tables.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pydatview/plugins/tests/test_standardizeUnits.py b/pydatview/plugins/tests/test_standardizeUnits.py index 35843c3..c809d33 100644 --- a/pydatview/plugins/tests/test_standardizeUnits.py +++ b/pydatview/plugins/tests/test_standardizeUnits.py @@ -14,10 +14,10 @@ def test_change_units(self): df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) tab=Table(data=df) changeUnits(tab, flavor='WE') - np.testing.assert_almost_equal(tab.data.values[:,0],[1]) - np.testing.assert_almost_equal(tab.data.values[:,1],[2]) - np.testing.assert_almost_equal(tab.data.values[:,2],[10]) - self.assertEqual(tab.columns, ['om [rpm]', 'F [kN]', 'angle [deg]']) + np.testing.assert_almost_equal(tab.data.values[:,1],[1]) + np.testing.assert_almost_equal(tab.data.values[:,2],[2]) + np.testing.assert_almost_equal(tab.data.values[:,3],[10]) + self.assertEqual(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) if __name__ == '__main__': diff --git a/tests/test_Tables.py b/tests/test_Tables.py index fcf8e77..5562ce7 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -46,7 +46,7 @@ def test_merge(self): tab2=Table(data=pd.DataFrame(data={'ID': np.arange(1,4),'ColB': [11,12,13]})) tablist = TableList([tab1,tab2]) # - name, df = tablist.mergeTabs(ICommonColPerTab=[0,0], extrap='nan') + name, df = tablist.mergeTabs(ICommonColPerTab=[1,1], extrap='nan') np.testing.assert_almost_equal(df['ID'] , [0 , 0.5 , 1.0 , 1.5 , 2.0 , 2.5 , 3.0]) np.testing.assert_almost_equal(df['ColA'] , [10 , 10.5 , 11 , 11.5 , 12 , 12.5 , np.nan] ) np.testing.assert_almost_equal(df['ColB'] , [np.nan , np.nan , 11 , 11.5 , 12 , 12.5 , 13.0] ) @@ -80,10 +80,10 @@ def test_change_units(self): df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) tab=Table(data=df) tab.changeUnits() - np.testing.assert_almost_equal(tab.data.values[:,0],[1]) - np.testing.assert_almost_equal(tab.data.values[:,1],[2]) - np.testing.assert_almost_equal(tab.data.values[:,2],[10]) - self.assertEqual(tab.columns, ['om [rpm]', 'F [kN]', 'angle [deg]']) + np.testing.assert_almost_equal(tab.data.values[:,1],[1]) + np.testing.assert_almost_equal(tab.data.values[:,2],[2]) + np.testing.assert_almost_equal(tab.data.values[:,3],[10]) + self.assertEqual(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) if __name__ == '__main__': From eea58292dc802c12572b33c7772942e1687caef9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 18 Sep 2022 20:37:13 -0400 Subject: [PATCH 040/178] Fix bug when ED file not present in radial postpro --- pydatview/fast/postpro.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index cee7cbb..ff9a445 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -867,13 +867,14 @@ def FASTRadialOutputs(FST_In, OutputCols=None): print('[WARN] The Elastodyn file couldn''t be found or read, from main file: '+FST_In) #raise Exception('The Elastodyn file couldn''t be found or read, from main file: '+FST_In) else: - R = fst.ED['TipRad'] - r_hub = fst.ED['HubRad'] - if fst.ED.hasNodal: - _, r_ED = ED_BldStations(fst.ED) - IR_ED =None - else: - r_ED, IR_ED = ED_BldGag(fst.ED) + if fst.ED is not None: + R = fst.ED['TipRad'] + r_hub = fst.ED['HubRad'] + if fst.ED.hasNodal: + _, r_ED = ED_BldStations(fst.ED) + IR_ED =None + else: + r_ED, IR_ED = ED_BldGag(fst.ED) # --- BeamDyn if fst.BD is not None: From 30c8182135be7d0834b81fdcc80c6b4219bc6491 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 18 Sep 2022 21:29:21 -0400 Subject: [PATCH 041/178] Bug fix SpinCtrlDouble needs 0 as minimum range, and digits (tentative fix) --- pydatview/GUITools.py | 20 +++++++++++++++++--- pydatview/tools/signal_analysis.py | 6 +++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 4d47ae4..44dd833 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -211,7 +211,8 @@ def __init__(self, parent): self.cbFilters = wx.ComboBox(self, choices=[filt['name'] for filt in self._DEFAULT_FILTERS], style=wx.CB_READONLY) self.lbParamName = wx.StaticText(self, -1, ' :') self.cbFilters.SetSelection(0) - self.tParam = wx.SpinCtrlDouble(self, value='11', size=wx.Size(60,-1)) +# self.tParam = wx.TextCtrl(self, wx.ID_ANY, '', style= wx.TE_PROCESS_ENTER, size=wx.Size(60,-1)) + self.tParam = wx.SpinCtrlDouble(self, value='11', size=wx.Size(80,-1)) self.lbInfo = wx.StaticText( self, -1, '') @@ -251,9 +252,12 @@ def __init__(self, parent): self.cbFilters.Bind(wx.EVT_COMBOBOX, self.onSelectFilt) self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow, self.tParam) self.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter, self.tParam) + try: + self.spintxt = self.tParam.Children[0] + except: + self.spintxt = None if platform.system()=='Windows': # See issue https://github.com/wxWidgets/Phoenix/issues/1762 - self.spintxt = self.tParam.Children[0] assert isinstance(self.spintxt, wx.TextCtrl) self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onParamChangeChar) @@ -273,8 +277,11 @@ def onSelectFilt(self, event=None): iFilt = self.cbFilters.GetSelection() filt = self._DEFAULT_FILTERS[iFilt] self.lbParamName.SetLabel(filt['paramName']+':') - self.tParam.SetRange(filt['paramRange'][0], filt['paramRange'][1]) + #self.tParam.SetRange(filt['paramRange'][0], filt['paramRange'][1]) + # NOTE: if min value for range is not 0, the Ctrl prevents you to enter 0.01 + self.tParam.SetRange(0, filt['paramRange'][1]) self.tParam.SetIncrement(filt['increment']) + self.tParam.SetDigits(filt['digits']) parentFilt=self.parent.plotDataOptions['Filter'] # Value @@ -282,6 +289,8 @@ def onSelectFilt(self, event=None): self.tParam.SetValue(parentFilt['param']) else: self.tParam.SetValue(filt['param']) + # Trigger plot if applied + self.onParamChange(self) def _GUI2Data(self): iFilt = self.cbFilters.GetSelection() @@ -290,6 +299,11 @@ def _GUI2Data(self): filt['param']=np.float(self.spintxt.Value) except: print('[WARN] pyDatView: Issue on Mac: GUITools.py/_GUI2Data. Help needed.') + filt['param']=np.float(self.tParam.Value) + if filt['param'] Date: Sun, 18 Sep 2022 21:51:08 -0400 Subject: [PATCH 042/178] Bug fix, with index in table was using wrong column (see #127) --- pydatview/Tables.py | 8 +++++--- tests/test_Tables.py | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 13c29d9..4f808a2 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -319,7 +319,7 @@ def applyResampling(self,iCol,sampDict,bAdd=True): errors=[] for i,t in enumerate(self._tabs): # try: - df_new, name_new = t.applyResampling(iCol,sampDict, bAdd=bAdd) + df_new, name_new = t.applyResampling(iCol, sampDict, bAdd=bAdd) if df_new is not None: # we don't append when string is empty dfs_new.append(df_new) @@ -505,9 +505,11 @@ def applyResampling(self, iCol, sampDict, bAdd=True): from pydatview.tools.signal_analysis import applySamplerDF if iCol==0: raise Exception('Cannot resample based on index') - colName=self.data.columns[iCol-1] + colName=self.data.columns[iCol] df_new =applySamplerDF(self.data, colName, sampDict=sampDict) - df_new + # Reindex afterwards + if df_new.columns[0]=='Index': + df_new['Index'] = np.arange(0,len(df_new)) if bAdd: name_new=self.raw_name+'_resampled' else: diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 5562ce7..3c58b2a 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -51,6 +51,23 @@ def test_merge(self): np.testing.assert_almost_equal(df['ColA'] , [10 , 10.5 , 11 , 11.5 , 12 , 12.5 , np.nan] ) np.testing.assert_almost_equal(df['ColB'] , [np.nan , np.nan , 11 , 11.5 , 12 , 12.5 , 13.0] ) + + def test_resample(self): + tab1=Table(data=pd.DataFrame(data={'BlSpn': [0,1,2],'Chord': [1,2,1]})) + tablist = TableList([tab1]) + #print(tablist) + #print(tab1.data) + + # Test Insertion of new values into table + icol=1 + opt = {'name': 'Insert', 'param': np.array([0.5, 1.5])} + dfs, names, errors = tablist.applyResampling(icol, opt, bAdd=True) + np.testing.assert_almost_equal(dfs[0]['Index'], [0,1,2,3,4]) + np.testing.assert_almost_equal(dfs[0]['BlSpn'], [0,0.5,1.0,1.5,2.0]) + np.testing.assert_almost_equal(dfs[0]['Chord'], [1,1.5,2.0,1.5,1.0]) + print(dfs) + + def test_load_files_misc_formats(self): tablist = TableList() files =[ @@ -88,7 +105,8 @@ def test_change_units(self): if __name__ == '__main__': # TestTable.setUpClass() - TestTable().test_merge() +# TestTable().test_merge() + TestTable().test_resample() # tt= TestTable() # tt.test_load_files_misc_formats() # unittest.main() From 397e7d3497d5e7cb8e0278fcb888318fbf03321f Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 18 Sep 2022 22:26:26 -0400 Subject: [PATCH 043/178] Fix mask (string sMask undefined), and applying on Enter --- pydatview/GUITools.py | 23 ++++++++++++++++++----- pydatview/Tables.py | 9 +++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 44dd833..76ce408 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -710,7 +710,7 @@ def __init__(self, parent): self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) self.cbTabs.SetSelection(0) - self.textMask = wx.TextCtrl(self, wx.ID_ANY, allMask) + self.textMask = wx.TextCtrl(self, wx.ID_ANY, allMask, style = wx.TE_PROCESS_ENTER) #self.textMask.SetValue('({Time}>100) & ({Time}<400)') #self.textMask.SetValue("{Date} > '2018-10-01'") @@ -734,7 +734,10 @@ def __init__(self, parent): self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) self.sizer.Add(vert_sizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) self.SetSizer(self.sizer) + # Bindings + # NOTE: getBtBitmap and getToggleBtBitmap already specify the binding self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) + self.textMask.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeAndPressEnter) def onTabChange(self,event=None): tabList = self.parent.selPanel.tabList @@ -750,13 +753,16 @@ def onTabChange(self,event=None): # self.textMask.SetValue(self.guessMask) # no known mask def guessMask(self,tabList): - cols=[c.lower() for c in tabList.get(0).columns_clean] - if 'time' in cols: + cols=tabList.get(0).columns_clean + if 'Time' in cols: return '{Time} > 100' - elif 'date' in cols: + elif 'Date' in cols: return "{Date} > '2017-01-01" else: - return '' + if len(cols)>1: + return '{'+cols[1]+'}>0' + else: + return '' def onClear(self,event=None): iSel = self.cbTabs.GetSelection() @@ -770,6 +776,13 @@ def onClear(self,event=None): mainframe.redraw() self.onTabChange() + def onParamChangeAndPressEnter(self, event=None): + # We apply + if self.applied: + self.onApply(self,bAdd=False) + else: + self.onToggleApplyMask(self) + def onToggleApplyMask(self,event=None): self.applied = not self.applied if self.applied: diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 4f808a2..280824d 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -297,6 +297,7 @@ def clearCommonMask(self): t.clearMask() def applyCommonMaskString(self,maskString,bAdd=True): + # Apply mask on tablist dfs_new = [] names_new = [] errors=[] @@ -473,7 +474,8 @@ def clearMask(self): self.maskString='' self.mask=None - def applyMaskString(self,maskString,bAdd=True): + def applyMaskString(self, sMask, bAdd=True): + # Apply mask on Table df = self.data for i,c in enumerate(self.columns): c_no_unit = no_unit(c).strip() @@ -495,9 +497,12 @@ def applyMaskString(self,maskString,bAdd=True): name_new=self.raw_name+'_masked' else: self.mask=mask - self.maskString=maskString + self.maskString=sMask except: raise Exception('Error: The mask failed for table: '+self.name) + if sum(mask)==0: + self.clearMask() + raise Exception('Error: The mask returned no value for table: '+self.name) return df_new, name_new # --- Important manipulation TODO MOVE THIS OUT OF HERE OR UNIFY From 1475ed6a422497c501ab2d6216117cd6234eccc7 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 18 Sep 2022 22:29:05 -0400 Subject: [PATCH 044/178] Stop applying tools on Add --- pydatview/GUITools.py | 10 ++++++++++ pydatview/Tables.py | 11 +++++------ tests/test_Tables.py | 5 ++--- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 76ce408..ca56737 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -355,6 +355,9 @@ def onAdd(self,event=None): if len(errors)>0: raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) + # We stop applying + self.onToggleApply() + def onPlot(self, event=None): """ Overlay on current axis the filter @@ -610,6 +613,9 @@ def onAdd(self,event=None): if len(errors)>0: raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) + # We stop applying + self.onToggleApply() + def onPlot(self,event=None): from pydatview.tools.signal_analysis import applySampler if len(self.parent.plotData)!=1: @@ -816,6 +822,10 @@ def onApply(self,event=None,bAdd=True): mainframe.redraw() self.updateTabList() + # We stop applying + if bAdd: + self.onToggleApplyMask() + def updateTabList(self,event=None): tabList = self.parent.selPanel.tabList diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 280824d..f204c53 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -507,9 +507,8 @@ def applyMaskString(self, sMask, bAdd=True): # --- Important manipulation TODO MOVE THIS OUT OF HERE OR UNIFY def applyResampling(self, iCol, sampDict, bAdd=True): + # Resample Table from pydatview.tools.signal_analysis import applySamplerDF - if iCol==0: - raise Exception('Cannot resample based on index') colName=self.data.columns[iCol] df_new =applySamplerDF(self.data, colName, sampDict=sampDict) # Reindex afterwards @@ -524,11 +523,11 @@ def applyResampling(self, iCol, sampDict, bAdd=True): def applyFiltering(self, iCol, options, bAdd=True): from pydatview.tools.signal_analysis import applyFilterDF - if iCol==0: - raise Exception('Cannot filter based on index') - colName=self.data.columns[iCol-1] + colName=self.data.columns[iCol] df_new =applyFilterDF(self.data, colName, options) - df_new + # Reindex afterwards + if df_new.columns[0]=='Index': + df_new['Index'] = np.arange(0,len(df_new)) if bAdd: name_new=self.raw_name+'_filtered' else: diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 3c58b2a..cb5bc28 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -65,7 +65,6 @@ def test_resample(self): np.testing.assert_almost_equal(dfs[0]['Index'], [0,1,2,3,4]) np.testing.assert_almost_equal(dfs[0]['BlSpn'], [0,0.5,1.0,1.5,2.0]) np.testing.assert_almost_equal(dfs[0]['Chord'], [1,1.5,2.0,1.5,1.0]) - print(dfs) def test_load_files_misc_formats(self): @@ -106,7 +105,7 @@ def test_change_units(self): if __name__ == '__main__': # TestTable.setUpClass() # TestTable().test_merge() - TestTable().test_resample() +# TestTable().test_resample() # tt= TestTable() # tt.test_load_files_misc_formats() -# unittest.main() + unittest.main() From 77d1ae84280dae7044eeb7bd01ec4f5434034b5b Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 18 Sep 2022 22:32:38 -0400 Subject: [PATCH 045/178] Fix digits for outlier removal --- pydatview/GUITools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index ca56737..c653ccf 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -112,6 +112,7 @@ def __init__(self, parent): self.tMD.SetValue(self.parent.plotDataOptions['OutliersMedianDeviation']) self.tMD.SetRange(0.0, 1000) self.tMD.SetIncrement(0.5) + self.tMD.SetDigits(1) self.lb = wx.StaticText( self, -1, '') self.sizer = wx.BoxSizer(wx.HORIZONTAL) From 1aa03c6a5274e6be9d38801be6d4dd400cae4ada Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 18 Sep 2022 22:55:12 -0400 Subject: [PATCH 046/178] Fix merge after reindexing (see #127) --- pydatview/Tables.py | 7 ++++++- tests/test_Tables.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index f204c53..32ee6b3 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -194,6 +194,7 @@ def mergeTabs(self, I=None, ICommonColPerTab=None, samplDict=None, extrap='nan') dfs = [self._tabs[i].data for i in I] if ICommonColPerTab is None: # --- Option 0 - Index concatenation + print('Using dataframe index concatenation...') df = pd.concat(dfs, axis=1) # Remove duplicated columns #df = df.loc[:,~df.columns.duplicated()].copy() @@ -212,10 +213,14 @@ def mergeTabs(self, I=None, ICommonColPerTab=None, samplDict=None, extrap='nan') dfs_new = [] for i, (col, df_old) in enumerate(zip(cols, dfs)): df = interpDF(x_new, col, df_old, extrap=extrap) + if 'Index' in df.columns: + df = df.drop(['Index'], axis=1) if i>0: - df = df.loc[:, df.columns!=col] # We remove the common columns + df = df.drop([col], axis=1) dfs_new.append(df) df = pd.concat(dfs_new, axis=1) + # Reindex at the end + df.insert(0, 'Index', np.arange(df.shape[0])) newName = self._tabs[I[0]].name+'_merged' self.append(Table(data=df, name=newName)) return newName, df diff --git a/tests/test_Tables.py b/tests/test_Tables.py index cb5bc28..0d386cc 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -50,6 +50,7 @@ def test_merge(self): np.testing.assert_almost_equal(df['ID'] , [0 , 0.5 , 1.0 , 1.5 , 2.0 , 2.5 , 3.0]) np.testing.assert_almost_equal(df['ColA'] , [10 , 10.5 , 11 , 11.5 , 12 , 12.5 , np.nan] ) np.testing.assert_almost_equal(df['ColB'] , [np.nan , np.nan , 11 , 11.5 , 12 , 12.5 , 13.0] ) + np.testing.assert_almost_equal(df['Index'], [0,1,2,3,4,5,6]) def test_resample(self): From 60f7941be630437dfb1ee452098b71e49b0ccd52 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 19 Sep 2022 00:23:17 -0400 Subject: [PATCH 047/178] Bug fix: table rename and column index export --- pydatview/GUISelectionPanel.py | 11 +++++------ pydatview/Tables.py | 4 +++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index af03262..d84a379 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -403,12 +403,11 @@ def OnRenameColumn(self, event=None): ITab,STab=main.selPanel.getSelectedTables() # TODO adapt me for Sim. tables mode iFull = self.parent.Filt2Full[iFilt] - if iFull>0: # Important since -1 would rename last column of table - if main.tabList.haveSameColumns(ITab): - for iTab,sTab in zip(ITab,STab): - main.tabList.get(iTab).renameColumn(iFull-1,newName) - else: - self.parent.tab.renameColumn(iFull-1,newName) + if main.tabList.haveSameColumns(ITab): + for iTab,sTab in zip(ITab,STab): + main.tabList.get(iTab).renameColumn(iFull,newName) + else: + self.parent.tab.renameColumn(iFull,newName) self.parent.updateColumn(iFilt,newName) #faster self.parent.selPanel.updateLayout() # a trigger for the plot is required but skipped for now diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 32ee6b3..d856f8a 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -727,7 +727,9 @@ def setColumnByFormula(self,sNewName,sFormula,i=-1): def export(self, path): if isinstance(self.data, pd.DataFrame): - df = self.data.drop('Index', axis=1) + df = self.data + if 'Index' in df.columns.values: + df = df.drop(['Index'], axis=1) df.to_csv(path, sep=',', index=False) else: raise NotImplementedError('Export of data that is not a dataframe') From 56e3ed2945886067096b77feb0ef546bc3c8a4db Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 26 Sep 2022 15:08:31 -0600 Subject: [PATCH 048/178] Adding small test for MacOS --- tests/mini.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/mini.py diff --git a/tests/mini.py b/tests/mini.py new file mode 100644 index 0000000..5806744 --- /dev/null +++ b/tests/mini.py @@ -0,0 +1,5 @@ +import wx +app = wx.App(False) +frame = wx.Frame(None, wx.ID_ANY, "Hello World") +frame.Show(True) +app.MainLoop() From 0e2dcaa4745e27af94944861851731b311241697 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 18 Oct 2022 10:57:04 -0600 Subject: [PATCH 049/178] FAST: support for ROSCO files with missing units --- pydatview/io/fast_input_file.py | 9 +++++++- pydatview/io/fast_output_file.py | 36 ++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index e5f493f..8e53900 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -1602,7 +1602,8 @@ def _toDataFrame(self): # Linear Cn try: - CnLin = self['C_nalpha'+labOffset]*(alpha-self['alpha0'+labOffset]*np.pi/180.) + CnLin_ = self['C_nalpha'+labOffset]*(alpha-self['alpha0'+labOffset]*np.pi/180.) + CnLin = CnLin_.copy() CnLin[alpha<-20*np.pi/180]=np.nan CnLin[alpha> 30*np.pi/180]=np.nan df['Cn_pot_[-]'] = CnLin @@ -1622,6 +1623,12 @@ def _toDataFrame(self): except: pass + # Cnf + try: + df['Cn_f_[-]'] = CnLin_ * ((1 + np.sqrt(0.7)) / 2) ** 2 + except: + pass + if len(dfs)==1: dfs=dfs[list(dfs.keys())[0]] return dfs diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index e3b3b1a..b79810a 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -43,12 +43,33 @@ class FASTOutputFile(File): @staticmethod def defaultExtensions(): - return ['.out','.outb','.elm','.elev'] + return ['.out','.outb','.elm','.elev','.dbg','.dbg2'] @staticmethod def formatName(): return 'FAST output file' + def __init__(self, filename=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # --- Calling (children) function to read + self._read(**kwargs) + def _read(self): def readline(iLine): with open(self.filename) as f: @@ -62,7 +83,7 @@ def readline(iLine): self.info={} self['binary']=False try: - if ext in ['.out','.elev']: + if ext in ['.out','.elev','.dbg','.dbg2']: self.data, self.info = load_ascii_output(self.filename) elif ext=='.outb': self.data, self.info = load_binary_output(self.filename) @@ -101,9 +122,16 @@ def _write(self): # TODO better.. f.write('\n'.join(['\t'.join(['{:10.4f}'.format(y[0])]+['{:10.3e}'.format(x) for x in y[1:]]) for y in self.data])) - def _toDataFrame(self): + def toDataFrame(self): + """ Returns object into one DataFrame, or a dictionary of DataFrames""" + # --- Example (returning one DataFrame): + # return pd.DataFrame(data=np.zeros((10,2)),columns=['Col1','Col2']) if self.info['attribute_units'] is not None: - cols=[n+'_['+u.replace('sec','s')+']' for n,u in zip(self.info['attribute_names'],self.info['attribute_units'])] + if len(self.info['attribute_names'])!=len(self.info['attribute_units']): + cols=self.info['attribute_names'] + print('[WARN] not all columns have units! Skipping units') + else: + cols=[n+'_['+u.replace('sec','s')+']' for n,u in zip(self.info['attribute_names'],self.info['attribute_units'])] else: cols=self.info['attribute_names'] if isinstance(self.data, pd.DataFrame): From 9be95261a56e52ebfd9ac32d79be34df926058af Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 7 Nov 2022 16:45:31 -0700 Subject: [PATCH 050/178] Bug Fix: crosshair does not erase itself (Close #129) --- pydatview/GUIToolBox.py | 111 +++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 48 deletions(-) diff --git a/pydatview/GUIToolBox.py b/pydatview/GUIToolBox.py index 8bca906..ff4a987 100644 --- a/pydatview/GUIToolBox.py +++ b/pydatview/GUIToolBox.py @@ -3,7 +3,7 @@ import matplotlib from matplotlib.backends.backend_wx import NavigationToolbar2Wx from matplotlib.backend_bases import NavigationToolbar2 -from matplotlib.widgets import Cursor, MultiCursor +from matplotlib.widgets import Cursor, MultiCursor, Widget # from matplotlib.widgets import AxesWidget def GetKeyString(evt): @@ -125,51 +125,38 @@ def TBAddTool(tb, label, defaultBitmap=None, callback=None, Type=None): # --------------------------------------------------------------------------------} # --- Plot Panel # --------------------------------------------------------------------------------{ -# class MyCursor(Cursor): -# def onmove(self, event): -# """on mouse motion draw the cursor if visible""" -# if self.ignore(event): -# return -# # MANU: Disabling lock below so that we can have cross hairwith zoom -# #if not self.canvas.widgetlock.available(self): -# # return -# if event.inaxes != self.ax: -# self.linev.set_visible(False) -# self.lineh.set_visible(False) -# -# if self.needclear: -# self.canvas.draw() -# self.needclear = False -# return -# self.needclear = True -# if not self.visible: -# return -# self.linev.set_xdata((event.xdata, event.xdata)) -# -# self.lineh.set_ydata((event.ydata, event.ydata)) -# self.linev.set_visible(self.visible and self.vertOn) -# self.lineh.set_visible(self.visible and self.horizOn) -# -# self._update() - -class MyMultiCursor(MultiCursor): - def __init__(self, canvas, axes, useblit=True, horizOn=False, vertOn=True, horizLocal=True, +class MyMultiCursor(Widget): + """ + Copy pasted from matplotlib.widgets.MultiCursor, version 3.6 + A change of interface occured between 3.5 and 3.6, it's simpler to just copy paste the whole class + The main changes are indicated with "MANU" below: + - adding a flag horizLocal (the horizontal cross hair is based on a given local axis (when having subplots)) + - setting the hlines and vlines per axes + - not returning when the zoom "widgetlock" is on to keep the cross hair in zoomed mode + """ + def __init__(self, canvas, axes, useblit=True, horizOn=False, vertOn=True, + horizLocal=True, # MANU **lineprops): - # Taken from matplotlib/widget.py but added horizLocal - super(MyMultiCursor,self).__init__(canvas, axes, useblit, horizOn, vertOn, **lineprops) - self.canvas = canvas self.axes = axes self.horizOn = horizOn self.vertOn = vertOn + + self._canvas_infos = { + ax.figure.canvas: {"cids": [], "background": None} for ax in axes} + self.visible = True - self.useblit = useblit and self.canvas.supports_blit - self.background = None + self.useblit = ( + useblit + and all(canvas.supports_blit for canvas in self._canvas_infos)) self.needclear = False + if self.useblit: lineprops['animated'] = True + + # MANU: xid and ymid are per axis basis + self.horizLocal = horizLocal self.vlines = [] self.hlines = [] - # MANU: xid and ymid are per axis basis for ax in axes: xmin, xmax = ax.get_xlim() ymin, ymax = ax.get_ylim() @@ -181,17 +168,38 @@ def __init__(self, canvas, axes, useblit=True, horizOn=False, vertOn=True, horiz self.hlines.append(ax.axhline(ymid, visible=False, **lineprops)) self.connect() - # --- - self.horizLocal = horizLocal - def onmove(self, event): + def connect(self): + """Connect events.""" + for canvas, info in self._canvas_infos.items(): + info["cids"] = [ + canvas.mpl_connect('motion_notify_event', self.onmove), + canvas.mpl_connect('draw_event', self.clear), + ] + + def disconnect(self): + """Disconnect events.""" + for canvas, info in self._canvas_infos.items(): + for cid in info["cids"]: + canvas.mpl_disconnect(cid) + info["cids"].clear() + + def clear(self, event): + """Clear the cursor.""" if self.ignore(event): return - if event.inaxes is None: + if self.useblit: + for canvas, info in self._canvas_infos.items(): + info["background"] = canvas.copy_from_bbox(canvas.figure.bbox) + for line in self.vlines + self.hlines: + line.set_visible(False) + + def onmove(self, event): + if (self.ignore(event) + or event.inaxes not in self.axes): + # MANU: Disabling lock below so that we can have cross hairwith zoom + # or not event.canvas.widgetlock.available(self)): return - # MANU: Disabling lock below so that we can have cross hairwith zoom - # if not self.canvas.widgetlock.available(self): - # return self.needclear = True if not self.visible: return @@ -203,12 +211,15 @@ def onmove(self, event): for line in self.hlines: line.set_ydata((event.ydata, event.ydata)) line.set_visible(self.visible) + #self._update() # MANU: adding current axes self._update(currentaxes=event.inaxes) - def _update(self,currentaxes=None): + + def _update(self, currentaxes=None): if self.useblit: - if self.background is not None: - self.canvas.restore_region(self.background) + for canvas, info in self._canvas_infos.items(): + if info["background"]: + canvas.restore_region(info["background"]) if self.vertOn: for ax, line in zip(self.axes, self.vlines): ax.draw_artist(line) @@ -217,9 +228,13 @@ def _update(self,currentaxes=None): for ax, line in zip(self.axes, self.hlines): if (self.horizLocal and currentaxes == ax) or (not self.horizLocal): ax.draw_artist(line) - self.canvas.blit(self.canvas.figure.bbox) + + for canvas in self._canvas_infos: + canvas.blit() else: - self.canvas.draw_idle() + for canvas in self._canvas_infos: + canvas.draw_idle() + class MyNavigationToolbar2Wx(NavigationToolbar2Wx): From 9f9b9befb057a3520828547fc82f40500fc99520 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 7 Nov 2022 16:48:07 -0700 Subject: [PATCH 051/178] Updates from welib --- pydatview/fast/fastfarm.py | 401 +++++++++++++++++++++---------- pydatview/fast/postpro.py | 23 +- pydatview/io/fast_input_file.py | 75 ++++-- pydatview/io/fast_output_file.py | 19 ++ pydatview/io/hawc2_htc_file.py | 2 +- pydatview/io/turbsim_file.py | 158 +++++++++++- pydatview/tools/fatigue.py | 42 ++++ 7 files changed, 559 insertions(+), 161 deletions(-) diff --git a/pydatview/fast/fastfarm.py b/pydatview/fast/fastfarm.py index 5e7dd07..1c48511 100644 --- a/pydatview/fast/fastfarm.py +++ b/pydatview/fast/fastfarm.py @@ -139,107 +139,180 @@ def rectangularLayoutSubDomains(D,Lx,Ly): return XWT, YWT, ZWT -def fastFarmTurbSimExtent(TurbSimFilename, HubHeight, D, xWT, yWT, Cmeander=1.9, Chord_max=3, extent_X=1.2, extent_Y=1.2): + +def fastFarmTurbSimExtent(TurbSimFilename, hubHeight, D, xWT, yWT, Cmeander=1.9, chord_max=3, extent_X=1.1, extent_YZ=1.1, meanUAtHubHeight=False): """ Determines "Ambient Wind" box parametesr for FastFarm, based on a TurbSimFile ('bts') + + Implements the guidelines listed here: + https://openfast.readthedocs.io/en/dev/source/user/fast.farm/ModelGuidance.html + + INPUTS: + - TurbSimFilename: name of the BTS file used in the FAST.Farm simulation + - hubHeight : Hub height [m] + - D : turbine diameter [m] + - xWT : vector of x positions of the wind turbines (e.g. [0,300,600]) + - yWT : vector of y positions of the wind turbines (e.g. [0,0,0]) + - Cmeander : parameter for meandering used in FAST.Farm [-] + - chord_max : maximum chord of the wind turbine blade. Used to determine the high resolution + - extent_X : x-extent of high res box in diamter around turbine location + - extent_YZ : y-extent of high res box in diamter around turbine location + """ # --- TurbSim data - ts = TurbSimFile(TurbSimFilename) - #iy,iz = ts.closestPoint(y=0,z=HubHeight) - #iy,iz = ts.closestPoint(y=0,z=HubHeight) - zMid, uMid = ts.midValues() - #print('uMid',uMid) - #meanU = ts['u'][0,:,iy,iz].mean() - meanU = uMid - dY_High = ts['y'][1]-ts['y'][0] - dZ_High = ts['z'][1]-ts['z'][0] - Z0_Low = ts['z'][0] - Z0_High = ts['z'][0] # we start at lowest to include tower - Width = ts['y'][-1]-ts['y'][0] - Height = ts['z'][-1]-ts['z'][0] - dT_High = ts['dt'] - #effSimLength = ts['t'][-1]-ts['t'][0] + Width/meanU - effSimLength = ts['t'][-1]-ts['t'][0] - - # Desired resolution, rule of thumbs - dX_High_desired = Chord_max + ts = TurbSimFile(TurbSimFilename) + + if meanUAtHubHeight: + # Use Hub Height to determine convection velocity + iy,iz = ts.closestPoint(y=0,z=hubHeight) + meanU = ts['u'][0,:,iy,iz].mean() + else: + # Use middle of the box to determine convection velocity + zMid, meanU = ts.midValues() + + return fastFarmBoxExtent(ts.y, ts.z, ts.t, meanU, hubHeight, D, xWT, yWT, Cmeander=Cmeander, chord_max=chord_max, extent_X=extent_X, extent_YZ=extent_YZ) + +def fastFarmBoxExtent(yBox, zBox, tBox, meanU, hubHeight, D, xWT, yWT, + Cmeander=1.9, chord_max=3, extent_X=1.1, extent_YZ=1.1, + extent_wake=8, LES=False): + """ + Determines "Ambient Wind" box parametesr for FastFarm, based on turbulence box parameters + INPUTS: + - yBox : y vector of grid points of the box + - zBox : z vector of grid points of the box + - tBox : time vector of the box + - meanU : mean velocity used to convect the box + - hubHeight : Hub height [m] + - D : turbine diameter [m] + - xWT : vector of x positions of the wind turbines (e.g. [0,300,600]) + - yWT : vector of y positions of the wind turbines (e.g. [0,0,0]) + - Cmeander : parameter for meandering used in FAST.Farm [-] + - chord_max : maximum chord of the wind turbine blade. Used to determine the high resolution + - extent_X : x-extent of high-res box (in diameter) around turbine location + - extent_YZ : y-extent of high-res box (in diameter) around turbine location + - extent_wake : extent of low-res box (in diameter) to add beyond the "last" wind turbine + - LES: False for TurbSim box, true for LES. Perform additional checks for LES. + """ + if LES: + raise NotImplementedError() + # --- Box resolution and extents + dY_Box = yBox[1]-yBox[0] + dZ_Box = zBox[1]-zBox[0] + dT_Box = tBox[1]-tBox[0] + dX_Box = dT_Box * meanU + Z0_Box = zBox[0] + LY_Box = yBox[-1]-yBox[0] + LZ_Box = zBox[-1]-zBox[0] + LT_Box = tBox[-1]-tBox[0] + LX_Box = LT_Box * meanU + + # --- Desired resolution, rules of thumb + dX_High_desired = chord_max dX_Low_desired = Cmeander*D*meanU/150.0 - dt_des = Cmeander*D/(10.0*meanU) - - # --- High domain - ZMax_High = HubHeight+extent_Y*D/2.0 - # high-box extent in x and y [D] - Xdist_High = extent_X*D - Ydist_High = extent_Y*D - Zdist_High = ZMax_High-Z0_High # we include the tower - X0_rel = Xdist_High/2.0 - Y0_rel = Ydist_High/2.0 - Length = effSimLength*meanU - nx = int(round(effSimLength/dT_High)) - dx_TS = Length/(nx-1) - #print('dx_TS',dx_TS) - dX_High = round(dX_High_desired/dx_TS)*dx_TS - #print('dX_High_desired',dX_High_desired, dX_High) + dY_Low_desired = dX_Low_desired + dZ_Low_desired = dX_Low_desired + dT_Low_desired = Cmeander*D/(10.0*meanU) + + # --- Suitable resolution for high res + dX_High = int(dX_High_desired/dX_Box)*dX_Box + if dX_High==0: raise Exception('The x-resolution of the box ({}) is too large and cannot satisfy the requirements for the high-res domain of dX~{} (based on chord_max). Reduce DX (or DT) of the box.'.format(dX_Box, dX_High_desired)) + dY_High = dY_Box # TODO? + dZ_High = dZ_Box # TODO? + dT_High = dT_Box # TODO? + + # --- Suitable resolution for Low res + dT_Low = int(dT_Low_desired/dT_Box )*dT_Box + dX_Low = int(dX_Low_desired/dX_High)*dX_High + dY_Low = int(dY_Low_desired/dY_High)*dY_High + dZ_Low = int(dZ_Low_desired/dZ_High)*dZ_High + if dT_Low==0: raise Exception('The time-resolution of the box ({}) is too large and cannot satisfy the requirements for the low-res domain of dT~{} (based on D & U). Reduce the DT of the box.'.format(dT_Box, dT_Low_desired)) + if dX_Low==0: raise Exception('The X-resolution of the box ({}) is too large and cannot satisfy the requirements for the low-res domain of dX~{} (based on D & U). Reduce the DX of the box.'.format(dX_Box, dX_Low_desired)) + if dY_Low==0: raise Exception('The Y-resolution of the box ({}) is too large and cannot satisfy the requirements for the low-res domain of dY~{} (based on D & U). Reduce the DY of the box.'.format(dY_Box, dY_Low_desired)) + if dZ_Low==0: raise Exception('The Z-resolution of the box ({}) is too large and cannot satisfy the requirements for the low-res domain of dZ~{} (based on D & U). Reduce the DZ of the box.'.format(dZ_Box, dZ_Low_desired)) + + # --- Low-res domain + # NOTE: more work is needed to make sure the domain encompass the turbines + # Also, we need to know the main flow direction to add a buffere with extent_wake + # Origin + nD_Before = extent_X/2 # Diameters before the first turbine to start the domain + X0_Low = np.floor( (min(xWT)-nD_Before*D-dX_Low)) # Starting on integer value for esthetics. With a dX_Low margin. + Y0_Low = np.floor( -LY_Box/2 ) # Starting on integer value for esthetics + Z0_Low = zBox[0] # we start at lowest to include tower + if LES: + if Y0_Low > min(yWT)-3*D: + Y0_Low = np.floor(min(yWT)-3*D) + # Extent NOTE: this assumes main flow about x. Might need to be changed - nX_High = int(round(Xdist_High/dX_High)+1) - nY_High = int(round(Ydist_High/dY_High)+1) - nZ_High = int(round(Zdist_High/dZ_High)+1) - - # --- High extent per turbine - nTurbs = len(xWT) - X0_des = np.asarray(xWT)-X0_rel - Y0_des = np.asarray(yWT)-Y0_rel - X0_High = np.around(np.round(X0_des/dX_High)*dX_High,3) - Y0_High = np.around(np.round(Y0_des/dY_High)*dY_High,3) - - # --- Low domain - dT_Low = round(dt_des/dT_High)*dT_High - dx_des = dX_Low_desired - dy_des = dX_Low_desired - dz_des = dX_Low_desired - X0_Low = round( (min(xWT)-2*D)/dX_High) *dX_High - Y0_Low = round( -Width/2 /dY_High) *dY_High - dX_Low = round( dx_des /dX_High)*dX_High - dY_Low = round( dy_des /dY_High)*dY_High - dZ_Low = round( dz_des /dZ_High)*dZ_High - Xdist = max(xWT)+8.0*D-X0_Low # Maximum extent - Ydist = Width - Zdist = Height - #print('dX_Low',dX_Low, dX_Low/dx_TS, dX_High/dx_TS) - - nX_Low = int(Xdist/dX_Low)+1; - nY_Low = int(Ydist/dY_Low)+1; - nZ_Low = int(Zdist/dZ_Low)+1; - - if (nX_Low*dX_Low>Xdist): - nX_Low=nX_Low-1 - if (nY_Low*dY_Low>Ydist): - nY_Low=nY_Low-1 - if (nZ_Low*dZ_Low>Zdist): - nZ_Low=nZ_Low-1 + XMax_Low = max(xWT) + extent_wake*D + LX_Low = XMax_Low-X0_Low + LY_Low = LY_Box + LZ_Low = LZ_Box + # Number of points + nX_Low = int(np.ceil(LX_Low/dX_Low)) + nY_Low = int(np.ceil(LY_Low/dY_Low)) + nZ_Low = int(np.ceil(LZ_Low/dZ_Low)) + # Make sure we don't exceed box in Y and Z + if (nY_Low*dY_Low>LY_Box): nY_Low=nY_Low-1 + if (nZ_Low*dZ_Low>LZ_Box): nZ_Low=nZ_Low-1 + + # --- High-res domain extent and number of points + ZMax_High = hubHeight+extent_YZ*D/2 + Z0_High = zBox[0] # we start at lowest to include tower + LX_High = extent_X*D + LY_High = min(LY_Box, extent_YZ*D ) # Bounding to not exceed the box dimension + LZ_High = min(LZ_Box, ZMax_High-Z0_High) # Bounding to not exceed the box dimension + nX_High = int(np.ceil(LX_High/dX_High)) + nY_High = int(np.ceil(LY_High/dY_High)) + nZ_High = int(np.ceil(LZ_High/dZ_High)) + # Make sure we don't exceed box in Y and Z + if (nY_High*dY_High>LY_Box): nY_High=nY_High-1 + if (nZ_High*dZ_High>LZ_Box): nZ_High=nZ_High-1 + + # --- High-res location per turbine + X0_desired = np.asarray(xWT)-LX_High/2 # high-res is centered on turbine location + Y0_desired = np.asarray(yWT)-LY_High/2 # high-res is centered on turbine location + X0_High = X0_Low + np.floor((X0_desired-X0_Low)/dX_High)*dX_High + Y0_High = Y0_Low + np.floor((Y0_desired-Y0_Low)/dY_High)*dY_High d = dict() - d['DT'] = np.around(dT_Low ,3) - d['DT_High'] = np.around(dT_High,3) - d['NX_Low'] = int(nX_Low) - d['NY_Low'] = int(nY_Low) - d['NZ_Low'] = int(nZ_Low) - d['X0_Low'] = np.around(X0_Low,3) - d['Y0_Low'] = np.around(Y0_Low,3) - d['Z0_Low'] = np.around(Z0_Low,3) - d['dX_Low'] = np.around(dX_Low,3) - d['dY_Low'] = np.around(dY_Low,3) - d['dZ_Low'] = np.around(dZ_Low,3) - d['NX_High'] = int(nX_High) - d['NY_High'] = int(nY_High) - d['NZ_High'] = int(nZ_High) + d['DT_Low'] = np.around(dT_Low ,4) + d['DT_High'] = np.around(dT_High,4) + d['NX_Low'] = nX_Low + d['NY_Low'] = nY_Low + d['NZ_Low'] = nZ_Low + d['X0_Low'] = np.around(X0_Low,4) + d['Y0_Low'] = np.around(Y0_Low,4) + d['Z0_Low'] = np.around(Z0_Low,4) + d['dX_Low'] = np.around(dX_Low,4) + d['dY_Low'] = np.around(dY_Low,4) + d['dZ_Low'] = np.around(dZ_Low,4) + d['NX_High'] = nX_High + d['NY_High'] = nY_High + d['NZ_High'] = nZ_High # --- High extent info for turbine outputs - d['dX_High'] = np.around(dX_High,3) - d['dY_High'] = np.around(dY_High,3) - d['dZ_High'] = np.around(dZ_High,3) - d['X0_High'] = X0_High - d['Y0_High'] = Y0_High - d['Z0_High'] = np.around(Z0_High,3) + d['dX_High'] = np.around(dX_High,4) + d['dY_High'] = np.around(dY_High,4) + d['dZ_High'] = np.around(dZ_High,4) + d['X0_High'] = np.around(X0_High,4) + d['Y0_High'] = np.around(Y0_High,4) + d['Z0_High'] = np.around(Z0_High,4) + # --- Misc + d['dX_des_High'] = dX_High_desired + d['dX_des_Low'] = dX_Low_desired + d['DT_des'] = dT_Low_desired + d['U_mean'] = meanU + + # --- Sanity check: check that the high res is at "almost" an integer location + X_rel = (np.array(d['X0_High'])-d['X0_Low'])/d['dX_High'] + Y_rel = (np.array(d['Y0_High'])-d['Y0_Low'])/d['dY_High'] + dX = X_rel - np.round(X_rel) # Should be close to zero + dY = Y_rel - np.round(Y_rel) # Should be close to zero + if any(abs(dX)>1e-3): + print('Deltas:',dX) + raise Exception('Some X0_High are not on an integer multiple of the high-res grid') + if any(abs(dY)>1e-3): + print('Deltas:',dY) + raise Exception('Some Y0_High are not on an integer multiple of the high-res grid') return d @@ -256,12 +329,13 @@ def writeFastFarm(outputFile, templateFile, xWT, yWT, zWT, FFTS=None, OutListT1= # --- Replace box extent values if FFTS is not None: fst['Mod_AmbWind'] = 2 - for k in ['DT', 'DT_High', 'NX_Low', 'NY_Low', 'NZ_Low', 'X0_Low', 'Y0_Low', 'Z0_Low', 'dX_Low', 'dY_Low', 'dZ_Low', 'NX_High', 'NY_High', 'NZ_High']: + ModVars = ['DT_Low', 'DT_High', 'NX_Low', 'NY_Low', 'NZ_Low', 'X0_Low', 'Y0_Low', 'Z0_Low', 'dX_Low', 'dY_Low', 'dZ_Low', 'NX_High', 'NY_High', 'NZ_High'] + for k in ModVars: if isinstance(FFTS[k],int): fst[k] = FFTS[k] else: fst[k] = np.around(FFTS[k],3) - fst['WrDisDT'] = FFTS['DT'] + fst['WrDisDT'] = FFTS['DT_Low'] # --- Set turbine names, position, and box extent nWT = len(xWT) @@ -305,49 +379,118 @@ def setFastFarmOutputs(fastFarmFile, OutListT1): fst.write(fastFarmFile) -def plotFastFarmSetup(fastFarmFile): +def plotFastFarmSetup(fastFarmFile, grid=True, fig=None, D=None, plane='XY', hubHeight=None): """ """ import matplotlib.pyplot as plt + + def col(i): + Colrs=plt.rcParams['axes.prop_cycle'].by_key()['color'] + return Colrs[ np.mod(i,len(Colrs)) ] + def boundingBox(x, y): + """ return x and y coordinates to form a box marked by the min and max of x and y""" + x_bound = [x[0],x[-1],x[-1],x[0] ,x[0]] + y_bound = [y[0],y[0] ,y[-1],y[-1],y[0]] + return x_bound, y_bound + + + # --- Read FAST.Farm input file fst=FASTInputFile(fastFarmFile) - fig = plt.figure(figsize=(13.5,10)) - ax = fig.add_subplot(111,aspect="equal") + if fig is None: + fig = plt.figure(figsize=(13.5,8)) + ax = fig.add_subplot(111,aspect="equal") WT=fst['WindTurbines'] - x = WT[:,0].astype(float) - y = WT[:,1].astype(float) + xWT = WT[:,0].astype(float) + yWT = WT[:,1].astype(float) + zWT = yWT*0 + if hubHeight is not None: + zWT += hubHeight + + if plane == 'XY': + pass + elif plane == 'XZ': + yWT = zWT + elif plane == 'YZ': + xWT = yWT + yWT = zWT + else: + raise Exception("Plane should be 'XY' 'XZ' or 'YZ'") if fst['Mod_AmbWind'] == 2: - xmax_low = fst['X0_Low']+fst['DX_Low']*fst['NX_Low'] - ymax_low = fst['Y0_Low']+fst['DY_Low']*fst['NY_Low'] - # low-res box - ax.plot([fst['X0_Low'],xmax_low,xmax_low,fst['X0_Low'],fst['X0_Low']], - [fst['Y0_Low'],fst['Y0_Low'],ymax_low,ymax_low,fst['Y0_Low']],'--k',lw=2,label='Low') + x_low = fst['X0_Low'] + np.arange(fst['NX_Low']+1)*fst['DX_Low'] + y_low = fst['Y0_Low'] + np.arange(fst['NY_Low']+1)*fst['DY_Low'] + z_low = fst['Z0_Low'] + np.arange(fst['NZ_Low']+1)*fst['DZ_Low'] + if plane == 'XZ': + y_low = z_low + elif plane == 'YZ': + x_low = y_low + y_low = z_low + # Plot low-res box + x_bound_low, y_bound_low = boundingBox(x_low, y_low) + ax.plot(x_bound_low, y_bound_low ,'--k',lw=2,label='Low-res') + # Plot Low res grid lines + if grid: + ax.vlines(x_low, ymin=y_low[0], ymax=y_low[-1], ls='-', lw=0.3, color=(0.3,0.3,0.3)) + ax.hlines(y_low, xmin=x_low[0], xmax=x_low[-1], ls='-', lw=0.3, color=(0.3,0.3,0.3)) + X0_High = WT[:,4].astype(float) Y0_High = WT[:,5].astype(float) + Z0_High = WT[:,6].astype(float) dX_High = WT[:,7].astype(float)[0] dY_High = WT[:,8].astype(float)[0] + dZ_High = WT[:,9].astype(float)[0] nX_High = fst['NX_High'] nY_High = fst['NY_High'] + nZ_High = fst['NZ_High'] + # high-res boxes - for wt in range(len(x)): - xmax_high = X0_High[wt]+dX_High*nX_High - ymax_high = Y0_High[wt]+dY_High*nY_High - ax.plot([X0_High[wt],xmax_high,xmax_high,X0_High[wt],X0_High[wt]], - [Y0_High[wt],Y0_High[wt],ymax_high,ymax_high,Y0_High[wt]], - '-', - label="HighT{0}".format(wt+1)) - ax.plot(x[wt],y[wt],'x',ms=8,mew=2,label="WT{0}".format(wt+1)) - else: - for wt in range(len(x)): - ax.plot(x[wt],y[wt],'x',ms=8,mew=2,label="WT{0}".format(wt+1)) - # - plt.legend(bbox_to_anchor=(1.05,1.015),frameon=False) - ax.set_xlabel("x-location [m]") - ax.set_ylabel("y-location [m]") + for wt in range(len(xWT)): + x_high = X0_High[wt] + np.arange(nX_High+1)*dX_High + y_high = Y0_High[wt] + np.arange(nY_High+1)*dY_High + z_high = Z0_High[wt] + np.arange(nZ_High+1)*dZ_High + if plane == 'XZ': + y_high = z_high + elif plane == 'YZ': + x_high = y_high + y_high = z_high + + x_bound_high, y_bound_high = boundingBox(x_high, y_high) + ax.plot(x_bound_high, y_bound_high, '-', lw=2, c=col(wt)) + # Plot High res grid lines + if grid: + ax.vlines(x_high, ymin=y_high[0], ymax=y_high[-1], ls='--', lw=0.4, color=col(wt)) + ax.hlines(y_high, xmin=x_high[0], xmax=x_high[-1], ls='--', lw=0.4, color=col(wt)) + + # Plot turbines + for wt in range(len(xWT)): + ax.plot(xWT[wt], yWT[wt], 'x', ms=8, mew=2, c=col(wt),label="WT{}".format(wt+1)) + if plane=='XY' and D is not None: + ax.plot([xWT[wt],xWT[wt]], [yWT[wt]-D/2,yWT[wt]+D/2], '-', lw=2, c=col(wt)) + elif plane=='XZ' and D is not None and hubHeight is not None: + ax.plot([xWT[wt],xWT[wt]], [yWT[wt]-D/2,yWT[wt]+D/2], '-', lw=2, c=col(wt)) + elif plane=='YZ' and D is not None and hubHeight is not None: + theta = np.linspace(0,2*np.pi, 40) + x = xWT[wt] + D/2*np.cos(theta) + y = yWT[wt] + D/2*np.sin(theta) + ax.plot(x, y, '-', lw=2, c=col(wt)) + + #plt.legend(bbox_to_anchor=(1.05,1.015),frameon=False) + ax.legend() + if plane=='XY': + ax.set_xlabel("x [m]") + ax.set_ylabel("y [m]") + elif plane=='XZ': + ax.set_xlabel("x [m]") + ax.set_ylabel("z [m]") + elif plane=='YZ': + ax.set_xlabel("y [m]") + ax.set_ylabel("z [m]") fig.tight_layout # fig.savefig('FFarmLayout.pdf',bbox_to_inches='tight',dpi=500) + return fig + # --------------------------------------------------------------------------------} # --- Tools for postpro # --------------------------------------------------------------------------------{ @@ -356,13 +499,13 @@ def spanwiseColFastFarm(Cols, nWT=9, nD=9): """ Return column info, available columns and indices that contain AD spanwise data""" FFSpanMap=dict() for i in np.arange(nWT): - FFSpanMap['^CtT{:d}N(\d*)_\[-\]'.format(i+1)]='CtT{:d}_[-]'.format(i+1) + FFSpanMap[r'^CtT{:d}N(\d*)_\[-\]'.format(i+1)]='CtT{:d}_[-]'.format(i+1) for i in np.arange(nWT): for k in np.arange(nD): - FFSpanMap['^WkDfVxT{:d}N(\d*)D{:d}_\[m/s\]'.format(i+1,k+1) ]='WkDfVxT{:d}D{:d}_[m/s]'.format(i+1, k+1) + FFSpanMap[r'^WkDfVxT{:d}N(\d*)D{:d}_\[m/s\]'.format(i+1,k+1) ]='WkDfVxT{:d}D{:d}_[m/s]'.format(i+1, k+1) for i in np.arange(nWT): for k in np.arange(nD): - FFSpanMap['^WkDfVrT{:d}N(\d*)D{:d}_\[m/s\]'.format(i+1,k+1) ]='WkDfVrT{:d}D{:d}_[m/s]'.format(i+1, k+1) + FFSpanMap[r'^WkDfVrT{:d}N(\d*)D{:d}_\[m/s\]'.format(i+1,k+1) ]='WkDfVrT{:d}D{:d}_[m/s]'.format(i+1, k+1) return fastlib.find_matching_columns(Cols, FFSpanMap) @@ -371,16 +514,16 @@ def diameterwiseColFastFarm(Cols, nWT=9): FFDiamMap=dict() for i in np.arange(nWT): for x in ['X','Y','Z']: - FFDiamMap['^WkAxs{}T{:d}D(\d*)_\[-\]'.format(x,i+1)] ='WkAxs{}T{:d}_[-]'.format(x,i+1) + FFDiamMap[r'^WkAxs{}T{:d}D(\d*)_\[-\]'.format(x,i+1)] ='WkAxs{}T{:d}_[-]'.format(x,i+1) for i in np.arange(nWT): for x in ['X','Y','Z']: - FFDiamMap['^WkPos{}T{:d}D(\d*)_\[m\]'.format(x,i+1)] ='WkPos{}T{:d}_[m]'.format(x,i+1) + FFDiamMap[r'^WkPos{}T{:d}D(\d*)_\[m\]'.format(x,i+1)] ='WkPos{}T{:d}_[m]'.format(x,i+1) for i in np.arange(nWT): for x in ['X','Y','Z']: - FFDiamMap['^WkVel{}T{:d}D(\d*)_\[m/s\]'.format(x,i+1)] ='WkVel{}T{:d}_[m/s]'.format(x,i+1) + FFDiamMap[r'^WkVel{}T{:d}D(\d*)_\[m/s\]'.format(x,i+1)] ='WkVel{}T{:d}_[m/s]'.format(x,i+1) for i in np.arange(nWT): for x in ['X','Y','Z']: - FFDiamMap['^WkDiam{}T{:d}D(\d*)_\[m\]'.format(x,i+1)] ='WkDiam{}T{:d}_[m]'.format(x,i+1) + FFDiamMap[r'^WkDiam{}T{:d}D(\d*)_\[m\]'.format(x,i+1)] ='WkDiam{}T{:d}_[m]'.format(x,i+1) return fastlib.find_matching_columns(Cols, FFDiamMap) def SensorsFARMRadial(nWT=3,nD=10,nR=30,signals=None): @@ -437,11 +580,11 @@ def spanwisePostProFF(fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1, # --- Opening input file and extracting inportant variables if fastfarm_input is None: # We don't have an input file, guess numbers of turbine, diameters, Nodes... - cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'T(\d+)') + cols, sIdx = fastlib.find_matching_pattern(df.columns.values, r'T(\d+)') nWT = np.array(sIdx).astype(int).max() - cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'D(\d+)') + cols, sIdx = fastlib.find_matching_pattern(df.columns.values, r'D(\d+)') nD = np.array(sIdx).astype(int).max() - cols, sIdx = fastlib.find_matching_pattern(df.columns.values, 'N(\d+)') + cols, sIdx = fastlib.find_matching_pattern(df.columns.values, r'N(\d+)') nr = np.array(sIdx).astype(int).max() vr=None vD=None diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index ff9a445..0279c03 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -163,8 +163,9 @@ def BD_BldStations(BD, BDBld): This will NOT match the "Initial Nodes" reported in the summary file. INPUTS: - BD: either: - - a filename of a ElastoDyn input file + - a filename of a BeamDyn input file - an instance of FileCl, as returned by reading the file, BD = weio.read(BD_filename) + - BDBld: same as BD but for the BeamDyn blade file OUTPUTS: - r_nodes: spanwise position from the balde root of the Blade stations """ @@ -177,7 +178,6 @@ def BD_BldStations(BD, BDBld): # BD['BldFile'].replace('"','')) - # --- Extract relevant info from BD files z_kp = BD['MemberGeom'][:,2] R = z_kp[-1]-z_kp[0] @@ -210,9 +210,7 @@ def BD_BldStations(BD, BDBld): r = np.concatenate( (rStations, rmid)) r = np.unique(np.sort(r)) else: - - raise NotImplementedError('BeamDyn with Gaussian quadrature points') - + raise NotImplementedError('Only Gauss and Trap quadrature implemented') return r def BD_BldGag(BD): @@ -867,14 +865,13 @@ def FASTRadialOutputs(FST_In, OutputCols=None): print('[WARN] The Elastodyn file couldn''t be found or read, from main file: '+FST_In) #raise Exception('The Elastodyn file couldn''t be found or read, from main file: '+FST_In) else: - if fst.ED is not None: - R = fst.ED['TipRad'] - r_hub = fst.ED['HubRad'] - if fst.ED.hasNodal: - _, r_ED = ED_BldStations(fst.ED) - IR_ED =None - else: - r_ED, IR_ED = ED_BldGag(fst.ED) + R = fst.ED['TipRad'] + r_hub = fst.ED['HubRad'] + if fst.ED.hasNodal: + _, r_ED = ED_BldStations(fst.ED) + IR_ED =None + else: + r_ED, IR_ED = ED_BldGag(fst.ED) # --- BeamDyn if fst.BD is not None: diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index 8e53900..3abbda9 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -16,6 +16,7 @@ class BrokenFormatError(Exception): pass TABTYPE_NUM_WITH_HEADERCOM = 2 TABTYPE_NUM_NO_HEADER = 4 TABTYPE_NUM_BEAMDYN = 5 +TABTYPE_NUM_SUBDYNOUT = 7 TABTYPE_MIX_WITH_HEADER = 6 TABTYPE_FIL = 3 TABTYPE_FMT = 9999 # TODO @@ -236,9 +237,15 @@ def __next__(self): # Python 2: def next(self) return self.data[self.iCurrent] # Making it behave like a dictionary - def __setitem__(self,key,item): + def __setitem__(self, key, item): I = self.getIDs(key) for i in I: + if self.data[i]['tabType'] != TABTYPE_NOT_A_TAB: + # For tables, we automatically update variable that stores the dimension + nRows = len(item) + dimVar = self.data[i]['tabDimVar'] + iDimVar = self.getID(dimVar) + self.data[iDimVar]['value'] = nRows # Avoiding a recursive call to __setitem__ here self.data[i]['value'] = item def __getitem__(self,key): @@ -290,11 +297,14 @@ def comment(self, comment): splits = comment.split('\n') for i,com in zip(self._IComment, splits): self.data[i]['value'] = com + self.data[i]['label'] = '' + self.data[i]['descr'] = '' + self.data[i]['isComment'] = True @property def _IComment(self): """ return indices of comment line""" - return [] # Typical OpenFAST files have comment on second line [1] + return [1] # Typical OpenFAST files have comment on second line [1] def read(self, filename=None): @@ -313,18 +323,23 @@ def _read(self): # --- Tables that can be detected based on the "Value" (first entry on line) # TODO members for BeamDyn with mutliple key point ####### TODO PropSetID is Duplicate SubDyn and used in HydroDyn - NUMTAB_FROM_VAL_DETECT = ['HtFract' , 'TwrElev' , 'BlFract' , 'Genspd_TLU' , 'BlSpn' , 'WndSpeed' , 'HvCoefID' , 'AxCoefID' , 'JointID' , 'Dpth' , 'FillNumM' , 'MGDpth' , 'SimplCd' , 'RNodes' , 'kp_xr' , 'mu1' , 'TwrHtFr' , 'TwrRe' , 'WT_X'] - NUMTAB_FROM_VAL_DIM_VAR = ['NTwInpSt' , 'NumTwrNds' , 'NBlInpSt' , 'DLL_NumTrq' , 'NumBlNds' , 'NumCases' , 'NHvCoef' , 'NAxCoef' , 'NJoints' , 'NCoefDpth' , 'NFillGroups' , 'NMGDepths' , 1 , 'BldNodes' , 'kp_total' , 1 , 'NTwrHt' , 'NTwrRe' , 'NumTurbines'] - NUMTAB_FROM_VAL_VARNAME = ['TowProp' , 'TowProp' , 'BldProp' , 'DLLProp' , 'BldAeroNodes' , 'Cases' , 'HvCoefs' , 'AxCoefs' , 'Joints' , 'DpthProp' , 'FillGroups' , 'MGProp' , 'SmplProp' , 'BldAeroNodes' , 'MemberGeom' , 'DampingCoeffs' , 'TowerProp' , 'TowerRe', 'WindTurbines'] - NUMTAB_FROM_VAL_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 1 , 2 , 2 , 1 , 1 , 2 ] - NUMTAB_FROM_VAL_TYPE = ['num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' , 'num' , 'mix'] + NUMTAB_FROM_VAL_DETECT = ['HtFract' , 'TwrElev' , 'BlFract' , 'Genspd_TLU' , 'BlSpn' , 'HvCoefID' , 'AxCoefID' , 'JointID' , 'Dpth' , 'FillNumM' , 'MGDpth' , 'SimplCd' , 'RNodes' , 'kp_xr' , 'mu1' , 'TwrHtFr' , 'TwrRe' , 'WT_X'] + NUMTAB_FROM_VAL_DIM_VAR = ['NTwInpSt' , 'NumTwrNds' , 'NBlInpSt' , 'DLL_NumTrq' , 'NumBlNds' , 'NHvCoef' , 'NAxCoef' , 'NJoints' , 'NCoefDpth' , 'NFillGroups' , 'NMGDepths' , 1 , 'BldNodes' , 'kp_total' , 1 , 'NTwrHt' , 'NTwrRe' , 'NumTurbines'] + NUMTAB_FROM_VAL_VARNAME = ['TowProp' , 'TowProp' , 'BldProp' , 'DLLProp' , 'BldAeroNodes' , 'HvCoefs' , 'AxCoefs' , 'Joints' , 'DpthProp' , 'FillGroups' , 'MGProp' , 'SmplProp' , 'BldAeroNodes' , 'MemberGeom' , 'DampingCoeffs' , 'TowerProp' , 'TowerRe', 'WindTurbines'] + NUMTAB_FROM_VAL_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 1 , 2 , 2 , 1 , 1 , 2 ] + NUMTAB_FROM_VAL_TYPE = ['num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' , 'num' , 'mix'] # SubDyn NUMTAB_FROM_VAL_DETECT += [ 'RJointID' , 'IJointID' , 'COSMID' , 'CMJointID' ] NUMTAB_FROM_VAL_DIM_VAR += [ 'NReact' , 'NInterf' , 'NCOSMs' , 'NCmass' ] NUMTAB_FROM_VAL_VARNAME += [ 'BaseJoints' , 'InterfaceJoints' , 'MemberCosineMatrix' , 'ConcentratedMasses'] NUMTAB_FROM_VAL_NHEADER += [ 2 , 2 , 2 , 2 ] NUMTAB_FROM_VAL_TYPE += [ 'mix' , 'num' , 'num' , 'num' ] - + # AD Driver old and new + NUMTAB_FROM_VAL_DETECT += [ 'WndSpeed' , 'HWndSpeed' ] + NUMTAB_FROM_VAL_DIM_VAR += [ 'NumCases' , 'NumCases' ] + NUMTAB_FROM_VAL_VARNAME += [ 'Cases' , 'Cases' ] + NUMTAB_FROM_VAL_NHEADER += [ 2 , 2 ] + NUMTAB_FROM_VAL_TYPE += [ 'num' , 'num' ] # --- Tables that can be detected based on the "Label" (second entry on line) # NOTE: MJointID1, used by SubDyn and HydroDyn @@ -333,7 +348,7 @@ def _read(self): NUMTAB_FROM_LAB_VARNAME = ['AFCoeff' , 'TMDspProp' , 'MemberProp' , 'Members' , 'MemberOuts' , 'MemberOuts' , 'SectionProp' ,'LineTypes' ,'ConnectionProp' ,'LineProp' ] NUMTAB_FROM_LAB_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 ] NUMTAB_FROM_LAB_NOFFSET = [0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ] - NUMTAB_FROM_LAB_TYPE = ['num' , 'num' , 'num' , 'mix' , 'num' , 'num' , 'num' ,'mix' ,'mix' ,'mix' ] + NUMTAB_FROM_LAB_TYPE = ['num' , 'num' , 'num' , 'mix' , 'num' , 'sdout' , 'num' ,'mix' ,'mix' ,'mix' ] # SubDyn NUMTAB_FROM_LAB_DETECT += ['GuyanDampSize' , 'YoungE' , 'YoungE' , 'EA' , 'MatDens' ] NUMTAB_FROM_LAB_DIM_VAR += [6 , 'NPropSets', 'NXPropSets', 'NCablePropSets' , 'NRigidPropSets'] @@ -403,7 +418,7 @@ def _read(self): # Parsing outlist, and then we continue at a new "i" (to read END etc.) OutList,i = parseFASTOutList(lines,i+1) d = getDict() - if self.hasNodal: + if self.hasNodal and not firstword.endswith('_Nodal'): d['label'] = firstword+'_Nodal' else: d['label'] = firstword @@ -579,6 +594,8 @@ def _read(self): else: if tab_type=='num': d['tabType'] = TABTYPE_NUM_WITH_HEADER + elif tab_type=='sdout': + d['tabType'] = TABTYPE_NUM_SUBDYNOUT else: d['tabType'] = TABTYPE_MIX_WITH_HEADER if isinstance(d['tabDimVar'],int): @@ -666,6 +683,16 @@ def toStringVLD(val,lab,descr): lab='{:13s}'.format(lab) return val+' '+lab+' - '+descr.strip().strip('-').strip()+'\n' + def toStringIntFloatStr(x): + try: + if int(x)==x: + s='{:15.0f}'.format(x) + else: + s='{:15.8e}'.format(x) + except: + s=x + return s + def beamdyn_section_mat_tostring(x,K,M): def mat_tostring(M,fmt='24.16e'): return '\n'.join([' '+' '.join(['{:24.16E}'.format(m) for m in M[i,:]]) for i in range(np.size(M,1))]) @@ -712,15 +739,18 @@ def mat_tostring(M,fmt='24.16e'): s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) if np.size(d['value'],0) > 0 : s+='\n' - s+='\n'.join('\t'.join('{}'.format(x) for x in y) for y in d['value']) + s+='\n'.join('\t'.join(toStringIntFloatStr(x) for x in y) for y in d['value']) elif d['tabType']==TABTYPE_NUM_WITH_HEADERCOM: s+='! {}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) s+='! {}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) s+='\n'.join('\t'.join('{:15.8e}'.format(x) for x in y) for y in d['value']) elif d['tabType']==TABTYPE_FIL: #f.write('{} {} {}\n'.format(d['value'][0],d['tabDetect'],d['descr'])) - s+='{} {} {}\n'.format(d['value'][0],d['label'],d['descr']) # TODO? - s+='\n'.join(fil for fil in d['value'][1:]) + if len(d['value'])==1: + s+='{} {} {}'.format(d['value'][0],d['label'],d['descr']) # TODO? + else: + s+='{} {} {}\n'.format(d['value'][0],d['label'],d['descr']) # TODO? + s+='\n'.join(fil for fil in d['value'][1:]) elif d['tabType']==TABTYPE_NUM_BEAMDYN: # TODO use dedicated sub-class data = d['value'] @@ -732,6 +762,11 @@ def mat_tostring(M,fmt='24.16e'): K = data['K'][i] M = data['M'][i] s += beamdyn_section_mat_tostring(x,K,M) + elif d['tabType']==TABTYPE_NUM_SUBDYNOUT: + data = d['value'] + s+='{}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) + s+='{}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) + s+='\n'.join('\t'.join('{:15.0f}'.format(x) for x in y) for y in data) else: raise Exception('Unknown table type for variable {}'.format(d)) if i0: + print('[WARN] Creating directory: ',dirname) + os.makedirs(dirname) + self._write() else: raise Exception('No filename provided') @@ -1242,6 +1282,13 @@ def parseFASTNumTable(filename,lines,n,iStart,nHeaders=2,tableType='num',nOffset # If all values are float, we convert to float if all([strIsFloat(x) for x in Tab.ravel()]): Tab=Tab.astype(float) + elif tableType=='sdout': + header = lines[0] + units = lines[1] + Tab=[] + for i in range(nHeaders+nOffset,n+nHeaders+nOffset): + l = cleanAfterChar(lines[i].lower(),'!') + Tab.append( np.array(l.split()).astype(int)) else: raise Exception('Unknown table type') @@ -1635,7 +1682,7 @@ def _toDataFrame(self): @property def comment(self): - return FASTInputFileBase.comment + return '\n'.join([self.data[i]['value'] for i in self._IComment]) @comment.setter def comment(self, comment): diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index b79810a..6db3bff 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -1,3 +1,15 @@ +""" +Tools to read/write OpenFAST output files + +Main content: + +- class FASTOutputFile() +- data, info = def load_output(filename) +- data, info = def load_ascii_output(filename) +- data, info = def load_binary_output(filename, use_buffer=True) +- def writeDataFrame(df, filename, binary=True) +- def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr='') +""" from itertools import takewhile import numpy as np import pandas as pd @@ -145,6 +157,13 @@ def toDataFrame(self): def writeDataFrame(self, df, filename, binary=True): writeDataFrame(df, filename, binary=binary) + def __repr__(self): + s='<{} object> with attributes:\n'.format(type(self).__name__) + s+=' - info ({})\n'.format(type(self.info)) + s+=' - data ({})\n'.format(type(self.data)) + s+='and keys: {}\n'.format(self.keys()) + return s + # -------------------------------------------------------------------------------- # --- Helper low level functions # -------------------------------------------------------------------------------- diff --git a/pydatview/io/hawc2_htc_file.py b/pydatview/io/hawc2_htc_file.py index 7a95c5d..0fefb1b 100644 --- a/pydatview/io/hawc2_htc_file.py +++ b/pydatview/io/hawc2_htc_file.py @@ -116,7 +116,7 @@ def _toDataFrame(self): if not os.path.exists(H2_stfile): print('[WARN] st file referenced in htc file was not found. St file: {}, htc file {}'.format(H2_stfile, self.filename)) else: - dfs_st = HAWC2StFile(H2_stfile).toDataFrame() + dfs_st = HAWC2StFile(H2_stfile).toDataFrame(extraCols=False) if 'set' in tim.keys(): mset = tim.set[0] iset = tim.set[1] diff --git a/pydatview/io/turbsim_file.py b/pydatview/io/turbsim_file.py index 0b421c5..99f33d0 100644 --- a/pydatview/io/turbsim_file.py +++ b/pydatview/io/turbsim_file.py @@ -53,7 +53,7 @@ def defaultExtensions(): def formatName(): return 'TurbSim binary' - def __init__(self,filename=None, **kwargs): + def __init__(self, filename=None, **kwargs): self.filename = None if filename: self.read(filename, **kwargs) @@ -174,13 +174,32 @@ def write(self, filename=None): # --- Convenient properties (matching Mann Box interface as well) # --------------------------------------------------------------------------------{ @property - def z(self): return self['z'] + def z(self): return self['z'] # np.arange(nz)*dz +zBottom @property - def y(self): return self['y'] + def y(self): return self['y'] # np.arange(ny)*dy - np.mean( np.arange(ny)*dy ) + + @property + def t(self): return self['t'] # np.arange(nt)*dt + + # NOTE: it would be best to use dz and dy as given in the file to avoid numerical issues + @property + def dz(self): return self['z'][1]-self['z'][0] + + @property + def dy(self): return self['y'][1]-self['y'][0] + + @property + def dt(self): return self['t'][1]-self['t'][0] @property - def t(self): return self['t'] + def nz(self): return len(self.z) + + @property + def ny(self): return len(self.y) + + @property + def nt(self): return len(self.t) # --------------------------------------------------------------------------------} # --- Extracting relevant "Line" data at one point @@ -660,6 +679,52 @@ def toDataFrame(self): # Useful converters + def fromAMRWind(self, filename, dt, nt): + """ + Convert current TurbSim file into one generated from AMR-Wind LES sampling data in .nc format + Assumes: + -- u, v, w (nt, nx * ny * nz) + -- u is aligned with x-axis (flow is not rotated) - this consideration needs to be added + + INPUTS: + - filename: (string) full path to .nc sampling data file + - plane_label: (string) name of sampling plane group from .inp file (e.g. "p_sw2") + - dt: timestep size [s] + - nt: number of timesteps (sequential) you want to read in, starting at the first timestep available + - y: user-defined vector of coordinate positions in y + - z: user-defined vector of coordinate positions in z + - uref: (float) reference mean velocity (e.g. 8.0 hub height mean velocity from input file) + - zref: (float) hub height (e.t. 150.0) + """ + import xarray as xr + + # read in sampling data plane + ds = xr.open_dataset(filename, + engine='netcdf4', + group=plane_label) + ny, nz, _ = ds.attrs['ijk_dims'] + noffsets = len(ds.attrs['offsets']) + t = np.arange(0, dt*(nt-0.5), dt) + print('max time [s] = ', t[-1]) + + self['u']=np.ndarray((3,nt,ny,nz)) #, buffer=shm.buf) + # read in AMRWind velocity data + self['u'][0,:,:,:] = ds['velocityx'].isel(num_time_steps=slice(0,nt)).values.reshape(nt,noffsets,ny,nz)[:,1,:,:] # last index = 1 refers to 2nd offset plane at -1200 m + self['u'][1,:,:,:] = ds['velocityy'].isel(num_time_steps=slice(0,nt)).values.reshape(nt,noffsets,ny,nz)[:,1,:,:] + self['u'][2,:,:,:] = ds['velocityz'].isel(num_time_steps=slice(0,nt)).values.reshape(nt,noffsets,ny,nz)[:,1,:,:] + self['t'] = t + self['y'] = y + self['z'] = z + self['dt'] = dt + # TODO + self['ID'] = 7 # ... + self['info'] = 'Converted from AMRWind fields {:s}.'.format(time.strftime('%d-%b-%Y at %H:%M:%S', time.localtime())) +# self['zTwr'] = np.array([]) +# self['uTwr'] = np.array([]) + self['zRef'] = zref #None + self['uRef'] = uref #None + self['zRef'], self['uRef'], bHub = self.hubValues() + def fromMannBox(self, u, v, w, dx, U, y, z, addU=None): """ Convert current TurbSim file into one generated from MannBox @@ -782,6 +847,91 @@ def fitPowerLaw(ts, z_ref=None, y_span='full', U_guess=10, alpha_guess=0.1): u_fit, pfit, model = fit_powerlaw_u_alpha(z, u, z_ref=z_ref, p0=(U_guess, alpha_guess)) return u_fit, pfit, model, z_ref +# Functions from BTS_File.py to be ported here +# def TI(self,y=None,z=None,j=None,k=None): +# """ +# If no argument is given, compute TI over entire grid and return array of size (ny,nz). Else, compute TI at the specified point. +# +# Parameters +# ---------- +# y : float, +# cross-stream position [m] +# z : float, +# vertical position AGL [m] +# j : int, +# grid index along cross-stream +# k : int, +# grid index along vertical +# """ +# if ((y==None) & (j==None)): +# return np.std(self.U,axis=0) / np.mean(self.U,axis=0) +# if ((y==None) & (j!=None)): +# return (np.std(self.U[:,j,k])/np.mean(self.U[:,j,k])) +# if ((y!=None) & (j==None)): +# uSeries = self.U[:,self.y2j(y),self.z2k(z)] +# return np.std(uSeries)/np.mean(uSeries) +# +# def visualize(self,component='U',time=0): +# """ +# Quick peak at the data for a given component, at a specific time. +# """ +# data = getattr(self,component)[time,:,:] +# plt.figure() ; +# plt.imshow(data) ; +# plt.colorbar() +# plt.show() +# +# def spectrum(self,component='u',y=None,z=None): +# """ +# Calculate spectrum of a specific component, given time series at ~ hub. +# +# Parameters +# ---------- +# component : string, +# which component to use +# y : float, +# y coordinate [m] of specific location +# z : float, +# z coordinate [m] of specific location +# +# """ +# if y==None: +# k = self.kHub +# j = self.jHub +# data = getattr(self,component) +# data = data[:,j,k] +# N = data.size +# freqs = fftpack.fftfreq(N,self.dT)[1:N/2] +# psd = (np.abs(fftpack.fft(data,N)[1:N/2]))**2 +# return [freqs, psd] +# +# def getRotorPoints(self): +# """ +# In the square y-z slice, return which points are at the edge of the rotor in the horizontal and vertical directions. +# +# Returns +# ------- +# jLeft : int, +# index for grid point that matches the left side of the rotor (when looking towards upstream) +# jRight : int, +# index for grid point that matches the right side of the rotor (when looking towards upstream) +# kBot : int, +# index for grid point that matches the bottom of the rotor +# kTop : int, +# index for grid point that matches the top of the rotor +# """ +# self.zBotRotor = self.zHub - self.R +# self.zTopRotor = self.zHub + self.R +# self.yLeftRotor = self.yHub - self.R +# self.yRightRotor = self.yHub + self.R +# self.jLeftRotor = self.y2j(self.yLeftRotor) +# self.jRightRotor = self.y2j(self.yRightRotor) +# self.kBotRotor = self.z2k(self.zBotRotor) +# self.kTopRotor = self.z2k(self.zTopRotor) +# + + + def fit_powerlaw_u_alpha(x, y, z_ref=100, p0=(10,0.1)): """ p[0] : u_ref diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index 2b70be2..8886c1f 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -31,6 +31,48 @@ __all__ = ['rainflow_astm', 'rainflow_windap','eq_load','eq_load_and_cycles','cycle_matrix','cycle_matrix2'] +def equivalent_load(signal, m=3, Teq=1, nBins=46, method='rainflow_windap'): + """Equivalent load calculation + + Calculate the equivalent loads for a list of Wohler exponent + + Parameters + ---------- + signals : array-like, the signal + m : Wohler exponent (default is 3) + Teq : The equivalent number of load cycles (default is 1, but normally the time duration in seconds is used) + nBins : Number of bins in rainflow count histogram + method: 'rainflow_windap, rainflow_astm, fatpack + + Returns + ------- + Leq : the equivalent load for given m and Tea + """ + signal = np.asarray(signal) + + rainflow_func_dict = {'rainflow_windap':rainflow_windap, 'rainflow_astm':rainflow_astm} + if method in rainflow_func_dict.keys(): + # Call wetb function for one m + Leq = eq_load(signal, m=[m], neq=Teq, no_bins=nBins, rainflow_func=rainflow_func_dict[method])[0][0] + + elif method=='fatpack': + import fatpack + # find rainflow ranges + ranges = fatpack.find_rainflow_ranges(signal) + # find range count and bin + Nrf, Srf = fatpack.find_range_count(ranges, nBins) + # get DEL + DELs = Srf**m * Nrf / Teq + Leq = DELs.sum() ** (1/m) + + else: + raise NotImplementedError(method) + + return Leq + + + + def check_signal(signal): # check input data validity if not type(signal).__name__ == 'ndarray': From 09971f6dcce337628457ad22fb5879366e33d58c Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 7 Nov 2022 16:49:39 -0700 Subject: [PATCH 052/178] Attempt to work around locale issue (#128) --- pydatview/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pydatview/main.py b/pydatview/main.py index 1f18be6..5f652f8 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -767,6 +767,13 @@ def __init__(self, redirect=False, filename=None): else: msg = 'Unable to create GUI' # TODO: more description is needed for wxMSW... raise SystemExit(msg) + def InitLocale(self): + if sys.platform.startswith('win') and sys.version_info > (3,8): + # See Bug #128 - Issue with wxPython 4.1 on Windows + import locale + locale.setlocale(locale.LC_ALL, "C") + print('[INFO] Setting locale to C') + #self.SetAssertMode(wx.APP_ASSERT_SUPPRESS) # Try this # --------------------------------------------------------------------------------} # --- Mains From 0d3cde6800038aef52ffd338903fdd407f3eda8f Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 8 Dec 2022 16:20:22 -0700 Subject: [PATCH 053/178] Adding linspace option for resampling (#130) --- pydatview/fast/postpro.py | 8 ++++---- pydatview/tools/signal_analysis.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 0279c03..3f6e1be 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -1419,16 +1419,16 @@ def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKee result = pd.DataFrame(np.nan, index=np.arange(len(outFiles)), columns=columns) result.iloc[i,:] = MeanValues.copy().values - if ColSort is not None: - # Sorting - result.sort_values([ColSort],inplace=True,ascending=True) - result.reset_index(drop=True,inplace=True) if len(invalidFiles)==len(outFiles): raise Exception('None of the files can be read (or exist)!. For instance, cannot find: {}'.format(invalidFiles[0])) elif len(invalidFiles)>0: print('[WARN] There were {} missing/invalid files: \n {}'.format(len(invalidFiles),'\n'.join(invalidFiles))) + if ColSort is not None: + # Sorting + result.sort_values([ColSort],inplace=True,ascending=True) + result.reset_index(drop=True,inplace=True) return result diff --git a/pydatview/tools/signal_analysis.py b/pydatview/tools/signal_analysis.py index 5ea9b53..e4ee470 100644 --- a/pydatview/tools/signal_analysis.py +++ b/pydatview/tools/signal_analysis.py @@ -20,6 +20,7 @@ {'name':'Insert', 'param':[], 'paramName':'Insert list'}, {'name':'Remove', 'param':[], 'paramName':'Remove list'}, {'name':'Every n', 'param':2 , 'paramName':'n'}, + {'name':'Linspace', 'param':[0,1,100] , 'paramName':'xmin, xmax, n'}, {'name':'Time-based', 'param':0.01 , 'paramName':'Sample time (s)'}, {'name':'Delta x', 'param':[0.1,np.nan,np.nan], 'paramName':'dx, xmin, xmax'}, ] @@ -230,6 +231,15 @@ def applySampler(x_old, y_old, sampDict, df_old=None): param = [dx, xmin, xmax] return x_new, resample_interp(x_old, x_new, y_old, df_old) + elif sampDict['name']=='Linspace': + if len(param)!=3: + raise Exception('Error: Provide three parameters for linspace: xmin, xmax, n') + xmin = float(param[0]) + xmax = float(param[1]) + n = int(param[2]) + x_new = np.linspace(xmin, xmax, n) + return x_new, resample_interp(x_old, x_new, y_old, df_old) + elif sampDict['name']=='Every n': if len(param)==0: raise Exception('Error: provide value for n') From 813499dfbbec87b144a0f00af0246c6b08735704 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 8 Dec 2022 16:22:50 -0700 Subject: [PATCH 054/178] GH: depreciating 3.6 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b921e10..ef2c60c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.8, 3.9] + python-version: [3.8, 3.9, 3.11] steps: # --- Install steps From 87e500500b059e9aa673b1bd62fde34636da5dc1 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 8 Dec 2022 16:24:17 -0700 Subject: [PATCH 055/178] IO: update from weio --- pydatview/io/fast_input_deck.py | 175 ++++++++++++++++++------------ pydatview/io/fast_input_file.py | 33 ++++-- pydatview/io/fast_output_file.py | 8 +- pydatview/io/mini_yaml.py | 179 +++++++++++++++++++++---------- 4 files changed, 261 insertions(+), 134 deletions(-) diff --git a/pydatview/io/fast_input_deck.py b/pydatview/io/fast_input_deck.py index fe39428..56119ad 100644 --- a/pydatview/io/fast_input_deck.py +++ b/pydatview/io/fast_input_deck.py @@ -212,15 +212,17 @@ def read(self, filename=None): if filename is not None: self.filename = filename - # Read OpenFAST files + # Read main file (.fst, or .drv) and store into key "Fst" self.fst_vt['Fst'] = self._read(self.FAST_InputFile, 'Fst') if self.fst_vt['Fst'] is None: raise Exception('Error reading main file {}'.format(self.filename)) keys = self.fst_vt['Fst'].keys() - + # Detect driver or OpenFAST version if 'NumTurbines' in keys: self.version='AD_driver' + elif 'DynamicSolve' in keys: + self.version='BD_driver' elif 'InterpOrder' in self.fst_vt['Fst'].keys(): self.version='OF2' else: @@ -235,6 +237,18 @@ def read(self, filename=None): self.readAD(key='AeroDyn15') + elif self.version=='BD_driver': + # --- BD driver + self.fst_vt['BeamDyn'] = self._read(self.fst_vt['Fst']['InputFile'],'BD') + if self.fst_vt['BeamDyn'] is not None: + # Blades + bld_file = os.path.join(os.path.dirname(self.fst_vt['Fst']['InputFile']), self.fst_vt['BeamDyn']['BldFile']) + print('bld_file', bld_file) + self.fst_vt['BeamDynBlade']= self._read(bld_file,'BDbld') + + del self.fst_vt['af_data'] + del self.fst_vt['ac_data'] + elif self.version=='OF2': # ---- Regular OpenFAST file # ElastoDyn @@ -336,85 +350,110 @@ def write(self, filename=None, prefix='', suffix='', directory=None): self.filename=filename if directory is None: directory = os.path.dirname(filename) + else: + # Making sure filename is within directory + filename = os.path.join(directory, os.path.basename(filename)) + if not os.path.exists(directory): + os.makedirs(directory) + basename = os.path.splitext(os.path.basename(filename))[0] fst = self.fst_vt['Fst'] - # Filenames - filename_ED = os.path.join(directory,prefix+'ED'+suffix+'.dat') if fst['CompElast']>0 else 'none' - filename_IW = os.path.join(directory,prefix+'IW'+suffix+'.dat') if fst['CompInflow']>0 else 'none' - filename_BD = os.path.join(directory,prefix+'BD'+suffix+'.dat') if fst['CompElast']==2 else 'none' - filename_AD = os.path.join(directory,prefix+'AD'+suffix+'.dat') if fst['CompAero']>0 else 'none' - filename_HD = os.path.join(directory,prefix+'HD'+suffix+'.dat') if fst['CompHydro']>0 else 'none' - filename_SD = os.path.join(directory,prefix+'SD'+suffix+'.dat') if fst['CompSub']>0 else 'none' - filename_MD = os.path.join(directory,prefix+'MD'+suffix+'.dat') if fst['CompMooring']>0 else 'none' - filename_SvD = os.path.join(directory,prefix+'SvD'+suffix+'.dat') if fst['CompServo']>0 else 'none' - filename_Ice = os.path.join(directory,prefix+'Ice'+suffix+'.dat') if fst['CompIce']>0 else 'none' - filename_ED_bld = os.path.join(directory,prefix+'ED_bld'+suffix+'.dat') if fst['CompElast']>0 else 'none' - filename_ED_twr = os.path.join(directory,prefix+'ED_twr'+suffix+'.dat') if fst['CompElast']>0 else 'none' - filename_BD_bld = os.path.join(directory,prefix+'BD_bld'+suffix+'.dat') if fst['CompElast']>0 else 'none' - # TODO AD Profiles and OLAF - - fst['EDFile'] = '"' + os.path.basename(filename_ED) + '"' - fst['BDBldFile(1)'] = '"' + os.path.basename(filename_BD) + '"' - fst['BDBldFile(2)'] = '"' + os.path.basename(filename_BD) + '"' - fst['BDBldFile(3)'] = '"' + os.path.basename(filename_BD) + '"' - fst['InflowFile'] = '"' + os.path.basename(filename_IW) + '"' - fst['AeroFile'] = '"' + os.path.basename(filename_AD) + '"' - fst['ServoFile'] = '"' + os.path.basename(filename_AD) + '"' - fst['HydroFile'] = '"' + os.path.basename(filename_HD) + '"' - fst['SubFile'] = '"' + os.path.basename(filename_SD) + '"' - fst['MooringFile'] = '"' + os.path.basename(filename_MD) + '"' - fst['IceFile'] = '"' + os.path.basename(filename_Ice)+ '"' - fst.write(filename) - - - ED = self.fst_vt['ElastoDyn'] - if fst['CompElast']>0: - ED['TwrFile'] = '"' + os.path.basename(filename_ED_twr)+ '"' - self.fst_vt['ElastoDynTower'].write(filename_ED_twr) - if fst['CompElast']==1: - if 'BldFile1' in ED.keys(): - ED['BldFile1'] = '"' + os.path.basename(filename_ED_bld)+ '"' - ED['BldFile2'] = '"' + os.path.basename(filename_ED_bld)+ '"' - ED['BldFile3'] = '"' + os.path.basename(filename_ED_bld)+ '"' - else: - ED['BldFile(1)'] = '"' + os.path.basename(filename_ED_bld)+ '"' - ED['BldFile(2)'] = '"' + os.path.basename(filename_ED_bld)+ '"' - ED['BldFile(3)'] = '"' + os.path.basename(filename_ED_bld)+ '"' - self.fst_vt['ElastoDynBlade'].write(filename_ED_bld) - elif fst['CompElast']==2: + if self.version=='AD_driver': + raise NotImplementedError() + + elif self.version=='BD_driver': + # --- BD driver + filename_BD = os.path.join(directory, prefix+'BD'+suffix+'.dat') + filename_BD_bld = os.path.join(directory, prefix+'BD_bld'+suffix+'.dat') + fst['InputFile'] = '"' + os.path.basename(filename_BD) + '"' + fst.write(filename) BD = self.fst_vt['BeamDyn'] BD['BldFile'] = '"'+os.path.basename(filename_BD_bld)+'"' self.fst_vt['BeamDynBlade'].write(filename_BD_bld) # TODO TODO pick up the proper blade file! BD.write(filename_BD) - ED.write(filename_ED) + elif self.version=='OF2': - if fst['CompInflow']>0: - self.fst_vt['InflowWind'].write(filename_IW) - - if fst['CompAero']>0: - self.fst_vt['AeroDyn15'].write(filename_AD) - # TODO other files - - if fst['CompServo']>0: - self.fst_vt['ServoDyn'].write(filename_SvD) - - if fst['CompHydro']==1: - self.fst_vt['HydroDyn'].write(filename_HD) - - if fst['CompSub']==1: - self.fst_vt['SubDyn'].write(filename_SD) - elif fst['CompSub']==2: - raise NotImplementedError() + # Filenames + filename_ED = os.path.join(directory,prefix+'ED'+suffix+'.dat') if fst['CompElast']>0 else 'none' + filename_IW = os.path.join(directory,prefix+'IW'+suffix+'.dat') if fst['CompInflow']>0 else 'none' + filename_BD = os.path.join(directory,prefix+'BD'+suffix+'.dat') if fst['CompElast']==2 else 'none' + filename_AD = os.path.join(directory,prefix+'AD'+suffix+'.dat') if fst['CompAero']>0 else 'none' + filename_HD = os.path.join(directory,prefix+'HD'+suffix+'.dat') if fst['CompHydro']>0 else 'none' + filename_SD = os.path.join(directory,prefix+'SD'+suffix+'.dat') if fst['CompSub']>0 else 'none' + filename_MD = os.path.join(directory,prefix+'MD'+suffix+'.dat') if fst['CompMooring']>0 else 'none' + filename_SvD = os.path.join(directory,prefix+'SvD'+suffix+'.dat') if fst['CompServo']>0 else 'none' + filename_Ice = os.path.join(directory,prefix+'Ice'+suffix+'.dat') if fst['CompIce']>0 else 'none' + filename_ED_bld = os.path.join(directory,prefix+'ED_bld'+suffix+'.dat') if fst['CompElast']>0 else 'none' + filename_ED_twr = os.path.join(directory,prefix+'ED_twr'+suffix+'.dat') if fst['CompElast']>0 else 'none' + filename_BD_bld = os.path.join(directory,prefix+'BD_bld'+suffix+'.dat') if fst['CompElast']>0 else 'none' + # TODO AD Profiles and OLAF + + fst['EDFile'] = '"' + os.path.basename(filename_ED) + '"' + fst['BDBldFile(1)'] = '"' + os.path.basename(filename_BD) + '"' + fst['BDBldFile(2)'] = '"' + os.path.basename(filename_BD) + '"' + fst['BDBldFile(3)'] = '"' + os.path.basename(filename_BD) + '"' + fst['InflowFile'] = '"' + os.path.basename(filename_IW) + '"' + fst['AeroFile'] = '"' + os.path.basename(filename_AD) + '"' + fst['ServoFile'] = '"' + os.path.basename(filename_AD) + '"' + fst['HydroFile'] = '"' + os.path.basename(filename_HD) + '"' + fst['SubFile'] = '"' + os.path.basename(filename_SD) + '"' + fst['MooringFile'] = '"' + os.path.basename(filename_MD) + '"' + fst['IceFile'] = '"' + os.path.basename(filename_Ice)+ '"' + fst.write(filename) + + + ED = self.fst_vt['ElastoDyn'] + if fst['CompElast']>0: + ED['TwrFile'] = '"' + os.path.basename(filename_ED_twr)+ '"' + self.fst_vt['ElastoDynTower'].write(filename_ED_twr) + if fst['CompElast']==1: + if 'BldFile1' in ED.keys(): + ED['BldFile1'] = '"' + os.path.basename(filename_ED_bld)+ '"' + ED['BldFile2'] = '"' + os.path.basename(filename_ED_bld)+ '"' + ED['BldFile3'] = '"' + os.path.basename(filename_ED_bld)+ '"' + else: + ED['BldFile(1)'] = '"' + os.path.basename(filename_ED_bld)+ '"' + ED['BldFile(2)'] = '"' + os.path.basename(filename_ED_bld)+ '"' + ED['BldFile(3)'] = '"' + os.path.basename(filename_ED_bld)+ '"' + self.fst_vt['ElastoDynBlade'].write(filename_ED_bld) + + elif fst['CompElast']==2: + BD = self.fst_vt['BeamDyn'] + BD['BldFile'] = '"'+os.path.basename(filename_BD_bld)+'"' + self.fst_vt['BeamDynBlade'].write(filename_BD_bld) # TODO TODO pick up the proper blade file! + BD.write(filename_BD) + ED.write(filename_ED) + + + if fst['CompInflow']>0: + self.fst_vt['InflowWind'].write(filename_IW) + + if fst['CompAero']>0: + self.fst_vt['AeroDyn15'].write(filename_AD) + # TODO other files + + if fst['CompServo']>0: + self.fst_vt['ServoDyn'].write(filename_SvD) + + if fst['CompHydro']==1: + self.fst_vt['HydroDyn'].write(filename_HD) + + if fst['CompSub']==1: + self.fst_vt['SubDyn'].write(filename_SD) + elif fst['CompSub']==2: + raise NotImplementedError() + + if fst['CompMooring']==1: + self.fst_vt['MAP'].write(filename_MD) + if self.fst_vt['Fst']['CompMooring']==2: + self.fst_vt['MoorDyn'].write(filename_MD) - if fst['CompMooring']==1: - self.fst_vt['MAP'].write(filename_MD) - if self.fst_vt['Fst']['CompMooring']==2: - self.fst_vt['MoorDyn'].write(filename_MD) + return filename diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index 3abbda9..44c7955 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -174,6 +174,13 @@ class FASTInputFileBase(File): f.write('AeroDyn_Changed.dat') """ + @staticmethod + def defaultExtensions(): + return ['.dat','.fst','.txt','.fstf','.dvr'] + + @staticmethod + def formatName(): + return 'FAST input file Base' def __init__(self, filename=None, **kwargs): self._size=None @@ -243,9 +250,12 @@ def __setitem__(self, key, item): if self.data[i]['tabType'] != TABTYPE_NOT_A_TAB: # For tables, we automatically update variable that stores the dimension nRows = len(item) - dimVar = self.data[i]['tabDimVar'] - iDimVar = self.getID(dimVar) - self.data[iDimVar]['value'] = nRows # Avoiding a recursive call to __setitem__ here + if 'tabDimVar' in self.data[i].keys(): + dimVar = self.data[i]['tabDimVar'] + iDimVar = self.getID(dimVar) + self.data[iDimVar]['value'] = nRows # Avoiding a recursive call to __setitem__ here + else: + pass self.data[i]['value'] = item def __getitem__(self,key): @@ -765,8 +775,10 @@ def mat_tostring(M,fmt='24.16e'): elif d['tabType']==TABTYPE_NUM_SUBDYNOUT: data = d['value'] s+='{}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) - s+='{}\n'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) - s+='\n'.join('\t'.join('{:15.0f}'.format(x) for x in y) for y in data) + s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabUnits']])) + if np.size(d['value'],0) > 0 : + s+='\n' + s+='\n'.join('\t'.join('{:15.0f}'.format(x) for x in y) for y in data) else: raise Exception('Unknown table type for variable {}'.format(d)) if iMAX : - raise Exception('More that 200 lines found in outlist') if i>=len(lines): print('[WARN] End of file reached while reading Outlist') #i=min(i+1,len(lines)) @@ -1443,7 +1452,7 @@ def _writeSanityChecks(self): # TODO double check this calculation with gradient dr = np.gradient(aeroNodes[:,0]) dx = np.gradient(aeroNodes[:,1]) - crvAng = np.degrees(np.arctan2(dx,dr))*np.pi/180 + crvAng = np.degrees(np.arctan2(dx,dr)) if np.mean(np.abs(crvAng-aeroNodes[:,3]))>0.1: print('[WARN] BlCrvAng might not be computed correctly') @@ -1512,6 +1521,10 @@ def _IComment(self): return [1] # --- AeroDyn Polar # --------------------------------------------------------------------------------{ class ADPolarFile(FASTInputFileBase): + @staticmethod + def formatName(): + return 'FAST AeroDyn polar file' + @classmethod def from_fast_input_file(cls, parent): self = cls() diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index 6db3bff..820a9ae 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -17,11 +17,12 @@ import os import re try: - from .file import File, WrongFormatError, BrokenReaderError, EmptyFileError + from .file import File, WrongFormatError, BrokenReaderError, EmptyFileError, BrokenFormatError except: File = dict class WrongFormatError(Exception): pass class WrongReaderError(Exception): pass + class BrokenFormatError(Exception): pass class EmptyFileError(Exception): pass try: from .csv_file import CSVFile @@ -150,6 +151,8 @@ def toDataFrame(self): df= self.data df.columns=cols else: + if len(cols)!=self.data.shape[1]: + raise BrokenFormatError('Inconstistent number of columns between headers ({}) and data ({}) for file {}'.format(len(cols), self.data.shape[1], self.filename)) df = pd.DataFrame(data=self.data,columns=cols) return df @@ -206,7 +209,8 @@ def load_ascii_output(filename): l = f.readline() if not l: raise Exception('Error finding the end of FAST out file header. Keyword Time missing.') - in_header= (l+' dummy').lower().split()[0] != 'time' + first_word = (l+' dummy').lower().split()[0] + in_header= (first_word != 'time') and (first_word != 'alpha') if in_header: header.append(l) else: diff --git a/pydatview/io/mini_yaml.py b/pydatview/io/mini_yaml.py index 2768f02..0650c0e 100644 --- a/pydatview/io/mini_yaml.py +++ b/pydatview/io/mini_yaml.py @@ -1,6 +1,6 @@ import numpy as np -def yaml_read(filename,dictIn=None): +def yaml_read(filename=None, dictIn=None, lines=None, text=None): """ read yaml files only supports: - Key value pairs: @@ -12,84 +12,155 @@ def yaml_read(filename,dictIn=None): - Comments are stripped based on first # found (in string or not) - Keys are found based on first : found (in string or not) """ - # Read all lines at once - with open(filename, 'r', errors="surrogateescape") as f: - lines=f.read().splitlines() - + # --- swtich depending on what the user provided + if filename is not None: + # Read all lines at once + with open(filename, 'r', errors="surrogateescape") as f: + lines=f.read().splitlines() + elif text is not None: + lines = text.split('\n') + elif lines is not None: + # OK + pass if dictIn is None: d=dict() else: d=dictIn - def cleanComment(l): - """ remove comments from a line""" - return l.split('#')[0].strip() - - def readDashList(iStart): - """ """ - i=iStart - while i0]) - try: - FirstElems=FirstElems.astype(int) - mytype=int - except: - try: - FirstElems=FirstElems.astype(float) - mytype=float - except: - raise Exception('Cannot convert line to float or int: {}'.format(lines[iStart])) - M = np.zeros((n,len(FirstElems)), mytype) - if len(FirstElems)>0: - for i in np.arange(iStart,iEnd+1): - elem = cleanComment(lines[i])[1:].replace(']','').replace('[','').split(',') - M[i-iStart,:] = np.array([v.strip() for v in elem if len(v)>0]).astype(mytype) - return M, iEnd+1 - + # --- Loop on lines i=0 while i0: + for i in np.arange(iStart,iEnd+1): + L,_ = _readInlineList(lines[i].lstrip()[1:], mytype=mytype) + M[i-iStart,:] = L + return M, iEnd+1 + +def _readInlineList(line, mytype=None): + """ + Parse a simple list of int, float or string + [a, b, c] + [a, b, c,] + """ + L = _cleanComment(line.replace(']','').replace('[','')).strip().rstrip(',').strip() + if len(L)==0: + return np.array([]), float + L=L.split(',') + L = np.asarray(L) + if mytype is None: + ## try to detect type + try: + L=L.astype(int) + mytype=int + except: + try: + L=L.astype(float) + mytype=float + except: + try: + L=L.astype(str) + mytype=str + L=np.array([c.strip() for c in L]) + except: + raise Exception('Cannot parse list from string: >{}<'.format(line)) + else: + try: + L = L.astype(mytype) + if mytype==str: + L=np.array([c.strip() for c in L]) + except: + raise Exception('Cannot parse list of type {} from string: >{}<'.format(mytype, line)) + + # + return L, mytype if __name__=='__main__': - d=read('test.yaml') - #d=yaml_read('TetraSpar_outputs_DOUBLE_PRECISION.SD.sum.yaml') - print(d.keys()) +# #d=yaml_read('test.SD.sum.yaml') + text = """ +# Comment +IS1: 40567 # int scalar +FS1: 40567.32 # float scalar +FA1: # Array1 + - [ 3.97887E+07, 0.00000E+00, 0.00000E+00] + - [ 0.00000E+00, 3.97887E+07, 0.00000E+00] + - [ 0.00000E+00, 0.00000E+00, 0.00000E+00] +FA2: # Array2 + - [ 1.E+00, 0.E+00, 0.E+00,] + - [ 0.E+00, 1.E+00, 0.E+00,] + - [ 0.E+00, 0.E+00, 1.E+00,] +FL1: [ 0.0, 0.0, 1.0 ] # FloatList1 +FL2: [ 0.0, 0.0, 1.0,] # FloatList2 +SL2: [ aa, bb , cc, dd, ] #string list +""" + text=""" +EL1: [ ] #empty list +EL2: #empty list2 + - [ ] # empty list +""" + d=yaml_read(text=text) print(d) - print(d['nNodes_I']) + for k,v, in d.items(): + if hasattr(v,'__len__'): + if len(v)>0: + print('{:12s} {:20s} {}'.format(k, str(type(v[0]))[6:], v[0])) + else: + print('{:12s} {:20s} {}'.format(k, str(type(v))[6:], v)) + else: + print('{:12s} {:20s} {}'.format(k, str(type(v))[6:], v)) + + From e32900bc49e000d51a0a667803891430f3c7af8e Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 20 Dec 2022 14:22:50 +0100 Subject: [PATCH 056/178] Adding loader options for day first in datetime (see #132) --- pydatview/GUIInfoPanel.py | 18 ++++++++++ pydatview/GUIPlotPanel.py | 38 +++++++++++++++++---- pydatview/Tables.py | 44 +++++++++++++++--------- pydatview/appdata.py | 63 +++++++++-------------------------- pydatview/main.py | 31 +++++++++++++++-- pydatview/plugins/__init__.py | 1 + 6 files changed, 124 insertions(+), 71 deletions(-) diff --git a/pydatview/GUIInfoPanel.py b/pydatview/GUIInfoPanel.py index e198a78..7017715 100644 --- a/pydatview/GUIInfoPanel.py +++ b/pydatview/GUIInfoPanel.py @@ -272,6 +272,24 @@ def __init__(self, parent, data=None): self.SetSizer(sizer) self.SetMaxSize((-1, 50)) + # --- GUI Data + def saveData(self, data): + data['ColumnsRegular'] = [c['name'] for c in self.ColsReg if c['s']] + data['ColumnsFFT'] = [c['name'] for c in self.ColsFFT if c['s']] + data['ColumnsMinMax'] = [c['name'] for c in self.ColsMinMax if c['s']] + data['ColumnsPDF'] = [c['name'] for c in self.ColsPDF if c['s']] + data['ColumnsCmp'] = [c['name'] for c in self.ColsCmp if c['s']] + + @staticmethod + def defaultData(): + data={} + data['ColumnsRegular'] = ['Column','Mean','Std','Min','Max','Range','dx','n'] + data['ColumnsFFT'] = ['Column','Mean','Std','Min','Max','Min(FFT)','Max(FFT)',u'\u222By(FFT)','dx(FFT)','xMax(FFT)','n(FFT)','n'] + data['ColumnsMinMax'] = ['Column','Mean','Std','Min','Max','Mean(MinMax)','Std(MinMax)',u'\u222By(MinMax)','n'] + data['ColumnsPDF'] = ['Column','Mean','Std','Min','Max','Min(PDF)','Max(PDF)',u'\u222By(PDF)','n(PDF)'] + data['ColumnsCmp'] = ['Column','Mean(Cmp)','Std(Cmp)','Min(Cmp)','Max(Cmp)','n(Cmp)'] + return data + def CopyToClipBoard(self, event): nCols=self.tbStats.GetColumnCount() headerLine='\t'.join([self.tbStats.GetColumn(j).GetText() for j in np.arange(nCols)] ) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index c73e6f5..074f139 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -376,7 +376,7 @@ def onFontOptionChange(self,event=None): class PlotPanel(wx.Panel): - def __init__(self, parent, selPanel,infoPanel=None, mainframe=None): + def __init__(self, parent, selPanel, infoPanel=None, mainframe=None, data=None): # Superclass constructor super(PlotPanel,self).__init__(parent) @@ -409,12 +409,11 @@ def __init__(self, parent, selPanel,infoPanel=None, mainframe=None): self.mainframe= mainframe self.plotData = [] self.plotDataOptions=dict() - try: - self.data = mainframe.data['plotPanel'] - except: + if data is not None: + self.data = data + else: print('>>> Using default settings for plot panel') - from .appdata import defaultPlotPanelData - self.data = defaultPlotPanelData() + self.data = self.defaultData() if self.selPanel is not None: bg=self.selPanel.BackgroundColour self.SetBackgroundColour(bg) # sowhow, our parent has a wrong color @@ -571,6 +570,31 @@ def __init__(self, parent, selPanel,infoPanel=None, mainframe=None): self.plotsizer=plotsizer; self.set_subplot_spacing(init=True) + + # --- GUI DATA + def saveData(self, data): + data['Grid'] = self.cbGrid.IsChecked() + data['CrossHair'] = self.cbXHair.IsChecked() + data['plotStyle']['Font'] = self.esthPanel.cbFont.GetValue() + data['plotStyle']['LegendFont'] = self.esthPanel.cbLgdFont.GetValue() + data['plotStyle']['LegendPosition'] = self.esthPanel.cbLegend.GetValue() + data['plotStyle']['LineWidth'] = self.esthPanel.cbLW.GetValue() + data['plotStyle']['MarkerSize'] = self.esthPanel.cbMS.GetValue() + + @staticmethod + def defaultData(): + data={} + data['CrossHair']=True + data['Grid']=False + plotStyle = dict() + plotStyle['Font'] = '11' + plotStyle['LegendFont'] = '11' + plotStyle['LegendPosition'] = 'Upper right' + plotStyle['LineWidth'] = '1.5' + plotStyle['MarkerSize'] = '2' + data['plotStyle']= plotStyle + return data + def onEsthToggle(self,event): self.esthToggle=not self.esthToggle if self.esthToggle: @@ -1449,7 +1473,7 @@ def getPlotDataSelection(self): selpanel=FakeSelPanel(self) # selpanel.SetBackgroundColour('blue') - p1=PlotPanel(self,selpanel) + p1=PlotPanel(self, selpanel, data=None) p1.load_and_draw() #p1=SpectralCtrlPanel(self) sizer = wx.BoxSizer(wx.VERTICAL) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index d856f8a..2e87301 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -14,11 +14,25 @@ # --- TabList # --------------------------------------------------------------------------------{ class TableList(object): # todo inherit list - def __init__(self, tabs=None): + + def __init__(self, tabs=None, options=None): if tabs is None: tabs =[] self._tabs = tabs - self.Naming = 'Ellude' + + self.options = self.defaultOptions() if options is None else options + + # --- Options + def saveOptions(self, optionts): + options['naming'] = self.options['naming'] + options['dayfirst'] = self.options['dayfirst'] + + @staticmethod + def defaultOptions(): + options={} + options['naming'] = 'Ellude' + options['dayfirst'] = False + return options # --- behaves like a list... def __iter__(self): @@ -51,7 +65,7 @@ def from_dataframes(self, dataframes=[], names=[], bAdd=False): # Returning a list of tables for df,name in zip(dataframes, names): if df is not None: - self.append(Table(data=df, name=name)) + self.append(Table(data=df, name=name, dayfirst=self.options['dayfirst'])) def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bReload=False, statusFunction=None): """ load multiple files into table list""" @@ -139,11 +153,11 @@ def _load_file_tabs(self, filename, fileformat=None, bReload=False): pass elif not isinstance(dfs,dict): if len(dfs)>0: - tabs=[Table(data=dfs, filename=filename, fileformat=fileformat)] + tabs=[Table(data=dfs, filename=filename, fileformat=fileformat, dayfirst=self.options['dayfirst'])] else: for k in list(dfs.keys()): if len(dfs[k])>0: - tabs.append(Table(data=dfs[k], name=str(k), filename=filename, fileformat=fileformat)) + tabs.append(Table(data=dfs[k], name=str(k), filename=filename, fileformat=fileformat, dayfirst=self.options['dayfirst'])) if len(tabs)<=0: warn='Warn: No dataframe found in file: '+filename+'\n' return tabs, warn @@ -233,10 +247,10 @@ def setActiveNames(self,names): t.active_name=tn def setNaming(self,naming): - self.Naming=naming + self.options['naming']=naming def getDisplayTabNames(self): - if self.Naming=='Ellude': + if self.options['naming']=='Ellude': # Temporary hack, using last names if all last names are unique names = [t.raw_name for t in self._tabs] last_names=[n.split('|')[-1] for n in names] @@ -244,10 +258,10 @@ def getDisplayTabNames(self): return ellude_common(last_names) else: return ellude_common(names) - elif self.Naming=='FileNames': + elif self.options['naming']=='FileNames': return [os.path.splitext(os.path.basename(t.filename))[0] for t in self._tabs] else: - raise Exception('Table naming unknown: {}'.format(self.Naming)) + raise Exception('Table naming unknown: {}'.format(self.options['naming'])) # --- Properties @property @@ -399,7 +413,7 @@ class Table(object): # active_name : # raw_name : # filename : - def __init__(self,data=None,name='',filename='',columns=[], fileformat=None): + def __init__(self,data=None,name='',filename='',columns=[], fileformat=None, dayfirst=False): # Default init self.maskString='' self.mask=None @@ -431,7 +445,7 @@ def __init__(self,data=None,name='',filename='',columns=[], fileformat=None): self.setupName(name=str(name)) - self.convertTimeColumns() + self.convertTimeColumns(dayfirst=dayfirst) def setupName(self,name=''): @@ -588,7 +602,7 @@ def changeUnits(self, flavor='WE'): from pydatview.plugins.data_standardizeUnits import changeUnits changeUnits(self, flavor=flavor) - def convertTimeColumns(self): + def convertTimeColumns(self, dayfirst=False): if len(self.data)>0: for i,c in enumerate(self.data.columns.values): y = self.data.iloc[:,i] @@ -596,7 +610,7 @@ def convertTimeColumns(self): if isinstance(y.values[0], str): # tring to convert to date try: - parser.parse(y.values[0]) + vals = parser.parse(y.values[0]) isDate=True except: if y.values[0]=='NaT': @@ -605,8 +619,8 @@ def convertTimeColumns(self): isDate=False if isDate: try: - self.data[c]=pd.to_datetime(self.data[c].values).to_pydatetime() - print('Column {} converted to datetime'.format(c)) + self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst).to_pydatetime() + print('Column {} converted to datetime, dayfirst: {}'.format(c, dayfirst)) except: # Happens if values are e.g. "Monday, Tuesday" print('Conversion to datetime failed, column {} inferred as string'.format(c)) diff --git a/pydatview/appdata.py b/pydatview/appdata.py index db0816a..cdaec7a 100644 --- a/pydatview/appdata.py +++ b/pydatview/appdata.py @@ -1,7 +1,12 @@ import json import os from pydatview.io import defaultUserDataDir + from .GUICommon import Error +from .GUICommon import getFontSize, getMonoFontSize +from .GUIPlotPanel import PlotPanel +from .GUIInfoPanel import InfoPanel +from .Tables import TableList def configFilePath(): @@ -66,15 +71,16 @@ def loadAppData(mainframe): return data def saveAppData(mainFrame, data): - from .GUICommon import getFontSize, getMonoFontSize if not mainFrame.datareset: # --- Import data from GUI data['fontSize'] = int(getFontSize()) data['monoFontSize'] = int(getMonoFontSize()) if hasattr(mainFrame, 'plotPanel'): - savePlotPanelData(data['plotPanel'], mainFrame.plotPanel) - if hasattr(mainFrame, 'plotPanel'): - saveInfoPanelData(data['infoPanel'], mainFrame.infoPanel) + mainFrame.plotPanel.saveData(data['plotPanel']) + if hasattr(mainFrame, 'infoPanel'): + mainFrame.infoPanel.saveData(data['infoPanel']) + if hasattr(mainFrame, 'tablist'): + mainFrame.tablist.saveOptions(data['loaderOptions']) # --- Write config file configFile = configFilePath() @@ -95,53 +101,16 @@ def defaultAppData(mainframe): data['windowSize'] = (900,700) data['monoFontSize'] = mainframe.systemFontSize data['fontSize'] = mainframe.systemFontSize + # Loader/Table + data['loaderOptions'] = TableList.defaultOptions() + # Pipeline #SIDE_COL = [160,160,300,420,530] #SIDE_COL_LARGE = [200,200,360,480,600] #BOT_PANL =85 - data['plotPanel']=defaultPlotPanelData() - data['infoPanel']=defaultInfoPanelData() - return data - -# --- Plot Panel -# TODO put this into plotPanel file? -def savePlotPanelData(data, plotPanel): - data['Grid'] = plotPanel.cbGrid.IsChecked() - data['CrossHair'] = plotPanel.cbXHair.IsChecked() - data['plotStyle']['Font'] = plotPanel.esthPanel.cbFont.GetValue() - data['plotStyle']['LegendFont'] = plotPanel.esthPanel.cbLgdFont.GetValue() - data['plotStyle']['LegendPosition'] = plotPanel.esthPanel.cbLegend.GetValue() - data['plotStyle']['LineWidth'] = plotPanel.esthPanel.cbLW.GetValue() - data['plotStyle']['MarkerSize'] = plotPanel.esthPanel.cbMS.GetValue() - -def defaultPlotPanelData(): - data={} - data['CrossHair']=True - data['Grid']=False - plotStyle = dict() - plotStyle['Font'] = '11' - plotStyle['LegendFont'] = '11' - plotStyle['LegendPosition'] = 'Upper right' - plotStyle['LineWidth'] = '1.5' - plotStyle['MarkerSize'] = '2' - data['plotStyle']= plotStyle + # GUI + data['plotPanel']=PlotPanel.defaultData() + data['infoPanel']=InfoPanel.defaultData() return data -# --- Info Panel -# TODO put this into infoPanel file? -def saveInfoPanelData(data, infoPanel): - data['ColumnsRegular'] = [c['name'] for c in infoPanel.ColsReg if c['s']] - data['ColumnsFFT'] = [c['name'] for c in infoPanel.ColsFFT if c['s']] - data['ColumnsMinMax'] = [c['name'] for c in infoPanel.ColsMinMax if c['s']] - data['ColumnsPDF'] = [c['name'] for c in infoPanel.ColsPDF if c['s']] - data['ColumnsCmp'] = [c['name'] for c in infoPanel.ColsCmp if c['s']] - -def defaultInfoPanelData(): - data={} - data['ColumnsRegular'] = ['Column','Mean','Std','Min','Max','Range','dx','n'] - data['ColumnsFFT'] = ['Column','Mean','Std','Min','Max','Min(FFT)','Max(FFT)',u'\u222By(FFT)','dx(FFT)','xMax(FFT)','n(FFT)','n'] - data['ColumnsMinMax'] = ['Column','Mean','Std','Min','Max','Mean(MinMax)','Std(MinMax)',u'\u222By(MinMax)','n'] - data['ColumnsPDF'] = ['Column','Mean','Std','Min','Max','Min(PDF)','Max(PDF)',u'\u222By(PDF)','n(PDF)'] - data['ColumnsCmp'] = ['Column','Mean(Cmp)','Std(Cmp)','Min(Cmp)','Max(Cmp)','n(Cmp)'] - return data diff --git a/pydatview/main.py b/pydatview/main.py index 5f652f8..6ac21ee 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -77,6 +77,23 @@ def OnDropFiles(self, x, y, filenames): return True +# --------------------------------------------------------------------------------} +# --- Loader Menu +# --------------------------------------------------------------------------------{ +class LoaderMenuPopup(wx.Menu): + def __init__(self, parent, data): + wx.Menu.__init__(self) + self.parent = parent + self.data = data + + # Populate menu + item = wx.MenuItem(self, -1, "Date format: dayfirst", kind=wx.ITEM_CHECK) + self.Append(item) + self.Bind(wx.EVT_MENU, lambda ev: self.setCheck(ev, 'dayfirst') ) + self.Check(item.GetId(), self.data['dayfirst']) # Checking the menu box + + def setCheck(self, event, label): + self.data['dayfirst'] = not self.data['dayfirst'] # --------------------------------------------------------------------------------} @@ -89,10 +106,10 @@ def __init__(self, data=None): # Hooking exceptions to display them to the user sys.excepthook = MyExceptionHook # --- Data - self.tabList=TableList() self.restore_formulas = [] self.systemFontSize = self.GetFont().GetPointSize() self.data = loadAppData(self) + self.tabList=TableList(options=self.data['loaderOptions']) self.datareset = False # Global variables... setFontSize(self.data['fontSize']) @@ -173,6 +190,11 @@ def __init__(self, data=None): tb.AddStretchableSpace() tb.AddControl( wx.StaticText(tb, -1, 'Format: ' ) ) tb.AddControl(self.comboFormats ) + # Menu for loader options + self.btLoaderMenu = wx.Button(tb, wx.ID_ANY, CHAR['menu'], style=wx.BU_EXACTFIT) + tb.AddControl(self.btLoaderMenu) + self.loaderMenu = LoaderMenuPopup(tb, self.data['loaderOptions']) + tb.Bind(wx.EVT_BUTTON, self.onShowLoaderMenu, self.btLoaderMenu) tb.AddSeparator() TBAddTool(tb, "Open" , 'ART_FILE_OPEN', self.onLoad) TBAddTool(tb, "Reload", 'ART_REDO' , self.onReload) @@ -325,7 +347,7 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): self.tSplitter = wx.SplitterWindow(self.vSplitter) #self.tSplitter.SetMinimumPaneSize(20) self.infoPanel = InfoPanel(self.tSplitter, data=self.data['infoPanel']) - self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, self) + self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, self, data=self.data['plotPanel']) self.tSplitter.SetSashGravity(0.9) self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) self.tSplitter.SetMinimumPaneSize(BOT_PANL) @@ -632,6 +654,11 @@ def onFormatChange(self, event=None): # ISel=self.selPanel.tabPanel.lbTab.GetSelections() pass + def onShowLoaderMenu(self, event=None): + #pos = (self.btLoaderMenu.GetPosition()[0], self.btLoaderMenu.GetPosition()[1] + self.btLoaderMenu.GetSize()[1]) + self.PopupMenu(self.loaderMenu) #, pos) + + def mainFrameUpdateLayout(self, event=None): if hasattr(self,'selPanel'): nWind=self.selPanel.splitter.nWindows diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index a4d272c..74c4158 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -24,6 +24,7 @@ def _data_binning(mainframe, event=None, label=''): dataPlugins=[ + # Name/label , callback , is a Panel ('Bin data' , _data_binning , True ), ('Standardize Units (SI)', _data_standardizeUnits, False), ('Standardize Units (WE)', _data_standardizeUnits, False), From 1f1f4cfd8bcced0f68c7943f14b183cdb53c8341 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 21 Dec 2022 16:26:17 +0100 Subject: [PATCH 057/178] IO: misc updates, naming as tablelist property --- pydatview/GUISelectionPanel.py | 8 +++--- pydatview/Tables.py | 40 +++++++++++++++++++++++++-- pydatview/io/fast_input_file.py | 13 +++++---- pydatview/io/fast_input_file_graph.py | 35 +++++++++++++++++------ pydatview/io/hawc2_htc_file.py | 6 +++- pydatview/io/wetb/hawc2/htc_file.py | 23 ++++++++++----- 6 files changed, 97 insertions(+), 28 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index d84a379..6175931 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -269,7 +269,7 @@ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=Fal self.itNameFile = wx.MenuItem(self, -1, "Naming: by file names", kind=wx.ITEM_CHECK) self.Append(self.itNameFile) self.Bind(wx.EVT_MENU, self.OnNaming, self.itNameFile) - self.Check(self.itNameFile.GetId(), self.parent.GetParent().tabList.Naming=='FileNames') # Checking the menu box + self.Check(self.itNameFile.GetId(), self.parent.GetParent().tabList.naming=='FileNames') # Checking the menu box item = wx.MenuItem(self, -1, "Sort by name") self.Append(item) @@ -291,7 +291,7 @@ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=Fal self.Bind(wx.EVT_MENU, self.OnDeleteTabs, item) if len(self.ISel)==1: - if self.tabPanel.tabList.Naming!='FileNames': + if self.tabPanel.tabList.naming!='FileNames': item = wx.MenuItem(self, -1, "Rename") self.Append(item) self.Bind(wx.EVT_MENU, self.OnRenameTab, item) @@ -303,9 +303,9 @@ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=Fal def OnNaming(self, event=None): if self.itNameFile.IsChecked(): - self.tabPanel.tabList.setNaming('FileNames') + self.tabPanel.tabList.naming='FileNames' else: - self.tabPanel.tabList.setNaming('Ellude') + self.tabPanel.tabList.naming='Ellude' self.tabPanel.updateTabNames() diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 2e87301..2650d6e 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -246,9 +246,6 @@ def setActiveNames(self,names): for t,tn in zip(self._tabs,names): t.active_name=tn - def setNaming(self,naming): - self.options['naming']=naming - def getDisplayTabNames(self): if self.options['naming']=='Ellude': # Temporary hack, using last names if all last names are unique @@ -291,6 +288,17 @@ def filenames_and_formats(self): fileformats.append(t.fileformat) return filenames, fileformats + @property + def naming(self): + return self.options['naming'] + + @naming.setter + def naming(self, naming): + if naming not in ['FileNames', 'Ellude']: + raise NotImplementedError('Naming',naming) + self.options['naming']=naming + + def clean(self): del self._tabs self._tabs=[] @@ -387,6 +395,15 @@ def get(self,i): + @staticmethod + def createDummyList(nTab=3): + tabs=[] + for iTab in range(nTab): + tabs.append( Table.createDummy() ) + tablist = TableList(tabs) + return tablist + + # --------------------------------------------------------------------------------} # --- Table # --------------------------------------------------------------------------------{ @@ -791,6 +808,23 @@ def nCols(self): @property def nRows(self): return len(self.data.iloc[:,0]) # TODO if not panda + + @staticmethod + def createDummy(n=5, lab=''): + """ create a dummy table of length n""" + t = np.arange(0,0.5*n,0.5) + x = t+10 + alpha_d = np.linspace(0, 360, n) + P = np.random.normal(0,100,n)+5000 + RPM = np.random.normal(-0.2,0.2,n) + 12. + + d={'Time_[s]':t, + 'x{}_[m]'.format(lab): x, + 'alpha{}_[deg]'.format(lab):alpha_d, + 'P{}_[W]'.format(lab):P, + 'RotSpeed{}_[rpm]'.format(lab):RPM} + df = pd.DataFrame(data=d) + return Table(data=df) if __name__ == '__main__': diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index 44c7955..1d4061e 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -100,7 +100,6 @@ def fixedFormat(self): #print('>>>>>>>>>>>> NO FILEFORMAT', KEYS) return self.basefile - def read(self, filename=None): return self.fixedfile.read(filename) @@ -116,8 +115,8 @@ def toString(self): def keys(self): return self.fixedfile.keys() - def toGraph(self): - return self.fixedfile.toGraph() + def toGraph(self, **kwargs): + return self.fixedfile.toGraph(**kwargs) @property def filename(self): @@ -883,9 +882,9 @@ def _toDataFrame(self): dfs=dfs[list(dfs.keys())[0]] return dfs - def toGraph(self): + def toGraph(self, **kwargs): from .fast_input_file_graph import fastToGraph - return fastToGraph(self) + return fastToGraph(self, **kwargs) @@ -1511,6 +1510,10 @@ def _toDataFrame(self): df['c2_Crv_Approx_[m]'] = prebend df['c2_Swp_Approx_[m]'] = sweep df['AC_Approx_[-]'] = ACloc + # --- Calc CvrAng + dr = np.gradient(aeroNodes[:,0]) + dx = np.gradient(aeroNodes[:,1]) + df['CrvAng_Calc_[-]'] = np.degrees(np.arctan2(dx,dr)) return df @property diff --git a/pydatview/io/fast_input_file_graph.py b/pydatview/io/fast_input_file_graph.py index cfb9767..3cb55a4 100644 --- a/pydatview/io/fast_input_file_graph.py +++ b/pydatview/io/fast_input_file_graph.py @@ -12,24 +12,27 @@ # --------------------------------------------------------------------------------} # --- Wrapper to convert a "fast" input file dictionary into a graph # --------------------------------------------------------------------------------{ -def fastToGraph(data): +def fastToGraph(data, **kwargs): if 'BeamProp' in data.keys(): - return subdynToGraph(data) + return subdynToGraph(data, **kwargs) if 'SmplProp' in data.keys(): - return hydrodynToGraph(data) + return hydrodynToGraph(data, **kwargs) if 'DOF2Nodes' in data.keys(): - return subdynSumToGraph(data) + return subdynSumToGraph(data, **kwargs) raise NotImplementedError('Graph for object with keys: {}'.format(data.keys())) # --------------------------------------------------------------------------------} # --- SubDyn # --------------------------------------------------------------------------------{ -def subdynToGraph(sd): +def subdynToGraph(sd, propToNodes=False, propToElem=False): """ sd: dict-like object as returned by weio + + -propToNodes: if True, the element properties are also transferred to the nodes for convenience. + NOTE: this is not the default because a same node can have two different diameters in SubDyn (it's by element) """ type2Color=[ (0.1,0.1,0.1), # Watchout based on background @@ -81,7 +84,11 @@ def subdynToGraph(sd): elem.data['color'] = type2Color[Type] Graph.addElement(elem) # Nodal prop data - #Graph.setElementNodalProp(elem, propset=PropSets[Type-1], propIDs=E[3:5]) + if propToNodes: + # NOTE: this is disallowed by default because a same node can have two different diameters in SubDyn (it's by element) + Graph.setElementNodalProp(elem, propset=PropSets[Type-1], propIDs=E[3:5]) + if propToElem: + Graph.setElementNodalPropToElem(elem) # TODO, this shouldn't be needed # --- Concentrated Masses (in global coordinates), node data for iC, CM in enumerate(sd['ConcentratedMasses']): @@ -127,9 +134,14 @@ def subdynToGraph(sd): # --------------------------------------------------------------------------------} # --- HydroDyn # --------------------------------------------------------------------------------{ -def hydrodynToGraph(hd): +def hydrodynToGraph(hd, propToNodes=False, propToElem=False): """ hd: dict-like object as returned by weio + + -propToNodes: if True, the element properties are also transferred to the nodes for convenience. + NOTE: this is not the default because a same node can have two different diameters in SubDyn (it's by element) + + - propToElem: This might be due to misunderstanding of graph.. """ def type2Color(Pot): if Pot: @@ -218,12 +230,19 @@ def type2Color(Pot): elem.data['color'] = type2Color(Pot) Graph.addElement(elem) # Nodal prop data NOTE: can't do that anymore for memebrs with different diameters at the same node - #Graph.setElementNodalProp(elem, propset='Section', propIDs=EE[3:5]) + if propToNodes: + # NOTE: not by default because of feature with members with different diameters at the same node + Graph.setElementNodalProp(elem, propset='Section', propIDs=EE[3:5]) + if propToElem: + Graph.setElementNodalPropToElem(elem) # TODO, this shouldn't be needed + if Type==1: # Simple Graph.setElementNodalProp(elem, propset='SimpleCoefs', propIDs=[1,1]) else: print('>>> TODO type DepthCoefs and MemberCoefs') + # NOTE: this is disallowed by default because a same node can have two different diameters in SubDyn (it's by element) + Graph.setElementNodalProp(elem, propset=PropSets[Type-1], propIDs=E[3:5]) return Graph diff --git a/pydatview/io/hawc2_htc_file.py b/pydatview/io/hawc2_htc_file.py index 0fefb1b..7cdb49e 100644 --- a/pydatview/io/hawc2_htc_file.py +++ b/pydatview/io/hawc2_htc_file.py @@ -114,7 +114,11 @@ def _toDataFrame(self): tim = bdy.timoschenko_input H2_stfile = os.path.join(simdir, tim.filename[0]) if not os.path.exists(H2_stfile): - print('[WARN] st file referenced in htc file was not found. St file: {}, htc file {}'.format(H2_stfile, self.filename)) + # Try with a parent directory.. + H2_stfile = os.path.join(simdir, '../',tim.filename[0]) + + if not os.path.exists(H2_stfile): + print('[WARN] st file referenced in htc file was not found for body {}.\nSt file: {}\nhtc file {}'.format(name, H2_stfile, self.filename)) else: dfs_st = HAWC2StFile(H2_stfile).toDataFrame(extraCols=False) if 'set' in tim.keys(): diff --git a/pydatview/io/wetb/hawc2/htc_file.py b/pydatview/io/wetb/hawc2/htc_file.py index 169e015..7648ac7 100644 --- a/pydatview/io/wetb/hawc2/htc_file.py +++ b/pydatview/io/wetb/hawc2/htc_file.py @@ -204,15 +204,24 @@ def readlines(self, filename): if self.modelpath == 'unknown': p = os.path.dirname(self.filename) - lu = [os.path.isfile(os.path.abspath(os.path.join(p, "../" * i, filename.replace("\\", "/")))) - for i in range(4)].index(True) - filename = os.path.join(p, "../" * lu, filename) + try: + lu = [os.path.isfile(os.path.abspath(os.path.join(p, "../" * i, filename.replace("\\", "/")))) + for i in range(4)].index(True) + filename = os.path.join(p, "../" * lu, filename) + except ValueError: + print('[FAIL] Cannot continue in file: {}'.format(filename)) + filename = None else: filename = os.path.join(self.modelpath, filename) - for line in self.readlines(filename): - if line.lstrip().lower().startswith('exit'): - break - htc_lines.append(line) + if not os.path.isfile(filename): + print('[FAIL] Cannot continue in file: {}'.format(filename)) + filename=None + if filename is not None: + #print('[INFO] Continuing in file: {}'.format(filename)) + for line in self.readlines(filename): + if line.lstrip().lower().startswith('exit'): + break + htc_lines.append(line) else: htc_lines.append(l) return htc_lines From 6f02e71cdcf36d43bb4b936c968f4754a71e88b0 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 21 Dec 2022 23:10:29 +0100 Subject: [PATCH 058/178] SelPanel: Introducing callbacks to reduce dependency with mainframe --- pydatview/GUIPlotPanel.py | 85 +- pydatview/GUISelectionPanel.py | 91 +- pydatview/Tables.py | 12 +- pydatview/main.py | 1689 ++++++++++++++++---------------- pydatview/plotdata.py | 1620 +++++++++++++++--------------- 5 files changed, 1792 insertions(+), 1705 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 074f139..9a78f19 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -38,12 +38,12 @@ import gc -from .common import * # unique, CHAR -from .plotdata import PlotData, compareMultiplePD -from .GUICommon import * -from .GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool -from .GUIMeasure import GUIMeasure -from . import icons +from pydatview.common import * # unique, CHAR +from pydatview.plotdata import PlotData, compareMultiplePD +from pydatview.GUICommon import * +from pydatview.GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool +from pydatview.GUIMeasure import GUIMeasure +import pydatview.icons as icons font = {'size' : 8} matplotlib_rc('font', **font) @@ -404,7 +404,8 @@ def __init__(self, parent, selPanel, infoPanel=None, mainframe=None, data=None): self.selPanel = selPanel # <<< dependency with selPanel should be minimum self.selMode = '' self.infoPanel=infoPanel - self.infoPanel.setPlotMatrixCallbacks(self._onPlotMatrixLeftClick, self._onPlotMatrixRightClick) + if self.infoPanel is not None: + self.infoPanel.setPlotMatrixCallbacks(self._onPlotMatrixLeftClick, self._onPlotMatrixRightClick) self.parent = parent self.mainframe= mainframe self.plotData = [] @@ -650,7 +651,8 @@ def set_subplot_spacing(self, init=False): self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98) def plot_matrix_select(self, event): - self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) + if self.infoPanel is not None: + self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) self.redraw_same_data() def measure_select(self, event): @@ -725,11 +727,13 @@ def onMouseRelease(self, event): self.cbAutoScale.SetValue(False) return if event.button == 1: - self.infoPanel.setMeasurements((x, y), None) + if self.infoPanel is not None: + self.infoPanel.setMeasurements((x, y), None) self.leftMeasure.set(ax_idx, x, y) self.leftMeasure.plot(ax, ax_idx) elif event.button == 3: - self.infoPanel.setMeasurements(None, (x, y)) + if self.infoPanel is not None: + self.infoPanel.setMeasurements(None, (x, y)) self.rightMeasure.set(ax_idx, x, y) self.rightMeasure.plot(ax, ax_idx) else: @@ -783,12 +787,12 @@ def showTool(self,toolName=''): else: raise Exception('Unknown tool {}'.format(toolName)) - def showToolPanel(self, panelClass): + def showToolPanel(self, action): """ Show a tool panel based on a panel class (should inherit from GUIToolPanel)""" - from .GUITools import TOOLS + panelClass = action.guiEditorClass self.Freeze() self.removeTools(Layout=False) - self.toolPanel=panelClass(parent=self) # calling the panel constructor + self.toolPanel=panelClass(parent=self, action=action) # calling the panel constructor self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) self.plotsizer.Layout() self.Thaw() @@ -980,7 +984,8 @@ def plot_all(self, keep_limits=True): if self.cbMeasure.GetValue() is False: for measure in [self.leftMeasure, self.rightMeasure]: measure.clear() - self.infoPanel.setMeasurements(None, None) + if self.infoPanel is not None: + self.infoPanel.setMeasurements(None, None) self.lbDeltaX.SetLabel('') self.lbDeltaY.SetLabel('') @@ -1035,11 +1040,15 @@ def plot_all(self, keep_limits=True): self.set_axes_lim(PD, ax_left) # Actually plot - pm = self.infoPanel.getPlotMatrix(PD, self.cbSub.IsChecked()) + if self.infoPanel is not None: + pm = self.infoPanel.getPlotMatrix(PD, self.cbSub.IsChecked()) + else: + pm = None __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) - self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) + if self.infoPanel is not None: + self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) for measure in [self.leftMeasure, self.rightMeasure]: measure.plot(ax_left, axis_idx) @@ -1233,11 +1242,15 @@ def findSubPlots(self,PD,mode): bCompare = self.pltTypePanel.cbCompare.GetValue() # NOTE bCompare somehow always 1Tab_nCols nSubPlots=1 spreadBy='none' - self.infoPanel.setTabMode(mode) + if self.infoPanel is not None: + self.infoPanel.setTabMode(mode) # TODO get rid of me if mode=='1Tab_nCols': if bSubPlots: if bCompare or len(uTabs)==1: - nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) + if self.infoPanel is not None: + nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) + else: + nSubPlots=len(usy) else: nSubPlots=len(usy) spreadBy='iy' @@ -1453,10 +1466,14 @@ def _restore_limits(self): if __name__ == '__main__': import pandas as pd; from Tables import Table,TableList + from pydatview.Tables import TableList + from pydatview.GUISelectionPanel import SelectionPanel + from pydatview.common import DummyMainFrame + tabList = TableList.createDummy(1) app = wx.App(False) - self=wx.Frame(None,-1,"Title") - self.SetSize((800, 600)) + self=wx.Frame(None,-1,"GUI Plot Panel Demo") + #self.SetBackgroundColour('red') class FakeSelPanel(wx.Panel): def __init__(self, parent): @@ -1471,14 +1488,26 @@ def getPlotDataSelection(self): ID.append([0,0,3,'x','ColC','tab']) return ID,True - selpanel=FakeSelPanel(self) - # selpanel.SetBackgroundColour('blue') - p1=PlotPanel(self, selpanel, data=None) - p1.load_and_draw() - #p1=SpectralCtrlPanel(self) - sizer = wx.BoxSizer(wx.VERTICAL) - sizer.Add(selpanel,0, flag = wx.EXPAND|wx.ALL,border = 10) - sizer.Add(p1,1, flag = wx.EXPAND|wx.ALL,border = 10) + mainframe = DummyMainFrame(self) + # --- Selection Panel + #selpanel = FakeSelPanel(self) + selPanel = SelectionPanel(self, tabList, mode='auto', mainframe=mainframe) + # selpanel.SetBackgroundColour('blue') + self.selPanel=selPanel + + + # --- Plot Panel + plotPanel=PlotPanel(self, selPanel, data=None) + plotPanel.load_and_draw() + self.plotPanel = plotPanel + + # --- Binding the two + selPanel.setRedrawCallback(self.plotPanel.load_and_draw) + + # --- Finalize GUI + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(selPanel ,0, flag = wx.EXPAND|wx.ALL,border = 10) + sizer.Add(plotPanel,1, flag = wx.EXPAND|wx.ALL,border = 10) self.SetSizer(sizer) self.Center() diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 6175931..37ec1fa 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -355,9 +355,10 @@ def OnSort(self, event): class ColumnPopup(wx.Menu): """ Popup Menu when right clicking on the column list """ - def __init__(self, parent, fullmenu=False): + def __init__(self, parent, selPanel, fullmenu=False): wx.Menu.__init__(self) self.parent = parent # parent is ColumnPanel + self.selPanel = selPanel # we need a selPanel self.ISel = self.parent.lbColumns.GetSelections() self.itShowID = wx.MenuItem(self, -1, "Show ID", kind=wx.ITEM_CHECK) @@ -399,13 +400,12 @@ def OnRenameColumn(self, event=None): dlg.CentreOnParent() if dlg.ShowModal() == wx.ID_OK: newName=dlg.GetValue() - main=self.parent.mainframe - ITab,STab=main.selPanel.getSelectedTables() + ITab,STab=self.selPanel.getSelectedTables() # TODO adapt me for Sim. tables mode iFull = self.parent.Filt2Full[iFilt] - if main.tabList.haveSameColumns(ITab): + if self.tabList.haveSameColumns(ITab): for iTab,sTab in zip(ITab,STab): - main.tabList.get(iTab).renameColumn(iFull,newName) + self.tabList.get(iTab).renameColumn(iFull,newName) else: self.parent.tab.renameColumn(iFull,newName) self.parent.updateColumn(iFilt,newName) #faster @@ -413,13 +413,12 @@ def OnRenameColumn(self, event=None): # a trigger for the plot is required but skipped for now def OnEditColumn(self, event): - main=self.parent.mainframe if len(self.ISel) != 1: raise ValueError('Only one signal can be edited!') - ITab, STab = main.selPanel.getSelectedTables() + ITab, STab = self.selPanel.getSelectedTables() for iTab,sTab in zip(ITab,STab): if sTab == self.parent.tab.active_name: - for f in main.tabList.get(iTab).formulas: + for f in self.tabList.get(iTab).formulas: if f['pos'] == self.ISel[0]: sName = f['name'] sFormula = f['formula'] @@ -429,29 +428,26 @@ def OnEditColumn(self, event): self.showFormulaDialog('Edit column', sName, sFormula, edit=True) def OnDeleteColumn(self, event): - main=self.parent.mainframe iX = self.parent.comboX.GetSelection() - ITab,STab=main.selPanel.getSelectedTables() + ITab,STab=self.selPanel.getSelectedTables() # TODO adapt me for Sim. tables mode IFull = [self.parent.Filt2Full[iFilt] for iFilt in self.ISel] IFull = [iFull for iFull in IFull if iFull>=0] - if main.tabList.haveSameColumns(ITab): + if self.tabList.haveSameColumns(ITab): for iTab,sTab in zip(ITab,STab): - main.tabList.get(iTab).deleteColumns(IFull) + self.tabList.get(iTab).deleteColumns(IFull) else: self.parent.tab.deleteColumns(IFull) self.parent.setColumns() self.parent.setGUIColumns(xSel=iX) - main.redraw() + self.redraw() def OnAddColumn(self, event): - main=self.parent.mainframe self.showFormulaDialog('Add a new column') def showFormulaDialog(self, title, name='', formula='', edit=False): bValid=False bCancelled=False - main=self.parent.mainframe sName=name sFormula=formula @@ -480,19 +476,19 @@ def showFormulaDialog(self, title, name='', formula='', edit=False): else: iFull = -1 - ITab,STab=main.selPanel.getSelectedTables() + ITab,STab = self.selPanel.getSelectedTables() #if main.tabList.haveSameColumns(ITab): sError='' nError=0 - haveSameColumns=main.tabList.haveSameColumns(ITab) + haveSameColumns= self.selPanel.tabList.haveSameColumns(ITab) for iTab,sTab in zip(ITab,STab): if haveSameColumns or self.parent.tab.active_name == sTab: # apply formula to all tables with same columns, otherwise only to active table if edit: - bValid=main.tabList.get(iTab).setColumnByFormula(sName,sFormula,iFull) + bValid=self.selPanel.tabList.get(iTab).setColumnByFormula(sName,sFormula,iFull) iOffset = 0 # we'll stay on this column that we are editing else: - bValid=main.tabList.get(iTab).addColumnByFormula(sName,sFormula,iFull) + bValid=self.selPanel.tabList.get(iTab).addColumnByFormula(sName,sFormula,iFull) iOffset = 1 # we'll select this newly created column if not bValid: sError+='The formula didn''t eval for table {}\n'.format(sTab) @@ -509,7 +505,7 @@ def showFormulaDialog(self, title, name='', formula='', edit=False): iX = self.parent.comboX.GetSelection() self.parent.setColumns() self.parent.setGUIColumns(xSel=iX,ySel=[iFull+iOffset]) - main.redraw() + self.selPanel.redraw() @@ -639,13 +635,13 @@ def __init__(self, parent, selPanel, mainframe): def showColumnMenu(self,event): if not self.bReadOnly: pos = (self.bt.GetPosition()[0], self.bt.GetPosition()[1] + self.bt.GetSize()[1]) - menu = ColumnPopup(self,fullmenu=True) + menu = ColumnPopup(self, selPanel=self.selPanel, fullmenu=True) self.PopupMenu(menu, pos) menu.Destroy() def OnColPopup(self,event): if not self.bReadOnly: - menu = ColumnPopup(self) + menu = ColumnPopup(self, selPanel=self.selPanel) self.PopupMenu(menu, event.GetPosition()) menu.Destroy() @@ -956,6 +952,10 @@ def __init__(self, parent, tabList, mode='auto', mainframe=None): self.currentMode = None self.nSplits = -1 self.IKeepPerTab=None + # Useful callBacks to be set by callee + self.redrawCallback=None + self.colSelectionChangeCallback=None + self.tabSelectionChangeCallback=None # GUI DATA self.splitter = MultiSplit(self, style=wx.SP_LIVE_UPDATE) @@ -975,9 +975,56 @@ def __init__(self, parent, tabList, mode='auto', mainframe=None): VertSizer.Add(self.splitter, 2, flag=wx.EXPAND, border=0) self.SetSizer(VertSizer) + # BINDINGS + self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.colPanel1.comboX ) + self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.colPanel1.lbColumns) + self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.colPanel2.comboX ) + self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.colPanel2.lbColumns) + self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.colPanel3.comboX ) + self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.colPanel3.lbColumns) + self.Bind(wx.EVT_LISTBOX, self.onTabSelectionChange, self.tabPanel.lbTab) + # TRIGGERS self.setTables(tabList) + # --- Callbacks + def setColSelectionChangeCallback(self, callback): + self.colSelectionChangeCallback = callback + + def setTabSelectionChangeCallback(self, callback): + self.tabSelectionChangeCallback = callback + + def setRedrawCallback(self, callBack): + self.redrawCallback = callBack + + def redraw(self): + if self.redrawCallback is not None: + self.redrawCallback() + + def onColSelectionChange(self, event=None): + # Letting selection panel handle the change + self.colSelectionChanged() + # We call the callback if it was set + if self.colSelectionChangeCallback is not None: + self.colSelectionChangeCallback() + # We redraw + self.redraw() + + def onTabSelectionChange(self, event=None): + ISel=self.tabPanel.lbTab.GetSelections() + if len(ISel)>0: + # Letting seletion panel handle the change + self.tabSelectionChanged() + + # We call the callback if it was set + if self.tabSelectionChangeCallback is not None: + self.tabSelectionChangeCallback() + + # We trigger a column selection change... + self.onColSelectionChange(event=None) + + + def updateLayout(self, mode=None): self.Freeze() if mode is None: diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 2650d6e..6506984 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -78,6 +78,7 @@ def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bRe # Loop through files, appending tables within files warnList=[] + newTabs=[] for i, (f,ff) in enumerate(zip(filenames, fileformats)): if statusFunction is not None: statusFunction(i) @@ -91,11 +92,12 @@ def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bRe if len(warnloc)>0: warnList.append(warnloc) self.append(tabs) + newTabs +=tabs - return warnList + return newTabs, warnList def _load_file_tabs(self, filename, fileformat=None, bReload=False): - """ load a single file, adds table """ + """ load a single file, returns a list (often of size one) of tables """ # Returning a list of tables tabs=[] warn='' @@ -396,10 +398,10 @@ def get(self,i): @staticmethod - def createDummyList(nTab=3): + def createDummy(nTab=3): tabs=[] for iTab in range(nTab): - tabs.append( Table.createDummy() ) + tabs.append( Table.createDummy(lab='_'+str(iTab)) ) tablist = TableList(tabs) return tablist @@ -824,7 +826,7 @@ def createDummy(n=5, lab=''): 'P{}_[W]'.format(lab):P, 'RotSpeed{}_[rpm]'.format(lab):RPM} df = pd.DataFrame(data=d) - return Table(data=df) + return Table(data=df, name='Dummy '+lab) if __name__ == '__main__': diff --git a/pydatview/main.py b/pydatview/main.py index 6ac21ee..990fe0e 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -1,843 +1,846 @@ -import numpy as np -import os.path -import sys -import traceback -import gc -try: - import pandas as pd -except: - print('') - print('') - print('Error: problem loading pandas package:') - print(' - Check if this package is installed ( e.g. type: `pip install pandas`)') - print(' - If you are using anaconda, try `conda update python.app`') - print(' - If none of the above work, contact the developer.') - print('') - print('') - sys.exit(-1) - #raise - - -# GUI -import wx -from .GUIPlotPanel import PlotPanel -from .GUISelectionPanel import SelectionPanel,SEL_MODES,SEL_MODES_ID -from .GUISelectionPanel import ColumnPopup,TablePopup -from .GUIInfoPanel import InfoPanel -from .GUIToolBox import GetKeyString, TBAddTool -from .Tables import TableList, Table -# Helper -from .common import * -from .GUICommon import * -import pydatview.io as weio # File Formats and File Readers -# Pluggins -from .plugins import dataPlugins -from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData - -# --------------------------------------------------------------------------------} -# --- GLOBAL -# --------------------------------------------------------------------------------{ -PROG_NAME='pyDatView' -PROG_VERSION='v0.3-local' -SIDE_COL = [160,160,300,420,530] -SIDE_COL_LARGE = [200,200,360,480,600] -BOT_PANL =85 -ISTAT = 0 # Index of Status bar where main status info is provided - -#matplotlib.rcParams['text.usetex'] = False -# matplotlib.rcParams['font.sans-serif'] = 'DejaVu Sans' -#matplotlib.rcParams['font.family'] = 'Arial' -#matplotlib.rcParams['font.sans-serif'] = 'Arial' -# matplotlib.rcParams['font.family'] = 'sans-serif' - - - - - -# --------------------------------------------------------------------------------} -# --- Drag and drop -# --------------------------------------------------------------------------------{ -# Implement File Drop Target class -class FileDropTarget(wx.FileDropTarget): - def __init__(self, parent): - wx.FileDropTarget.__init__(self) - self.parent = parent - def OnDropFiles(self, x, y, filenames): - filenames = [f for f in filenames if not os.path.isdir(f)] - filenames.sort() - if len(filenames)>0: - # If Ctrl is pressed we add - bAdd= wx.GetKeyState(wx.WXK_CONTROL); - iFormat=self.parent.comboFormats.GetSelection() - if iFormat==0: # auto-format - Format = None - else: - Format = self.parent.FILE_FORMATS[iFormat-1] - self.parent.load_files(filenames, fileformats=[Format]*len(filenames), bAdd=bAdd, bPlot=True) - return True - - -# --------------------------------------------------------------------------------} -# --- Loader Menu -# --------------------------------------------------------------------------------{ -class LoaderMenuPopup(wx.Menu): - def __init__(self, parent, data): - wx.Menu.__init__(self) - self.parent = parent - self.data = data - - # Populate menu - item = wx.MenuItem(self, -1, "Date format: dayfirst", kind=wx.ITEM_CHECK) - self.Append(item) - self.Bind(wx.EVT_MENU, lambda ev: self.setCheck(ev, 'dayfirst') ) - self.Check(item.GetId(), self.data['dayfirst']) # Checking the menu box - - def setCheck(self, event, label): - self.data['dayfirst'] = not self.data['dayfirst'] - - -# --------------------------------------------------------------------------------} -# --- Main Frame -# --------------------------------------------------------------------------------{ -class MainFrame(wx.Frame): - def __init__(self, data=None): - # Parent constructor - wx.Frame.__init__(self, None, -1, PROG_NAME+' '+PROG_VERSION) - # Hooking exceptions to display them to the user - sys.excepthook = MyExceptionHook - # --- Data - self.restore_formulas = [] - self.systemFontSize = self.GetFont().GetPointSize() - self.data = loadAppData(self) - self.tabList=TableList(options=self.data['loaderOptions']) - self.datareset = False - # Global variables... - setFontSize(self.data['fontSize']) - setMonoFontSize(self.data['monoFontSize']) - - # --- GUI - #font = self.GetFont() - #print(font.GetFamily(),font.GetStyle(),font.GetPointSize()) - #font.SetFamily(wx.FONTFAMILY_DEFAULT) - #font.SetFamily(wx.FONTFAMILY_MODERN) - #font.SetFamily(wx.FONTFAMILY_SWISS) - #font.SetPointSize(8) - #print(font.GetFamily(),font.GetStyle(),font.GetPointSize()) - #self.SetFont(font) - self.SetFont(getFont(self)) - # --- Menu - menuBar = wx.MenuBar() - - fileMenu = wx.Menu() - loadMenuItem = fileMenu.Append(wx.ID_NEW,"Open file" ,"Open file" ) - exptMenuItem = fileMenu.Append(-1 ,"Export table" ,"Export table" ) - saveMenuItem = fileMenu.Append(wx.ID_SAVE,"Save figure" ,"Save figure" ) - exitMenuItem = fileMenu.Append(wx.ID_EXIT, 'Quit', 'Quit application') - menuBar.Append(fileMenu, "&File") - self.Bind(wx.EVT_MENU,self.onExit ,exitMenuItem) - self.Bind(wx.EVT_MENU,self.onLoad ,loadMenuItem) - self.Bind(wx.EVT_MENU,self.onExport,exptMenuItem) - self.Bind(wx.EVT_MENU,self.onSave ,saveMenuItem) - - dataMenu = wx.Menu() - menuBar.Append(dataMenu, "&Data") - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e, 'Mask') , dataMenu.Append(wx.ID_ANY, 'Mask')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Outlier'), dataMenu.Append(wx.ID_ANY, 'Outliers removal')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Filter') , dataMenu.Append(wx.ID_ANY, 'Filter')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Resample') , dataMenu.Append(wx.ID_ANY, 'Resample')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'FASTRadialAverage'), dataMenu.Append(wx.ID_ANY, 'FAST - Radial average')) - - # --- Data Plugins - for string, function, isPanel in dataPlugins: - self.Bind(wx.EVT_MENU, lambda e, s_loc=string: self.onDataPlugin(e, s_loc), dataMenu.Append(wx.ID_ANY, string)) - - toolMenu = wx.Menu() - menuBar.Append(toolMenu, "&Tools") - self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'CurveFitting'), toolMenu.Append(wx.ID_ANY, 'Curve fitting')) - self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'LogDec') , toolMenu.Append(wx.ID_ANY, 'Damping from decay')) - - helpMenu = wx.Menu() - aboutMenuItem = helpMenu.Append(wx.NewId(), 'About', 'About') - resetMenuItem = helpMenu.Append(wx.NewId(), 'Reset options', 'Rest options') - menuBar.Append(helpMenu, "&Help") - self.SetMenuBar(menuBar) - self.Bind(wx.EVT_MENU,self.onAbout, aboutMenuItem) - self.Bind(wx.EVT_MENU,self.onReset, resetMenuItem) - - - io_userpath = os.path.join(weio.defaultUserDataDir(), 'pydatview_io') - self.FILE_FORMATS, errors= weio.fileFormats(userpath=io_userpath, ignoreErrors=True, verbose=False) - if len(errors)>0: - for e in errors: - Warn(self, e) - - self.FILE_FORMATS_EXTENSIONS = [['.*']]+[f.extensions for f in self.FILE_FORMATS] - self.FILE_FORMATS_NAMES = ['auto (any supported file)'] + [f.name for f in self.FILE_FORMATS] - self.FILE_FORMATS_NAMEXT =['{} ({})'.format(n,','.join(e)) for n,e in zip(self.FILE_FORMATS_NAMES,self.FILE_FORMATS_EXTENSIONS)] - - # --- ToolBar - tb = self.CreateToolBar(wx.TB_HORIZONTAL|wx.TB_TEXT|wx.TB_HORZ_LAYOUT) - self.toolBar = tb - self.comboFormats = wx.ComboBox(tb, choices = self.FILE_FORMATS_NAMEXT, style=wx.CB_READONLY) - self.comboFormats.SetSelection(0) - self.comboMode = wx.ComboBox(tb, choices = SEL_MODES, style=wx.CB_READONLY) - self.comboMode.SetSelection(0) - self.Bind(wx.EVT_COMBOBOX, self.onModeChange, self.comboMode ) - self.Bind(wx.EVT_COMBOBOX, self.onFormatChange, self.comboFormats ) - tb.AddSeparator() - tb.AddControl( wx.StaticText(tb, -1, 'Mode: ' ) ) - tb.AddControl( self.comboMode ) - tb.AddStretchableSpace() - tb.AddControl( wx.StaticText(tb, -1, 'Format: ' ) ) - tb.AddControl(self.comboFormats ) - # Menu for loader options - self.btLoaderMenu = wx.Button(tb, wx.ID_ANY, CHAR['menu'], style=wx.BU_EXACTFIT) - tb.AddControl(self.btLoaderMenu) - self.loaderMenu = LoaderMenuPopup(tb, self.data['loaderOptions']) - tb.Bind(wx.EVT_BUTTON, self.onShowLoaderMenu, self.btLoaderMenu) - tb.AddSeparator() - TBAddTool(tb, "Open" , 'ART_FILE_OPEN', self.onLoad) - TBAddTool(tb, "Reload", 'ART_REDO' , self.onReload) - TBAddTool(tb, "Add" , 'ART_PLUS' , self.onAdd) - #bmp = wx.Bitmap('help.png') #wx.Bitmap("NEW.BMP", wx.BITMAP_TYPE_BMP) - #self.AddTBBitmapTool(tb,"Debug" ,wx.ArtProvider.GetBitmap(wx.ART_ERROR),self.onDEBUG) - tb.AddStretchableSpace() - tb.Realize() - - # --- Status bar - self.statusbar=self.CreateStatusBar(3, style=0) - self.statusbar.SetStatusWidths([150, -1, 70]) - - # --- Main Panel and Notebook - self.MainPanel = wx.Panel(self) - #self.MainPanel = wx.Panel(self, style=wx.RAISED_BORDER) - #self.MainPanel.SetBackgroundColour((200,0,0)) - - #self.nb = wx.Notebook(self.MainPanel) - #self.nb.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.on_tab_change) - - - sizer = wx.BoxSizer() - #sizer.Add(self.nb, 1, flag=wx.EXPAND) - self.MainPanel.SetSizer(sizer) - - # --- Drag and drop - dd = FileDropTarget(self) - self.SetDropTarget(dd) - - # --- Main Frame (self) - self.FrameSizer = wx.BoxSizer(wx.VERTICAL) - slSep = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) - self.FrameSizer.Add(slSep ,0, flag=wx.EXPAND|wx.BOTTOM,border=0) - self.FrameSizer.Add(self.MainPanel,1, flag=wx.EXPAND,border=0) - self.SetSizer(self.FrameSizer) - - self.SetSize(self.data['windowSize']) - self.Center() - self.Show() - self.Bind(wx.EVT_SIZE, self.OnResizeWindow) - self.Bind(wx.EVT_CLOSE, self.onClose) - - # Shortcuts - idFilter=wx.NewId() - self.Bind(wx.EVT_MENU, self.onFilter, id=idFilter) - - accel_tbl = wx.AcceleratorTable( - [(wx.ACCEL_CTRL, ord('F'), idFilter )] - ) - self.SetAcceleratorTable(accel_tbl) - - def onFilter(self,event): - if hasattr(self,'selPanel'): - self.selPanel.colPanel1.tFilter.SetFocus() - event.Skip() - - def clean_memory(self,bReload=False): - #print('Clean memory') - # force Memory cleanup - self.tabList.clean() - if not bReload: - if hasattr(self,'selPanel'): - self.selPanel.clean_memory() - if hasattr(self,'infoPanel'): - self.infoPanel.clean() - if hasattr(self,'plotPanel'): - self.plotPanel.cleanPlot() - gc.collect() - - def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, bPlot=True): - """ load multiple files, only trigger the plot at the end """ - if bReload: - if hasattr(self,'selPanel'): - self.selPanel.saveSelection() # TODO move to tables - else: - self.statusbar.SetStatusText('Loading files...', ISTAT) - - # A function to update the status bar while we load files - statusFunction = lambda i: self.statusbar.SetStatusText('Loading files {}/{}'.format(i+1,len(filenames)), ISTAT) - - if not bAdd: - self.clean_memory(bReload=bReload) - - - if fileformats is None: - fileformats=[None]*len(filenames) - assert type(fileformats)==list, 'fileformats must be a list' - assert len(fileformats)==len(filenames), 'fileformats and filenames must have the same lengths' - - # Sorting files in alphabetical order in base_filenames order - base_filenames = [os.path.basename(f) for f in filenames] - I = np.argsort(base_filenames) - filenames = list(np.array(filenames)[I]) - fileformats = list(np.array(fileformats)[I]) - #filenames = [f for __, f in sorted(zip(base_filenames, filenames))] - - # Load the tables - warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformats=fileformats, bAdd=bAdd, bReload=bReload, statusFunction=statusFunction) - if bReload: - # Restore formulas that were previously added - for tab in self.tabList: - if tab.raw_name in self.restore_formulas.keys(): - for f in self.restore_formulas[tab.raw_name]: - tab.addColumnByFormula(f['name'], f['formula'], f['pos']-1) - self.restore_formulas = {} - # Display warnings - for warn in warnList: - Warn(self,warn) - # Load tables into the GUI - if self.tabList.len()>0: - self.load_tabs_into_GUI(bReload=bReload, bAdd=bAdd, bPlot=bPlot) - - def load_df(self, df, name=None, bAdd=False, bPlot=True): - if bAdd: - self.tabList.append(Table(data=df, name=name)) - else: - self.tabList = TableList( [Table(data=df, name=name)] ) - self.load_tabs_into_GUI(bAdd=bAdd, bPlot=bPlot) - if hasattr(self,'selPanel'): - self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) - - def load_dfs(self, dfs, names, bAdd=False): - self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) - self.load_tabs_into_GUI(bAdd=bAdd, bPlot=True) - if hasattr(self,'selPanel'): - self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) - - def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): - if bAdd: - if not hasattr(self,'selPanel'): - bAdd=False - - if (not bReload) and (not bAdd): - self.cleanGUI() - if (bReload): - self.statusbar.SetStatusText('Done reloading.', ISTAT) - self.Freeze() - # Setting status bar - self.setStatusBar() - - if bReload or bAdd: - self.selPanel.update_tabs(self.tabList) - else: - # --- Create a selPanel, plotPanel and infoPanel - mode = SEL_MODES_ID[self.comboMode.GetSelection()] - #self.vSplitter = wx.SplitterWindow(self.nb) - self.vSplitter = wx.SplitterWindow(self.MainPanel) - self.selPanel = SelectionPanel(self.vSplitter, self.tabList, mode=mode, mainframe=self) - self.tSplitter = wx.SplitterWindow(self.vSplitter) - #self.tSplitter.SetMinimumPaneSize(20) - self.infoPanel = InfoPanel(self.tSplitter, data=self.data['infoPanel']) - self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, self, data=self.data['plotPanel']) - self.tSplitter.SetSashGravity(0.9) - self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) - self.tSplitter.SetMinimumPaneSize(BOT_PANL) - self.tSplitter.SetSashGravity(1) - self.tSplitter.SetSashPosition(400) - - self.vSplitter.SplitVertically(self.selPanel, self.tSplitter) - self.vSplitter.SetMinimumPaneSize(SIDE_COL[0]) - self.tSplitter.SetSashPosition(SIDE_COL[0]) - - #self.nb.AddPage(self.vSplitter, "Plot") - #self.nb.SendSizeEvent() - - sizer = self.MainPanel.GetSizer() - sizer.Add(self.vSplitter, 1, flag=wx.EXPAND,border=0) - self.MainPanel.SetSizer(sizer) - self.FrameSizer.Layout() - - self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel1.comboX ) - self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel1.lbColumns) - self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel2.comboX ) - self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel2.lbColumns) - self.Bind(wx.EVT_COMBOBOX, self.onColSelectionChange, self.selPanel.colPanel3.comboX ) - self.Bind(wx.EVT_LISTBOX , self.onColSelectionChange, self.selPanel.colPanel3.lbColumns) - self.Bind(wx.EVT_LISTBOX , self.onTabSelectionChange, self.selPanel.tabPanel.lbTab) - self.Bind(wx.EVT_SPLITTER_SASH_POS_CHANGED, self.onSashChangeMain, self.vSplitter) - - # plot trigger - if bPlot: - self.mainFrameUpdateLayout() - self.onColSelectionChange(event=None) - try: - self.Thaw() - except: - pass - # Hack - #self.onShowTool(tool='Filter') - #self.onShowTool(tool='Resample') - #self.onDataPlugin(toolName='Bin data') - - def setStatusBar(self, ISel=None): - nTabs=self.tabList.len() - if ISel is None: - ISel = list(np.arange(nTabs)) - if nTabs<0: - self.statusbar.SetStatusText('', ISTAT) # Format - self.statusbar.SetStatusText('', ISTAT+1) # Filenames - self.statusbar.SetStatusText('', ISTAT+2) # Shape - elif nTabs==1: - self.statusbar.SetStatusText(self.tabList.get(0).fileformat_name, ISTAT+0) - self.statusbar.SetStatusText(self.tabList.get(0).filename , ISTAT+1) - self.statusbar.SetStatusText(self.tabList.get(0).shapestring , ISTAT+2) - elif len(ISel)==1: - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).fileformat_name , ISTAT+0) - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).filename , ISTAT+1) - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).shapestring , ISTAT+2) - else: - self.statusbar.SetStatusText('{} tables loaded'.format(nTabs) ,ISTAT+0) - self.statusbar.SetStatusText(", ".join(list(set([self.tabList.filenames[i] for i in ISel]))),ISTAT+1) - self.statusbar.SetStatusText('' ,ISTAT+2) - - # --- Table Actions - TODO consider a table handler, or doing only the triggers - def renameTable(self, iTab, newName): - oldName = self.tabList.renameTable(iTab, newName) - self.selPanel.renameTable(iTab, oldName, newName) - - def sortTabs(self, method='byName'): - self.tabList.sort(method=method) - # Updating tables - self.selPanel.update_tabs(self.tabList) - # Trigger a replot - self.onTabSelectionChange() - - def mergeTabsTrigger(self): - if hasattr(self,'selPanel'): - # Select the newly created table - self.selPanel.tabPanel.lbTab.SetSelection(-1) # Empty selection - self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)-1) # Select new/last table - # Trigger a replot - self.onTabSelectionChange() - - def deleteTabs(self, I): - self.tabList.deleteTabs(I) - if len(self.tabList)==0: - self.cleanGUI() - return - - # Invalidating selections - self.selPanel.tabPanel.lbTab.SetSelection(-1) - # Until we have something better, we empty plot - self.plotPanel.empty() - self.infoPanel.empty() - self.selPanel.clean_memory() - # Updating tables - self.selPanel.update_tabs(self.tabList) - # Trigger a replot - self.onTabSelectionChange() - - - def exportTab(self, iTab): - tab=self.tabList.get(iTab) - default_filename=tab.basename +'.csv' - with wx.FileDialog(self, "Save to CSV file",defaultFile=default_filename, - style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as dlg: - #, wildcard="CSV files (*.csv)|*.csv", - dlg.CentreOnParent() - if dlg.ShowModal() == wx.ID_CANCEL: - return # the user changed their mind - tab.export(dlg.GetPath()) - - def onShowTool(self, event=None, tool=''): - """ - Show tool - tool in 'Outlier', 'Filter', 'LogDec','FASTRadialAverage', 'Mask', 'CurveFitting' - """ - if not hasattr(self,'plotPanel'): - Error(self,'Plot some data first') - return - self.plotPanel.showTool(tool) - - def onDataPlugin(self, event=None, toolName=''): - """ - Dispatcher to apply plugins to data: - - simple plugins are directly exectued - - plugins that are panels are sent over to plotPanel to show them - TODO merge with onShowTool - """ - if not hasattr(self,'plotPanel'): - Error(self,'Plot some data first') - return - - for thisToolName, function, isPanel in dataPlugins: - if toolName == thisToolName: - if isPanel: - panelClass = function(self, event, toolName) # getting panelClass - self.plotPanel.showToolPanel(panelClass) - else: - function(self, event, toolName) # calling the data function - return - raise NotImplementedError('Tool: ',toolName) - - - def onSashChangeMain(self,event=None): - pass - # doent work because size is not communicated yet - #if hasattr(self,'selPanel'): - # print('ON SASH') - # self.selPanel.setEquiSash(event) - - def onTabSelectionChange(self,event=None): - # TODO This can be cleaned-up - ISel=self.selPanel.tabPanel.lbTab.GetSelections() - if len(ISel)>0: - # Letting seletion panel handle the change - self.selPanel.tabSelectionChanged() - # Update of status bar - self.setStatusBar(ISel) - # Trigger the colSelection Event - self.onColSelectionChange(event=None) - - def onColSelectionChange(self,event=None): - if hasattr(self,'plotPanel'): - # Letting selection panel handle the change - self.selPanel.colSelectionChanged() - # Redrawing - self.plotPanel.load_and_draw() - # --- Stats trigger - #self.showStats() - - def redraw(self): - if hasattr(self,'plotPanel'): - self.plotPanel.load_and_draw() -# def showStats(self): -# self.infoPanel.showStats(self.plotPanel.plotData,self.plotPanel.pltTypePanel.plotType()) - - def onExit(self, event): - self.Close() - - def onClose(self, event): - saveAppData(self, self.data) - event.Skip() - - def cleanGUI(self, event=None): - if hasattr(self,'plotPanel'): - del self.plotPanel - if hasattr(self,'selPanel'): - del self.selPanel - if hasattr(self,'infoPanel'): - del self.infoPanel - #self.deletePages() - try: - self.MainPanel.GetSizer().Clear(delete_windows=True) # Delete Windows - except: - self.MainPanel.GetSizer().Clear() - self.FrameSizer.Layout() - gc.collect() - - def onSave(self, event=None): - # using the navigation toolbar save functionality - self.plotPanel.navTB.save_figure() - - def onAbout(self, event=None): - io_userpath = os.path.join(weio.defaultUserDataDir(), 'pydatview_io') - About(self,PROG_NAME+' '+PROG_VERSION+'\n\n' - 'pyDatView config file:\n {}\n'.format(configFilePath())+ - 'pyDatView io data directory:\n {}\n'.format(io_userpath)+ - '\n\nVisit http://github.com/ebranlard/pyDatView for documentation.') - - def onReset (self, event=None): - configFile = configFilePath() - result = YesNo(self, - 'The options of pyDatView will be reset to default.\nThe changes will be noticeable the next time you open pyDatView.\n\n'+ - 'This action will overwrite the user settings file:\n {}\n\n'.format(configFile)+ - 'pyDatView will then close.\n\n' - 'Are you sure you want to continue?', caption = 'Reset settings?') - if result: - try: - os.remove(configFile) - except: - pass - self.data = defaultAppData(self) - self.datareset = True - self.onExit(event=None) - - def onReload(self, event=None): - filenames, fileformats = self.tabList.filenames_and_formats - self.statusbar.SetStatusText('Reloading...', ISTAT) - if len(filenames)>0: - # If only one file, use the comboBox to decide which fileformat to use - if len(filenames)==1: - iFormat=self.comboFormats.GetSelection() - if iFormat==0: # auto-format - fileformats = [None] - else: - fileformats = [self.FILE_FORMATS[iFormat-1]] - - # Save formulas to restore them after reload with sorted tabs - self.restore_formulas = {} - for tab in self.tabList._tabs: - f = tab.formulas # list of dict('pos','formula','name') - f = sorted(f, key=lambda k: k['pos']) # Sort formulae by position in list of formua - self.restore_formulas[tab.raw_name]=f # we use raw_name as key - # Actually load files (read and add in GUI) - self.load_files(filenames, fileformats=fileformats, bReload=True, bAdd=False, bPlot=True) - else: - Error(self,'Open one or more file first.') - - def onDEBUG(self, event=None): - #self.clean_memory() - self.plotPanel.ctrlPanel.Refresh() - self.plotPanel.cb_sizer.ForceRefresh() - - def onExport(self, event=None): - ISel=[] - try: - ISel = self.selPanel.tabPanel.lbTab.GetSelections() - except: - pass - if len(ISel)>0: - self.exportTab(ISel[0]) - else: - Error(self,'Open a file and select a table first.') - - def onLoad(self, event=None): - self.selectFile(bAdd=False) - - def onAdd(self, event=None): - self.selectFile(bAdd=self.tabList.len()>0) - - def selectFile(self,bAdd=False): - # --- File Format extension - iFormat=self.comboFormats.GetSelection() - sFormat=self.comboFormats.GetStringSelection() - if iFormat==0: # auto-format - Format = None - #wildcard = 'all (*.*)|*.*' - wildcard='|'.join([n+'|*'+';*'.join(e) for n,e in zip(self.FILE_FORMATS_NAMEXT,self.FILE_FORMATS_EXTENSIONS)]) - #wildcard = sFormat + extensions+'|all (*.*)|*.*' - else: - Format = self.FILE_FORMATS[iFormat-1] - extensions = '|*'+';*'.join(self.FILE_FORMATS[iFormat-1].extensions) - wildcard = sFormat + extensions+'|all (*.*)|*.*' - - with wx.FileDialog(self, "Open file", wildcard=wildcard, - style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST | wx.FD_MULTIPLE) as dlg: - #other options: wx.CHANGE_DIR - #dlg.SetSize((100,100)) - #dlg.Center() - if dlg.ShowModal() == wx.ID_CANCEL: - return # the user changed their mind - filenames = dlg.GetPaths() - self.load_files(filenames,fileformats=[Format]*len(filenames),bAdd=bAdd, bPlot=True) - - def onModeChange(self, event=None): - if hasattr(self,'selPanel'): - self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) - self.mainFrameUpdateLayout() - # --- Trigger to check number of columns - self.onTabSelectionChange() - - def onFormatChange(self, event=None): - """ The user changed the format """ - #if hasattr(self,'selPanel'): - # ISel=self.selPanel.tabPanel.lbTab.GetSelections() - pass - - def onShowLoaderMenu(self, event=None): - #pos = (self.btLoaderMenu.GetPosition()[0], self.btLoaderMenu.GetPosition()[1] + self.btLoaderMenu.GetSize()[1]) - self.PopupMenu(self.loaderMenu) #, pos) - - - def mainFrameUpdateLayout(self, event=None): - if hasattr(self,'selPanel'): - nWind=self.selPanel.splitter.nWindows - if self.Size[0]<=800: - sash=SIDE_COL[nWind] - else: - sash=SIDE_COL_LARGE[nWind] - self.resizeSideColumn(sash) - - def OnResizeWindow(self, event): - try: - self.mainFrameUpdateLayout() - self.Layout() - except: - pass - # NOTE: doesn't work... - #if hasattr(self,'plotPanel'): - # Subplot spacing changes based on figure size - #print('>>> RESIZE WINDOW') - #self.redraw() - - # --- Side column - def resizeSideColumn(self,width): - # To force the replot we do an epic unsplit/split... - #self.vSplitter.Unsplit() - #self.vSplitter.SplitVertically(self.selPanel, self.tSplitter) - self.vSplitter.SetMinimumPaneSize(width) - self.vSplitter.SetSashPosition(width) - #self.selPanel.splitter.setEquiSash() - - # --- NOTEBOOK - #def deletePages(self): - # for index in reversed(range(self.nb.GetPageCount())): - # self.nb.DeletePage(index) - # self.nb.SendSizeEvent() - # gc.collect() - #def on_tab_change(self, event=None): - # page_to_select = event.GetSelection() - # wx.CallAfter(self.fix_focus, page_to_select) - # event.Skip(True) - #def fix_focus(self, page_to_select): - # page = self.nb.GetPage(page_to_select) - # page.SetFocus() - -#---------------------------------------------------------------------- -def MyExceptionHook(etype, value, trace): - """ - Handler for all unhandled exceptions. - :param `etype`: the exception type (`SyntaxError`, `ZeroDivisionError`, etc...); - :type `etype`: `Exception` - :param string `value`: the exception error message; - :param string `trace`: the traceback header, if any (otherwise, it prints the - standard Python header: ``Traceback (most recent call last)``. - """ - from wx._core import wxAssertionError - # Printing exception - traceback.print_exception(etype, value, trace) - if etype==wxAssertionError: - if wx.Platform == '__WXMAC__': - # We skip these exceptions on macos (likely bitmap size 0) - return - # Then showing to user the last error - frame = wx.GetApp().GetTopWindow() - tmp = traceback.format_exception(etype, value, trace) - if tmp[-1].find('Exception: Error:')==0: - Error(frame,tmp[-1][18:]) - elif tmp[-1].find('Exception: Warn:')==0: - Warn(frame,tmp[-1][17:]) - else: - exception = 'The following exception occured:\n\n'+ tmp[-1] + '\n'+tmp[-2].strip() - Error(frame,exception) - try: - frame.Thaw() # Make sure any freeze event is stopped - except: - pass - -# --------------------------------------------------------------------------------} -# --- Tests -# --------------------------------------------------------------------------------{ -def test(filenames=None): - if filenames is not None: - app = wx.App(False) - frame = MainFrame() - frame.load_files(filenames,fileformats=None, bPlot=True) - return - -# --------------------------------------------------------------------------------} -# --- Wrapped WxApp -# --------------------------------------------------------------------------------{ -class MyWxApp(wx.App): - def __init__(self, redirect=False, filename=None): - try: - wx.App.__init__(self, redirect, filename) - except: - if wx.Platform == '__WXMAC__': - #msg = """This program needs access to the screen. - # Please run with 'pythonw', not 'python', and only when you are logged - # in on the main display of your Mac.""" - msg= """ -MacOS Error: - This program needs access to the screen. Please run with a - Framework build of python, and only when you are logged in - on the main display of your Mac. - -pyDatView help: - You see the error above because you are using a Mac and - the python executable you are using does not have access to - your screen. This is a Mac issue, not a pyDatView issue. - Instead of calling 'python pyDatView.py', you need to find - another python and do '/path/python pyDatView.py' - You can try './pythonmac pyDatView.py', a script provided - in this repository to detect the path (in some cases) - - You can find additional help in the file 'README.md'. - - For quick reference, here are some typical cases: - - Your python was installed with 'brew', then likely use - /usr/lib/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX; - - Your python is an anaconda python, use something like:; - /anaconda3/bin/python.app (NOTE: the '.app'! -""" - - elif wx.Platform == '__WXGTK__': - msg =""" -Error: - Unable to access the X Display, is $DISPLAY set properly? - -pyDatView help: - You are probably running this application on a server accessed via ssh. - Use `ssh -X` or `ssh -Y` to access the server. - Else, try setting up $DISPLAY before doing the ssh connection. -""" - else: - msg = 'Unable to create GUI' # TODO: more description is needed for wxMSW... - raise SystemExit(msg) - def InitLocale(self): - if sys.platform.startswith('win') and sys.version_info > (3,8): - # See Bug #128 - Issue with wxPython 4.1 on Windows - import locale - locale.setlocale(locale.LC_ALL, "C") - print('[INFO] Setting locale to C') - #self.SetAssertMode(wx.APP_ASSERT_SUPPRESS) # Try this - -# --------------------------------------------------------------------------------} -# --- Mains -# --------------------------------------------------------------------------------{ -def showApp(firstArg=None, dataframes=None, filenames=[], names=None): - """ - The main function to start the pyDatView GUI and loads - Call this function with: - - filenames : list of filenames or a single filename (string) - OR - - dataframes: list of dataframes or a single dataframe - - names: list of names to be used for the multiple dataframes - """ - app = MyWxApp(False) - frame = MainFrame() - # Optional first argument - if firstArg is not None: - if isinstance(firstArg,list): - if isinstance(firstArg[0],str): - filenames=firstArg - else: - dataframes=firstArg - elif isinstance(firstArg,str): - filenames=[firstArg] - elif isinstance(firstArg, pd.DataFrame): - dataframes=[firstArg] - # Load files or dataframe depending on interface - if (dataframes is not None) and (len(dataframes)>0): - if names is None: - names=['df{}'.format(i+1) for i in range(len(dataframes))] - frame.load_dfs(dataframes, names) - elif len(filenames)>0: - frame.load_files(filenames, fileformats=None, bPlot=True) - app.MainLoop() - -def cmdline(): - if len(sys.argv)>1: - pydatview(filename=sys.argv[1]) - else: - pydatview() +import numpy as np +import os.path +import sys +import traceback +import gc +try: + import pandas as pd +except: + print('') + print('') + print('Error: problem loading pandas package:') + print(' - Check if this package is installed ( e.g. type: `pip install pandas`)') + print(' - If you are using anaconda, try `conda update python.app`') + print(' - If none of the above work, contact the developer.') + print('') + print('') + sys.exit(-1) + #raise + + +# GUI +import wx +from .GUIPlotPanel import PlotPanel +from .GUISelectionPanel import SelectionPanel,SEL_MODES,SEL_MODES_ID +from .GUISelectionPanel import ColumnPopup,TablePopup +from .GUIInfoPanel import InfoPanel +from .GUIToolBox import GetKeyString, TBAddTool +from .Tables import TableList, Table +# Helper +from .common import * +from .GUICommon import * +import pydatview.io as weio # File Formats and File Readers +# Pluggins +from .plugins import dataPlugins +from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData + +# --------------------------------------------------------------------------------} +# --- GLOBAL +# --------------------------------------------------------------------------------{ +PROG_NAME='pyDatView' +PROG_VERSION='v0.3-local' +SIDE_COL = [160,160,300,420,530] +SIDE_COL_LARGE = [200,200,360,480,600] +BOT_PANL =85 +ISTAT = 0 # Index of Status bar where main status info is provided + +#matplotlib.rcParams['text.usetex'] = False +# matplotlib.rcParams['font.sans-serif'] = 'DejaVu Sans' +#matplotlib.rcParams['font.family'] = 'Arial' +#matplotlib.rcParams['font.sans-serif'] = 'Arial' +# matplotlib.rcParams['font.family'] = 'sans-serif' + + + + + +# --------------------------------------------------------------------------------} +# --- Drag and drop +# --------------------------------------------------------------------------------{ +# Implement File Drop Target class +class FileDropTarget(wx.FileDropTarget): + def __init__(self, parent): + wx.FileDropTarget.__init__(self) + self.parent = parent + def OnDropFiles(self, x, y, filenames): + filenames = [f for f in filenames if not os.path.isdir(f)] + filenames.sort() + if len(filenames)>0: + # If Ctrl is pressed we add + bAdd= wx.GetKeyState(wx.WXK_CONTROL); + iFormat=self.parent.comboFormats.GetSelection() + if iFormat==0: # auto-format + Format = None + else: + Format = self.parent.FILE_FORMATS[iFormat-1] + self.parent.load_files(filenames, fileformats=[Format]*len(filenames), bAdd=bAdd, bPlot=True) + return True + + +# --------------------------------------------------------------------------------} +# --- Loader Menu +# --------------------------------------------------------------------------------{ +class LoaderMenuPopup(wx.Menu): + def __init__(self, parent, data): + wx.Menu.__init__(self) + self.parent = parent + self.data = data + + # Populate menu + item = wx.MenuItem(self, -1, "Date format: dayfirst", kind=wx.ITEM_CHECK) + self.Append(item) + self.Bind(wx.EVT_MENU, lambda ev: self.setCheck(ev, 'dayfirst') ) + self.Check(item.GetId(), self.data['dayfirst']) # Checking the menu box + + def setCheck(self, event, label): + self.data['dayfirst'] = not self.data['dayfirst'] + + +# --------------------------------------------------------------------------------} +# --- Main Frame +# --------------------------------------------------------------------------------{ +class MainFrame(wx.Frame): + def __init__(self, data=None): + # Parent constructor + wx.Frame.__init__(self, None, -1, PROG_NAME+' '+PROG_VERSION) + # Hooking exceptions to display them to the user + sys.excepthook = MyExceptionHook + # --- Data + self.restore_formulas = [] + self.systemFontSize = self.GetFont().GetPointSize() + self.data = loadAppData(self) + self.tabList=TableList(options=self.data['loaderOptions']) + self.datareset = False + # Global variables... + setFontSize(self.data['fontSize']) + setMonoFontSize(self.data['monoFontSize']) + + # --- GUI + #font = self.GetFont() + #print(font.GetFamily(),font.GetStyle(),font.GetPointSize()) + #font.SetFamily(wx.FONTFAMILY_DEFAULT) + #font.SetFamily(wx.FONTFAMILY_MODERN) + #font.SetFamily(wx.FONTFAMILY_SWISS) + #font.SetPointSize(8) + #print(font.GetFamily(),font.GetStyle(),font.GetPointSize()) + #self.SetFont(font) + self.SetFont(getFont(self)) + # --- Menu + menuBar = wx.MenuBar() + + fileMenu = wx.Menu() + loadMenuItem = fileMenu.Append(wx.ID_NEW,"Open file" ,"Open file" ) + exptMenuItem = fileMenu.Append(-1 ,"Export table" ,"Export table" ) + saveMenuItem = fileMenu.Append(wx.ID_SAVE,"Save figure" ,"Save figure" ) + exitMenuItem = fileMenu.Append(wx.ID_EXIT, 'Quit', 'Quit application') + menuBar.Append(fileMenu, "&File") + self.Bind(wx.EVT_MENU,self.onExit ,exitMenuItem) + self.Bind(wx.EVT_MENU,self.onLoad ,loadMenuItem) + self.Bind(wx.EVT_MENU,self.onExport,exptMenuItem) + self.Bind(wx.EVT_MENU,self.onSave ,saveMenuItem) + + dataMenu = wx.Menu() + menuBar.Append(dataMenu, "&Data") + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e, 'Mask') , dataMenu.Append(wx.ID_ANY, 'Mask')) + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Outlier'), dataMenu.Append(wx.ID_ANY, 'Outliers removal')) + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Filter') , dataMenu.Append(wx.ID_ANY, 'Filter')) + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Resample') , dataMenu.Append(wx.ID_ANY, 'Resample')) + self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'FASTRadialAverage'), dataMenu.Append(wx.ID_ANY, 'FAST - Radial average')) + + # --- Data Plugins + for string, function, isPanel in dataPlugins: + self.Bind(wx.EVT_MENU, lambda e, s_loc=string: self.onDataPlugin(e, s_loc), dataMenu.Append(wx.ID_ANY, string)) + + toolMenu = wx.Menu() + menuBar.Append(toolMenu, "&Tools") + self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'CurveFitting'), toolMenu.Append(wx.ID_ANY, 'Curve fitting')) + self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'LogDec') , toolMenu.Append(wx.ID_ANY, 'Damping from decay')) + + helpMenu = wx.Menu() + aboutMenuItem = helpMenu.Append(wx.NewId(), 'About', 'About') + resetMenuItem = helpMenu.Append(wx.NewId(), 'Reset options', 'Rest options') + menuBar.Append(helpMenu, "&Help") + self.SetMenuBar(menuBar) + self.Bind(wx.EVT_MENU,self.onAbout, aboutMenuItem) + self.Bind(wx.EVT_MENU,self.onReset, resetMenuItem) + + + io_userpath = os.path.join(weio.defaultUserDataDir(), 'pydatview_io') + self.FILE_FORMATS, errors= weio.fileFormats(userpath=io_userpath, ignoreErrors=True, verbose=False) + if len(errors)>0: + for e in errors: + Warn(self, e) + + self.FILE_FORMATS_EXTENSIONS = [['.*']]+[f.extensions for f in self.FILE_FORMATS] + self.FILE_FORMATS_NAMES = ['auto (any supported file)'] + [f.name for f in self.FILE_FORMATS] + self.FILE_FORMATS_NAMEXT =['{} ({})'.format(n,','.join(e)) for n,e in zip(self.FILE_FORMATS_NAMES,self.FILE_FORMATS_EXTENSIONS)] + + # --- ToolBar + tb = self.CreateToolBar(wx.TB_HORIZONTAL|wx.TB_TEXT|wx.TB_HORZ_LAYOUT) + self.toolBar = tb + self.comboFormats = wx.ComboBox(tb, choices = self.FILE_FORMATS_NAMEXT, style=wx.CB_READONLY) + self.comboFormats.SetSelection(0) + self.comboMode = wx.ComboBox(tb, choices = SEL_MODES, style=wx.CB_READONLY) + self.comboMode.SetSelection(0) + self.Bind(wx.EVT_COMBOBOX, self.onModeChange, self.comboMode ) + self.Bind(wx.EVT_COMBOBOX, self.onFormatChange, self.comboFormats ) + tb.AddSeparator() + tb.AddControl( wx.StaticText(tb, -1, 'Mode: ' ) ) + tb.AddControl( self.comboMode ) + tb.AddStretchableSpace() + tb.AddControl( wx.StaticText(tb, -1, 'Format: ' ) ) + tb.AddControl(self.comboFormats ) + # Menu for loader options + self.btLoaderMenu = wx.Button(tb, wx.ID_ANY, CHAR['menu'], style=wx.BU_EXACTFIT) + tb.AddControl(self.btLoaderMenu) + self.loaderMenu = LoaderMenuPopup(tb, self.data['loaderOptions']) + tb.Bind(wx.EVT_BUTTON, self.onShowLoaderMenu, self.btLoaderMenu) + tb.AddSeparator() + TBAddTool(tb, "Open" , 'ART_FILE_OPEN', self.onLoad) + TBAddTool(tb, "Reload", 'ART_REDO' , self.onReload) + TBAddTool(tb, "Add" , 'ART_PLUS' , self.onAdd) + #bmp = wx.Bitmap('help.png') #wx.Bitmap("NEW.BMP", wx.BITMAP_TYPE_BMP) + #self.AddTBBitmapTool(tb,"Debug" ,wx.ArtProvider.GetBitmap(wx.ART_ERROR),self.onDEBUG) + tb.AddStretchableSpace() + tb.Realize() + + # --- Status bar + self.statusbar=self.CreateStatusBar(3, style=0) + self.statusbar.SetStatusWidths([150, -1, 70]) + + # --- Main Panel and Notebook + self.MainPanel = wx.Panel(self) + #self.MainPanel = wx.Panel(self, style=wx.RAISED_BORDER) + #self.MainPanel.SetBackgroundColour((200,0,0)) + + #self.nb = wx.Notebook(self.MainPanel) + #self.nb.Bind(wx.EVT_NOTEBOOK_PAGE_CHANGED, self.on_tab_change) + + + sizer = wx.BoxSizer() + #sizer.Add(self.nb, 1, flag=wx.EXPAND) + self.MainPanel.SetSizer(sizer) + + # --- Drag and drop + dd = FileDropTarget(self) + self.SetDropTarget(dd) + + # --- Main Frame (self) + self.FrameSizer = wx.BoxSizer(wx.VERTICAL) + slSep = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) + self.FrameSizer.Add(slSep ,0, flag=wx.EXPAND|wx.BOTTOM,border=0) + self.FrameSizer.Add(self.MainPanel,1, flag=wx.EXPAND,border=0) + self.SetSizer(self.FrameSizer) + + self.SetSize(self.data['windowSize']) + self.Center() + self.Show() + self.Bind(wx.EVT_SIZE, self.OnResizeWindow) + self.Bind(wx.EVT_CLOSE, self.onClose) + + # Shortcuts + idFilter=wx.NewId() + self.Bind(wx.EVT_MENU, self.onFilter, id=idFilter) + + accel_tbl = wx.AcceleratorTable( + [(wx.ACCEL_CTRL, ord('F'), idFilter )] + ) + self.SetAcceleratorTable(accel_tbl) + + def onFilter(self,event): + if hasattr(self,'selPanel'): + self.selPanel.colPanel1.tFilter.SetFocus() + event.Skip() + + def clean_memory(self,bReload=False): + #print('Clean memory') + # force Memory cleanup + self.tabList.clean() + if not bReload: + if hasattr(self,'selPanel'): + self.selPanel.clean_memory() + if hasattr(self,'infoPanel'): + self.infoPanel.clean() + if hasattr(self,'plotPanel'): + self.plotPanel.cleanPlot() + gc.collect() + + def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, bPlot=True): + """ load multiple files, only trigger the plot at the end """ + if bReload: + if hasattr(self,'selPanel'): + self.selPanel.saveSelection() # TODO move to tables + else: + self.statusbar.SetStatusText('Loading files...', ISTAT) + + # A function to update the status bar while we load files + statusFunction = lambda i: self.statusbar.SetStatusText('Loading files {}/{}'.format(i+1,len(filenames)), ISTAT) + + if not bAdd: + self.clean_memory(bReload=bReload) + + + if fileformats is None: + fileformats=[None]*len(filenames) + assert type(fileformats)==list, 'fileformats must be a list' + assert len(fileformats)==len(filenames), 'fileformats and filenames must have the same lengths' + + # Sorting files in alphabetical order in base_filenames order + base_filenames = [os.path.basename(f) for f in filenames] + I = np.argsort(base_filenames) + filenames = list(np.array(filenames)[I]) + fileformats = list(np.array(fileformats)[I]) + #filenames = [f for __, f in sorted(zip(base_filenames, filenames))] + + # Load the tables + newTabs, warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformats=fileformats, bAdd=bAdd, bReload=bReload, statusFunction=statusFunction) + if bReload: + # Restore formulas that were previously added + for tab in self.tabList: + if tab.raw_name in self.restore_formulas.keys(): + for f in self.restore_formulas[tab.raw_name]: + tab.addColumnByFormula(f['name'], f['formula'], f['pos']-1) + self.restore_formulas = {} + # Display warnings + for warn in warnList: + Warn(self,warn) + # Load tables into the GUI + if self.tabList.len()>0: + self.load_tabs_into_GUI(bReload=bReload, bAdd=bAdd, bPlot=bPlot) + + def load_df(self, df, name=None, bAdd=False, bPlot=True): + if bAdd: + self.tabList.append(Table(data=df, name=name)) + else: + self.tabList = TableList( [Table(data=df, name=name)] ) + self.load_tabs_into_GUI(bAdd=bAdd, bPlot=bPlot) + if hasattr(self,'selPanel'): + self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) + + def load_dfs(self, dfs, names, bAdd=False): + self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) + self.load_tabs_into_GUI(bAdd=bAdd, bPlot=True) + if hasattr(self,'selPanel'): + self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) + + def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): + if bAdd: + if not hasattr(self,'selPanel'): + bAdd=False + + if (not bReload) and (not bAdd): + self.cleanGUI() + if (bReload): + self.statusbar.SetStatusText('Done reloading.', ISTAT) + self.Freeze() + # Setting status bar + self.setStatusBar() + + if bReload or bAdd: + self.selPanel.update_tabs(self.tabList) + else: + # --- Create a selPanel, plotPanel and infoPanel + mode = SEL_MODES_ID[self.comboMode.GetSelection()] + #self.vSplitter = wx.SplitterWindow(self.nb) + self.vSplitter = wx.SplitterWindow(self.MainPanel) + self.selPanel = SelectionPanel(self.vSplitter, self.tabList, mode=mode, mainframe=self) + self.tSplitter = wx.SplitterWindow(self.vSplitter) + #self.tSplitter.SetMinimumPaneSize(20) + self.infoPanel = InfoPanel(self.tSplitter, data=self.data['infoPanel']) + self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, self, data=self.data['plotPanel']) + self.tSplitter.SetSashGravity(0.9) + self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) + self.tSplitter.SetMinimumPaneSize(BOT_PANL) + self.tSplitter.SetSashGravity(1) + self.tSplitter.SetSashPosition(400) + + self.vSplitter.SplitVertically(self.selPanel, self.tSplitter) + self.vSplitter.SetMinimumPaneSize(SIDE_COL[0]) + self.tSplitter.SetSashPosition(SIDE_COL[0]) + + #self.nb.AddPage(self.vSplitter, "Plot") + #self.nb.SendSizeEvent() + + sizer = self.MainPanel.GetSizer() + sizer.Add(self.vSplitter, 1, flag=wx.EXPAND,border=0) + self.MainPanel.SetSizer(sizer) + self.FrameSizer.Layout() + + + # --- Bind + # The selPanel does the binding, but the callback is stored here because it involves plotPanel... TODO, rethink it + #self.selPanel.bindColSelectionChange(self.onColSelectionChangeCallBack) + self.selPanel.setTabSelectionChangeCallback(self.onTabSelectionChangeTrigger) + self.selPanel.setRedrawCallback(self.redrawCallback) + self.Bind(wx.EVT_SPLITTER_SASH_POS_CHANGED, self.onSashChangeMain, self.vSplitter) + + # plot trigger + if bPlot: + self.mainFrameUpdateLayout() + self.onColSelectionChange(event=None) + try: + self.Thaw() + except: + pass + # Hack + #self.onShowTool(tool='Filter') + #self.onShowTool(tool='Resample') + #self.onDataPlugin(toolName='Bin data') + + def setStatusBar(self, ISel=None): + nTabs=self.tabList.len() + if ISel is None: + ISel = list(np.arange(nTabs)) + if nTabs<0: + self.statusbar.SetStatusText('', ISTAT) # Format + self.statusbar.SetStatusText('', ISTAT+1) # Filenames + self.statusbar.SetStatusText('', ISTAT+2) # Shape + elif nTabs==1: + self.statusbar.SetStatusText(self.tabList.get(0).fileformat_name, ISTAT+0) + self.statusbar.SetStatusText(self.tabList.get(0).filename , ISTAT+1) + self.statusbar.SetStatusText(self.tabList.get(0).shapestring , ISTAT+2) + elif len(ISel)==1: + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).fileformat_name , ISTAT+0) + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).filename , ISTAT+1) + self.statusbar.SetStatusText(self.tabList.get(ISel[0]).shapestring , ISTAT+2) + else: + self.statusbar.SetStatusText('{} tables loaded'.format(nTabs) ,ISTAT+0) + self.statusbar.SetStatusText(", ".join(list(set([self.tabList.filenames[i] for i in ISel]))),ISTAT+1) + self.statusbar.SetStatusText('' ,ISTAT+2) + + # --- Table Actions - TODO consider a table handler, or doing only the triggers + def renameTable(self, iTab, newName): + oldName = self.tabList.renameTable(iTab, newName) + self.selPanel.renameTable(iTab, oldName, newName) + + def sortTabs(self, method='byName'): + self.tabList.sort(method=method) + # Updating tables + self.selPanel.update_tabs(self.tabList) + # Trigger a replot + self.onTabSelectionChange() + + def mergeTabsTrigger(self): + if hasattr(self,'selPanel'): + # Select the newly created table + self.selPanel.tabPanel.lbTab.SetSelection(-1) # Empty selection + self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)-1) # Select new/last table + # Trigger a replot + self.onTabSelectionChange() + + def deleteTabs(self, I): + self.tabList.deleteTabs(I) + if len(self.tabList)==0: + self.cleanGUI() + return + + # Invalidating selections + self.selPanel.tabPanel.lbTab.SetSelection(-1) + # Until we have something better, we empty plot + self.plotPanel.empty() + self.infoPanel.empty() + self.selPanel.clean_memory() + # Updating tables + self.selPanel.update_tabs(self.tabList) + # Trigger a replot + self.onTabSelectionChange() + + + def exportTab(self, iTab): + tab=self.tabList.get(iTab) + default_filename=tab.basename +'.csv' + with wx.FileDialog(self, "Save to CSV file",defaultFile=default_filename, + style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as dlg: + #, wildcard="CSV files (*.csv)|*.csv", + dlg.CentreOnParent() + if dlg.ShowModal() == wx.ID_CANCEL: + return # the user changed their mind + tab.export(dlg.GetPath()) + + def onShowTool(self, event=None, tool=''): + """ + Show tool + tool in 'Outlier', 'Filter', 'LogDec','FASTRadialAverage', 'Mask', 'CurveFitting' + """ + if not hasattr(self,'plotPanel'): + Error(self,'Plot some data first') + return + self.plotPanel.showTool(tool) + + def onDataPlugin(self, event=None, toolName=''): + """ + Dispatcher to apply plugins to data: + - simple plugins are directly exectued + - plugins that are panels are sent over to plotPanel to show them + TODO merge with onShowTool + """ + if not hasattr(self,'plotPanel'): + Error(self,'Plot some data first') + return + + for thisToolName, function, isPanel in dataPlugins: + if toolName == thisToolName: + if isPanel: + panelClass = function(self, event, toolName) # getting panelClass + self.plotPanel.showToolPanel(panelClass) + else: + function(self, event, toolName) # calling the data function + return + raise NotImplementedError('Tool: ',toolName) + + + def onSashChangeMain(self, event=None): + pass + # doent work because size is not communicated yet + #if hasattr(self,'selPanel'): + # print('ON SASH') + # self.selPanel.setEquiSash(event) + + + def onTabSelectionChange(self, event=None): + # TODO get rid of me + self.selPanel.onTabSelectionChange() + + def onColSelectionChange(self, event=None): + # TODO get rid of me + self.selPanel.onColSelectionChange() + + def redraw(self): + # TODO get rid of me + self.redrawCallback() + + # --- CallBacks sent to panels + def onTabSelectionChangeTrigger(self, event=None): + # Update of status bar + ISel=self.selPanel.tabPanel.lbTab.GetSelections() + if len(ISel)>0: + self.setStatusBar(ISel) + + def onColSelectionChangeTrigger(self, event=None): + pass + + def redrawCallback(self): + if hasattr(self,'plotPanel'): + self.plotPanel.load_and_draw() + +# def showStats(self): +# self.infoPanel.showStats(self.plotPanel.plotData,self.plotPanel.pltTypePanel.plotType()) + + def onExit(self, event): + self.Close() + + def onClose(self, event): + saveAppData(self, self.data) + event.Skip() + + def cleanGUI(self, event=None): + if hasattr(self,'plotPanel'): + del self.plotPanel + if hasattr(self,'selPanel'): + del self.selPanel + if hasattr(self,'infoPanel'): + del self.infoPanel + #self.deletePages() + try: + self.MainPanel.GetSizer().Clear(delete_windows=True) # Delete Windows + except: + self.MainPanel.GetSizer().Clear() + self.FrameSizer.Layout() + gc.collect() + + def onSave(self, event=None): + # using the navigation toolbar save functionality + self.plotPanel.navTB.save_figure() + + def onAbout(self, event=None): + io_userpath = os.path.join(weio.defaultUserDataDir(), 'pydatview_io') + About(self,PROG_NAME+' '+PROG_VERSION+'\n\n' + 'pyDatView config file:\n {}\n'.format(configFilePath())+ + 'pyDatView io data directory:\n {}\n'.format(io_userpath)+ + '\n\nVisit http://github.com/ebranlard/pyDatView for documentation.') + + def onReset (self, event=None): + configFile = configFilePath() + result = YesNo(self, + 'The options of pyDatView will be reset to default.\nThe changes will be noticeable the next time you open pyDatView.\n\n'+ + 'This action will overwrite the user settings file:\n {}\n\n'.format(configFile)+ + 'pyDatView will then close.\n\n' + 'Are you sure you want to continue?', caption = 'Reset settings?') + if result: + try: + os.remove(configFile) + except: + pass + self.data = defaultAppData(self) + self.datareset = True + self.onExit(event=None) + + def onReload(self, event=None): + filenames, fileformats = self.tabList.filenames_and_formats + self.statusbar.SetStatusText('Reloading...', ISTAT) + if len(filenames)>0: + # If only one file, use the comboBox to decide which fileformat to use + if len(filenames)==1: + iFormat=self.comboFormats.GetSelection() + if iFormat==0: # auto-format + fileformats = [None] + else: + fileformats = [self.FILE_FORMATS[iFormat-1]] + + # Save formulas to restore them after reload with sorted tabs + self.restore_formulas = {} + for tab in self.tabList._tabs: + f = tab.formulas # list of dict('pos','formula','name') + f = sorted(f, key=lambda k: k['pos']) # Sort formulae by position in list of formua + self.restore_formulas[tab.raw_name]=f # we use raw_name as key + # Actually load files (read and add in GUI) + self.load_files(filenames, fileformats=fileformats, bReload=True, bAdd=False, bPlot=True) + else: + Error(self,'Open one or more file first.') + + def onDEBUG(self, event=None): + #self.clean_memory() + self.plotPanel.ctrlPanel.Refresh() + self.plotPanel.cb_sizer.ForceRefresh() + + def onExport(self, event=None): + ISel=[] + try: + ISel = self.selPanel.tabPanel.lbTab.GetSelections() + except: + pass + if len(ISel)>0: + self.exportTab(ISel[0]) + else: + Error(self,'Open a file and select a table first.') + + def onLoad(self, event=None): + self.selectFile(bAdd=False) + + def onAdd(self, event=None): + self.selectFile(bAdd=self.tabList.len()>0) + + def selectFile(self,bAdd=False): + # --- File Format extension + iFormat=self.comboFormats.GetSelection() + sFormat=self.comboFormats.GetStringSelection() + if iFormat==0: # auto-format + Format = None + #wildcard = 'all (*.*)|*.*' + wildcard='|'.join([n+'|*'+';*'.join(e) for n,e in zip(self.FILE_FORMATS_NAMEXT,self.FILE_FORMATS_EXTENSIONS)]) + #wildcard = sFormat + extensions+'|all (*.*)|*.*' + else: + Format = self.FILE_FORMATS[iFormat-1] + extensions = '|*'+';*'.join(self.FILE_FORMATS[iFormat-1].extensions) + wildcard = sFormat + extensions+'|all (*.*)|*.*' + + with wx.FileDialog(self, "Open file", wildcard=wildcard, + style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST | wx.FD_MULTIPLE) as dlg: + #other options: wx.CHANGE_DIR + #dlg.SetSize((100,100)) + #dlg.Center() + if dlg.ShowModal() == wx.ID_CANCEL: + return # the user changed their mind + filenames = dlg.GetPaths() + self.load_files(filenames,fileformats=[Format]*len(filenames),bAdd=bAdd, bPlot=True) + + def onModeChange(self, event=None): + if hasattr(self,'selPanel'): + self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) + self.mainFrameUpdateLayout() + # --- Trigger to check number of columns + self.onTabSelectionChange() + + def onFormatChange(self, event=None): + """ The user changed the format """ + #if hasattr(self,'selPanel'): + # ISel=self.selPanel.tabPanel.lbTab.GetSelections() + pass + + def onShowLoaderMenu(self, event=None): + #pos = (self.btLoaderMenu.GetPosition()[0], self.btLoaderMenu.GetPosition()[1] + self.btLoaderMenu.GetSize()[1]) + self.PopupMenu(self.loaderMenu) #, pos) + + + def mainFrameUpdateLayout(self, event=None): + if hasattr(self,'selPanel'): + nWind=self.selPanel.splitter.nWindows + if self.Size[0]<=800: + sash=SIDE_COL[nWind] + else: + sash=SIDE_COL_LARGE[nWind] + self.resizeSideColumn(sash) + + def OnResizeWindow(self, event): + try: + self.mainFrameUpdateLayout() + self.Layout() + except: + pass + # NOTE: doesn't work... + #if hasattr(self,'plotPanel'): + # Subplot spacing changes based on figure size + #print('>>> RESIZE WINDOW') + #self.redraw() + + # --- Side column + def resizeSideColumn(self,width): + # To force the replot we do an epic unsplit/split... + #self.vSplitter.Unsplit() + #self.vSplitter.SplitVertically(self.selPanel, self.tSplitter) + self.vSplitter.SetMinimumPaneSize(width) + self.vSplitter.SetSashPosition(width) + #self.selPanel.splitter.setEquiSash() + + # --- NOTEBOOK + #def deletePages(self): + # for index in reversed(range(self.nb.GetPageCount())): + # self.nb.DeletePage(index) + # self.nb.SendSizeEvent() + # gc.collect() + #def on_tab_change(self, event=None): + # page_to_select = event.GetSelection() + # wx.CallAfter(self.fix_focus, page_to_select) + # event.Skip(True) + #def fix_focus(self, page_to_select): + # page = self.nb.GetPage(page_to_select) + # page.SetFocus() + +#---------------------------------------------------------------------- +def MyExceptionHook(etype, value, trace): + """ + Handler for all unhandled exceptions. + :param `etype`: the exception type (`SyntaxError`, `ZeroDivisionError`, etc...); + :type `etype`: `Exception` + :param string `value`: the exception error message; + :param string `trace`: the traceback header, if any (otherwise, it prints the + standard Python header: ``Traceback (most recent call last)``. + """ + from wx._core import wxAssertionError + # Printing exception + traceback.print_exception(etype, value, trace) + if etype==wxAssertionError: + if wx.Platform == '__WXMAC__': + # We skip these exceptions on macos (likely bitmap size 0) + return + # Then showing to user the last error + frame = wx.GetApp().GetTopWindow() + tmp = traceback.format_exception(etype, value, trace) + if tmp[-1].find('Exception: Error:')==0: + Error(frame,tmp[-1][18:]) + elif tmp[-1].find('Exception: Warn:')==0: + Warn(frame,tmp[-1][17:]) + else: + exception = 'The following exception occured:\n\n'+ tmp[-1] + '\n'+tmp[-2].strip() + Error(frame,exception) + try: + frame.Thaw() # Make sure any freeze event is stopped + except: + pass + +# --------------------------------------------------------------------------------} +# --- Tests +# --------------------------------------------------------------------------------{ +def test(filenames=None): + if filenames is not None: + app = wx.App(False) + frame = MainFrame() + frame.load_files(filenames,fileformats=None, bPlot=True) + return + +# --------------------------------------------------------------------------------} +# --- Wrapped WxApp +# --------------------------------------------------------------------------------{ +class MyWxApp(wx.App): + def __init__(self, redirect=False, filename=None): + try: + wx.App.__init__(self, redirect, filename) + except: + if wx.Platform == '__WXMAC__': + #msg = """This program needs access to the screen. + # Please run with 'pythonw', not 'python', and only when you are logged + # in on the main display of your Mac.""" + msg= """ +MacOS Error: + This program needs access to the screen. Please run with a + Framework build of python, and only when you are logged in + on the main display of your Mac. + +pyDatView help: + You see the error above because you are using a Mac and + the python executable you are using does not have access to + your screen. This is a Mac issue, not a pyDatView issue. + Instead of calling 'python pyDatView.py', you need to find + another python and do '/path/python pyDatView.py' + You can try './pythonmac pyDatView.py', a script provided + in this repository to detect the path (in some cases) + + You can find additional help in the file 'README.md'. + + For quick reference, here are some typical cases: + - Your python was installed with 'brew', then likely use + /usr/lib/Cellar/python/XXXXX/Frameworks/python.framework/Versions/XXXX/bin/pythonXXX; + - Your python is an anaconda python, use something like:; + /anaconda3/bin/python.app (NOTE: the '.app'! +""" + + elif wx.Platform == '__WXGTK__': + msg =""" +Error: + Unable to access the X Display, is $DISPLAY set properly? + +pyDatView help: + You are probably running this application on a server accessed via ssh. + Use `ssh -X` or `ssh -Y` to access the server. + Else, try setting up $DISPLAY before doing the ssh connection. +""" + else: + msg = 'Unable to create GUI' # TODO: more description is needed for wxMSW... + raise SystemExit(msg) + def InitLocale(self): + if sys.platform.startswith('win') and sys.version_info > (3,8): + # See Bug #128 - Issue with wxPython 4.1 on Windows + import locale + locale.setlocale(locale.LC_ALL, "C") + print('[INFO] Setting locale to C') + #self.SetAssertMode(wx.APP_ASSERT_SUPPRESS) # Try this + +# --------------------------------------------------------------------------------} +# --- Mains +# --------------------------------------------------------------------------------{ +def showApp(firstArg=None, dataframes=None, filenames=[], names=None): + """ + The main function to start the pyDatView GUI and loads + Call this function with: + - filenames : list of filenames or a single filename (string) + OR + - dataframes: list of dataframes or a single dataframe + - names: list of names to be used for the multiple dataframes + """ + app = MyWxApp(False) + frame = MainFrame() + # Optional first argument + if firstArg is not None: + if isinstance(firstArg,list): + if isinstance(firstArg[0],str): + filenames=firstArg + else: + dataframes=firstArg + elif isinstance(firstArg,str): + filenames=[firstArg] + elif isinstance(firstArg, pd.DataFrame): + dataframes=[firstArg] + # Load files or dataframe depending on interface + if (dataframes is not None) and (len(dataframes)>0): + if names is None: + names=['df{}'.format(i+1) for i in range(len(dataframes))] + frame.load_dfs(dataframes, names) + elif len(filenames)>0: + frame.load_files(filenames, fileformats=None, bPlot=True) + app.MainLoop() + +def cmdline(): + if len(sys.argv)>1: + pydatview(filename=sys.argv[1]) + else: + pydatview() diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index edfe5c2..a2394b3 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -1,807 +1,813 @@ -import os -import numpy as np -from .common import no_unit, unit, inverse_unit, has_chinese_char -from .common import isString, isDate, getDt -from .common import unique, pretty_num, pretty_time -from .GUIMeasure import find_closest # Should not depend on wx - -class PlotData(): - """ - Class for plot data - - For now, relies on some "indices" related to Tables/Columns/ and maybe Selection panel - Not really elegant. These dependencies should be removed in the future - """ - def __init__(PD, x=None, y=None, sx='', sy=''): - """ Dummy init for now """ - PD.id=-1 - PD.it=-1 # tablx index - PD.ix=-1 # column index - PD.iy=-1 # column index - PD.sx='' # x label - PD.sy='' # y label - PD.st='' # table label - PD.syl='' # y label for legend - PD.filename = '' - PD.tabname = '' - PD.x =[] # x data - PD.y =[] # y data - PD.xIsString=False # true if strings - PD.xIsDate =False # true if dates - PD.yIsString=False # true if strings - PD.yIsDate =False # true if dates - - if x is not None and y is not None: - PD.fromXY(x,y,sx,sy) - - def fromIDs(PD, tabs, i, idx, SameCol, Options={}): - """ Nasty initialization of plot data from "IDs" """ - PD.id = i - PD.it = idx[0] # table index - PD.ix = idx[1] # x index - PD.iy = idx[2] # y index - PD.sx = idx[3] # x label - PD.sy = idx[4] # y label - PD.syl = '' # y label for legend - PD.st = idx[5] # table label - PD.filename = tabs[PD.it].filename - PD.tabname = tabs[PD.it].active_name - PD.SameCol = SameCol - PD.x, PD.xIsString, PD.xIsDate,_ = tabs[PD.it].getColumn(PD.ix) # actual x data, with info - PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info - PD.c =c # raw values, used by PDF - - PD._post_init(Options=Options) - - def fromXY(PD, x, y, sx='', sy=''): - PD.x = x - PD.y = y - PD.c = y - PD.sx = sx - PD.sy = sy - PD.xIsString = isString(x) - PD.yIsString = isString(y) - PD.xIsDate = isDate (x) - PD.yIsDate = isDate (y) - - PD._post_init() - - - def _post_init(PD, Options={}): - # --- Perform data manipulation on the fly - #[print(k,v) for k,v in Options.items()] - keys=Options.keys() - # TODO setup an "Order" - if 'RemoveOutliers' in keys: - if Options['RemoveOutliers']: - from pydatview.tools.signal_analysis import reject_outliers - try: - PD.x, PD.y = reject_outliers(PD.y, PD.x, m=Options['OutliersMedianDeviation']) - except: - raise Exception('Warn: Outlier removal failed. Desactivate it or use a different signal. ') - if 'Filter' in keys: - if Options['Filter']: - from pydatview.tools.signal_analysis import applyFilter - PD.y = applyFilter(PD.x, PD.y, Options['Filter']) - - if 'Sampler' in keys: - if Options['Sampler']: - from pydatview.tools.signal_analysis import applySampler - PD.x, PD.y = applySampler(PD.x, PD.y, Options['Sampler']) - - if 'Binning' in keys: - if Options['Binning']: - if Options['Binning']['active']: - PD.x, PD.y = Options['Binning']['applyCallBack'](PD.x, PD.y, Options['Binning']) - - # --- Store stats - n=len(PD.y) - if n>1000: - if (PD.xIsString): - raise Exception('Error: x values contain more than 1000 string. This is not suitable for plotting.\n\nPlease select another column for table: {}\nProblematic column: {}\n'.format(PD.st,PD.sx)) - if (PD.yIsString): - raise Exception('Error: y values contain more than 1000 string. This is not suitable for plotting.\n\nPlease select another column for table: {}\nProblematic column: {}\n'.format(PD.st,PD.sy)) - - PD.needChineseFont = has_chinese_char(PD.sy) or has_chinese_char(PD.sx) - # Stats of the raw data (computed once and for all, since it can be expensive for large dataset - PD.computeRange() - # Store the values of the original data (labelled "0"), since the data might be modified later by PDF or MinMax etc. - PD._y0Min = PD._yMin - PD._y0Max = PD._yMax - PD._x0Min = PD._xMin - PD._x0Max = PD._xMax - PD._x0AtYMin = PD._xAtYMin - PD._x0AtYMax = PD._xAtYMax - PD._y0Std = PD.yStd() - PD._y0Mean = PD.yMean() - PD._n0 = (n,'{:d}'.format(n)) - PD.x0 =PD.x - PD.y0 =PD.y - # Store xyMeas input values so we don't need to recompute xyMeas in case they didn't change - PD.xyMeasInput1, PD.xyMeasInput2 = None, None - PD.xyMeas1, PD.xyMeas2 = None, None - - def __repr__(s): - s1='id:{}, it:{}, ix:{}, iy:{}, sx:"{}", sy:"{}", st:{}, syl:{}\n'.format(s.id,s.it,s.ix,s.iy,s.sx,s.sy,s.st,s.syl) - return s1 - - def toPDF(PD, nBins=30, smooth=False): - """ Convert y-data to Probability density function (PDF) as function of x - Uses "stats" library (from welib/pybra) - NOTE: inPlace - """ - from pydatview.tools.stats import pdf_gaussian_kde, pdf_histogram - - n=len(PD.y) - if PD.yIsString: - if n>100: - raise Exception('Warn: Dataset has string format and is too large to display') - vc = PD.c.value_counts().sort_index() - PD.x = vc.keys().tolist() - PD.y = vc/n # TODO counts/PDF option - PD.yIsString=False - PD.xIsString=True - elif PD.yIsDate: - raise Exception('Warn: Cannot plot PDF of dates') - else: - if nBins>=n: - nBins=n - if smooth: - try: - PD.x, PD.y = pdf_gaussian_kde(PD.y, nOut=nBins) - except np.linalg.LinAlgError as e: - PD.x, PD.y = pdf_histogram(PD.y, nBins=nBins, norm=True, count=False) - else: - PD.x, PD.y = pdf_histogram(PD.y, nBins=nBins, norm=True, count=False) - PD.xIsString=False - PD.yIsString=False - - PD.sx = PD.sy; - PD.sy = 'PDF('+no_unit(PD.sy)+')' - iu = inverse_unit(PD.sy) - if len(iu)>0: - PD.sy += ' ['+ iu +']' - - # Compute min max once and for all - PD.computeRange() - - return nBins - - - def toMinMax(PD, xScale=False, yScale=True): - """ Convert plot data to MinMax data based on GUI options - NOTE: inPlace - """ - if yScale: - if PD.yIsString: - raise Exception('Warn: Cannot compute min-max for strings') - mi = PD._y0Min[0] #mi= np.nanmin(PD.y) - mx = PD._y0Max[0] #mx= np.nanmax(PD.y) - if mi == mx: - PD.y=PD.y*0 - else: - PD.y = (PD.y-mi)/(mx-mi) - PD._yMin=0,'0' - PD._yMax=1,'1' - if xScale: - if PD.xIsString: - raise Exception('Warn: Cannot compute min-max for strings') - mi= PD._x0Min[0] - mx= PD._x0Max[0] - if mi == mx: - PD.x=PD.x*0 - else: - PD.x = (PD.x-mi)/(mx-mi) - PD._xMin=0,'0' - PD._xMax=1,'1' - - # Compute min max once and for all - #PD.computeRange() - - return None - - - def toFFT(PD, yType='Amplitude', xType='1/x', avgMethod='Welch', avgWindow='Hamming', bDetrend=True, nExp=8, nPerDecade=10): - """ - Uses spectral.fft_wrap to generate a "FFT" plot data, with various options: - yType : amplitude, PSD, f x PSD - xType : 1/x, x, 2pi/x - avgMethod : None, Welch - avgWindow : Hamming, Hann, Rectangular - see module spectral for more - - NOTE: inplace (modifies itself), does not return a new instance - """ - from pydatview.tools.spectral import fft_wrap - - # --- TODO, make this independent of GUI - if PD.yIsString or PD.yIsDate: - raise Exception('Warn: Cannot plot FFT of dates or strings') - elif PD.xIsString: - raise Exception('Warn: Cannot plot FFT if x axis is string') - - dt=None - if PD.xIsDate: - dt = getDt(PD.x) - # --- Computing fft - x is freq, y is Amplitude - PD.x, PD.y, Info = fft_wrap(PD.x, PD.y, dt=dt, output_type=yType,averaging=avgMethod, averaging_window=avgWindow,detrend=bDetrend,nExp=nExp, nPerDecade=nPerDecade) - # --- Setting plot options - PD._Info=Info - PD.xIsDate=False - # y label - if yType=='PSD': - PD.sy= 'PSD({}) [({})^2/{}]'.format(no_unit(PD.sy), unit(PD.sy), unit(PD.sx)) - elif yType=='f x PSD': - PD.sy= 'f-weighted PSD({}) [({})^2]'.format(no_unit(PD.sy), unit(PD.sy)) - elif yType=='Amplitude': - PD.sy= 'FFT({}) [{}]'.format(no_unit(PD.sy), unit(PD.sy)) - else: - raise Exception('Unsupported FFT type {} '.format(yType)) - # x label - if xType=='1/x': - if unit(PD.sx)=='s': - PD.sx= 'Frequency [Hz]' - else: - PD.sx= '' - elif xType=='x': - PD.x=1/PD.x - if unit(PD.sx)=='s': - PD.sx= 'Period [s]' - else: - PD.sx= '' - elif xType=='2pi/x': - PD.x=2*np.pi*PD.x - if unit(PD.sx)=='s': - PD.sx= 'Cyclic frequency [rad/s]' - else: - PD.sx= '' - else: - raise Exception('Unsupported x-type {} '.format(xType)) - - PD.computeRange() - return Info - - def computeRange(PD): - """ Compute min max of data once and for all and store - From the performance tests, this ends up having a non negligible cost for large dataset, - so we store it to reuse these as much as possible. - If possible, should be used for the plotting as well, so that matplotlib don't - have to compute them again - NOTE: each variable is a tuple (v,s), with a float and its string representation - """ - PD._xMin = PD._xMinCalc() - PD._xMax = PD._xMaxCalc() - PD._yMin = PD._yMinCalc() - PD._yMax = PD._yMaxCalc() - PD._xAtYMin = PD._xAtYMinCalc(PD._yMin[0]) - PD._xAtYMax = PD._xAtYMaxCalc(PD._yMax[0]) - - - # --------------------------------------------------------------------------------} - # --- Stats functions that should only becalled once, could maybe use @attributes.. - # --------------------------------------------------------------------------------{ - def _yMinCalc(PD): - if PD.yIsString: - return PD.y[0],PD.y[0].strip() - elif PD.yIsDate: - return PD.y[0],'{}'.format(PD.y[0]) - else: - v=np.nanmin(PD.y) - s=pretty_num(v) - return (v,s) - - def _yMaxCalc(PD): - if PD.yIsString: - return PD.y[-1],PD.y[-1].strip() - elif PD.yIsDate: - return PD.y[-1],'{}'.format(PD.y[-1]) - else: - v=np.nanmax(PD.y) - s=pretty_num(v) - return (v,s) - - def _xAtYMinCalc(PD, yMin): - if PD.xIsString: - return PD.x[0],PD.x[0].strip() - elif PD.xIsDate: - return PD.x[0],'{}'.format(PD.x[0]) - else: - try: - v = PD.x[np.where(PD.y == yMin)[0][0]] # Might fail if all nan - except: - v = PD.x[0] - s=pretty_num(v) - return (v,s) - - def _xAtYMaxCalc(PD, yMax): - if PD.xIsString: - return PD.x[-1],PD.x[-1].strip() - elif PD.xIsDate: - return PD.x[-1],'{}'.format(PD.x[-1]) - else: - try: - v = PD.x[np.where(PD.y == yMax)[0][0]] # Might fail if all nan - except: - v = PD.x[0] - s=pretty_num(v) - return (v,s) - - def _xMinCalc(PD): - if PD.xIsString: - return PD.x[0],PD.x[0].strip() - elif PD.xIsDate: - return PD.x[0],'{}'.format(PD.x[0]) - else: - v=np.nanmin(PD.x) - s=pretty_num(v) - return (v,s) - - def _xMaxCalc(PD): - if PD.xIsString: - return PD.x[-1],PD.x[-1].strip() - elif PD.xIsDate: - return PD.x[-1],'{}'.format(PD.x[-1]) - else: - v=np.nanmax(PD.x) - s=pretty_num(v) - return (v,s) - - def xMin(PD): - return PD._xMin - - def xMax(PD): - return PD._xMax - - def xAtYMin(PD): - return PD._xAtYMin - - def xAtYMax(PD): - return PD._xAtYMax - - def yMin(PD): - return PD._yMin - - def yMax(PD): - return PD._yMax - - def y0Min(PD): - return PD._y0Min - - def y0Max(PD): - return PD._y0Max - - def y0Mean(PD): - return PD._y0Mean - - def y0Std(PD): - return PD._y0Std - - def n0(PD): - return PD._n0 - - # --------------------------------------------------------------------------------} - # --- Stats functions - # --------------------------------------------------------------------------------{ - def yMean(PD): - if PD.yIsString or PD.yIsDate: - return None,'NA' - else: - v=np.nanmean(PD.y) - s=pretty_num(v) - return (v,s) - - def yMedian(PD): - if PD.yIsString or PD.yIsDate: - return None,'NA' - else: - v=np.nanmedian(PD.y) - s=pretty_num(v) - return (v,s) - - def yStd(PD): - if PD.yIsString or PD.yIsDate: - return None,'NA' - else: - v=np.nanstd(PD.y) - s=pretty_num(v) - return (v,s) - - def yName(PD): - return PD.sy, PD.sy - - def fileName(PD): - return os.path.basename(PD.filename), os.path.basename(PD.filename) - - def baseDir(PD): - return os.path.dirname(PD.filename),os.path.join(os.path.dirname(PD.filename),'') - - def tabName(PD): - return PD.tabname, PD.tabname - - def ylen(PD): - v=len(PD.y) - s='{:d}'.format(v) - return v,s - - - def y0Var(PD): - if PD._y0Std[0] is not None: - v=PD._y0Std[0]**2 - s=pretty_num(v) - else: - v=None - s='NA' - return v,s - - def y0TI(PD): - v=PD._y0Std[0]/PD._y0Mean[0] - s=pretty_num(v) - return v,s - - - def yRange(PD): - if PD.yIsString: - return 'NA','NA' - elif PD.yIsDate: - dtAll=getDt([PD.x[-1]-PD.x[0]]) - return '',pretty_time(dtAll) - else: - v=np.nanmax(PD.y)-np.nanmin(PD.y) - s=pretty_num(v) - return v,s - - def yAbsMax(PD): - if PD.yIsString or PD.yIsDate: - return 'NA','NA' - else: - v=max(np.abs(PD._y0Min[0]),np.abs(PD._y0Max[0])) - s=pretty_num(v) - return v,s - - - def xRange(PD): - if PD.xIsString: - return 'NA','NA' - elif PD.xIsDate: - dtAll=getDt([PD.x[-1]-PD.x[0]]) - return '',pretty_time(dtAll) - else: - v=np.nanmax(PD.x)-np.nanmin(PD.x) - s=pretty_num(v) - return v,s - - - def inty(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y,x=PD.x) - s=pretty_num(v) - return v,s - - def intyintdx(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y,x=PD.x)/np.trapz(y=PD.x*0+1,x=PD.x) - s=pretty_num(v) - return v,s - - def intyx1(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y*PD.x,x=PD.x) - s=pretty_num(v) - return v,s - - def intyx1_scaled(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y*PD.x,x=PD.x) - v=v/np.trapz(y=PD.y,x=PD.x) - s=pretty_num(v) - return v,s - - def intyx2(PD): - if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: - return None,'NA' - else: - v=np.trapz(y=PD.y*PD.x**2,x=PD.x) - s=pretty_num(v) - return v,s - - def meas1(PD, xymeas1, xymeas2): - if PD.xyMeasInput1 is not None and PD.xyMeasInput1 == xymeas1: - yv = PD.xyMeas1[1] - s = pretty_num(yv) - else: - xv, yv, s = PD._meas(xymeas1) - PD.xyMeas1 = [xv, yv] - PD.xyMeasInput1 = xymeas1 - return yv, s - - def meas2(PD, xymeas1, xymeas2): - if PD.xyMeasInput2 is not None and PD.xyMeasInput2 == xymeas2: - yv = PD.xyMeas2[1] - s = pretty_num(yv) - else: - xv, yv, s = PD._meas(xymeas2) - PD.xyMeas2 = [xv, yv] - PD.xyMeasInput2 = xymeas2 - return yv, s - - def yMeanMeas(PD): - return PD._measCalc('mean') - - def yMinMeas(PD): - return PD._measCalc('min') - - def yMaxMeas(PD): - return PD._measCalc('max') - - def xAtYMinMeas(PD): - return PD._measCalc('xmin') - - def xAtYMaxMeas(PD): - return PD._measCalc('xmax') - - def _meas(PD, xymeas): - try: - xv, yv = 'NA', 'NA' - xy = np.array([PD.x, PD.y]).transpose() - points = find_closest(xy, [xymeas[0], xymeas[1]], False) - if points.ndim == 1: - xv, yv = points[0:2] - s = pretty_num(yv) - else: - xv, yv = points[0, 0], points[0, 1] - s = ' / '.join([str(p) for p in points[:, 1]]) - except (IndexError, TypeError): - xv, yv = 'NA', 'NA' - s='NA' - return xv, yv, s - - def _measCalc(PD, mode): - if PD.xyMeas1 is None or PD.xyMeas2 is None: - return 'NA', 'NA' - try: - v = 'NA' - left_index = np.where(PD.x == PD.xyMeas1[0])[0][0] - right_index = np.where(PD.x == PD.xyMeas2[0])[0][0] - if left_index == right_index: - raise IndexError - if left_index > right_index: - left_index, right_index = right_index, left_index - if mode == 'mean': - v = np.nanmean(PD.y[left_index:right_index]) - elif mode == 'min': - v = np.nanmin(PD.y[left_index:right_index]) - elif mode == 'max': - v = np.nanmax(PD.y[left_index:right_index]) - elif mode == 'xmin': - v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmin(PD.y[left_index:right_index]))[0][0]] - elif mode == 'xmax': - v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmax(PD.y[left_index:right_index]))[0][0]] - else: - raise NotImplementedError('Error: Mode ' + mode + ' not implemented') - s = pretty_num(v) - except (IndexError, TypeError): - v = 'NA' - s = 'NA' - return v, s - - def dx(PD): - if len(PD.x)<=1: - return 'NA','NA' - if PD.xIsString: - return None,'NA' - elif PD.xIsDate: - dt=getDt(PD.x) - return dt,pretty_time(dt) - else: - v=PD.x[1]-PD.x[0] - s=pretty_num(v) - return v,s - - def xMax(PD): - if PD.xIsString: - return PD.x[-1],PD.x[-1] - elif PD.xIsDate: - return PD.x[-1],'{}'.format(PD.x[-1]) - else: - v=np.nanmax(PD.x) - s=pretty_num(v) - return v,s - def xMin(PD): - if PD.xIsString: - return PD.x[0],PD.x[0] - elif PD.xIsDate: - return PD.x[0],'{}'.format(PD.x[0]) - else: - v=np.nanmin(PD.x) - s=pretty_num(v) - return v,s - - def leq(PD,m): - from pydatview.tools.fatigue import eq_load - if PD.yIsString or PD.yIsDate: - return 'NA','NA' - else: - T,_=PD.xRange() - v=eq_load(PD.y, m=m, neq=T)[0][0] - return v,pretty_num(v) - - def Info(PD,var): - if var=='LSeg': - return '','{:d}'.format(PD._Info.LSeg) - elif var=='LWin': - return '','{:d}'.format(PD._Info.LWin) - elif var=='LOvlp': - return '','{:d}'.format(PD._Info.LOvlp) - elif var=='nFFT': - return '','{:d}'.format(PD._Info.nFFT) - - -# --------------------------------------------------------------------------------} -# --- -# --------------------------------------------------------------------------------{ -def compareMultiplePD(PD, mode, sComp): - """ - PD: list of PlotData - sComp: string in ['Relative', '|Relative|', 'Ratio', 'Absolute' - mode: plot mode, nTabs_1Col, nTabs_SameCols, nTabs_SimCols - - return: - PD_comp : new PlotData list that compares the input list PD - - """ - # --- Helper function - def getError(y,yref,method): - if len(y)!=len(yref): - raise NotImplementedError('Cannot compare signals of different lengths') - if sComp=='Relative': - if np.mean(np.abs(yref))<1e-7: - Error=(y-yRef)/(yRef+1)*100 - else: - Error=(y-yRef)/yRef*100 - elif sComp=='|Relative|': - if np.mean(np.abs(yref))<1e-7: - Error=abs((y-yRef)/(yRef+1))*100 - else: - Error=abs((y-yRef)/yRef)*100 - elif sComp=='Ratio': - if np.mean(np.abs(yref))<1e-7: - Error=(y+1)/(yRef+1) - else: - Error=y/yRef - elif sComp=='Absolute': - Error=y-yRef - else: - raise Exception('Something wrong '+sComp) - return Error - - def getErrorLabel(ylab=''): - if len(ylab)>0: - ylab=no_unit(ylab) - ylab='in '+ylab+' ' - if sComp=='Relative': - return 'Relative error '+ylab+'[%]'; - elif sComp=='|Relative|': - return 'Abs. relative error '+ylab+'[%]'; - if sComp=='Ratio': - return 'Ratio '+ylab.replace('in','of')+'[-]'; - elif sComp=='Absolute': - usy = unique([pd.sy for pd in PD]) - yunits= unique([unit(sy) for sy in usy]) - if len(yunits)==1 and len(yunits[0])>0: - return 'Absolute error '+ylab+'['+yunits[0]+']' - else: - return 'Absolute error '+ylab; - elif sComp=='Y-Y': - return PD[0].sy - - xlabelAll=PD[0].sx - - - if any([pd.yIsString for pd in PD]): - raise Exception('Warn: Cannot compare strings') - if any([pd.yIsDate for pd in PD]): - raise Exception('Warn: Cannot compare dates with other values') - - if mode=='nTabs_1Col': - ylabelAll=getErrorLabel(PD[1].sy) - usy = unique([pd.sy for pd in PD]) - #print('Compare - different tabs - 1 col') - st = [pd.st for pd in PD] - if len(usy)==1: - SS=usy[0] + ', '+ ' wrt. '.join(st[::-1]) - if sComp=='Y-Y': - xlabelAll=PD[0].st+', '+PD[0].sy - ylabelAll=PD[1].st+', '+PD[1].sy - else: - SS=' wrt. '.join(usy[::-1]) - if sComp=='Y-Y': - xlabelAll=PD[0].sy - ylabelAll=PD[1].sy - - xRef = PD[0].x - yRef = PD[0].y - PD[1].syl=SS - y=np.interp(xRef,PD[1].x,PD[1].y) - if sComp=='Y-Y': - PD[1].x=yRef - PD[1].y=y - else: - Error = getError(y,yRef,sComp) - PD[1].x=xRef - PD[1].y=Error - PD[1].sx=xlabelAll - PD[1].sy=ylabelAll - PD_comp=[PD[1]] # return - - elif mode=='1Tab_nCols': - # --- Compare one table - different columns - #print('One Tab, different columns') - ylabelAll=getErrorLabel() - xRef = PD[0].x - yRef = PD[0].y - pdRef=PD[0] - for pd in PD[1:]: - if sComp=='Y-Y': - pd.syl = no_unit(pd.sy)+' wrt. '+no_unit(pdRef.sy) - pd.x = yRef - pd.sx = PD[0].sy - else: - pd.syl = no_unit(pd.sy)+' wrt. '+no_unit(pdRef.sy) - pd.sx = xlabelAll - pd.sy = ylabelAll - Error = getError(pd.y,yRef,sComp) - pd.x=xRef - pd.y=Error - PD_comp=PD[1:] - elif mode =='nTabs_SameCols': - # --- Compare different tables, same column - #print('Several Tabs, same columns') - uiy=unique([pd.iy for pd in PD]) - uit=unique([pd.it for pd in PD]) - PD_comp=[] - for iy in uiy: - PD_SameCol=[pd for pd in PD if pd.iy==iy] - xRef = PD_SameCol[0].x - yRef = PD_SameCol[0].y - ylabelAll=getErrorLabel(PD_SameCol[0].sy) - for pd in PD_SameCol[1:]: - if pd.xIsString: - if len(xRef)==len(pd.x): - pass # fine able to interpolate - else: - raise Exception('X values have different length and are strings, cannot interpolate string. Use `Index` for x instead.') - else: - pd.y=np.interp(xRef,pd.x,pd.y) - if sComp=='Y-Y': - pd.x=yRef - pd.sx=PD_SameCol[0].st+', '+PD_SameCol[0].sy - if len(PD_SameCol)==1: - pd.sy =pd.st+', '+pd.sy - else: - pd.syl= pd.st - else: - if len(uit)<=2: - pd.syl = pd.st+' wrt. '+PD_SameCol[0].st+', '+pd.sy - else: - pd.syl = pd.st+'|'+pd.sy - pd.sx = xlabelAll - pd.sy = ylabelAll - Error = getError(pd.y,yRef,sComp) - pd.x=xRef - pd.y=Error - PD_comp.append(pd) - elif mode =='nTabs_SimCols': - # --- Compare different tables, similar columns - print('Several Tabs, similar columns, TODO') - PD_comp=[] - - return PD_comp - +import os +import numpy as np +from .common import no_unit, unit, inverse_unit, has_chinese_char +from .common import isString, isDate, getDt +from .common import unique, pretty_num, pretty_time +from .GUIMeasure import find_closest # Should not depend on wx + +class PlotData(): + """ + Class for plot data + + For now, relies on some "indices" related to Tables/Columns/ and maybe Selection panel + Not really elegant. These dependencies should be removed in the future + """ + def __init__(PD, x=None, y=None, sx='', sy=''): + """ Dummy init for now """ + PD.id=-1 + PD.it=-1 # tablx index + PD.ix=-1 # column index + PD.iy=-1 # column index + PD.sx='' # x label + PD.sy='' # y label + PD.st='' # table label + PD.syl='' # y label for legend + PD.filename = '' + PD.tabname = '' + PD.x =[] # x data + PD.y =[] # y data + PD.xIsString=False # true if strings + PD.xIsDate =False # true if dates + PD.yIsString=False # true if strings + PD.yIsDate =False # true if dates + + if x is not None and y is not None: + PD.fromXY(x,y,sx,sy) + + def fromIDs(PD, tabs, i, idx, SameCol, Options={}): + """ Nasty initialization of plot data from "IDs" """ + PD.id = i + PD.it = idx[0] # table index + PD.ix = idx[1] # x index + PD.iy = idx[2] # y index + PD.sx = idx[3] # x label + PD.sy = idx[4] # y label + PD.syl = '' # y label for legend + PD.st = idx[5] # table label + PD.filename = tabs[PD.it].filename + PD.tabname = tabs[PD.it].active_name + PD.SameCol = SameCol + PD.x, PD.xIsString, PD.xIsDate,_ = tabs[PD.it].getColumn(PD.ix) # actual x data, with info + PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info + PD.c =c # raw values, used by PDF + + PD._post_init(Options=Options) + + def fromXY(PD, x, y, sx='', sy=''): + PD.x = x + PD.y = y + PD.c = y + PD.sx = sx + PD.sy = sy + PD.xIsString = isString(x) + PD.yIsString = isString(y) + PD.xIsDate = isDate (x) + PD.yIsDate = isDate (y) + + PD._post_init() + + + def _post_init(PD, Options={}): + # --- Perform data manipulation on the fly + #[print(k,v) for k,v in Options.items()] + keys=Options.keys() + # TODO setup an "Order" + if 'RemoveOutliers' in keys: + if Options['RemoveOutliers']: + from pydatview.tools.signal_analysis import reject_outliers + try: + PD.x, PD.y = reject_outliers(PD.y, PD.x, m=Options['OutliersMedianDeviation']) + except: + raise Exception('Warn: Outlier removal failed. Desactivate it or use a different signal. ') + if 'Filter' in keys: + if Options['Filter']: + from pydatview.tools.signal_analysis import applyFilter + PD.y = applyFilter(PD.x, PD.y, Options['Filter']) + + if 'Sampler' in keys: + if Options['Sampler']: + from pydatview.tools.signal_analysis import applySampler + PD.x, PD.y = applySampler(PD.x, PD.y, Options['Sampler']) + + if 'Binning' in keys: + if Options['Binning']: + if Options['Binning']['active']: + PD.x, PD.y = Options['Binning']['applyCallBack'](PD.x, PD.y, Options['Binning']) + + # --- Store stats + n=len(PD.y) + if n>1000: + if (PD.xIsString): + raise Exception('Error: x values contain more than 1000 string. This is not suitable for plotting.\n\nPlease select another column for table: {}\nProblematic column: {}\n'.format(PD.st,PD.sx)) + if (PD.yIsString): + raise Exception('Error: y values contain more than 1000 string. This is not suitable for plotting.\n\nPlease select another column for table: {}\nProblematic column: {}\n'.format(PD.st,PD.sy)) + + PD.needChineseFont = has_chinese_char(PD.sy) or has_chinese_char(PD.sx) + # Stats of the raw data (computed once and for all, since it can be expensive for large dataset + PD.computeRange() + # Store the values of the original data (labelled "0"), since the data might be modified later by PDF or MinMax etc. + PD._y0Min = PD._yMin + PD._y0Max = PD._yMax + PD._x0Min = PD._xMin + PD._x0Max = PD._xMax + PD._x0AtYMin = PD._xAtYMin + PD._x0AtYMax = PD._xAtYMax + PD._y0Std = PD.yStd() + PD._y0Mean = PD.yMean() + PD._n0 = (n,'{:d}'.format(n)) + PD.x0 =PD.x + PD.y0 =PD.y + # Store xyMeas input values so we don't need to recompute xyMeas in case they didn't change + PD.xyMeasInput1, PD.xyMeasInput2 = None, None + PD.xyMeas1, PD.xyMeas2 = None, None + + def __repr__(s): + s1='id:{}, it:{}, ix:{}, iy:{}, sx:"{}", sy:"{}", st:{}, syl:{}\n'.format(s.id,s.it,s.ix,s.iy,s.sx,s.sy,s.st,s.syl) + return s1 + + def toPDF(PD, nBins=30, smooth=False): + """ Convert y-data to Probability density function (PDF) as function of x + Uses "stats" library (from welib/pybra) + NOTE: inPlace + """ + from pydatview.tools.stats import pdf_gaussian_kde, pdf_histogram + + n=len(PD.y) + if PD.yIsString: + if n>100: + raise Exception('Warn: Dataset has string format and is too large to display') + vc = PD.c.value_counts().sort_index() + PD.x = vc.keys().tolist() + PD.y = vc/n # TODO counts/PDF option + PD.yIsString=False + PD.xIsString=True + elif PD.yIsDate: + raise Exception('Warn: Cannot plot PDF of dates') + else: + if nBins>=n: + nBins=n + if smooth: + try: + PD.x, PD.y = pdf_gaussian_kde(PD.y, nOut=nBins) + except np.linalg.LinAlgError as e: + PD.x, PD.y = pdf_histogram(PD.y, nBins=nBins, norm=True, count=False) + else: + PD.x, PD.y = pdf_histogram(PD.y, nBins=nBins, norm=True, count=False) + PD.xIsString=False + PD.yIsString=False + + PD.sx = PD.sy; + PD.sy = 'PDF('+no_unit(PD.sy)+')' + iu = inverse_unit(PD.sy) + if len(iu)>0: + PD.sy += ' ['+ iu +']' + + # Compute min max once and for all + PD.computeRange() + + return nBins + + + def toMinMax(PD, xScale=False, yScale=True): + """ Convert plot data to MinMax data based on GUI options + NOTE: inPlace + """ + if yScale: + if PD.yIsString: + raise Exception('Warn: Cannot compute min-max for strings') + mi = PD._y0Min[0] #mi= np.nanmin(PD.y) + mx = PD._y0Max[0] #mx= np.nanmax(PD.y) + if mi == mx: + PD.y=PD.y*0 + else: + PD.y = (PD.y-mi)/(mx-mi) + PD._yMin=0,'0' + PD._yMax=1,'1' + if xScale: + if PD.xIsString: + raise Exception('Warn: Cannot compute min-max for strings') + mi= PD._x0Min[0] + mx= PD._x0Max[0] + if mi == mx: + PD.x=PD.x*0 + else: + PD.x = (PD.x-mi)/(mx-mi) + PD._xMin=0,'0' + PD._xMax=1,'1' + + # Compute min max once and for all + #PD.computeRange() + + return None + + + def toFFT(PD, yType='Amplitude', xType='1/x', avgMethod='Welch', avgWindow='Hamming', bDetrend=True, nExp=8, nPerDecade=10): + """ + Uses spectral.fft_wrap to generate a "FFT" plot data, with various options: + yType : amplitude, PSD, f x PSD + xType : 1/x, x, 2pi/x + avgMethod : None, Welch + avgWindow : Hamming, Hann, Rectangular + see module spectral for more + + NOTE: inplace (modifies itself), does not return a new instance + """ + from pydatview.tools.spectral import fft_wrap + + # --- TODO, make this independent of GUI + if PD.yIsString or PD.yIsDate: + raise Exception('Warn: Cannot plot FFT of dates or strings') + elif PD.xIsString: + raise Exception('Warn: Cannot plot FFT if x axis is string') + + dt=None + if PD.xIsDate: + dt = getDt(PD.x) + # --- Computing fft - x is freq, y is Amplitude + PD.x, PD.y, Info = fft_wrap(PD.x, PD.y, dt=dt, output_type=yType,averaging=avgMethod, averaging_window=avgWindow,detrend=bDetrend,nExp=nExp, nPerDecade=nPerDecade) + # --- Setting plot options + PD._Info=Info + PD.xIsDate=False + # y label + if yType=='PSD': + PD.sy= 'PSD({}) [({})^2/{}]'.format(no_unit(PD.sy), unit(PD.sy), unit(PD.sx)) + elif yType=='f x PSD': + PD.sy= 'f-weighted PSD({}) [({})^2]'.format(no_unit(PD.sy), unit(PD.sy)) + elif yType=='Amplitude': + PD.sy= 'FFT({}) [{}]'.format(no_unit(PD.sy), unit(PD.sy)) + else: + raise Exception('Unsupported FFT type {} '.format(yType)) + # x label + if xType=='1/x': + if unit(PD.sx)=='s': + PD.sx= 'Frequency [Hz]' + else: + PD.sx= '' + elif xType=='x': + PD.x=1/PD.x + if unit(PD.sx)=='s': + PD.sx= 'Period [s]' + else: + PD.sx= '' + elif xType=='2pi/x': + PD.x=2*np.pi*PD.x + if unit(PD.sx)=='s': + PD.sx= 'Cyclic frequency [rad/s]' + else: + PD.sx= '' + else: + raise Exception('Unsupported x-type {} '.format(xType)) + + PD.computeRange() + return Info + + def computeRange(PD): + """ Compute min max of data once and for all and store + From the performance tests, this ends up having a non negligible cost for large dataset, + so we store it to reuse these as much as possible. + If possible, should be used for the plotting as well, so that matplotlib don't + have to compute them again + NOTE: each variable is a tuple (v,s), with a float and its string representation + """ + PD._xMin = PD._xMinCalc() + PD._xMax = PD._xMaxCalc() + PD._yMin = PD._yMinCalc() + PD._yMax = PD._yMaxCalc() + PD._xAtYMin = PD._xAtYMinCalc(PD._yMin[0]) + PD._xAtYMax = PD._xAtYMaxCalc(PD._yMax[0]) + + + # --------------------------------------------------------------------------------} + # --- Stats functions that should only becalled once, could maybe use @attributes.. + # --------------------------------------------------------------------------------{ + def _yMinCalc(PD): + if PD.yIsString: + return PD.y[0],PD.y[0].strip() + elif PD.yIsDate: + return PD.y[0],'{}'.format(PD.y[0]) + else: + v=np.nanmin(PD.y) + s=pretty_num(v) + return (v,s) + + def _yMaxCalc(PD): + if PD.yIsString: + return PD.y[-1],PD.y[-1].strip() + elif PD.yIsDate: + return PD.y[-1],'{}'.format(PD.y[-1]) + else: + v=np.nanmax(PD.y) + s=pretty_num(v) + return (v,s) + + def _xAtYMinCalc(PD, yMin): + if PD.xIsString: + return PD.x[0],PD.x[0].strip() + elif PD.xIsDate: + return PD.x[0],'{}'.format(PD.x[0]) + else: + try: + v = PD.x[np.where(PD.y == yMin)[0][0]] # Might fail if all nan + except: + v = PD.x[0] + s=pretty_num(v) + return (v,s) + + def _xAtYMaxCalc(PD, yMax): + if PD.xIsString: + return PD.x[-1],PD.x[-1].strip() + elif PD.xIsDate: + return PD.x[-1],'{}'.format(PD.x[-1]) + else: + try: + v = PD.x[np.where(PD.y == yMax)[0][0]] # Might fail if all nan + except: + v = PD.x[0] + s=pretty_num(v) + return (v,s) + + def _xMinCalc(PD): + if PD.xIsString: + return PD.x[0],PD.x[0].strip() + elif PD.xIsDate: + return PD.x[0],'{}'.format(PD.x[0]) + else: + v=np.nanmin(PD.x) + s=pretty_num(v) + return (v,s) + + def _xMaxCalc(PD): + if PD.xIsString: + return PD.x[-1],PD.x[-1].strip() + elif PD.xIsDate: + return PD.x[-1],'{}'.format(PD.x[-1]) + else: + v=np.nanmax(PD.x) + s=pretty_num(v) + return (v,s) + + def xMin(PD): + return PD._xMin + + def xMax(PD): + return PD._xMax + + def xAtYMin(PD): + return PD._xAtYMin + + def xAtYMax(PD): + return PD._xAtYMax + + def yMin(PD): + return PD._yMin + + def yMax(PD): + return PD._yMax + + def y0Min(PD): + return PD._y0Min + + def y0Max(PD): + return PD._y0Max + + def y0Mean(PD): + return PD._y0Mean + + def y0Std(PD): + return PD._y0Std + + def n0(PD): + return PD._n0 + + # --------------------------------------------------------------------------------} + # --- Stats functions + # --------------------------------------------------------------------------------{ + def yMean(PD): + if PD.yIsString or PD.yIsDate: + return None,'NA' + else: + v=np.nanmean(PD.y) + s=pretty_num(v) + return (v,s) + + def yMedian(PD): + if PD.yIsString or PD.yIsDate: + return None,'NA' + else: + v=np.nanmedian(PD.y) + s=pretty_num(v) + return (v,s) + + def yStd(PD): + if PD.yIsString or PD.yIsDate: + return None,'NA' + else: + v=np.nanstd(PD.y) + s=pretty_num(v) + return (v,s) + + def yName(PD): + return PD.sy, PD.sy + + def fileName(PD): + return os.path.basename(PD.filename), os.path.basename(PD.filename) + + def baseDir(PD): + return os.path.dirname(PD.filename),os.path.join(os.path.dirname(PD.filename),'') + + def tabName(PD): + return PD.tabname, PD.tabname + + def ylen(PD): + v=len(PD.y) + s='{:d}'.format(v) + return v,s + + + def y0Var(PD): + if PD._y0Std[0] is not None: + v=PD._y0Std[0]**2 + s=pretty_num(v) + else: + v=None + s='NA' + return v,s + + def y0TI(PD): + v=PD._y0Std[0]/PD._y0Mean[0] + s=pretty_num(v) + return v,s + + + def yRange(PD): + if PD.yIsString: + return 'NA','NA' + elif PD.yIsDate: + dtAll=getDt([PD.x[-1]-PD.x[0]]) + return '',pretty_time(dtAll) + else: + v=np.nanmax(PD.y)-np.nanmin(PD.y) + s=pretty_num(v) + return v,s + + def yAbsMax(PD): + if PD.yIsString or PD.yIsDate: + return 'NA','NA' + else: + v=max(np.abs(PD._y0Min[0]),np.abs(PD._y0Max[0])) + s=pretty_num(v) + return v,s + + + def xRange(PD): + if PD.xIsString: + return 'NA','NA' + elif PD.xIsDate: + dtAll=getDt([PD.x[-1]-PD.x[0]]) + return '',pretty_time(dtAll) + else: + v=np.nanmax(PD.x)-np.nanmin(PD.x) + s=pretty_num(v) + return v,s + + + def inty(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y,x=PD.x) + s=pretty_num(v) + return v,s + + def intyintdx(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y,x=PD.x)/np.trapz(y=PD.x*0+1,x=PD.x) + s=pretty_num(v) + return v,s + + def intyx1(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y*PD.x,x=PD.x) + s=pretty_num(v) + return v,s + + def intyx1_scaled(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y*PD.x,x=PD.x) + v=v/np.trapz(y=PD.y,x=PD.x) + s=pretty_num(v) + return v,s + + def intyx2(PD): + if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: + return None,'NA' + else: + v=np.trapz(y=PD.y*PD.x**2,x=PD.x) + s=pretty_num(v) + return v,s + + def meas1(PD, xymeas1, xymeas2): + if PD.xyMeasInput1 is not None and PD.xyMeasInput1 == xymeas1: + yv = PD.xyMeas1[1] + s = pretty_num(yv) + else: + xv, yv, s = PD._meas(xymeas1) + PD.xyMeas1 = [xv, yv] + PD.xyMeasInput1 = xymeas1 + return yv, s + + def meas2(PD, xymeas1, xymeas2): + if PD.xyMeasInput2 is not None and PD.xyMeasInput2 == xymeas2: + yv = PD.xyMeas2[1] + s = pretty_num(yv) + else: + xv, yv, s = PD._meas(xymeas2) + PD.xyMeas2 = [xv, yv] + PD.xyMeasInput2 = xymeas2 + return yv, s + + def yMeanMeas(PD): + return PD._measCalc('mean') + + def yMinMeas(PD): + return PD._measCalc('min') + + def yMaxMeas(PD): + return PD._measCalc('max') + + def xAtYMinMeas(PD): + return PD._measCalc('xmin') + + def xAtYMaxMeas(PD): + return PD._measCalc('xmax') + + def _meas(PD, xymeas): + try: + xv, yv = 'NA', 'NA' + xy = np.array([PD.x, PD.y]).transpose() + points = find_closest(xy, [xymeas[0], xymeas[1]], False) + if points.ndim == 1: + xv, yv = points[0:2] + s = pretty_num(yv) + else: + xv, yv = points[0, 0], points[0, 1] + s = ' / '.join([str(p) for p in points[:, 1]]) + except (IndexError, TypeError): + xv, yv = 'NA', 'NA' + s='NA' + return xv, yv, s + + def _measCalc(PD, mode): + if PD.xyMeas1 is None or PD.xyMeas2 is None: + return 'NA', 'NA' + try: + v = 'NA' + left_index = np.where(PD.x == PD.xyMeas1[0])[0][0] + right_index = np.where(PD.x == PD.xyMeas2[0])[0][0] + if left_index == right_index: + raise IndexError + if left_index > right_index: + left_index, right_index = right_index, left_index + if mode == 'mean': + v = np.nanmean(PD.y[left_index:right_index]) + elif mode == 'min': + v = np.nanmin(PD.y[left_index:right_index]) + elif mode == 'max': + v = np.nanmax(PD.y[left_index:right_index]) + elif mode == 'xmin': + v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmin(PD.y[left_index:right_index]))[0][0]] + elif mode == 'xmax': + v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmax(PD.y[left_index:right_index]))[0][0]] + else: + raise NotImplementedError('Error: Mode ' + mode + ' not implemented') + s = pretty_num(v) + except (IndexError, TypeError): + v = 'NA' + s = 'NA' + return v, s + + def dx(PD): + if len(PD.x)<=1: + return 'NA','NA' + if PD.xIsString: + return None,'NA' + elif PD.xIsDate: + dt=getDt(PD.x) + return dt,pretty_time(dt) + else: + v=PD.x[1]-PD.x[0] + s=pretty_num(v) + return v,s + + def xMax(PD): + if PD.xIsString: + return PD.x[-1],PD.x[-1] + elif PD.xIsDate: + return PD.x[-1],'{}'.format(PD.x[-1]) + else: + v=np.nanmax(PD.x) + s=pretty_num(v) + return v,s + def xMin(PD): + if PD.xIsString: + return PD.x[0],PD.x[0] + elif PD.xIsDate: + return PD.x[0],'{}'.format(PD.x[0]) + else: + v=np.nanmin(PD.x) + s=pretty_num(v) + return v,s + + def leq(PD,m): + from pydatview.tools.fatigue import eq_load + if PD.yIsString or PD.yIsDate: + return 'NA','NA' + else: + T,_=PD.xRange() + v=eq_load(PD.y, m=m, neq=T)[0][0] + return v,pretty_num(v) + + def Info(PD,var): + if var=='LSeg': + return '','{:d}'.format(PD._Info.LSeg) + elif var=='LWin': + return '','{:d}'.format(PD._Info.LWin) + elif var=='LOvlp': + return '','{:d}'.format(PD._Info.LOvlp) + elif var=='nFFT': + return '','{:d}'.format(PD._Info.nFFT) + + @staticmethod + def createDummy(n=30): + x = np.linspace(0,4*np.pi,n) + y = np.sin(x) + return PlotData(x=x, y=y, sx='time [s]', sy='Signal [m]') + + +# --------------------------------------------------------------------------------} +# --- +# --------------------------------------------------------------------------------{ +def compareMultiplePD(PD, mode, sComp): + """ + PD: list of PlotData + sComp: string in ['Relative', '|Relative|', 'Ratio', 'Absolute' + mode: plot mode, nTabs_1Col, nTabs_SameCols, nTabs_SimCols + + return: + PD_comp : new PlotData list that compares the input list PD + + """ + # --- Helper function + def getError(y,yref,method): + if len(y)!=len(yref): + raise NotImplementedError('Cannot compare signals of different lengths') + if sComp=='Relative': + if np.mean(np.abs(yref))<1e-7: + Error=(y-yRef)/(yRef+1)*100 + else: + Error=(y-yRef)/yRef*100 + elif sComp=='|Relative|': + if np.mean(np.abs(yref))<1e-7: + Error=abs((y-yRef)/(yRef+1))*100 + else: + Error=abs((y-yRef)/yRef)*100 + elif sComp=='Ratio': + if np.mean(np.abs(yref))<1e-7: + Error=(y+1)/(yRef+1) + else: + Error=y/yRef + elif sComp=='Absolute': + Error=y-yRef + else: + raise Exception('Something wrong '+sComp) + return Error + + def getErrorLabel(ylab=''): + if len(ylab)>0: + ylab=no_unit(ylab) + ylab='in '+ylab+' ' + if sComp=='Relative': + return 'Relative error '+ylab+'[%]'; + elif sComp=='|Relative|': + return 'Abs. relative error '+ylab+'[%]'; + if sComp=='Ratio': + return 'Ratio '+ylab.replace('in','of')+'[-]'; + elif sComp=='Absolute': + usy = unique([pd.sy for pd in PD]) + yunits= unique([unit(sy) for sy in usy]) + if len(yunits)==1 and len(yunits[0])>0: + return 'Absolute error '+ylab+'['+yunits[0]+']' + else: + return 'Absolute error '+ylab; + elif sComp=='Y-Y': + return PD[0].sy + + xlabelAll=PD[0].sx + + + if any([pd.yIsString for pd in PD]): + raise Exception('Warn: Cannot compare strings') + if any([pd.yIsDate for pd in PD]): + raise Exception('Warn: Cannot compare dates with other values') + + if mode=='nTabs_1Col': + ylabelAll=getErrorLabel(PD[1].sy) + usy = unique([pd.sy for pd in PD]) + #print('Compare - different tabs - 1 col') + st = [pd.st for pd in PD] + if len(usy)==1: + SS=usy[0] + ', '+ ' wrt. '.join(st[::-1]) + if sComp=='Y-Y': + xlabelAll=PD[0].st+', '+PD[0].sy + ylabelAll=PD[1].st+', '+PD[1].sy + else: + SS=' wrt. '.join(usy[::-1]) + if sComp=='Y-Y': + xlabelAll=PD[0].sy + ylabelAll=PD[1].sy + + xRef = PD[0].x + yRef = PD[0].y + PD[1].syl=SS + y=np.interp(xRef,PD[1].x,PD[1].y) + if sComp=='Y-Y': + PD[1].x=yRef + PD[1].y=y + else: + Error = getError(y,yRef,sComp) + PD[1].x=xRef + PD[1].y=Error + PD[1].sx=xlabelAll + PD[1].sy=ylabelAll + PD_comp=[PD[1]] # return + + elif mode=='1Tab_nCols': + # --- Compare one table - different columns + #print('One Tab, different columns') + ylabelAll=getErrorLabel() + xRef = PD[0].x + yRef = PD[0].y + pdRef=PD[0] + for pd in PD[1:]: + if sComp=='Y-Y': + pd.syl = no_unit(pd.sy)+' wrt. '+no_unit(pdRef.sy) + pd.x = yRef + pd.sx = PD[0].sy + else: + pd.syl = no_unit(pd.sy)+' wrt. '+no_unit(pdRef.sy) + pd.sx = xlabelAll + pd.sy = ylabelAll + Error = getError(pd.y,yRef,sComp) + pd.x=xRef + pd.y=Error + PD_comp=PD[1:] + elif mode =='nTabs_SameCols': + # --- Compare different tables, same column + #print('Several Tabs, same columns') + uiy=unique([pd.iy for pd in PD]) + uit=unique([pd.it for pd in PD]) + PD_comp=[] + for iy in uiy: + PD_SameCol=[pd for pd in PD if pd.iy==iy] + xRef = PD_SameCol[0].x + yRef = PD_SameCol[0].y + ylabelAll=getErrorLabel(PD_SameCol[0].sy) + for pd in PD_SameCol[1:]: + if pd.xIsString: + if len(xRef)==len(pd.x): + pass # fine able to interpolate + else: + raise Exception('X values have different length and are strings, cannot interpolate string. Use `Index` for x instead.') + else: + pd.y=np.interp(xRef,pd.x,pd.y) + if sComp=='Y-Y': + pd.x=yRef + pd.sx=PD_SameCol[0].st+', '+PD_SameCol[0].sy + if len(PD_SameCol)==1: + pd.sy =pd.st+', '+pd.sy + else: + pd.syl= pd.st + else: + if len(uit)<=2: + pd.syl = pd.st+' wrt. '+PD_SameCol[0].st+', '+pd.sy + else: + pd.syl = pd.st+'|'+pd.sy + pd.sx = xlabelAll + pd.sy = ylabelAll + Error = getError(pd.y,yRef,sComp) + pd.x=xRef + pd.y=Error + PD_comp.append(pd) + elif mode =='nTabs_SimCols': + # --- Compare different tables, similar columns + print('Several Tabs, similar columns, TODO') + PD_comp=[] + + return PD_comp + From 5bef8b0d3e0b24a0c515bfcf6e41f14b874717d2 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 21 Dec 2022 23:15:37 +0100 Subject: [PATCH 059/178] Starting pipeline class and GUI --- pydatview/GUIPipelinePanel.py | 232 ++++++++++++++++++++++++++++++++++ pydatview/appdata.py | 4 + pydatview/common.py | 12 ++ pydatview/main.py | 14 ++ pydatview/pipeline.py | 169 +++++++++++++++++++++++++ tests/test_pipeline.py | 52 ++++++++ 6 files changed, 483 insertions(+) create mode 100644 pydatview/GUIPipelinePanel.py create mode 100644 pydatview/pipeline.py create mode 100644 tests/test_pipeline.py diff --git a/pydatview/GUIPipelinePanel.py b/pydatview/GUIPipelinePanel.py new file mode 100644 index 0000000..acb5913 --- /dev/null +++ b/pydatview/GUIPipelinePanel.py @@ -0,0 +1,232 @@ +import wx + +from pydatview.common import CHAR, Info +import wx.lib.agw.hyperlink as hl + + + + +class ActionPanel(wx.Panel): + def __init__(self, parent, action, style=wx.TAB_TRAVERSAL): + wx.Panel.__init__(self, parent, -1, style=style) + #self.SetBackgroundColour((0,100,0)) + + # --- Data + self.action=action + name = action.name + + # --- GUI + ##lko = wx.Button(self, wx.ID_ANY, name, style=wx.BU_EXACTFIT) + lko = wx.StaticText(self, -1, name) + ##lko = hl.HyperLinkCtrl(self, -1, name) + ##lko.AutoBrowse(False) + ##lko.SetUnderlines(False, False, False) + ##lko.SetColours(wx.BLACK, wx.BLACK, wx.BLACK) + ##lko.DoPopup(False) + ###lko.SetBold(True) + ##lko.SetToolTip(wx.ToolTip('Change "'+name+'"')) + ##lko.UpdateLink() # To update text properties + + lkc = hl.HyperLinkCtrl(self, -1, 'x') + lkc.AutoBrowse(False) + lkc.EnableRollover(True) + lkc.SetColours(wx.BLACK, wx.BLACK, (200,0,0)) + lkc.DoPopup(False) + #lkc.SetBold(True) + lkc.SetToolTip(wx.ToolTip('Remove "'+name+'"')) + lkc.UpdateLink() # To update text properties + + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(lko, 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 0) + sizer.AddSpacer(3) + sizer.Add(lkc, 0, wx.ALIGN_LEFT|wx.ALIGN_TOP , 1) + sizer.AddSpacer(2) + + txt = wx.StaticText(self, -1, '>') + sizer.AddSpacer(3) + sizer.Add(txt, 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, 0) + sizer.AddSpacer(3) + + self.SetSizer(sizer) + + self.Bind(hl.EVT_HYPERLINK_LEFT, lambda ev: parent.onCloseAction(ev, action, self) , lkc) + + def __repr__(self): + s='\n'.format(self.action.name) + return s + +class ErrorPanel(wx.Panel): + def __init__(self, parent, pipeline, style=wx.TAB_TRAVERSAL): + wx.Panel.__init__(self, parent, -1, style=style) + #self.SetBackgroundColour((100,0,0)) + # --- Data + self.parent=parent + self.pipeline=pipeline + # --- GUI + lke = hl.HyperLinkCtrl(self, -1, 'Errors (0)') + lke.AutoBrowse(False) + lke.EnableRollover(True) + lke.SetColours(wx.BLACK, wx.BLACK, (200,0,0)) + lke.DoPopup(False) + #lkc.SetBold(True) + lke.SetToolTip(wx.ToolTip('View errors.')) + lke.UpdateLink() # To update text properties + self.lke=lke + + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(lke, 0, wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + self.sizer=sizer + self.SetSizer(sizer) + + self.Bind(hl.EVT_HYPERLINK_LEFT, self.showErrors, lke) + + self.update() + + def showErrors(self, event=None): + if len(self.pipeline.errorList)>0: + message='\n'.join(self.pipeline.errorList) + else: + message='No errors' + Info(self.parent, message, caption = 'Errors when applying the pipeline actions:') + + def update(self): + self.lke.SetLabel('Errors ({})'.format(len(self.pipeline.errorList))) + self.sizer.Layout() + + +class PipelinePanel(wx.Panel): + """ Display the pipeline of actions, allow user to edit it """ + + def __init__(self, parent, pipeline, style=wx.TAB_TRAVERSAL): + #style=wx.RAISED_BORDER + wx.Panel.__init__(self, parent, -1, style=style) + #self.SetBackgroundColour(wx.BLUE) + + # --- Data + self.parent = parent + self.pipeline = pipeline + self.actionPanels=[] + + # --- GUI + #self.btSave = wx.Button(self, wx.ID_ANY, 'Save', style=wx.BU_EXACTFIT) + #self.btLoad = wx.Button(self, wx.ID_ANY, 'Load', style=wx.BU_EXACTFIT) + #self.btClear = wx.Button(self, wx.ID_ANY, CHAR['sun']+' Clear', style=wx.BU_EXACTFIT) + txt = wx.StaticText(self, -1, "Pipeline:") + leftSizer = wx.BoxSizer(wx.HORIZONTAL) + #leftSizer.Add(self.btSave , 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL |wx.LEFT, border=2) + #leftSizer.Add(self.btLoad , 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL |wx.LEFT, border=2) + #leftSizer.Add(self.btClear, 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL |wx.LEFT, border=2) + leftSizer.Add(txt , 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, border=0) + + self.wrapSizer = wx.WrapSizer(orient=wx.HORIZONTAL) + + self.ep = ErrorPanel(self, self.pipeline) + + self.Sizer = wx.BoxSizer(wx.HORIZONTAL) + self.Sizer.Add(leftSizer , 0, wx.ALIGN_CENTER_VERTICAL|wx.LEFT , 1) + self.Sizer.Add(self.wrapSizer, 1, wx.ALIGN_CENTER_VERTICAL|wx.LEFT|wx.RIGHT, 1) + self.Sizer.Add(self.ep , 0, wx.ALIGN_CENTER_VERTICAL| wx.LEFT, 1) + + self.SetSizer(self.Sizer) + + self.populate() + + def populate(self): + # Delete everything in wrapSizer + self.wrapSizer.Clear(True) + self.wrapSizer.Add(wx.StaticText(self, -1, ' '), wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, 0) + + # Add all actions to panel + for ia, action in enumerate(self.pipeline.actions): + self._addPanel(action) + self.ep.update() + self.wrapSizer.Layout() + self.Sizer.Layout() + + def _addPanel(self, action): + ap = ActionPanel(self, action) + self.wrapSizer.Add(ap, 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, 0) + + def _deletePanel(self, action): + for child in self.wrapSizer.Children: + win = child.GetWindow() + if win is not None: + if hasattr(win,'action'): + if win.action==action: + actionPanel=win + self.wrapSizer.Hide(actionPanel) #actionPanel.Destroy() + self.wrapSizer.Layout() + self.Sizer.Layout() + + def onCloseAction(self, event, action=None, actionPanel=None): + self.remove(action) + + # --- Wrap the data class + def apply(self, tablist, force=False, applyToAll=False): + self.pipeline.apply(tablist, force=force, applyToAll=applyToAll) + + self.ep.update() + self.Sizer.Layout() + + def append(self, action): + # Delete action is already present and if it's a "unique" action + ac = self.pipeline.find(action.name) + if ac is not None: + if ac.unique: + print('>>> Deleting unique action before inserting it again', ac.name) + self.delete(ac, silent=True) + # Add to pipeline + print('>>> Adding action',action.name) + self.pipeline.append(action) + # Add to GUI + self._addPanel(action) + self.Sizer.Layout() + + def remove(self, action, silent=False): + """ NOTE: the action is only removed from the pipeline, not deleted. """ + print('>>> Deleting action',action.name) + # Remove From Data + self.pipeline.remove(action) + # Remove From GUI + self._deletePanel(action) + + if action.removeNeedReload: + if not silent: + Info(self.parent, 'A reload is required now that the action "{}" has been removed.'.format(action.name)) + # TODO trigger reload/reapply + # TODO trigger GUI update + self.ep.update() + + +if __name__ == '__main__': + """ """ + from pydatview.pipeline import Pipeline, Action, IrreversibleAction, PlotDataAction + + pl = Pipeline() + + app = wx.App(False) + self=wx.Frame(None,-1,"GUIPipelinePanel main") + + p = PipelinePanel(self, pl) + p.append(PlotDataAction('PlotData Do X')) + p.append(PlotDataAction('PlotData Do Y')) + p.append(Action('Change units')) + p.append(IrreversibleAction('Rename columns')) + + pl.errorList=['This is a first error','This is a second error'] + + p.populate() + + self.SetSize((800, 200)) + self.Center() + self.Show() + + + + #d ={'ColA': np.random.normal(0,1,100)+1,'ColB':np.random.normal(0,1,100)+2} + #df = pd.DataFrame(data=d) + #tab=Table(data=df) + #p.showStats(None,[tab],[0],[0,1],tab.columns,0,erase=True) + + app.MainLoop() + diff --git a/pydatview/appdata.py b/pydatview/appdata.py index cdaec7a..3b6a8e3 100644 --- a/pydatview/appdata.py +++ b/pydatview/appdata.py @@ -7,6 +7,7 @@ from .GUIPlotPanel import PlotPanel from .GUIInfoPanel import InfoPanel from .Tables import TableList +from .pipeline import Pipeline def configFilePath(): @@ -81,6 +82,8 @@ def saveAppData(mainFrame, data): mainFrame.infoPanel.saveData(data['infoPanel']) if hasattr(mainFrame, 'tablist'): mainFrame.tablist.saveOptions(data['loaderOptions']) + if hasattr(mainFrame, 'pipeline'): + mainFrame.pipeline.saveData(data['pipeline']) # --- Write config file configFile = configFilePath() @@ -104,6 +107,7 @@ def defaultAppData(mainframe): # Loader/Table data['loaderOptions'] = TableList.defaultOptions() # Pipeline + data['pipeline'] = Pipeline.defaultData() #SIDE_COL = [160,160,300,420,530] #SIDE_COL_LARGE = [200,200,360,480,600] #BOT_PANL =85 diff --git a/pydatview/common.py b/pydatview/common.py index 59f1340..c5fa2af 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -4,6 +4,7 @@ import platform import datetime import re +import inspect CHAR={ 'menu' : u'\u2630', @@ -438,3 +439,14 @@ def isString(x): def isDate(x): return np.issubdtype(x.dtype, np.datetime64) + + +# Create a Dummy Main Frame Class for testing purposes (e.g. of plugins) + +class DummyMainFrame(): + def __init__(self, parent): self.parent=parent + def addAction (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) + def removeAction (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) + def load_df (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) + def load_dfs (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) + def mainFrameUpdateLayout(self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) diff --git a/pydatview/main.py b/pydatview/main.py index 990fe0e..687b0de 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -24,6 +24,7 @@ from .GUISelectionPanel import SelectionPanel,SEL_MODES,SEL_MODES_ID from .GUISelectionPanel import ColumnPopup,TablePopup from .GUIInfoPanel import InfoPanel +from .GUIPipelinePanel import PipelinePanel from .GUIToolBox import GetKeyString, TBAddTool from .Tables import TableList, Table # Helper @@ -31,6 +32,7 @@ from .GUICommon import * import pydatview.io as weio # File Formats and File Readers # Pluggins +from .pipeline import Pipeline from .plugins import dataPlugins from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData @@ -110,6 +112,7 @@ def __init__(self, data=None): self.systemFontSize = self.GetFont().GetPointSize() self.data = loadAppData(self) self.tabList=TableList(options=self.data['loaderOptions']) + self.pipeline=Pipeline(data = self.data['pipeline']) self.datareset = False # Global variables... setFontSize(self.data['fontSize']) @@ -208,6 +211,9 @@ def __init__(self, data=None): self.statusbar=self.CreateStatusBar(3, style=0) self.statusbar.SetStatusWidths([150, -1, 70]) + # --- Pipeline + self.pipePanel = PipelinePanel(self, self.pipeline) + # --- Main Panel and Notebook self.MainPanel = wx.Panel(self) #self.MainPanel = wx.Panel(self, style=wx.RAISED_BORDER) @@ -230,6 +236,7 @@ def __init__(self, data=None): slSep = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) self.FrameSizer.Add(slSep ,0, flag=wx.EXPAND|wx.BOTTOM,border=0) self.FrameSizer.Add(self.MainPanel,1, flag=wx.EXPAND,border=0) + self.FrameSizer.Add(self.pipePanel,0, flag=wx.EXPAND,border=0) self.SetSizer(self.FrameSizer) self.SetSize(self.data['windowSize']) @@ -488,6 +495,13 @@ def onDataPlugin(self, event=None, toolName=''): return raise NotImplementedError('Tool: ',toolName) + # --- Pipeline + def addAction(self, action): + self.pipePanel.append(action) + def removeAction(self, action): + self.pipePanel.remove(action) + def applyPipeline(self, *args, **kwargs): + self.pipePanel.apply(*args, **kwargs) def onSashChangeMain(self, event=None): pass diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py new file mode 100644 index 0000000..b2f8484 --- /dev/null +++ b/pydatview/pipeline.py @@ -0,0 +1,169 @@ +""" +pipelines and actions +""" + +class Action(): + def __init__(self, name, + tableFunction=None, guiCallBack=None, + guiEditorClass=None, + data=None, + onPlotData=False, unique=True, removeNeedReload=False): + + self.name = name + # + self.tableFunction = tableFunction + + self.guiCallBack = guiCallBack + self.guiEditorClass = guiEditorClass # Class that can be used to edit this action + + self.data = data if data is not None else {} + + self.applied=False # TODO this needs to be sotred per table + + # Behavior + self.onPlotData = onPlotData + self.unique = unique + self.removeNeedReload = removeNeedReload + + self.errorList=[] + + def apply(self, tablist, force=False, applyToAll=False): + self.errorList=[] + if self.tableFunction is None: + raise Exception('tableFunction was not specified for action: {}'.format(self.name)) + + for t in tablist: + print('>>> Applying action', self.name, 'to', t.name) + try: + self.tableFunction(t) + except e: + err = 'Failed to apply action {} to table {}.'.format(self.name, t.name) + self.errorList.append(e) + + self.applied = True + return tablist + + def updateGUI(self): + if self.guiCallBack is not None: + print('>>> Calling GUI callback, action', self.name) + self.guiCallBack() + + def __repr__(self): + s=''.format(self.name) + return s + + +class PlotDataAction(Action): + def __init__(self, name, **kwargs): + Action.__init__(self, name, onPlotData=True, **kwargs) + + def applyOnPlotData(self): + print('>>> Apply On Plot Data') + +class IrreversibleAction(Action): + + def __init__(self, name, **kwargs): + Action.__init__(self, name, deleteNeedReload=True, **kwargs) + + def apply(self, tablist, force=False, applyToAll=False): + if force: + self.applied = False + if self.applied: + print('>>> Skipping irreversible action', self.name) + return + Action.apply(self, tablist) + + def __repr__(self): + s=''.format(self.name, self.applied) + return s + +class FilterAction(Action): + def cancel(self, tablist): + raise NotImplementedError() + return tablist + + +class Pipeline(object): + + def __init__(self, data=[]): + self.actionsData = [] + self.actionsPlot = [] + self.errorList = [] + + @property + def actions(self): + return self.actionsData+self.actionsPlot # order matters + + def apply(self, tablist, force=False, applyToAll=False): + """ + Apply the pipeline to the tablist + If "force", then actions that are "one time only" are still applied + If applyToAll, then the action is applied to all the tables, irrespectively of the tablist stored by the action + """ + for action in self.actionsData: + action.apply(tablist, force=force, applyToAll=applyToAll) + # + for action in self.actionsPlot: + action.apply(tablist, force=force, applyToAll=applyToAll) + # + self.collectErrors() + + def collectErrors(self): + self.errorList=[] + for action in self.actions: + self.errorList+= action.errorList + + # --- Behave like a list.. + def append(self, action): + if action.onPlotData: + self.actionsPlot.append(action) + else: + self.actionsData.append(action) + + def remove(self, a): + """ NOTE: the action is removed, not deleted fully (it might be readded to the pipeline later)""" + try: + i = self.actionsData.index(a) + a = self.actionsData.pop(i) + except ValueError: + i = self.actionsPlot.index(a) + a = self.actionsPlot.pop(i) + return a + + def find(self, name): + for action in self.actions: + if action.name==name: + return action + return None + + # --- Data/Options + def loadFromFile(self, filename): + pass + + def saveToFile(self, filename): + pass + + def loadData(self, data): + pass + + def saveData(self, data): + data['actionsData'] = {} + data['actionsPlot'] = {} + for ac in self.actionsData: + data['actionsData'][ac.name] = ac.data + for ac in self.actions: + data['actionsPlot'][ac.name] = ac.data + #data[] = self.Naming + + @staticmethod + def defaultData(): + return {} + + def __repr__(self): + s=': ' + s+=' > '.join([ac.name for ac in self.actionsData]) + s+=' + ' + s+=' > '.join([ac.name for ac in self.actionsPlot]) + return s + + diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..afa763d --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,52 @@ +import unittest +import numpy as np +#from pydatview.Tables import Table +import os +import matplotlib.pyplot as plt + + +from pydatview.Tables import TableList, Table +from pydatview.pipeline import Pipeline, Action +from pydatview.plotdata import PlotData + +class TestPipeline(unittest.TestCase): + + def test_pipeline(self): + pass + + +if __name__ == '__main__': + + from pydatview.plugins import getDataPluginsDict + + DPD = getDataPluginsDict() + + + tablist = TableList.createDummyList(1) + print(tablist._tabs[0].data) + + pipeline = Pipeline() + + action = DPD['Standardize Units (SI)']['callback']() + pipeline.append(action) + + print(pipeline) + + pipeline.apply(tablist) + + print(tablist._tabs[0].data) + + pipeline.apply(tablist) + print(tablist._tabs[0].data) + + + + action = DPD['Standardize Units (WE)']['callback']() + pipeline.append(action) + + pipeline.apply(tablist) + print(tablist._tabs[0].data) + + + pipeline.apply(tablist, force=True) + print(tablist._tabs[0].data) From 13d2fa9bde6ef43ef566ee5446d1556da39ec716 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 21 Dec 2022 23:46:13 +0100 Subject: [PATCH 060/178] PlotPanel/SelPanel: less dependency with mainframe --- pydatview/GUIPlotPanel.py | 51 +++++++---------------------- pydatview/GUISelectionPanel.py | 59 +++++++++++++++++++++------------- pydatview/main.py | 16 --------- 3 files changed, 48 insertions(+), 78 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 9a78f19..a251e66 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -376,7 +376,7 @@ def onFontOptionChange(self,event=None): class PlotPanel(wx.Panel): - def __init__(self, parent, selPanel, infoPanel=None, mainframe=None, data=None): + def __init__(self, parent, selPanel, infoPanel=None, data=None): # Superclass constructor super(PlotPanel,self).__init__(parent) @@ -407,7 +407,6 @@ def __init__(self, parent, selPanel, infoPanel=None, mainframe=None, data=None): if self.infoPanel is not None: self.infoPanel.setPlotMatrixCallbacks(self._onPlotMatrixLeftClick, self._onPlotMatrixRightClick) self.parent = parent - self.mainframe= mainframe self.plotData = [] self.plotDataOptions=dict() if data is not None: @@ -1468,56 +1467,28 @@ def _restore_limits(self): from Tables import Table,TableList from pydatview.Tables import TableList from pydatview.GUISelectionPanel import SelectionPanel - from pydatview.common import DummyMainFrame - tabList = TableList.createDummy(1) + # --- Data + tabList = TableList.createDummy(1) app = wx.App(False) self=wx.Frame(None,-1,"GUI Plot Panel Demo") - #self.SetBackgroundColour('red') - class FakeSelPanel(wx.Panel): - def __init__(self, parent): - super(FakeSelPanel,self).__init__(parent) - d ={'ColA': np.linspace(0,1,100)+1,'ColB': np.random.normal(0,1,100)+0,'ColC':np.random.normal(0,1,100)+1} - df = pd.DataFrame(data=d) - self.tabList=TableList([Table(data=df)]) - - def getPlotDataSelection(self): - ID=[] - ID.append([0,0,2,'x','ColB','tab']) - ID.append([0,0,3,'x','ColC','tab']) - return ID,True - - mainframe = DummyMainFrame(self) - # --- Selection Panel - #selpanel = FakeSelPanel(self) - selPanel = SelectionPanel(self, tabList, mode='auto', mainframe=mainframe) - # selpanel.SetBackgroundColour('blue') - self.selPanel=selPanel - - - # --- Plot Panel - plotPanel=PlotPanel(self, selPanel, data=None) - plotPanel.load_and_draw() - self.plotPanel = plotPanel + # --- Panels + self.selPanel = SelectionPanel(self, tabList, mode='auto') + self.plotPanel = PlotPanel(self, self.selPanel) + self.plotPanel.load_and_draw() # <<< Important # --- Binding the two - selPanel.setRedrawCallback(self.plotPanel.load_and_draw) + self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # --- Finalize GUI sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(selPanel ,0, flag = wx.EXPAND|wx.ALL,border = 10) - sizer.Add(plotPanel,1, flag = wx.EXPAND|wx.ALL,border = 10) + sizer.Add(self.selPanel ,0, flag = wx.EXPAND|wx.ALL,border = 5) + sizer.Add(self.plotPanel,1, flag = wx.EXPAND|wx.ALL,border = 5) self.SetSizer(sizer) - self.Center() - self.Layout() - self.SetSize((800, 600)) + self.SetSize((900, 600)) self.Show() - self.SendSizeEvent() - - #p1.showStats(None,[tab],[0],[0,1],tab.columns,0,erase=False) - app.MainLoop() diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 37ec1fa..f9934db 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -257,6 +257,7 @@ def onCancel(self, event): class TablePopup(wx.Menu): """ Popup Menu when right clicking on the table list """ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=False): + # TODO remove mainframe for TablePopup, handle most of it with Tables and some callbacks wx.Menu.__init__(self) self.parent = parent # parent is listbox self.tabPanel = tabPanel @@ -333,8 +334,12 @@ def OnMergeTabs(self, event): # Updating tables self.selPanel.update_tabs(self.tabList) # TODO select latest - if self.mainframe: - self.mainframe.mergeTabsTrigger() + # Select the newly created table + self.selPanel.tabPanel.lbTab.SetSelection(-1) # Empty selection + self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)-1) # Select new/last table + # Trigger a replot + self.selPanel.onTabSelectionChange() + def OnDeleteTabs(self, event): self.mainframe.deleteTabs(self.ISel) @@ -351,7 +356,11 @@ def OnExportTab(self, event): self.mainframe.exportTab(self.ISel[0]); def OnSort(self, event): - self.mainframe.sortTabs() + self.tabList.sort(method=method) + # Updating tables + self.update_tabs(self.tabList) + # Trigger a replot + self.onTabSelectionChange() class ColumnPopup(wx.Menu): """ Popup Menu when right clicking on the column list """ @@ -433,9 +442,9 @@ def OnDeleteColumn(self, event): # TODO adapt me for Sim. tables mode IFull = [self.parent.Filt2Full[iFilt] for iFilt in self.ISel] IFull = [iFull for iFull in IFull if iFull>=0] - if self.tabList.haveSameColumns(ITab): + if self.selPanel.tabList.haveSameColumns(ITab): for iTab,sTab in zip(ITab,STab): - self.tabList.get(iTab).deleteColumns(IFull) + self.selPanel.tabList.get(iTab).deleteColumns(IFull) else: self.parent.tab.deleteColumns(IFull) self.parent.setColumns() @@ -515,6 +524,7 @@ def showFormulaDialog(self, title, name='', formula='', edit=False): class TablePanel(wx.Panel): """ Display list of tables """ def __init__(self, parent, selPanel, mainframe, tabList): + # TODO get rid of mainframe # Superclass constructor super(TablePanel,self).__init__(parent) # DATA @@ -576,13 +586,12 @@ def empty(self): # --------------------------------------------------------------------------------{ class ColumnPanel(wx.Panel): """ A list of columns for x and y axis """ - def __init__(self, parent, selPanel, mainframe): + def __init__(self, parent, selPanel): # Superclass constructor super(ColumnPanel,self).__init__(parent) self.selPanel = selPanel; # Data self.tab=None - self.mainframe=mainframe self.columns=[] # All the columns available (may be different from the displayed ones) self.Filt2Full=None # Index of GUI columns in self.columns self.bShowID=False @@ -940,7 +949,6 @@ def __init__(self, parent, tabList, mode='auto', mainframe=None): # Superclass constructor super(SelectionPanel,self).__init__(parent) # DATA - self.mainframe = mainframe self.tabList = None self.itabForCol = None self.parent = parent @@ -953,17 +961,18 @@ def __init__(self, parent, tabList, mode='auto', mainframe=None): self.nSplits = -1 self.IKeepPerTab=None # Useful callBacks to be set by callee - self.redrawCallback=None - self.colSelectionChangeCallback=None - self.tabSelectionChangeCallback=None + self.redrawCallback = None # called when new data is selected + self.colSelectionChangeCallback = None # called after a column selection has changed + self.tabSelectionChangeCallback = None # called after a table selection has changed + self.updateLayoutCallback = None # called after the panel changes its Layout (TODO maybe knoledge of the parentt sizer is enough?) # GUI DATA self.splitter = MultiSplit(self, style=wx.SP_LIVE_UPDATE) self.splitter.SetMinimumPaneSize(70) self.tabPanel = TablePanel (self.splitter, self, mainframe, tabList) - self.colPanel1 = ColumnPanel(self.splitter, self, mainframe); - self.colPanel2 = ColumnPanel(self.splitter, self, mainframe); - self.colPanel3 = ColumnPanel(self.splitter, self, mainframe); + self.colPanel1 = ColumnPanel(self.splitter, self); + self.colPanel2 = ColumnPanel(self.splitter, self); + self.colPanel3 = ColumnPanel(self.splitter, self); self.tabPanel.Hide() self.colPanel1.Hide() self.colPanel2.Hide() @@ -997,10 +1006,18 @@ def setTabSelectionChangeCallback(self, callback): def setRedrawCallback(self, callBack): self.redrawCallback = callBack + def setUpdateLayoutCallback(self, callBack): + self.updateLayoutCallback = callBack + + # --- Important Signals def redraw(self): if self.redrawCallback is not None: self.redrawCallback() + def parentUpdateLayout(self): + if self.updateLayoutCallback is not None: + self.updateLayoutCallback() + def onColSelectionChange(self, event=None): # Letting selection panel handle the change self.colSelectionChanged() @@ -1082,8 +1099,7 @@ def sameColumnsMode(self): if self.tabList.len()>1: self.splitter.AppendWindow(self.tabPanel) self.splitter.AppendWindow(self.colPanel1) - if self.mainframe is not None: - self.mainframe.mainFrameUpdateLayout() + self.parentUpdateLayout() if self.tabList is not None: if self.tabList.len()<=1: self.nSplits=0 @@ -1099,8 +1115,8 @@ def simColumnsMode(self): self.splitter.AppendWindow(self.colPanel2) self.splitter.AppendWindow(self.colPanel1) self.splitter.setEquiSash() - if self.nSplits<2 and self.mainframe is not None: - self.mainframe.mainFrameUpdateLayout() + if self.nSplits<2: + self.parentUpdateLayout() self.nSplits=2 def twoColumnsMode(self): @@ -1112,8 +1128,8 @@ def twoColumnsMode(self): self.splitter.AppendWindow(self.colPanel2) self.splitter.AppendWindow(self.colPanel1) self.splitter.setEquiSash() - if self.nSplits<2 and self.mainframe is not None: - self.mainframe.mainFrameUpdateLayout() + if self.nSplits<2: + self.parentUpdateLayout() self.nSplits=2 def threeColumnsMode(self): @@ -1126,8 +1142,7 @@ def threeColumnsMode(self): self.splitter.AppendWindow(self.colPanel2) self.splitter.AppendWindow(self.colPanel1) self.splitter.setEquiSash() - if self.mainframe is not None: - self.mainframe.mainFrameUpdateLayout() + self.parentUpdateLayout() self.nSplits=3 def setTables(self,tabList,update=False): diff --git a/pydatview/main.py b/pydatview/main.py index 687b0de..789eb6d 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -420,21 +420,6 @@ def renameTable(self, iTab, newName): oldName = self.tabList.renameTable(iTab, newName) self.selPanel.renameTable(iTab, oldName, newName) - def sortTabs(self, method='byName'): - self.tabList.sort(method=method) - # Updating tables - self.selPanel.update_tabs(self.tabList) - # Trigger a replot - self.onTabSelectionChange() - - def mergeTabsTrigger(self): - if hasattr(self,'selPanel'): - # Select the newly created table - self.selPanel.tabPanel.lbTab.SetSelection(-1) # Empty selection - self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)-1) # Select new/last table - # Trigger a replot - self.onTabSelectionChange() - def deleteTabs(self, I): self.tabList.deleteTabs(I) if len(self.tabList)==0: @@ -452,7 +437,6 @@ def deleteTabs(self, I): # Trigger a replot self.onTabSelectionChange() - def exportTab(self, iTab): tab=self.tabList.get(iTab) default_filename=tab.basename +'.csv' From 757e36fdb07e24c7864d10aa27ada713f0f12e5d Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 00:32:37 +0100 Subject: [PATCH 061/178] PlotPanel: adding callback to addTables... --- pydatview/GUIPlotPanel.py | 39 ++++++-- pydatview/GUITools.py | 30 +++--- pydatview/Tables.py | 25 ++--- pydatview/common.py | 1 - pydatview/main.py | 48 ++++++---- pydatview/pipeline.py | 2 + pydatview/plugins/__init__.py | 35 +++++-- pydatview/plugins/data_binning.py | 105 +++++++++++++++++---- pydatview/plugins/data_standardizeUnits.py | 31 ++++-- tests/prof_all.py | 2 +- 10 files changed, 226 insertions(+), 92 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index a251e66..700d34a 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -422,7 +422,9 @@ def __init__(self, parent, selPanel, infoPanel=None, data=None): self.rightMeasure = GUIMeasure(2, 'darkgreen') self.xlim_prev = [[0, 1]] self.ylim_prev = [[0, 1]] - # GUI + self.addTablesCallback = None + + # --- GUI self.fig = Figure(facecolor="white", figsize=(1, 1)) register_matplotlib_converters() self.canvas = FigureCanvas(self, -1, self.fig) @@ -570,6 +572,16 @@ def __init__(self, parent, selPanel, infoPanel=None, data=None): self.plotsizer=plotsizer; self.set_subplot_spacing(init=True) + # --- Bindings/callback + def setAddTablesCallback(self, callback): + self.addTablesCallback = callback + + def addTables(self, *args, **kwargs): + if self.addTablesCallback is not None: + self.addTablesCallback(*args, **kwargs) + else: + print('[WARN] callback to add tables to parent was not set.') + # --- GUI DATA def saveData(self, data): @@ -779,19 +791,30 @@ def removeTools(self,event=None,Layout=True): if Layout: self.plotsizer.Layout() - def showTool(self,toolName=''): + def showTool(self, toolName=''): from .GUITools import TOOLS if toolName in TOOLS.keys(): - self.showToolPanel(TOOLS[toolName]) + self.showToolPanel(panelClass=TOOLS[toolName]) else: raise Exception('Unknown tool {}'.format(toolName)) - def showToolPanel(self, action): + def showToolAction(self, action): + """ Show a tool panel based on an action""" + self.showToolPanel(panelClass=action.guiEditorClass, action=action) + + + def showToolPanel(self, panelClass=None, panel=None, action=None): """ Show a tool panel based on a panel class (should inherit from GUIToolPanel)""" - panelClass = action.guiEditorClass self.Freeze() self.removeTools(Layout=False) - self.toolPanel=panelClass(parent=self, action=action) # calling the panel constructor + if panel is not None: + self.toolPanel=panel # use the panel directly + else: + if action is None: + print('NOTE: calling a panel without action') + self.toolPanel=panelClass(parent=self) # calling the panel constructor + else: + self.toolPanel=panelClass(parent=self, action=action) # calling the panel constructor self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) self.plotsizer.Layout() self.Thaw() @@ -1477,9 +1500,7 @@ def _restore_limits(self): self.selPanel = SelectionPanel(self, tabList, mode='auto') self.plotPanel = PlotPanel(self, self.selPanel) self.plotPanel.load_and_draw() # <<< Important - - # --- Binding the two - self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) + self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # Binding the two # --- Finalize GUI sizer = wx.BoxSizer(wx.HORIZONTAL) diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index c653ccf..8eb4e67 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -341,16 +341,15 @@ def onToggleApply(self, event=None, init=False): def onAdd(self,event=None): iSel = self.cbTabs.GetSelection() tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe icol, colname = self.parent.selPanel.xCol opt = self._GUI2Data() errors=[] if iSel==0: dfs, names, errors = tabList.applyFiltering(icol, opt, bAdd=True) - mainframe.load_dfs(dfs,names,bAdd=True) + self.parent.addTables(dfs,names,bAdd=True) else: df, name = tabList.get(iSel-1).applyFiltering(icol, opt, bAdd=True) - mainframe.load_df(df,name,bAdd=True) + self.parent.addTables([df], [name], bAdd=True) self.updateTabList() if len(errors)>0: @@ -598,17 +597,15 @@ def onToggleApply(self, event=None, init=False): def onAdd(self,event=None): iSel = self.cbTabs.GetSelection() tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe icol, colname = self.parent.selPanel.xCol - print(icol,colname) opt = self._GUI2Data() errors=[] if iSel==0: dfs, names, errors = tabList.applyResampling(icol, opt, bAdd=True) - mainframe.load_dfs(dfs,names,bAdd=True) + self.parent.addTables(dfs,names,bAdd=True) else: df, name = tabList.get(iSel-1).applyResampling(icol, opt, bAdd=True) - mainframe.load_df(df,name,bAdd=True) + self.parent.addTables([df],[name], bAdd=True) self.updateTabList() if len(errors)>0: @@ -774,13 +771,12 @@ def guessMask(self,tabList): def onClear(self,event=None): iSel = self.cbTabs.GetSelection() tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe if iSel==0: tabList.clearCommonMask() else: tabList.get(iSel-1).clearMask() - mainframe.redraw() + self.parent.load_and_draw() self.onTabChange() def onParamChangeAndPressEnter(self, event=None): @@ -806,21 +802,20 @@ def onApply(self,event=None,bAdd=True): maskString = self.textMask.GetLineText(0) iSel = self.cbTabs.GetSelection() tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe if iSel==0: dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=bAdd) if bAdd: - mainframe.load_dfs(dfs,names,bAdd=bAdd) + self.parent.addTables(dfs,names,bAdd=bAdd) else: - mainframe.redraw() + self.parent.load_and_draw() if len(errors)>0: raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) else: dfs, name = tabList.get(iSel-1).applyMaskString(maskString, bAdd=bAdd) if bAdd: - mainframe.load_df(df,name,bAdd=bAdd) + self.parent.addTables([df],[name], bAdd=bAdd) else: - mainframe.redraw() + self.parent.load_and_draw() self.updateTabList() # We stop applying @@ -897,15 +892,14 @@ def onApply(self,event=None): iSel = self.cbTabs.GetSelection() avgMethod = AVG_METHODS[self.cbMethod.GetSelection()] tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe if iSel==0: dfs, names, errors = tabList.radialAvg(avgMethod,avgParam) - mainframe.load_dfs(dfs,names,bAdd=True) + self.parent.addTables(dfs,names,bAdd=True) if len(errors)>0: raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) else: dfs, names = tabList.get(iSel-1).radialAvg(avgMethod,avgParam) - mainframe.load_dfs(dfs,names,bAdd=True) + self.parent.addTables([dfs],[names], bAdd=True) self.updateTabList() @@ -1148,7 +1142,7 @@ def onAdd(self,event=None): name='model_fit' if self.x is not None and self.y_fit is not None: df=pd.DataFrame({self.sx:self.x, self.sy:self.y_fit}) - self.parent.mainframe.load_df(df,name,bAdd=True) + self.parent.addTables([df], [name], bAdd=True) def onHelp(self,event=None): Info(self,"""Curve fitting is still in beta. diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 6506984..657202a 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -398,10 +398,13 @@ def get(self,i): @staticmethod - def createDummy(nTab=3): + def createDummy(nTabs=3, n=30, addLabel=True): tabs=[] - for iTab in range(nTab): - tabs.append( Table.createDummy(lab='_'+str(iTab)) ) + label='' + for iTab in range(nTabs): + if addLabel: + label='_'+str(iTab) + tabs.append( Table.createDummy(n=n, label=label)) tablist = TableList(tabs) return tablist @@ -812,21 +815,21 @@ def nRows(self): return len(self.data.iloc[:,0]) # TODO if not panda @staticmethod - def createDummy(n=5, lab=''): + def createDummy(n, label=''): """ create a dummy table of length n""" - t = np.arange(0,0.5*n,0.5) - x = t+10 + t = np.linspace(0, 4*np.pi, n) + x = np.sin(t)+10 alpha_d = np.linspace(0, 360, n) P = np.random.normal(0,100,n)+5000 RPM = np.random.normal(-0.2,0.2,n) + 12. d={'Time_[s]':t, - 'x{}_[m]'.format(lab): x, - 'alpha{}_[deg]'.format(lab):alpha_d, - 'P{}_[W]'.format(lab):P, - 'RotSpeed{}_[rpm]'.format(lab):RPM} + 'x{}_[m]'.format(label): x, + 'alpha{}_[deg]'.format(label):alpha_d, + 'P{}_[W]'.format(label):P, + 'RotSpeed{}_[rpm]'.format(label):RPM} df = pd.DataFrame(data=d) - return Table(data=df, name='Dummy '+lab) + return Table(data=df, name='Dummy '+label) if __name__ == '__main__': diff --git a/pydatview/common.py b/pydatview/common.py index c5fa2af..fcd94df 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -447,6 +447,5 @@ class DummyMainFrame(): def __init__(self, parent): self.parent=parent def addAction (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) def removeAction (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) - def load_df (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) def load_dfs (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) def mainFrameUpdateLayout(self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) diff --git a/pydatview/main.py b/pydatview/main.py index 789eb6d..4444cc2 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -301,6 +301,13 @@ def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, # Load the tables newTabs, warnList = self.tabList.load_tables_from_files(filenames=filenames, fileformats=fileformats, bAdd=bAdd, bReload=bReload, statusFunction=statusFunction) + + # Apply postLoad pipeline + if bReload: + self.applyPipeline(self.tabList, force=True) # we force on reload + else: + self.applyPipeline(newTabs, force=True, applyToAll=True) # we apply only on newTabs + if bReload: # Restore formulas that were previously added for tab in self.tabList: @@ -315,18 +322,15 @@ def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, if self.tabList.len()>0: self.load_tabs_into_GUI(bReload=bReload, bAdd=bAdd, bPlot=bPlot) - def load_df(self, df, name=None, bAdd=False, bPlot=True): - if bAdd: - self.tabList.append(Table(data=df, name=name)) - else: - self.tabList = TableList( [Table(data=df, name=name)] ) - self.load_tabs_into_GUI(bAdd=bAdd, bPlot=bPlot) - if hasattr(self,'selPanel'): - self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) - - def load_dfs(self, dfs, names, bAdd=False): + def load_dfs(self, dfs, names=None, bAdd=False, bPlot=True): + """ Load one or multiple dataframes intoGUI """ + # + if not isinstance(dfs,list): + dfs=[dfs] + if not isinstance(names,list): + names=[names] self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) - self.load_tabs_into_GUI(bAdd=bAdd, bPlot=True) + self.load_tabs_into_GUI(bAdd=bAdd, bPlot=bPlot) if hasattr(self,'selPanel'): self.selPanel.updateLayout(SEL_MODES_ID[self.comboMode.GetSelection()]) @@ -354,7 +358,7 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): self.tSplitter = wx.SplitterWindow(self.vSplitter) #self.tSplitter.SetMinimumPaneSize(20) self.infoPanel = InfoPanel(self.tSplitter, data=self.data['infoPanel']) - self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, self, data=self.data['plotPanel']) + self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, data=self.data['plotPanel']) self.tSplitter.SetSashGravity(0.9) self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) self.tSplitter.SetMinimumPaneSize(BOT_PANL) @@ -379,6 +383,9 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): #self.selPanel.bindColSelectionChange(self.onColSelectionChangeCallBack) self.selPanel.setTabSelectionChangeCallback(self.onTabSelectionChangeTrigger) self.selPanel.setRedrawCallback(self.redrawCallback) + self.selPanel.setUpdateLayoutCallback(self.mainFrameUpdateLayout) + self.plotPanel.setAddTablesCallback(self.load_dfs) + self.Bind(wx.EVT_SPLITTER_SASH_POS_CHANGED, self.onSashChangeMain, self.vSplitter) # plot trigger @@ -471,11 +478,20 @@ def onDataPlugin(self, event=None, toolName=''): for thisToolName, function, isPanel in dataPlugins: if toolName == thisToolName: - if isPanel: - panelClass = function(self, event, toolName) # getting panelClass - self.plotPanel.showToolPanel(panelClass) + if isPanel: # This is more of a "hasPanel" + # Check to see if the pipeline already contains this action + action = self.pipeline.find(toolName) # old action to edit + if action is None: + action = function(label=toolName, mainframe=self) # getting brand new action + self.plotPanel.showToolAction(action) + # The panel will have the responsability to apply/delete the action, updateGUI, etc else: - function(self, event, toolName) # calling the data function + action = function(label=toolName, mainframe=self) # calling the data function + # Here we apply the action directly + action.apply(self.tabList) # the action will chose that to apply it on + self.addAction(action) + action.updateGUI() + return raise NotImplementedError('Tool: ',toolName) diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index b2f8484..aa64629 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -7,6 +7,7 @@ def __init__(self, name, tableFunction=None, guiCallBack=None, guiEditorClass=None, data=None, + mainframe=None, onPlotData=False, unique=True, removeNeedReload=False): self.name = name @@ -17,6 +18,7 @@ def __init__(self, name, self.guiEditorClass = guiEditorClass # Class that can be used to edit this action self.data = data if data is not None else {} + self.mainframe=mainframe # If possible, dont use that... self.applied=False # TODO this needs to be sotred per table diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 74c4158..2de282f 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -14,18 +14,33 @@ def _function_name(mainframe, event=None, label='') See working examples in this file and this directory. """ -def _data_standardizeUnits(mainframe, event=None, label=''): - from .data_standardizeUnits import standardizeUnitsPlugin - standardizeUnitsPlugin(mainframe, event, label) +def _data_standardizeUnitsSI(label, mainframe=None): + from .data_standardizeUnits import standardizeUnitsAction + return standardizeUnitsAction(label, mainframe, flavor='SI') -def _data_binning(mainframe, event=None, label=''): - from .data_binning import BinningToolPanel - return BinningToolPanel +def _data_standardizeUnitsWE(label, mainframe=None): + from .data_standardizeUnits import standardizeUnitsAction + return standardizeUnitsAction(label, mainframe, flavor='WE') + +def _data_binning(label, mainframe): + from .data_binning import binningAction + return binningAction(label, mainframe) dataPlugins=[ - # Name/label , callback , is a Panel - ('Bin data' , _data_binning , True ), - ('Standardize Units (SI)', _data_standardizeUnits, False), - ('Standardize Units (WE)', _data_standardizeUnits, False), + # Name/label , callback , is a Panel + ('Bin data' , _data_binning , True ), + ('Standardize Units (SI)', _data_standardizeUnitsSI, False), + ('Standardize Units (WE)', _data_standardizeUnitsWE, False), ] + + + + + +# --- +def getDataPluginsDict(): + d={} + for toolName, function, isPanel in dataPlugins: + d[toolName]={'callback':function, 'isPanel':isPanel} + return d diff --git a/pydatview/plugins/data_binning.py b/pydatview/plugins/data_binning.py index 22a1c25..5a46f89 100644 --- a/pydatview/plugins/data_binning.py +++ b/pydatview/plugins/data_binning.py @@ -7,24 +7,50 @@ # For log dec tool from pydatview.GUITools import GUIToolPanel, TOOL_BORDER from pydatview.common import CHAR, Error, Info, pretty_num_short +from pydatview.common import DummyMainFrame from pydatview.plotdata import PlotData # from pydatview.tools.damping import logDecFromDecay # from pydatview.tools.curve_fitting import model_fit, extract_key_miscnum, extract_key_num, MODELS, FITTERS, set_common_keys +from pydatview.pipeline import PlotDataAction # --------------------------------------------------------------------------------} -# --- GUI +# --- Action # --------------------------------------------------------------------------------{ -class BinningToolPanel(GUIToolPanel): - def __init__(self, parent): - super(BinningToolPanel,self).__init__(parent) +def binningAction(label, mainframe=None, data=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + The action is also edited and created by the GUI Editor + """ + if data is None: + data=_DEFAULT_DICT + + action = PlotDataAction( + name=label, + guiEditorClass = BinningToolPanel, + data = data, + mainframe=mainframe + ) + return action +# --------------------------------------------------------------------------------} +# --- GUI to Edit Plugin and control the Action +# --------------------------------------------------------------------------------{ +class BinningToolPanel(GUIToolPanel): + def __init__(self, parent, action): + super(BinningToolPanel, self).__init__(parent) + + # --- Creating "Fake data" for testing only! + if action is None: + print('[WARN] Calling GUI without an action! Creating one.') + mainframe = DummyMainFrame(parent) + action = binningAction(label='dummyAction', mainframe=mainframe) # --- Data from other modules self.parent = parent # parent is GUIPlotPanel - # Getting states from parent - if 'Binning' not in self.parent.plotDataOptions.keys() or self.parent.plotDataOptions['Binning'] is None: - self.parent.plotDataOptions['Binning'] =_DEFAULT_DICT.copy() - self.data = self.parent.plotDataOptions['Binning'] + self.mainframe = action.mainframe + + self.data = action.data + self.action = action self.data['selectionChangeCallBack'] = self.selectionChange @@ -81,10 +107,9 @@ def __init__(self, parent): vsizer.Add(msizer,0, flag = wx.TOP ,border = 1) vsizer.Add(msizer2,0, flag = wx.TOP|wx.EXPAND ,border = 1) - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) - self.sizer.Add(vsizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) + self.sizer.Add(btSizer ,0, flag = wx.LEFT , border = 5) + self.sizer.Add(vsizer ,1, flag = wx.LEFT|wx.EXPAND , border = TOOL_BORDER) #self.sizer.Add(msizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) self.SetSizer(self.sizer) @@ -146,14 +171,27 @@ def onToggleApply(self, event=None, init=False): self.btPlot.Enable(False) self.btClear.Enable(False) self.btApply.SetLabel(CHAR['sun']+' Clear') + # We add our action to the pipeline + if self.mainframe is not None: + self.mainframe.addAction(self.action) + else: + print('[WARN] Running data_binning without a main frame') else: - self.parent.plotDataOptions['Binning'] = None + print('>>>> TODO Remove Action') + # We remove our action from the pipeline + if not init: + if self.mainframe is not None: + self.mainframe.removeAction(self.action) + else: + print('[WARN] Running data_binning without a main frame') + #self.data = None + #self.action = None self.btPlot.Enable(True) self.btClear.Enable(True) self.btApply.SetLabel(CHAR['cloud']+' Apply') if not init: - self.parent.plotDataOptions['Binning'] = self.data + # This is a "plotData" action, we don't need to do anything self.parent.load_and_draw() # Data will change based on plotData @@ -161,7 +199,6 @@ def onAdd(self,event=None): from pydatview.tools.stats import bin_DF iSel = self.cbTabs.GetSelection() tabList = self.parent.selPanel.tabList - mainframe = self.parent.mainframe icol, colname = self.parent.selPanel.xCol if self.parent.selPanel.currentMode=='simColumnsMode': # The difficulty here is that we have to use @@ -189,12 +226,12 @@ def onAdd(self,event=None): names_new.append(name_new) else: errors.append(tab.active_name) - mainframe.load_dfs(dfs_new, names_new, bAdd=True) + self.parent.addTables(dfs_new, names_new, bAdd=True) else: tab = tabList.get(iSel-1) df_new, name_new = bin_tab(tab, icol, colname, self.data, bAdd=True) if df_new is not None: - mainframe.load_df(df_new, name_new, bAdd=True) + self.parent.addTables([df_new], [name_new], bAdd=True) else: errors.append(tab.active_name) self.updateTabList() @@ -294,7 +331,6 @@ def bin_tab(tab, iCol, colName, opts, bAdd=True): return df_new, name_new - _DEFAULT_DICT={ 'active':False, 'xMin':None, @@ -305,3 +341,38 @@ def bin_tab(tab, iCol, colName, opts, bAdd=True): 'selectionChangeCallBack':None, } + +if __name__ == '__main__': + from pydatview.Tables import TableList + from pydatview.plotdata import PlotData + from pydatview.GUIPlotPanel import PlotPanel + from pydatview.GUISelectionPanel import SelectionPanel + + + # --- Data + tabList = TableList.createDummy(nTabs=2, n=100, addLabel=False) + app = wx.App(False) + self = wx.Frame(None,-1,"Data Binning GUI") + + # --- Panels + self.selPanel = SelectionPanel(self, tabList, mode='auto') + self.plotPanel = PlotPanel(self, self.selPanel) + self.plotPanel.load_and_draw() # <<< Important + self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # Binding the two + + p = BinningToolPanel(self.plotPanel, action=None) + + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(self.selPanel ,0, wx.EXPAND|wx.ALL, border=5) + sizer.Add(self.plotPanel,1, wx.EXPAND|wx.ALL, border=5) + #sizer.Add(p) + self.SetSizer(sizer) + self.SetSize((900, 600)) + self.Center() + self.Show() + + self.plotPanel.showToolPanel(panel=p) + + app.MainLoop() + + diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py index b08fcbf..a6eb9aa 100644 --- a/pydatview/plugins/data_standardizeUnits.py +++ b/pydatview/plugins/data_standardizeUnits.py @@ -1,21 +1,34 @@ import unittest import numpy as np from pydatview.common import splitunit +from pydatview.pipeline import Action, IrreversibleAction -def standardizeUnitsPlugin(mainframe, event=None, label='Standardize Units (SI)'): +def standardizeUnitsAction(label, mainframe=None, flavor='SI'): """ Main entry point of the plugin """ - flavor = label.split('(')[1][0:2] + guiCallBack=None + if mainframe is not None: + def guiCallBack(): + if hasattr(mainframe,'selPanel'): + mainframe.selPanel.colPanel1.setColumns() + mainframe.selPanel.colPanel2.setColumns() + mainframe.selPanel.colPanel3.setColumns() + mainframe.onTabSelectionChange() # trigger replot + if hasattr(mainframe,'pipePanel'): + pass - for t in mainframe.tabList: - changeUnits(t, flavor=flavor) + # Function that will be applied to all tables + tableFunction = lambda t: changeUnits(t, flavor=flavor) - if hasattr(mainframe,'selPanel'): - mainframe.selPanel.colPanel1.setColumns() - mainframe.selPanel.colPanel2.setColumns() - mainframe.selPanel.colPanel3.setColumns() - mainframe.onTabSelectionChange() # trigger replot + action = IrreversibleAction( + name=label, + tableFunction=tableFunction, + guiCallBack=guiCallBack, + mainframe=mainframe, # shouldnt be needed + ) + + return action def changeUnits(tab, flavor='SI'): """ Change units of a table diff --git a/tests/prof_all.py b/tests/prof_all.py index 5f0188c..4503e95 100644 --- a/tests/prof_all.py +++ b/tests/prof_all.py @@ -28,7 +28,7 @@ def test_heavy(): with PerfMon('Plot 1'): app = wx.App(False) frame = MainFrame() - frame.load_df(df) + frame.load_dfs([df]) del df time.sleep(dt) with PerfMon('Redraw 1'): From 5a93daef8959a4cc87bad7a239b38f74e5943cd6 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 00:42:56 +0100 Subject: [PATCH 062/178] Bug fix: data binning failed since removal of fake index (see #127) --- pydatview/plugins/data_binning.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pydatview/plugins/data_binning.py b/pydatview/plugins/data_binning.py index 22a1c25..44cf5be 100644 --- a/pydatview/plugins/data_binning.py +++ b/pydatview/plugins/data_binning.py @@ -273,11 +273,15 @@ def bin_plot(x, y, opts): def bin_tab(tab, iCol, colName, opts, bAdd=True): # TODO, make it such as it's only handling a dataframe instead of a table from pydatview.tools.stats import bin_DF - colName = tab.data.columns[iCol-1] + colName = tab.data.columns[iCol] error='' xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) # try: df_new =bin_DF(tab.data, xbins=xBins, colBin=colName) + # Remove index if present + if df_new.columns[0].lower().find('index')>=0: + df_new = df_new.iloc[:, 1:] # We don't use "drop" in case of duplicate "index" + # Setting bin column as first columns colNames = list(df_new.columns.values) colNames.remove(colName) From fd22142e20240c4bbcd51148841a7d03ac7cf24f Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 02:48:09 +0100 Subject: [PATCH 063/178] Pipeline and plugin for plot data filters --- pydatview/GUIPipelinePanel.py | 22 +++---- pydatview/GUIPlotPanel.py | 12 +++- pydatview/main.py | 10 ++-- pydatview/pipeline.py | 98 +++++++++++++++++++++++++------ pydatview/plotdata.py | 17 +++--- pydatview/plugins/data_binning.py | 41 +++++++++---- 6 files changed, 147 insertions(+), 53 deletions(-) diff --git a/pydatview/GUIPipelinePanel.py b/pydatview/GUIPipelinePanel.py index acb5913..f9af14e 100644 --- a/pydatview/GUIPipelinePanel.py +++ b/pydatview/GUIPipelinePanel.py @@ -168,19 +168,21 @@ def apply(self, tablist, force=False, applyToAll=False): self.ep.update() self.Sizer.Layout() - def append(self, action): - # Delete action is already present and if it's a "unique" action - ac = self.pipeline.find(action.name) - if ac is not None: - if ac.unique: - print('>>> Deleting unique action before inserting it again', ac.name) - self.delete(ac, silent=True) + def append(self, action, cancelIfPresent=False): + if not cancelIfPresent: + # Delete action is already present and if it's a "unique" action + ac = self.pipeline.find(action.name) + if ac is not None: + if ac.unique: + print('>>> Deleting unique action before inserting it again', ac.name) + self.remove(ac, silent=True) # Add to pipeline print('>>> Adding action',action.name) - self.pipeline.append(action) + self.pipeline.append(action, cancelIfPresent=cancelIfPresent) # Add to GUI - self._addPanel(action) - self.Sizer.Layout() + self.populate() # NOTE: we populate because of the change of order between actionsData and actionsPlot.. + #self._addPanel(action) + #self.Sizer.Layout() def remove(self, action, silent=False): """ NOTE: the action is only removed from the pipeline, not deleted. """ diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 700d34a..3fe7ac0 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -376,7 +376,7 @@ def onFontOptionChange(self,event=None): class PlotPanel(wx.Panel): - def __init__(self, parent, selPanel, infoPanel=None, data=None): + def __init__(self, parent, selPanel, pipeline=None, infoPanel=None, data=None): # Superclass constructor super(PlotPanel,self).__init__(parent) @@ -402,6 +402,7 @@ def __init__(self, parent, selPanel, infoPanel=None, data=None): break # data self.selPanel = selPanel # <<< dependency with selPanel should be minimum + self.pipeline = pipeline # self.selMode = '' self.infoPanel=infoPanel if self.infoPanel is not None: @@ -785,6 +786,9 @@ def removeTools(self,event=None,Layout=True): # Python2 if hasattr(self,'toolPanel'): self.toolSizer.Remove(self.toolPanel) + if hasattr(self.toolPanel,'action'): + self.toolPanel.action.guiEditorObj = None # Important + self.toolPanel.action = None self.toolPanel.Destroy() del self.toolPanel self.toolSizer.Clear() # Delete Windows @@ -815,6 +819,7 @@ def showToolPanel(self, panelClass=None, panel=None, action=None): self.toolPanel=panelClass(parent=self) # calling the panel constructor else: self.toolPanel=panelClass(parent=self, action=action) # calling the panel constructor + action.guiEditorObj = self.toolPanel self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) self.plotsizer.Layout() self.Thaw() @@ -874,16 +879,19 @@ def transformPlotData(self,PD): self.setPD_FFT(PD) def getPlotData(self,plotType): + ID,SameCol,selMode=self.selPanel.getPlotDataSelection() + self.selMode=selMode # we store the selection mode del self.plotData self.plotData=[] tabs=self.selPanel.tabList.getTabs() # TODO, selPanel should just return the PlotData... + try: for i,idx in enumerate(ID): # Initialize each plotdata based on selected table and selected id channels pd=PlotData(); - pd.fromIDs(tabs,i,idx,SameCol, self.plotDataOptions) + pd.fromIDs(tabs, i, idx, SameCol, self.plotDataOptions, pipeline=self.pipeline) # Possible change of data if plotType=='MinMax': self.setPD_MinMax(pd) diff --git a/pydatview/main.py b/pydatview/main.py index 4444cc2..dc15660 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -358,7 +358,7 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): self.tSplitter = wx.SplitterWindow(self.vSplitter) #self.tSplitter.SetMinimumPaneSize(20) self.infoPanel = InfoPanel(self.tSplitter, data=self.data['infoPanel']) - self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, self.infoPanel, data=self.data['plotPanel']) + self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, infoPanel=self.infoPanel, pipeline=self.pipeline, data=self.data['plotPanel']) self.tSplitter.SetSashGravity(0.9) self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) self.tSplitter.SetMinimumPaneSize(BOT_PANL) @@ -483,8 +483,10 @@ def onDataPlugin(self, event=None, toolName=''): action = self.pipeline.find(toolName) # old action to edit if action is None: action = function(label=toolName, mainframe=self) # getting brand new action + else: + print('>>> The action already exists, we use it for the GUI') self.plotPanel.showToolAction(action) - # The panel will have the responsability to apply/delete the action, updateGUI, etc + # The panel will have the responsibility to apply/delete the action, updateGUI, etc else: action = function(label=toolName, mainframe=self) # calling the data function # Here we apply the action directly @@ -496,8 +498,8 @@ def onDataPlugin(self, event=None, toolName=''): raise NotImplementedError('Tool: ',toolName) # --- Pipeline - def addAction(self, action): - self.pipePanel.append(action) + def addAction(self, action, cancelIfPresent=False): + self.pipePanel.append(action, cancelIfPresent=cancelIfPresent) def removeAction(self, action): self.pipePanel.remove(action) def applyPipeline(self, *args, **kwargs): diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index aa64629..7e765d8 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -1,23 +1,35 @@ """ pipelines and actions """ +import numpy as np class Action(): def __init__(self, name, - tableFunction=None, guiCallBack=None, + tableFunction = None, + plotDataFunction = None, + guiCallBack=None, guiEditorClass=None, data=None, mainframe=None, onPlotData=False, unique=True, removeNeedReload=False): + """ + tableFunction: signature: f(tab) # TODO that's inplace + plotDataFunction: signature: xnew, ynew = f(x, y, data) + # TODO + + """ self.name = name # - self.tableFunction = tableFunction + self.tableFunction = tableFunction # applies to a full table + self.plotDataFunction = plotDataFunction # applies to x,y arrays only - self.guiCallBack = guiCallBack + self.guiCallBack = guiCallBack # callback to update GUI after the action, + # TODO remove me, replace with generic "redraw", "update tab list" self.guiEditorClass = guiEditorClass # Class that can be used to edit this action + self.guiEditorObj = None # Instance of guiEditorClass that can be used to edit this action - self.data = data if data is not None else {} + self.data = data if data is not None else {} # Data needed by the action, can be saved to file so that the action can be restored self.mainframe=mainframe # If possible, dont use that... self.applied=False # TODO this needs to be sotred per table @@ -32,6 +44,7 @@ def __init__(self, name, def apply(self, tablist, force=False, applyToAll=False): self.errorList=[] if self.tableFunction is None: + # NOTE: this does not applyt to plotdataActions.. raise Exception('tableFunction was not specified for action: {}'.format(self.name)) for t in tablist: @@ -45,6 +58,7 @@ def apply(self, tablist, force=False, applyToAll=False): self.applied = True return tablist + def updateGUI(self): if self.guiCallBack is not None: print('>>> Calling GUI callback, action', self.name) @@ -59,13 +73,17 @@ class PlotDataAction(Action): def __init__(self, name, **kwargs): Action.__init__(self, name, onPlotData=True, **kwargs) - def applyOnPlotData(self): - print('>>> Apply On Plot Data') + def apply(self, *args, **kwargs): + pass # not pretty + + def applyOnPlotData(self, x, y): + x, y = self.plotDataFunction(x, y, self.data) + return x, y class IrreversibleAction(Action): def __init__(self, name, **kwargs): - Action.__init__(self, name, deleteNeedReload=True, **kwargs) + Action.__init__(self, name, removeNeedReload=True, **kwargs) def apply(self, tablist, force=False, applyToAll=False): if force: @@ -89,12 +107,13 @@ class Pipeline(object): def __init__(self, data=[]): self.actionsData = [] - self.actionsPlot = [] + self.actionsPlotFilters = [] self.errorList = [] + self.plotFiltersData=[] # list of data for plot data filters, that plotData.py will use @property def actions(self): - return self.actionsData+self.actionsPlot # order matters + return self.actionsData+self.actionsPlotFilters # order matters def apply(self, tablist, force=False, applyToAll=False): """ @@ -105,31 +124,64 @@ def apply(self, tablist, force=False, applyToAll=False): for action in self.actionsData: action.apply(tablist, force=force, applyToAll=applyToAll) # - for action in self.actionsPlot: + for action in self.actionsPlotFilters: action.apply(tablist, force=force, applyToAll=applyToAll) # self.collectErrors() + + def applyOnPlotData(self, x, y): + x = np.copy(x) + y = np.copy(y) + for action in self.actionsPlotFilters: + x, y = action.applyOnPlotData(x, y) + return x, y + + + def collectErrors(self): self.errorList=[] for action in self.actions: self.errorList+= action.errorList +# def setPlotFiltersData(self): +# print('>>> Setting plotFiltersData') +# self.plotFiltersData=[] +# for action in self.actionsPlotFilters: +# self.plotFiltersData.append(action.data) +# print(action.data) + # --- Behave like a list.. - def append(self, action): + def append(self, action, cancelIfPresent=False): + if cancelIfPresent: + i = self.index(action) + if i>=0: + print('>>> Not adding action, its already present') + return if action.onPlotData: - self.actionsPlot.append(action) + self.actionsPlotFilters.append(action) + # Trigger +# self.setPlotFiltersData() else: self.actionsData.append(action) + def remove(self, a): """ NOTE: the action is removed, not deleted fully (it might be readded to the pipeline later)""" try: i = self.actionsData.index(a) a = self.actionsData.pop(i) except ValueError: - i = self.actionsPlot.index(a) - a = self.actionsPlot.pop(i) + i = self.actionsPlotFilters.index(a) + a = self.actionsPlotFilters.pop(i) + # Trigger +# self.setPlotFiltersData() + + # Cancel the action in Editor + if a.guiEditorObj is not None: + print('>>> Canceling action in guiEditor') + a.guiEditorObj.cancelAction() + return a def find(self, name): @@ -138,6 +190,13 @@ def find(self, name): return action return None + def index(self, action_): + for i, action in enumerate(self.actions): + if action==action_: + return i + else: + return -1 + # --- Data/Options def loadFromFile(self, filename): pass @@ -150,11 +209,11 @@ def loadData(self, data): def saveData(self, data): data['actionsData'] = {} - data['actionsPlot'] = {} + data['actionsPlotFilters'] = {} for ac in self.actionsData: data['actionsData'][ac.name] = ac.data for ac in self.actions: - data['actionsPlot'][ac.name] = ac.data + data['actionsPlotFilters'][ac.name] = ac.data #data[] = self.Naming @staticmethod @@ -165,7 +224,12 @@ def __repr__(self): s=': ' s+=' > '.join([ac.name for ac in self.actionsData]) s+=' + ' - s+=' > '.join([ac.name for ac in self.actionsPlot]) + s+=' > '.join([ac.name for ac in self.actionsPlotFilters]) return s + def __reprFilters__(self): + s=': ' + s+=' > '.join([ac.name for ac in self.actionsPlotFilters]) + return s + diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index a2394b3..d1dcc26 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -34,7 +34,7 @@ def __init__(PD, x=None, y=None, sx='', sy=''): if x is not None and y is not None: PD.fromXY(x,y,sx,sy) - def fromIDs(PD, tabs, i, idx, SameCol, Options={}): + def fromIDs(PD, tabs, i, idx, SameCol, Options=None, pipeline=None): """ Nasty initialization of plot data from "IDs" """ PD.id = i PD.it = idx[0] # table index @@ -51,7 +51,7 @@ def fromIDs(PD, tabs, i, idx, SameCol, Options={}): PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info PD.c =c # raw values, used by PDF - PD._post_init(Options=Options) + PD._post_init(Options=Options, pipeline=pipeline) def fromXY(PD, x, y, sx='', sy=''): PD.x = x @@ -67,7 +67,9 @@ def fromXY(PD, x, y, sx='', sy=''): PD._post_init() - def _post_init(PD, Options={}): + def _post_init(PD, Options=None, pipeline=None): + if Options is None: + Options = {} # --- Perform data manipulation on the fly #[print(k,v) for k,v in Options.items()] keys=Options.keys() @@ -89,10 +91,11 @@ def _post_init(PD, Options={}): from pydatview.tools.signal_analysis import applySampler PD.x, PD.y = applySampler(PD.x, PD.y, Options['Sampler']) - if 'Binning' in keys: - if Options['Binning']: - if Options['Binning']['active']: - PD.x, PD.y = Options['Binning']['applyCallBack'](PD.x, PD.y, Options['Binning']) + # --- Apply filters from pipeline + print('[PDat]', pipeline.__reprFilters__()) + if pipeline is not None: + PD.x, PD.y = pipeline.applyOnPlotData(PD.x, PD.y) + # --- Store stats n=len(PD.y) diff --git a/pydatview/plugins/data_binning.py b/pydatview/plugins/data_binning.py index 0f58267..0fac1ed 100644 --- a/pydatview/plugins/data_binning.py +++ b/pydatview/plugins/data_binning.py @@ -27,6 +27,7 @@ def binningAction(label, mainframe=None, data=None): action = PlotDataAction( name=label, + plotDataFunction = bin_plot, guiEditorClass = BinningToolPanel, data = data, mainframe=mainframe @@ -51,11 +52,10 @@ def __init__(self, parent, action): self.data = action.data self.action = action - self.data['selectionChangeCallBack'] = self.selectionChange - + # self.data['selectionChangeCallBack'] = self.selectionChange # TODO # --- GUI elements - self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) + self.btClose = self.getBtBitmap(self, 'Close','close', self.onClose) self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) @@ -129,6 +129,17 @@ def __init__(self, parent, action): self.updateTabList() self.onParamChange() + # --- External Calls + def cancelAction(self, redraw=True): + """ do cancel the action""" + self.btPlot.Enable(True) + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + self.btApply.SetValue(False) + self.data['active'] = False + if redraw: + self.parent.load_and_draw() # Data will change based on plotData + # def reset(self, event=None): self.setXRange() self.updateTabList() # might as well until we add a nice callback/button.. @@ -171,11 +182,17 @@ def onToggleApply(self, event=None, init=False): self.btPlot.Enable(False) self.btClear.Enable(False) self.btApply.SetLabel(CHAR['sun']+' Clear') - # We add our action to the pipeline + self.btApply.SetValue(True) + # The action is now active we add it to the pipeline, unless it's already in it if self.mainframe is not None: - self.mainframe.addAction(self.action) + self.mainframe.addAction(self.action, cancelIfPresent=True) else: print('[WARN] Running data_binning without a main frame') + + if not init: + # This is a "plotData" action, we don't need to do anything + print('>>> redraw') + self.parent.load_and_draw() # Data will change based on plotData else: print('>>>> TODO Remove Action') # We remove our action from the pipeline @@ -186,13 +203,8 @@ def onToggleApply(self, event=None, init=False): print('[WARN] Running data_binning without a main frame') #self.data = None #self.action = None - self.btPlot.Enable(True) - self.btClear.Enable(True) - self.btApply.SetLabel(CHAR['cloud']+' Apply') + self.cancelAction(redraw=not init) - if not init: - # This is a "plotData" action, we don't need to do anything - self.parent.load_and_draw() # Data will change based on plotData def onAdd(self,event=None): @@ -276,6 +288,11 @@ def updateTabList(self,event=None): except RuntimeError: pass + def onClose(self, event=None): + # cleanup action calls + self.action.guiEditorObj=None + self.destroy() + def onHelp(self,event=None): Info(self,"""Binning. @@ -341,8 +358,6 @@ def bin_tab(tab, iCol, colName, opts, bAdd=True): 'xMax':None, 'nBins':50, 'dx':0, - 'applyCallBack':bin_plot, - 'selectionChangeCallBack':None, } From ce89a6d8e9533ef3c42ab1afe0745c79ea603def Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 04:45:47 +0100 Subject: [PATCH 064/178] Pipeline: sampler as a plugin/action --- pydatview/GUIPipelinePanel.py | 8 +- pydatview/GUIPlotPanel.py | 3034 ++++++++++---------- pydatview/GUITools.py | 250 +- pydatview/main.py | 1 - pydatview/pipeline.py | 9 +- pydatview/plotdata.py | 8 +- pydatview/plugins/__init__.py | 4 + pydatview/plugins/data_binning.py | 215 +- pydatview/plugins/data_sampler.py | 321 +++ pydatview/plugins/data_standardizeUnits.py | 7 +- 10 files changed, 1967 insertions(+), 1890 deletions(-) create mode 100644 pydatview/plugins/data_sampler.py diff --git a/pydatview/GUIPipelinePanel.py b/pydatview/GUIPipelinePanel.py index f9af14e..f006fa2 100644 --- a/pydatview/GUIPipelinePanel.py +++ b/pydatview/GUIPipelinePanel.py @@ -195,8 +195,12 @@ def remove(self, action, silent=False): if action.removeNeedReload: if not silent: Info(self.parent, 'A reload is required now that the action "{}" has been removed.'.format(action.name)) - # TODO trigger reload/reapply - # TODO trigger GUI update + + # trigger GUI update (guiCallback) + action.updateGUI() + + + # Update list of errors self.ep.update() diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 3fe7ac0..47963d8 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -1,1523 +1,1511 @@ -import os -import numpy as np -import wx -import wx.lib.buttons as buttons -import dateutil # required by matplotlib -#from matplotlib import pyplot as plt -import matplotlib -# Backends: -# ['GTK3Agg', 'GTK3Cairo', 'GTK4Agg', 'GTK4Cairo', 'MacOSX', 'nbAgg', 'QtAgg', 'QtCairo', 'Qt5Agg', 'Qt5Cairo', 'TkAgg', 'TkCairo', 'WebAgg', 'WX', 'WXAgg', 'WXCairo', 'agg', 'cairo', 'pdf', 'pgf', 'ps', 'svg', 'template'] -matplotlib.use('WX') # Important for Windows version of installer. NOTE: changed from Agg to wxAgg, then to WX -from matplotlib import rc as matplotlib_rc -try: - from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas -except Exception as e: - print('') - print('Error: problem importing `matplotlib.backends.backend_wx`.') - import platform - if platform.system()=='Darwin': - print('') - print('pyDatView help:') - print(' This is a typical issue on MacOS, most likely you are') - print(' using the native MacOS python with the native matplolib') - print(' library, which is incompatible with `wxPython`.') - print('') - print(' You can solve this by either:') - print(' - using python3, and pip3 e.g. installing it with brew') - print(' - using a virtual environment with python 3') - print(' - using anaconda with python 3'); - print('') - import sys - sys.exit(1) - else: - raise e -from matplotlib.figure import Figure -from matplotlib.pyplot import rcParams as pyplot_rc -from matplotlib import font_manager -from pandas.plotting import register_matplotlib_converters - -import gc - -from pydatview.common import * # unique, CHAR -from pydatview.plotdata import PlotData, compareMultiplePD -from pydatview.GUICommon import * -from pydatview.GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool -from pydatview.GUIMeasure import GUIMeasure -import pydatview.icons as icons - -font = {'size' : 8} -matplotlib_rc('font', **font) -pyplot_rc['agg.path.chunksize'] = 20000 - - -class PDFCtrlPanel(wx.Panel): - def __init__(self, parent): - super(PDFCtrlPanel,self).__init__(parent) - self.parent = parent - lb = wx.StaticText( self, -1, 'Number of bins:') - self.scBins = wx.SpinCtrl(self, value='51',size=wx.Size(70,-1), style=wx.TE_RIGHT) - self.scBins.SetRange(3, 10000) - self.cbSmooth = wx.CheckBox(self, -1, 'Smooth',(10,10)) - self.cbSmooth.SetValue(False) - dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer.Add(lb ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.scBins ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.cbSmooth ,0, flag = wx.CENTER|wx.LEFT,border = 6) - self.SetSizer(dummy_sizer) - self.Bind(wx.EVT_TEXT , self.onPDFOptionChange, self.scBins) - self.Bind(wx.EVT_CHECKBOX, self.onPDFOptionChange) - self.Hide() - - def onPDFOptionChange(self,event=None): - self.parent.load_and_draw(); # DATA HAS CHANGED - -class MinMaxPanel(wx.Panel): - def __init__(self, parent): - super(MinMaxPanel,self).__init__(parent) - self.parent = parent - self.cbxMinMax = wx.CheckBox(self, -1, 'xMinMax',(10,10)) - self.cbyMinMax = wx.CheckBox(self, -1, 'yMinMax',(10,10)) - self.cbxMinMax.SetValue(False) - self.cbyMinMax.SetValue(True) - dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer.Add(self.cbxMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) - dummy_sizer.Add(self.cbyMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) - self.SetSizer(dummy_sizer) - self.Bind(wx.EVT_CHECKBOX, self.onMinMaxChange) - self.Hide() - - def onMinMaxChange(self,event=None): - self.parent.load_and_draw(); # DATA HAS CHANGED - -class CompCtrlPanel(wx.Panel): - def __init__(self, parent): - super(CompCtrlPanel,self).__init__(parent) - self.parent = parent - lblList = ['Relative', '|Relative|','Ratio','Absolute','Y-Y'] - self.rbType = wx.RadioBox(self, label = 'Type', choices = lblList, - majorDimension = 1, style = wx.RA_SPECIFY_ROWS) - dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer.Add(self.rbType ,0, flag = wx.CENTER|wx.LEFT,border = 1) - self.SetSizer(dummy_sizer) - self.rbType.Bind(wx.EVT_RADIOBOX,self.onTypeChange) - self.Hide() - - def onTypeChange(self,e): - self.parent.load_and_draw(); # DATA HAS CHANGED - - -class SpectralCtrlPanel(wx.Panel): - def __init__(self, parent): - super(SpectralCtrlPanel,self).__init__(parent) - self.parent = parent - # --- GUI widgets - lb = wx.StaticText( self, -1, 'Type:') - self.cbType = wx.ComboBox(self, choices=['PSD','f x PSD','Amplitude'] , style=wx.CB_READONLY) - self.cbType.SetSelection(0) - lbAveraging = wx.StaticText( self, -1, 'Avg.:') - self.cbAveraging = wx.ComboBox(self, choices=['None','Welch','Binning'] , style=wx.CB_READONLY) - self.cbAveraging.SetSelection(1) - self.lbAveragingMethod = wx.StaticText( self, -1, 'Window:') - self.cbAveragingMethod = wx.ComboBox(self, choices=['Hamming','Hann','Rectangular'] , style=wx.CB_READONLY) - self.cbAveragingMethod.SetSelection(0) - self.lbP2 = wx.StaticText( self, -1, '2^n:') - self.scP2 = wx.SpinCtrl(self, value='11',size=wx.Size(40,-1)) - self.lbWinLength = wx.StaticText( self, -1, '(2048) ') - self.scP2.SetRange(3, 50) - self.previousNExp = 8 - self.previousNDec = 20 - lbMaxFreq = wx.StaticText( self, -1, 'Xlim:') - self.tMaxFreq = wx.TextCtrl(self,size = (30,-1),style=wx.TE_PROCESS_ENTER) - self.tMaxFreq.SetValue("-1") - self.cbDetrend = wx.CheckBox(self, -1, 'Detrend',(10,10)) - lbX = wx.StaticText( self, -1, 'x:') - self.cbTypeX = wx.ComboBox(self, choices=['1/x','2pi/x','x'] , style=wx.CB_READONLY) - self.cbTypeX.SetSelection(0) - # Layout - dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer.Add(lb ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.cbType ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbAveraging ,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.cbAveraging ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.lbAveragingMethod,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.cbAveragingMethod,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.lbP2 ,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.scP2 ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.lbWinLength ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbMaxFreq ,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.tMaxFreq ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbX ,0, flag = wx.CENTER|wx.LEFT,border = 6) - dummy_sizer.Add(self.cbTypeX ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.cbDetrend ,0, flag = wx.CENTER|wx.LEFT,border = 7) - self.SetSizer(dummy_sizer) - self.Bind(wx.EVT_COMBOBOX ,self.onSpecCtrlChange) - self.Bind(wx.EVT_TEXT ,self.onP2ChangeText ,self.scP2 ) - self.Bind(wx.EVT_TEXT_ENTER,self.onXlimChange ,self.tMaxFreq ) - self.Bind(wx.EVT_CHECKBOX ,self.onDetrendChange ,self.cbDetrend) - self.Hide() - - def onXlimChange(self,event=None): - self.parent.redraw_same_data(); - def onSpecCtrlChange(self,event=None): - if self.cbAveraging.GetStringSelection()=='None': - self.scP2.Enable(False) - self.cbAveragingMethod.Enable(False) - self.lbP2.SetLabel('') - self.lbWinLength.SetLabel('') - elif self.cbAveraging.GetStringSelection()=='Binning': - self.previousNExp= self.scP2.GetValue() - self.scP2.SetValue(self.previousNDec) - self.scP2.Enable(True) - self.cbAveragingMethod.Enable(False) - self.lbP2.SetLabel('n:') - self.lbWinLength.SetLabel('') - else: - self.previousDec= self.scP2.GetValue() - self.scP2.SetValue(self.previousNExp) - self.lbP2.SetLabel('2^n:') - self.scP2.Enable(True) - self.cbAveragingMethod.Enable(True) - self.onP2ChangeText(event=None) - self.parent.load_and_draw() # Data changes - - def onDetrendChange(self,event=None): - self.parent.load_and_draw() # Data changes - - def onP2ChangeText(self,event=None): - if self.cbAveraging.GetStringSelection()=='Binning': - pass - else: - nExp=self.scP2.GetValue() - self.updateP2(nExp) - self.parent.load_and_draw() # Data changes - - def updateP2(self,P2): - self.lbWinLength.SetLabel("({})".format(2**P2)) - - - - -class PlotTypePanel(wx.Panel): - def __init__(self, parent): - # Superclass constructor - super(PlotTypePanel,self).__init__(parent) - #self.SetBackgroundColour('yellow') - # data - self.parent = parent - # --- Ctrl Panel - self.cbRegular = wx.RadioButton(self, -1, 'Regular',style=wx.RB_GROUP) - self.cbPDF = wx.RadioButton(self, -1, 'PDF' , ) - self.cbFFT = wx.RadioButton(self, -1, 'FFT' , ) - self.cbMinMax = wx.RadioButton(self, -1, 'MinMax' , ) - self.cbCompare = wx.RadioButton(self, -1, 'Compare', ) - self.cbRegular.SetValue(True) - self.Bind(wx.EVT_RADIOBUTTON, self.pdf_select , self.cbPDF ) - self.Bind(wx.EVT_RADIOBUTTON, self.fft_select , self.cbFFT ) - self.Bind(wx.EVT_RADIOBUTTON, self.minmax_select , self.cbMinMax ) - self.Bind(wx.EVT_RADIOBUTTON, self.compare_select, self.cbCompare) - self.Bind(wx.EVT_RADIOBUTTON, self.regular_select, self.cbRegular) - # LAYOUT - cb_sizer = wx.FlexGridSizer(rows=5, cols=1, hgap=0, vgap=0) - cb_sizer.Add(self.cbRegular , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbPDF , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbFFT , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbMinMax , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbCompare , 0, flag=wx.ALL, border=1) - self.SetSizer(cb_sizer) - - def plotType(self): - plotType='Regular' - if self.cbMinMax.GetValue(): - plotType='MinMax' - elif self.cbPDF.GetValue(): - plotType='PDF' - elif self.cbFFT.GetValue(): - plotType='FFT' - elif self.cbCompare.GetValue(): - plotType='Compare' - return plotType - - def regular_select(self, event=None): - self.clear_measures() - self.parent.cbLogY.SetValue(False) - # - self.parent.spcPanel.Hide(); - self.parent.pdfPanel.Hide(); - self.parent.cmpPanel.Hide(); - self.parent.mmxPanel.Hide(); - self.parent.slEsth.Hide(); - self.parent.plotsizer.Layout() - # - self.parent.load_and_draw() # Data changes - - def compare_select(self, event=None): - self.clear_measures() - self.parent.cbLogY.SetValue(False) - self.parent.show_hide(self.parent.cmpPanel, self.cbCompare.GetValue()) - self.parent.spcPanel.Hide(); - self.parent.pdfPanel.Hide(); - self.parent.mmxPanel.Hide(); - self.parent.plotsizer.Layout() - self.parent.load_and_draw() # Data changes - - def fft_select(self, event=None): - self.clear_measures() - self.parent.show_hide(self.parent.spcPanel, self.cbFFT.GetValue()) - self.parent.cbLogY.SetValue(self.cbFFT.GetValue()) - self.parent.pdfPanel.Hide(); - self.parent.mmxPanel.Hide(); - self.parent.plotsizer.Layout() - self.parent.load_and_draw() # Data changes - - def pdf_select(self, event=None): - self.clear_measures() - self.parent.cbLogX.SetValue(False) - self.parent.cbLogY.SetValue(False) - self.parent.show_hide(self.parent.pdfPanel, self.cbPDF.GetValue()) - self.parent.spcPanel.Hide(); - self.parent.cmpPanel.Hide(); - self.parent.mmxPanel.Hide(); - self.parent.plotsizer.Layout() - self.parent.load_and_draw() # Data changes - - def minmax_select(self, event): - self.clear_measures() - self.parent.cbLogY.SetValue(False) - self.parent.show_hide(self.parent.mmxPanel, self.cbMinMax.GetValue()) - self.parent.spcPanel.Hide(); - self.parent.pdfPanel.Hide(); - self.parent.cmpPanel.Hide(); - self.parent.plotsizer.Layout() - self.parent.load_and_draw() # Data changes - - def clear_measures(self): - self.parent.rightMeasure.clear() - self.parent.leftMeasure.clear() - self.parent.lbDeltaX.SetLabel('') - self.parent.lbDeltaY.SetLabel('') - -class EstheticsPanel(wx.Panel): - def __init__(self, parent, data): - wx.Panel.__init__(self, parent) - self.parent=parent - #self.SetBackgroundColour('red') - - # Font - lbFont = wx.StaticText( self, -1, 'Font:') - fontChoices = ['6','7','8','9','10','11','12','13','14','15','16','17','18'] - self.cbFont = wx.ComboBox(self, choices=fontChoices , style=wx.CB_READONLY) - try: - i = fontChoices.index(str(data['Font'])) - except ValueError: - i = 2 - self.cbFont.SetSelection(i) - # Legend - # NOTE: we don't offer "best" since best is slow - lbLegend = wx.StaticText( self, -1, 'Legend:') - lbChoices = ['None','Upper right','Upper left','Lower left','Lower right','Right','Center left','Center right','Lower center','Upper center','Center'] - self.cbLegend = wx.ComboBox(self, choices=lbChoices, style=wx.CB_READONLY) - try: - i = lbChoices.index(str(data['LegendPosition'])) - except ValueError: - i=1 - self.cbLegend.SetSelection(i) - # Legend Font - lbLgdFont = wx.StaticText( self, -1, 'Legend font:') - self.cbLgdFont = wx.ComboBox(self, choices=fontChoices, style=wx.CB_READONLY) - try: - i = fontChoices.index(str(data['LegendFont'])) - except ValueError: - i = 2 - self.cbLgdFont.SetSelection(i) - # Line Width Font - lbLW = wx.StaticText( self, -1, 'Line width:') - LWChoices = ['0.5','1.0','1.25','1.5','2.0','2.5','3.0'] - self.cbLW = wx.ComboBox(self, choices=LWChoices , style=wx.CB_READONLY) - try: - i = LWChoices.index(str(data['LineWidth'])) - except ValueError: - i = 3 - self.cbLW.SetSelection(i) - # Marker Size - lbMS = wx.StaticText( self, -1, 'Marker size:') - MSChoices = ['0.5','1','2','3','4','5','6','7','8'] - self.cbMS= wx.ComboBox(self, choices=MSChoices, style=wx.CB_READONLY) - try: - i = MSChoices.index(str(data['MarkerSize'])) - except ValueError: - i = 2 - self.cbMS.SetSelection(i) - - # Layout - #dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) - dummy_sizer = wx.WrapSizer(orient=wx.HORIZONTAL) - dummy_sizer.Add(lbFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(self.cbFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbLW ,0, flag = wx.CENTER|wx.LEFT,border = 5) - dummy_sizer.Add(self.cbLW ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbMS ,0, flag = wx.CENTER|wx.LEFT,border = 5) - dummy_sizer.Add(self.cbMS ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbLegend ,0, flag = wx.CENTER|wx.LEFT,border = 5) - dummy_sizer.Add(self.cbLegend ,0, flag = wx.CENTER|wx.LEFT,border = 1) - dummy_sizer.Add(lbLgdFont ,0, flag = wx.CENTER|wx.LEFT,border = 5) - dummy_sizer.Add(self.cbLgdFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) - self.SetSizer(dummy_sizer) - self.Hide() - # Callbacks - self.Bind(wx.EVT_COMBOBOX ,self.onAnyEsthOptionChange) - self.cbFont.Bind(wx.EVT_COMBOBOX ,self.onFontOptionChange) - - def onAnyEsthOptionChange(self,event=None): - self.parent.redraw_same_data() - - def onFontOptionChange(self,event=None): - matplotlib_rc('font', **{'size':int(self.cbFont.Value) }) # affect all (including ticks) - self.onAnyEsthOptionChange() - - -class PlotPanel(wx.Panel): - def __init__(self, parent, selPanel, pipeline=None, infoPanel=None, data=None): - - # Superclass constructor - super(PlotPanel,self).__init__(parent) - - # Font handling - font = parent.GetFont() - font.SetPointSize(font.GetPointSize()-1) - self.SetFont(font) - # Preparing a special font manager for chinese characters - self.specialFont=None - try: - pyplot_path = matplotlib.get_data_path() - except: - pyplot_path = pyplot_rc['datapath'] - CH_F_PATHS = [ - os.path.join(pyplot_path, 'fonts/ttf/SimHei.ttf'), - os.path.join(os.path.dirname(__file__),'../SimHei.ttf')] - for fpath in CH_F_PATHS: - if os.path.exists(fpath): - fontP = font_manager.FontProperties(fname=fpath) - fontP.set_size(font.GetPointSize()) - self.specialFont=fontP - break - # data - self.selPanel = selPanel # <<< dependency with selPanel should be minimum - self.pipeline = pipeline # - self.selMode = '' - self.infoPanel=infoPanel - if self.infoPanel is not None: - self.infoPanel.setPlotMatrixCallbacks(self._onPlotMatrixLeftClick, self._onPlotMatrixRightClick) - self.parent = parent - self.plotData = [] - self.plotDataOptions=dict() - if data is not None: - self.data = data - else: - print('>>> Using default settings for plot panel') - self.data = self.defaultData() - if self.selPanel is not None: - bg=self.selPanel.BackgroundColour - self.SetBackgroundColour(bg) # sowhow, our parent has a wrong color - #self.SetBackgroundColour('red') - self.leftMeasure = GUIMeasure(1, 'firebrick') - self.rightMeasure = GUIMeasure(2, 'darkgreen') - self.xlim_prev = [[0, 1]] - self.ylim_prev = [[0, 1]] - self.addTablesCallback = None - - # --- GUI - self.fig = Figure(facecolor="white", figsize=(1, 1)) - register_matplotlib_converters() - self.canvas = FigureCanvas(self, -1, self.fig) - self.canvas.mpl_connect('motion_notify_event', self.onMouseMove) - self.canvas.mpl_connect('button_press_event', self.onMouseClick) - self.canvas.mpl_connect('button_release_event', self.onMouseRelease) - self.canvas.mpl_connect('draw_event', self.onDraw) - self.clickLocation = (None, 0, 0) - - self.navTBTop = MyNavigationToolbar2Wx(self.canvas, ['Home', 'Pan']) - self.navTBBottom = MyNavigationToolbar2Wx(self.canvas, ['Subplots', 'Save']) - TBAddCheckTool(self.navTBBottom,'', icons.chart.GetBitmap(), self.onEsthToggle) - self.esthToggle=False - - self.navTBBottom.Realize() - - #self.navTB = wx.ToolBar(self, style=wx.TB_HORIZONTAL|wx.TB_HORZ_LAYOUT|wx.TB_NODIVIDER|wx.TB_FLAT) - #self.navTB.SetMargins(0,0) - #self.navTB.SetToolPacking(0) - #self.navTB.AddCheckTool(-1, label='', bitmap1=icons.chart.GetBitmap()) - #self.navTB.Realize() - - self.toolbar_sizer = wx.BoxSizer(wx.VERTICAL) - self.toolbar_sizer.Add(self.navTBTop) - self.toolbar_sizer.Add(self.navTBBottom) - - - # --- Tool Panel - self.toolSizer= wx.BoxSizer(wx.VERTICAL) - # --- PlotType Panel - self.pltTypePanel= PlotTypePanel(self); - # --- Plot type specific options - self.spcPanel = SpectralCtrlPanel(self) - self.pdfPanel = PDFCtrlPanel(self) - self.cmpPanel = CompCtrlPanel(self) - self.mmxPanel = MinMaxPanel(self) - # --- Esthetics panel - self.esthPanel = EstheticsPanel(self, data=self.data['plotStyle']) - - - # --- Ctrl Panel - self.ctrlPanel= wx.Panel(self) - #self.ctrlPanel.SetBackgroundColour('blue') - # Check Boxes - self.cbCurveType = wx.ComboBox(self.ctrlPanel, choices=['Plain','LS','Markers','Mix'] , style=wx.CB_READONLY) - self.cbCurveType.SetSelection(1) - self.cbSub = wx.CheckBox(self.ctrlPanel, -1, 'Subplot',(10,10)) - self.cbLogX = wx.CheckBox(self.ctrlPanel, -1, 'Log-x',(10,10)) - self.cbLogY = wx.CheckBox(self.ctrlPanel, -1, 'Log-y',(10,10)) - self.cbSync = wx.CheckBox(self.ctrlPanel, -1, 'Sync-x',(10,10)) - self.cbXHair = wx.CheckBox(self.ctrlPanel, -1, 'CrossHair',(10,10)) - self.cbPlotMatrix = wx.CheckBox(self.ctrlPanel, -1, 'Matrix',(10,10)) - self.cbAutoScale = wx.CheckBox(self.ctrlPanel, -1, 'AutoScale',(10,10)) - self.cbGrid = wx.CheckBox(self.ctrlPanel, -1, 'Grid',(10,10)) - self.cbStepPlot = wx.CheckBox(self.ctrlPanel, -1, 'StepPlot',(10,10)) - self.cbMeasure = wx.CheckBox(self.ctrlPanel, -1, 'Measure',(10,10)) - #self.cbSub.SetValue(True) # DEFAULT TO SUB? - self.cbSync.SetValue(True) - self.cbXHair.SetValue(self.data['CrossHair']) # Have cross hair by default - self.cbAutoScale.SetValue(True) - self.cbGrid.SetValue(self.data['Grid']) - # Callbacks - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSub ) - self.Bind(wx.EVT_COMBOBOX, self.redraw_event , self.cbCurveType) - self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogX ) - self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogY ) - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSync ) - self.Bind(wx.EVT_CHECKBOX, self.crosshair_event , self.cbXHair ) - self.Bind(wx.EVT_CHECKBOX, self.plot_matrix_select, self.cbPlotMatrix ) - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbAutoScale ) - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbGrid ) - self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbStepPlot ) - self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) - self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) - # LAYOUT - cb_sizer = wx.FlexGridSizer(rows=4, cols=3, hgap=0, vgap=0) - cb_sizer.Add(self.cbCurveType , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbSub , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbAutoScale , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbLogX , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbLogY , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbStepPlot , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbXHair , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbGrid , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbSync , 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbPlotMatrix, 0, flag=wx.ALL, border=1) - cb_sizer.Add(self.cbMeasure , 0, flag=wx.ALL, border=1) - - self.ctrlPanel.SetSizer(cb_sizer) - - # --- Crosshair Panel - crossHairPanel= wx.Panel(self) - self.lbCrossHairX = wx.StaticText(crossHairPanel, -1, 'x = ... ') - self.lbCrossHairY = wx.StaticText(crossHairPanel, -1, 'y = ... ') - self.lbDeltaX = wx.StaticText(crossHairPanel, -1, ' ') - self.lbDeltaY = wx.StaticText(crossHairPanel, -1, ' ') - self.lbCrossHairX.SetFont(getMonoFont(self)) - self.lbCrossHairY.SetFont(getMonoFont(self)) - self.lbDeltaX.SetFont(getMonoFont(self)) - self.lbDeltaY.SetFont(getMonoFont(self)) - cbCH = wx.FlexGridSizer(rows=4, cols=1, hgap=0, vgap=0) - cbCH.Add(self.lbCrossHairX , 0, flag=wx.ALL, border=1) - cbCH.Add(self.lbCrossHairY , 0, flag=wx.ALL, border=1) - cbCH.Add(self.lbDeltaX , 0, flag=wx.ALL, border=1) - cbCH.Add(self.lbDeltaY , 0, flag=wx.ALL, border=1) - crossHairPanel.SetSizer(cbCH) - - # --- layout of panels - row_sizer = wx.BoxSizer(wx.HORIZONTAL) - sl2 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) - sl3 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) - sl4 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) - row_sizer.Add(self.pltTypePanel , 0 , flag=wx.LEFT|wx.RIGHT|wx.CENTER , border=1) - row_sizer.Add(sl2 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) - row_sizer.Add(self.toolbar_sizer, 0 , flag=wx.LEFT|wx.RIGHT|wx.CENTER , border=1) - row_sizer.Add(sl3 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) - row_sizer.Add(self.ctrlPanel , 1 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) - row_sizer.Add(sl4 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) - row_sizer.Add(crossHairPanel , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=1) - - plotsizer = wx.BoxSizer(wx.VERTICAL) - self.slCtrl = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) - self.slCtrl.Hide() - self.slEsth = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) - self.slEsth.Hide() - sl1 = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) - plotsizer.Add(self.toolSizer,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.canvas ,1,flag = wx.EXPAND,border = 5 ) - plotsizer.Add(sl1 ,0,flag = wx.EXPAND,border = 0) - plotsizer.Add(self.spcPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.pdfPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.cmpPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.mmxPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.slEsth ,0,flag = wx.EXPAND,border = 0) - plotsizer.Add(self.esthPanel,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) - plotsizer.Add(self.slCtrl ,0,flag = wx.EXPAND,border = 0) - plotsizer.Add(row_sizer ,0,flag = wx.EXPAND|wx.NORTH ,border = 2) - - self.show_hide(self.spcPanel, self.pltTypePanel.cbFFT.GetValue()) - self.show_hide(self.cmpPanel, self.pltTypePanel.cbCompare.GetValue()) - self.show_hide(self.pdfPanel, self.pltTypePanel.cbPDF.GetValue()) - self.show_hide(self.mmxPanel, self.pltTypePanel.cbMinMax.GetValue()) - - self.SetSizer(plotsizer) - self.plotsizer=plotsizer; - self.set_subplot_spacing(init=True) - - # --- Bindings/callback - def setAddTablesCallback(self, callback): - self.addTablesCallback = callback - - def addTables(self, *args, **kwargs): - if self.addTablesCallback is not None: - self.addTablesCallback(*args, **kwargs) - else: - print('[WARN] callback to add tables to parent was not set.') - - - # --- GUI DATA - def saveData(self, data): - data['Grid'] = self.cbGrid.IsChecked() - data['CrossHair'] = self.cbXHair.IsChecked() - data['plotStyle']['Font'] = self.esthPanel.cbFont.GetValue() - data['plotStyle']['LegendFont'] = self.esthPanel.cbLgdFont.GetValue() - data['plotStyle']['LegendPosition'] = self.esthPanel.cbLegend.GetValue() - data['plotStyle']['LineWidth'] = self.esthPanel.cbLW.GetValue() - data['plotStyle']['MarkerSize'] = self.esthPanel.cbMS.GetValue() - - @staticmethod - def defaultData(): - data={} - data['CrossHair']=True - data['Grid']=False - plotStyle = dict() - plotStyle['Font'] = '11' - plotStyle['LegendFont'] = '11' - plotStyle['LegendPosition'] = 'Upper right' - plotStyle['LineWidth'] = '1.5' - plotStyle['MarkerSize'] = '2' - data['plotStyle']= plotStyle - return data - - def onEsthToggle(self,event): - self.esthToggle=not self.esthToggle - if self.esthToggle: - self.slCtrl.Show() - self.esthPanel.Show() - else: - self.slCtrl.Hide() - self.esthPanel.Hide() - self.plotsizer.Layout() - event.Skip() - - def set_subplot_spacing(self, init=False): - """ - Handle default subplot spacing - - NOTE: - - Tight fails when the ylabel is too long, especially for fft with multiplt signals - - might need to change depending on window size/resizing - - need to change if right axis needed - - this will override the user settings - """ - #self.fig.set_tight_layout(True) # NOTE: works almost fine, but problem with FFT multiple - # TODO this is definitely not generic, but tight fails.. - if init: - # NOTE: at init size is (20,20) because sizer is not initialized yet - bottom = 0.12 - left = 0.12 - else: - if self.Size[1]<300: - bottom=0.20 - elif self.Size[1]<350: - bottom=0.18 - elif self.Size[1]<430: - bottom=0.16 - elif self.Size[1]<600: - bottom=0.13 - elif self.Size[1]<800: - bottom=0.09 - else: - bottom=0.07 - if self.Size[0]<300: - left=0.22 - elif self.Size[0]<450: - left=0.20 - elif self.Size[0]<950: - left=0.12 - else: - left=0.06 - #print(self.Size,'bottom', bottom, 'left',left) - if self.cbPlotMatrix.GetValue(): # TODO detect it - self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98-left) - else: - self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98) - - def plot_matrix_select(self, event): - if self.infoPanel is not None: - self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) - self.redraw_same_data() - - def measure_select(self, event): - if self.cbMeasure.IsChecked(): - self.cbAutoScale.SetValue(False) - self.redraw_same_data() - - def redraw_event(self, event): - self.redraw_same_data() - - def log_select(self, event): - if self.pltTypePanel.cbPDF.GetValue(): - self.cbLogX.SetValue(False) - self.cbLogY.SetValue(False) - else: - self.redraw_same_data() - - def crosshair_event(self, event): - try: - self.multiCursors.vertOn =self.cbXHair.GetValue() - self.multiCursors.horizOn=self.cbXHair.GetValue() - self.multiCursors._update() - except: - pass - - def show_hide(self,panel,bShow): - if bShow: - panel.Show() - self.slEsth.Show() - else: - self.slEsth.Hide() - panel.Hide() - - @property - def sharex(self): - return self.cbSync.IsChecked() and (not self.pltTypePanel.cbPDF.GetValue()) - - def set_subplots(self,nPlots): - self.set_subplot_spacing() - # Creating subplots - for ax in self.fig.axes: - self.fig.delaxes(ax) - sharex=None - for i in range(nPlots): - # Vertical stack - if i==0: - ax=self.fig.add_subplot(nPlots,1,i+1) - if self.sharex: - sharex=ax - else: - ax=self.fig.add_subplot(nPlots,1,i+1,sharex=sharex) - # Horizontal stack - #self.fig.add_subplot(1,nPlots,i+1) - - def onMouseMove(self, event): - if event.inaxes: - x, y = event.xdata, event.ydata - self.lbCrossHairX.SetLabel('x =' + self.formatLabelValue(x)) - self.lbCrossHairY.SetLabel('y =' + self.formatLabelValue(y)) - - def onMouseClick(self, event): - self.clickLocation = (event.inaxes, event.xdata, event.ydata) - - def onMouseRelease(self, event): - if self.cbMeasure.GetValue(): - for ax, ax_idx in zip(self.fig.axes, range(len(self.fig.axes))): - if event.inaxes == ax: - x, y = event.xdata, event.ydata - if self.clickLocation != (ax, x, y): - # Ignore measurements for zoom-actions. Possibly add small tolerance. - # Zoom-actions disable autoscale - self.cbAutoScale.SetValue(False) - return - if event.button == 1: - if self.infoPanel is not None: - self.infoPanel.setMeasurements((x, y), None) - self.leftMeasure.set(ax_idx, x, y) - self.leftMeasure.plot(ax, ax_idx) - elif event.button == 3: - if self.infoPanel is not None: - self.infoPanel.setMeasurements(None, (x, y)) - self.rightMeasure.set(ax_idx, x, y) - self.rightMeasure.plot(ax, ax_idx) - else: - return - if not self.cbAutoScale.IsChecked(): - self._restore_limits() - - if self.leftMeasure.axis_idx == self.rightMeasure.axis_idx and self.leftMeasure.axis_idx != -1: - self.lbDeltaX.SetLabel('dx=' + self.formatLabelValue(self.rightMeasure.x - self.leftMeasure.x)) - self.lbDeltaY.SetLabel('dy=' + self.formatLabelValue(self.rightMeasure.y - self.leftMeasure.y)) - else: - self.lbDeltaX.SetLabel('') - self.lbDeltaY.SetLabel('') - return - - def onDraw(self, event): - self._store_limits() - - def formatLabelValue(self, value): - try: - if abs(value)<1000 and abs(value)>1e-4: - s = '{:10.5f}'.format(value) - else: - s = '{:10.3e}'.format(value) - except TypeError: - s = ' ' - return s - - def removeTools(self,event=None,Layout=True): - try: - self.toolPanel.destroy() # call the "destroy" function which might clean up data - except: - pass - try: - # Python3 - self.toolSizer.Clear(delete_windows=True) # Delete Windows - except: - # Python2 - if hasattr(self,'toolPanel'): - self.toolSizer.Remove(self.toolPanel) - if hasattr(self.toolPanel,'action'): - self.toolPanel.action.guiEditorObj = None # Important - self.toolPanel.action = None - self.toolPanel.Destroy() - del self.toolPanel - self.toolSizer.Clear() # Delete Windows - if Layout: - self.plotsizer.Layout() - - def showTool(self, toolName=''): - from .GUITools import TOOLS - if toolName in TOOLS.keys(): - self.showToolPanel(panelClass=TOOLS[toolName]) - else: - raise Exception('Unknown tool {}'.format(toolName)) - - def showToolAction(self, action): - """ Show a tool panel based on an action""" - self.showToolPanel(panelClass=action.guiEditorClass, action=action) - - - def showToolPanel(self, panelClass=None, panel=None, action=None): - """ Show a tool panel based on a panel class (should inherit from GUIToolPanel)""" - self.Freeze() - self.removeTools(Layout=False) - if panel is not None: - self.toolPanel=panel # use the panel directly - else: - if action is None: - print('NOTE: calling a panel without action') - self.toolPanel=panelClass(parent=self) # calling the panel constructor - else: - self.toolPanel=panelClass(parent=self, action=action) # calling the panel constructor - action.guiEditorObj = self.toolPanel - self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) - self.plotsizer.Layout() - self.Thaw() - - - def setPD_PDF(self,PD,c): - """ Convert plot data to PDF data based on GUI options""" - # ---PDF - nBins = self.pdfPanel.scBins.GetValue() - bSmooth = self.pdfPanel.cbSmooth.GetValue() - nBins_out= PD.toPDF(nBins,bSmooth) - if nBins_out!=nBins: - self.pdfPanel.scBins.SetValue(nBins) - - def setPD_MinMax(self,PD): - """ Convert plot data to MinMax data based on GUI options""" - yScale=self.mmxPanel.cbyMinMax.IsChecked() - xScale=self.mmxPanel.cbxMinMax.IsChecked() - try: - PD.toMinMax(xScale,yScale) - except Exception as e: - self.mmxPanel.cbxMinMax.SetValue(False) - raise e # Used to be Warn - - def setPD_FFT(self,pd): - """ Convert plot data to FFT data based on GUI options""" - yType = self.spcPanel.cbType.GetStringSelection() - xType = self.spcPanel.cbTypeX.GetStringSelection() - avgMethod = self.spcPanel.cbAveraging.GetStringSelection() - avgWindow = self.spcPanel.cbAveragingMethod.GetStringSelection() - bDetrend = self.spcPanel.cbDetrend.IsChecked() - nExp = self.spcPanel.scP2.GetValue() - nPerDecade = self.spcPanel.scP2.GetValue() - # Convert plotdata to FFT data - try: - Info = pd.toFFT(yType=yType, xType=xType, avgMethod=avgMethod, avgWindow=avgWindow, bDetrend=bDetrend, nExp=nExp, nPerDecade=nPerDecade) - # Trigger - if hasattr(Info,'nExp') and Info.nExp!=nExp: - self.spcPanel.scP2.SetValue(Info.nExp) - self.spcPanel.updateP2(Info.nExp) - except Exception as e: - self.spcPanel.Hide(); - self.plotsizer.Layout() - raise e - - - def transformPlotData(self,PD): - """" - Apply MinMax, PDF or FFT transform to plot based on GUI data - """ - plotType=self.pltTypePanel.plotType() - if plotType=='MinMax': - self.setPD_MinMax(PD) - elif plotType=='PDF': - self.setPD_PDF(PD,PD.c) - elif plotType=='FFT': - self.setPD_FFT(PD) - - def getPlotData(self,plotType): - - ID,SameCol,selMode=self.selPanel.getPlotDataSelection() - - self.selMode=selMode # we store the selection mode - del self.plotData - self.plotData=[] - tabs=self.selPanel.tabList.getTabs() # TODO, selPanel should just return the PlotData... - - try: - for i,idx in enumerate(ID): - # Initialize each plotdata based on selected table and selected id channels - pd=PlotData(); - pd.fromIDs(tabs, i, idx, SameCol, self.plotDataOptions, pipeline=self.pipeline) - # Possible change of data - if plotType=='MinMax': - self.setPD_MinMax(pd) - elif plotType=='PDF': - self.setPD_PDF(pd,pd.c) - elif plotType=='FFT': - self.setPD_FFT(pd) - self.plotData.append(pd) - except Exception as e: - self.plotData=[] - raise e - - def PD_Compare(self,mode): - """ Perform comparison of the selected PlotData, returns new plotData with the comparison. """ - sComp = self.cmpPanel.rbType.GetStringSelection() - try: - self.plotData = compareMultiplePD(self.plotData,mode, sComp) - except Exception as e: - self.pltTypePanel.cbRegular.SetValue(True) - raise e - - def _onPlotMatrixLeftClick(self, event): - """Toggle plot-states from None, to left-axis, to right-axis. - Left-click goes forwards, right-click goes backwards. - IndexError to avoid "holes" in matrix with outer adjacent populated entries - """ - btn = event.GetEventObject() - label = btn.GetLabelText() - if label == '-': - btn.SetLabel('1') - try: - self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) - except IndexError: - btn.SetLabel('-') - elif label == '1': - btn.SetLabel('2') - else: - btn.SetLabel('-') - try: - self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) - except IndexError: - btn.SetLabel('1') - self.redraw_same_data() - - def _onPlotMatrixRightClick(self, event): - btn = event.GetEventObject() - label = btn.GetLabelText() - if label == '-': - btn.SetLabel('2') - try: - self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) - except IndexError: - btn.SetLabel('-') - elif label == '1': - btn.SetLabel('-') - try: - self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) - except IndexError: - btn.SetLabel('2') - else: - btn.SetLabel('1') - self.redraw_same_data() - - def set_axes_lim(self, PDs, axis): - """ - It's usually faster to set the axis limits first (before plotting) - and disable autoscaling. This way the limits are not recomputed when plot data are added. - Also, we already have computed the min and max, so we leverage that. - NOTE: - doesnt not work with strings - doesnt not work for FFT and compare - - INPUTS: - PDs: list of plot data - """ - # TODO option for tight axes - tight=False - - plotType=self.pltTypePanel.plotType() - if plotType in ['FFT','Compare']: - axis.autoscale(True, axis='both', tight=tight) - return - vXString=[PDs[i].xIsString for i in axis.iPD] - vYString=[PDs[i].yIsString for i in axis.iPD] - if not any(vXString) and not self.cbLogX.IsChecked(): - try: - xMin=np.min([PDs[i]._xMin[0] for i in axis.iPD]) - xMax=np.max([PDs[i]._xMax[0] for i in axis.iPD]) - if np.isclose(xMin,xMax): - delta=1 if np.isclose(xMax,0) else 0.1*xMax - else: - if tight: - delta=0 - else: - delta = (xMax-xMin)*pyplot_rc['axes.xmargin'] - axis.set_xlim(xMin-delta,xMax+delta) - axis.autoscale(False, axis='x', tight=False) - except: - pass - if not any(vYString) and not self.cbLogY.IsChecked(): - try: - yMin=np.min([PDs[i]._yMin[0] for i in axis.iPD]) - yMax=np.max([PDs[i]._yMax[0] for i in axis.iPD]) - delta = (yMax-yMin)*pyplot_rc['axes.ymargin'] - if np.isclose(yMin,yMax): - delta=1 if np.isclose(yMax,0) else 0.1*yMax - else: - if tight: - delta=0 - else: - delta = (yMax-yMin)*pyplot_rc['axes.xmargin'] - axis.set_ylim(yMin-delta,yMax+delta) - axis.autoscale(False, axis='y', tight=False) - except: - pass - - def plot_all(self, keep_limits=True): - self.multiCursors=[] - - if self.cbMeasure.GetValue() is False: - for measure in [self.leftMeasure, self.rightMeasure]: - measure.clear() - if self.infoPanel is not None: - self.infoPanel.setMeasurements(None, None) - self.lbDeltaX.SetLabel('') - self.lbDeltaY.SetLabel('') - - axes=self.fig.axes - PD=self.plotData - - - # --- Plot options - bStep = self.cbStepPlot.IsChecked() - plot_options = dict() - plot_options['lw']=float(self.esthPanel.cbLW.Value) - plot_options['ms']=float(self.esthPanel.cbMS.Value) - if self.cbCurveType.Value=='Plain': - plot_options['LineStyles'] = ['-'] - plot_options['Markers'] = [''] - elif self.cbCurveType.Value=='LS': - plot_options['LineStyles'] = ['-','--','-.',':'] - plot_options['Markers'] = [''] - elif self.cbCurveType.Value=='Markers': - plot_options['LineStyles'] = [''] - plot_options['Markers'] = ['o','d','v','^','s'] - elif self.cbCurveType.Value=='Mix': # NOTE, can be improved - plot_options['LineStyles'] = ['-','--', '-','-','-'] - plot_options['Markers'] = ['' ,'' ,'o','^','s'] - else: - # Combination of linestyles markers, colors, etc. - # But at that stage, if the user really want this, then we can implement an option to set styles per plot. Not high priority. - raise Exception('Not implemented') - - - - # --- Font options - font_options = dict() - font_options_legd = dict() - font_options['size'] = int(self.esthPanel.cbFont.Value) # affect labels - font_options_legd['fontsize'] = int(self.esthPanel.cbLgdFont.Value) - needChineseFont = any([pd.needChineseFont for pd in PD]) - if needChineseFont and self.specialFont is not None: - font_options['fontproperties']= self.specialFont - font_options_legd['prop'] = self.specialFont - - # --- Loop on axes. Either use ax.iPD to chose the plot data, or rely on plotmatrix - for axis_idx, ax_left in enumerate(axes): - ax_right = None - # Checks - vDate=[PD[i].yIsDate for i in ax_left.iPD] - if any(vDate) and len(vDate)>1: - Error(self,'Cannot plot date and other value on the same axis') - return - - # Set limit before plot when possible, for optimization - self.set_axes_lim(PD, ax_left) - - # Actually plot - if self.infoPanel is not None: - pm = self.infoPanel.getPlotMatrix(PD, self.cbSub.IsChecked()) - else: - pm = None - __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) - ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) - - if self.infoPanel is not None: - self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) - for measure in [self.leftMeasure, self.rightMeasure]: - measure.plot(ax_left, axis_idx) - - # Log Axes - if self.cbLogX.IsChecked(): - try: - ax_left.set_xscale("log", nonpositive='clip') # latest - except: - ax_left.set_xscale("log", nonposx='clip') # legacy - - if self.cbLogY.IsChecked(): - if bAllNegLeft is False: - try: - ax_left.set_yscale("log", nonpositive='clip') # latest - except: - ax_left.set_yscale("log", nonposy='clip') - if bAllNegRight is False and ax_right is not None: - try: - ax_right.set_yscale("log", nonpositive='clip') # latest - except: - ax_left.set_yscale("log", nonposy='clip') # legacy - - # XLIM - TODO FFT ONLY NASTY - if self.pltTypePanel.cbFFT.GetValue(): - try: - if self.cbAutoScale.IsChecked(): - xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) - if xlim>0: - ax_left.set_xlim([0,xlim]) - pd=PD[ax_left.iPD[0]] - I=pd.x 0 and len(yleft_labels) <= 3: - ax_left.set_ylabel(' and '.join(yleft_labels), **font_options) - elif ax_left is not None: - ax_left.set_ylabel('') - if len(yright_labels) > 0 and len(yright_labels) <= 3: - ax_right.set_ylabel(' and '.join(yright_labels), **font_options) - elif ax_right is not None: - ax_right.set_ylabel('') - - # Legends - lgdLoc = self.esthPanel.cbLegend.Value.lower() - if (self.pltTypePanel.cbCompare.GetValue() or - ((len(yleft_legends) + len(yright_legends)) > 1)): - if lgdLoc !='none': - if len(yleft_legends) > 0: - ax_left.legend(fancybox=False, loc=lgdLoc, **font_options_legd) - if ax_right is not None and len(yright_legends) > 0: - ax_right.legend(fancybox=False, loc=4, **font_options_legd) - elif len(axes)>1 and len(axes)==len(PD): - # TODO: can this be removed? If there is only one unique signal - # per subplot, normally only ylabel is displayed and no legend. - # Special case when we have subplots and all plots have the same label - if lgdLoc !='none': - usy = unique([pd.sy for pd in PD]) - if len(usy)==1: - for ax in axes: - ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) - - axes[-1].set_xlabel(PD[axes[-1].iPD[0]].sx, **font_options) - - #print('sy :',[pd.sy for pd in PD]) - #print('syl:',[pd.syl for pd in PD]) - - # --- Cursors for each individual plot - # NOTE: cursors needs to be stored in the object! - #for ax_left in self.fig.axes: - # self.cursors.append(MyCursor(ax_left,horizOn=True, vertOn=False, useblit=True, color='gray', linewidth=0.5, linestyle=':')) - # Vertical cusor for all, commonly - bXHair = self.cbXHair.GetValue() - self.multiCursors = MyMultiCursor(self.canvas, tuple(self.fig.axes), useblit=True, horizOn=bXHair, vertOn=bXHair, color='gray', linewidth=0.5, linestyle=':') - - def plotSignals(self, ax, axis_idx, PD, pm, left_right, is_step, opts): - axis = None - bAllNeg = True - if pm is None: - loop_range = ax.iPD - else: - loop_range = range(len(PD)) - - iPlot=-1 - for signal_idx in loop_range: - do_plot = False - if left_right == 1 and (pm is None or pm[signal_idx][axis_idx] == left_right): - do_plot = True - axis = ax - elif left_right == 2 and pm is not None and pm[signal_idx][axis_idx] == left_right: - do_plot = True - if axis is None: - axis = ax.twinx() - ax.set_zorder(axis.get_zorder()+1) - ax.patch.set_visible(False) - axis._get_lines.prop_cycler = ax._get_lines.prop_cycler - pd=PD[signal_idx] - if do_plot: - iPlot+=1 - # --- styling per plot - if len(pd.x)==1: - marker='o'; ls='' - else: - # TODO allow PlotData to override for "per plot" options in the future - marker = opts['Markers'][np.mod(iPlot,len(opts['Markers']))] - ls = opts['LineStyles'][np.mod(iPlot,len(opts['LineStyles']))] - if is_step: - plot = axis.step - else: - plot = axis.plot - plot(pd.x,pd.y,label=pd.syl,ms=opts['ms'], lw=opts['lw'], marker=marker, ls=ls) - try: - bAllNeg = bAllNeg and all(pd.y<=0) - except: - pass # Dates or strings - return axis, bAllNeg - - def findPlotMode(self,PD): - uTabs = unique([pd.it for pd in PD]) - usy = unique([pd.sy for pd in PD]) - uiy = unique([pd.iy for pd in PD]) - if len(uTabs)<=0: - raise Exception('No Table. Contact developer') - if len(uTabs)==1: - mode='1Tab_nCols' - else: - if PD[0].SameCol: - mode='nTabs_SameCols' - else: - # Now that we allow multiple selections detecting "simColumns" is more difficult - if len(uTabs) == len(PD): - mode='nTabs_1Col' - elif self.selMode=='simColumnsMode': - mode='nTabs_SimCols' - else: - mode='nTabs_mCols' - return mode - - def findSubPlots(self,PD,mode): - uTabs = unique([pd.it for pd in PD]) - usy = unique([pd.sy for pd in PD]) - bSubPlots = self.cbSub.IsChecked() - bCompare = self.pltTypePanel.cbCompare.GetValue() # NOTE bCompare somehow always 1Tab_nCols - nSubPlots=1 - spreadBy='none' - if self.infoPanel is not None: - self.infoPanel.setTabMode(mode) # TODO get rid of me - if mode=='1Tab_nCols': - if bSubPlots: - if bCompare or len(uTabs)==1: - if self.infoPanel is not None: - nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) - else: - nSubPlots=len(usy) - else: - nSubPlots=len(usy) - spreadBy='iy' - elif mode=='nTabs_SameCols': - if bSubPlots: - if bCompare: - print('>>>TODO ',mode,len(usy),len(uTabs)) - else: - if len(usy)==1: - # Temporary hack until we have an option for spread by tabs or col - nSubPlots=len(uTabs) - spreadBy='it' - else: - nSubPlots=len(usy) - spreadBy='iy' - elif mode=='nTabs_SimCols': - if bSubPlots: - if bCompare: - print('>>>TODO ',mode,len(usy),len(uTabs)) - else: - nSubPlots=int(len(PD)/len(uTabs)) - spreadBy='mod-ip' - elif mode=='nTabs_mCols': - if bSubPlots: - if bCompare: - print('>>>TODO ',mode,len(usy),len(uTabs)) - else: - if bCompare or len(uTabs)==1: - nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) - else: - nSubPlots=len(PD) - spreadBy='mod-ip' - elif mode=='nTabs_1Col': - if bSubPlots: - if bCompare: - print('>>> TODO',mode,len(uTabs)) - else: - nSubPlots=len(uTabs) - spreadBy='it' - else: - raise Exception('Unknown mode, contact developer.') - return nSubPlots,spreadBy - - def distributePlots(self,mode,nSubPlots,spreadBy): - """ Assigns plot data to axes and axes to plot data """ - axes=self.fig.axes - - # Link plot data to axes - if nSubPlots==1 or spreadBy=='none': - axes[0].iPD=[i for i in range(len(self.plotData))] - else: - for ax in axes: - ax.iPD=[] - PD=self.plotData - uTabs=unique([pd.it for pd in PD]) - uiy=unique([pd.iy for pd in PD]) - if spreadBy=='iy': - for ipd,pd in enumerate(PD): - i=uiy.index(pd.iy) - if i < len(axes): - axes[i].iPD.append(ipd) - elif spreadBy=='it': - for ipd,pd in enumerate(PD): - i=uTabs.index(pd.it) - axes[i].iPD.append(ipd) - elif spreadBy=='mod-ip': - for ipd,pd in enumerate(PD): - i=np.mod(ipd, nSubPlots) - axes[i].iPD.append(ipd) - else: - raise Exception('Wrong spreadby value') - - def setLegendLabels(self,mode): - """ Set labels for legend """ - if mode=='1Tab_nCols': - for pd in self.plotData: - if self.pltTypePanel.cbMinMax.GetValue(): - pd.syl = no_unit(pd.sy) - else: - pd.syl = pd.sy - - elif mode=='nTabs_SameCols': - for pd in self.plotData: - pd.syl=pd.st - - elif mode=='nTabs_1Col': - usy=unique([pd.sy for pd in self.plotData]) - if len(usy)==1: - for pd in self.plotData: - pd.syl=pd.st - else: - for pd in self.plotData: - if self.pltTypePanel.cbMinMax.GetValue(): - pd.syl=no_unit(pd.sy) - else: - pd.syl=pd.sy #pd.syl=pd.st + ' - '+pd.sy - elif mode=='nTabs_SimCols': - bSubPlots = self.cbSub.IsChecked() - if bSubPlots: # spread by table name - for pd in self.plotData: - pd.syl=pd.st - else: - for pd in self.plotData: - pd.syl=pd.st + ' - '+pd.sy - elif mode=='nTabs_mCols': - usy=unique([pd.sy for pd in self.plotData]) - bSubPlots = self.cbSub.IsChecked() - if bSubPlots and len(usy)==1: # spread by table name - for pd in self.plotData: - pd.syl=pd.st - else: - for pd in self.plotData: - pd.syl=pd.st + ' - '+pd.sy - else: - raise Exception('Unknown mode {}'.format(mode)) - - - def empty(self): - self.cleanPlot() - - def clean_memory(self): - if hasattr(self,'plotData'): - del self.plotData - self.plotData=[] - for ax in self.fig.axes: - ax.iPD=[] - self.fig.delaxes(ax) - gc.collect() - - def clean_memory_plot(self): - pass - - def cleanPlot(self): - for ax in self.fig.axes: - if hasattr(ax,'iPD'): - del ax.iPD - self.fig.delaxes(ax) - gc.collect() - self.fig.add_subplot(111) - ax = self.fig.axes[0] - ax.set_axis_off() - #ax.plot(1,1) - self.canvas.draw() - gc.collect() - - def load_and_draw(self): - """ Full draw event: - - Get plot data based on selection - - Plot them - - Trigger changes to infoPanel - - """ - self.clean_memory() - self.getPlotData(self.pltTypePanel.plotType()) - if len(self.plotData)==0: - self.cleanPlot(); - return - mode=self.findPlotMode(self.plotData) - if self.pltTypePanel.cbCompare.GetValue(): - self.PD_Compare(mode) - if len(self.plotData)==0: - self.cleanPlot(); - return - self.redraw_same_data() - if self.infoPanel is not None: - self.infoPanel.showStats(self.plotData,self.pltTypePanel.plotType()) - - def redraw_same_data(self, keep_limits=True): - if len(self.plotData)==0: - self.cleanPlot(); - return - elif len(self.plotData) == 1: - if self.plotData[0].xIsString or self.plotData[0].yIsString or self.plotData[0].xIsDate or self.plotData[0].yIsDate: - self.cbAutoScale.SetValue(True) - else: - if len(self.xlim_prev)==0: # Might occur if some date didn't plot before (e.g. strings) - self.cbAutoScale.SetValue(True) - elif rectangleOverlap(self.plotData[0]._xMin[0], self.plotData[0]._yMin[0], - self.plotData[0]._xMax[0], self.plotData[0]._yMax[0], - self.xlim_prev[0][0], self.ylim_prev[0][0], - self.xlim_prev[0][1], self.ylim_prev[0][1]): - pass - else: - self.cbAutoScale.SetValue(True) - - mode=self.findPlotMode(self.plotData) - nPlots,spreadBy=self.findSubPlots(self.plotData,mode) - - self.clean_memory_plot() - self.set_subplots(nPlots) - self.distributePlots(mode,nPlots,spreadBy) - - if not self.pltTypePanel.cbCompare.GetValue(): - self.setLegendLabels(mode) - - self.plot_all(keep_limits) - self.canvas.draw() - - - def _store_limits(self): - self.xlim_prev = [] - self.ylim_prev = [] - for ax in self.fig.axes: - self.xlim_prev.append(ax.get_xlim()) - self.ylim_prev.append(ax.get_ylim()) - - def _restore_limits(self): - for ax, xlim, ylim in zip(self.fig.axes, self.xlim_prev, self.ylim_prev): - ax.set_xlim(xlim) - ax.set_ylim(ylim) - - -if __name__ == '__main__': - import pandas as pd; - from Tables import Table,TableList - from pydatview.Tables import TableList - from pydatview.GUISelectionPanel import SelectionPanel - - # --- Data - tabList = TableList.createDummy(1) - app = wx.App(False) - self=wx.Frame(None,-1,"GUI Plot Panel Demo") - - # --- Panels - self.selPanel = SelectionPanel(self, tabList, mode='auto') - self.plotPanel = PlotPanel(self, self.selPanel) - self.plotPanel.load_and_draw() # <<< Important - self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # Binding the two - - # --- Finalize GUI - sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(self.selPanel ,0, flag = wx.EXPAND|wx.ALL,border = 5) - sizer.Add(self.plotPanel,1, flag = wx.EXPAND|wx.ALL,border = 5) - self.SetSizer(sizer) - self.Center() - self.SetSize((900, 600)) - self.Show() - app.MainLoop() - - +import os +import numpy as np +import wx +import wx.lib.buttons as buttons +import dateutil # required by matplotlib +#from matplotlib import pyplot as plt +import matplotlib +# Backends: +# ['GTK3Agg', 'GTK3Cairo', 'GTK4Agg', 'GTK4Cairo', 'MacOSX', 'nbAgg', 'QtAgg', 'QtCairo', 'Qt5Agg', 'Qt5Cairo', 'TkAgg', 'TkCairo', 'WebAgg', 'WX', 'WXAgg', 'WXCairo', 'agg', 'cairo', 'pdf', 'pgf', 'ps', 'svg', 'template'] +matplotlib.use('WX') # Important for Windows version of installer. NOTE: changed from Agg to wxAgg, then to WX +from matplotlib import rc as matplotlib_rc +try: + from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas +except Exception as e: + print('') + print('Error: problem importing `matplotlib.backends.backend_wx`.') + import platform + if platform.system()=='Darwin': + print('') + print('pyDatView help:') + print(' This is a typical issue on MacOS, most likely you are') + print(' using the native MacOS python with the native matplolib') + print(' library, which is incompatible with `wxPython`.') + print('') + print(' You can solve this by either:') + print(' - using python3, and pip3 e.g. installing it with brew') + print(' - using a virtual environment with python 3') + print(' - using anaconda with python 3'); + print('') + import sys + sys.exit(1) + else: + raise e +from matplotlib.figure import Figure +from matplotlib.pyplot import rcParams as pyplot_rc +from matplotlib import font_manager +from pandas.plotting import register_matplotlib_converters + +import gc + +from pydatview.common import * # unique, CHAR +from pydatview.plotdata import PlotData, compareMultiplePD +from pydatview.GUICommon import * +from pydatview.GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool +from pydatview.GUIMeasure import GUIMeasure +import pydatview.icons as icons + +font = {'size' : 8} +matplotlib_rc('font', **font) +pyplot_rc['agg.path.chunksize'] = 20000 + + +class PDFCtrlPanel(wx.Panel): + def __init__(self, parent): + super(PDFCtrlPanel,self).__init__(parent) + self.parent = parent + lb = wx.StaticText( self, -1, 'Number of bins:') + self.scBins = wx.SpinCtrl(self, value='51',size=wx.Size(70,-1), style=wx.TE_RIGHT) + self.scBins.SetRange(3, 10000) + self.cbSmooth = wx.CheckBox(self, -1, 'Smooth',(10,10)) + self.cbSmooth.SetValue(False) + dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer.Add(lb ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.scBins ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.cbSmooth ,0, flag = wx.CENTER|wx.LEFT,border = 6) + self.SetSizer(dummy_sizer) + self.Bind(wx.EVT_TEXT , self.onPDFOptionChange, self.scBins) + self.Bind(wx.EVT_CHECKBOX, self.onPDFOptionChange) + self.Hide() + + def onPDFOptionChange(self,event=None): + self.parent.load_and_draw(); # DATA HAS CHANGED + +class MinMaxPanel(wx.Panel): + def __init__(self, parent): + super(MinMaxPanel,self).__init__(parent) + self.parent = parent + self.cbxMinMax = wx.CheckBox(self, -1, 'xMinMax',(10,10)) + self.cbyMinMax = wx.CheckBox(self, -1, 'yMinMax',(10,10)) + self.cbxMinMax.SetValue(False) + self.cbyMinMax.SetValue(True) + dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer.Add(self.cbxMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) + dummy_sizer.Add(self.cbyMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) + self.SetSizer(dummy_sizer) + self.Bind(wx.EVT_CHECKBOX, self.onMinMaxChange) + self.Hide() + + def onMinMaxChange(self,event=None): + self.parent.load_and_draw(); # DATA HAS CHANGED + +class CompCtrlPanel(wx.Panel): + def __init__(self, parent): + super(CompCtrlPanel,self).__init__(parent) + self.parent = parent + lblList = ['Relative', '|Relative|','Ratio','Absolute','Y-Y'] + self.rbType = wx.RadioBox(self, label = 'Type', choices = lblList, + majorDimension = 1, style = wx.RA_SPECIFY_ROWS) + dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer.Add(self.rbType ,0, flag = wx.CENTER|wx.LEFT,border = 1) + self.SetSizer(dummy_sizer) + self.rbType.Bind(wx.EVT_RADIOBOX,self.onTypeChange) + self.Hide() + + def onTypeChange(self,e): + self.parent.load_and_draw(); # DATA HAS CHANGED + + +class SpectralCtrlPanel(wx.Panel): + def __init__(self, parent): + super(SpectralCtrlPanel,self).__init__(parent) + self.parent = parent + # --- GUI widgets + lb = wx.StaticText( self, -1, 'Type:') + self.cbType = wx.ComboBox(self, choices=['PSD','f x PSD','Amplitude'] , style=wx.CB_READONLY) + self.cbType.SetSelection(0) + lbAveraging = wx.StaticText( self, -1, 'Avg.:') + self.cbAveraging = wx.ComboBox(self, choices=['None','Welch','Binning'] , style=wx.CB_READONLY) + self.cbAveraging.SetSelection(1) + self.lbAveragingMethod = wx.StaticText( self, -1, 'Window:') + self.cbAveragingMethod = wx.ComboBox(self, choices=['Hamming','Hann','Rectangular'] , style=wx.CB_READONLY) + self.cbAveragingMethod.SetSelection(0) + self.lbP2 = wx.StaticText( self, -1, '2^n:') + self.scP2 = wx.SpinCtrl(self, value='11',size=wx.Size(40,-1)) + self.lbWinLength = wx.StaticText( self, -1, '(2048) ') + self.scP2.SetRange(3, 50) + self.previousNExp = 8 + self.previousNDec = 20 + lbMaxFreq = wx.StaticText( self, -1, 'Xlim:') + self.tMaxFreq = wx.TextCtrl(self,size = (30,-1),style=wx.TE_PROCESS_ENTER) + self.tMaxFreq.SetValue("-1") + self.cbDetrend = wx.CheckBox(self, -1, 'Detrend',(10,10)) + lbX = wx.StaticText( self, -1, 'x:') + self.cbTypeX = wx.ComboBox(self, choices=['1/x','2pi/x','x'] , style=wx.CB_READONLY) + self.cbTypeX.SetSelection(0) + # Layout + dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer.Add(lb ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.cbType ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbAveraging ,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.cbAveraging ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.lbAveragingMethod,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.cbAveragingMethod,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.lbP2 ,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.scP2 ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.lbWinLength ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbMaxFreq ,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.tMaxFreq ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbX ,0, flag = wx.CENTER|wx.LEFT,border = 6) + dummy_sizer.Add(self.cbTypeX ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.cbDetrend ,0, flag = wx.CENTER|wx.LEFT,border = 7) + self.SetSizer(dummy_sizer) + self.Bind(wx.EVT_COMBOBOX ,self.onSpecCtrlChange) + self.Bind(wx.EVT_TEXT ,self.onP2ChangeText ,self.scP2 ) + self.Bind(wx.EVT_TEXT_ENTER,self.onXlimChange ,self.tMaxFreq ) + self.Bind(wx.EVT_CHECKBOX ,self.onDetrendChange ,self.cbDetrend) + self.Hide() + + def onXlimChange(self,event=None): + self.parent.redraw_same_data(); + def onSpecCtrlChange(self,event=None): + if self.cbAveraging.GetStringSelection()=='None': + self.scP2.Enable(False) + self.cbAveragingMethod.Enable(False) + self.lbP2.SetLabel('') + self.lbWinLength.SetLabel('') + elif self.cbAveraging.GetStringSelection()=='Binning': + self.previousNExp= self.scP2.GetValue() + self.scP2.SetValue(self.previousNDec) + self.scP2.Enable(True) + self.cbAveragingMethod.Enable(False) + self.lbP2.SetLabel('n:') + self.lbWinLength.SetLabel('') + else: + self.previousDec= self.scP2.GetValue() + self.scP2.SetValue(self.previousNExp) + self.lbP2.SetLabel('2^n:') + self.scP2.Enable(True) + self.cbAveragingMethod.Enable(True) + self.onP2ChangeText(event=None) + self.parent.load_and_draw() # Data changes + + def onDetrendChange(self,event=None): + self.parent.load_and_draw() # Data changes + + def onP2ChangeText(self,event=None): + if self.cbAveraging.GetStringSelection()=='Binning': + pass + else: + nExp=self.scP2.GetValue() + self.updateP2(nExp) + self.parent.load_and_draw() # Data changes + + def updateP2(self,P2): + self.lbWinLength.SetLabel("({})".format(2**P2)) + + + + +class PlotTypePanel(wx.Panel): + def __init__(self, parent): + # Superclass constructor + super(PlotTypePanel,self).__init__(parent) + #self.SetBackgroundColour('yellow') + # data + self.parent = parent + # --- Ctrl Panel + self.cbRegular = wx.RadioButton(self, -1, 'Regular',style=wx.RB_GROUP) + self.cbPDF = wx.RadioButton(self, -1, 'PDF' , ) + self.cbFFT = wx.RadioButton(self, -1, 'FFT' , ) + self.cbMinMax = wx.RadioButton(self, -1, 'MinMax' , ) + self.cbCompare = wx.RadioButton(self, -1, 'Compare', ) + self.cbRegular.SetValue(True) + self.Bind(wx.EVT_RADIOBUTTON, self.pdf_select , self.cbPDF ) + self.Bind(wx.EVT_RADIOBUTTON, self.fft_select , self.cbFFT ) + self.Bind(wx.EVT_RADIOBUTTON, self.minmax_select , self.cbMinMax ) + self.Bind(wx.EVT_RADIOBUTTON, self.compare_select, self.cbCompare) + self.Bind(wx.EVT_RADIOBUTTON, self.regular_select, self.cbRegular) + # LAYOUT + cb_sizer = wx.FlexGridSizer(rows=5, cols=1, hgap=0, vgap=0) + cb_sizer.Add(self.cbRegular , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbPDF , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbFFT , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbMinMax , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbCompare , 0, flag=wx.ALL, border=1) + self.SetSizer(cb_sizer) + + def plotType(self): + plotType='Regular' + if self.cbMinMax.GetValue(): + plotType='MinMax' + elif self.cbPDF.GetValue(): + plotType='PDF' + elif self.cbFFT.GetValue(): + plotType='FFT' + elif self.cbCompare.GetValue(): + plotType='Compare' + return plotType + + def regular_select(self, event=None): + self.clear_measures() + self.parent.cbLogY.SetValue(False) + # + self.parent.spcPanel.Hide(); + self.parent.pdfPanel.Hide(); + self.parent.cmpPanel.Hide(); + self.parent.mmxPanel.Hide(); + self.parent.slEsth.Hide(); + self.parent.plotsizer.Layout() + # + self.parent.load_and_draw() # Data changes + + def compare_select(self, event=None): + self.clear_measures() + self.parent.cbLogY.SetValue(False) + self.parent.show_hide(self.parent.cmpPanel, self.cbCompare.GetValue()) + self.parent.spcPanel.Hide(); + self.parent.pdfPanel.Hide(); + self.parent.mmxPanel.Hide(); + self.parent.plotsizer.Layout() + self.parent.load_and_draw() # Data changes + + def fft_select(self, event=None): + self.clear_measures() + self.parent.show_hide(self.parent.spcPanel, self.cbFFT.GetValue()) + self.parent.cbLogY.SetValue(self.cbFFT.GetValue()) + self.parent.pdfPanel.Hide(); + self.parent.mmxPanel.Hide(); + self.parent.plotsizer.Layout() + self.parent.load_and_draw() # Data changes + + def pdf_select(self, event=None): + self.clear_measures() + self.parent.cbLogX.SetValue(False) + self.parent.cbLogY.SetValue(False) + self.parent.show_hide(self.parent.pdfPanel, self.cbPDF.GetValue()) + self.parent.spcPanel.Hide(); + self.parent.cmpPanel.Hide(); + self.parent.mmxPanel.Hide(); + self.parent.plotsizer.Layout() + self.parent.load_and_draw() # Data changes + + def minmax_select(self, event): + self.clear_measures() + self.parent.cbLogY.SetValue(False) + self.parent.show_hide(self.parent.mmxPanel, self.cbMinMax.GetValue()) + self.parent.spcPanel.Hide(); + self.parent.pdfPanel.Hide(); + self.parent.cmpPanel.Hide(); + self.parent.plotsizer.Layout() + self.parent.load_and_draw() # Data changes + + def clear_measures(self): + self.parent.rightMeasure.clear() + self.parent.leftMeasure.clear() + self.parent.lbDeltaX.SetLabel('') + self.parent.lbDeltaY.SetLabel('') + +class EstheticsPanel(wx.Panel): + def __init__(self, parent, data): + wx.Panel.__init__(self, parent) + self.parent=parent + #self.SetBackgroundColour('red') + + # Font + lbFont = wx.StaticText( self, -1, 'Font:') + fontChoices = ['6','7','8','9','10','11','12','13','14','15','16','17','18'] + self.cbFont = wx.ComboBox(self, choices=fontChoices , style=wx.CB_READONLY) + try: + i = fontChoices.index(str(data['Font'])) + except ValueError: + i = 2 + self.cbFont.SetSelection(i) + # Legend + # NOTE: we don't offer "best" since best is slow + lbLegend = wx.StaticText( self, -1, 'Legend:') + lbChoices = ['None','Upper right','Upper left','Lower left','Lower right','Right','Center left','Center right','Lower center','Upper center','Center'] + self.cbLegend = wx.ComboBox(self, choices=lbChoices, style=wx.CB_READONLY) + try: + i = lbChoices.index(str(data['LegendPosition'])) + except ValueError: + i=1 + self.cbLegend.SetSelection(i) + # Legend Font + lbLgdFont = wx.StaticText( self, -1, 'Legend font:') + self.cbLgdFont = wx.ComboBox(self, choices=fontChoices, style=wx.CB_READONLY) + try: + i = fontChoices.index(str(data['LegendFont'])) + except ValueError: + i = 2 + self.cbLgdFont.SetSelection(i) + # Line Width Font + lbLW = wx.StaticText( self, -1, 'Line width:') + LWChoices = ['0.5','1.0','1.25','1.5','2.0','2.5','3.0'] + self.cbLW = wx.ComboBox(self, choices=LWChoices , style=wx.CB_READONLY) + try: + i = LWChoices.index(str(data['LineWidth'])) + except ValueError: + i = 3 + self.cbLW.SetSelection(i) + # Marker Size + lbMS = wx.StaticText( self, -1, 'Marker size:') + MSChoices = ['0.5','1','2','3','4','5','6','7','8'] + self.cbMS= wx.ComboBox(self, choices=MSChoices, style=wx.CB_READONLY) + try: + i = MSChoices.index(str(data['MarkerSize'])) + except ValueError: + i = 2 + self.cbMS.SetSelection(i) + + # Layout + #dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) + dummy_sizer = wx.WrapSizer(orient=wx.HORIZONTAL) + dummy_sizer.Add(lbFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(self.cbFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbLW ,0, flag = wx.CENTER|wx.LEFT,border = 5) + dummy_sizer.Add(self.cbLW ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbMS ,0, flag = wx.CENTER|wx.LEFT,border = 5) + dummy_sizer.Add(self.cbMS ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbLegend ,0, flag = wx.CENTER|wx.LEFT,border = 5) + dummy_sizer.Add(self.cbLegend ,0, flag = wx.CENTER|wx.LEFT,border = 1) + dummy_sizer.Add(lbLgdFont ,0, flag = wx.CENTER|wx.LEFT,border = 5) + dummy_sizer.Add(self.cbLgdFont ,0, flag = wx.CENTER|wx.LEFT,border = 1) + self.SetSizer(dummy_sizer) + self.Hide() + # Callbacks + self.Bind(wx.EVT_COMBOBOX ,self.onAnyEsthOptionChange) + self.cbFont.Bind(wx.EVT_COMBOBOX ,self.onFontOptionChange) + + def onAnyEsthOptionChange(self,event=None): + self.parent.redraw_same_data() + + def onFontOptionChange(self,event=None): + matplotlib_rc('font', **{'size':int(self.cbFont.Value) }) # affect all (including ticks) + self.onAnyEsthOptionChange() + + +class PlotPanel(wx.Panel): + def __init__(self, parent, selPanel, pipeline=None, infoPanel=None, data=None): + + # Superclass constructor + super(PlotPanel,self).__init__(parent) + + # Font handling + font = parent.GetFont() + font.SetPointSize(font.GetPointSize()-1) + self.SetFont(font) + # Preparing a special font manager for chinese characters + self.specialFont=None + try: + pyplot_path = matplotlib.get_data_path() + except: + pyplot_path = pyplot_rc['datapath'] + CH_F_PATHS = [ + os.path.join(pyplot_path, 'fonts/ttf/SimHei.ttf'), + os.path.join(os.path.dirname(__file__),'../SimHei.ttf')] + for fpath in CH_F_PATHS: + if os.path.exists(fpath): + fontP = font_manager.FontProperties(fname=fpath) + fontP.set_size(font.GetPointSize()) + self.specialFont=fontP + break + # data + self.selPanel = selPanel # <<< dependency with selPanel should be minimum + self.pipeline = pipeline # + self.selMode = '' + self.infoPanel=infoPanel + if self.infoPanel is not None: + self.infoPanel.setPlotMatrixCallbacks(self._onPlotMatrixLeftClick, self._onPlotMatrixRightClick) + self.parent = parent + self.plotData = [] + self.plotDataOptions=dict() # TODO remove me + self.toolPanel=None + if data is not None: + self.data = data + else: + print('>>> Using default settings for plot panel') + self.data = self.defaultData() + if self.selPanel is not None: + bg=self.selPanel.BackgroundColour + self.SetBackgroundColour(bg) # sowhow, our parent has a wrong color + #self.SetBackgroundColour('red') + self.leftMeasure = GUIMeasure(1, 'firebrick') + self.rightMeasure = GUIMeasure(2, 'darkgreen') + self.xlim_prev = [[0, 1]] + self.ylim_prev = [[0, 1]] + self.addTablesCallback = None + + # --- GUI + self.fig = Figure(facecolor="white", figsize=(1, 1)) + register_matplotlib_converters() + self.canvas = FigureCanvas(self, -1, self.fig) + self.canvas.mpl_connect('motion_notify_event', self.onMouseMove) + self.canvas.mpl_connect('button_press_event', self.onMouseClick) + self.canvas.mpl_connect('button_release_event', self.onMouseRelease) + self.canvas.mpl_connect('draw_event', self.onDraw) + self.clickLocation = (None, 0, 0) + + self.navTBTop = MyNavigationToolbar2Wx(self.canvas, ['Home', 'Pan']) + self.navTBBottom = MyNavigationToolbar2Wx(self.canvas, ['Subplots', 'Save']) + TBAddCheckTool(self.navTBBottom,'', icons.chart.GetBitmap(), self.onEsthToggle) + self.esthToggle=False + + self.navTBBottom.Realize() + + #self.navTB = wx.ToolBar(self, style=wx.TB_HORIZONTAL|wx.TB_HORZ_LAYOUT|wx.TB_NODIVIDER|wx.TB_FLAT) + #self.navTB.SetMargins(0,0) + #self.navTB.SetToolPacking(0) + #self.navTB.AddCheckTool(-1, label='', bitmap1=icons.chart.GetBitmap()) + #self.navTB.Realize() + + self.toolbar_sizer = wx.BoxSizer(wx.VERTICAL) + self.toolbar_sizer.Add(self.navTBTop) + self.toolbar_sizer.Add(self.navTBBottom) + + + # --- Tool Panel + self.toolSizer= wx.BoxSizer(wx.VERTICAL) + # --- PlotType Panel + self.pltTypePanel= PlotTypePanel(self); + # --- Plot type specific options + self.spcPanel = SpectralCtrlPanel(self) + self.pdfPanel = PDFCtrlPanel(self) + self.cmpPanel = CompCtrlPanel(self) + self.mmxPanel = MinMaxPanel(self) + # --- Esthetics panel + self.esthPanel = EstheticsPanel(self, data=self.data['plotStyle']) + + + # --- Ctrl Panel + self.ctrlPanel= wx.Panel(self) + #self.ctrlPanel.SetBackgroundColour('blue') + # Check Boxes + self.cbCurveType = wx.ComboBox(self.ctrlPanel, choices=['Plain','LS','Markers','Mix'] , style=wx.CB_READONLY) + self.cbCurveType.SetSelection(1) + self.cbSub = wx.CheckBox(self.ctrlPanel, -1, 'Subplot',(10,10)) + self.cbLogX = wx.CheckBox(self.ctrlPanel, -1, 'Log-x',(10,10)) + self.cbLogY = wx.CheckBox(self.ctrlPanel, -1, 'Log-y',(10,10)) + self.cbSync = wx.CheckBox(self.ctrlPanel, -1, 'Sync-x',(10,10)) + self.cbXHair = wx.CheckBox(self.ctrlPanel, -1, 'CrossHair',(10,10)) + self.cbPlotMatrix = wx.CheckBox(self.ctrlPanel, -1, 'Matrix',(10,10)) + self.cbAutoScale = wx.CheckBox(self.ctrlPanel, -1, 'AutoScale',(10,10)) + self.cbGrid = wx.CheckBox(self.ctrlPanel, -1, 'Grid',(10,10)) + self.cbStepPlot = wx.CheckBox(self.ctrlPanel, -1, 'StepPlot',(10,10)) + self.cbMeasure = wx.CheckBox(self.ctrlPanel, -1, 'Measure',(10,10)) + #self.cbSub.SetValue(True) # DEFAULT TO SUB? + self.cbSync.SetValue(True) + self.cbXHair.SetValue(self.data['CrossHair']) # Have cross hair by default + self.cbAutoScale.SetValue(True) + self.cbGrid.SetValue(self.data['Grid']) + # Callbacks + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSub ) + self.Bind(wx.EVT_COMBOBOX, self.redraw_event , self.cbCurveType) + self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogX ) + self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogY ) + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSync ) + self.Bind(wx.EVT_CHECKBOX, self.crosshair_event , self.cbXHair ) + self.Bind(wx.EVT_CHECKBOX, self.plot_matrix_select, self.cbPlotMatrix ) + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbAutoScale ) + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbGrid ) + self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbStepPlot ) + self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) + self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) + # LAYOUT + cb_sizer = wx.FlexGridSizer(rows=4, cols=3, hgap=0, vgap=0) + cb_sizer.Add(self.cbCurveType , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbSub , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbAutoScale , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbLogX , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbLogY , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbStepPlot , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbXHair , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbGrid , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbSync , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbPlotMatrix, 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbMeasure , 0, flag=wx.ALL, border=1) + + self.ctrlPanel.SetSizer(cb_sizer) + + # --- Crosshair Panel + crossHairPanel= wx.Panel(self) + self.lbCrossHairX = wx.StaticText(crossHairPanel, -1, 'x = ... ') + self.lbCrossHairY = wx.StaticText(crossHairPanel, -1, 'y = ... ') + self.lbDeltaX = wx.StaticText(crossHairPanel, -1, ' ') + self.lbDeltaY = wx.StaticText(crossHairPanel, -1, ' ') + self.lbCrossHairX.SetFont(getMonoFont(self)) + self.lbCrossHairY.SetFont(getMonoFont(self)) + self.lbDeltaX.SetFont(getMonoFont(self)) + self.lbDeltaY.SetFont(getMonoFont(self)) + cbCH = wx.FlexGridSizer(rows=4, cols=1, hgap=0, vgap=0) + cbCH.Add(self.lbCrossHairX , 0, flag=wx.ALL, border=1) + cbCH.Add(self.lbCrossHairY , 0, flag=wx.ALL, border=1) + cbCH.Add(self.lbDeltaX , 0, flag=wx.ALL, border=1) + cbCH.Add(self.lbDeltaY , 0, flag=wx.ALL, border=1) + crossHairPanel.SetSizer(cbCH) + + # --- layout of panels + row_sizer = wx.BoxSizer(wx.HORIZONTAL) + sl2 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) + sl3 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) + sl4 = wx.StaticLine(self, -1, size=wx.Size(1,-1), style=wx.LI_VERTICAL) + row_sizer.Add(self.pltTypePanel , 0 , flag=wx.LEFT|wx.RIGHT|wx.CENTER , border=1) + row_sizer.Add(sl2 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) + row_sizer.Add(self.toolbar_sizer, 0 , flag=wx.LEFT|wx.RIGHT|wx.CENTER , border=1) + row_sizer.Add(sl3 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) + row_sizer.Add(self.ctrlPanel , 1 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) + row_sizer.Add(sl4 , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=0) + row_sizer.Add(crossHairPanel , 0 , flag=wx.LEFT|wx.RIGHT|wx.EXPAND|wx.CENTER, border=1) + + plotsizer = wx.BoxSizer(wx.VERTICAL) + self.slCtrl = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) + self.slCtrl.Hide() + self.slEsth = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) + self.slEsth.Hide() + sl1 = wx.StaticLine(self, -1, size=wx.Size(-1,1), style=wx.LI_HORIZONTAL) + plotsizer.Add(self.toolSizer,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.canvas ,1,flag = wx.EXPAND,border = 5 ) + plotsizer.Add(sl1 ,0,flag = wx.EXPAND,border = 0) + plotsizer.Add(self.spcPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.pdfPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.cmpPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.mmxPanel ,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.slEsth ,0,flag = wx.EXPAND,border = 0) + plotsizer.Add(self.esthPanel,0,flag = wx.EXPAND|wx.CENTER|wx.TOP|wx.BOTTOM,border = 10) + plotsizer.Add(self.slCtrl ,0,flag = wx.EXPAND,border = 0) + plotsizer.Add(row_sizer ,0,flag = wx.EXPAND|wx.NORTH ,border = 2) + + self.show_hide(self.spcPanel, self.pltTypePanel.cbFFT.GetValue()) + self.show_hide(self.cmpPanel, self.pltTypePanel.cbCompare.GetValue()) + self.show_hide(self.pdfPanel, self.pltTypePanel.cbPDF.GetValue()) + self.show_hide(self.mmxPanel, self.pltTypePanel.cbMinMax.GetValue()) + + self.SetSizer(plotsizer) + self.plotsizer=plotsizer; + self.set_subplot_spacing(init=True) + + # --- Bindings/callback + def setAddTablesCallback(self, callback): + self.addTablesCallback = callback + + def addTables(self, *args, **kwargs): + if self.addTablesCallback is not None: + self.addTablesCallback(*args, **kwargs) + else: + print('[WARN] callback to add tables to parent was not set.') + + + # --- GUI DATA + def saveData(self, data): + data['Grid'] = self.cbGrid.IsChecked() + data['CrossHair'] = self.cbXHair.IsChecked() + data['plotStyle']['Font'] = self.esthPanel.cbFont.GetValue() + data['plotStyle']['LegendFont'] = self.esthPanel.cbLgdFont.GetValue() + data['plotStyle']['LegendPosition'] = self.esthPanel.cbLegend.GetValue() + data['plotStyle']['LineWidth'] = self.esthPanel.cbLW.GetValue() + data['plotStyle']['MarkerSize'] = self.esthPanel.cbMS.GetValue() + + @staticmethod + def defaultData(): + data={} + data['CrossHair']=True + data['Grid']=False + plotStyle = dict() + plotStyle['Font'] = '11' + plotStyle['LegendFont'] = '11' + plotStyle['LegendPosition'] = 'Upper right' + plotStyle['LineWidth'] = '1.5' + plotStyle['MarkerSize'] = '2' + data['plotStyle']= plotStyle + return data + + def onEsthToggle(self,event): + self.esthToggle=not self.esthToggle + if self.esthToggle: + self.slCtrl.Show() + self.esthPanel.Show() + else: + self.slCtrl.Hide() + self.esthPanel.Hide() + self.plotsizer.Layout() + event.Skip() + + def set_subplot_spacing(self, init=False): + """ + Handle default subplot spacing + + NOTE: + - Tight fails when the ylabel is too long, especially for fft with multiplt signals + - might need to change depending on window size/resizing + - need to change if right axis needed + - this will override the user settings + """ + #self.fig.set_tight_layout(True) # NOTE: works almost fine, but problem with FFT multiple + # TODO this is definitely not generic, but tight fails.. + if init: + # NOTE: at init size is (20,20) because sizer is not initialized yet + bottom = 0.12 + left = 0.12 + else: + if self.Size[1]<300: + bottom=0.20 + elif self.Size[1]<350: + bottom=0.18 + elif self.Size[1]<430: + bottom=0.16 + elif self.Size[1]<600: + bottom=0.13 + elif self.Size[1]<800: + bottom=0.09 + else: + bottom=0.07 + if self.Size[0]<300: + left=0.22 + elif self.Size[0]<450: + left=0.20 + elif self.Size[0]<950: + left=0.12 + else: + left=0.06 + #print(self.Size,'bottom', bottom, 'left',left) + if self.cbPlotMatrix.GetValue(): # TODO detect it + self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98-left) + else: + self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98) + + def plot_matrix_select(self, event): + if self.infoPanel is not None: + self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) + self.redraw_same_data() + + def measure_select(self, event): + if self.cbMeasure.IsChecked(): + self.cbAutoScale.SetValue(False) + self.redraw_same_data() + + def redraw_event(self, event): + self.redraw_same_data() + + def log_select(self, event): + if self.pltTypePanel.cbPDF.GetValue(): + self.cbLogX.SetValue(False) + self.cbLogY.SetValue(False) + else: + self.redraw_same_data() + + def crosshair_event(self, event): + try: + self.multiCursors.vertOn =self.cbXHair.GetValue() + self.multiCursors.horizOn=self.cbXHair.GetValue() + self.multiCursors._update() + except: + pass + + def show_hide(self,panel,bShow): + if bShow: + panel.Show() + self.slEsth.Show() + else: + self.slEsth.Hide() + panel.Hide() + + @property + def sharex(self): + return self.cbSync.IsChecked() and (not self.pltTypePanel.cbPDF.GetValue()) + + def set_subplots(self,nPlots): + self.set_subplot_spacing() + # Creating subplots + for ax in self.fig.axes: + self.fig.delaxes(ax) + sharex=None + for i in range(nPlots): + # Vertical stack + if i==0: + ax=self.fig.add_subplot(nPlots,1,i+1) + if self.sharex: + sharex=ax + else: + ax=self.fig.add_subplot(nPlots,1,i+1,sharex=sharex) + # Horizontal stack + #self.fig.add_subplot(1,nPlots,i+1) + + def onMouseMove(self, event): + if event.inaxes: + x, y = event.xdata, event.ydata + self.lbCrossHairX.SetLabel('x =' + self.formatLabelValue(x)) + self.lbCrossHairY.SetLabel('y =' + self.formatLabelValue(y)) + + def onMouseClick(self, event): + self.clickLocation = (event.inaxes, event.xdata, event.ydata) + + def onMouseRelease(self, event): + if self.cbMeasure.GetValue(): + for ax, ax_idx in zip(self.fig.axes, range(len(self.fig.axes))): + if event.inaxes == ax: + x, y = event.xdata, event.ydata + if self.clickLocation != (ax, x, y): + # Ignore measurements for zoom-actions. Possibly add small tolerance. + # Zoom-actions disable autoscale + self.cbAutoScale.SetValue(False) + return + if event.button == 1: + if self.infoPanel is not None: + self.infoPanel.setMeasurements((x, y), None) + self.leftMeasure.set(ax_idx, x, y) + self.leftMeasure.plot(ax, ax_idx) + elif event.button == 3: + if self.infoPanel is not None: + self.infoPanel.setMeasurements(None, (x, y)) + self.rightMeasure.set(ax_idx, x, y) + self.rightMeasure.plot(ax, ax_idx) + else: + return + if not self.cbAutoScale.IsChecked(): + self._restore_limits() + + if self.leftMeasure.axis_idx == self.rightMeasure.axis_idx and self.leftMeasure.axis_idx != -1: + self.lbDeltaX.SetLabel('dx=' + self.formatLabelValue(self.rightMeasure.x - self.leftMeasure.x)) + self.lbDeltaY.SetLabel('dy=' + self.formatLabelValue(self.rightMeasure.y - self.leftMeasure.y)) + else: + self.lbDeltaX.SetLabel('') + self.lbDeltaY.SetLabel('') + return + + def onDraw(self, event): + self._store_limits() + + def formatLabelValue(self, value): + try: + if abs(value)<1000 and abs(value)>1e-4: + s = '{:10.5f}'.format(value) + else: + s = '{:10.3e}'.format(value) + except TypeError: + s = ' ' + return s + + def removeTools(self, event=None, Layout=True): + + if self.toolPanel is not None: + self.toolPanel.destroyData() # clean destroy of data (action callbacks) + self.toolSizer.Clear(delete_windows=True) # Delete Windows + if Layout: + self.plotsizer.Layout() + + def showTool(self, toolName=''): + from .GUITools import TOOLS + if toolName in TOOLS.keys(): + self.showToolPanel(panelClass=TOOLS[toolName]) + else: + raise Exception('Unknown tool {}'.format(toolName)) + + def showToolAction(self, action): + """ Show a tool panel based on an action""" + self.showToolPanel(panelClass=action.guiEditorClass, action=action) + + + def showToolPanel(self, panelClass=None, panel=None, action=None): + """ Show a tool panel based on a panel class (should inherit from GUIToolPanel)""" + self.Freeze() + self.removeTools(Layout=False) + if panel is not None: + self.toolPanel=panel # use the panel directly + else: + if action is None: + print('NOTE: calling a panel without action') + self.toolPanel=panelClass(parent=self) # calling the panel constructor + else: + self.toolPanel=panelClass(parent=self, action=action) # calling the panel constructor + action.guiEditorObj = self.toolPanel + self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) + self.plotsizer.Layout() + self.Thaw() + + + def setPD_PDF(self,PD,c): + """ Convert plot data to PDF data based on GUI options""" + # ---PDF + nBins = self.pdfPanel.scBins.GetValue() + bSmooth = self.pdfPanel.cbSmooth.GetValue() + nBins_out= PD.toPDF(nBins,bSmooth) + if nBins_out!=nBins: + self.pdfPanel.scBins.SetValue(nBins) + + def setPD_MinMax(self,PD): + """ Convert plot data to MinMax data based on GUI options""" + yScale=self.mmxPanel.cbyMinMax.IsChecked() + xScale=self.mmxPanel.cbxMinMax.IsChecked() + try: + PD.toMinMax(xScale,yScale) + except Exception as e: + self.mmxPanel.cbxMinMax.SetValue(False) + raise e # Used to be Warn + + def setPD_FFT(self,pd): + """ Convert plot data to FFT data based on GUI options""" + yType = self.spcPanel.cbType.GetStringSelection() + xType = self.spcPanel.cbTypeX.GetStringSelection() + avgMethod = self.spcPanel.cbAveraging.GetStringSelection() + avgWindow = self.spcPanel.cbAveragingMethod.GetStringSelection() + bDetrend = self.spcPanel.cbDetrend.IsChecked() + nExp = self.spcPanel.scP2.GetValue() + nPerDecade = self.spcPanel.scP2.GetValue() + # Convert plotdata to FFT data + try: + Info = pd.toFFT(yType=yType, xType=xType, avgMethod=avgMethod, avgWindow=avgWindow, bDetrend=bDetrend, nExp=nExp, nPerDecade=nPerDecade) + # Trigger + if hasattr(Info,'nExp') and Info.nExp!=nExp: + self.spcPanel.scP2.SetValue(Info.nExp) + self.spcPanel.updateP2(Info.nExp) + except Exception as e: + self.spcPanel.Hide(); + self.plotsizer.Layout() + raise e + + + def transformPlotData(self,PD): + """" + Apply MinMax, PDF or FFT transform to plot based on GUI data + """ + plotType=self.pltTypePanel.plotType() + if plotType=='MinMax': + self.setPD_MinMax(PD) + elif plotType=='PDF': + self.setPD_PDF(PD,PD.c) + elif plotType=='FFT': + self.setPD_FFT(PD) + + def getPlotData(self,plotType): + + ID,SameCol,selMode=self.selPanel.getPlotDataSelection() + + self.selMode=selMode # we store the selection mode + del self.plotData + self.plotData=[] + tabs=self.selPanel.tabList.getTabs() # TODO, selPanel should just return the PlotData... + + try: + for i,idx in enumerate(ID): + # Initialize each plotdata based on selected table and selected id channels + pd=PlotData(); + pd.fromIDs(tabs, i, idx, SameCol, self.plotDataOptions, pipeline=self.pipeline) + # Possible change of data + if plotType=='MinMax': + self.setPD_MinMax(pd) + elif plotType=='PDF': + self.setPD_PDF(pd,pd.c) + elif plotType=='FFT': + self.setPD_FFT(pd) + self.plotData.append(pd) + except Exception as e: + self.plotData=[] + raise e + + def PD_Compare(self,mode): + """ Perform comparison of the selected PlotData, returns new plotData with the comparison. """ + sComp = self.cmpPanel.rbType.GetStringSelection() + try: + self.plotData = compareMultiplePD(self.plotData,mode, sComp) + except Exception as e: + self.pltTypePanel.cbRegular.SetValue(True) + raise e + + def _onPlotMatrixLeftClick(self, event): + """Toggle plot-states from None, to left-axis, to right-axis. + Left-click goes forwards, right-click goes backwards. + IndexError to avoid "holes" in matrix with outer adjacent populated entries + """ + btn = event.GetEventObject() + label = btn.GetLabelText() + if label == '-': + btn.SetLabel('1') + try: + self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) + except IndexError: + btn.SetLabel('-') + elif label == '1': + btn.SetLabel('2') + else: + btn.SetLabel('-') + try: + self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) + except IndexError: + btn.SetLabel('1') + self.redraw_same_data() + + def _onPlotMatrixRightClick(self, event): + btn = event.GetEventObject() + label = btn.GetLabelText() + if label == '-': + btn.SetLabel('2') + try: + self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) + except IndexError: + btn.SetLabel('-') + elif label == '1': + btn.SetLabel('-') + try: + self.infoPanel.getPlotMatrix(self.plotData, self.cbSub.IsChecked()) + except IndexError: + btn.SetLabel('2') + else: + btn.SetLabel('1') + self.redraw_same_data() + + def set_axes_lim(self, PDs, axis): + """ + It's usually faster to set the axis limits first (before plotting) + and disable autoscaling. This way the limits are not recomputed when plot data are added. + Also, we already have computed the min and max, so we leverage that. + NOTE: + doesnt not work with strings + doesnt not work for FFT and compare + + INPUTS: + PDs: list of plot data + """ + # TODO option for tight axes + tight=False + + plotType=self.pltTypePanel.plotType() + if plotType in ['FFT','Compare']: + axis.autoscale(True, axis='both', tight=tight) + return + vXString=[PDs[i].xIsString for i in axis.iPD] + vYString=[PDs[i].yIsString for i in axis.iPD] + if not any(vXString) and not self.cbLogX.IsChecked(): + try: + xMin=np.min([PDs[i]._xMin[0] for i in axis.iPD]) + xMax=np.max([PDs[i]._xMax[0] for i in axis.iPD]) + if np.isclose(xMin,xMax): + delta=1 if np.isclose(xMax,0) else 0.1*xMax + else: + if tight: + delta=0 + else: + delta = (xMax-xMin)*pyplot_rc['axes.xmargin'] + axis.set_xlim(xMin-delta,xMax+delta) + axis.autoscale(False, axis='x', tight=False) + except: + pass + if not any(vYString) and not self.cbLogY.IsChecked(): + try: + yMin=np.min([PDs[i]._yMin[0] for i in axis.iPD]) + yMax=np.max([PDs[i]._yMax[0] for i in axis.iPD]) + delta = (yMax-yMin)*pyplot_rc['axes.ymargin'] + if np.isclose(yMin,yMax): + delta=1 if np.isclose(yMax,0) else 0.1*yMax + else: + if tight: + delta=0 + else: + delta = (yMax-yMin)*pyplot_rc['axes.xmargin'] + axis.set_ylim(yMin-delta,yMax+delta) + axis.autoscale(False, axis='y', tight=False) + except: + pass + + def plot_all(self, keep_limits=True): + self.multiCursors=[] + + if self.cbMeasure.GetValue() is False: + for measure in [self.leftMeasure, self.rightMeasure]: + measure.clear() + if self.infoPanel is not None: + self.infoPanel.setMeasurements(None, None) + self.lbDeltaX.SetLabel('') + self.lbDeltaY.SetLabel('') + + axes=self.fig.axes + PD=self.plotData + + + # --- Plot options + bStep = self.cbStepPlot.IsChecked() + plot_options = dict() + plot_options['lw']=float(self.esthPanel.cbLW.Value) + plot_options['ms']=float(self.esthPanel.cbMS.Value) + if self.cbCurveType.Value=='Plain': + plot_options['LineStyles'] = ['-'] + plot_options['Markers'] = [''] + elif self.cbCurveType.Value=='LS': + plot_options['LineStyles'] = ['-','--','-.',':'] + plot_options['Markers'] = [''] + elif self.cbCurveType.Value=='Markers': + plot_options['LineStyles'] = [''] + plot_options['Markers'] = ['o','d','v','^','s'] + elif self.cbCurveType.Value=='Mix': # NOTE, can be improved + plot_options['LineStyles'] = ['-','--', '-','-','-'] + plot_options['Markers'] = ['' ,'' ,'o','^','s'] + else: + # Combination of linestyles markers, colors, etc. + # But at that stage, if the user really want this, then we can implement an option to set styles per plot. Not high priority. + raise Exception('Not implemented') + + + + # --- Font options + font_options = dict() + font_options_legd = dict() + font_options['size'] = int(self.esthPanel.cbFont.Value) # affect labels + font_options_legd['fontsize'] = int(self.esthPanel.cbLgdFont.Value) + needChineseFont = any([pd.needChineseFont for pd in PD]) + if needChineseFont and self.specialFont is not None: + font_options['fontproperties']= self.specialFont + font_options_legd['prop'] = self.specialFont + + # --- Loop on axes. Either use ax.iPD to chose the plot data, or rely on plotmatrix + for axis_idx, ax_left in enumerate(axes): + ax_right = None + # Checks + vDate=[PD[i].yIsDate for i in ax_left.iPD] + if any(vDate) and len(vDate)>1: + Error(self,'Cannot plot date and other value on the same axis') + return + + # Set limit before plot when possible, for optimization + self.set_axes_lim(PD, ax_left) + + # Actually plot + if self.infoPanel is not None: + pm = self.infoPanel.getPlotMatrix(PD, self.cbSub.IsChecked()) + else: + pm = None + __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) + ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) + + if self.infoPanel is not None: + self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) + for measure in [self.leftMeasure, self.rightMeasure]: + measure.plot(ax_left, axis_idx) + + # Log Axes + if self.cbLogX.IsChecked(): + try: + ax_left.set_xscale("log", nonpositive='clip') # latest + except: + ax_left.set_xscale("log", nonposx='clip') # legacy + + if self.cbLogY.IsChecked(): + if bAllNegLeft is False: + try: + ax_left.set_yscale("log", nonpositive='clip') # latest + except: + ax_left.set_yscale("log", nonposy='clip') + if bAllNegRight is False and ax_right is not None: + try: + ax_right.set_yscale("log", nonpositive='clip') # latest + except: + ax_left.set_yscale("log", nonposy='clip') # legacy + + # XLIM - TODO FFT ONLY NASTY + if self.pltTypePanel.cbFFT.GetValue(): + try: + if self.cbAutoScale.IsChecked(): + xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) + if xlim>0: + ax_left.set_xlim([0,xlim]) + pd=PD[ax_left.iPD[0]] + I=pd.x 0 and len(yleft_labels) <= 3: + ax_left.set_ylabel(' and '.join(yleft_labels), **font_options) + elif ax_left is not None: + ax_left.set_ylabel('') + if len(yright_labels) > 0 and len(yright_labels) <= 3: + ax_right.set_ylabel(' and '.join(yright_labels), **font_options) + elif ax_right is not None: + ax_right.set_ylabel('') + + # Legends + lgdLoc = self.esthPanel.cbLegend.Value.lower() + if (self.pltTypePanel.cbCompare.GetValue() or + ((len(yleft_legends) + len(yright_legends)) > 1)): + if lgdLoc !='none': + if len(yleft_legends) > 0: + ax_left.legend(fancybox=False, loc=lgdLoc, **font_options_legd) + if ax_right is not None and len(yright_legends) > 0: + ax_right.legend(fancybox=False, loc=4, **font_options_legd) + elif len(axes)>1 and len(axes)==len(PD): + # TODO: can this be removed? If there is only one unique signal + # per subplot, normally only ylabel is displayed and no legend. + # Special case when we have subplots and all plots have the same label + if lgdLoc !='none': + usy = unique([pd.sy for pd in PD]) + if len(usy)==1: + for ax in axes: + ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) + + axes[-1].set_xlabel(PD[axes[-1].iPD[0]].sx, **font_options) + + #print('sy :',[pd.sy for pd in PD]) + #print('syl:',[pd.syl for pd in PD]) + + # --- Cursors for each individual plot + # NOTE: cursors needs to be stored in the object! + #for ax_left in self.fig.axes: + # self.cursors.append(MyCursor(ax_left,horizOn=True, vertOn=False, useblit=True, color='gray', linewidth=0.5, linestyle=':')) + # Vertical cusor for all, commonly + bXHair = self.cbXHair.GetValue() + self.multiCursors = MyMultiCursor(self.canvas, tuple(self.fig.axes), useblit=True, horizOn=bXHair, vertOn=bXHair, color='gray', linewidth=0.5, linestyle=':') + + def plotSignals(self, ax, axis_idx, PD, pm, left_right, is_step, opts): + axis = None + bAllNeg = True + if pm is None: + loop_range = ax.iPD + else: + loop_range = range(len(PD)) + + iPlot=-1 + for signal_idx in loop_range: + do_plot = False + if left_right == 1 and (pm is None or pm[signal_idx][axis_idx] == left_right): + do_plot = True + axis = ax + elif left_right == 2 and pm is not None and pm[signal_idx][axis_idx] == left_right: + do_plot = True + if axis is None: + axis = ax.twinx() + ax.set_zorder(axis.get_zorder()+1) + ax.patch.set_visible(False) + axis._get_lines.prop_cycler = ax._get_lines.prop_cycler + pd=PD[signal_idx] + if do_plot: + iPlot+=1 + # --- styling per plot + if len(pd.x)==1: + marker='o'; ls='' + else: + # TODO allow PlotData to override for "per plot" options in the future + marker = opts['Markers'][np.mod(iPlot,len(opts['Markers']))] + ls = opts['LineStyles'][np.mod(iPlot,len(opts['LineStyles']))] + if is_step: + plot = axis.step + else: + plot = axis.plot + plot(pd.x,pd.y,label=pd.syl,ms=opts['ms'], lw=opts['lw'], marker=marker, ls=ls) + try: + bAllNeg = bAllNeg and all(pd.y<=0) + except: + pass # Dates or strings + return axis, bAllNeg + + def findPlotMode(self,PD): + uTabs = unique([pd.it for pd in PD]) + usy = unique([pd.sy for pd in PD]) + uiy = unique([pd.iy for pd in PD]) + if len(uTabs)<=0: + raise Exception('No Table. Contact developer') + if len(uTabs)==1: + mode='1Tab_nCols' + else: + if PD[0].SameCol: + mode='nTabs_SameCols' + else: + # Now that we allow multiple selections detecting "simColumns" is more difficult + if len(uTabs) == len(PD): + mode='nTabs_1Col' + elif self.selMode=='simColumnsMode': + mode='nTabs_SimCols' + else: + mode='nTabs_mCols' + return mode + + def findSubPlots(self,PD,mode): + uTabs = unique([pd.it for pd in PD]) + usy = unique([pd.sy for pd in PD]) + bSubPlots = self.cbSub.IsChecked() + bCompare = self.pltTypePanel.cbCompare.GetValue() # NOTE bCompare somehow always 1Tab_nCols + nSubPlots=1 + spreadBy='none' + if self.infoPanel is not None: + self.infoPanel.setTabMode(mode) # TODO get rid of me + if mode=='1Tab_nCols': + if bSubPlots: + if bCompare or len(uTabs)==1: + if self.infoPanel is not None: + nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) + else: + nSubPlots=len(usy) + else: + nSubPlots=len(usy) + spreadBy='iy' + elif mode=='nTabs_SameCols': + if bSubPlots: + if bCompare: + print('>>>TODO ',mode,len(usy),len(uTabs)) + else: + if len(usy)==1: + # Temporary hack until we have an option for spread by tabs or col + nSubPlots=len(uTabs) + spreadBy='it' + else: + nSubPlots=len(usy) + spreadBy='iy' + elif mode=='nTabs_SimCols': + if bSubPlots: + if bCompare: + print('>>>TODO ',mode,len(usy),len(uTabs)) + else: + nSubPlots=int(len(PD)/len(uTabs)) + spreadBy='mod-ip' + elif mode=='nTabs_mCols': + if bSubPlots: + if bCompare: + print('>>>TODO ',mode,len(usy),len(uTabs)) + else: + if bCompare or len(uTabs)==1: + nSubPlots = self.infoPanel.getNumberOfSubplots(PD, bSubPlots) + else: + nSubPlots=len(PD) + spreadBy='mod-ip' + elif mode=='nTabs_1Col': + if bSubPlots: + if bCompare: + print('>>> TODO',mode,len(uTabs)) + else: + nSubPlots=len(uTabs) + spreadBy='it' + else: + raise Exception('Unknown mode, contact developer.') + return nSubPlots,spreadBy + + def distributePlots(self,mode,nSubPlots,spreadBy): + """ Assigns plot data to axes and axes to plot data """ + axes=self.fig.axes + + # Link plot data to axes + if nSubPlots==1 or spreadBy=='none': + axes[0].iPD=[i for i in range(len(self.plotData))] + else: + for ax in axes: + ax.iPD=[] + PD=self.plotData + uTabs=unique([pd.it for pd in PD]) + uiy=unique([pd.iy for pd in PD]) + if spreadBy=='iy': + for ipd,pd in enumerate(PD): + i=uiy.index(pd.iy) + if i < len(axes): + axes[i].iPD.append(ipd) + elif spreadBy=='it': + for ipd,pd in enumerate(PD): + i=uTabs.index(pd.it) + axes[i].iPD.append(ipd) + elif spreadBy=='mod-ip': + for ipd,pd in enumerate(PD): + i=np.mod(ipd, nSubPlots) + axes[i].iPD.append(ipd) + else: + raise Exception('Wrong spreadby value') + + def setLegendLabels(self,mode): + """ Set labels for legend """ + if mode=='1Tab_nCols': + for pd in self.plotData: + if self.pltTypePanel.cbMinMax.GetValue(): + pd.syl = no_unit(pd.sy) + else: + pd.syl = pd.sy + + elif mode=='nTabs_SameCols': + for pd in self.plotData: + pd.syl=pd.st + + elif mode=='nTabs_1Col': + usy=unique([pd.sy for pd in self.plotData]) + if len(usy)==1: + for pd in self.plotData: + pd.syl=pd.st + else: + for pd in self.plotData: + if self.pltTypePanel.cbMinMax.GetValue(): + pd.syl=no_unit(pd.sy) + else: + pd.syl=pd.sy #pd.syl=pd.st + ' - '+pd.sy + elif mode=='nTabs_SimCols': + bSubPlots = self.cbSub.IsChecked() + if bSubPlots: # spread by table name + for pd in self.plotData: + pd.syl=pd.st + else: + for pd in self.plotData: + pd.syl=pd.st + ' - '+pd.sy + elif mode=='nTabs_mCols': + usy=unique([pd.sy for pd in self.plotData]) + bSubPlots = self.cbSub.IsChecked() + if bSubPlots and len(usy)==1: # spread by table name + for pd in self.plotData: + pd.syl=pd.st + else: + for pd in self.plotData: + pd.syl=pd.st + ' - '+pd.sy + else: + raise Exception('Unknown mode {}'.format(mode)) + + + def empty(self): + self.cleanPlot() + + def clean_memory(self): + if hasattr(self,'plotData'): + del self.plotData + self.plotData=[] + for ax in self.fig.axes: + ax.iPD=[] + self.fig.delaxes(ax) + gc.collect() + + def clean_memory_plot(self): + pass + + def cleanPlot(self): + for ax in self.fig.axes: + if hasattr(ax,'iPD'): + del ax.iPD + self.fig.delaxes(ax) + gc.collect() + self.fig.add_subplot(111) + ax = self.fig.axes[0] + ax.set_axis_off() + #ax.plot(1,1) + self.canvas.draw() + gc.collect() + + def load_and_draw(self): + """ Full draw event: + - Get plot data based on selection + - Plot them + - Trigger changes to infoPanel + + """ + self.clean_memory() + self.getPlotData(self.pltTypePanel.plotType()) + if len(self.plotData)==0: + self.cleanPlot(); + return + mode=self.findPlotMode(self.plotData) + if self.pltTypePanel.cbCompare.GetValue(): + self.PD_Compare(mode) + if len(self.plotData)==0: + self.cleanPlot(); + return + self.redraw_same_data() + if self.infoPanel is not None: + self.infoPanel.showStats(self.plotData,self.pltTypePanel.plotType()) + + def redraw_same_data(self, keep_limits=True): + if len(self.plotData)==0: + self.cleanPlot(); + return + elif len(self.plotData) == 1: + if self.plotData[0].xIsString or self.plotData[0].yIsString or self.plotData[0].xIsDate or self.plotData[0].yIsDate: + self.cbAutoScale.SetValue(True) + else: + if len(self.xlim_prev)==0: # Might occur if some date didn't plot before (e.g. strings) + self.cbAutoScale.SetValue(True) + elif rectangleOverlap(self.plotData[0]._xMin[0], self.plotData[0]._yMin[0], + self.plotData[0]._xMax[0], self.plotData[0]._yMax[0], + self.xlim_prev[0][0], self.ylim_prev[0][0], + self.xlim_prev[0][1], self.ylim_prev[0][1]): + pass + else: + self.cbAutoScale.SetValue(True) + + mode=self.findPlotMode(self.plotData) + nPlots,spreadBy=self.findSubPlots(self.plotData,mode) + + self.clean_memory_plot() + self.set_subplots(nPlots) + self.distributePlots(mode,nPlots,spreadBy) + + if not self.pltTypePanel.cbCompare.GetValue(): + self.setLegendLabels(mode) + + self.plot_all(keep_limits) + self.canvas.draw() + + + def _store_limits(self): + self.xlim_prev = [] + self.ylim_prev = [] + for ax in self.fig.axes: + self.xlim_prev.append(ax.get_xlim()) + self.ylim_prev.append(ax.get_ylim()) + + def _restore_limits(self): + for ax, xlim, ylim in zip(self.fig.axes, self.xlim_prev, self.ylim_prev): + ax.set_xlim(xlim) + ax.set_ylim(ylim) + + +if __name__ == '__main__': + import pandas as pd; + from Tables import Table,TableList + from pydatview.Tables import TableList + from pydatview.GUISelectionPanel import SelectionPanel + + # --- Data + tabList = TableList.createDummy(1) + app = wx.App(False) + self=wx.Frame(None,-1,"GUI Plot Panel Demo") + + # --- Panels + self.selPanel = SelectionPanel(self, tabList, mode='auto') + self.plotPanel = PlotPanel(self, self.selPanel) + self.plotPanel.load_and_draw() # <<< Important + self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # Binding the two + + # --- Finalize GUI + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(self.selPanel ,0, flag = wx.EXPAND|wx.ALL,border = 5) + sizer.Add(self.plotPanel,1, flag = wx.EXPAND|wx.ALL,border = 5) + self.SetSizer(sizer) + self.Center() + self.SetSize((900, 600)) + self.Show() + app.MainLoop() + + diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 8eb4e67..04010e3 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -22,7 +22,15 @@ def __init__(self, parent): super(GUIToolPanel,self).__init__(parent) self.parent = parent - def destroy(self,event=None): + def destroyData(self): + if hasattr(self, 'action'): + if self.action is not None: + # cleanup action calls + self.action.guiEditorObj = None + self.action = None + + def destroy(self,event=None): # TODO rename to something less close to Destroy + self.destroyData() self.parent.removeTools() def getBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): @@ -444,245 +452,6 @@ def onHelp(self,event=None): """) -# --------------------------------------------------------------------------------} -# --- Resample -# --------------------------------------------------------------------------------{ -class ResampleToolPanel(GUIToolPanel): - def __init__(self, parent): - super(ResampleToolPanel,self).__init__(parent) - - # --- Data from other modules - from pydatview.tools.signal_analysis import SAMPLERS - self.parent = parent # parent is GUIPlotPanel - self._SAMPLERS=SAMPLERS - # Setting default states to parent - if 'Sampler' not in self.parent.plotDataOptions.keys(): - self.parent.plotDataOptions['Sampler']=None - self._applied = type(self.parent.plotDataOptions['Sampler'])==dict - - - # --- GUI elements - self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) - self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) - self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) - - #self.lb = wx.StaticText( self, -1, """ Click help """) - self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) - self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS], style=wx.CB_READONLY) - - self.lbNewX = wx.StaticText(self, -1, 'New x: ') - self.textNewX = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER) - self.textOldX = wx.TextCtrl(self, wx.ID_ANY|wx.TE_READONLY) - self.textOldX.Enable(False) - - # --- Layout - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) - - msizer = wx.FlexGridSizer(rows=2, cols=4, hgap=2, vgap=0) - msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(wx.StaticText(self, -1, 'Current x: '), 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.textOldX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) - msizer.Add(wx.StaticText(self, -1, 'Method:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.cbMethods , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.lbNewX , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.textNewX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) - msizer.AddGrowableCol(3,1) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) - self.sizer.Add(msizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) - self.SetSizer(self.sizer) - - # --- Events - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) - self.cbMethods.Bind(wx.EVT_COMBOBOX, self.onMethodChange) - self.textNewX.Bind(wx.EVT_TEXT_ENTER,self.onParamChange) - - # --- Init triggers - self.cbMethods.SetSelection(3) - self.onMethodChange(init=True) - self.onToggleApply(init=True) - self.updateTabList() - self.textNewX.SetValue('2') - - def setCurrentX(self, x=None): - if x is None: - x= self.parent.plotData[0].x - if len(x)<50: - s=np.array2string(x, separator=', ') - else: - s =np.array2string(x[[0,1,2,3]], separator=', ') - s+=', ..., ' - s+=np.array2string(x[[-3,-2,-1]], separator=', ') - s=s.replace('[','').replace(']','').replace(' ','').replace(',',', ') - - self.textOldX.SetValue(s) - - def onMethodChange(self, event=None, init=True): - """ Select the method, but does not applied it to the plotData - User data and option is unchanged - But if the user already has some options, they are used - """ - iOpt = self.cbMethods.GetSelection() - opt = self._SAMPLERS[iOpt] - self.lbNewX.SetLabel(opt['paramName']+':') - - parentOpt=self.parent.plotDataOptions['Sampler'] - # Value - if len(self.textNewX.Value)==0: - if type(parentOpt)==dict: - self.textNewX.SetValue(str(parentOpt['param'])[1:-1]) - else: - self.textNewX.SetValue(str(opt['param'])[2:-2]) - self.onParamChange() - - def onParamChange(self, event=None): - if self._applied: - self.parent.plotDataOptions['Sampler'] =self._GUI2Data() - self.parent.load_and_draw() # Data will change - self.setCurrentX() - - def _GUI2Data(self): - iOpt = self.cbMethods.GetSelection() - opt = self._SAMPLERS[iOpt].copy() - s= self.textNewX.Value.strip().replace('[','').replace(']','') - if len(s)>0: - if s.find(','): - opt['param']=np.array(s.split(',')).astype(float) - else: - opt['param']=np.array(s.split('')).astype(float) - return opt - - def onToggleApply(self, event=None, init=False): - """ - apply sampler based on GUI Data - """ - parentFilt=self.parent.plotDataOptions['Sampler'] - if not init: - self._applied = not self._applied - - if self._applied: - self.parent.plotDataOptions['Sampler'] =self._GUI2Data() - #print('Apply', self.parent.plotDataOptions['Sampler']) - #self.lbInfo.SetLabel( - # 'Sampler is now applied on the fly. Change parameter live. Click "Clear" to stop. ' - # ) - self.btPlot.Enable(False) - self.btClear.Enable(False) - self.btApply.SetLabel(CHAR['sun']+' Clear') - else: - self.parent.plotDataOptions['Sampler'] = None - #self.lbInfo.SetLabel( - # 'Click on "Apply" to set filter on the fly for all plots. '+ - # 'Click on "Plot" to try a filter on the current plot.' - # ) - self.btPlot.Enable(True) - self.btClear.Enable(True) - self.btApply.SetLabel(CHAR['cloud']+' Apply') - - if not init: - self.parent.load_and_draw() # Data will change - self.setCurrentX() - - def onAdd(self,event=None): - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - icol, colname = self.parent.selPanel.xCol - opt = self._GUI2Data() - errors=[] - if iSel==0: - dfs, names, errors = tabList.applyResampling(icol, opt, bAdd=True) - self.parent.addTables(dfs,names,bAdd=True) - else: - df, name = tabList.get(iSel-1).applyResampling(icol, opt, bAdd=True) - self.parent.addTables([df],[name], bAdd=True) - self.updateTabList() - - if len(errors)>0: - raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) - - # We stop applying - self.onToggleApply() - - def onPlot(self,event=None): - from pydatview.tools.signal_analysis import applySampler - if len(self.parent.plotData)!=1: - Error(self,'Plotting only works for a single plot. Plot less data.') - return - opts=self._GUI2Data() - PD = self.parent.plotData[0] - x_new, y_new = applySampler(PD.x0, PD.y0, opts) - ax = self.parent.fig.axes[0] - - PD_new = PlotData() - PD_new.fromXY(x_new, y_new) - self.parent.transformPlotData(PD_new) - ax.plot(PD_new.x, PD_new.y, '-') - self.setCurrentX(x_new) - - self.parent.canvas.draw() - - def onClear(self,event=None): - self.parent.load_and_draw() # Data will change - # Update Current X - self.setCurrentX() - # Update Table list - self.updateTabList() - - - def onTabChange(self,event=None): - #tabList = self.parent.selPanel.tabList - #iSel=self.cbTabs.GetSelection() - pass - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - - def onHelp(self,event=None): - Info(self,"""Resampling. - -The resampling operation changes the "x" values of a table/plot and -adapt the "y" values accordingly. - -To resample perform the following step: - -- Chose a resampling method: - - replace: specify all the new x-values - - insert : insert a list of x values to the existing ones - - delete : delete a list of x values from the existing ones - - every-n : use every n values - - time-based: downsample using sample averaging or upsample using - linear interpolation, x-axis must already be in seconds - - delta x : specify a delta for uniform spacing of x values - -- Specify the x values as a space or comma separated list - -- Click on one of the following buttons: - - Plot: will display the resampled data on the figure - - Apply: will perform the resampling on the fly for all new plots - - Add: will create new table(s) with resampled values for all - signals. This process might take some time. - Select a table or choose all (default) -""") - # --------------------------------------------------------------------------------} @@ -1169,7 +938,6 @@ def onHelp(self,event=None): 'LogDec': LogDecToolPanel, 'Outlier': OutlierToolPanel, 'Filter': FilterToolPanel, - 'Resample': ResampleToolPanel, 'Mask': MaskToolPanel, 'FASTRadialAverage': RadialToolPanel, 'CurveFitting': CurveFitToolPanel, diff --git a/pydatview/main.py b/pydatview/main.py index dc15660..428d5db 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -147,7 +147,6 @@ def __init__(self, data=None): self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e, 'Mask') , dataMenu.Append(wx.ID_ANY, 'Mask')) self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Outlier'), dataMenu.Append(wx.ID_ANY, 'Outliers removal')) self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Filter') , dataMenu.Append(wx.ID_ANY, 'Filter')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Resample') , dataMenu.Append(wx.ID_ANY, 'Resample')) self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'FASTRadialAverage'), dataMenu.Append(wx.ID_ANY, 'FAST - Radial average')) # --- Data Plugins diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index 7e765d8..a174382 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -7,7 +7,7 @@ class Action(): def __init__(self, name, tableFunction = None, plotDataFunction = None, - guiCallBack=None, + guiCallback=None, guiEditorClass=None, data=None, mainframe=None, @@ -24,7 +24,7 @@ def __init__(self, name, self.tableFunction = tableFunction # applies to a full table self.plotDataFunction = plotDataFunction # applies to x,y arrays only - self.guiCallBack = guiCallBack # callback to update GUI after the action, + self.guiCallback = guiCallback # callback to update GUI after the action, # TODO remove me, replace with generic "redraw", "update tab list" self.guiEditorClass = guiEditorClass # Class that can be used to edit this action self.guiEditorObj = None # Instance of guiEditorClass that can be used to edit this action @@ -60,9 +60,10 @@ def apply(self, tablist, force=False, applyToAll=False): def updateGUI(self): - if self.guiCallBack is not None: + """ Typically called by a calleed after append""" + if self.guiCallback is not None: print('>>> Calling GUI callback, action', self.name) - self.guiCallBack() + self.guiCallback() def __repr__(self): s=''.format(self.name) diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index d1dcc26..c4c349d 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -86,13 +86,9 @@ def _post_init(PD, Options=None, pipeline=None): from pydatview.tools.signal_analysis import applyFilter PD.y = applyFilter(PD.x, PD.y, Options['Filter']) - if 'Sampler' in keys: - if Options['Sampler']: - from pydatview.tools.signal_analysis import applySampler - PD.x, PD.y = applySampler(PD.x, PD.y, Options['Sampler']) - # --- Apply filters from pipeline - print('[PDat]', pipeline.__reprFilters__()) + if pipeline is not None: + print('[PDat]', pipeline.__reprFilters__()) if pipeline is not None: PD.x, PD.y = pipeline.applyOnPlotData(PD.x, PD.y) diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 2de282f..8f64f78 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -26,9 +26,13 @@ def _data_binning(label, mainframe): from .data_binning import binningAction return binningAction(label, mainframe) +def _data_sampler(label, mainframe): + from .data_sampler import samplerAction + return samplerAction(label, mainframe) dataPlugins=[ # Name/label , callback , is a Panel + ('Resample' , _data_sampler , True ), ('Bin data' , _data_binning , True ), ('Standardize Units (SI)', _data_standardizeUnitsSI, False), ('Standardize Units (WE)', _data_standardizeUnitsWE, False), diff --git a/pydatview/plugins/data_binning.py b/pydatview/plugins/data_binning.py index 0fac1ed..ab14759 100644 --- a/pydatview/plugins/data_binning.py +++ b/pydatview/plugins/data_binning.py @@ -1,19 +1,19 @@ import wx import numpy as np -import pandas as pd -# import copy -# import platform -# from collections import OrderedDict -# For log dec tool from pydatview.GUITools import GUIToolPanel, TOOL_BORDER from pydatview.common import CHAR, Error, Info, pretty_num_short from pydatview.common import DummyMainFrame from pydatview.plotdata import PlotData -# from pydatview.tools.damping import logDecFromDecay -# from pydatview.tools.curve_fitting import model_fit, extract_key_miscnum, extract_key_num, MODELS, FITTERS, set_common_keys - from pydatview.pipeline import PlotDataAction - +# --------------------------------------------------------------------------------} +# --- Data +# --------------------------------------------------------------------------------{ +_DEFAULT_DICT={ + 'active':False, + 'xMin':None, + 'xMax':None, + 'nBins':50 +} # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ @@ -23,23 +23,72 @@ def binningAction(label, mainframe=None, data=None): The action is also edited and created by the GUI Editor """ if data is None: + # NOTE: if we don't do copy below, we will end up remembering even after the action was deleted + # its not a bad feature, but we might want to think it through + # One issue is that "active" is kept in memory data=_DEFAULT_DICT + data['active'] = False #<<< Important + + guiCallback=None + if mainframe is not None: + guiCallback = mainframe.redraw action = PlotDataAction( name=label, plotDataFunction = bin_plot, guiEditorClass = BinningToolPanel, + guiCallback = guiCallback, data = data, mainframe=mainframe ) return action +# --------------------------------------------------------------------------------} +# --- Main method +# --------------------------------------------------------------------------------{ +def bin_plot(x, y, opts): + from pydatview.tools.stats import bin_signal + xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) + if xBins[0]>xBins[1]: + raise Exception('xmin must be lower than xmax') + x_new, y_new = bin_signal(x, y, xbins=xBins) + return x_new, y_new + +def bin_tab(tab, iCol, colName, opts, bAdd=True): + # TODO, make it such as it's only handling a dataframe instead of a table + from pydatview.tools.stats import bin_DF + colName = tab.data.columns[iCol] + error='' + xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) +# try: + df_new =bin_DF(tab.data, xbins=xBins, colBin=colName) + # Remove index if present + if df_new.columns[0].lower().find('index')>=0: + df_new = df_new.iloc[:, 1:] # We don't use "drop" in case of duplicate "index" + + # Setting bin column as first columns + colNames = list(df_new.columns.values) + colNames.remove(colName) + colNames.insert(0, colName) + df_new=df_new.reindex(columns=colNames) + if bAdd: + name_new=tab.raw_name+'_binned' + else: + name_new=None + tab.data=df_new +# except: +# df_new = None +# name_new = None + + return df_new, name_new + + # --------------------------------------------------------------------------------} -# --- GUI to Edit Plugin and control the Action +# --- GUI to edit plugin and control the action # --------------------------------------------------------------------------------{ class BinningToolPanel(GUIToolPanel): - def __init__(self, parent, action): - super(BinningToolPanel, self).__init__(parent) + def __init__(self, parent, action=None): + GUIToolPanel.__init__(self, parent) # --- Creating "Fake data" for testing only! if action is None: @@ -49,24 +98,24 @@ def __init__(self, parent, action): # --- Data from other modules self.parent = parent # parent is GUIPlotPanel self.mainframe = action.mainframe - self.data = action.data self.action = action - # self.data['selectionChangeCallBack'] = self.selectionChange # TODO # --- GUI elements - self.btClose = self.getBtBitmap(self, 'Close','close', self.onClose) + self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) + self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) #self.lb = wx.StaticText( self, -1, """ Click help """) self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better self.scBins = wx.SpinCtrl(self, value='50', style=wx.TE_RIGHT, size=wx.Size(60,-1) ) self.textXMin = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) self.textXMax = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) + self.btXRange = self.getBtBitmap(self, 'Default','compute', self.reset) self.lbDX = wx.StaticText(self, -1, '') self.scBins.SetRange(3, 10000) @@ -124,22 +173,12 @@ def __init__(self, parent, action): self.setXRange(x=[self.data['xMin'], self.data['xMax']]) else: self.setXRange() - self.scBins.SetValue(self.data['nBins']) + self._Data2GUI() self.onToggleApply(init=True) self.updateTabList() self.onParamChange() - # --- External Calls - def cancelAction(self, redraw=True): - """ do cancel the action""" - self.btPlot.Enable(True) - self.btClear.Enable(True) - self.btApply.SetLabel(CHAR['cloud']+' Apply') - self.btApply.SetValue(False) - self.data['active'] = False - if redraw: - self.parent.load_and_draw() # Data will change based on plotData - # + # --- Implementation specific def reset(self, event=None): self.setXRange() self.updateTabList() # might as well until we add a nice callback/button.. @@ -163,6 +202,35 @@ def selectionChange(self): print('>>> Binning selectionChange callback, TODO') self.setXRange() + # --- Table related + def onTabChange(self,event=None): + #tabList = self.parent.selPanel.tabList + #iSel=self.cbTabs.GetSelection() + pass + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + try: + iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + except RuntimeError: + pass + + # --- External Calls + def cancelAction(self, redraw=True): + """ do cancel the action""" + self.btPlot.Enable(True) + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + self.btApply.SetValue(False) + self.data['active'] = False + if redraw: + self.parent.load_and_draw() # Data will change based on plotData + + # --- Fairly generic def _GUI2Data(self): def zero_if_empty(s): return 0 if len(s)==0 else s @@ -170,6 +238,13 @@ def zero_if_empty(s): self.data['xMin'] = float(zero_if_empty(self.textXMin.Value)) self.data['xMax'] = float(zero_if_empty(self.textXMax.Value)) + def _Data2GUI(self): + if self.data['active']: + self.lbDX.SetLabel(pretty_num_short((self.data['xMax']- self.data['xMin'])/self.data['nBins'])) + self.textXMin.SetValue(pretty_num_short(self.data['xMin'])) + self.textXMax.SetValue(pretty_num_short(self.data['xMax'])) + self.scBins.SetValue(self.data['nBins']) + def onToggleApply(self, event=None, init=False): """ apply sampler based on GUI Data @@ -186,27 +261,15 @@ def onToggleApply(self, event=None, init=False): # The action is now active we add it to the pipeline, unless it's already in it if self.mainframe is not None: self.mainframe.addAction(self.action, cancelIfPresent=True) - else: - print('[WARN] Running data_binning without a main frame') - if not init: - # This is a "plotData" action, we don't need to do anything - print('>>> redraw') - self.parent.load_and_draw() # Data will change based on plotData + self.parent.load_and_draw() # filter will be applied in plotData.py else: - print('>>>> TODO Remove Action') # We remove our action from the pipeline if not init: if self.mainframe is not None: self.mainframe.removeAction(self.action) - else: - print('[WARN] Running data_binning without a main frame') - #self.data = None - #self.action = None self.cancelAction(redraw=not init) - - def onAdd(self,event=None): from pydatview.tools.stats import bin_DF iSel = self.cbTabs.GetSelection() @@ -225,7 +288,6 @@ def onAdd(self,event=None): self._GUI2Data() errors=[] - if iSel==0: # Looping on all tables and adding new table dfs_new = [] @@ -272,27 +334,6 @@ def onClear(self,event=None): # Update Table list self.updateTabList() - def onTabChange(self,event=None): - #tabList = self.parent.selPanel.tabList - #iSel=self.cbTabs.GetSelection() - pass - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - - def onClose(self, event=None): - # cleanup action calls - self.action.guiEditorObj=None - self.destroy() - def onHelp(self,event=None): Info(self,"""Binning. @@ -313,52 +354,6 @@ def onHelp(self,event=None): """) -# --------------------------------------------------------------------------------} -# --- DATA -# --------------------------------------------------------------------------------{ -def bin_plot(x, y, opts): - from pydatview.tools.stats import bin_signal - xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) - if xBins[0]>xBins[1]: - raise Exception('xmin must be lower than xmax') - x_new, y_new = bin_signal(x, y, xbins=xBins) - return x_new, y_new - -def bin_tab(tab, iCol, colName, opts, bAdd=True): - # TODO, make it such as it's only handling a dataframe instead of a table - from pydatview.tools.stats import bin_DF - colName = tab.data.columns[iCol] - error='' - xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) -# try: - df_new =bin_DF(tab.data, xbins=xBins, colBin=colName) - # Remove index if present - if df_new.columns[0].lower().find('index')>=0: - df_new = df_new.iloc[:, 1:] # We don't use "drop" in case of duplicate "index" - - # Setting bin column as first columns - colNames = list(df_new.columns.values) - colNames.remove(colName) - colNames.insert(0, colName) - df_new=df_new.reindex(columns=colNames) - if bAdd: - name_new=tab.raw_name+'_binned' - else: - name_new=None - tab.data=df_new -# except: -# df_new = None -# name_new = None - - return df_new, name_new - -_DEFAULT_DICT={ - 'active':False, - 'xMin':None, - 'xMax':None, - 'nBins':50, - 'dx':0, -} if __name__ == '__main__': diff --git a/pydatview/plugins/data_sampler.py b/pydatview/plugins/data_sampler.py new file mode 100644 index 0000000..7f4f7fe --- /dev/null +++ b/pydatview/plugins/data_sampler.py @@ -0,0 +1,321 @@ +import wx +import numpy as np +from pydatview.GUITools import GUIToolPanel, TOOL_BORDER +from pydatview.common import CHAR, Error, Info, pretty_num_short +from pydatview.common import DummyMainFrame +from pydatview.plotdata import PlotData +from pydatview.pipeline import PlotDataAction +# --------------------------------------------------------------------------------} +# --- Data +# --------------------------------------------------------------------------------{ +# See SAMPLERS in signal_analysis +_DEFAULT_DICT={ + 'active':False, + 'name':'Every n', + 'param':2, + 'paramName':'n' +} +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def samplerAction(label, mainframe=None, data=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + The action is also edited and created by the GUI Editor + """ + if data is None: + # NOTE: if we don't do copy below, we will end up remembering even after the action was deleted + # its not a bad feature, but we might want to think it through + # One issue is that "active" is kept in memory + data=_DEFAULT_DICT + data['active'] = False #<<< Important + + guiCallback=None + if mainframe is not None: + guiCallback = mainframe.redraw + + action = PlotDataAction( + name=label, + plotDataFunction = samplerXY, + guiEditorClass = SamplerToolPanel, + guiCallback = guiCallback, + data = data, + mainframe=mainframe + ) + return action +# --------------------------------------------------------------------------------} +# --- Main method +# --------------------------------------------------------------------------------{ +def samplerXY(x, y, opts): + from pydatview.tools.signal_analysis import applySampler + x_new, y_new = applySampler(x, y, opts) + return x_new, y_new + +# --------------------------------------------------------------------------------} +# --- GUI to edit plugin and control the action +# --------------------------------------------------------------------------------{ +class SamplerToolPanel(GUIToolPanel): + def __init__(self, parent, action=None): + GUIToolPanel.__init__(self, parent) + + # --- Creating "Fake data" for testing only! + if action is None: + print('[WARN] Calling GUI without an action! Creating one.') + mainframe = DummyMainFrame(parent) + action = binningAction(label='dummyAction', mainframe=mainframe) + # --- Data from other modules + self.parent = parent # parent is GUIPlotPanel + self.mainframe = action.mainframe + self.data = action.data + self.action = action + from pydatview.tools.signal_analysis import SAMPLERS + self._SAMPLERS_DEFAULT = SAMPLERS.copy() + self._SAMPLERS_USER = SAMPLERS.copy() + + # --- GUI elements + self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) + self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) + self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) + self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) + self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) + + #self.lb = wx.StaticText( self, -1, """ Click help """) + self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better + self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS_DEFAULT], style=wx.CB_READONLY) + + self.lbNewX = wx.StaticText(self, -1, 'New x: ') + self.textNewX = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER) + self.textOldX = wx.TextCtrl(self, wx.ID_ANY|wx.TE_READONLY) + self.textOldX.Enable(False) + + # --- Layout + btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) + btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) + + msizer = wx.FlexGridSizer(rows=2, cols=4, hgap=2, vgap=0) + msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(wx.StaticText(self, -1, 'Current x: '), 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.textOldX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) + msizer.Add(wx.StaticText(self, -1, 'Method:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.cbMethods , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.lbNewX , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.textNewX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) + msizer.AddGrowableCol(3,1) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + self.sizer.Add(msizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) + self.SetSizer(self.sizer) + + # --- Events + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + self.cbMethods.Bind(wx.EVT_COMBOBOX, self.onMethodChange) + self.textNewX.Bind(wx.EVT_TEXT_ENTER,self.onParamChange) + + # --- Init triggers + self._Data2GUI() + self.onMethodChange(init=True) + self.onToggleApply(init=True) + self.updateTabList() + + # --- Implementation specific + def getSamplerIndex(self, name): + for i, samp in enumerate(self._SAMPLERS_USER): + if samp['name'] == name: + return i + return -1 + + def setCurrentX(self, x=None): + if x is None: + x= self.parent.plotData[0].x + if len(x)<50: + s=np.array2string(x, separator=', ') + else: + s =np.array2string(x[[0,1,2,3]], separator=', ') + s+=', ..., ' + s+=np.array2string(x[[-3,-2,-1]], separator=', ') + s=s.replace('[','').replace(']','').replace(' ','').replace(',',', ') + + self.textOldX.SetValue(s) + + def onMethodChange(self, event=None, init=True): + """ Select the method, but does not applied it to the plotData + User data and option is unchanged + But if the user already has some options, they are used + """ + iOpt = self.cbMethods.GetSelection() + opt_default = self._SAMPLERS_DEFAULT[iOpt] + opt_user = self._SAMPLERS_USER[iOpt] + self.lbNewX.SetLabel(opt_default['paramName']+':') + + # Value + if len(self.textNewX.Value)==0: + self.textNewX.SetValue(str(opt_user['param'])[1:-1]) + #if type(parentOpt)==dict: + # self.textNewX.SetValue(str(parentOpt['param'])[1:-1]) + #else: + # self.textNewX.SetValue(str(opt['param'])[2:-2]) + self.onParamChange() + + def onParamChange(self, event=None): + self._GUI2Data() + if self.data['active']: + self.parent.load_and_draw() # Data will change + self.setCurrentX() + + # --- Table related + def onTabChange(self,event=None): + #tabList = self.parent.selPanel.tabList + #iSel=self.cbTabs.GetSelection() + pass + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + try: + iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + except RuntimeError: + pass + + # --- External Calls + def cancelAction(self, redraw=True): + """ do cancel the action""" + self.btPlot.Enable(True) + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + self.btApply.SetValue(False) + self.data['active'] = False + if redraw: + self.parent.load_and_draw() # Data will change based on plotData + + # --- Fairly generic + def _GUI2Data(self): + iOpt = self.cbMethods.GetSelection() + # Store GUI into samplers_user list + opt = self._SAMPLERS_USER[iOpt] + s= self.textNewX.Value.strip().replace('[','').replace(']','') + if len(s)>0: + if s.find(','): + opt['param']=np.array(s.split(',')).astype(float) + else: + opt['param']=np.array(s.split('')).astype(float) + # Then update our main data dictionary + self.data.update(opt) + return opt + + def _Data2GUI(self): + i = self.getSamplerIndex(self.data['name']) + if i==-1: + raise Exception('Unknown sampling method ', self.data['name']) + self.cbMethods.SetSelection(i) + param = self.data['param'] + self.textNewX.SetValue(str(param).lstrip('[').rstrip(']')) + + def onToggleApply(self, event=None, init=False): + """ + apply sampler based on GUI Data + """ + if not init: + self.data['active'] = not self.data['active'] + + if self.data['active']: + self._GUI2Data() + self.btPlot.Enable(False) + self.btClear.Enable(False) + self.btApply.SetLabel(CHAR['sun']+' Clear') + self.btApply.SetValue(True) + # The action is now active we add it to the pipeline, unless it's already in it + if self.mainframe is not None: + self.mainframe.addAction(self.action, cancelIfPresent=True) + if not init: + self.parent.load_and_draw() # filter will be applied in plotData.py + else: + # We remove our action from the pipeline + if not init: + if self.mainframe is not None: + self.mainframe.removeAction(self.action) + self.cancelAction(redraw=not init) + self.setCurrentX() + + def onAdd(self,event=None): + iSel = self.cbTabs.GetSelection() + tabList = self.parent.selPanel.tabList + icol, colname = self.parent.selPanel.xCol + self._GUI2Data() + errors=[] + if iSel==0: + dfs, names, errors = tabList.applyResampling(icol, self.data, bAdd=True) + self.parent.addTables(dfs,names,bAdd=True) + else: + df, name = tabList.get(iSel-1).applyResampling(icol, self.data, bAdd=True) + self.parent.addTables([df],[name], bAdd=True) + self.updateTabList() + + if len(errors)>0: + raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) + + # We stop applying + self.onToggleApply() + + def onPlot(self,event=None): + from pydatview.tools.signal_analysis import applySampler + if len(self.parent.plotData)!=1: + Error(self,'Plotting only works for a single plot. Plot less data.') + return + self._GUI2Data() + PD = self.parent.plotData[0] + x_new, y_new = samplerXY(PD.x0, PD.y0, self.data) + + ax = self.parent.fig.axes[0] + PD_new = PlotData() + PD_new.fromXY(x_new, y_new) + self.parent.transformPlotData(PD_new) + ax.plot(PD_new.x, PD_new.y, '-') + self.setCurrentX(x_new) + + self.parent.canvas.draw() + + def onClear(self,event=None): + self.parent.load_and_draw() # Data will change + # Update Current X + self.setCurrentX() + # Update Table list + self.updateTabList() + + def onHelp(self,event=None): + Info(self,"""Resampling. + +The resampling operation changes the "x" values of a table/plot and +adapt the "y" values accordingly. + +To resample perform the following step: + +- Chose a resampling method: + - replace: specify all the new x-values + - insert : insert a list of x values to the existing ones + - delete : delete a list of x values from the existing ones + - every-n : use every n values + - time-based: downsample using sample averaging or upsample using + linear interpolation, x-axis must already be in seconds + - delta x : specify a delta for uniform spacing of x values + +- Specify the x values as a space or comma separated list + +- Click on one of the following buttons: + - Plot: will display the resampled data on the figure + - Apply: will perform the resampling on the fly for all new plots + - Add: will create new table(s) with resampled values for all + signals. This process might take some time. + Select a table or choose all (default) +""") diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py index a6eb9aa..c014460 100644 --- a/pydatview/plugins/data_standardizeUnits.py +++ b/pydatview/plugins/data_standardizeUnits.py @@ -7,9 +7,10 @@ def standardizeUnitsAction(label, mainframe=None, flavor='SI'): """ Main entry point of the plugin """ - guiCallBack=None + guiCallback=None if mainframe is not None: - def guiCallBack(): + # TODO TODO TODO Clean this up + def guiCallback(): if hasattr(mainframe,'selPanel'): mainframe.selPanel.colPanel1.setColumns() mainframe.selPanel.colPanel2.setColumns() @@ -24,7 +25,7 @@ def guiCallBack(): action = IrreversibleAction( name=label, tableFunction=tableFunction, - guiCallBack=guiCallBack, + guiCallback=guiCallback, mainframe=mainframe, # shouldnt be needed ) From 17c47aeeae9fad5688fa69565510f8ad21cae853 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 05:14:09 +0100 Subject: [PATCH 065/178] Pipeline: filter as a plugin/action --- pydatview/GUITools.py | 269 ----------------------- pydatview/plugins/__init__.py | 5 + pydatview/plugins/data_filter.py | 349 ++++++++++++++++++++++++++++++ pydatview/plugins/data_sampler.py | 7 +- 4 files changed, 358 insertions(+), 272 deletions(-) create mode 100644 pydatview/plugins/data_filter.py diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 04010e3..9ceef1f 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -186,274 +186,6 @@ def onMDChangeChar(self, event): -# --------------------------------------------------------------------------------} -# --- Filter Tool -# --------------------------------------------------------------------------------{ -class FilterToolPanel(GUIToolPanel): - """ - Moving average/Filters - A quick and dirty solution to manipulate plotData - I need to think of a better way to do that - """ - def __init__(self, parent): - from pydatview.tools.signal_analysis import FILTERS - super(FilterToolPanel,self).__init__(parent) - self.parent = parent # parent is GUIPlotPanel - - self._DEFAULT_FILTERS=FILTERS - - # Setting default states to parent - if 'Filter' not in self.parent.plotDataOptions.keys(): - self.parent.plotDataOptions['Filter']=None - self._filterApplied = type(self.parent.plotDataOptions['Filter'])==dict - - - self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) - self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) - self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear) - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) - - self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) - - self.cbFilters = wx.ComboBox(self, choices=[filt['name'] for filt in self._DEFAULT_FILTERS], style=wx.CB_READONLY) - self.lbParamName = wx.StaticText(self, -1, ' :') - self.cbFilters.SetSelection(0) -# self.tParam = wx.TextCtrl(self, wx.ID_ANY, '', style= wx.TE_PROCESS_ENTER, size=wx.Size(60,-1)) - self.tParam = wx.SpinCtrlDouble(self, value='11', size=wx.Size(80,-1)) - self.lbInfo = wx.StaticText( self, -1, '') - - - # --- Layout - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) - - - horzSizerT = wx.BoxSizer(wx.HORIZONTAL) - horzSizerT.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) - horzSizerT.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) - - horzSizer = wx.BoxSizer(wx.HORIZONTAL) - horzSizer.Add(wx.StaticText(self, -1, 'Filter:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) - horzSizer.Add(self.cbFilters , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) - horzSizer.Add(self.lbParamName , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) - horzSizer.Add(self.tParam , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) - - - vertSizer = wx.BoxSizer(wx.VERTICAL) - vertSizer.Add(self.lbInfo ,0, flag = wx.LEFT ,border = 5) - vertSizer.Add(horzSizerT ,1, flag = wx.LEFT|wx.EXPAND,border = 1) - vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) - self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) - self.SetSizer(self.sizer) - - # --- Events - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) - self.cbFilters.Bind(wx.EVT_COMBOBOX, self.onSelectFilt) - self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow, self.tParam) - self.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter, self.tParam) - try: - self.spintxt = self.tParam.Children[0] - except: - self.spintxt = None - if platform.system()=='Windows': - # See issue https://github.com/wxWidgets/Phoenix/issues/1762 - assert isinstance(self.spintxt, wx.TextCtrl) - self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onParamChangeChar) - - # --- Init triggers - self.updateTabList() - self.onSelectFilt() - self.onToggleApply(init=True) - - def destroy(self,event=None): - super(FilterToolPanel,self).destroy() - - def onSelectFilt(self, event=None): - """ Select the filter, but does not applied it to the plotData - parentFilt is unchanged - But if the parent already has - """ - iFilt = self.cbFilters.GetSelection() - filt = self._DEFAULT_FILTERS[iFilt] - self.lbParamName.SetLabel(filt['paramName']+':') - #self.tParam.SetRange(filt['paramRange'][0], filt['paramRange'][1]) - # NOTE: if min value for range is not 0, the Ctrl prevents you to enter 0.01 - self.tParam.SetRange(0, filt['paramRange'][1]) - self.tParam.SetIncrement(filt['increment']) - self.tParam.SetDigits(filt['digits']) - - parentFilt=self.parent.plotDataOptions['Filter'] - # Value - if type(parentFilt)==dict and parentFilt['name']==filt['name']: - self.tParam.SetValue(parentFilt['param']) - else: - self.tParam.SetValue(filt['param']) - # Trigger plot if applied - self.onParamChange(self) - - def _GUI2Data(self): - iFilt = self.cbFilters.GetSelection() - filt = self._DEFAULT_FILTERS[iFilt].copy() - try: - filt['param']=np.float(self.spintxt.Value) - except: - print('[WARN] pyDatView: Issue on Mac: GUITools.py/_GUI2Data. Help needed.') - filt['param']=np.float(self.tParam.Value) - if filt['param']0: - raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) - - # We stop applying - self.onToggleApply() - - def onPlot(self, event=None): - """ - Overlay on current axis the filter - """ - from pydatview.tools.signal_analysis import applyFilter - if len(self.parent.plotData)!=1: - Error(self,'Plotting only works for a single plot. Plot less data.') - return - filt=self._GUI2Data() - - PD = self.parent.plotData[0] - y_filt = applyFilter(PD.x0, PD.y0, filt) - ax = self.parent.fig.axes[0] - - PD_new = PlotData() - PD_new.fromXY(PD.x0, y_filt) - self.parent.transformPlotData(PD_new) - ax.plot(PD_new.x, PD_new.y, '-') - self.parent.canvas.draw() - - def onClear(self, event): - self.parent.load_and_draw() # Data will change - - def onParamChange(self, event=None): - if self._filterApplied: - self.parent.plotDataOptions['Filter'] =self._GUI2Data() - #print('OnParamChange', self.parent.plotDataOptions['Filter']) - self.parent.load_and_draw() # Data will change - - def onParamChangeArrow(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeEnter(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeChar(self, event): - event.Skip() - code = event.GetKeyCode() - if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: - #print(self.spintxt.Value) - self.tParam.SetValue(self.spintxt.Value) - self.onParamChangeEnter(event) - - def onTabChange(self,event=None): - #tabList = self.parent.selPanel.tabList - #iSel=self.cbTabs.GetSelection() - pass - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - - def onHelp(self,event=None): - Info(self,"""Filtering. - -The filtering operation changes the "y" values of a table/plot, -applying a given filter (typically cutting off some frequencies). - -To filter perform the following step: - -- Chose a filtering method: - - Moving average: apply a moving average filter, with - a length specified by the window size (in indices) - - High pass 1st order: apply a first oder high-pass filter, - passing the frequencies above the cutoff frequency parameter. - - Low pass 1st order: apply a first oder low-pass filter, - passing the frequencies below the cutoff frequency parameter. - -- Click on one of the following buttons: - - Plot: will display the filtered data on the figure - - Apply: will perform the filtering on the fly for all new plots - - Add: will create new table(s) with filtered values for all - signals. This process might take some time. - Currently done for all tables. -""") - - - - # --------------------------------------------------------------------------------} # --- Mask # --------------------------------------------------------------------------------{ @@ -937,7 +669,6 @@ def onHelp(self,event=None): TOOLS={ 'LogDec': LogDecToolPanel, 'Outlier': OutlierToolPanel, - 'Filter': FilterToolPanel, 'Mask': MaskToolPanel, 'FASTRadialAverage': RadialToolPanel, 'CurveFitting': CurveFitToolPanel, diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 8f64f78..f3cc15e 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -26,12 +26,17 @@ def _data_binning(label, mainframe): from .data_binning import binningAction return binningAction(label, mainframe) +def _data_filter(label, mainframe): + from .data_filter import filterAction + return filterAction(label, mainframe) + def _data_sampler(label, mainframe): from .data_sampler import samplerAction return samplerAction(label, mainframe) dataPlugins=[ # Name/label , callback , is a Panel + ('Filter' , _data_filter , True ), ('Resample' , _data_sampler , True ), ('Bin data' , _data_binning , True ), ('Standardize Units (SI)', _data_standardizeUnitsSI, False), diff --git a/pydatview/plugins/data_filter.py b/pydatview/plugins/data_filter.py new file mode 100644 index 0000000..72abebe --- /dev/null +++ b/pydatview/plugins/data_filter.py @@ -0,0 +1,349 @@ +import wx +import numpy as np +from pydatview.GUITools import GUIToolPanel, TOOL_BORDER +from pydatview.common import CHAR, Error, Info, pretty_num_short +from pydatview.common import DummyMainFrame +from pydatview.plotdata import PlotData +from pydatview.pipeline import PlotDataAction +import platform +# --------------------------------------------------------------------------------} +# --- Data +# --------------------------------------------------------------------------------{ +# See FILTERS in signal_analysis +_DEFAULT_DICT={ + 'active':False, + 'name':'Moving average', + 'param':100, + 'paramName':'Window Size', + 'paramRange':[1, 100000], + 'increment':1, + 'digits':0 +} +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def filterAction(label, mainframe=None, data=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + The action is also edited and created by the GUI Editor + """ + if data is None: + # NOTE: if we don't do copy below, we will end up remembering even after the action was deleted + # its not a bad feature, but we might want to think it through + # One issue is that "active" is kept in memory + data=_DEFAULT_DICT + data['active'] = False #<<< Important + + guiCallback=None + if mainframe is not None: + guiCallback = mainframe.redraw + + action = PlotDataAction( + name=label, + plotDataFunction = filterXY, + guiEditorClass = FilterToolPanel, + guiCallback = guiCallback, + data = data, + mainframe=mainframe + ) + return action +# --------------------------------------------------------------------------------} +# --- Main method +# --------------------------------------------------------------------------------{ +def filterXY(x, y, opts): + from pydatview.tools.signal_analysis import applyFilter + y_new = applyFilter(x, y, opts) + return x, y_new + +# --------------------------------------------------------------------------------} +# --- GUI to edit plugin and control the action +# --------------------------------------------------------------------------------{ +class FilterToolPanel(GUIToolPanel): + + def __init__(self, parent, action=None): + GUIToolPanel.__init__(self, parent) + + # --- Creating "Fake data" for testing only! + if action is None: + print('[WARN] Calling GUI without an action! Creating one.') + mainframe = DummyMainFrame(parent) + action = binningAction(label='dummyAction', mainframe=mainframe) + + # --- Data + self.parent = parent # parent is GUIPlotPanel + self.mainframe = action.mainframe + self.data = action.data + self.action = action + from pydatview.tools.signal_analysis import FILTERS + self._FILTERS_USER=FILTERS .copy() + + # --- GUI elements + self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) + self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) + self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) + self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear) + self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) + + self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better + self.cbFilters = wx.ComboBox(self, choices=[filt['name'] for filt in self._FILTERS_USER], style=wx.CB_READONLY) + self.lbParamName = wx.StaticText(self, -1, ' :') + self.cbFilters.SetSelection(0) +# self.tParam = wx.TextCtrl(self, wx.ID_ANY, '', style= wx.TE_PROCESS_ENTER, size=wx.Size(60,-1)) + self.tParam = wx.SpinCtrlDouble(self, value='11', size=wx.Size(80,-1)) + self.lbInfo = wx.StaticText( self, -1, '') + + + # --- Layout + btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) + btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) + + + horzSizerT = wx.BoxSizer(wx.HORIZONTAL) + horzSizerT.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) + horzSizerT.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) + + horzSizer = wx.BoxSizer(wx.HORIZONTAL) + horzSizer.Add(wx.StaticText(self, -1, 'Filter:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) + horzSizer.Add(self.cbFilters , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) + horzSizer.Add(self.lbParamName , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) + horzSizer.Add(self.tParam , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) + + + vertSizer = wx.BoxSizer(wx.VERTICAL) + vertSizer.Add(self.lbInfo ,0, flag = wx.LEFT ,border = 5) + vertSizer.Add(horzSizerT ,1, flag = wx.LEFT|wx.EXPAND,border = 1) + vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) + self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) + self.SetSizer(self.sizer) + + # --- Events + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + self.cbFilters.Bind(wx.EVT_COMBOBOX, self.onSelectFilt) + self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow, self.tParam) + self.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter, self.tParam) + try: + self.spintxt = self.tParam.Children[0] + except: + self.spintxt = None + if platform.system()=='Windows': + # See issue https://github.com/wxWidgets/Phoenix/issues/1762 + assert isinstance(self.spintxt, wx.TextCtrl) + self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onParamChangeChar) + + # --- Init triggers + self._Data2GUI() + self.updateTabList() + self.onSelectFilt() + self.onToggleApply(init=True) + + # --- Implementation specific + def onSelectFilt(self, event=None): + """ Select the filter, but does not applied it to the plotData + parentFilt is unchanged + But if the parent already has + """ + iFilt = self.cbFilters.GetSelection() + filt = self._FILTERS_USER[iFilt] + self.lbParamName.SetLabel(filt['paramName']+':') + #self.tParam.SetRange(filt['paramRange'][0], filt['paramRange'][1]) + # NOTE: if min value for range is not 0, the Ctrl prevents you to enter 0.01 + self.tParam.SetRange(0, filt['paramRange'][1]) + self.tParam.SetIncrement(filt['increment']) + self.tParam.SetDigits(filt['digits']) + + parentFilt=self.data + # Value + if type(parentFilt)==dict and parentFilt['name']==filt['name']: + self.tParam.SetValue(parentFilt['param']) + else: + self.tParam.SetValue(filt['param']) + # Trigger plot if applied + self.onParamChange(self) + + # --- Bindings for plot triggers on parameters changes + def onParamChange(self, event=None): + self._GUI2Data() + if self.data['active']: + self.parent.load_and_draw() # Data will change + + def onParamChangeArrow(self, event): + self.onParamChange() + event.Skip() + + def onParamChangeEnter(self, event): + self.onParamChange() + event.Skip() + + def onParamChangeChar(self, event): + event.Skip() + code = event.GetKeyCode() + if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: + #print(self.spintxt.Value) + self.tParam.SetValue(self.spintxt.Value) + self.onParamChangeEnter(event) + + # --- Table related + def onTabChange(self,event=None): + #tabList = self.parent.selPanel.tabList + #iSel=self.cbTabs.GetSelection() + pass + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + try: + iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + except RuntimeError: + pass + + # --- External Calls + def cancelAction(self, redraw=True): + """ do cancel the action""" + self.lbInfo.SetLabel( + 'Click on "Apply" to set filter on the fly for all plots. '+ + 'Click on "Plot" to try a filter on the current plot.' + ) + self.btPlot.Enable(True) + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + self.btApply.SetValue(False) + self.data['active'] = False + if redraw: + self.parent.load_and_draw() # Data will change based on plotData + + # --- Fairly generic + def _GUI2Data(self): + iFilt = self.cbFilters.GetSelection() + opt = self._FILTERS_USER[iFilt] + try: + opt['param']=np.float(self.spintxt.Value) + except: + print('[WARN] pyDatView: Issue on Mac: GUITools.py/_GUI2Data. Help needed.') + opt['param']=np.float(self.tParam.Value) + if opt['param']0: + raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) + + # We stop applying + self.onToggleApply() + + def onPlot(self, event=None): + """ + Overlay on current axis the filter + """ + + if len(self.parent.plotData)!=1: + Error(self,'Plotting only works for a single plot. Plot less data.') + return + self._GUI2Data() + PD = self.parent.plotData[0] + x_new, y_new = filterXY(PD.x0, PD.y0, self.data) + + ax = self.parent.fig.axes[0] + PD_new = PlotData() + PD_new.fromXY(x_new, y_new) + self.parent.transformPlotData(PD_new) + ax.plot(PD_new.x, PD_new.y, '-') + self.parent.canvas.draw() + + def onClear(self, event): + self.parent.load_and_draw() # Data will change + + + def onHelp(self,event=None): + Info(self,"""Filtering. + +The filtering operation changes the "y" values of a table/plot, +applying a given filter (typically cutting off some frequencies). + +To filter perform the following step: + +- Chose a filtering method: + - Moving average: apply a moving average filter, with + a length specified by the window size (in indices) + - High pass 1st order: apply a first oder high-pass filter, + passing the frequencies above the cutoff frequency parameter. + - Low pass 1st order: apply a first oder low-pass filter, + passing the frequencies below the cutoff frequency parameter. + +- Click on one of the following buttons: + - Plot: will display the filtered data on the figure + - Apply: will perform the filtering on the fly for all new plots + - Add: will create new table(s) with filtered values for all + signals. This process might take some time. + Currently done for all tables. +""") + + + + diff --git a/pydatview/plugins/data_sampler.py b/pydatview/plugins/data_sampler.py index 7f4f7fe..db01a74 100644 --- a/pydatview/plugins/data_sampler.py +++ b/pydatview/plugins/data_sampler.py @@ -63,7 +63,8 @@ def __init__(self, parent, action=None): print('[WARN] Calling GUI without an action! Creating one.') mainframe = DummyMainFrame(parent) action = binningAction(label='dummyAction', mainframe=mainframe) - # --- Data from other modules + + # --- Data self.parent = parent # parent is GUIPlotPanel self.mainframe = action.mainframe self.data = action.data @@ -80,7 +81,6 @@ def __init__(self, parent, action=None): self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) - #self.lb = wx.StaticText( self, -1, """ Click help """) self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS_DEFAULT], style=wx.CB_READONLY) @@ -165,6 +165,7 @@ def onMethodChange(self, event=None, init=True): # self.textNewX.SetValue(str(opt['param'])[2:-2]) self.onParamChange() + # --- Bindings for plot triggers on parameters changes def onParamChange(self, event=None): self._GUI2Data() if self.data['active']: @@ -246,6 +247,7 @@ def onToggleApply(self, event=None, init=False): if self.mainframe is not None: self.mainframe.removeAction(self.action) self.cancelAction(redraw=not init) + self.setCurrentX() def onAdd(self,event=None): @@ -269,7 +271,6 @@ def onAdd(self,event=None): self.onToggleApply() def onPlot(self,event=None): - from pydatview.tools.signal_analysis import applySampler if len(self.parent.plotData)!=1: Error(self,'Plotting only works for a single plot. Plot less data.') return From 0c9e109e402ce8e320de29b5a0ad0958143c9bc7 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 05:37:49 +0100 Subject: [PATCH 066/178] Pipeline: outliers as a plugin/action --- pydatview/GUIPlotPanel.py | 3 +- pydatview/GUITools.py | 93 ------------ pydatview/main.py | 2 - pydatview/plotdata.py | 27 +--- pydatview/plugins/__init__.py | 28 ++-- pydatview/plugins/data_removeOutliers.py | 173 +++++++++++++++++++++++ 6 files changed, 196 insertions(+), 130 deletions(-) create mode 100644 pydatview/plugins/data_removeOutliers.py diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 47963d8..33f2945 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -409,7 +409,6 @@ def __init__(self, parent, selPanel, pipeline=None, infoPanel=None, data=None): self.infoPanel.setPlotMatrixCallbacks(self._onPlotMatrixLeftClick, self._onPlotMatrixRightClick) self.parent = parent self.plotData = [] - self.plotDataOptions=dict() # TODO remove me self.toolPanel=None if data is not None: self.data = data @@ -879,7 +878,7 @@ def getPlotData(self,plotType): for i,idx in enumerate(ID): # Initialize each plotdata based on selected table and selected id channels pd=PlotData(); - pd.fromIDs(tabs, i, idx, SameCol, self.plotDataOptions, pipeline=self.pipeline) + pd.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeline) # Possible change of data if plotType=='MinMax': self.setPD_MinMax(pd) diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 9ceef1f..1a6c61e 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -93,98 +93,6 @@ def onCompute(self,event=None): self.lb.SetLabel('Failed. The signal needs to look like the decay of a first order system.') #self.parent.load_and_draw(); # DATA HAS CHANGED -# --------------------------------------------------------------------------------} -# --- Outliers -# --------------------------------------------------------------------------------{ -class OutlierToolPanel(GUIToolPanel): - """ - A quick and dirty solution to manipulate plotData - I need to think of a better way to do that - """ - def __init__(self, parent): - super(OutlierToolPanel,self).__init__(parent) - self.parent = parent # parent is GUIPlotPanel - - # Setting default states to parent - if 'RemoveOutliers' not in self.parent.plotDataOptions.keys(): - self.parent.plotDataOptions['RemoveOutliers']=False - if 'OutliersMedianDeviation' not in self.parent.plotDataOptions.keys(): - self.parent.plotDataOptions['OutliersMedianDeviation']=5 - - btClose = self.getBtBitmap(self,'Close','close',self.destroy) - self.btComp = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleCompute) - - lb1 = wx.StaticText(self, -1, 'Median deviation:') -# self.tMD = wx.TextCtrl(self, wx.ID_ANY,, size = (30,-1), style=wx.TE_PROCESS_ENTER) - self.tMD = wx.SpinCtrlDouble(self, value='11', size=wx.Size(60,-1)) - self.tMD.SetValue(self.parent.plotDataOptions['OutliersMedianDeviation']) - self.tMD.SetRange(0.0, 1000) - self.tMD.SetIncrement(0.5) - self.tMD.SetDigits(1) - - self.lb = wx.StaticText( self, -1, '') - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btClose ,0,flag = wx.LEFT|wx.CENTER,border = 1) - self.sizer.Add(self.btComp,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(lb1 ,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(self.tMD ,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(self.lb ,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.SetSizer(self.sizer) - - self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onMDChangeArrow, self.tMD) - self.Bind(wx.EVT_TEXT_ENTER, self.onMDChangeEnter, self.tMD) - - if platform.system()=='Windows': - # See issue https://github.com/wxWidgets/Phoenix/issues/1762 - self.spintxt = self.tMD.Children[0] - assert isinstance(self.spintxt, wx.TextCtrl) - self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onMDChangeChar) - - self.onToggleCompute(init=True) - - def destroy(self,event=None): - self.parent.plotDataOptions['RemoveOutliers']=False - super(OutlierToolPanel,self).destroy() - - def onToggleCompute(self,event=None, init=False): - self.parent.plotDataOptions['OutliersMedianDeviation'] = float(self.tMD.Value) - - if not init: - self.parent.plotDataOptions['RemoveOutliers']= not self.parent.plotDataOptions['RemoveOutliers'] - - if self.parent.plotDataOptions['RemoveOutliers']: - self.lb.SetLabel('Outliers are now removed on the fly. Click "Clear" to stop.') - self.btComp.SetLabel(CHAR['sun']+' Clear') - else: - self.lb.SetLabel('Click on "Apply" to remove outliers on the fly for all new plot.') - self.btComp.SetLabel(CHAR['cloud']+' Apply') - - if not init: - self.parent.load_and_draw() # Data will change - - def onMDChange(self, event=None): - #print(self.tMD.Value) - self.parent.plotDataOptions['OutliersMedianDeviation'] = float(self.tMD.Value) - if self.parent.plotDataOptions['RemoveOutliers']: - self.parent.load_and_draw() # Data will change - - def onMDChangeArrow(self, event): - self.onMDChange() - event.Skip() - - def onMDChangeEnter(self, event): - self.onMDChange() - event.Skip() - - def onMDChangeChar(self, event): - event.Skip() - code = event.GetKeyCode() - if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: - #print(self.spintxt.Value) - self.tMD.SetValue(self.spintxt.Value) - self.onMDChangeEnter(event) - - # --------------------------------------------------------------------------------} # --- Mask @@ -668,7 +576,6 @@ def onHelp(self,event=None): TOOLS={ 'LogDec': LogDecToolPanel, - 'Outlier': OutlierToolPanel, 'Mask': MaskToolPanel, 'FASTRadialAverage': RadialToolPanel, 'CurveFitting': CurveFitToolPanel, diff --git a/pydatview/main.py b/pydatview/main.py index 428d5db..6a8d617 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -145,8 +145,6 @@ def __init__(self, data=None): dataMenu = wx.Menu() menuBar.Append(dataMenu, "&Data") self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e, 'Mask') , dataMenu.Append(wx.ID_ANY, 'Mask')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Outlier'), dataMenu.Append(wx.ID_ANY, 'Outliers removal')) - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'Filter') , dataMenu.Append(wx.ID_ANY, 'Filter')) self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'FASTRadialAverage'), dataMenu.Append(wx.ID_ANY, 'FAST - Radial average')) # --- Data Plugins diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index c4c349d..0bb8996 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -34,7 +34,7 @@ def __init__(PD, x=None, y=None, sx='', sy=''): if x is not None and y is not None: PD.fromXY(x,y,sx,sy) - def fromIDs(PD, tabs, i, idx, SameCol, Options=None, pipeline=None): + def fromIDs(PD, tabs, i, idx, SameCol, pipeline=None): """ Nasty initialization of plot data from "IDs" """ PD.id = i PD.it = idx[0] # table index @@ -51,7 +51,7 @@ def fromIDs(PD, tabs, i, idx, SameCol, Options=None, pipeline=None): PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info PD.c =c # raw values, used by PDF - PD._post_init(Options=Options, pipeline=pipeline) + PD._post_init(pipeline=pipeline) def fromXY(PD, x, y, sx='', sy=''): PD.x = x @@ -67,26 +67,9 @@ def fromXY(PD, x, y, sx='', sy=''): PD._post_init() - def _post_init(PD, Options=None, pipeline=None): - if Options is None: - Options = {} - # --- Perform data manipulation on the fly - #[print(k,v) for k,v in Options.items()] - keys=Options.keys() - # TODO setup an "Order" - if 'RemoveOutliers' in keys: - if Options['RemoveOutliers']: - from pydatview.tools.signal_analysis import reject_outliers - try: - PD.x, PD.y = reject_outliers(PD.y, PD.x, m=Options['OutliersMedianDeviation']) - except: - raise Exception('Warn: Outlier removal failed. Desactivate it or use a different signal. ') - if 'Filter' in keys: - if Options['Filter']: - from pydatview.tools.signal_analysis import applyFilter - PD.y = applyFilter(PD.x, PD.y, Options['Filter']) - - # --- Apply filters from pipeline + def _post_init(PD, pipeline=None): + + # --- Apply filters from pipeline on the fly if pipeline is not None: print('[PDat]', pipeline.__reprFilters__()) if pipeline is not None: diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index f3cc15e..852666d 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -14,6 +14,22 @@ def _function_name(mainframe, event=None, label='') See working examples in this file and this directory. """ +def _data_filter(label, mainframe): + from .data_filter import filterAction + return filterAction(label, mainframe) + +def _data_sampler(label, mainframe): + from .data_sampler import samplerAction + return samplerAction(label, mainframe) + +def _data_binning(label, mainframe): + from .data_binning import binningAction + return binningAction(label, mainframe) + +def _data_removeOutliers(label, mainframe): + from .data_removeOutliers import removeOutliersAction + return removeOutliersAction(label, mainframe) + def _data_standardizeUnitsSI(label, mainframe=None): from .data_standardizeUnits import standardizeUnitsAction return standardizeUnitsAction(label, mainframe, flavor='SI') @@ -22,20 +38,10 @@ def _data_standardizeUnitsWE(label, mainframe=None): from .data_standardizeUnits import standardizeUnitsAction return standardizeUnitsAction(label, mainframe, flavor='WE') -def _data_binning(label, mainframe): - from .data_binning import binningAction - return binningAction(label, mainframe) - -def _data_filter(label, mainframe): - from .data_filter import filterAction - return filterAction(label, mainframe) - -def _data_sampler(label, mainframe): - from .data_sampler import samplerAction - return samplerAction(label, mainframe) dataPlugins=[ # Name/label , callback , is a Panel + ('Remove Outliers' , _data_removeOutliers , True ), ('Filter' , _data_filter , True ), ('Resample' , _data_sampler , True ), ('Bin data' , _data_binning , True ), diff --git a/pydatview/plugins/data_removeOutliers.py b/pydatview/plugins/data_removeOutliers.py new file mode 100644 index 0000000..8160116 --- /dev/null +++ b/pydatview/plugins/data_removeOutliers.py @@ -0,0 +1,173 @@ +import wx +import numpy as np +from pydatview.GUITools import GUIToolPanel, TOOL_BORDER +from pydatview.common import CHAR, Error, Info, pretty_num_short +from pydatview.common import DummyMainFrame +from pydatview.plotdata import PlotData +from pydatview.pipeline import PlotDataAction +import platform +# --------------------------------------------------------------------------------} +# --- Data +# --------------------------------------------------------------------------------{ +_DEFAULT_DICT={ + 'active':False, + 'medianDeviation':5 +} +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def removeOutliersAction(label, mainframe=None, data=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + The action is also edited and created by the GUI Editor + """ + if data is None: + # NOTE: if we don't do copy below, we will end up remembering even after the action was deleted + # its not a bad feature, but we might want to think it through + # One issue is that "active" is kept in memory + data=_DEFAULT_DICT + data['active'] = False #<<< Important + + guiCallback=None + if mainframe is not None: + guiCallback = mainframe.redraw + + action = PlotDataAction( + name = label, + plotDataFunction = removeOutliersXY, + guiEditorClass = RemoveOutliersToolPanel, + guiCallback = guiCallback, + data = data, + mainframe = mainframe + ) + return action +# --------------------------------------------------------------------------------} +# --- Main method +# --------------------------------------------------------------------------------{ +def removeOutliersXY(x, y, opts): + from pydatview.tools.signal_analysis import reject_outliers + try: + x, y = reject_outliers(y, x, m=opts['medianDeviation']) + except: + raise Exception('Warn: Outlier removal failed. Desactivate it or use a different signal. ') + return x, y + +# --------------------------------------------------------------------------------} +# --- GUI to edit plugin and control the action +# --------------------------------------------------------------------------------{ +class RemoveOutliersToolPanel(GUIToolPanel): + + def __init__(self, parent, action): + GUIToolPanel.__init__(self, parent) + + # --- Creating "Fake data" for testing only! + if action is None: + print('[WARN] Calling GUI without an action! Creating one.') + mainframe = DummyMainFrame(parent) + action = binningAction(label='dummyAction', mainframe=mainframe) + + # --- Data + self.parent = parent # parent is GUIPlotPanel + self.mainframe = action.mainframe + self.data = action.data + self.action = action + # --- GUI elements + self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) + + lb1 = wx.StaticText(self, -1, 'Median deviation:') +# self.tMD = wx.TextCtrl(self, wx.ID_ANY,, size = (30,-1), style=wx.TE_PROCESS_ENTER) + self.tMD = wx.SpinCtrlDouble(self, value='11', size=wx.Size(60,-1)) + self.tMD.SetValue(5) + self.tMD.SetRange(0.0, 1000) + self.tMD.SetIncrement(0.5) + self.tMD.SetDigits(1) + self.lb = wx.StaticText( self, -1, '') + + # --- Layout + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(self.btClose,0,flag = wx.LEFT|wx.CENTER,border = 1) + self.sizer.Add(self.btApply,0,flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(lb1 ,0,flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(self.tMD ,0,flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(self.lb ,0,flag = wx.LEFT|wx.CENTER,border = 5) + self.SetSizer(self.sizer) + + # --- Events + self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow, self.tMD) + self.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter, self.tMD) + + if platform.system()=='Windows': + # See issue https://github.com/wxWidgets/Phoenix/issues/1762 + self.spintxt = self.tMD.Children[0] + assert isinstance(self.spintxt, wx.TextCtrl) + self.spintxt.Bind(wx.EVT_CHAR_HOOK, self.onParamChangeChar) + + # --- Init triggers + self._Data2GUI() + self.onToggleApply(init=True) + + # --- Implementation specific + + # --- Bindings for plot triggers on parameters changes + def onParamChange(self, event=None): + self._GUI2Data() + if self.data['active']: + self.parent.load_and_draw() # Data will change + + def onParamChangeArrow(self, event): + self.onParamChange() + event.Skip() + + def onParamChangeEnter(self, event): + self.onParamChange() + event.Skip() + + def onParamChangeChar(self, event): + event.Skip() + code = event.GetKeyCode() + if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: + #print(self.spintxt.Value) + self.tMD.SetValue(self.spintxt.Value) + self.onParamChangeEnter(event) + + # --- Table related + # --- External Calls + def cancelAction(self, redraw=True): + """ do cancel the action""" + self.lb.SetLabel('Click on "Apply" to remove outliers on the fly for all new plot.') + self.btApply.SetLabel(CHAR['cloud']+' Apply') + self.btApply.SetValue(False) + self.data['active'] = False + if redraw: + self.parent.load_and_draw() # Data will change based on plotData + + # --- Fairly generic + def _GUI2Data(self): + self.data['medianDeviation'] = float(self.tMD.Value) + + def _Data2GUI(self): + self.tMD.SetValue(self.data['medianDeviation']) + + def onToggleApply(self, event=None, init=False): + + if not init: + self.data['active'] = not self.data['active'] + + if self.data['active']: + self._GUI2Data() + self.lb.SetLabel('Outliers are now removed on the fly. Click "Clear" to stop.') + self.btApply.SetLabel(CHAR['sun']+' Clear') + self.btApply.SetValue(True) + # The action is now active we add it to the pipeline, unless it's already in it + if self.mainframe is not None: + self.mainframe.addAction(self.action, cancelIfPresent=True) + if not init: + self.parent.load_and_draw() # filter will be applied in plotData.py + else: + # We remove our action from the pipeline + if not init: + if self.mainframe is not None: + self.mainframe.removeAction(self.action) + self.cancelAction(redraw= not init) + From c9c1804e0752eb1e656214520bcb9aca5f511c00 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 11:11:50 +0100 Subject: [PATCH 067/178] Pipeline: preparing application per table --- pydatview/GUIPlotPanel.py | 2 +- pydatview/GUISelectionPanel.py | 52 +++++++++++++++---------------- pydatview/GUITools.py | 14 ++++----- pydatview/Tables.py | 20 +++++++----- pydatview/main.py | 14 ++++----- pydatview/pipeline.py | 7 +++-- pydatview/plotdata.py | 6 ++-- pydatview/plugins/data_binning.py | 2 +- pydatview/plugins/data_filter.py | 2 +- pydatview/plugins/data_sampler.py | 2 +- 10 files changed, 64 insertions(+), 57 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 33f2945..62ff612 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -872,7 +872,7 @@ def getPlotData(self,plotType): self.selMode=selMode # we store the selection mode del self.plotData self.plotData=[] - tabs=self.selPanel.tabList.getTabs() # TODO, selPanel should just return the PlotData... + tabs=self.selPanel.tabList try: for i,idx in enumerate(ID): diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index f9934db..7f9a9d3 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -270,7 +270,7 @@ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=Fal self.itNameFile = wx.MenuItem(self, -1, "Naming: by file names", kind=wx.ITEM_CHECK) self.Append(self.itNameFile) self.Bind(wx.EVT_MENU, self.OnNaming, self.itNameFile) - self.Check(self.itNameFile.GetId(), self.parent.GetParent().tabList.naming=='FileNames') # Checking the menu box + self.Check(self.itNameFile.GetId(), self.tabList.naming=='FileNames') # Checking the menu box item = wx.MenuItem(self, -1, "Sort by name") self.Append(item) @@ -312,7 +312,7 @@ def OnNaming(self, event=None): def OnMergeTabs(self, event): # --- Figure out the common columns - tabs = [self.tabList.get(i) for i in self.ISel] + tabs = [self.tabList[i] for i in self.ISel] IKeepPerTab, IMissPerTab, IDuplPerTab, _ = getTabCommonColIndices(tabs) nCommonCols = len(IKeepPerTab[0]) commonCol = None @@ -366,7 +366,7 @@ class ColumnPopup(wx.Menu): """ Popup Menu when right clicking on the column list """ def __init__(self, parent, selPanel, fullmenu=False): wx.Menu.__init__(self) - self.parent = parent # parent is ColumnPanel + self.parent = parent # parent is ColumnPanel self.selPanel = selPanel # we need a selPanel self.ISel = self.parent.lbColumns.GetSelections() @@ -412,9 +412,9 @@ def OnRenameColumn(self, event=None): ITab,STab=self.selPanel.getSelectedTables() # TODO adapt me for Sim. tables mode iFull = self.parent.Filt2Full[iFilt] - if self.tabList.haveSameColumns(ITab): + if self.selPanel.tabList.haveSameColumns(ITab): for iTab,sTab in zip(ITab,STab): - self.tabList.get(iTab).renameColumn(iFull,newName) + self.selPanel.tabList[iTab].renameColumn(iFull,newName) else: self.parent.tab.renameColumn(iFull,newName) self.parent.updateColumn(iFilt,newName) #faster @@ -427,7 +427,7 @@ def OnEditColumn(self, event): ITab, STab = self.selPanel.getSelectedTables() for iTab,sTab in zip(ITab,STab): if sTab == self.parent.tab.active_name: - for f in self.tabList.get(iTab).formulas: + for f in self.selPanel.tabList[iTab].formulas: if f['pos'] == self.ISel[0]: sName = f['name'] sFormula = f['formula'] @@ -444,12 +444,12 @@ def OnDeleteColumn(self, event): IFull = [iFull for iFull in IFull if iFull>=0] if self.selPanel.tabList.haveSameColumns(ITab): for iTab,sTab in zip(ITab,STab): - self.selPanel.tabList.get(iTab).deleteColumns(IFull) + self.selPanel.tabList[iTab].deleteColumns(IFull) else: self.parent.tab.deleteColumns(IFull) self.parent.setColumns() self.parent.setGUIColumns(xSel=iX) - self.redraw() + self.selPanel.redraw() def OnAddColumn(self, event): self.showFormulaDialog('Add a new column') @@ -494,10 +494,10 @@ def showFormulaDialog(self, title, name='', formula='', edit=False): if haveSameColumns or self.parent.tab.active_name == sTab: # apply formula to all tables with same columns, otherwise only to active table if edit: - bValid=self.selPanel.tabList.get(iTab).setColumnByFormula(sName,sFormula,iFull) + bValid=self.selPanel.tabList[iTab].setColumnByFormula(sName,sFormula,iFull) iOffset = 0 # we'll stay on this column that we are editing else: - bValid=self.selPanel.tabList.get(iTab).addColumnByFormula(sName,sFormula,iFull) + bValid=self.selPanel.tabList[iTab].addColumnByFormula(sName,sFormula,iFull) iOffset = 1 # we'll select this newly created column if not bValid: sError+='The formula didn''t eval for table {}\n'.format(sTab) @@ -523,15 +523,14 @@ def showFormulaDialog(self, title, name='', formula='', edit=False): # --------------------------------------------------------------------------------{ class TablePanel(wx.Panel): """ Display list of tables """ - def __init__(self, parent, selPanel, mainframe, tabList): + def __init__(self, parent, selPanel, mainframe): # TODO get rid of mainframe # Superclass constructor super(TablePanel,self).__init__(parent) # DATA - self.parent = parent # splitter self.selPanel = selPanel self.mainframe = mainframe - self.tabList = tabList + self.tabList = selPanel.tabList # GUI tb = wx.ToolBar(self,wx.ID_ANY,style=wx.TB_HORIZONTAL|wx.TB_TEXT|wx.TB_HORZ_LAYOUT|wx.TB_NODIVIDER) self.bt=wx.Button(tb,wx.ID_ANY,CHAR['menu'], style=wx.BU_EXACTFIT) @@ -951,7 +950,6 @@ def __init__(self, parent, tabList, mode='auto', mainframe=None): # DATA self.tabList = None self.itabForCol = None - self.parent = parent self.tabSelections = {} # x-Y-Columns selected for each table self.simTabSelection = {} # selection for simTable case self.filterSelection = ['','',''] # filters @@ -969,10 +967,10 @@ def __init__(self, parent, tabList, mode='auto', mainframe=None): # GUI DATA self.splitter = MultiSplit(self, style=wx.SP_LIVE_UPDATE) self.splitter.SetMinimumPaneSize(70) - self.tabPanel = TablePanel (self.splitter, self, mainframe, tabList) - self.colPanel1 = ColumnPanel(self.splitter, self); - self.colPanel2 = ColumnPanel(self.splitter, self); - self.colPanel3 = ColumnPanel(self.splitter, self); + self.tabPanel = TablePanel (self.splitter, selPanel=self, mainframe=mainframe) + self.colPanel1 = ColumnPanel(self.splitter, selPanel=self); + self.colPanel2 = ColumnPanel(self.splitter, selPanel=self); + self.colPanel3 = ColumnPanel(self.splitter, selPanel=self); self.tabPanel.Hide() self.colPanel1.Hide() self.colPanel2.Hide() @@ -1077,7 +1075,7 @@ def autoMode(self): self.twoColumnsMode() else: # See if tables are quite similar - IKeepPerTab, IMissPerTab, IDuplPerTab, nCols = getTabCommonColIndices([self.tabList.get(i) for i in ISel]) + IKeepPerTab, IMissPerTab, IDuplPerTab, nCols = getTabCommonColIndices([self.tabList[i] for i in ISel]) if np.all(np.array([len(I) for I in IMissPerTab]))=2): self.simColumnsMode() elif len(ISel)==2: @@ -1200,7 +1198,7 @@ def setTables(self,tabList,update=False): self.updateLayout(self.modeRequested) def setTabForCol(self,iTabSel,iPanel): - t = self.tabList.get(iTabSel) + t = self.tabList[iTabSel] ts = self.tabSelections[t.name] if iPanel==1: self.colPanel1.setTab(t,ts['xSel'],ts['ySel'], sFilter=self.filterSelection[0]) @@ -1213,7 +1211,7 @@ def setTabForCol(self,iTabSel,iPanel): def setColForSimTab(self,ISel): """ Set column panels for similar tables """ - tabs = [self.tabList.get(i) for i in ISel] + tabs = [self.tabList[i] for i in ISel] IKeepPerTab, IMissPerTab, IDuplPerTab, _ = getTabCommonColIndices(tabs) LenMiss = np.array([len(I) for I in IMissPerTab]) LenKeep = np.array([len(I) for I in IKeepPerTab]) @@ -1392,20 +1390,20 @@ def saveSelection(self): # --- Save selected columns for each tab if self.tabList.haveSameColumns(ISel): for ii in ISel: - t=self.tabList.get(ii) + t=self.tabList[ii] self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() else: if len(ISel)>=1: - t=self.tabList.get(ISel[0]) + t=self.tabList[ISel[0]] self.tabSelections[t.name]['xSel'] = self.colPanel1.comboX.GetSelection() self.tabSelections[t.name]['ySel'] = self.colPanel1.lbColumns.GetSelections() if len(ISel)>=2: - t=self.tabList.get(ISel[1]) + t=self.tabList[ISel[1]] self.tabSelections[t.name]['xSel'] = self.colPanel2.comboX.GetSelection() self.tabSelections[t.name]['ySel'] = self.colPanel2.lbColumns.GetSelections() if len(ISel)>=3: - t=self.tabList.get(ISel[2]) + t=self.tabList[ISel[2]] self.tabSelections[t.name]['xSel'] = self.colPanel3.comboX.GetSelection() self.tabSelections[t.name]['ySel'] = self.colPanel3.lbColumns.GetSelections() self.tabSelected = self.tabPanel.lbTab.GetSelections(); @@ -1434,9 +1432,9 @@ def getPlotDataSelection(self): IKeep=self.IKeepPerTab[i] for j,(iiy,ssy) in enumerate(zip(IY1,SY1)): iy = IKeep[iiy] - sy = self.tabList.get(itab).columns[IKeep[iiy]] + sy = self.tabList[itab].columns[IKeep[iiy]] iX1 = IKeep[iiX1] - sX1 = self.tabList.get(itab).columns[IKeep[iiX1]] + sX1 = self.tabList[itab].columns[IKeep[iiX1]] ID.append([itab,iX1,iy,sX1,sy,stab]) else: iX1,IY1,sX1,SY1 = self.colPanel1.getColumnSelection() diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index 1a6c61e..af7543e 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -156,9 +156,9 @@ def onTabChange(self,event=None): tabList = self.parent.selPanel.tabList iSel=self.cbTabs.GetSelection() if iSel==0: - maskString = tabList.commonMaskString + maskString = tabList.commonMaskString # for "all" else: - maskString= tabList.get(iSel-1).maskString + maskString= tabList[iSel-1].maskString # -1, because "0" is for "all" if len(maskString)>0: self.textMask.SetValue(maskString) #else: @@ -166,7 +166,7 @@ def onTabChange(self,event=None): # self.textMask.SetValue(self.guessMask) # no known mask def guessMask(self,tabList): - cols=tabList.get(0).columns_clean + cols=tabList[0].columns_clean if 'Time' in cols: return '{Time} > 100' elif 'Date' in cols: @@ -183,7 +183,7 @@ def onClear(self,event=None): if iSel==0: tabList.clearCommonMask() else: - tabList.get(iSel-1).clearMask() + tabList[iSel-1].clearMask() self.parent.load_and_draw() self.onTabChange() @@ -220,9 +220,9 @@ def onApply(self,event=None,bAdd=True): if len(errors)>0: raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) else: - dfs, name = tabList.get(iSel-1).applyMaskString(maskString, bAdd=bAdd) + dfs, name = tabList[iSel-1].applyMaskString(maskString, bAdd=bAdd) if bAdd: - self.parent.addTables([df],[name], bAdd=bAdd) + self.parent.addTables([dfs],[name], bAdd=bAdd) else: self.parent.load_and_draw() self.updateTabList() @@ -307,7 +307,7 @@ def onApply(self,event=None): if len(errors)>0: raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) else: - dfs, names = tabList.get(iSel-1).radialAvg(avgMethod,avgParam) + dfs, names = tabList[iSel-1].radialAvg(avgMethod,avgParam) self.parent.addTables([dfs],[names], bAdd=True) self.updateTabList() diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 657202a..c6c43c6 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -35,6 +35,16 @@ def defaultOptions(): return options # --- behaves like a list... + #def __delitem__(self, key): + # self.__delattr__(key) + + def __getitem__(self, key): + return self._tabs[key] + + def __setitem__(self, key, value): + raise Exception('Setting not allowed') + self._tabs[key] = value + def __iter__(self): self.__n = 0 return self @@ -164,11 +174,6 @@ def _load_file_tabs(self, filename, fileformat=None, bReload=False): warn='Warn: No dataframe found in file: '+filename+'\n' return tabs, warn - def getTabs(self): - # TODO remove me later - return self._tabs - - def haveSameColumns(self,I=None): if I is None: I=list(range(len(self._tabs))) @@ -390,9 +395,9 @@ def radialAvg(self,avgMethod,avgParam): names_new.append(n) return dfs_new, names_new, errors - # --- Element--related functions def get(self,i): + print('.>> GET') return self._tabs[i] @@ -419,7 +424,8 @@ class Table(object): - data - columns - name - - raw_name + - raw_name # Should be unique and can be used for identification + - ID # Should be unique and can be used for identification - active_name - filename - fileformat diff --git a/pydatview/main.py b/pydatview/main.py index 6a8d617..44a21c0 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -407,13 +407,13 @@ def setStatusBar(self, ISel=None): self.statusbar.SetStatusText('', ISTAT+1) # Filenames self.statusbar.SetStatusText('', ISTAT+2) # Shape elif nTabs==1: - self.statusbar.SetStatusText(self.tabList.get(0).fileformat_name, ISTAT+0) - self.statusbar.SetStatusText(self.tabList.get(0).filename , ISTAT+1) - self.statusbar.SetStatusText(self.tabList.get(0).shapestring , ISTAT+2) + self.statusbar.SetStatusText(self.tabList[0].fileformat_name, ISTAT+0) + self.statusbar.SetStatusText(self.tabList[0].filename , ISTAT+1) + self.statusbar.SetStatusText(self.tabList[0].shapestring , ISTAT+2) elif len(ISel)==1: - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).fileformat_name , ISTAT+0) - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).filename , ISTAT+1) - self.statusbar.SetStatusText(self.tabList.get(ISel[0]).shapestring , ISTAT+2) + self.statusbar.SetStatusText(self.tabList[ISel[0]].fileformat_name , ISTAT+0) + self.statusbar.SetStatusText(self.tabList[ISel[0]].filename , ISTAT+1) + self.statusbar.SetStatusText(self.tabList[ISel[0]].shapestring , ISTAT+2) else: self.statusbar.SetStatusText('{} tables loaded'.format(nTabs) ,ISTAT+0) self.statusbar.SetStatusText(", ".join(list(set([self.tabList.filenames[i] for i in ISel]))),ISTAT+1) @@ -442,7 +442,7 @@ def deleteTabs(self, I): self.onTabSelectionChange() def exportTab(self, iTab): - tab=self.tabList.get(iTab) + tab=self.tabList[iTab] default_filename=tab.basename +'.csv' with wx.FileDialog(self, "Save to CSV file",defaultFile=default_filename, style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as dlg: diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index a174382..1f7d1f6 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -77,7 +77,8 @@ def __init__(self, name, **kwargs): def apply(self, *args, **kwargs): pass # not pretty - def applyOnPlotData(self, x, y): + def applyOnPlotData(self, x, y, tabID): + # TODO apply only based on tabID x, y = self.plotDataFunction(x, y, self.data) return x, y @@ -131,11 +132,11 @@ def apply(self, tablist, force=False, applyToAll=False): self.collectErrors() - def applyOnPlotData(self, x, y): + def applyOnPlotData(self, x, y, tabID): x = np.copy(x) y = np.copy(y) for action in self.actionsPlotFilters: - x, y = action.applyOnPlotData(x, y) + x, y = action.applyOnPlotData(x, y, tabID) return x, y diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 0bb8996..94d2b34 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -15,7 +15,7 @@ class PlotData(): def __init__(PD, x=None, y=None, sx='', sy=''): """ Dummy init for now """ PD.id=-1 - PD.it=-1 # tablx index + PD.it=-1 # table index PD.ix=-1 # column index PD.iy=-1 # column index PD.sx='' # x label @@ -24,6 +24,7 @@ def __init__(PD, x=None, y=None, sx='', sy=''): PD.syl='' # y label for legend PD.filename = '' PD.tabname = '' + PD.tabID = -1 PD.x =[] # x data PD.y =[] # y data PD.xIsString=False # true if strings @@ -46,6 +47,7 @@ def fromIDs(PD, tabs, i, idx, SameCol, pipeline=None): PD.st = idx[5] # table label PD.filename = tabs[PD.it].filename PD.tabname = tabs[PD.it].active_name + PD.tabID = -1 # TODO PD.SameCol = SameCol PD.x, PD.xIsString, PD.xIsDate,_ = tabs[PD.it].getColumn(PD.ix) # actual x data, with info PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info @@ -73,7 +75,7 @@ def _post_init(PD, pipeline=None): if pipeline is not None: print('[PDat]', pipeline.__reprFilters__()) if pipeline is not None: - PD.x, PD.y = pipeline.applyOnPlotData(PD.x, PD.y) + PD.x, PD.y = pipeline.applyOnPlotData(PD.x, PD.y, PD.tabID) # TODO pass the tabID # --- Store stats diff --git a/pydatview/plugins/data_binning.py b/pydatview/plugins/data_binning.py index ab14759..4fbda6c 100644 --- a/pydatview/plugins/data_binning.py +++ b/pydatview/plugins/data_binning.py @@ -302,7 +302,7 @@ def onAdd(self,event=None): errors.append(tab.active_name) self.parent.addTables(dfs_new, names_new, bAdd=True) else: - tab = tabList.get(iSel-1) + tab = tabList[iSel-1] df_new, name_new = bin_tab(tab, icol, colname, self.data, bAdd=True) if df_new is not None: self.parent.addTables([df_new], [name_new], bAdd=True) diff --git a/pydatview/plugins/data_filter.py b/pydatview/plugins/data_filter.py index 72abebe..f65c03d 100644 --- a/pydatview/plugins/data_filter.py +++ b/pydatview/plugins/data_filter.py @@ -287,7 +287,7 @@ def onAdd(self,event=None): dfs, names, errors = tabList.applyFiltering(icol, self.data, bAdd=True) self.parent.addTables(dfs,names,bAdd=True) else: - df, name = tabList.get(iSel-1).applyFiltering(icol, self.data, bAdd=True) + df, name = tabList[iSel-1].applyFiltering(icol, self.data, bAdd=True) self.parent.addTables([df], [name], bAdd=True) self.updateTabList() diff --git a/pydatview/plugins/data_sampler.py b/pydatview/plugins/data_sampler.py index db01a74..178e4e5 100644 --- a/pydatview/plugins/data_sampler.py +++ b/pydatview/plugins/data_sampler.py @@ -260,7 +260,7 @@ def onAdd(self,event=None): dfs, names, errors = tabList.applyResampling(icol, self.data, bAdd=True) self.parent.addTables(dfs,names,bAdd=True) else: - df, name = tabList.get(iSel-1).applyResampling(icol, self.data, bAdd=True) + df, name = tabList[iSel-1].applyResampling(icol, self.data, bAdd=True) self.parent.addTables([df],[name], bAdd=True) self.updateTabList() From 6bce70205b6ac7d526fd4ba41f18e1dbfe9afd33 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 11:26:52 +0100 Subject: [PATCH 068/178] AppData: sanitize json outputs --- pydatview/appdata.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pydatview/appdata.py b/pydatview/appdata.py index 3b6a8e3..71b9a44 100644 --- a/pydatview/appdata.py +++ b/pydatview/appdata.py @@ -1,4 +1,6 @@ import json +import numpy as np +import pandas as pd import os from pydatview.io import defaultUserDataDir @@ -85,6 +87,9 @@ def saveAppData(mainFrame, data): if hasattr(mainFrame, 'pipeline'): mainFrame.pipeline.saveData(data['pipeline']) + # --- Sanitize data + data = _sanitize(data) + # --- Write config file configFile = configFilePath() #print('>>> Writing configFile', configFile) @@ -98,6 +103,19 @@ def saveAppData(mainFrame, data): except: pass +def _sanitize(data): + """ + Replace numpy arrays with list + TODO: remove any callbacks/lambda + """ + # --- Level 1 + for k1,v1 in data.items(): + if type(v1) is dict: + data[k1] = _sanitize(v1) + elif isinstance(v1, (pd.core.series.Series,np.ndarray)): + data[k1]=list(v1) + return data + def defaultAppData(mainframe): data={} # --- Main frame data From 61a8d80ab600378bc6b01271296ed330525d3059 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 16:29:05 +0100 Subject: [PATCH 069/178] Pipeline: mask as a plugin/action --- pydatview/GUIPipelinePanel.py | 22 +- pydatview/GUITools.py | 150 ------------- pydatview/main.py | 15 +- pydatview/pipeline.py | 165 ++++++++++----- pydatview/plugins/__init__.py | 4 + pydatview/plugins/data_binning.py | 9 +- pydatview/plugins/data_filter.py | 4 +- pydatview/plugins/data_mask.py | 231 +++++++++++++++++++++ pydatview/plugins/data_removeOutliers.py | 2 +- pydatview/plugins/data_sampler.py | 2 +- pydatview/plugins/data_standardizeUnits.py | 29 ++- 11 files changed, 388 insertions(+), 245 deletions(-) create mode 100644 pydatview/plugins/data_mask.py diff --git a/pydatview/GUIPipelinePanel.py b/pydatview/GUIPipelinePanel.py index f006fa2..ecbadb0 100644 --- a/pydatview/GUIPipelinePanel.py +++ b/pydatview/GUIPipelinePanel.py @@ -168,27 +168,29 @@ def apply(self, tablist, force=False, applyToAll=False): self.ep.update() self.Sizer.Layout() - def append(self, action, cancelIfPresent=False): - if not cancelIfPresent: + def append(self, action, overwrite=True, apply=True, updateGUI=True, tabList=None): + if not overwrite: # Delete action is already present and if it's a "unique" action ac = self.pipeline.find(action.name) if ac is not None: if ac.unique: print('>>> Deleting unique action before inserting it again', ac.name) - self.remove(ac, silent=True) + self.remove(ac, silent=True, updateGUI=False) # Add to pipeline - print('>>> Adding action',action.name) - self.pipeline.append(action, cancelIfPresent=cancelIfPresent) + print('>>> GUIPipeline: Adding action',action.name) + self.pipeline.append(action, overwrite=overwrite, apply=apply, updateGUI=updateGUI, tabList=tabList) # Add to GUI self.populate() # NOTE: we populate because of the change of order between actionsData and actionsPlot.. #self._addPanel(action) #self.Sizer.Layout() + # Update list of errors + self.ep.update() - def remove(self, action, silent=False): + def remove(self, action, silent=False, cancel=True, updateGUI=True, tabList=None): """ NOTE: the action is only removed from the pipeline, not deleted. """ - print('>>> Deleting action',action.name) + print('>>> Deleting action', action.name) # Remove From Data - self.pipeline.remove(action) + self.pipeline.remove(action, cancel=cancel, updateGUI=updateGUI, tabList=tabList) # Remove From GUI self._deletePanel(action) @@ -196,10 +198,6 @@ def remove(self, action, silent=False): if not silent: Info(self.parent, 'A reload is required now that the action "{}" has been removed.'.format(action.name)) - # trigger GUI update (guiCallback) - action.updateGUI() - - # Update list of errors self.ep.update() diff --git a/pydatview/GUITools.py b/pydatview/GUITools.py index af7543e..3a0ad6c 100644 --- a/pydatview/GUITools.py +++ b/pydatview/GUITools.py @@ -94,155 +94,6 @@ def onCompute(self,event=None): #self.parent.load_and_draw(); # DATA HAS CHANGED -# --------------------------------------------------------------------------------} -# --- Mask -# --------------------------------------------------------------------------------{ -class MaskToolPanel(GUIToolPanel): - def __init__(self, parent): - super(MaskToolPanel,self).__init__(parent) - - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - - allMask = tabList.commonMaskString - if len(allMask)==0: - allMask=self.guessMask(tabList) # no known mask, we guess one to help the user - self.applied=False - else: - self.applied=True - - btClose = self.getBtBitmap(self, 'Close','close', self.destroy) - btComp = self.getBtBitmap(self, u'Mask (add)','add' , self.onApply) - if self.applied: - self.btCompMask = self.getToggleBtBitmap(self, 'Clear','sun', self.onToggleApplyMask) - self.btCompMask.SetValue(True) - else: - self.btCompMask = self.getToggleBtBitmap(self, 'Mask','cloud', self.onToggleApplyMask) - - self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'")""") - self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) - self.cbTabs.SetSelection(0) - - self.textMask = wx.TextCtrl(self, wx.ID_ANY, allMask, style = wx.TE_PROCESS_ENTER) - #self.textMask.SetValue('({Time}>100) & ({Time}<400)') - #self.textMask.SetValue("{Date} > '2018-10-01'") - - btSizer = wx.FlexGridSizer(rows=2, cols=2, hgap=2, vgap=0) - btSizer.Add(btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(wx.StaticText(self, -1, '') ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btComp ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btCompMask ,0,flag = wx.ALL|wx.EXPAND, border = 1) - - row_sizer = wx.BoxSizer(wx.HORIZONTAL) - row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) - row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) - row_sizer.Add(wx.StaticText(self, -1, 'Mask:'), 0, wx.CENTER|wx.LEFT, 5) - row_sizer.Add(self.textMask, 1, wx.CENTER|wx.LEFT|wx.EXPAND, 5) - - vert_sizer = wx.BoxSizer(wx.VERTICAL) - vert_sizer.Add(self.lb ,0, flag = wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) - vert_sizer.Add(row_sizer ,1, flag = wx.EXPAND|wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) - self.sizer.Add(vert_sizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) - self.SetSizer(self.sizer) - # Bindings - # NOTE: getBtBitmap and getToggleBtBitmap already specify the binding - self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) - self.textMask.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeAndPressEnter) - - def onTabChange(self,event=None): - tabList = self.parent.selPanel.tabList - iSel=self.cbTabs.GetSelection() - if iSel==0: - maskString = tabList.commonMaskString # for "all" - else: - maskString= tabList[iSel-1].maskString # -1, because "0" is for "all" - if len(maskString)>0: - self.textMask.SetValue(maskString) - #else: - # self.textMask.SetValue('') # no known mask - # self.textMask.SetValue(self.guessMask) # no known mask - - def guessMask(self,tabList): - cols=tabList[0].columns_clean - if 'Time' in cols: - return '{Time} > 100' - elif 'Date' in cols: - return "{Date} > '2017-01-01" - else: - if len(cols)>1: - return '{'+cols[1]+'}>0' - else: - return '' - - def onClear(self,event=None): - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - if iSel==0: - tabList.clearCommonMask() - else: - tabList[iSel-1].clearMask() - - self.parent.load_and_draw() - self.onTabChange() - - def onParamChangeAndPressEnter(self, event=None): - # We apply - if self.applied: - self.onApply(self,bAdd=False) - else: - self.onToggleApplyMask(self) - - def onToggleApplyMask(self,event=None): - self.applied = not self.applied - if self.applied: - self.btCompMask.SetLabel(CHAR['sun']+' Clear') - else: - self.btCompMask.SetLabel(CHAR['cloud']+' Mask') - - if self.applied: - self.onApply(event,bAdd=False) - else: - self.onClear() - - def onApply(self,event=None,bAdd=True): - maskString = self.textMask.GetLineText(0) - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - if iSel==0: - dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=bAdd) - if bAdd: - self.parent.addTables(dfs,names,bAdd=bAdd) - else: - self.parent.load_and_draw() - if len(errors)>0: - raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) - else: - dfs, name = tabList[iSel-1].applyMaskString(maskString, bAdd=bAdd) - if bAdd: - self.parent.addTables([dfs],[name], bAdd=bAdd) - else: - self.parent.load_and_draw() - self.updateTabList() - - # We stop applying - if bAdd: - self.onToggleApplyMask() - - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - # --------------------------------------------------------------------------------} # --- Radial # --------------------------------------------------------------------------------{ @@ -576,7 +427,6 @@ def onHelp(self,event=None): TOOLS={ 'LogDec': LogDecToolPanel, - 'Mask': MaskToolPanel, 'FASTRadialAverage': RadialToolPanel, 'CurveFitting': CurveFitToolPanel, } diff --git a/pydatview/main.py b/pydatview/main.py index 44a21c0..c4b753a 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -144,7 +144,6 @@ def __init__(self, data=None): dataMenu = wx.Menu() menuBar.Append(dataMenu, "&Data") - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e, 'Mask') , dataMenu.Append(wx.ID_ANY, 'Mask')) self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'FASTRadialAverage'), dataMenu.Append(wx.ID_ANY, 'FAST - Radial average')) # --- Data Plugins @@ -397,6 +396,7 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): #self.onShowTool(tool='Filter') #self.onShowTool(tool='Resample') #self.onDataPlugin(toolName='Bin data') + self.onDataPlugin(toolName='Mask') def setStatusBar(self, ISel=None): nTabs=self.tabList.len() @@ -487,18 +487,17 @@ def onDataPlugin(self, event=None, toolName=''): else: action = function(label=toolName, mainframe=self) # calling the data function # Here we apply the action directly - action.apply(self.tabList) # the action will chose that to apply it on - self.addAction(action) - action.updateGUI() + # We can't overwrite, so we'll delete by name.. + self.addAction(action, overwrite=False, apply=True, tabList=self.tabList, updateGUI=True) return raise NotImplementedError('Tool: ',toolName) # --- Pipeline - def addAction(self, action, cancelIfPresent=False): - self.pipePanel.append(action, cancelIfPresent=cancelIfPresent) - def removeAction(self, action): - self.pipePanel.remove(action) + def addAction(self, action, **kwargs): + self.pipePanel.append(action, **kwargs) + def removeAction(self, action, **kwargs): + self.pipePanel.remove(action, **kwargs) def applyPipeline(self, *args, **kwargs): self.pipePanel.apply(*args, **kwargs) diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index 1f7d1f6..3d0ff8c 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -4,8 +4,10 @@ import numpy as np class Action(): + # TODO: store data per table and for all def __init__(self, name, - tableFunction = None, + tableFunctionApply = None, + tableFunctionCancel = None, plotDataFunction = None, guiCallback=None, guiEditorClass=None, @@ -21,48 +23,53 @@ def __init__(self, name, self.name = name # - self.tableFunction = tableFunction # applies to a full table - self.plotDataFunction = plotDataFunction # applies to x,y arrays only + self.tableFunctionApply = tableFunctionApply # applies to a full table + self.tableFunctionCancel = tableFunctionCancel # cancel action on a full table + self.plotDataFunction = plotDataFunction # applies to x,y arrays only self.guiCallback = guiCallback # callback to update GUI after the action, # TODO remove me, replace with generic "redraw", "update tab list" self.guiEditorClass = guiEditorClass # Class that can be used to edit this action self.guiEditorObj = None # Instance of guiEditorClass that can be used to edit this action - self.data = data if data is not None else {} # Data needed by the action, can be saved to file so that the action can be restored self.mainframe=mainframe # If possible, dont use that... + # TODO this needs to be stored per table + self.data = data if data is not None else {} # Data needed by the action, can be saved to file so that the action can be restored self.applied=False # TODO this needs to be sotred per table # Behavior - self.onPlotData = onPlotData + self.onPlotData = onPlotData # True for plotDataActions... self.unique = unique self.removeNeedReload = removeNeedReload self.errorList=[] - def apply(self, tablist, force=False, applyToAll=False): + def apply(self, tabList, force=False, applyToAll=False): self.errorList=[] - if self.tableFunction is None: + if self.tableFunctionApply is None: # NOTE: this does not applyt to plotdataActions.. raise Exception('tableFunction was not specified for action: {}'.format(self.name)) - for t in tablist: + if tabList is None: + raise Exception('{}: cannot apply on None tabList'.format(self)) + + for t in tabList: print('>>> Applying action', self.name, 'to', t.name) try: - self.tableFunction(t) - except e: + self.tableFunctionApply(t, data=self.data) + except: err = 'Failed to apply action {} to table {}.'.format(self.name, t.name) - self.errorList.append(e) + self.errorList.append(err) self.applied = True - return tablist + return tabList def updateGUI(self): - """ Typically called by a calleed after append""" + """ Typically called by a callee after append""" if self.guiCallback is not None: - print('>>> Calling GUI callback, action', self.name) + print('>>> Action: Calling GUI callback, action', self.name) self.guiCallback() def __repr__(self): @@ -70,41 +77,84 @@ def __repr__(self): return s +# --------------------------------------------------------------------------------} +# --- Plot data actions (applied on the fly) +# --------------------------------------------------------------------------------{ +# TODO: handle how they generate new tables +# TODO: handle how they are applied to few tables class PlotDataAction(Action): def __init__(self, name, **kwargs): Action.__init__(self, name, onPlotData=True, **kwargs) def apply(self, *args, **kwargs): - pass # not pretty + pass # nothing to do + + def cancel(self, *args, **kwargs): + pass # nothing to do def applyOnPlotData(self, x, y, tabID): # TODO apply only based on tabID x, y = self.plotDataFunction(x, y, self.data) return x, y -class IrreversibleAction(Action): + def __repr__(self): + s=''.format(self.name, self.applied) + return s +# --------------------------------------------------------------------------------} +# --- Table actions (apply on a full table) +# --------------------------------------------------------------------------------{ +# TODO: store data per table and for all +class IrreversibleTableAction(Action): def __init__(self, name, **kwargs): Action.__init__(self, name, removeNeedReload=True, **kwargs) - def apply(self, tablist, force=False, applyToAll=False): + def apply(self, tabList, force=False, applyToAll=False): if force: self.applied = False if self.applied: - print('>>> Skipping irreversible action', self.name) + print('>>> Action: Skipping irreversible action', self.name) return - Action.apply(self, tablist) + Action.apply(self, tabList) + + def cancel(self, *args, **kwargs): + print('>>> Action: Cancel: skipping irreversible action', self.name) + pass def __repr__(self): - s=''.format(self.name, self.applied) + s=''.format(self.name, self.applied) return s -class FilterAction(Action): - def cancel(self, tablist): - raise NotImplementedError() - return tablist +class ReversibleTableAction(Action): + + def __init__(self, name, tableFunctionApply, tableFunctionCancel, **kwargs): + Action.__init__(self, name, tableFunctionApply=tableFunctionApply, tableFunctionCancel=tableFunctionCancel, **kwargs) + + def apply(self, tabList, force=False, applyToAll=False): + if force: + self.applied = False + if self.applied: + print('>>> Action: Apply: Skipping irreversible action', self.name) + return + Action.apply(self, tabList) + + def cancel(self, tabList): + self.errorList=[] + for t in tabList: + print('>>> Action: Cancel: ', self, 'to', t.name) + try: + self.tableFunctionCancel(t, data=self.data) + except: + self.errorList.append('Failed to apply action {} to table {}.'.format(self.name, t.name)) + + def __repr__(self): + s=''.format(self.name, self.applied) + return s +# --------------------------------------------------------------------------------} +# --- Pipeline +# --------------------------------------------------------------------------------{ class Pipeline(object): def __init__(self, data=[]): @@ -117,17 +167,17 @@ def __init__(self, data=[]): def actions(self): return self.actionsData+self.actionsPlotFilters # order matters - def apply(self, tablist, force=False, applyToAll=False): + def apply(self, tabList, force=False, applyToAll=False): """ - Apply the pipeline to the tablist + Apply the pipeline to the tabList If "force", then actions that are "one time only" are still applied - If applyToAll, then the action is applied to all the tables, irrespectively of the tablist stored by the action + If applyToAll, then the action is applied to all the tables, irrespectively of the tabList stored by the action """ for action in self.actionsData: - action.apply(tablist, force=force, applyToAll=applyToAll) + action.apply(tabList, force=force, applyToAll=applyToAll) # for action in self.actionsPlotFilters: - action.apply(tablist, force=force, applyToAll=applyToAll) + action.apply(tabList, force=force, applyToAll=applyToAll) # self.collectErrors() @@ -139,50 +189,55 @@ def applyOnPlotData(self, x, y, tabID): x, y = action.applyOnPlotData(x, y, tabID) return x, y - - def collectErrors(self): self.errorList=[] for action in self.actions: self.errorList+= action.errorList -# def setPlotFiltersData(self): -# print('>>> Setting plotFiltersData') -# self.plotFiltersData=[] -# for action in self.actionsPlotFilters: -# self.plotFiltersData.append(action.data) -# print(action.data) - # --- Behave like a list.. - def append(self, action, cancelIfPresent=False): - if cancelIfPresent: - i = self.index(action) - if i>=0: - print('>>> Not adding action, its already present') - return - if action.onPlotData: - self.actionsPlotFilters.append(action) - # Trigger -# self.setPlotFiltersData() + def append(self, action, overwrite=False, apply=True, updateGUI=True, tabList=None): + i = self.index(action) + if i>=0 and not overwrite: + print('[Pipe] Not adding action, its already present') else: - self.actionsData.append(action) + if action.onPlotData: + self.actionsPlotFilters.append(action) + else: + self.actionsData.append(action) + + if apply: + action.apply(tabList=tabList, force=True) + self.collectErrors() + + # trigger GUI update (guiCallback) + if updateGUI: + action.updateGUI() - def remove(self, a): - """ NOTE: the action is removed, not deleted fully (it might be readded to the pipeline later)""" + def remove(self, a, cancel=True, updateGUI=True, tabList=None): + """ NOTE: the action is removed, not deleted fully (it might be readded to the pipeline later) + - If a GUI edtor is attached to this action, we make sure that it shows the action as cancelled + """ try: i = self.actionsData.index(a) a = self.actionsData.pop(i) except ValueError: i = self.actionsPlotFilters.index(a) a = self.actionsPlotFilters.pop(i) - # Trigger -# self.setPlotFiltersData() + + # Cancel the action + if cancel: + a.cancel(tabList) + self.collectErrors() # Cancel the action in Editor if a.guiEditorObj is not None: - print('>>> Canceling action in guiEditor') - a.guiEditorObj.cancelAction() + print('[Pipe] Canceling action in guiEditor because the action is removed') + a.guiEditorObj.cancelAction() # NOTE: should not trigger a plot + + # trigger GUI update (guiCallback) + if updateGUI: + a.updateGUI() return a diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 852666d..3dc0f0d 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -13,6 +13,9 @@ def _function_name(mainframe, event=None, label='') See working examples in this file and this directory. """ +def _data_mask(label, mainframe): + from .data_mask import maskAction + return maskAction(label, mainframe) def _data_filter(label, mainframe): from .data_filter import filterAction @@ -41,6 +44,7 @@ def _data_standardizeUnitsWE(label, mainframe=None): dataPlugins=[ # Name/label , callback , is a Panel + ('Mask' , _data_mask , True ), ('Remove Outliers' , _data_removeOutliers , True ), ('Filter' , _data_filter , True ), ('Resample' , _data_sampler , True ), diff --git a/pydatview/plugins/data_binning.py b/pydatview/plugins/data_binning.py index 4fbda6c..83f7c0c 100644 --- a/pydatview/plugins/data_binning.py +++ b/pydatview/plugins/data_binning.py @@ -95,7 +95,7 @@ def __init__(self, parent, action=None): print('[WARN] Calling GUI without an action! Creating one.') mainframe = DummyMainFrame(parent) action = binningAction(label='dummyAction', mainframe=mainframe) - # --- Data from other modules + # --- Data self.parent = parent # parent is GUIPlotPanel self.mainframe = action.mainframe self.data = action.data @@ -159,7 +159,6 @@ def __init__(self, parent, action=None): self.sizer = wx.BoxSizer(wx.HORIZONTAL) self.sizer.Add(btSizer ,0, flag = wx.LEFT , border = 5) self.sizer.Add(vsizer ,1, flag = wx.LEFT|wx.EXPAND , border = TOOL_BORDER) - #self.sizer.Add(msizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) self.SetSizer(self.sizer) # --- Events @@ -227,8 +226,8 @@ def cancelAction(self, redraw=True): self.btApply.SetLabel(CHAR['cloud']+' Apply') self.btApply.SetValue(False) self.data['active'] = False - if redraw: - self.parent.load_and_draw() # Data will change based on plotData + #if redraw: + # self.parent.load_and_draw() # Data will change based on plotData # --- Fairly generic def _GUI2Data(self): @@ -260,7 +259,7 @@ def onToggleApply(self, event=None, init=False): self.btApply.SetValue(True) # The action is now active we add it to the pipeline, unless it's already in it if self.mainframe is not None: - self.mainframe.addAction(self.action, cancelIfPresent=True) + self.mainframe.addAction(self.action, overwrite=True) if not init: self.parent.load_and_draw() # filter will be applied in plotData.py else: diff --git a/pydatview/plugins/data_filter.py b/pydatview/plugins/data_filter.py index f65c03d..3fac83f 100644 --- a/pydatview/plugins/data_filter.py +++ b/pydatview/plugins/data_filter.py @@ -267,7 +267,7 @@ def onToggleApply(self, event=None, init=False): self.btApply.SetValue(True) # The action is now active we add it to the pipeline, unless it's already in it if self.mainframe is not None: - self.mainframe.addAction(self.action, cancelIfPresent=True) + self.mainframe.addAction(self.action, overwrite=True) if not init: self.parent.load_and_draw() # filter will be applied in plotData.py else: @@ -277,7 +277,7 @@ def onToggleApply(self, event=None, init=False): self.mainframe.removeAction(self.action) self.cancelAction(redraw=not init) - def onAdd(self,event=None): + def onAdd(self, event=None): iSel = self.cbTabs.GetSelection() tabList = self.parent.selPanel.tabList icol, colname = self.parent.selPanel.xCol diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py new file mode 100644 index 0000000..a13fc0f --- /dev/null +++ b/pydatview/plugins/data_mask.py @@ -0,0 +1,231 @@ +import wx +import numpy as np +from pydatview.GUITools import GUIToolPanel, TOOL_BORDER +from pydatview.common import CHAR, Error, Info, pretty_num_short +from pydatview.common import DummyMainFrame +from pydatview.pipeline import ReversibleTableAction +# --------------------------------------------------------------------------------} +# --- Data +# --------------------------------------------------------------------------------{ +_DEFAULT_DICT={ + 'active':False, + 'maskString': '' + # 'nBins':50 +} + +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def maskAction(label, mainframe=None, data=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + The action is also edited and created by the GUI Editor + """ + if data is None: + # NOTE: if we don't do copy below, we will end up remembering even after the action was deleted + # its not a bad feature, but we might want to think it through + # One issue is that "active" is kept in memory + data=_DEFAULT_DICT + data['active'] = False #<<< Important + + guiCallback=None + if mainframe is not None: + guiCallback = mainframe.redraw + + action = ReversibleTableAction( + name=label, + tableFunctionApply = applyMask, + tableFunctionCancel = removeMask, + guiEditorClass = MaskToolPanel, + guiCallback = guiCallback, + data = data, + mainframe=mainframe + ) + return action +# --------------------------------------------------------------------------------} +# --- Main method +# --------------------------------------------------------------------------------{ +def applyMask(tab, data): + # dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=False) + dfs, name = tab.applyMaskString(data['maskString'], bAdd=False) + +def removeMask(tab, data): + tab.clearMask() + # tabList.clearCommonMask() + +# --------------------------------------------------------------------------------} +# --- Mask +# --------------------------------------------------------------------------------{ +class MaskToolPanel(GUIToolPanel): + def __init__(self, parent, action): + GUIToolPanel.__init__(self, parent) + + # --- Creating "Fake data" for testing only! + if action is None: + print('[WARN] Calling GUI without an action! Creating one.') + action = maskAction(label='dummyAction', mainframe=DummyMainFrame(parent)) + + # --- Data + self.data = action.data + self.action = action + + # --- Unfortunate data + self.tabList = action.mainframe.tabList # <<< + self.addTablesHandle = action.mainframe.load_dfs # <<< + self.addActionHandle = action.mainframe.addAction # <<< + self.removeActionHandle = action.mainframe.removeAction + self.redrawHandle = action.mainframe.redraw # or GUIPanel.load_and_draw() + + # --- GUI elements + tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() + + self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) + self.btAdd = self.getBtBitmap(self, u'Mask (add)','add' , self.onAdd) + self.btApply = self.getToggleBtBitmap(self, 'Clear','sun', self.onToggleApply) + + self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'")""") + self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) + self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better + self.cbTabs.SetSelection(0) + + self.textMask = wx.TextCtrl(self, wx.ID_ANY, 'Dummy', style = wx.TE_PROCESS_ENTER) + #self.textMask.SetValue('({Time}>100) & ({Time}<400)') + #self.textMask.SetValue("{Date} > '2018-10-01'") + + btSizer = wx.FlexGridSizer(rows=2, cols=2, hgap=2, vgap=0) + btSizer.Add(self.btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(wx.StaticText(self, -1, '') ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btAdd ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btApply ,0,flag = wx.ALL|wx.EXPAND, border = 1) + + row_sizer = wx.BoxSizer(wx.HORIZONTAL) + row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) + row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Mask:'), 0, wx.CENTER|wx.LEFT, 5) + row_sizer.Add(self.textMask, 1, wx.CENTER|wx.LEFT|wx.EXPAND, 5) + + vert_sizer = wx.BoxSizer(wx.VERTICAL) + vert_sizer.Add(self.lb ,0, flag = wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) + vert_sizer.Add(row_sizer ,1, flag = wx.EXPAND|wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + self.sizer.Add(vert_sizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) + self.SetSizer(self.sizer) + + # --- Events + # NOTE: getBtBitmap and getToggleBtBitmap already specify the binding + self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) + self.textMask.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeAndPressEnter) + + # --- Init triggers + self._Data2GUI() + + # --- Implementation specific + def guessMask(self): + cols=self.tabList[0].columns_clean + if 'Time' in cols: + return '{Time} > 10' + elif 'Date' in cols: + return "{Date} > '2017-01-01" + else: + if len(cols)>1: + return '{'+cols[1]+'}>0' + else: + return '' + + # --- Bindings for plot triggers on parameters changes + def onParamChangeAndPressEnter(self, event=None): + # We apply + if self.data['active']: + self.action.apply(self.tabList, force=True) + self.action.updateGUI() # We call the guiCallback + else: + # We assume that "enter" means apply + self.onToggleApply() + + # --- Table related + def onTabChange(self,event=None): + iSel=self.cbTabs.GetSelection() + # TODO need a way to retrieve "data" from action, perTab + if iSel==0: + maskString = self.tabList.commonMaskString # for "all" + else: + maskString = self.tabList[iSel-1].maskString # -1, because "0" is for "all" + if len(maskString)>0: + self.textMask.SetValue(maskString) + + def updateTabList(self,event=None): + tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() + #try: + iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + #except RuntimeError: + # pass + + # --- External Calls + def cancelAction(self, redraw=True): + """ do cancel the action""" + self.btApply.SetLabel(CHAR['cloud']+' Mask') + self.btApply.SetValue(False) + self.data['active'] = False + + def guiApplyAction(self, redraw=True): + self.btApply.SetLabel(CHAR['sun']+' Clear') + self.btApply.SetValue(True) + self.data['active'] = True + + # --- Fairly generic + def _GUI2Data(self): + self.data['maskString'] = self.textMask.GetLineText(0) + #iSel = self.cbTabs.GetSelection() + #tabList = self.parent.selPanel.tabList + + def _Data2GUI(self): + if len(self.data['maskString'])==0: + self.data['maskString'] = self.guessMask() # no known mask, we guess one to help the user + self.textMask.SetValue(self.data['maskString']) + + def onToggleApply(self, event=None, init=False): + self.data['active'] = not self.data['active'] + print('') + print('') + print('') + if self.data['active']: + self._GUI2Data() + # We update the GUI + self.guiApplyAction() + # Add action to pipeline, apply it, update the GUI + self.addActionHandle(self.action, overwrite=True, apply=True, tabList=self.tabList, updateGUI=True) + #if iSel==0: + # dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=False) + # if len(errors)>0: + # raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) + #else: + # dfs, name = tabList[iSel-1].applyMaskString(maskString, bAdd=False) + else: + if not init: + # Remove action from pipeline, cancel it, update the GUI + self.removeActionHandle(self.action, cancel=True, tabList=self.tabList, updateGUI=True) + + def onAdd(self, event=None): + self._GUI2Data() + iSel = self.cbTabs.GetSelection() + # TODO this should be handled by the action + dfs, names, errors = self.tabList.applyCommonMaskString(self.data['maskString'], bAdd=True) + if len(errors)>0: + raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) + + # We stop applying if we were applying it: + if self.data['active']: + self.onToggleApply() + + self.addTablesHandle(dfs, names, bAdd=True, bPlot=False) # Triggers a redraw of the whole panel... + #if iSel==0: + #else: + # dfs, name = tabList[iSel-1].applyMaskString(self.data['maskString'], bAdd=True) + # self.parent.addTables([dfs],[name], bAdd=True) + #self.updateTabList() + diff --git a/pydatview/plugins/data_removeOutliers.py b/pydatview/plugins/data_removeOutliers.py index 8160116..779d430 100644 --- a/pydatview/plugins/data_removeOutliers.py +++ b/pydatview/plugins/data_removeOutliers.py @@ -161,7 +161,7 @@ def onToggleApply(self, event=None, init=False): self.btApply.SetValue(True) # The action is now active we add it to the pipeline, unless it's already in it if self.mainframe is not None: - self.mainframe.addAction(self.action, cancelIfPresent=True) + self.mainframe.addAction(self.action, overwrite=True) if not init: self.parent.load_and_draw() # filter will be applied in plotData.py else: diff --git a/pydatview/plugins/data_sampler.py b/pydatview/plugins/data_sampler.py index 178e4e5..f281ba4 100644 --- a/pydatview/plugins/data_sampler.py +++ b/pydatview/plugins/data_sampler.py @@ -238,7 +238,7 @@ def onToggleApply(self, event=None, init=False): self.btApply.SetValue(True) # The action is now active we add it to the pipeline, unless it's already in it if self.mainframe is not None: - self.mainframe.addAction(self.action, cancelIfPresent=True) + self.mainframe.addAction(self.action, overwrite=True) if not init: self.parent.load_and_draw() # filter will be applied in plotData.py else: diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py index c014460..3edadb4 100644 --- a/pydatview/plugins/data_standardizeUnits.py +++ b/pydatview/plugins/data_standardizeUnits.py @@ -1,12 +1,17 @@ import unittest import numpy as np from pydatview.common import splitunit -from pydatview.pipeline import Action, IrreversibleAction +from pydatview.pipeline import IrreversibleTableAction +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ def standardizeUnitsAction(label, mainframe=None, flavor='SI'): """ - Main entry point of the plugin + Return an "action" for the current plugin, to be used in the pipeline. """ + data = {'flavor':flavor} + guiCallback=None if mainframe is not None: # TODO TODO TODO Clean this up @@ -18,35 +23,37 @@ def guiCallback(): mainframe.onTabSelectionChange() # trigger replot if hasattr(mainframe,'pipePanel'): pass - # Function that will be applied to all tables - tableFunction = lambda t: changeUnits(t, flavor=flavor) - action = IrreversibleAction( + action = IrreversibleTableAction( name=label, - tableFunction=tableFunction, + tableFunctionApply=changeUnits, guiCallback=guiCallback, mainframe=mainframe, # shouldnt be needed + data = data ) return action -def changeUnits(tab, flavor='SI'): +# --------------------------------------------------------------------------------} +# --- Main method +# --------------------------------------------------------------------------------{ +def changeUnits(tab, data): """ Change units of a table NOTE: it relies on the Table class, which may change interface in the future.. """ - if flavor=='WE': + if data['flavor']=='WE': for i, colname in enumerate(tab.columns): colname, tab.data.iloc[:,i] = change_units_to_WE(colname, tab.data.iloc[:,i]) tab.columns[i] = colname # TODO, use a dataframe everywhere.. tab.data.columns = tab.columns - elif flavor=='SI': + elif data['flavor']=='SI': for i, colname in enumerate(tab.columns): colname, tab.data.iloc[:,i] = change_units_to_SI(colname, tab.data.iloc[:,i]) tab.columns[i] = colname # TODO, use a dataframe everywhere.. tab.data.columns = tab.columns else: - raise NotImplementedError(flavor) + raise NotImplementedError(data['flavor']) def change_units_to_WE(s, c): @@ -109,7 +116,7 @@ def test_change_units(self): data[:,2] *= 10*np.pi/180 # rad df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) tab=Table(data=df) - changeUnits(tab, flavor='WE') + changeUnits(tab, {'flavor':'WE'}) np.testing.assert_almost_equal(tab.data.values[:,0],[1]) np.testing.assert_almost_equal(tab.data.values[:,1],[2]) np.testing.assert_almost_equal(tab.data.values[:,2],[10]) From 6ac532368be15e65fa32363d5e886ac079e398ad Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 22 Dec 2022 21:21:53 +0100 Subject: [PATCH 070/178] Pipeline: using a parent class for plotdata plugins --- pydatview/GUIPipelinePanel.py | 56 +- pydatview/GUIPlotPanel.py | 8 +- pydatview/appdata.py | 4 +- pydatview/common.py | 3 +- pydatview/main.py | 12 +- pydatview/pipeline.py | 30 + pydatview/plugins/__init__.py | 10 +- pydatview/plugins/data_mask.py | 70 +- .../{data_binning.py => plotdata_binning.py} | 620 +++++++----------- pydatview/plugins/plotdata_default_plugin.py | 281 ++++++++ .../{data_filter.py => plotdata_filter.py} | 238 ++----- ...Outliers.py => plotdata_removeOutliers.py} | 153 +++-- .../{data_sampler.py => plotdata_sampler.py} | 179 ++--- 13 files changed, 812 insertions(+), 852 deletions(-) rename pydatview/plugins/{data_binning.py => plotdata_binning.py} (50%) create mode 100644 pydatview/plugins/plotdata_default_plugin.py rename pydatview/plugins/{data_filter.py => plotdata_filter.py} (52%) rename pydatview/plugins/{data_removeOutliers.py => plotdata_removeOutliers.py} (51%) rename pydatview/plugins/{data_sampler.py => plotdata_sampler.py} (55%) diff --git a/pydatview/GUIPipelinePanel.py b/pydatview/GUIPipelinePanel.py index ecbadb0..a08cd22 100644 --- a/pydatview/GUIPipelinePanel.py +++ b/pydatview/GUIPipelinePanel.py @@ -1,5 +1,6 @@ import wx +from pydatview.pipeline import Pipeline from pydatview.common import CHAR, Info import wx.lib.agw.hyperlink as hl @@ -60,7 +61,6 @@ def __init__(self, parent, pipeline, style=wx.TAB_TRAVERSAL): wx.Panel.__init__(self, parent, -1, style=style) #self.SetBackgroundColour((100,0,0)) # --- Data - self.parent=parent self.pipeline=pipeline # --- GUI lke = hl.HyperLinkCtrl(self, -1, 'Errors (0)') @@ -87,25 +87,29 @@ def showErrors(self, event=None): message='\n'.join(self.pipeline.errorList) else: message='No errors' - Info(self.parent, message, caption = 'Errors when applying the pipeline actions:') + Info(self, message, caption = 'Errors when applying the pipeline actions:') def update(self): self.lke.SetLabel('Errors ({})'.format(len(self.pipeline.errorList))) self.sizer.Layout() -class PipelinePanel(wx.Panel): +# --------------------------------------------------------------------------------} +# --- PipelinePanel +# --------------------------------------------------------------------------------{ +class PipelinePanel(wx.Panel, Pipeline): """ Display the pipeline of actions, allow user to edit it """ - def __init__(self, parent, pipeline, style=wx.TAB_TRAVERSAL): - #style=wx.RAISED_BORDER + def __init__(self, parent, data=None, tabList=None, style=wx.TAB_TRAVERSAL): + # Init parent classes wx.Panel.__init__(self, parent, -1, style=style) + Pipeline.__init__(self, data=data) #self.SetBackgroundColour(wx.BLUE) - # --- Data - self.parent = parent - self.pipeline = pipeline + # --- Important GUI Data + self.tabList = tabList self.actionPanels=[] + self.ep = ErrorPanel(self, pipeline=self) # --- GUI #self.btSave = wx.Button(self, wx.ID_ANY, 'Save', style=wx.BU_EXACTFIT) @@ -120,7 +124,6 @@ def __init__(self, parent, pipeline, style=wx.TAB_TRAVERSAL): self.wrapSizer = wx.WrapSizer(orient=wx.HORIZONTAL) - self.ep = ErrorPanel(self, self.pipeline) self.Sizer = wx.BoxSizer(wx.HORIZONTAL) self.Sizer.Add(leftSizer , 0, wx.ALIGN_CENTER_VERTICAL|wx.LEFT , 1) @@ -137,7 +140,7 @@ def populate(self): self.wrapSizer.Add(wx.StaticText(self, -1, ' '), wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, 0) # Add all actions to panel - for ia, action in enumerate(self.pipeline.actions): + for ia, action in enumerate(self.actions): self._addPanel(action) self.ep.update() self.wrapSizer.Layout() @@ -159,26 +162,28 @@ def _deletePanel(self, action): self.Sizer.Layout() def onCloseAction(self, event, action=None, actionPanel=None): - self.remove(action) + self.remove(action, tabList=self.tabList) # TODO # --- Wrap the data class def apply(self, tablist, force=False, applyToAll=False): - self.pipeline.apply(tablist, force=force, applyToAll=applyToAll) - + # Call parent class (data) + Pipeline.apply(self, tablist, force=force, applyToAll=applyToAll) + # Update GUI self.ep.update() self.Sizer.Layout() def append(self, action, overwrite=True, apply=True, updateGUI=True, tabList=None): if not overwrite: # Delete action is already present and if it's a "unique" action - ac = self.pipeline.find(action.name) + ac = self.find(action.name) if ac is not None: if ac.unique: print('>>> Deleting unique action before inserting it again', ac.name) self.remove(ac, silent=True, updateGUI=False) # Add to pipeline print('>>> GUIPipeline: Adding action',action.name) - self.pipeline.append(action, overwrite=overwrite, apply=apply, updateGUI=updateGUI, tabList=tabList) + # Call parent class (data) + Pipeline.append(self, action, overwrite=overwrite, apply=apply, updateGUI=updateGUI, tabList=tabList) # Add to GUI self.populate() # NOTE: we populate because of the change of order between actionsData and actionsPlot.. #self._addPanel(action) @@ -189,14 +194,14 @@ def append(self, action, overwrite=True, apply=True, updateGUI=True, tabList=Non def remove(self, action, silent=False, cancel=True, updateGUI=True, tabList=None): """ NOTE: the action is only removed from the pipeline, not deleted. """ print('>>> Deleting action', action.name) - # Remove From Data - self.pipeline.remove(action, cancel=cancel, updateGUI=updateGUI, tabList=tabList) + # Call parent class (data) + Pipeline.remove(self, action, cancel=cancel, updateGUI=updateGUI, tabList=tabList) # Remove From GUI self._deletePanel(action) if action.removeNeedReload: if not silent: - Info(self.parent, 'A reload is required now that the action "{}" has been removed.'.format(action.name)) + Info(self, 'A reload is required now that the action "{}" has been removed.'.format(action.name)) # Update list of errors self.ep.update() @@ -204,20 +209,19 @@ def remove(self, action, silent=False, cancel=True, updateGUI=True, tabList=None if __name__ == '__main__': """ """ - from pydatview.pipeline import Pipeline, Action, IrreversibleAction, PlotDataAction + from pydatview.pipeline import Pipeline, Action, IrreversibleTableAction, PlotDataAction - pl = Pipeline() app = wx.App(False) self=wx.Frame(None,-1,"GUIPipelinePanel main") - p = PipelinePanel(self, pl) - p.append(PlotDataAction('PlotData Do X')) - p.append(PlotDataAction('PlotData Do Y')) - p.append(Action('Change units')) - p.append(IrreversibleAction('Rename columns')) + p = PipelinePanel(self) + p.append(PlotDataAction('PlotData Do X'), apply=False) + p.append(PlotDataAction('PlotData Do Y'), apply=False) + p.append(Action('Change units'), apply=False) + p.append(IrreversibleTableAction('Rename columns'), apply=False) - pl.errorList=['This is a first error','This is a second error'] + p.errorList=['This is a first error','This is a second error'] p.populate() diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 62ff612..60e5d9d 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -376,7 +376,7 @@ def onFontOptionChange(self,event=None): class PlotPanel(wx.Panel): - def __init__(self, parent, selPanel, pipeline=None, infoPanel=None, data=None): + def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): # Superclass constructor super(PlotPanel,self).__init__(parent) @@ -402,7 +402,7 @@ def __init__(self, parent, selPanel, pipeline=None, infoPanel=None, data=None): break # data self.selPanel = selPanel # <<< dependency with selPanel should be minimum - self.pipeline = pipeline # + self.pipeLike = pipeLike # self.selMode = '' self.infoPanel=infoPanel if self.infoPanel is not None: @@ -805,7 +805,7 @@ def showToolPanel(self, panelClass=None, panel=None, action=None): print('NOTE: calling a panel without action') self.toolPanel=panelClass(parent=self) # calling the panel constructor else: - self.toolPanel=panelClass(parent=self, action=action) # calling the panel constructor + self.toolPanel=panelClass(parent=self, action=action, plotPanel=self, pipeLike=self.pipeLike) # calling the panel constructor action.guiEditorObj = self.toolPanel self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) self.plotsizer.Layout() @@ -878,7 +878,7 @@ def getPlotData(self,plotType): for i,idx in enumerate(ID): # Initialize each plotdata based on selected table and selected id channels pd=PlotData(); - pd.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeline) + pd.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeLike) # Possible change of data if plotType=='MinMax': self.setPD_MinMax(pd) diff --git a/pydatview/appdata.py b/pydatview/appdata.py index 71b9a44..03d4339 100644 --- a/pydatview/appdata.py +++ b/pydatview/appdata.py @@ -84,8 +84,8 @@ def saveAppData(mainFrame, data): mainFrame.infoPanel.saveData(data['infoPanel']) if hasattr(mainFrame, 'tablist'): mainFrame.tablist.saveOptions(data['loaderOptions']) - if hasattr(mainFrame, 'pipeline'): - mainFrame.pipeline.saveData(data['pipeline']) + if hasattr(mainFrame, 'pipePanel'): + mainFrame.pipePanel.saveData(data['pipeline']) # --- Sanitize data data = _sanitize(data) diff --git a/pydatview/common.py b/pydatview/common.py index fcd94df..5e7cb6d 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -444,8 +444,9 @@ def isDate(x): # Create a Dummy Main Frame Class for testing purposes (e.g. of plugins) class DummyMainFrame(): - def __init__(self, parent): self.parent=parent + def __init__(self, parent): self.parent=parent; def addAction (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) def removeAction (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) def load_dfs (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) def mainFrameUpdateLayout(self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) + def redraw (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) diff --git a/pydatview/main.py b/pydatview/main.py index c4b753a..f677e76 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -32,7 +32,6 @@ from .GUICommon import * import pydatview.io as weio # File Formats and File Readers # Pluggins -from .pipeline import Pipeline from .plugins import dataPlugins from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData @@ -112,7 +111,6 @@ def __init__(self, data=None): self.systemFontSize = self.GetFont().GetPointSize() self.data = loadAppData(self) self.tabList=TableList(options=self.data['loaderOptions']) - self.pipeline=Pipeline(data = self.data['pipeline']) self.datareset = False # Global variables... setFontSize(self.data['fontSize']) @@ -208,7 +206,7 @@ def __init__(self, data=None): self.statusbar.SetStatusWidths([150, -1, 70]) # --- Pipeline - self.pipePanel = PipelinePanel(self, self.pipeline) + self.pipePanel = PipelinePanel(self, data=self.data['pipeline'], tabList=self.tabList) # --- Main Panel and Notebook self.MainPanel = wx.Panel(self) @@ -354,7 +352,7 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): self.tSplitter = wx.SplitterWindow(self.vSplitter) #self.tSplitter.SetMinimumPaneSize(20) self.infoPanel = InfoPanel(self.tSplitter, data=self.data['infoPanel']) - self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, infoPanel=self.infoPanel, pipeline=self.pipeline, data=self.data['plotPanel']) + self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, infoPanel=self.infoPanel, pipeLike=self.pipePanel, data=self.data['plotPanel']) self.tSplitter.SetSashGravity(0.9) self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) self.tSplitter.SetMinimumPaneSize(BOT_PANL) @@ -395,8 +393,10 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): # Hack #self.onShowTool(tool='Filter') #self.onShowTool(tool='Resample') + #self.onDataPlugin(toolName='Mask') #self.onDataPlugin(toolName='Bin data') - self.onDataPlugin(toolName='Mask') + #self.onDataPlugin(toolName='Remove Outliers') + #self.onDataPlugin(toolName='Filter') def setStatusBar(self, ISel=None): nTabs=self.tabList.len() @@ -477,7 +477,7 @@ def onDataPlugin(self, event=None, toolName=''): if toolName == thisToolName: if isPanel: # This is more of a "hasPanel" # Check to see if the pipeline already contains this action - action = self.pipeline.find(toolName) # old action to edit + action = self.pipePanel.find(toolName) # old action to edit if action is None: action = function(label=toolName, mainframe=self) # getting brand new action else: diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index 3d0ff8c..731161a 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -8,6 +8,7 @@ class Action(): def __init__(self, name, tableFunctionApply = None, tableFunctionCancel = None, + tableFunctionAdd = None, plotDataFunction = None, guiCallback=None, guiEditorClass=None, @@ -23,6 +24,7 @@ def __init__(self, name, self.name = name # + self.tableFunctionAdd = tableFunctionAdd # applies to a full table, create a new one self.tableFunctionApply = tableFunctionApply # applies to a full table self.tableFunctionCancel = tableFunctionCancel # cancel action on a full table self.plotDataFunction = plotDataFunction # applies to x,y arrays only @@ -66,6 +68,30 @@ def apply(self, tabList, force=False, applyToAll=False): return tabList + def applyAndAdd(self, tabList): + """ + Loop through tabList, perform the action and create new tables + """ + if self.tableFunctionAdd is None: + raise Exception('tableFunctionAdd was not specified for action: {}'.format(self.name)) + + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(tabList): +# try: + df_new, name_new = self.tableFunctionAdd(t, self.data) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) +# except: +# errors.append('Resampling failed for table: '+t.active_name) # TODO + return dfs_new, names_new, errors + + + + def updateGUI(self): """ Typically called by a callee after append""" if self.guiCallback is not None: @@ -140,6 +166,9 @@ def apply(self, tabList, force=False, applyToAll=False): def cancel(self, tabList): self.errorList=[] + if tabList is None: + print('[WARN] Cannot cancel action {} on None tablist'.format(self)) + return for t in tabList: print('>>> Action: Cancel: ', self, 'to', t.name) try: @@ -231,6 +260,7 @@ def remove(self, a, cancel=True, updateGUI=True, tabList=None): self.collectErrors() # Cancel the action in Editor + print('>>> GUI EDITOR', a.guiEditorObj) if a.guiEditorObj is not None: print('[Pipe] Canceling action in guiEditor because the action is removed') a.guiEditorObj.cancelAction() # NOTE: should not trigger a plot diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 3dc0f0d..6bea115 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -17,22 +17,24 @@ def _data_mask(label, mainframe): from .data_mask import maskAction return maskAction(label, mainframe) +# --- plotDataActions def _data_filter(label, mainframe): - from .data_filter import filterAction + from .plotdata_filter import filterAction return filterAction(label, mainframe) def _data_sampler(label, mainframe): - from .data_sampler import samplerAction + from .plotdata_sampler import samplerAction return samplerAction(label, mainframe) def _data_binning(label, mainframe): - from .data_binning import binningAction + from .plotdata_binning import binningAction return binningAction(label, mainframe) def _data_removeOutliers(label, mainframe): - from .data_removeOutliers import removeOutliersAction + from .plotdata_removeOutliers import removeOutliersAction return removeOutliersAction(label, mainframe) +# --- Irreversible actions def _data_standardizeUnitsSI(label, mainframe=None): from .data_standardizeUnits import standardizeUnitsAction return standardizeUnitsAction(label, mainframe, flavor='SI') diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index a13fc0f..ab0d613 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -54,10 +54,10 @@ def removeMask(tab, data): # tabList.clearCommonMask() # --------------------------------------------------------------------------------} -# --- Mask +# --- GUI to edit plugin and control a plot data action # --------------------------------------------------------------------------------{ class MaskToolPanel(GUIToolPanel): - def __init__(self, parent, action): + def __init__(self, parent, action, plotPanel, pipeLike): GUIToolPanel.__init__(self, parent) # --- Creating "Fake data" for testing only! @@ -66,32 +66,35 @@ def __init__(self, parent, action): action = maskAction(label='dummyAction', mainframe=DummyMainFrame(parent)) # --- Data - self.data = action.data - self.action = action + self.data = action.data + self.action = action + self.plotPanel = plotPanel + self.pipeLike = pipeLike + self.tabList = plotPanel.selPanel.tabList # a bit unfortunate - # --- Unfortunate data - self.tabList = action.mainframe.tabList # <<< - self.addTablesHandle = action.mainframe.load_dfs # <<< - self.addActionHandle = action.mainframe.addAction # <<< - self.removeActionHandle = action.mainframe.removeAction - self.redrawHandle = action.mainframe.redraw # or GUIPanel.load_and_draw() + # --- Unfortunate data to remove/manage + self.addTablesHandle = action.mainframe.load_dfs + self.addActionHandle = pipeLike.append + self.removeActionHandle = pipeLike.remove + self.redrawHandle = plotPanel.load_and_draw # or action.guiCallback - # --- GUI elements - tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() + # Register ourselves to the action to be safe + self.action.guiEditorObj = self + # --- GUI elements self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) self.btAdd = self.getBtBitmap(self, u'Mask (add)','add' , self.onAdd) - self.btApply = self.getToggleBtBitmap(self, 'Clear','sun', self.onToggleApply) + self.btApply = self.getToggleBtBitmap(self, 'Apply','cloud', self.onToggleApply) - self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'")""") - self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) + self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better - self.cbTabs.SetSelection(0) + self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'")""") self.textMask = wx.TextCtrl(self, wx.ID_ANY, 'Dummy', style = wx.TE_PROCESS_ENTER) #self.textMask.SetValue('({Time}>100) & ({Time}<400)') #self.textMask.SetValue("{Date} > '2018-10-01'") + # --- Layout btSizer = wx.FlexGridSizer(rows=2, cols=2, hgap=2, vgap=0) btSizer.Add(self.btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) btSizer.Add(wx.StaticText(self, -1, '') ,0,flag = wx.ALL|wx.EXPAND, border = 1) @@ -115,11 +118,13 @@ def __init__(self, parent, action): # --- Events # NOTE: getBtBitmap and getToggleBtBitmap already specify the binding - self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) self.textMask.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeAndPressEnter) # --- Init triggers self._Data2GUI() + self.onToggleApply(init=True) + self.updateTabList() # --- Implementation specific def guessMask(self): @@ -166,13 +171,14 @@ def updateTabList(self,event=None): # pass # --- External Calls - def cancelAction(self, redraw=True): - """ do cancel the action""" + def cancelAction(self): + """ set the GUI state when the action is cancelled""" self.btApply.SetLabel(CHAR['cloud']+' Mask') self.btApply.SetValue(False) self.data['active'] = False - def guiApplyAction(self, redraw=True): + def guiActionAppliedState(self): + """ set the GUI state when the action is applied""" self.btApply.SetLabel(CHAR['sun']+' Clear') self.btApply.SetValue(True) self.data['active'] = True @@ -189,28 +195,26 @@ def _Data2GUI(self): self.textMask.SetValue(self.data['maskString']) def onToggleApply(self, event=None, init=False): - self.data['active'] = not self.data['active'] - print('') - print('') - print('') + if not init: + self.data['active'] = not self.data['active'] + if self.data['active']: self._GUI2Data() # We update the GUI - self.guiApplyAction() + self.guiActionAppliedState() # Add action to pipeline, apply it, update the GUI self.addActionHandle(self.action, overwrite=True, apply=True, tabList=self.tabList, updateGUI=True) - #if iSel==0: - # dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=False) - # if len(errors)>0: - # raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) - #else: - # dfs, name = tabList[iSel-1].applyMaskString(maskString, bAdd=False) else: if not init: - # Remove action from pipeline, cancel it, update the GUI + # Remove action from pipeline, cancel it, update the GUI self.removeActionHandle(self.action, cancel=True, tabList=self.tabList, updateGUI=True) + else: + self.cancelAction() def onAdd(self, event=None): + """ + Apply tableFunction on all selected tables, create new tables, add them to the GUI + """ self._GUI2Data() iSel = self.cbTabs.GetSelection() # TODO this should be handled by the action @@ -218,7 +222,7 @@ def onAdd(self, event=None): if len(errors)>0: raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) - # We stop applying if we were applying it: + # We stop applying if we were applying it (NOTE: doing this before adding table due to redraw trigger of the whole panel) if self.data['active']: self.onToggleApply() diff --git a/pydatview/plugins/data_binning.py b/pydatview/plugins/plotdata_binning.py similarity index 50% rename from pydatview/plugins/data_binning.py rename to pydatview/plugins/plotdata_binning.py index 83f7c0c..ea94e7a 100644 --- a/pydatview/plugins/data_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -1,391 +1,229 @@ -import wx -import numpy as np -from pydatview.GUITools import GUIToolPanel, TOOL_BORDER -from pydatview.common import CHAR, Error, Info, pretty_num_short -from pydatview.common import DummyMainFrame -from pydatview.plotdata import PlotData -from pydatview.pipeline import PlotDataAction -# --------------------------------------------------------------------------------} -# --- Data -# --------------------------------------------------------------------------------{ -_DEFAULT_DICT={ - 'active':False, - 'xMin':None, - 'xMax':None, - 'nBins':50 -} -# --------------------------------------------------------------------------------} -# --- Action -# --------------------------------------------------------------------------------{ -def binningAction(label, mainframe=None, data=None): - """ - Return an "action" for the current plugin, to be used in the pipeline. - The action is also edited and created by the GUI Editor - """ - if data is None: - # NOTE: if we don't do copy below, we will end up remembering even after the action was deleted - # its not a bad feature, but we might want to think it through - # One issue is that "active" is kept in memory - data=_DEFAULT_DICT - data['active'] = False #<<< Important - - guiCallback=None - if mainframe is not None: - guiCallback = mainframe.redraw - - action = PlotDataAction( - name=label, - plotDataFunction = bin_plot, - guiEditorClass = BinningToolPanel, - guiCallback = guiCallback, - data = data, - mainframe=mainframe - ) - return action -# --------------------------------------------------------------------------------} -# --- Main method -# --------------------------------------------------------------------------------{ -def bin_plot(x, y, opts): - from pydatview.tools.stats import bin_signal - xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) - if xBins[0]>xBins[1]: - raise Exception('xmin must be lower than xmax') - x_new, y_new = bin_signal(x, y, xbins=xBins) - return x_new, y_new - -def bin_tab(tab, iCol, colName, opts, bAdd=True): - # TODO, make it such as it's only handling a dataframe instead of a table - from pydatview.tools.stats import bin_DF - colName = tab.data.columns[iCol] - error='' - xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) -# try: - df_new =bin_DF(tab.data, xbins=xBins, colBin=colName) - # Remove index if present - if df_new.columns[0].lower().find('index')>=0: - df_new = df_new.iloc[:, 1:] # We don't use "drop" in case of duplicate "index" - - # Setting bin column as first columns - colNames = list(df_new.columns.values) - colNames.remove(colName) - colNames.insert(0, colName) - df_new=df_new.reindex(columns=colNames) - if bAdd: - name_new=tab.raw_name+'_binned' - else: - name_new=None - tab.data=df_new -# except: -# df_new = None -# name_new = None - - return df_new, name_new - - - -# --------------------------------------------------------------------------------} -# --- GUI to edit plugin and control the action -# --------------------------------------------------------------------------------{ -class BinningToolPanel(GUIToolPanel): - def __init__(self, parent, action=None): - GUIToolPanel.__init__(self, parent) - - # --- Creating "Fake data" for testing only! - if action is None: - print('[WARN] Calling GUI without an action! Creating one.') - mainframe = DummyMainFrame(parent) - action = binningAction(label='dummyAction', mainframe=mainframe) - # --- Data - self.parent = parent # parent is GUIPlotPanel - self.mainframe = action.mainframe - self.data = action.data - self.action = action - - # --- GUI elements - self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) - self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) - self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) - - #self.lb = wx.StaticText( self, -1, """ Click help """) - self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) - self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better - self.scBins = wx.SpinCtrl(self, value='50', style=wx.TE_RIGHT, size=wx.Size(60,-1) ) - self.textXMin = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) - self.textXMax = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) - - self.btXRange = self.getBtBitmap(self, 'Default','compute', self.reset) - self.lbDX = wx.StaticText(self, -1, '') - self.scBins.SetRange(3, 10000) - - boldFont = self.GetFont().Bold() - lbInputs = wx.StaticText(self, -1, 'Inputs: ') - lbInputs.SetFont(boldFont) - - # --- Layout - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) - - msizer = wx.FlexGridSizer(rows=1, cols=3, hgap=2, vgap=0) - msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) -# msizer.Add(self.btXRange , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.LEFT, 1) - - msizer2 = wx.FlexGridSizer(rows=2, cols=5, hgap=2, vgap=1) - - msizer2.Add(lbInputs , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 0) - msizer2.Add(wx.StaticText(self, -1, '#bins: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 1) - msizer2.Add(self.scBins , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, 1) - msizer2.Add(wx.StaticText(self, -1, 'dx: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 8) - msizer2.Add(self.lbDX , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) - msizer2.Add(self.btXRange , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 0) - msizer2.Add(wx.StaticText(self, -1, 'xmin: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 1) - msizer2.Add(self.textXMin , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) - msizer2.Add(wx.StaticText(self, -1, 'xmax: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 8) - msizer2.Add(self.textXMax , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) - #msizer2.AddGrowableCol(4,1) - - vsizer = wx.BoxSizer(wx.VERTICAL) - vsizer.Add(msizer,0, flag = wx.TOP ,border = 1) - vsizer.Add(msizer2,0, flag = wx.TOP|wx.EXPAND ,border = 1) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT , border = 5) - self.sizer.Add(vsizer ,1, flag = wx.LEFT|wx.EXPAND , border = TOOL_BORDER) - self.SetSizer(self.sizer) - - # --- Events - self.scBins.Bind(wx.EVT_TEXT, self.onParamChange) - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) - self.textXMin.Bind(wx.EVT_TEXT_ENTER, self.onParamChange) - self.textXMax.Bind(wx.EVT_TEXT_ENTER, self.onParamChange) - - # --- Init triggers - if self.data['active']: - self.setXRange(x=[self.data['xMin'], self.data['xMax']]) - else: - self.setXRange() - self._Data2GUI() - self.onToggleApply(init=True) - self.updateTabList() - self.onParamChange() - - # --- Implementation specific - def reset(self, event=None): - self.setXRange() - self.updateTabList() # might as well until we add a nice callback/button.. - - def setXRange(self, x=None): - if x is None: - x= self.parent.plotData[0].x0 - xmin, xmax = np.nanmin(x), np.nanmax(x) - self.textXMin.SetValue(pretty_num_short(xmin)) - self.textXMax.SetValue(pretty_num_short(xmax)) - - def onParamChange(self, event=None): - self._GUI2Data() - self.lbDX.SetLabel(pretty_num_short((self.data['xMax']- self.data['xMin'])/self.data['nBins'])) - - if self.data['active']: - self.parent.load_and_draw() # Data will change - - def selectionChange(self): - """ function called if user change tables/columns""" - print('>>> Binning selectionChange callback, TODO') - self.setXRange() - - # --- Table related - def onTabChange(self,event=None): - #tabList = self.parent.selPanel.tabList - #iSel=self.cbTabs.GetSelection() - pass - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - - # --- External Calls - def cancelAction(self, redraw=True): - """ do cancel the action""" - self.btPlot.Enable(True) - self.btClear.Enable(True) - self.btApply.SetLabel(CHAR['cloud']+' Apply') - self.btApply.SetValue(False) - self.data['active'] = False - #if redraw: - # self.parent.load_and_draw() # Data will change based on plotData - - # --- Fairly generic - def _GUI2Data(self): - def zero_if_empty(s): - return 0 if len(s)==0 else s - self.data['nBins'] = int (self.scBins.Value) - self.data['xMin'] = float(zero_if_empty(self.textXMin.Value)) - self.data['xMax'] = float(zero_if_empty(self.textXMax.Value)) - - def _Data2GUI(self): - if self.data['active']: - self.lbDX.SetLabel(pretty_num_short((self.data['xMax']- self.data['xMin'])/self.data['nBins'])) - self.textXMin.SetValue(pretty_num_short(self.data['xMin'])) - self.textXMax.SetValue(pretty_num_short(self.data['xMax'])) - self.scBins.SetValue(self.data['nBins']) - - def onToggleApply(self, event=None, init=False): - """ - apply sampler based on GUI Data - """ - if not init: - self.data['active'] = not self.data['active'] - - if self.data['active']: - self._GUI2Data() - self.btPlot.Enable(False) - self.btClear.Enable(False) - self.btApply.SetLabel(CHAR['sun']+' Clear') - self.btApply.SetValue(True) - # The action is now active we add it to the pipeline, unless it's already in it - if self.mainframe is not None: - self.mainframe.addAction(self.action, overwrite=True) - if not init: - self.parent.load_and_draw() # filter will be applied in plotData.py - else: - # We remove our action from the pipeline - if not init: - if self.mainframe is not None: - self.mainframe.removeAction(self.action) - self.cancelAction(redraw=not init) - - def onAdd(self,event=None): - from pydatview.tools.stats import bin_DF - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - icol, colname = self.parent.selPanel.xCol - if self.parent.selPanel.currentMode=='simColumnsMode': - # The difficulty here is that we have to use - # self.parent.selPanel.IKeepPerTab - # or maybe just do it for the first table to get the x column name, - # but there is no guarantee that other tables will have the exact same column name. - Error(self, 'Cannot add tables in "simColumnsMode" for now. Go back to 1 table mode, and add tables individually.') - return - if icol==0: - Error(self, 'Cannot resample based on index') - return - - self._GUI2Data() - errors=[] - if iSel==0: - # Looping on all tables and adding new table - dfs_new = [] - names_new = [] - for itab, tab in enumerate(tabList): - df_new, name_new = bin_tab(tab, icol, colname, self.data, bAdd=True) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) - else: - errors.append(tab.active_name) - self.parent.addTables(dfs_new, names_new, bAdd=True) - else: - tab = tabList[iSel-1] - df_new, name_new = bin_tab(tab, icol, colname, self.data, bAdd=True) - if df_new is not None: - self.parent.addTables([df_new], [name_new], bAdd=True) - else: - errors.append(tab.active_name) - self.updateTabList() - - if len(errors)>0: - Error(self, 'The binning failed on some tables:\n\n'+'\n'.join(errors)) - return - - def onPlot(self,event=None): - if len(self.parent.plotData)!=1: - Error(self,'Plotting only works for a single plot. Plot less data.') - return - self._GUI2Data() - PD = self.parent.plotData[0] - x_new, y_new = bin_plot(PD.x0, PD.y0, self.data) - - ax = self.parent.fig.axes[0] - PD_new = PlotData() - PD_new.fromXY(x_new, y_new) - self.parent.transformPlotData(PD_new) - ax.plot(PD_new.x, PD_new.y, '-') - self.parent.canvas.draw() - - def onClear(self,event=None): - self.parent.load_and_draw() # Data will change - # Update Table list - self.updateTabList() - - def onHelp(self,event=None): - Info(self,"""Binning. - -The binning operation computes average y values for a set of x ranges. - -To bin perform the following step: - -- Specify the number of bins (#bins) -- Specify the min and max of the x values (or click on "Default") - -- Click on one of the following buttons: - - Plot: will display the binned data on the figure - - Apply: will perform the binning on the fly for all new plots - (click on Clear to stop applying) - - Add: will create new table(s) with biined values for all - signals. This process might take some time. - Select a table or choose all (default) -""") - - - - -if __name__ == '__main__': - from pydatview.Tables import TableList - from pydatview.plotdata import PlotData - from pydatview.GUIPlotPanel import PlotPanel - from pydatview.GUISelectionPanel import SelectionPanel - - - # --- Data - tabList = TableList.createDummy(nTabs=2, n=100, addLabel=False) - app = wx.App(False) - self = wx.Frame(None,-1,"Data Binning GUI") - - # --- Panels - self.selPanel = SelectionPanel(self, tabList, mode='auto') - self.plotPanel = PlotPanel(self, self.selPanel) - self.plotPanel.load_and_draw() # <<< Important - self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # Binding the two - - p = BinningToolPanel(self.plotPanel, action=None) - - sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(self.selPanel ,0, wx.EXPAND|wx.ALL, border=5) - sizer.Add(self.plotPanel,1, wx.EXPAND|wx.ALL, border=5) - #sizer.Add(p) - self.SetSizer(sizer) - self.SetSize((900, 600)) - self.Center() - self.Show() - - self.plotPanel.showToolPanel(panel=p) - - app.MainLoop() - - +import wx +import numpy as np +from pydatview.GUITools import TOOL_BORDER +from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor +from pydatview.common import Error, Info, pretty_num_short +from pydatview.pipeline import PlotDataAction +# --------------------------------------------------------------------------------} +# --- Data +# --------------------------------------------------------------------------------{ +_DEFAULT_DICT={ + 'active':False, + 'xMin':None, + 'xMax':None, + 'nBins':50 +} +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def binningAction(label, mainframe=None, data=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + The action is also edited and created by the GUI Editor + """ + if data is None: + # NOTE: if we don't do copy below, we will end up remembering even after the action was deleted + # its not a bad feature, but we might want to think it through + # One issue is that "active" is kept in memory + data=_DEFAULT_DICT + data['active'] = False #<<< Important + + guiCallback=None + if mainframe is not None: + guiCallback = mainframe.redraw + + action = PlotDataAction( + name=label, + tableFunctionAdd = binTabAdd, + plotDataFunction = bin_plot, + guiEditorClass = BinningToolPanel, + guiCallback = guiCallback, + data = data, + mainframe=mainframe + ) + return action +# --------------------------------------------------------------------------------} +# --- Main method +# --------------------------------------------------------------------------------{ +def bin_plot(x, y, opts): + from pydatview.tools.stats import bin_signal + xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) + if xBins[0]>xBins[1]: + raise Exception('xmin must be lower than xmax') + x_new, y_new = bin_signal(x, y, xbins=xBins) + return x_new, y_new + +def bin_tab(tab, iCol, colName, opts, bAdd=True): + # TODO, make it such as it's only handling a dataframe instead of a table + from pydatview.tools.stats import bin_DF + colName = tab.data.columns[iCol] + error='' + xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) +# try: + df_new =bin_DF(tab.data, xbins=xBins, colBin=colName) + # Remove index if present + if df_new.columns[0].lower().find('index')>=0: + df_new = df_new.iloc[:, 1:] # We don't use "drop" in case of duplicate "index" + + # Setting bin column as first columns + colNames = list(df_new.columns.values) + colNames.remove(colName) + colNames.insert(0, colName) + df_new=df_new.reindex(columns=colNames) + if bAdd: + name_new=tab.raw_name+'_binned' + else: + name_new=None + tab.data=df_new +# except: +# df_new = None +# name_new = None + + return df_new, name_new + +def binTabAdd(tab, data): + return bin_tab(tab, data['icol'], data['colname'], data, bAdd=True) + +# --------------------------------------------------------------------------------} +# --- GUI to edit plugin and control the action +# --------------------------------------------------------------------------------{ +class BinningToolPanel(PlotDataActionEditor): + + def __init__(self, parent, action, plotPanel, pipeLike, **kwargs): + PlotDataActionEditor.__init__(self, parent, action, plotPanel, pipeLike, tables=True) + + # --- GUI elements + self.scBins = wx.SpinCtrl(self, value='50', style=wx.TE_RIGHT, size=wx.Size(60,-1) ) + self.textXMin = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) + self.textXMax = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) + + self.btXRange = self.getBtBitmap(self, 'Default','compute', self.reset) + self.lbDX = wx.StaticText(self, -1, '') + self.scBins.SetRange(3, 10000) + + boldFont = self.GetFont().Bold() + lbInputs = wx.StaticText(self, -1, 'Inputs: ') + lbInputs.SetFont(boldFont) + + # --- Layout + msizer = wx.FlexGridSizer(rows=1, cols=3, hgap=2, vgap=0) + msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) +# msizer.Add(self.btXRange , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.LEFT, 1) + + msizer2 = wx.FlexGridSizer(rows=2, cols=5, hgap=2, vgap=1) + + msizer2.Add(lbInputs , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 0) + msizer2.Add(wx.StaticText(self, -1, '#bins: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 1) + msizer2.Add(self.scBins , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, 1) + msizer2.Add(wx.StaticText(self, -1, 'dx: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 8) + msizer2.Add(self.lbDX , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) + msizer2.Add(self.btXRange , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 0) + msizer2.Add(wx.StaticText(self, -1, 'xmin: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 1) + msizer2.Add(self.textXMin , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) + msizer2.Add(wx.StaticText(self, -1, 'xmax: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 8) + msizer2.Add(self.textXMax , 1, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.EXPAND, 1) + #msizer2.AddGrowableCol(4,1) + + vsizer = wx.BoxSizer(wx.VERTICAL) + vsizer.Add(msizer,0, flag = wx.TOP ,border = 1) + vsizer.Add(msizer2,0, flag = wx.TOP|wx.EXPAND ,border = 1) + + #self.sizer = wx.BoxSizer(wx.HORIZONTAL) + #self.sizer.Add(btSizer ,0, flag = wx.LEFT , border = 5) + self.sizer.Add(vsizer ,1, flag = wx.LEFT|wx.EXPAND , border = TOOL_BORDER) + self.SetSizer(self.sizer) + + # --- Events + self.scBins.Bind (wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow) + self.scBins.Bind (wx.EVT_TEXT_ENTER, self.onParamChangeEnter) + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + self.textXMin.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter) + self.textXMax.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter) + + # --- Init triggers + if self.data['active']: + self.setXRange(x=[self.data['xMin'], self.data['xMax']]) + else: + self.setXRange() + self._Data2GUI() + self.onToggleApply(init=True) + #self.updateTabList() + + # --- Implementation specific + def reset(self, event=None): + self.setXRange() + self.updateTabList() # might as well until we add a nice callback/button.. + + def setXRange(self, x=None): + if x is None: + x= self.plotPanel.plotData[0].x0 + xmin, xmax = np.nanmin(x), np.nanmax(x) + self.textXMin.SetValue(pretty_num_short(xmin)) + self.textXMax.SetValue(pretty_num_short(xmax)) + self.lbDX.SetLabel(pretty_num_short((xmax-xmin)/int (self.scBins.Value))) + + # --- Bindings for plot triggers on parameters changes + def onParamChange(self, event=None): + PlotDataActionEditor.onParamChange(self) + self.lbDX.SetLabel(pretty_num_short((self.data['xMax']- self.data['xMin'])/self.data['nBins'])) + + # --- Fairly generic + def _GUI2Data(self): + def zero_if_empty(s): + return 0 if len(s)==0 else s + self.data['nBins'] = int (self.scBins.Value) + self.data['xMin'] = float(zero_if_empty(self.textXMin.Value)) + self.data['xMax'] = float(zero_if_empty(self.textXMax.Value)) + + def _Data2GUI(self): + if self.data['active']: + self.lbDX.SetLabel(pretty_num_short((self.data['xMax']- self.data['xMin'])/self.data['nBins'])) + self.textXMin.SetValue(pretty_num_short(self.data['xMin'])) + self.textXMax.SetValue(pretty_num_short(self.data['xMax'])) + self.scBins.SetValue(self.data['nBins']) + + def onAdd(self,event=None): + if self.plotPanel.selPanel.currentMode=='simColumnsMode': + # The difficulty here is that we have to use + # self.plotPanel.selPanel.IKeepPerTab + # or maybe just do it for the first table to get the x column name, + # but there is no guarantee that other tables will have the exact same column name. + Error(self, 'Cannot add tables in "simColumnsMode" for now. Go back to 1 table mode, and add tables individually.') + return + + # TODO put this in GUI2Data??? + iSel = self.cbTabs.GetSelection() + icol, colname = self.plotPanel.selPanel.xCol + self.data['icol'] = icol + self.data['colname'] = colname + + # Call parent class + PlotDataActionEditor.onAdd(self) + + def onHelp(self,event=None): + Info(self,"""Binning. + +The binning operation computes average y values for a set of x ranges. + +To bin perform the following step: + +- Specify the number of bins (#bins) +- Specify the min and max of the x values (or click on "Default") + +- Click on one of the following buttons: + - Plot: will display the binned data on the figure + - Apply: will perform the binning on the fly for all new plots + (click on Clear to stop applying) + - Add: will create new table(s) with biined values for all + signals. This process might take some time. + Select a table or choose all (default) +""") + + + + +if __name__ == '__main__': + from pydatview.plugins.plotdata_default_plugin import demoPlotDataActionPanel + + demoPlotDataActionPanel(BinningToolPanel, plotDataFunction=bin_plot, data=_DEFAULT_DICT, tableFunctionAdd=binTabAdd) diff --git a/pydatview/plugins/plotdata_default_plugin.py b/pydatview/plugins/plotdata_default_plugin.py new file mode 100644 index 0000000..c72a43d --- /dev/null +++ b/pydatview/plugins/plotdata_default_plugin.py @@ -0,0 +1,281 @@ +import wx +import numpy as np +from pydatview.GUITools import GUIToolPanel, TOOL_BORDER +from pydatview.common import CHAR, Error, Info, pretty_num_short +from pydatview.common import DummyMainFrame +from pydatview.plotdata import PlotData +from pydatview.pipeline import PlotDataAction + + +# --------------------------------------------------------------------------------} +# --- GUI to edit plugin and control a plot data action +# --------------------------------------------------------------------------------{ +class PlotDataActionEditor(GUIToolPanel): + + def __init__(self, parent, action, plotPanel, pipeLike, buttons=None, tables=True): + """ + """ + GUIToolPanel.__init__(self, parent) + + # --- Data + self.data = action.data + self.action = action + self.plotPanel = plotPanel + self.pipeLike = pipeLike + self.tabList = plotPanel.selPanel.tabList # a bit unfortunate + + # --- Unfortunate data to remove/manage + self.addTablesHandle = action.mainframe.load_dfs + self.addActionHandle = pipeLike.append + self.removeActionHandle = pipeLike.remove + self.redrawHandle = plotPanel.load_and_draw # or action.guiCallback + + # Register ourselves to the action to be safe + self.action.guiEditorObj = self + + # --- GUI elements + if buttons is None: + buttons=['Close', 'Add', 'Help', 'Clear', 'Plot', 'Apply'] + self.btAdd = None + self.btHelp = None + self.btClear = None + self.btPlot = None + nButtons = 0 + self.btClose = self.getBtBitmap(self,'Close','close',self.destroy); nButtons+=1 + if 'Add' in buttons: + self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd); nButtons+=1 + if 'Help' in buttons: + self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp); nButtons+=1 + if 'Clear' in buttons: + self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear); nButtons+=1 + if 'Plot' in buttons: + self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot); nButtons+=1 + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply); nButtons+=1 + + + if tables: + self.cbTabs= wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better + else: + self.cbTabs = None + + # --- Layout + btSizer = wx.FlexGridSizer(rows=int(nButtons/2), cols=2, hgap=2, vgap=0) + btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if self.btClear is not None: + btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if self.btAdd is not None: + btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if self.btPlot is not None: + btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if self.btHelp is not None: + btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) + # Child sizer will go here + # TODO cbTabs has no sizer + self.SetSizer(self.sizer) + + # --- Events + if tables: + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + + ## --- Init triggers + #self._Data2GUI() + #self.onSelectFilt() + #self.onToggleApply(init=True) + #self.updateTabList() + + # --- Bindings for plot triggers on parameters changes + def onParamChange(self, event=None): + self._GUI2Data() + if self.data['active']: + self.redrawHandle() + + def onParamChangeArrow(self, event): + self.onParamChange() + event.Skip() + + def onParamChangeEnter(self, event): + """ Action when the user presses Enter: we activate the action """ + if not self.data['active']: + self.onToggleApply() + else: + self.onParamChange() + event.Skip() + + def onParamChangeChar(self, event): + event.Skip() + code = event.GetKeyCode() + if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: + #self.tParam.SetValue(self.spintxt.Value) # TODO + self.onParamChangeEnter(event) + + # --- Table related + def onTabChange(self,event=None): + #tabList = self.parent.selPanel.tabList + #iSel=self.cbTabs.GetSelection() + pass + + def updateTabList(self,event=None): + if self.cbTabs is not None: + tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() + #try: + iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + #except RuntimeError: + # pass + + # --- External Calls + def cancelAction(self): + """ set the GUI state when the action is cancelled""" + if self.btPlot is not None: + self.btPlot.Enable(True) + if self.btClear is not None: + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + self.btApply.SetValue(False) + self.data['active'] = False + + def guiActionAppliedState(self): + """ set the GUI state when the action is applied""" + if self.btPlot is not None: + self.btPlot.Enable(False) + if self.btClear is not None: + self.btClear.Enable(False) + self.btApply.SetLabel(CHAR['sun']+' Clear') + self.btApply.SetValue(True) + self.data['active'] = True + + # --- Fairly generic + def _GUI2Data(self): + pass + + def _Data2GUI(self): + pass + + def onToggleApply(self, event=None, init=False): + if not init: + self.data['active'] = not self.data['active'] + + if self.data['active']: + self._GUI2Data() + # We update the GUI + self.guiActionAppliedState() # TODO remove me + # Add action to pipeline, no need to apply it, update the GUI (redraw) + self.addActionHandle(self.action, overwrite=True, apply=False, updateGUI=True) + else: + if not init: + # Remove action from pipeline, no need to cancel it, update the GUI + self.removeActionHandle(self.action, cancel=False, updateGUI=True) + else: + self.cancelAction() + + def onAdd(self, event=None): + """ + Apply tableFunction on all selected tables, create new tables, add them to the GUI + """ + #iSel = self.cbTabs.GetSelection() + #icol, colname = self.plotPanel.selPanel.xCol + self._GUI2Data() + + dfs, names, errors = self.action.applyAndAdd(self.tabList) + if len(errors)>0: + raise Exception('Error: The action {} failed on some tables:\n\n'.format(action.name)+'\n'.join(errors)) + + # We stop applying if we were applying it (NOTE: doing this before adding table due to redraw trigger of the whole panel) + if self.data['active']: + self.onToggleApply() + + self.addTablesHandle(dfs, names, bAdd=True, bPlot=False) # Triggers a redraw of the whole panel... + # df, name = self.tabList[iSel-1].applyFiltering(icol, self.data, bAdd=True) + # self.parent.addTables([df], [name], bAdd=True) + #self.updateTabList() + + + def onPlot(self, event=None): + self._GUI2Data() + # Loop on axes and plotdata of each axes + for ax in self.plotPanel.fig.axes: + for iPD in ax.iPD: + PD = self.plotPanel.plotData[iPD] + # Apply the plotDataFunction + x_new, y_new = self.action.plotDataFunction(PD.x0, PD.y0, self.data) + # Go through plotPanel own pipeline + PD_new = PlotData() + PD_new.fromXY(x_new, y_new) + self.plotPanel.transformPlotData(PD_new) + # Plot + ax.plot(PD_new.x, PD_new.y, '-') + # Simple canvas draw (a redraw would remove what we just added) + self.plotPanel.canvas.draw() + + def onClear(self, event=None): + self.redrawHandle() + + def onHelp(self,event=None): + Info(self, """Dummy help""") + + + +def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableFunctionAdd=None): + """ Function to demonstrate behavior of a plotdata plugin""" + from pydatview.Tables import TableList + from pydatview.pipeline import Pipeline + from pydatview.GUIPlotPanel import PlotPanel + from pydatview.GUISelectionPanel import SelectionPanel + if data is None: + data={'active':False} + + if plotDataFunction is None: + plotDataFunction = lambda x,y,opts: (x+0.1, 1.01*y) + + + # --- Data + tabList = TableList.createDummy(nTabs=2, n=100, addLabel=False) + app = wx.App(False) + self = wx.Frame(None,-1,"Data Binning GUI") + pipeline = Pipeline() + + # --- Panels + self.selPanel = SelectionPanel(self, tabList, mode='auto') + self.plotPanel = PlotPanel(self, self.selPanel, pipeLike=pipeline) + self.plotPanel.load_and_draw() # <<< Important + self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # Binding the two + + # --- Dummy mainframe and action.. + mainframe = DummyMainFrame(self) + guiCallback = self.plotPanel.load_and_draw + action = PlotDataAction( + name = 'Dummy Action', + tableFunctionAdd = tableFunctionAdd, + plotDataFunction = plotDataFunction, + guiEditorClass = panelClass, + guiCallback = guiCallback, + data = data, + mainframe = mainframe + ) + + # --- Create main object to be tested + p = panelClass(self.plotPanel, action=action, plotPanel=self.plotPanel, pipeLike=pipeline, tables=False) + + # --- Finalize GUI + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(self.selPanel ,0, wx.EXPAND|wx.ALL, border=5) + sizer.Add(self.plotPanel,1, wx.EXPAND|wx.ALL, border=5) + self.SetSizer(sizer) + self.SetSize((900, 600)) + self.Center() + self.Show() + self.plotPanel.showToolPanel(panel=p) # <<< Show + app.MainLoop() + +if __name__ == '__main__': + + demoPlotDataActionPanel(PlotDataActionEditor) + + diff --git a/pydatview/plugins/data_filter.py b/pydatview/plugins/plotdata_filter.py similarity index 52% rename from pydatview/plugins/data_filter.py rename to pydatview/plugins/plotdata_filter.py index 3fac83f..18b0604 100644 --- a/pydatview/plugins/data_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -1,9 +1,8 @@ import wx import numpy as np -from pydatview.GUITools import GUIToolPanel, TOOL_BORDER -from pydatview.common import CHAR, Error, Info, pretty_num_short -from pydatview.common import DummyMainFrame -from pydatview.plotdata import PlotData +from pydatview.GUITools import TOOL_BORDER +from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor +from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction import platform # --------------------------------------------------------------------------------} @@ -39,72 +38,50 @@ def filterAction(label, mainframe=None, data=None): guiCallback = mainframe.redraw action = PlotDataAction( - name=label, + name = label, + tableFunctionAdd = filterTabAdd, plotDataFunction = filterXY, - guiEditorClass = FilterToolPanel, - guiCallback = guiCallback, - data = data, - mainframe=mainframe + guiEditorClass = FilterToolPanel, + guiCallback = guiCallback, + data = data, + mainframe = mainframe ) return action # --------------------------------------------------------------------------------} # --- Main method # --------------------------------------------------------------------------------{ def filterXY(x, y, opts): + """ Apply action on a x and y array """ from pydatview.tools.signal_analysis import applyFilter y_new = applyFilter(x, y, opts) return x, y_new +def filterTabAdd(tab, opts): + """ Apply action on a a table and return a new one with a new name + df_new, name_new = f(t, opts) + """ + return tab.applyFiltering(opts['icol'], opts, bAdd=True) + # --------------------------------------------------------------------------------} # --- GUI to edit plugin and control the action # --------------------------------------------------------------------------------{ -class FilterToolPanel(GUIToolPanel): +class FilterToolPanel(PlotDataActionEditor): - def __init__(self, parent, action=None): - GUIToolPanel.__init__(self, parent) + def __init__(self, parent, action, plotPanel, pipeLike, **kwargs): + PlotDataActionEditor.__init__(self, parent, action, plotPanel, pipeLike, tables=True) - # --- Creating "Fake data" for testing only! - if action is None: - print('[WARN] Calling GUI without an action! Creating one.') - mainframe = DummyMainFrame(parent) - action = binningAction(label='dummyAction', mainframe=mainframe) - # --- Data - self.parent = parent # parent is GUIPlotPanel - self.mainframe = action.mainframe - self.data = action.data - self.action = action from pydatview.tools.signal_analysis import FILTERS self._FILTERS_USER=FILTERS .copy() # --- GUI elements - self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) - self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) - self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear) - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) - - self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) - self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better self.cbFilters = wx.ComboBox(self, choices=[filt['name'] for filt in self._FILTERS_USER], style=wx.CB_READONLY) self.lbParamName = wx.StaticText(self, -1, ' :') self.cbFilters.SetSelection(0) -# self.tParam = wx.TextCtrl(self, wx.ID_ANY, '', style= wx.TE_PROCESS_ENTER, size=wx.Size(60,-1)) self.tParam = wx.SpinCtrlDouble(self, value='11', size=wx.Size(80,-1)) self.lbInfo = wx.StaticText( self, -1, '') - # --- Layout - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) - - horzSizerT = wx.BoxSizer(wx.HORIZONTAL) horzSizerT.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) horzSizerT.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) @@ -121,13 +98,13 @@ def __init__(self, parent, action=None): vertSizer.Add(horzSizerT ,1, flag = wx.LEFT|wx.EXPAND,border = 1) vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) + # self.sizer = wx.BoxSizer(wx.HORIZONTAL) + # self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) - self.SetSizer(self.sizer) + self.sizer.Layout() + # --- Events - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) self.cbFilters.Bind(wx.EVT_COMBOBOX, self.onSelectFilt) self.Bind(wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow, self.tParam) self.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter, self.tParam) @@ -142,13 +119,12 @@ def __init__(self, parent, action=None): # --- Init triggers self._Data2GUI() - self.updateTabList() - self.onSelectFilt() self.onToggleApply(init=True) + self.onSelectFilt() # --- Implementation specific def onSelectFilt(self, event=None): - """ Select the filter, but does not applied it to the plotData + """ Select the filter, but does not apply it to the plotData parentFilt is unchanged But if the parent already has """ @@ -170,59 +146,49 @@ def onSelectFilt(self, event=None): # Trigger plot if applied self.onParamChange(self) - # --- Bindings for plot triggers on parameters changes - def onParamChange(self, event=None): - self._GUI2Data() - if self.data['active']: - self.parent.load_and_draw() # Data will change - - def onParamChangeArrow(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeEnter(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeChar(self, event): - event.Skip() - code = event.GetKeyCode() - if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: - #print(self.spintxt.Value) - self.tParam.SetValue(self.spintxt.Value) - self.onParamChangeEnter(event) - - # --- Table related - def onTabChange(self,event=None): - #tabList = self.parent.selPanel.tabList - #iSel=self.cbTabs.GetSelection() - pass - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - +# # --- Bindings for plot triggers on parameters changes +# def onParamChange(self, event=None): +# self._GUI2Data() +# if self.data['active']: +# self.parent.load_and_draw() # Data will change +# +# def onParamChangeArrow(self, event): +# self.onParamChange() +# event.Skip() +# +# def onParamChangeEnter(self, event): +# if not self.data['active']: +# self.onToggleApply() +# else: +# self.onParamChange() +# event.Skip() +# +# def onParamChangeChar(self, event): +# event.Skip() +# code = event.GetKeyCode() +# if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: +# #print(self.spintxt.Value) +# self.tParam.SetValue(self.spintxt.Value) +# self.onParamChangeEnter(event) +# # --- External Calls - def cancelAction(self, redraw=True): - """ do cancel the action""" + def cancelAction(self): + """ set the GUI state when the action is cancelled""" + # Call parent class + PlotDataActionEditor.cancelAction(self) + # Update GUI self.lbInfo.SetLabel( 'Click on "Apply" to set filter on the fly for all plots. '+ 'Click on "Plot" to try a filter on the current plot.' ) - self.btPlot.Enable(True) - self.btClear.Enable(True) - self.btApply.SetLabel(CHAR['cloud']+' Apply') - self.btApply.SetValue(False) - self.data['active'] = False - if redraw: - self.parent.load_and_draw() # Data will change based on plotData + def guiActionAppliedState(self): + """ set the GUI state when the action is applied""" + # Call parent class + PlotDataActionEditor.guiActionAppliedState(self) + # Update GUI + self.lbInfo.SetLabel( + 'Filter is now applied on the fly. Change parameter live. Click "Clear" to stop. ' + ) # --- Fairly generic def _GUI2Data(self): @@ -248,77 +214,14 @@ def _Data2GUI(self): self.tParam.SetIncrement(self.data['increment']) self.tParam.SetDigits(self.data['digits']) - def onToggleApply(self, event=None, init=False): - """ - apply Filter based on GUI Data - """ - if not init: - self.data['active'] = not self.data['active'] - - if self.data['active']: - self._GUI2Data() - - self.lbInfo.SetLabel( - 'Filter is now applied on the fly. Change parameter live. Click "Clear" to stop. ' - ) - self.btPlot.Enable(False) - self.btClear.Enable(False) - self.btApply.SetLabel(CHAR['sun']+' Clear') - self.btApply.SetValue(True) - # The action is now active we add it to the pipeline, unless it's already in it - if self.mainframe is not None: - self.mainframe.addAction(self.action, overwrite=True) - if not init: - self.parent.load_and_draw() # filter will be applied in plotData.py - else: - # We remove our action from the pipeline - if not init: - if self.mainframe is not None: - self.mainframe.removeAction(self.action) - self.cancelAction(redraw=not init) - def onAdd(self, event=None): - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - icol, colname = self.parent.selPanel.xCol - self._GUI2Data() - errors=[] - if iSel==0: - dfs, names, errors = tabList.applyFiltering(icol, self.data, bAdd=True) - self.parent.addTables(dfs,names,bAdd=True) - else: - df, name = tabList[iSel-1].applyFiltering(icol, self.data, bAdd=True) - self.parent.addTables([df], [name], bAdd=True) - self.updateTabList() - - if len(errors)>0: - raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) - - # We stop applying - self.onToggleApply() - - def onPlot(self, event=None): - """ - Overlay on current axis the filter - """ - - if len(self.parent.plotData)!=1: - Error(self,'Plotting only works for a single plot. Plot less data.') - return - self._GUI2Data() - PD = self.parent.plotData[0] - x_new, y_new = filterXY(PD.x0, PD.y0, self.data) - - ax = self.parent.fig.axes[0] - PD_new = PlotData() - PD_new.fromXY(x_new, y_new) - self.parent.transformPlotData(PD_new) - ax.plot(PD_new.x, PD_new.y, '-') - self.parent.canvas.draw() - - def onClear(self, event): - self.parent.load_and_draw() # Data will change + # TODO put this in GUI2Data??? + iSel = self.cbTabs.GetSelection() + icol, colname = self.plotPanel.selPanel.xCol + self.data['icol'] = icol + # Call parent class + PlotDataActionEditor.onAdd(self) def onHelp(self,event=None): Info(self,"""Filtering. @@ -346,4 +249,7 @@ def onHelp(self,event=None): +if __name__ == '__main__': + from pydatview.plugins.plotdata_default_plugin import demoPlotDataActionPanel + demoPlotDataActionPanel(FilterToolPanel, plotDataFunction=filterXY, data=_DEFAULT_DICT, tableFunctionAdd=filterTabAdd) diff --git a/pydatview/plugins/data_removeOutliers.py b/pydatview/plugins/plotdata_removeOutliers.py similarity index 51% rename from pydatview/plugins/data_removeOutliers.py rename to pydatview/plugins/plotdata_removeOutliers.py index 779d430..0ca09f9 100644 --- a/pydatview/plugins/data_removeOutliers.py +++ b/pydatview/plugins/plotdata_removeOutliers.py @@ -1,9 +1,8 @@ import wx import numpy as np -from pydatview.GUITools import GUIToolPanel, TOOL_BORDER -from pydatview.common import CHAR, Error, Info, pretty_num_short -from pydatview.common import DummyMainFrame -from pydatview.plotdata import PlotData +from pydatview.GUITools import TOOL_BORDER +from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor +from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction import platform # --------------------------------------------------------------------------------} @@ -55,28 +54,16 @@ def removeOutliersXY(x, y, opts): # --------------------------------------------------------------------------------} # --- GUI to edit plugin and control the action # --------------------------------------------------------------------------------{ -class RemoveOutliersToolPanel(GUIToolPanel): - - def __init__(self, parent, action): - GUIToolPanel.__init__(self, parent) - - # --- Creating "Fake data" for testing only! - if action is None: - print('[WARN] Calling GUI without an action! Creating one.') - mainframe = DummyMainFrame(parent) - action = binningAction(label='dummyAction', mainframe=mainframe) - - # --- Data - self.parent = parent # parent is GUIPlotPanel - self.mainframe = action.mainframe - self.data = action.data - self.action = action +class RemoveOutliersToolPanel(PlotDataActionEditor): + + def __init__(self, parent, action, plotPanel, pipeLike, **kwargs): + PlotDataActionEditor.__init__(self, parent, action, plotPanel, pipeLike, tables=False, buttons=['']) + # --- GUI elements - self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) + #self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) + #self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) lb1 = wx.StaticText(self, -1, 'Median deviation:') -# self.tMD = wx.TextCtrl(self, wx.ID_ANY,, size = (30,-1), style=wx.TE_PROCESS_ENTER) self.tMD = wx.SpinCtrlDouble(self, value='11', size=wx.Size(60,-1)) self.tMD.SetValue(5) self.tMD.SetRange(0.0, 1000) @@ -85,12 +72,14 @@ def __init__(self, parent, action): self.lb = wx.StaticText( self, -1, '') # --- Layout - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(self.btClose,0,flag = wx.LEFT|wx.CENTER,border = 1) - self.sizer.Add(self.btApply,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(lb1 ,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(self.tMD ,0,flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(self.lb ,0,flag = wx.LEFT|wx.CENTER,border = 5) + hsizer = wx.BoxSizer(wx.HORIZONTAL) +# self.sizer.Add(self.btClose,0,flag = wx.LEFT|wx.CENTER,border = 1) +# self.sizer.Add(self.btApply,0,flag = wx.LEFT|wx.CENTER,border = 5) + hsizer.Add(lb1 ,0,flag = wx.LEFT|wx.CENTER,border = 5) + hsizer.Add(self.tMD ,0,flag = wx.LEFT|wx.CENTER,border = 5) + hsizer.Add(self.lb ,0,flag = wx.LEFT|wx.CENTER,border = 5) + + self.sizer.Add(hsizer ,0,flag = wx.LEFT|wx.CENTER,border = 5) self.SetSizer(self.sizer) # --- Events @@ -107,40 +96,40 @@ def __init__(self, parent, action): self._Data2GUI() self.onToggleApply(init=True) - # --- Implementation specific - # --- Bindings for plot triggers on parameters changes - def onParamChange(self, event=None): - self._GUI2Data() - if self.data['active']: - self.parent.load_and_draw() # Data will change - - def onParamChangeArrow(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeEnter(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeChar(self, event): - event.Skip() - code = event.GetKeyCode() - if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: - #print(self.spintxt.Value) - self.tMD.SetValue(self.spintxt.Value) - self.onParamChangeEnter(event) +# def onParamChange(self, event=None): +# self._GUI2Data() +# if self.data['active']: +# self.parent.load_and_draw() # Data will change +# +# def onParamChangeArrow(self, event): +# self.onParamChange() +# event.Skip() +# +# def onParamChangeEnter(self, event): +# self.onParamChange() +# event.Skip() +# +# def onParamChangeChar(self, event): +# event.Skip() +# code = event.GetKeyCode() +# if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: +# #print(self.spintxt.Value) +# self.tMD.SetValue(self.spintxt.Value) +# self.onParamChangeEnter(event) # --- Table related # --- External Calls - def cancelAction(self, redraw=True): + def cancelAction(self): """ do cancel the action""" self.lb.SetLabel('Click on "Apply" to remove outliers on the fly for all new plot.') - self.btApply.SetLabel(CHAR['cloud']+' Apply') - self.btApply.SetValue(False) - self.data['active'] = False - if redraw: - self.parent.load_and_draw() # Data will change based on plotData + PlotDataActionEditor.cancelAction(self) + +# self.btApply.SetLabel(CHAR['cloud']+' Apply') +# self.btApply.SetValue(False) +# self.data['active'] = False +# if redraw: +# self.parent.load_and_draw() # Data will change based on plotData # --- Fairly generic def _GUI2Data(self): @@ -148,26 +137,30 @@ def _GUI2Data(self): def _Data2GUI(self): self.tMD.SetValue(self.data['medianDeviation']) - - def onToggleApply(self, event=None, init=False): - - if not init: - self.data['active'] = not self.data['active'] - - if self.data['active']: - self._GUI2Data() - self.lb.SetLabel('Outliers are now removed on the fly. Click "Clear" to stop.') - self.btApply.SetLabel(CHAR['sun']+' Clear') - self.btApply.SetValue(True) - # The action is now active we add it to the pipeline, unless it's already in it - if self.mainframe is not None: - self.mainframe.addAction(self.action, overwrite=True) - if not init: - self.parent.load_and_draw() # filter will be applied in plotData.py - else: - # We remove our action from the pipeline - if not init: - if self.mainframe is not None: - self.mainframe.removeAction(self.action) - self.cancelAction(redraw= not init) - +# +# def onToggleApply(self, event=None, init=False): +# +# if not init: +# self.data['active'] = not self.data['active'] +# +# if self.data['active']: +# self._GUI2Data() +# self.lb.SetLabel('Outliers are now removed on the fly. Click "Clear" to stop.') +# self.btApply.SetLabel(CHAR['sun']+' Clear') +# self.btApply.SetValue(True) +# # The action is now active we add it to the pipeline, unless it's already in it +# if self.mainframe is not None: +# self.mainframe.addAction(self.action, overwrite=True) +# if not init: +# self.parent.load_and_draw() # filter will be applied in plotData.py +# else: +# # We remove our action from the pipeline +# if not init: +# if self.mainframe is not None: +# self.mainframe.removeAction(self.action) +# self.cancelAction(redraw= not init) + +if __name__ == '__main__': + from pydatview.plugins.plotdata_default_plugin import demoPlotDataActionPanel + + demoPlotDataActionPanel(RemoveOutliersToolPanel, plotDataFunction=removeOutliersXY, data=_DEFAULT_DICT) diff --git a/pydatview/plugins/data_sampler.py b/pydatview/plugins/plotdata_sampler.py similarity index 55% rename from pydatview/plugins/data_sampler.py rename to pydatview/plugins/plotdata_sampler.py index f281ba4..4ceb6e9 100644 --- a/pydatview/plugins/data_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -1,9 +1,8 @@ import wx import numpy as np -from pydatview.GUITools import GUIToolPanel, TOOL_BORDER -from pydatview.common import CHAR, Error, Info, pretty_num_short -from pydatview.common import DummyMainFrame -from pydatview.plotdata import PlotData +from pydatview.GUITools import TOOL_BORDER +from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor +from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction # --------------------------------------------------------------------------------} # --- Data @@ -35,12 +34,13 @@ def samplerAction(label, mainframe=None, data=None): guiCallback = mainframe.redraw action = PlotDataAction( - name=label, + name = label, + tableFunctionAdd = samplerTabAdd, plotDataFunction = samplerXY, - guiEditorClass = SamplerToolPanel, - guiCallback = guiCallback, - data = data, - mainframe=mainframe + guiEditorClass = SamplerToolPanel, + guiCallback = guiCallback, + data = data, + mainframe = mainframe ) return action # --------------------------------------------------------------------------------} @@ -51,38 +51,26 @@ def samplerXY(x, y, opts): x_new, y_new = applySampler(x, y, opts) return x_new, y_new +def samplerTabAdd(tab, opts): + """ Apply action on a a table and return a new one with a new name + df_new, name_new = f(t, opts) + """ + return tab.applyResampling(opts['icol'], opts, bAdd=True) + # --------------------------------------------------------------------------------} # --- GUI to edit plugin and control the action # --------------------------------------------------------------------------------{ -class SamplerToolPanel(GUIToolPanel): - def __init__(self, parent, action=None): - GUIToolPanel.__init__(self, parent) - - # --- Creating "Fake data" for testing only! - if action is None: - print('[WARN] Calling GUI without an action! Creating one.') - mainframe = DummyMainFrame(parent) - action = binningAction(label='dummyAction', mainframe=mainframe) - +class SamplerToolPanel(PlotDataActionEditor): + + def __init__(self, parent, action, plotPanel, pipeLike, **kwargs): + PlotDataActionEditor.__init__(self, parent, action, plotPanel, pipeLike, tables=True) + # --- Data - self.parent = parent # parent is GUIPlotPanel - self.mainframe = action.mainframe - self.data = action.data - self.action = action from pydatview.tools.signal_analysis import SAMPLERS self._SAMPLERS_DEFAULT = SAMPLERS.copy() self._SAMPLERS_USER = SAMPLERS.copy() # --- GUI elements - self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) - self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) - self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun', self.onClear) - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot) - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply) - - self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) - self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS_DEFAULT], style=wx.CB_READONLY) self.lbNewX = wx.StaticText(self, -1, 'New x: ') @@ -91,14 +79,6 @@ def __init__(self, parent, action=None): self.textOldX.Enable(False) # --- Layout - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) - msizer = wx.FlexGridSizer(rows=2, cols=4, hgap=2, vgap=0) msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) @@ -110,21 +90,20 @@ def __init__(self, parent, action=None): msizer.Add(self.textNewX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) msizer.AddGrowableCol(3,1) - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + #self.sizer = wx.BoxSizer(wx.HORIZONTAL) + #self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) self.sizer.Add(msizer ,1, flag = wx.LEFT|wx.EXPAND ,border = TOOL_BORDER) self.SetSizer(self.sizer) # --- Events - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) self.cbMethods.Bind(wx.EVT_COMBOBOX, self.onMethodChange) - self.textNewX.Bind(wx.EVT_TEXT_ENTER,self.onParamChange) + self.textNewX.Bind(wx.EVT_TEXT_ENTER,self.onParamChangeEnter) # --- Init triggers self._Data2GUI() self.onMethodChange(init=True) self.onToggleApply(init=True) - self.updateTabList() + self.setCurrentX() # --- Implementation specific def getSamplerIndex(self, name): @@ -135,7 +114,7 @@ def getSamplerIndex(self, name): def setCurrentX(self, x=None): if x is None: - x= self.parent.plotData[0].x + x= self.plotPanel.plotData[0].x if len(x)<50: s=np.array2string(x, separator=', ') else: @@ -172,34 +151,6 @@ def onParamChange(self, event=None): self.parent.load_and_draw() # Data will change self.setCurrentX() - # --- Table related - def onTabChange(self,event=None): - #tabList = self.parent.selPanel.tabList - #iSel=self.cbTabs.GetSelection() - pass - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - try: - iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - except RuntimeError: - pass - - # --- External Calls - def cancelAction(self, redraw=True): - """ do cancel the action""" - self.btPlot.Enable(True) - self.btClear.Enable(True) - self.btApply.SetLabel(CHAR['cloud']+' Apply') - self.btApply.SetValue(False) - self.data['active'] = False - if redraw: - self.parent.load_and_draw() # Data will change based on plotData - # --- Fairly generic def _GUI2Data(self): iOpt = self.cbMethods.GetSelection() @@ -223,76 +174,21 @@ def _Data2GUI(self): param = self.data['param'] self.textNewX.SetValue(str(param).lstrip('[').rstrip(']')) - def onToggleApply(self, event=None, init=False): - """ - apply sampler based on GUI Data - """ - if not init: - self.data['active'] = not self.data['active'] - if self.data['active']: - self._GUI2Data() - self.btPlot.Enable(False) - self.btClear.Enable(False) - self.btApply.SetLabel(CHAR['sun']+' Clear') - self.btApply.SetValue(True) - # The action is now active we add it to the pipeline, unless it's already in it - if self.mainframe is not None: - self.mainframe.addAction(self.action, overwrite=True) - if not init: - self.parent.load_and_draw() # filter will be applied in plotData.py - else: - # We remove our action from the pipeline - if not init: - if self.mainframe is not None: - self.mainframe.removeAction(self.action) - self.cancelAction(redraw=not init) - + def onToggleApply(self, event=None, init=False): self.setCurrentX() + # Call parent class + PlotDataActionEditor.onToggleApply(self, event=event, init=init) - def onAdd(self,event=None): - iSel = self.cbTabs.GetSelection() - tabList = self.parent.selPanel.tabList - icol, colname = self.parent.selPanel.xCol - self._GUI2Data() - errors=[] - if iSel==0: - dfs, names, errors = tabList.applyResampling(icol, self.data, bAdd=True) - self.parent.addTables(dfs,names,bAdd=True) - else: - df, name = tabList[iSel-1].applyResampling(icol, self.data, bAdd=True) - self.parent.addTables([df],[name], bAdd=True) - self.updateTabList() - - if len(errors)>0: - raise Exception('Error: The resampling failed on some tables:\n\n'+'\n'.join(errors)) - - # We stop applying - self.onToggleApply() - def onPlot(self,event=None): - if len(self.parent.plotData)!=1: - Error(self,'Plotting only works for a single plot. Plot less data.') - return - self._GUI2Data() - PD = self.parent.plotData[0] - x_new, y_new = samplerXY(PD.x0, PD.y0, self.data) + def onAdd(self, event=None): + # TODO put this in GUI2Data??? + iSel = self.cbTabs.GetSelection() + icol, colname = self.plotPanel.selPanel.xCol + self.data['icol'] = icol - ax = self.parent.fig.axes[0] - PD_new = PlotData() - PD_new.fromXY(x_new, y_new) - self.parent.transformPlotData(PD_new) - ax.plot(PD_new.x, PD_new.y, '-') - self.setCurrentX(x_new) - - self.parent.canvas.draw() - - def onClear(self,event=None): - self.parent.load_and_draw() # Data will change - # Update Current X - self.setCurrentX() - # Update Table list - self.updateTabList() + # Call parent class + PlotDataActionEditor.onAdd(self) def onHelp(self,event=None): Info(self,"""Resampling. @@ -320,3 +216,8 @@ def onHelp(self,event=None): signals. This process might take some time. Select a table or choose all (default) """) + +if __name__ == '__main__': + from pydatview.plugins.plotdata_default_plugin import demoPlotDataActionPanel + + demoPlotDataActionPanel(SamplerToolPanel, plotDataFunction=samplerXY, data=_DEFAULT_DICT, tableFunctionAdd=samplerTabAdd) From 3fa5b6a2b118fe5a8af02877ce948bf5d42e5b08 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 23 Dec 2022 11:24:38 +0100 Subject: [PATCH 071/178] Action: adding mainframe to action... --- pydatview/GUIPlotPanel.py | 2 +- pydatview/plugins/data_mask.py | 20 +++++++++----------- pydatview/plugins/plotdata_binning.py | 8 +++----- pydatview/plugins/plotdata_default_plugin.py | 17 +++++++++-------- pydatview/plugins/plotdata_filter.py | 10 ++++------ pydatview/plugins/plotdata_removeOutliers.py | 6 ++---- pydatview/plugins/plotdata_sampler.py | 10 ++++------ 7 files changed, 32 insertions(+), 41 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 60e5d9d..4af9014 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -805,7 +805,7 @@ def showToolPanel(self, panelClass=None, panel=None, action=None): print('NOTE: calling a panel without action') self.toolPanel=panelClass(parent=self) # calling the panel constructor else: - self.toolPanel=panelClass(parent=self, action=action, plotPanel=self, pipeLike=self.pipeLike) # calling the panel constructor + self.toolPanel=panelClass(parent=self, action=action) # calling the panel constructor action.guiEditorObj = self.toolPanel self.toolSizer.Add(self.toolPanel, 0, wx.EXPAND|wx.ALL, 5) self.plotsizer.Layout() diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index ab0d613..3231b1a 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -16,7 +16,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def maskAction(label, mainframe=None, data=None): +def maskAction(label, mainframe, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -28,9 +28,7 @@ def maskAction(label, mainframe=None, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback=None - if mainframe is not None: - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw action = ReversibleTableAction( name=label, @@ -57,7 +55,7 @@ def removeMask(tab, data): # --- GUI to edit plugin and control a plot data action # --------------------------------------------------------------------------------{ class MaskToolPanel(GUIToolPanel): - def __init__(self, parent, action, plotPanel, pipeLike): + def __init__(self, parent, action): GUIToolPanel.__init__(self, parent) # --- Creating "Fake data" for testing only! @@ -68,15 +66,15 @@ def __init__(self, parent, action, plotPanel, pipeLike): # --- Data self.data = action.data self.action = action - self.plotPanel = plotPanel - self.pipeLike = pipeLike - self.tabList = plotPanel.selPanel.tabList # a bit unfortunate + self.plotPanel = action.mainframe.plotPanel + self.pipeLike = action.mainframe.plotPanel.pipeLike + self.tabList = action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate # --- Unfortunate data to remove/manage + self.addActionHandle = self.pipeLike.append + self.removeActionHandle = self.pipeLike.remove self.addTablesHandle = action.mainframe.load_dfs - self.addActionHandle = pipeLike.append - self.removeActionHandle = pipeLike.remove - self.redrawHandle = plotPanel.load_and_draw # or action.guiCallback + self.redrawHandle = action.mainframe.plotPanel.load_and_draw # or action.guiCallback # Register ourselves to the action to be safe self.action.guiEditorObj = self diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index ea94e7a..357b370 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -28,9 +28,7 @@ def binningAction(label, mainframe=None, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback=None - if mainframe is not None: - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw action = PlotDataAction( name=label, @@ -89,8 +87,8 @@ def binTabAdd(tab, data): # --------------------------------------------------------------------------------{ class BinningToolPanel(PlotDataActionEditor): - def __init__(self, parent, action, plotPanel, pipeLike, **kwargs): - PlotDataActionEditor.__init__(self, parent, action, plotPanel, pipeLike, tables=True) + def __init__(self, parent, action, **kwargs): + PlotDataActionEditor.__init__(self, parent, action, tables=True) # --- GUI elements self.scBins = wx.SpinCtrl(self, value='50', style=wx.TE_RIGHT, size=wx.Size(60,-1) ) diff --git a/pydatview/plugins/plotdata_default_plugin.py b/pydatview/plugins/plotdata_default_plugin.py index c72a43d..04bcf5f 100644 --- a/pydatview/plugins/plotdata_default_plugin.py +++ b/pydatview/plugins/plotdata_default_plugin.py @@ -12,7 +12,7 @@ # --------------------------------------------------------------------------------{ class PlotDataActionEditor(GUIToolPanel): - def __init__(self, parent, action, plotPanel, pipeLike, buttons=None, tables=True): + def __init__(self, parent, action, buttons=None, tables=True): """ """ GUIToolPanel.__init__(self, parent) @@ -20,15 +20,15 @@ def __init__(self, parent, action, plotPanel, pipeLike, buttons=None, tables=Tru # --- Data self.data = action.data self.action = action - self.plotPanel = plotPanel - self.pipeLike = pipeLike - self.tabList = plotPanel.selPanel.tabList # a bit unfortunate + self.plotPanel = action.mainframe.plotPanel # NOTE: we need mainframe if plotPlanel is respawned.. + self.pipeLike = action.mainframe.plotPanel.pipeLike + self.tabList = action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate # --- Unfortunate data to remove/manage + self.addActionHandle = self.pipeLike.append + self.removeActionHandle = self.pipeLike.remove self.addTablesHandle = action.mainframe.load_dfs - self.addActionHandle = pipeLike.append - self.removeActionHandle = pipeLike.remove - self.redrawHandle = plotPanel.load_and_draw # or action.guiCallback + self.redrawHandle = action.mainframe.plotPanel.load_and_draw # or action.guiCallback # Register ourselves to the action to be safe self.action.guiEditorObj = self @@ -249,6 +249,7 @@ def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableF # --- Dummy mainframe and action.. mainframe = DummyMainFrame(self) + mainframe.plotPanel = self.plotPanel guiCallback = self.plotPanel.load_and_draw action = PlotDataAction( name = 'Dummy Action', @@ -261,7 +262,7 @@ def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableF ) # --- Create main object to be tested - p = panelClass(self.plotPanel, action=action, plotPanel=self.plotPanel, pipeLike=pipeline, tables=False) + p = panelClass(self.plotPanel, action=action, tables=False) # --- Finalize GUI sizer = wx.BoxSizer(wx.HORIZONTAL) diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index 18b0604..a87506d 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -21,7 +21,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def filterAction(label, mainframe=None, data=None): +def filterAction(label, mainframe, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -33,9 +33,7 @@ def filterAction(label, mainframe=None, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback=None - if mainframe is not None: - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw action = PlotDataAction( name = label, @@ -67,8 +65,8 @@ def filterTabAdd(tab, opts): # --------------------------------------------------------------------------------{ class FilterToolPanel(PlotDataActionEditor): - def __init__(self, parent, action, plotPanel, pipeLike, **kwargs): - PlotDataActionEditor.__init__(self, parent, action, plotPanel, pipeLike, tables=True) + def __init__(self, parent, action, **kwargs): + PlotDataActionEditor.__init__(self, parent, action, tables=True) # --- Data from pydatview.tools.signal_analysis import FILTERS diff --git a/pydatview/plugins/plotdata_removeOutliers.py b/pydatview/plugins/plotdata_removeOutliers.py index 0ca09f9..7b138b1 100644 --- a/pydatview/plugins/plotdata_removeOutliers.py +++ b/pydatview/plugins/plotdata_removeOutliers.py @@ -15,7 +15,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def removeOutliersAction(label, mainframe=None, data=None): +def removeOutliersAction(label, mainframe, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -27,9 +27,7 @@ def removeOutliersAction(label, mainframe=None, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback=None - if mainframe is not None: - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw action = PlotDataAction( name = label, diff --git a/pydatview/plugins/plotdata_sampler.py b/pydatview/plugins/plotdata_sampler.py index 4ceb6e9..7530382 100644 --- a/pydatview/plugins/plotdata_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -17,7 +17,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def samplerAction(label, mainframe=None, data=None): +def samplerAction(label, mainframe, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -29,9 +29,7 @@ def samplerAction(label, mainframe=None, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback=None - if mainframe is not None: - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw action = PlotDataAction( name = label, @@ -62,8 +60,8 @@ def samplerTabAdd(tab, opts): # --------------------------------------------------------------------------------{ class SamplerToolPanel(PlotDataActionEditor): - def __init__(self, parent, action, plotPanel, pipeLike, **kwargs): - PlotDataActionEditor.__init__(self, parent, action, plotPanel, pipeLike, tables=True) + def __init__(self, parent, action, **kwargs): + PlotDataActionEditor.__init__(self, parent, action, tables=True) # --- Data from pydatview.tools.signal_analysis import SAMPLERS From b22631907d6350e478e2e9ec8a9c9e79b557a13c Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 26 Dec 2022 11:49:23 +0100 Subject: [PATCH 072/178] Pipeline: GUIPipeline opening action editor --- pydatview/GUICommon.py | 12 ++++ pydatview/GUIPipelinePanel.py | 51 ++++++++++----- pydatview/pipeline.py | 22 ++++--- pydatview/plotdata.py | 4 +- pydatview/plugins/data_mask.py | 17 +++-- pydatview/plugins/plotdata_binning.py | 25 ++++--- pydatview/plugins/plotdata_default_plugin.py | 33 ++++++---- pydatview/plugins/plotdata_filter.py | 69 +++++--------------- pydatview/plugins/plotdata_removeOutliers.py | 4 +- pydatview/plugins/plotdata_sampler.py | 61 ++++++++--------- pydatview/tools/signal_analysis.py | 12 ++-- 11 files changed, 164 insertions(+), 146 deletions(-) diff --git a/pydatview/GUICommon.py b/pydatview/GUICommon.py index 59470be..623da7d 100644 --- a/pydatview/GUICommon.py +++ b/pydatview/GUICommon.py @@ -41,6 +41,18 @@ def getMonoFont(widget): font.SetPointSize(_MONOFONTSIZE) return font + +class DummyPanel(wx.Panel): + def __init__(self, parent): + wx.Panel.__init__(self, parent, -1) + self.SetBackgroundColour((0,100,0)) + txt = wx.StaticText(self, -1, 'This is a dummy panel') + + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(txt, 0, wx.ALL , 10) + sizer.AddSpacer(3) + self.SetSizer(sizer) + # --------------------------------------------------------------------------------} # --- Helper functions # --------------------------------------------------------------------------------{ diff --git a/pydatview/GUIPipelinePanel.py b/pydatview/GUIPipelinePanel.py index a08cd22..f49b7a4 100644 --- a/pydatview/GUIPipelinePanel.py +++ b/pydatview/GUIPipelinePanel.py @@ -5,8 +5,6 @@ import wx.lib.agw.hyperlink as hl - - class ActionPanel(wx.Panel): def __init__(self, parent, action, style=wx.TAB_TRAVERSAL): wx.Panel.__init__(self, parent, -1, style=style) @@ -17,16 +15,17 @@ def __init__(self, parent, action, style=wx.TAB_TRAVERSAL): name = action.name # --- GUI - ##lko = wx.Button(self, wx.ID_ANY, name, style=wx.BU_EXACTFIT) - lko = wx.StaticText(self, -1, name) - ##lko = hl.HyperLinkCtrl(self, -1, name) - ##lko.AutoBrowse(False) - ##lko.SetUnderlines(False, False, False) - ##lko.SetColours(wx.BLACK, wx.BLACK, wx.BLACK) - ##lko.DoPopup(False) - ###lko.SetBold(True) - ##lko.SetToolTip(wx.ToolTip('Change "'+name+'"')) - ##lko.UpdateLink() # To update text properties + if action.guiEditorClass is None: + lko = wx.StaticText(self, -1, name) + else: + lko = hl.HyperLinkCtrl(self, -1, name) + lko.AutoBrowse(False) + lko.SetUnderlines(False, False, False) + lko.SetColours(wx.BLACK, wx.BLACK, wx.BLACK) + lko.DoPopup(False) + #lko.SetBold(True) + lko.SetToolTip(wx.ToolTip('Change "'+name+'"')) + lko.UpdateLink() # To update text properties lkc = hl.HyperLinkCtrl(self, -1, 'x') lkc.AutoBrowse(False) @@ -50,7 +49,9 @@ def __init__(self, parent, action, style=wx.TAB_TRAVERSAL): self.SetSizer(sizer) - self.Bind(hl.EVT_HYPERLINK_LEFT, lambda ev: parent.onCloseAction(ev, action, self) , lkc) + self.Bind(hl.EVT_HYPERLINK_LEFT, lambda ev: parent.onCloseAction(ev, action) , lkc) + if action.guiEditorClass is not None: + self.Bind(hl.EVT_HYPERLINK_LEFT, lambda ev: parent.onOpenAction(ev, action) , lko) def __repr__(self): s='\n'.format(self.action.name) @@ -136,7 +137,7 @@ def __init__(self, parent, data=None, tabList=None, style=wx.TAB_TRAVERSAL): def populate(self): # Delete everything in wrapSizer - self.wrapSizer.Clear(True) + self.wrapSizer.Clear(delete_windows=True) self.wrapSizer.Add(wx.StaticText(self, -1, ' '), wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL, 0) # Add all actions to panel @@ -161,8 +162,18 @@ def _deletePanel(self, action): self.wrapSizer.Layout() self.Sizer.Layout() - def onCloseAction(self, event, action=None, actionPanel=None): + def onCloseAction(self, event, action=None): self.remove(action, tabList=self.tabList) # TODO + try: + action.mainframe.plotPanel.removeTools() + except: + print('[FAIL] removing tool from plotPanel') + + def onOpenAction(self, event, action=None): + """ Opens the Action GUI Editor """ + event.Skip() + action.mainframe.plotPanel.showToolAction(action) + # --- Wrap the data class def apply(self, tablist, force=False, applyToAll=False): @@ -173,6 +184,7 @@ def apply(self, tablist, force=False, applyToAll=False): self.Sizer.Layout() def append(self, action, overwrite=True, apply=True, updateGUI=True, tabList=None): + i = self.index(action) if not overwrite: # Delete action is already present and if it's a "unique" action ac = self.find(action.name) @@ -185,7 +197,9 @@ def append(self, action, overwrite=True, apply=True, updateGUI=True, tabList=Non # Call parent class (data) Pipeline.append(self, action, overwrite=overwrite, apply=apply, updateGUI=updateGUI, tabList=tabList) # Add to GUI - self.populate() # NOTE: we populate because of the change of order between actionsData and actionsPlot.. + if i<0: + # NOTE: populating when "modifying" can result to some kind of segfault (likely due to the window deletion) + self.populate() # NOTE: we populate because of the change of order between actionsData and actionsPlot.. #self._addPanel(action) #self.Sizer.Layout() # Update list of errors @@ -197,7 +211,10 @@ def remove(self, action, silent=False, cancel=True, updateGUI=True, tabList=None # Call parent class (data) Pipeline.remove(self, action, cancel=cancel, updateGUI=updateGUI, tabList=tabList) # Remove From GUI - self._deletePanel(action) + try: + self._deletePanel(action) + except: + print('[FAIL] GUIPipeline: could not delete action') if action.removeNeedReload: if not silent: diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index 731161a..f9e0b0e 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -95,8 +95,10 @@ def applyAndAdd(self, tabList): def updateGUI(self): """ Typically called by a callee after append""" if self.guiCallback is not None: - print('>>> Action: Calling GUI callback, action', self.name) - self.guiCallback() + try: + self.guiCallback() + except: + print('[FAIL] Action: failed to call GUI callback, action', self.name) def __repr__(self): s=''.format(self.name) @@ -226,7 +228,7 @@ def collectErrors(self): # --- Behave like a list.. def append(self, action, overwrite=False, apply=True, updateGUI=True, tabList=None): i = self.index(action) - if i>=0 and not overwrite: + if i>=0 and overwrite: print('[Pipe] Not adding action, its already present') else: if action.onPlotData: @@ -247,6 +249,14 @@ def remove(self, a, cancel=True, updateGUI=True, tabList=None): """ NOTE: the action is removed, not deleted fully (it might be readded to the pipeline later) - If a GUI edtor is attached to this action, we make sure that it shows the action as cancelled """ + # Cancel the action in Editor + if a.guiEditorObj is not None: + try: + print('[Pipe] Canceling action in guiEditor because the action is removed') + a.guiEditorObj.cancelAction() # NOTE: should not trigger a plot + except: + print('[FAIL] Pipeline: Failed to call cancelAction() in GUI.') + try: i = self.actionsData.index(a) a = self.actionsData.pop(i) @@ -259,12 +269,6 @@ def remove(self, a, cancel=True, updateGUI=True, tabList=None): a.cancel(tabList) self.collectErrors() - # Cancel the action in Editor - print('>>> GUI EDITOR', a.guiEditorObj) - if a.guiEditorObj is not None: - print('[Pipe] Canceling action in guiEditor because the action is removed') - a.guiEditorObj.cancelAction() # NOTE: should not trigger a plot - # trigger GUI update (guiCallback) if updateGUI: a.updateGUI() diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 94d2b34..127b8f4 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -72,8 +72,8 @@ def fromXY(PD, x, y, sx='', sy=''): def _post_init(PD, pipeline=None): # --- Apply filters from pipeline on the fly - if pipeline is not None: - print('[PDat]', pipeline.__reprFilters__()) + #if pipeline is not None: + # print('[PDat]', pipeline.__reprFilters__()) if pipeline is not None: PD.x, PD.y = pipeline.applyOnPlotData(PD.x, PD.y, PD.tabID) # TODO pass the tabID diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index 3231b1a..1b2d3d1 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -66,15 +66,11 @@ def __init__(self, parent, action): # --- Data self.data = action.data self.action = action - self.plotPanel = action.mainframe.plotPanel - self.pipeLike = action.mainframe.plotPanel.pipeLike - self.tabList = action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate # --- Unfortunate data to remove/manage self.addActionHandle = self.pipeLike.append self.removeActionHandle = self.pipeLike.remove self.addTablesHandle = action.mainframe.load_dfs - self.redrawHandle = action.mainframe.plotPanel.load_and_draw # or action.guiCallback # Register ourselves to the action to be safe self.action.guiEditorObj = self @@ -124,6 +120,16 @@ def __init__(self, parent, action): self.onToggleApply(init=True) self.updateTabList() + # --- unfortunate data + @property + def plotPanel(self): return self.action.mainframe.plotPanel # NOTE: we need mainframe if plotPlanel is respawned.. + @property + def redrawHandle(self): return self.action.mainframe.plotPanel.load_and_draw # or action.guiCallback + @property + def pipeLike(self): return self.action.mainframe.plotPanel.pipeLike + @property + def tabList(self): return self.action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate + # --- Implementation specific def guessMask(self): cols=self.tabList[0].columns_clean @@ -196,6 +202,9 @@ def onToggleApply(self, event=None, init=False): if not init: self.data['active'] = not self.data['active'] + # Check if action is already in pipeline + i = self.pipeLike.index(self.action) + if self.data['active']: self._GUI2Data() # We update the GUI diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index 357b370..6ce208c 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -88,10 +88,10 @@ def binTabAdd(tab, data): class BinningToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): - PlotDataActionEditor.__init__(self, parent, action, tables=True) + PlotDataActionEditor.__init__(self, parent, action, tables=True, **kwargs) # --- GUI elements - self.scBins = wx.SpinCtrl(self, value='50', style=wx.TE_RIGHT, size=wx.Size(60,-1) ) + self.scBins = wx.SpinCtrl(self, value='50', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(60,-1) ) self.textXMin = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) self.textXMax = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) @@ -133,8 +133,9 @@ def __init__(self, parent, action, **kwargs): self.SetSizer(self.sizer) # --- Events + self.scBins.Bind (wx.EVT_SPINCTRL , self.onParamChangeArrow) self.scBins.Bind (wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow) - self.scBins.Bind (wx.EVT_TEXT_ENTER, self.onParamChangeEnter) + self.scBins.Bind (wx.EVT_TEXT_ENTER, self.onParamChangeEnter) self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) self.textXMin.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter) self.textXMax.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter) @@ -146,7 +147,6 @@ def __init__(self, parent, action, **kwargs): self.setXRange() self._Data2GUI() self.onToggleApply(init=True) - #self.updateTabList() # --- Implementation specific def reset(self, event=None): @@ -164,15 +164,22 @@ def setXRange(self, x=None): # --- Bindings for plot triggers on parameters changes def onParamChange(self, event=None): PlotDataActionEditor.onParamChange(self) - self.lbDX.SetLabel(pretty_num_short((self.data['xMax']- self.data['xMin'])/self.data['nBins'])) + data = {} + data = self._GUI2Data(data) + #if data['nBins']<3: + # self.scBins.SetValue(3) # NOTE: this does not work, we might need to access the text? + self.lbDX.SetLabel(pretty_num_short((data['xMax']- data['xMin'])/data['nBins'])) # --- Fairly generic - def _GUI2Data(self): + def _GUI2Data(self, data=None): + if data is None: + data = self.data def zero_if_empty(s): return 0 if len(s)==0 else s - self.data['nBins'] = int (self.scBins.Value) - self.data['xMin'] = float(zero_if_empty(self.textXMin.Value)) - self.data['xMax'] = float(zero_if_empty(self.textXMax.Value)) + data['nBins'] = int (self.scBins.Value) + data['xMin'] = float(zero_if_empty(self.textXMin.Value)) + data['xMax'] = float(zero_if_empty(self.textXMax.Value)) + return data def _Data2GUI(self): if self.data['active']: diff --git a/pydatview/plugins/plotdata_default_plugin.py b/pydatview/plugins/plotdata_default_plugin.py index 04bcf5f..d5664ac 100644 --- a/pydatview/plugins/plotdata_default_plugin.py +++ b/pydatview/plugins/plotdata_default_plugin.py @@ -20,15 +20,10 @@ def __init__(self, parent, action, buttons=None, tables=True): # --- Data self.data = action.data self.action = action - self.plotPanel = action.mainframe.plotPanel # NOTE: we need mainframe if plotPlanel is respawned.. - self.pipeLike = action.mainframe.plotPanel.pipeLike - self.tabList = action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate - # --- Unfortunate data to remove/manage self.addActionHandle = self.pipeLike.append self.removeActionHandle = self.pipeLike.remove self.addTablesHandle = action.mainframe.load_dfs - self.redrawHandle = action.mainframe.plotPanel.load_and_draw # or action.guiCallback # Register ourselves to the action to be safe self.action.guiEditorObj = self @@ -83,15 +78,22 @@ def __init__(self, parent, action, buttons=None, tables=True): self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) ## --- Init triggers - #self._Data2GUI() - #self.onSelectFilt() - #self.onToggleApply(init=True) #self.updateTabList() + + # --- unfortunate data + @property + def plotPanel(self): return self.action.mainframe.plotPanel # NOTE: we need mainframe if plotPlanel is respawned.. + @property + def redrawHandle(self): return self.action.mainframe.plotPanel.load_and_draw # or action.guiCallback + @property + def pipeLike(self): return self.action.mainframe.plotPanel.pipeLike + @property + def tabList(self): return self.action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate # --- Bindings for plot triggers on parameters changes def onParamChange(self, event=None): - self._GUI2Data() if self.data['active']: + self._GUI2Data() self.redrawHandle() def onParamChangeArrow(self, event): @@ -100,10 +102,9 @@ def onParamChangeArrow(self, event): def onParamChangeEnter(self, event): """ Action when the user presses Enter: we activate the action """ + self.onParamChange() if not self.data['active']: self.onToggleApply() - else: - self.onParamChange() event.Skip() def onParamChangeChar(self, event): @@ -161,16 +162,20 @@ def _Data2GUI(self): def onToggleApply(self, event=None, init=False): if not init: self.data['active'] = not self.data['active'] + + # Check if action is already in pipeline + i = self.pipeLike.index(self.action) if self.data['active']: self._GUI2Data() # We update the GUI - self.guiActionAppliedState() # TODO remove me + self.guiActionAppliedState() # Add action to pipeline, no need to apply it, update the GUI (redraw) - self.addActionHandle(self.action, overwrite=True, apply=False, updateGUI=True) + if i<0: + self.addActionHandle(self.action, overwrite=True, apply=False, updateGUI=True) else: if not init: - # Remove action from pipeline, no need to cancel it, update the GUI + # Remove action from pipeline, no need to cancel it, update the GUI (redraw) self.removeActionHandle(self.action, cancel=False, updateGUI=True) else: self.cancelAction() diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index a87506d..d551d27 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -66,7 +66,7 @@ def filterTabAdd(tab, opts): class FilterToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): - PlotDataActionEditor.__init__(self, parent, action, tables=True) + PlotDataActionEditor.__init__(self, parent, action, tables=True, **kwargs) # --- Data from pydatview.tools.signal_analysis import FILTERS @@ -123,53 +123,19 @@ def __init__(self, parent, action, **kwargs): # --- Implementation specific def onSelectFilt(self, event=None): """ Select the filter, but does not apply it to the plotData - parentFilt is unchanged - But if the parent already has + self.data should be unchanged! + Uses the action data only if the selection is the same. """ iFilt = self.cbFilters.GetSelection() - filt = self._FILTERS_USER[iFilt] - self.lbParamName.SetLabel(filt['paramName']+':') - #self.tParam.SetRange(filt['paramRange'][0], filt['paramRange'][1]) - # NOTE: if min value for range is not 0, the Ctrl prevents you to enter 0.01 - self.tParam.SetRange(0, filt['paramRange'][1]) - self.tParam.SetIncrement(filt['increment']) - self.tParam.SetDigits(filt['digits']) - - parentFilt=self.data - # Value - if type(parentFilt)==dict and parentFilt['name']==filt['name']: - self.tParam.SetValue(parentFilt['param']) - else: - self.tParam.SetValue(filt['param']) + opts = self._FILTERS_USER[iFilt] + # check if selection is the same as the one currently used + if self.data['name']==opts['name']: + opts['param'] = self.data['param'] + self._Data2GUI(opts) # Trigger plot if applied self.onParamChange(self) -# # --- Bindings for plot triggers on parameters changes -# def onParamChange(self, event=None): -# self._GUI2Data() -# if self.data['active']: -# self.parent.load_and_draw() # Data will change -# -# def onParamChangeArrow(self, event): -# self.onParamChange() -# event.Skip() -# -# def onParamChangeEnter(self, event): -# if not self.data['active']: -# self.onToggleApply() -# else: -# self.onParamChange() -# event.Skip() -# -# def onParamChangeChar(self, event): -# event.Skip() -# code = event.GetKeyCode() -# if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: -# #print(self.spintxt.Value) -# self.tParam.SetValue(self.spintxt.Value) -# self.onParamChangeEnter(event) -# - # --- External Calls + # --- External Calls def cancelAction(self): """ set the GUI state when the action is cancelled""" # Call parent class @@ -184,9 +150,7 @@ def guiActionAppliedState(self): # Call parent class PlotDataActionEditor.guiActionAppliedState(self) # Update GUI - self.lbInfo.SetLabel( - 'Filter is now applied on the fly. Change parameter live. Click "Clear" to stop. ' - ) + self.lbInfo.SetLabel( 'Filter is now applied on the fly. Change parameter live. Click "Clear" to stop. ') # --- Fairly generic def _GUI2Data(self): @@ -204,13 +168,16 @@ def _GUI2Data(self): return opt - def _Data2GUI(self): - self.lbParamName.SetLabel(self.data['paramName']+':') + def _Data2GUI(self, data=None): + if data is None: + data = self.data + self.lbParamName.SetLabel(data['paramName']+':') #self.tParam.SetRange(filt['paramRange'][0], filt['paramRange'][1]) # NOTE: if min value for range is not 0, the Ctrl prevents you to enter 0.01 - self.tParam.SetRange(0, self.data['paramRange'][1]) - self.tParam.SetIncrement(self.data['increment']) - self.tParam.SetDigits(self.data['digits']) + self.tParam.SetRange(0, data['paramRange'][1]) + self.tParam.SetIncrement(data['increment']) + self.tParam.SetDigits(data['digits']) + self.tParam.SetValue(self.data['param']) def onAdd(self, event=None): # TODO put this in GUI2Data??? diff --git a/pydatview/plugins/plotdata_removeOutliers.py b/pydatview/plugins/plotdata_removeOutliers.py index 7b138b1..9fe0240 100644 --- a/pydatview/plugins/plotdata_removeOutliers.py +++ b/pydatview/plugins/plotdata_removeOutliers.py @@ -54,8 +54,8 @@ def removeOutliersXY(x, y, opts): # --------------------------------------------------------------------------------{ class RemoveOutliersToolPanel(PlotDataActionEditor): - def __init__(self, parent, action, plotPanel, pipeLike, **kwargs): - PlotDataActionEditor.__init__(self, parent, action, plotPanel, pipeLike, tables=False, buttons=['']) + def __init__(self, parent, action, **kwargs): + PlotDataActionEditor.__init__(self, parent, action, tables=False, buttons=[''], **kwargs) # --- GUI elements #self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) diff --git a/pydatview/plugins/plotdata_sampler.py b/pydatview/plugins/plotdata_sampler.py index 7530382..8ed579a 100644 --- a/pydatview/plugins/plotdata_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -65,11 +65,10 @@ def __init__(self, parent, action, **kwargs): # --- Data from pydatview.tools.signal_analysis import SAMPLERS - self._SAMPLERS_DEFAULT = SAMPLERS.copy() self._SAMPLERS_USER = SAMPLERS.copy() # --- GUI elements - self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS_DEFAULT], style=wx.CB_READONLY) + self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS_USER], style=wx.CB_READONLY) self.lbNewX = wx.StaticText(self, -1, 'New x: ') self.textNewX = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER) @@ -110,17 +109,16 @@ def getSamplerIndex(self, name): return i return -1 - def setCurrentX(self, x=None): - if x is None: - x= self.plotPanel.plotData[0].x + def setCurrentX(self): + if len(self.plotPanel.plotData)==0: + return + x= np.array(self.plotPanel.plotData[0].x).astype(str) if len(x)<50: - s=np.array2string(x, separator=', ') + s=', '.join(x) else: - s =np.array2string(x[[0,1,2,3]], separator=', ') + s=', '.join(x[[0,1,2,3]]) s+=', ..., ' - s+=np.array2string(x[[-3,-2,-1]], separator=', ') - s=s.replace('[','').replace(']','').replace(' ','').replace(',',', ') - + s+=', '.join(x[[-3,-2,-1]]) self.textOldX.SetValue(s) def onMethodChange(self, event=None, init=True): @@ -129,23 +127,18 @@ def onMethodChange(self, event=None, init=True): But if the user already has some options, they are used """ iOpt = self.cbMethods.GetSelection() - opt_default = self._SAMPLERS_DEFAULT[iOpt] - opt_user = self._SAMPLERS_USER[iOpt] - self.lbNewX.SetLabel(opt_default['paramName']+':') - - # Value - if len(self.textNewX.Value)==0: - self.textNewX.SetValue(str(opt_user['param'])[1:-1]) - #if type(parentOpt)==dict: - # self.textNewX.SetValue(str(parentOpt['param'])[1:-1]) - #else: - # self.textNewX.SetValue(str(opt['param'])[2:-2]) + opts = self._SAMPLERS_USER[iOpt] + # check if selection is the same as the one currently used + if self.data['name'] == opts['name']: + opts['param'] = self.data['param'] + self._Data2GUI(opts) + # Trigger plot if applied self.onParamChange() # --- Bindings for plot triggers on parameters changes def onParamChange(self, event=None): - self._GUI2Data() if self.data['active']: + self._GUI2Data() self.parent.load_and_draw() # Data will change self.setCurrentX() @@ -156,27 +149,31 @@ def _GUI2Data(self): opt = self._SAMPLERS_USER[iOpt] s= self.textNewX.Value.strip().replace('[','').replace(']','') if len(s)>0: - if s.find(','): - opt['param']=np.array(s.split(',')).astype(float) + if s.find(',')>=0: + opt['param']=np.array([v for v in s.split(',') if len(v)>0]).astype(float) else: - opt['param']=np.array(s.split('')).astype(float) + opt['param']=np.array([v for v in s.split() if len(v)>0]).astype(float) # Then update our main data dictionary self.data.update(opt) return opt - def _Data2GUI(self): - i = self.getSamplerIndex(self.data['name']) + def _Data2GUI(self, data=None): + if data is None: + data = self.data + i = self.getSamplerIndex(data['name']) if i==-1: - raise Exception('Unknown sampling method ', self.data['name']) + raise Exception('Unknown sampling method ', data['name']) + self.lbNewX.SetLabel(data['paramName']+':') self.cbMethods.SetSelection(i) - param = self.data['param'] - self.textNewX.SetValue(str(param).lstrip('[').rstrip(']')) - + param = data['param'] + sParam = ', '.join(list(np.atleast_1d(param).astype(str))) + self.textNewX.SetValue(sParam) def onToggleApply(self, event=None, init=False): - self.setCurrentX() # Call parent class PlotDataActionEditor.onToggleApply(self, event=event, init=init) + # + self.setCurrentX() def onAdd(self, event=None): diff --git a/pydatview/tools/signal_analysis.py b/pydatview/tools/signal_analysis.py index e4ee470..bcf1d24 100644 --- a/pydatview/tools/signal_analysis.py +++ b/pydatview/tools/signal_analysis.py @@ -213,15 +213,15 @@ def applySampler(x_old, y_old, sampDict, df_old=None): raise Exception('Error: `dx` cannot be 0') if len(param)==1: # NOTE: not using min/max if data loops (like airfoil) - xmin = np.min(x_old) - xmax = np.max(x_old) + dx/2 + xmin = np.nanmin(x_old) + xmax = np.nanmax(x_old) + dx/2 elif len(param)==3: xmin = param[1] xmax = param[2] - if xmin is np.nan: - xmin = np.min(x_old) - if xmax is np.nan: - xmax = np.max(x_old) + dx/2 + if np.isnan(xmin): + xmin = np.nanmin(x_old) + if np.isnan(xmax): + xmax = np.nanmax(x_old) + dx/2 else: raise Exception('Error: the sampling parameters should be a list of three values `dx, xmin, xmax`') x_new = np.arange(xmin, xmax, dx) From 14eb37736ea0cfc2f830724fd52387638ce5dc8e Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 27 Dec 2022 11:07:12 +0100 Subject: [PATCH 073/178] Plugins: moving tools to plugins folder --- pydatview/GUIPlotPanel.py | 2 +- pydatview/plugins/__init__.py | 21 + pydatview/plugins/base_plugin.py | 52 ++ pydatview/plugins/data_mask.py | 2 +- pydatview/plugins/plotdata_binning.py | 2 +- pydatview/plugins/plotdata_default_plugin.py | 2 +- pydatview/plugins/plotdata_filter.py | 4 +- pydatview/plugins/plotdata_removeOutliers.py | 2 +- pydatview/plugins/plotdata_sampler.py | 2 +- .../tool_curvefitting.py} | 690 +++++++----------- pydatview/plugins/tool_logdec.py | 39 + pydatview/plugins/tool_radialavg.py | 81 ++ 12 files changed, 459 insertions(+), 440 deletions(-) create mode 100644 pydatview/plugins/base_plugin.py rename pydatview/{GUITools.py => plugins/tool_curvefitting.py} (60%) create mode 100644 pydatview/plugins/tool_logdec.py create mode 100644 pydatview/plugins/tool_radialavg.py diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 4af9014..ca061bd 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -783,7 +783,7 @@ def removeTools(self, event=None, Layout=True): self.plotsizer.Layout() def showTool(self, toolName=''): - from .GUITools import TOOLS + from pydatview.plugins import TOOLS if toolName in TOOLS.keys(): self.showToolPanel(panelClass=TOOLS[toolName]) else: diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 6bea115..c9aee60 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -44,6 +44,20 @@ def _data_standardizeUnitsWE(label, mainframe=None): return standardizeUnitsAction(label, mainframe, flavor='WE') +# --- Tools +def _tool_logdec(*args, **kwargs): + from .tool_logdec import LogDecToolPanel + return LogDecToolPanel(*args, **kwargs) + +def _tool_curvefitting(*args, **kwargs): + from .tool_curvefitting import CurveFitToolPanel + return CurveFitToolPanel(*args, **kwargs) + +def _tool_radialavg(*args, **kwargs): + from .tool_radialavg import RadialToolPanel + return RadialToolPanel(*args, **kwargs) + + dataPlugins=[ # Name/label , callback , is a Panel ('Mask' , _data_mask , True ), @@ -56,6 +70,13 @@ def _data_standardizeUnitsWE(label, mainframe=None): ] +TOOLS={ + 'LogDec': _tool_logdec, + 'FASTRadialAverage': _tool_radialavg, + 'CurveFitting': _tool_curvefitting, +} + + diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py new file mode 100644 index 0000000..7ab7ab2 --- /dev/null +++ b/pydatview/plugins/base_plugin.py @@ -0,0 +1,52 @@ +import wx +import numpy as np +from pydatview.common import CHAR, Error, Info, pretty_num_short +from pydatview.common import DummyMainFrame +# from pydatview.plotdata import PlotData +# from pydatview.pipeline import PlotDataAction + + +TOOL_BORDER=15 + +# --------------------------------------------------------------------------------} +# --- Default class for tools +# --------------------------------------------------------------------------------{ +class GUIToolPanel(wx.Panel): + def __init__(self, parent): + super(GUIToolPanel,self).__init__(parent) + self.parent = parent + + def destroyData(self): + if hasattr(self, 'action'): + if self.action is not None: + # cleanup action calls + self.action.guiEditorObj = None + self.action = None + + def destroy(self,event=None): # TODO rename to something less close to Destroy + self.destroyData() + self.parent.removeTools() + + def getBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): + if Type is not None: + label=CHAR[Type]+' '+label + + bt=wx.Button(par,wx.ID_ANY, label, style=wx.BU_EXACTFIT) + #try: + # if bitmap is not None: + # bt.SetBitmapLabel(wx.ArtProvider.GetBitmap(bitmap)) #,size=(12,12))) + # else: + #except: + # pass + if callback is not None: + par.Bind(wx.EVT_BUTTON, callback, bt) + return bt + + def getToggleBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): + if Type is not None: + label=CHAR[Type]+' '+label + bt=wx.ToggleButton(par,wx.ID_ANY, label, style=wx.BU_EXACTFIT) + if callback is not None: + par.Bind(wx.EVT_TOGGLEBUTTON, callback, bt) + return bt + diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index 1b2d3d1..aecb098 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -1,6 +1,6 @@ import wx import numpy as np -from pydatview.GUITools import GUIToolPanel, TOOL_BORDER +from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER from pydatview.common import CHAR, Error, Info, pretty_num_short from pydatview.common import DummyMainFrame from pydatview.pipeline import ReversibleTableAction diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index 6ce208c..36d9ae4 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -1,6 +1,6 @@ import wx import numpy as np -from pydatview.GUITools import TOOL_BORDER +from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor from pydatview.common import Error, Info, pretty_num_short from pydatview.pipeline import PlotDataAction diff --git a/pydatview/plugins/plotdata_default_plugin.py b/pydatview/plugins/plotdata_default_plugin.py index d5664ac..9c167e2 100644 --- a/pydatview/plugins/plotdata_default_plugin.py +++ b/pydatview/plugins/plotdata_default_plugin.py @@ -1,6 +1,6 @@ import wx import numpy as np -from pydatview.GUITools import GUIToolPanel, TOOL_BORDER +from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER from pydatview.common import CHAR, Error, Info, pretty_num_short from pydatview.common import DummyMainFrame from pydatview.plotdata import PlotData diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index d551d27..4ebd43a 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -1,6 +1,6 @@ import wx import numpy as np -from pydatview.GUITools import TOOL_BORDER +from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction @@ -159,7 +159,7 @@ def _GUI2Data(self): try: opt['param']=np.float(self.spintxt.Value) except: - print('[WARN] pyDatView: Issue on Mac: GUITools.py/_GUI2Data. Help needed.') + print('[WARN] pyDatView: Issue on Mac: plotdata_filter.py/_GUI2Data. Help needed.') opt['param']=np.float(self.tParam.Value) if opt['param']0: - raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) - else: - dfs, names = tabList[iSel-1].radialAvg(avgMethod,avgParam) - self.parent.addTables([dfs],[names], bAdd=True) - - self.updateTabList() - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - - -# --------------------------------------------------------------------------------} -# --- Curve Fitting -# --------------------------------------------------------------------------------{ -MODELS_EXAMPLE =[ - {'label':'User defined model', 'id':'eval:', - 'formula':'{a}*x**2 + {b}', - 'coeffs':None, - 'consts':None, - 'bounds':None }, - ] -MODELS_EXTRA =[ -# {'label':'Exponential decay', 'id':'eval:', -# 'formula':'{A}*exp(-{k}*x)+{B}', -# 'coeffs' :'k=1, A=1, B=0', -# 'consts' :None, -# 'bounds' :None}, -] - -class CurveFitToolPanel(GUIToolPanel): - def __init__(self, parent): - super(CurveFitToolPanel,self).__init__(parent) - - # Data - self.x = None - self.y_fit = None - - # GUI Objecst - btClose = self.getBtBitmap(self, 'Close','close', self.destroy) - btClear = self.getBtBitmap(self, 'Clear','sun', self.onClear) # DELETE - btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) - btCompFit = self.getBtBitmap(self, 'Fit','check', self.onCurveFit) - btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) - - boldFont = self.GetFont().Bold() - lbOutputs = wx.StaticText(self, -1, 'Outputs') - lbInputs = wx.StaticText(self, -1, 'Inputs ') - lbOutputs.SetFont(boldFont) - lbInputs.SetFont(boldFont) - - self.textFormula = wx.TextCtrl(self, wx.ID_ANY, '') - self.textGuess = wx.TextCtrl(self, wx.ID_ANY, '') - self.textBounds = wx.TextCtrl(self, wx.ID_ANY, '') - self.textConstants = wx.TextCtrl(self, wx.ID_ANY, '') - - self.textFormulaNum = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) - self.textCoeffs = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) - self.textInfo = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) - - - self.Models=copy.deepcopy(MODELS_EXAMPLE) + copy.deepcopy(FITTERS) + copy.deepcopy(MODELS) + copy.deepcopy(MODELS_EXTRA) - sModels=[d['label'] for d in self.Models] - - - self.cbModels = wx.ComboBox(self, choices=sModels, style=wx.CB_READONLY) - self.cbModels.SetSelection(0) - - btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) - btSizer.Add(btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btClear ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btAdd ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btCompFit ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btHelp ,0,flag = wx.ALL|wx.EXPAND, border = 1) - - inputSizer = wx.FlexGridSizer(rows=5, cols=2, hgap=0, vgap=0) - inputSizer.Add(lbInputs ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.cbModels ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.Add(wx.StaticText(self, -1, 'Formula:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.textFormula ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.Add(wx.StaticText(self, -1, 'Guess:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.textGuess ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.Add(wx.StaticText(self, -1, 'Bounds:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.textBounds ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.Add(wx.StaticText(self, -1, 'Constants:'),0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - inputSizer.Add(self.textConstants ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - inputSizer.AddGrowableCol(1,1) - - outputSizer = wx.FlexGridSizer(rows=5, cols=2, hgap=0, vgap=0) - outputSizer.Add(lbOutputs ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(wx.StaticText(self, -1, '') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(wx.StaticText(self, -1, 'Formula:'),0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(self.textFormulaNum ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - outputSizer.Add(wx.StaticText(self, -1, 'Parameters:') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(self.textCoeffs ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - outputSizer.Add(wx.StaticText(self, -1, 'Accuracy:') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) - outputSizer.Add(self.textInfo ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) - outputSizer.AddGrowableCol(1,0.5) - - horzSizer = wx.BoxSizer(wx.HORIZONTAL) - horzSizer.Add(inputSizer ,1.0, flag = wx.LEFT|wx.EXPAND,border = 2) - horzSizer.Add(outputSizer ,1.0, flag = wx.LEFT|wx.EXPAND,border = 9) - - vertSizer = wx.BoxSizer(wx.VERTICAL) -# vertSizer.Add(self.lbHelp ,0, flag = wx.LEFT ,border = 1) - vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) -# self.sizer.Add(vertSizerCB ,0, flag = wx.LEFT ,border = 1) - self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) - self.SetSizer(self.sizer) - - self.Bind(wx.EVT_COMBOBOX, self.onModelChange, self.cbModels) - - self.onModelChange() - - def onModelChange(self,event=None): - iModel = self.cbModels.GetSelection() - d = self.Models[iModel] - self.textFormula.SetEditable(True) - - if d['id'].find('fitter:')==0 : - self.textGuess.Enable(False) - self.textGuess.SetValue('') - self.textFormula.Enable(False) - self.textFormula.SetValue(d['formula']) - self.textBounds.Enable(False) - self.textBounds.SetValue('') - self.textConstants.Enable(True) - # NOTE: conversion to string works with list, and tuples, not numpy array - val = ', '.join([k+'='+str(v) for k,v in d['consts'].items()]) - self.textConstants.SetValue(val) - else: - # Formula - if d['id'].find('eval:')==0 : - self.textFormula.Enable(True) - self.textFormula.SetEditable(True) - else: - #self.textFormula.Enable(False) - self.textFormula.Enable(True) - self.textFormula.SetEditable(False) - self.textFormula.SetValue(d['formula']) - - # Guess - if d['coeffs'] is None: - self.textGuess.SetValue('') - else: - self.textGuess.SetValue(d['coeffs']) - - # Constants - if d['consts'] is None or len(d['consts'].strip())==0: - self.textConstants.Enable(False) - self.textConstants.SetValue('') - else: - self.textConstants.Enable(True) - self.textConstants.SetValue(d['consts']) - - # Bounds - self.textBounds.Enable(True) - if d['bounds'] is None or len(d['bounds'].strip())==0: - self.textBounds.SetValue('all=(-np.inf, np.inf)') - else: - self.textBounds.SetValue(d['bounds']) - - # Outputs - self.textFormulaNum.SetValue('(Click on Fit)') - self.textCoeffs.SetValue('') - self.textInfo.SetValue('') - - def onCurveFit(self,event=None): - self.x = None - self.y_fit = None - if len(self.parent.plotData)!=1: - Error(self,'Curve fitting tool only works with a single curve. Plot less data.') - return - PD =self.parent.plotData[0] - ax =self.parent.fig.axes[0] - # Restricting data to axes visible bounds on the x axis - xlim= ax.get_xlim() - b=np.logical_and(PD.x>=xlim[0], PD.x<=xlim[1]) - - iModel = self.cbModels.GetSelection() - d = self.Models[iModel] - - if d['id'].find('fitter:')==0 : - sFunc=d['id'] - p0=None - bounds=None - fun_kwargs=extract_key_miscnum(self.textConstants.GetLineText(0).replace('np.inf','inf')) - else: - # Formula - sFunc=d['id'] - if sFunc=='eval:': - sFunc+=self.textFormula.GetLineText(0) - # Bounds - bounds=self.textBounds.GetLineText(0).replace('np.inf','inf') - # Guess - p0=self.textGuess.GetLineText(0).replace('np.inf','inf') - fun_kwargs=extract_key_num(self.textConstants.GetLineText(0).replace('np.inf','inf')) - #print('>>> Model fit sFunc :',sFunc ) - #print('>>> Model fit p0 :',p0 ) - #print('>>> Model fit bounds:',bounds ) - #print('>>> Model fit kwargs:',fun_kwargs) - # Performing fit - y_fit, pfit, fitter = model_fit(sFunc, PD.x[b], PD.y[b], p0=p0, bounds=bounds,**fun_kwargs) - - formatter = lambda x: pretty_num_short(x, digits=3) - formula_num = fitter.formula_num(fmt=formatter) - # Update info - self.textFormulaNum.SetValue(formula_num) - self.textCoeffs.SetValue(', '.join(['{}={:s}'.format(k,formatter(v)) for k,v in fitter.model['coeffs'].items()])) - self.textInfo.SetValue('R2 = {:.3f} '.format(fitter.model['R2'])) - - # Saving - d['formula'] = self.textFormula.GetLineText(0) - d['bounds'] = self.textBounds.GetLineText(0).replace('np.inf','inf') - d['coeffs'] = self.textGuess.GetLineText(0).replace('np.inf','inf') - if d['id'].find('fitter:')==0 : - d['consts'], _ = set_common_keys(d['consts'],fun_kwargs) - else: - d['consts']= self.textConstants.GetLineText(0).replace('np.inf','inf') - - - # Plot - ax=self.parent.fig.axes[0] - ax.plot(PD.x[b],y_fit,'o', ms=4) - self.parent.canvas.draw() - - self.x=PD.x[b] - self.y_fit=y_fit - self.sx=PD.sx - self.sy=PD.sy - - def onClear(self,event=None): - self.parent.load_and_draw() # DATA HAS CHANGED - self.onModelChange() - - def onAdd(self,event=None): - name='model_fit' - if self.x is not None and self.y_fit is not None: - df=pd.DataFrame({self.sx:self.x, self.sy:self.y_fit}) - self.parent.addTables([df], [name], bAdd=True) - - def onHelp(self,event=None): - Info(self,"""Curve fitting is still in beta. - -To perform a curve fit, adjusts the "Inputs section on the left": -- Select a predefined equation to fit, using the scrolldown menu. -- Adjust the initial gues for the parameters (if wanted) -- (Only for few models: set constants values) -- Click on "Fit" - -If you select a user-defined model: -- Equation parameters are specified using curly brackets -- Numpy functions are available using "np" - -Buttons: -- Clear: remove the fit from the plot -- Add: add the fit data to the list of tables (can then be exported) - -""") - - - -TOOLS={ - 'LogDec': LogDecToolPanel, - 'FASTRadialAverage': RadialToolPanel, - 'CurveFitting': CurveFitToolPanel, -} +import wx +import numpy as np +import copy +from pydatview.plugins.base_plugin import GUIToolPanel +from pydatview.common import Error, Info, pretty_num_short +from pydatview.tools.curve_fitting import model_fit, extract_key_miscnum, extract_key_num, MODELS, FITTERS, set_common_keys + + +# --------------------------------------------------------------------------------} +# --- Curve Fitting +# --------------------------------------------------------------------------------{ +MODELS_EXAMPLE =[ + {'label':'User defined model', 'id':'eval:', + 'formula':'{a}*x**2 + {b}', + 'coeffs':None, + 'consts':None, + 'bounds':None }, + ] +MODELS_EXTRA =[ +# {'label':'Exponential decay', 'id':'eval:', +# 'formula':'{A}*exp(-{k}*x)+{B}', +# 'coeffs' :'k=1, A=1, B=0', +# 'consts' :None, +# 'bounds' :None}, +] + +class CurveFitToolPanel(GUIToolPanel): + def __init__(self, parent): + super(CurveFitToolPanel,self).__init__(parent) + + # Data + self.x = None + self.y_fit = None + + # GUI Objecst + btClose = self.getBtBitmap(self, 'Close','close', self.destroy) + btClear = self.getBtBitmap(self, 'Clear','sun', self.onClear) # DELETE + btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd) + btCompFit = self.getBtBitmap(self, 'Fit','check', self.onCurveFit) + btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) + + boldFont = self.GetFont().Bold() + lbOutputs = wx.StaticText(self, -1, 'Outputs') + lbInputs = wx.StaticText(self, -1, 'Inputs ') + lbOutputs.SetFont(boldFont) + lbInputs.SetFont(boldFont) + + self.textFormula = wx.TextCtrl(self, wx.ID_ANY, '') + self.textGuess = wx.TextCtrl(self, wx.ID_ANY, '') + self.textBounds = wx.TextCtrl(self, wx.ID_ANY, '') + self.textConstants = wx.TextCtrl(self, wx.ID_ANY, '') + + self.textFormulaNum = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) + self.textCoeffs = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) + self.textInfo = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) + + + self.Models=copy.deepcopy(MODELS_EXAMPLE) + copy.deepcopy(FITTERS) + copy.deepcopy(MODELS) + copy.deepcopy(MODELS_EXTRA) + sModels=[d['label'] for d in self.Models] + + + self.cbModels = wx.ComboBox(self, choices=sModels, style=wx.CB_READONLY) + self.cbModels.SetSelection(0) + + btSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) + btSizer.Add(btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btClear ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btAdd ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btCompFit ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btHelp ,0,flag = wx.ALL|wx.EXPAND, border = 1) + + inputSizer = wx.FlexGridSizer(rows=5, cols=2, hgap=0, vgap=0) + inputSizer.Add(lbInputs ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.cbModels ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.Add(wx.StaticText(self, -1, 'Formula:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.textFormula ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.Add(wx.StaticText(self, -1, 'Guess:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.textGuess ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.Add(wx.StaticText(self, -1, 'Bounds:') ,0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.textBounds ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.Add(wx.StaticText(self, -1, 'Constants:'),0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + inputSizer.Add(self.textConstants ,1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + inputSizer.AddGrowableCol(1,1) + + outputSizer = wx.FlexGridSizer(rows=5, cols=2, hgap=0, vgap=0) + outputSizer.Add(lbOutputs ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(wx.StaticText(self, -1, '') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(wx.StaticText(self, -1, 'Formula:'),0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(self.textFormulaNum ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + outputSizer.Add(wx.StaticText(self, -1, 'Parameters:') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(self.textCoeffs ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + outputSizer.Add(wx.StaticText(self, -1, 'Accuracy:') ,0 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + outputSizer.Add(self.textInfo ,1 , flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + outputSizer.AddGrowableCol(1,0.5) + + horzSizer = wx.BoxSizer(wx.HORIZONTAL) + horzSizer.Add(inputSizer ,1.0, flag = wx.LEFT|wx.EXPAND,border = 2) + horzSizer.Add(outputSizer ,1.0, flag = wx.LEFT|wx.EXPAND,border = 9) + + vertSizer = wx.BoxSizer(wx.VERTICAL) +# vertSizer.Add(self.lbHelp ,0, flag = wx.LEFT ,border = 1) + vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) +# self.sizer.Add(vertSizerCB ,0, flag = wx.LEFT ,border = 1) + self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) + self.SetSizer(self.sizer) + + self.Bind(wx.EVT_COMBOBOX, self.onModelChange, self.cbModels) + + self.onModelChange() + + def onModelChange(self,event=None): + iModel = self.cbModels.GetSelection() + d = self.Models[iModel] + self.textFormula.SetEditable(True) + + if d['id'].find('fitter:')==0 : + self.textGuess.Enable(False) + self.textGuess.SetValue('') + self.textFormula.Enable(False) + self.textFormula.SetValue(d['formula']) + self.textBounds.Enable(False) + self.textBounds.SetValue('') + self.textConstants.Enable(True) + # NOTE: conversion to string works with list, and tuples, not numpy array + val = ', '.join([k+'='+str(v) for k,v in d['consts'].items()]) + self.textConstants.SetValue(val) + else: + # Formula + if d['id'].find('eval:')==0 : + self.textFormula.Enable(True) + self.textFormula.SetEditable(True) + else: + #self.textFormula.Enable(False) + self.textFormula.Enable(True) + self.textFormula.SetEditable(False) + self.textFormula.SetValue(d['formula']) + + # Guess + if d['coeffs'] is None: + self.textGuess.SetValue('') + else: + self.textGuess.SetValue(d['coeffs']) + + # Constants + if d['consts'] is None or len(d['consts'].strip())==0: + self.textConstants.Enable(False) + self.textConstants.SetValue('') + else: + self.textConstants.Enable(True) + self.textConstants.SetValue(d['consts']) + + # Bounds + self.textBounds.Enable(True) + if d['bounds'] is None or len(d['bounds'].strip())==0: + self.textBounds.SetValue('all=(-np.inf, np.inf)') + else: + self.textBounds.SetValue(d['bounds']) + + # Outputs + self.textFormulaNum.SetValue('(Click on Fit)') + self.textCoeffs.SetValue('') + self.textInfo.SetValue('') + + def onCurveFit(self,event=None): + self.x = None + self.y_fit = None + if len(self.parent.plotData)!=1: + Error(self,'Curve fitting tool only works with a single curve. Plot less data.') + return + PD =self.parent.plotData[0] + ax =self.parent.fig.axes[0] + # Restricting data to axes visible bounds on the x axis + xlim= ax.get_xlim() + b=np.logical_and(PD.x>=xlim[0], PD.x<=xlim[1]) + + iModel = self.cbModels.GetSelection() + d = self.Models[iModel] + + if d['id'].find('fitter:')==0 : + sFunc=d['id'] + p0=None + bounds=None + fun_kwargs=extract_key_miscnum(self.textConstants.GetLineText(0).replace('np.inf','inf')) + else: + # Formula + sFunc=d['id'] + if sFunc=='eval:': + sFunc+=self.textFormula.GetLineText(0) + # Bounds + bounds=self.textBounds.GetLineText(0).replace('np.inf','inf') + # Guess + p0=self.textGuess.GetLineText(0).replace('np.inf','inf') + fun_kwargs=extract_key_num(self.textConstants.GetLineText(0).replace('np.inf','inf')) + #print('>>> Model fit sFunc :',sFunc ) + #print('>>> Model fit p0 :',p0 ) + #print('>>> Model fit bounds:',bounds ) + #print('>>> Model fit kwargs:',fun_kwargs) + # Performing fit + y_fit, pfit, fitter = model_fit(sFunc, PD.x[b], PD.y[b], p0=p0, bounds=bounds,**fun_kwargs) + + formatter = lambda x: pretty_num_short(x, digits=3) + formula_num = fitter.formula_num(fmt=formatter) + # Update info + self.textFormulaNum.SetValue(formula_num) + self.textCoeffs.SetValue(', '.join(['{}={:s}'.format(k,formatter(v)) for k,v in fitter.model['coeffs'].items()])) + self.textInfo.SetValue('R2 = {:.3f} '.format(fitter.model['R2'])) + + # Saving + d['formula'] = self.textFormula.GetLineText(0) + d['bounds'] = self.textBounds.GetLineText(0).replace('np.inf','inf') + d['coeffs'] = self.textGuess.GetLineText(0).replace('np.inf','inf') + if d['id'].find('fitter:')==0 : + d['consts'], _ = set_common_keys(d['consts'],fun_kwargs) + else: + d['consts']= self.textConstants.GetLineText(0).replace('np.inf','inf') + + + # Plot + ax=self.parent.fig.axes[0] + ax.plot(PD.x[b],y_fit,'o', ms=4) + self.parent.canvas.draw() + + self.x=PD.x[b] + self.y_fit=y_fit + self.sx=PD.sx + self.sy=PD.sy + + def onClear(self,event=None): + self.parent.load_and_draw() # DATA HAS CHANGED + self.onModelChange() + + def onAdd(self,event=None): + name='model_fit' + if self.x is not None and self.y_fit is not None: + df=pd.DataFrame({self.sx:self.x, self.sy:self.y_fit}) + self.parent.addTables([df], [name], bAdd=True) + + def onHelp(self,event=None): + Info(self,"""Curve fitting is still in beta. + +To perform a curve fit, adjusts the "Inputs section on the left": +- Select a predefined equation to fit, using the scrolldown menu. +- Adjust the initial gues for the parameters (if wanted) +- (Only for few models: set constants values) +- Click on "Fit" + +If you select a user-defined model: +- Equation parameters are specified using curly brackets +- Numpy functions are available using "np" + +Buttons: +- Clear: remove the fit from the plot +- Add: add the fit data to the list of tables (can then be exported) + +""") diff --git a/pydatview/plugins/tool_logdec.py b/pydatview/plugins/tool_logdec.py new file mode 100644 index 0000000..815cf64 --- /dev/null +++ b/pydatview/plugins/tool_logdec.py @@ -0,0 +1,39 @@ +import wx +from pydatview.plugins.base_plugin import GUIToolPanel +from pydatview.tools.damping import logDecFromDecay + +# --------------------------------------------------------------------------------} +# --- Log Dec +# --------------------------------------------------------------------------------{ +class LogDecToolPanel(GUIToolPanel): + def __init__(self, parent): + super(LogDecToolPanel,self).__init__(parent) + btClose = self.getBtBitmap(self,'Close' ,'close' ,self.destroy ) + btComp = self.getBtBitmap(self,'Compute','compute',self.onCompute) + self.lb = wx.StaticText( self, -1, ' ') + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btClose ,0, flag = wx.LEFT|wx.CENTER,border = 1) + self.sizer.Add(btComp ,0, flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(self.lb ,0, flag = wx.LEFT|wx.CENTER,border = 5) + self.SetSizer(self.sizer) + + def onCompute(self,event=None): + if len(self.parent.plotData)!=1: + Error(self,'Log Dec tool only works with a single plot.') + return + pd =self.parent.plotData[0] + try: + logdec,DampingRatio,T,fn,fd,IPos,INeg,epos,eneg=logDecFromDecay(pd.y,pd.x) + lab='LogDec.: {:.4f} - Damping ratio: {:.4f} - F_n: {:.4f} - F_d: {:.4f} - T:{:.3f}'.format(logdec,DampingRatio,fn,fd,T) + self.lb.SetLabel(lab) + self.sizer.Layout() + ax=self.parent.fig.axes[0] + ax.plot(pd.x[IPos],pd.y[IPos],'o') + ax.plot(pd.x[INeg],pd.y[INeg],'o') + ax.plot(pd.x ,epos,'k--') + ax.plot(pd.x ,eneg,'k--') + self.parent.canvas.draw() + except: + self.lb.SetLabel('Failed. The signal needs to look like the decay of a first order system.') + #self.parent.load_and_draw(); # DATA HAS CHANGED + diff --git a/pydatview/plugins/tool_radialavg.py b/pydatview/plugins/tool_radialavg.py new file mode 100644 index 0000000..f8a4494 --- /dev/null +++ b/pydatview/plugins/tool_radialavg.py @@ -0,0 +1,81 @@ +import wx +import numpy as np +from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER +# --------------------------------------------------------------------------------} +# --- Radial +# --------------------------------------------------------------------------------{ +sAVG_METHODS = ['Last `n` seconds','Last `n` periods'] +AVG_METHODS = ['constantwindow','periods'] + +class RadialToolPanel(GUIToolPanel): + def __init__(self, parent): + super(RadialToolPanel,self).__init__(parent) + + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + + # --- GUI + btClose = self.getBtBitmap(self,'Close' ,'close' , self.destroy) + btComp = self.getBtBitmap(self,'Average','compute', self.onApply) # ART_PLUS + + self.lb = wx.StaticText( self, -1, """Select tables, averaging method and average parameter (`Period` methods uses the `azimuth` signal) """) + self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) + self.cbMethod = wx.ComboBox(self, choices=sAVG_METHODS, style=wx.CB_READONLY) + self.cbTabs.SetSelection(0) + self.cbMethod.SetSelection(0) + + self.textAverageParam = wx.TextCtrl(self, wx.ID_ANY, '2',size = (36,-1), style=wx.TE_PROCESS_ENTER) + + btSizer = wx.FlexGridSizer(rows=2, cols=1, hgap=0, vgap=0) + #btSizer = wx.BoxSizer(wx.VERTICAL) + btSizer.Add(btClose ,0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(btComp ,0, flag = wx.ALL|wx.EXPAND, border = 1) + + row_sizer = wx.BoxSizer(wx.HORIZONTAL) + row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) + row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Method:'), 0, wx.CENTER|wx.LEFT, 5) + row_sizer.Add(self.cbMethod , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Param:') , 0, wx.CENTER|wx.LEFT, 5) + row_sizer.Add(self.textAverageParam , 0, wx.CENTER|wx.LEFT|wx.RIGHT| wx.EXPAND, 2) + + vert_sizer = wx.BoxSizer(wx.VERTICAL) + vert_sizer.Add(self.lb ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) + vert_sizer.Add(row_sizer ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + self.sizer.Add(vert_sizer ,0, flag = wx.LEFT|wx.EXPAND,border = TOOL_BORDER) + self.SetSizer(self.sizer) + self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) + + def onTabChange(self,event=None): + tabList = self.parent.selPanel.tabList + + def onApply(self,event=None): + try: + avgParam = float(self.textAverageParam.GetLineText(0)) + except: + raise Exception('Error: the averaging parameter needs to be an integer or a float') + iSel = self.cbTabs.GetSelection() + avgMethod = AVG_METHODS[self.cbMethod.GetSelection()] + tabList = self.parent.selPanel.tabList + if iSel==0: + dfs, names, errors = tabList.radialAvg(avgMethod,avgParam) + self.parent.addTables(dfs,names,bAdd=True) + if len(errors)>0: + raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) + else: + dfs, names = tabList[iSel-1].radialAvg(avgMethod,avgParam) + self.parent.addTables([dfs],[names], bAdd=True) + + self.updateTabList() + + def updateTabList(self,event=None): + tabList = self.parent.selPanel.tabList + tabListNames = ['All opened tables']+tabList.getDisplayTabNames() + iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + From 6399a28606e4cea7304137f39d08319f657eac17 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 27 Dec 2022 12:21:17 +0100 Subject: [PATCH 074/178] Plugins: introducing more base classes and starting to unify tools and data plugins --- pydatview/GUIPlotPanel.py | 5 +- pydatview/Tables.py | 7 +- pydatview/main.py | 62 ++-- pydatview/pipeline.py | 2 + pydatview/plugins/__init__.py | 58 +-- pydatview/plugins/base_plugin.py | 345 +++++++++++++++++- pydatview/plugins/data_mask.py | 83 +---- pydatview/plugins/plotdata_binning.py | 5 +- pydatview/plugins/plotdata_default_plugin.py | 287 --------------- pydatview/plugins/plotdata_filter.py | 5 +- pydatview/plugins/plotdata_removeOutliers.py | 5 +- pydatview/plugins/plotdata_sampler.py | 5 +- .../plugins/tests/test_standardizeUnits.py | 2 +- tests/test_pipeline.py | 15 +- 14 files changed, 447 insertions(+), 439 deletions(-) delete mode 100644 pydatview/plugins/plotdata_default_plugin.py diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index ca061bd..7d3e470 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -775,7 +775,6 @@ def formatLabelValue(self, value): return s def removeTools(self, event=None, Layout=True): - if self.toolPanel is not None: self.toolPanel.destroyData() # clean destroy of data (action callbacks) self.toolSizer.Clear(delete_windows=True) # Delete Windows @@ -784,8 +783,11 @@ def removeTools(self, event=None, Layout=True): def showTool(self, toolName=''): from pydatview.plugins import TOOLS + from pydatview.plugins import DATA_TOOLS # TODO remove me if toolName in TOOLS.keys(): self.showToolPanel(panelClass=TOOLS[toolName]) + elif toolName in DATA_TOOLS.keys(): + self.showToolPanel(panelClass=DATA_TOOLS[toolName]) else: raise Exception('Unknown tool {}'.format(toolName)) @@ -793,7 +795,6 @@ def showToolAction(self, action): """ Show a tool panel based on an action""" self.showToolPanel(panelClass=action.guiEditorClass, action=action) - def showToolPanel(self, panelClass=None, panel=None, action=None): """ Show a tool panel based on a panel class (should inherit from GUIToolPanel)""" self.Freeze() diff --git a/pydatview/Tables.py b/pydatview/Tables.py index c6c43c6..95b2fe1 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -511,6 +511,7 @@ def __repr__(self): s+=' - fileformat_name : {}\n'.format(self.fileformat_name) s+=' - columns : {}\n'.format(self.columns) s+=' - nCols x nRows: {}x{}\n'.format(self.nCols, self.nRows) + s+=' - maskString: {}\n'.format(self.maskString) return s def columnsFromDF(self,df): @@ -624,11 +625,13 @@ def radialAvg(self,avgMethod, avgParam): names_new=[self.raw_name+'_AD', self.raw_name+'_ED', self.raw_name+'_BD'] return dfs_new, names_new - def changeUnits(self, flavor='WE'): + def changeUnits(self, data=None): """ Change units of the table """ + if data is None: + data={'flavor':'WE'} # NOTE: moved to a plugin, but interface kept from pydatview.plugins.data_standardizeUnits import changeUnits - changeUnits(self, flavor=flavor) + changeUnits(self, data=data) def convertTimeColumns(self, dayfirst=False): if len(self.data)>0: diff --git a/pydatview/main.py b/pydatview/main.py index f677e76..9c8ab80 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -32,7 +32,7 @@ from .GUICommon import * import pydatview.io as weio # File Formats and File Readers # Pluggins -from .plugins import dataPlugins +from .plugins import DATA_PLUGINS_WITH_EDITOR, DATA_PLUGINS_SIMPLE, DATA_TOOLS, TOOLS from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData # --------------------------------------------------------------------------------} @@ -140,18 +140,24 @@ def __init__(self, data=None): self.Bind(wx.EVT_MENU,self.onExport,exptMenuItem) self.Bind(wx.EVT_MENU,self.onSave ,saveMenuItem) + # --- Data Plugins + # NOTE: very important, need "s_loc" otherwise the lambda function take the last toolName dataMenu = wx.Menu() menuBar.Append(dataMenu, "&Data") - self.Bind(wx.EVT_MENU, lambda e: self.onShowTool(e,'FASTRadialAverage'), dataMenu.Append(wx.ID_ANY, 'FAST - Radial average')) + for toolName in DATA_TOOLS.keys(): # TODO remove me, should be an action + self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onShowTool(e, s_loc), dataMenu.Append(wx.ID_ANY, toolName)) - # --- Data Plugins - for string, function, isPanel in dataPlugins: - self.Bind(wx.EVT_MENU, lambda e, s_loc=string: self.onDataPlugin(e, s_loc), dataMenu.Append(wx.ID_ANY, string)) + for toolName in DATA_PLUGINS_WITH_EDITOR.keys(): + self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onDataPlugin(e, s_loc), dataMenu.Append(wx.ID_ANY, toolName)) + for toolName in DATA_PLUGINS_SIMPLE.keys(): + self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onDataPlugin(e, s_loc), dataMenu.Append(wx.ID_ANY, toolName)) + + # --- Tools Plugins toolMenu = wx.Menu() menuBar.Append(toolMenu, "&Tools") - self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'CurveFitting'), toolMenu.Append(wx.ID_ANY, 'Curve fitting')) - self.Bind(wx.EVT_MENU,lambda e: self.onShowTool(e, 'LogDec') , toolMenu.Append(wx.ID_ANY, 'Damping from decay')) + for toolName in TOOLS.keys(): + self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onShowTool(e, s_loc), toolMenu.Append(wx.ID_ANY, toolName)) helpMenu = wx.Menu() aboutMenuItem = helpMenu.Append(wx.NewId(), 'About', 'About') @@ -452,7 +458,7 @@ def exportTab(self, iTab): return # the user changed their mind tab.export(dlg.GetPath()) - def onShowTool(self, event=None, tool=''): + def onShowTool(self, event=None, toolName=''): """ Show tool tool in 'Outlier', 'Filter', 'LogDec','FASTRadialAverage', 'Mask', 'CurveFitting' @@ -460,7 +466,7 @@ def onShowTool(self, event=None, tool=''): if not hasattr(self,'plotPanel'): Error(self,'Plot some data first') return - self.plotPanel.showTool(tool) + self.plotPanel.showTool(toolName) def onDataPlugin(self, event=None, toolName=''): """ @@ -473,25 +479,25 @@ def onDataPlugin(self, event=None, toolName=''): Error(self,'Plot some data first') return - for thisToolName, function, isPanel in dataPlugins: - if toolName == thisToolName: - if isPanel: # This is more of a "hasPanel" - # Check to see if the pipeline already contains this action - action = self.pipePanel.find(toolName) # old action to edit - if action is None: - action = function(label=toolName, mainframe=self) # getting brand new action - else: - print('>>> The action already exists, we use it for the GUI') - self.plotPanel.showToolAction(action) - # The panel will have the responsibility to apply/delete the action, updateGUI, etc - else: - action = function(label=toolName, mainframe=self) # calling the data function - # Here we apply the action directly - # We can't overwrite, so we'll delete by name.. - self.addAction(action, overwrite=False, apply=True, tabList=self.tabList, updateGUI=True) - - return - raise NotImplementedError('Tool: ',toolName) + if toolName in DATA_PLUGINS_WITH_EDITOR.keys(): + # Check to see if the pipeline already contains this action + action = self.pipePanel.find(toolName) # old action to edit + if action is None: + function = DATA_PLUGINS_WITH_EDITOR[toolName] + action = function(label=toolName, mainframe=self) # getting brand new action + else: + print('>>> The action already exists, we use it for the GUI') + self.plotPanel.showToolAction(action) + # The panel will have the responsibility to apply/delete the action, updateGUI, etc + elif toolName in DATA_PLUGINS_SIMPLE.keys(): + print('>>> toolName') + function = DATA_PLUGINS_SIMPLE[toolName] + action = function(label=toolName, mainframe=self) # calling the data function + # Here we apply the action directly + # We can't overwrite, so we'll delete by name.. + self.addAction(action, overwrite=False, apply=True, tabList=self.tabList, updateGUI=True) + else: + raise NotImplementedError('Tool: ',toolName) # --- Pipeline def addAction(self, action, **kwargs): diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index f9e0b0e..91a792e 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -115,9 +115,11 @@ def __init__(self, name, **kwargs): Action.__init__(self, name, onPlotData=True, **kwargs) def apply(self, *args, **kwargs): + #print('[INFO] Action: Skipping apply (plotdata)') pass # nothing to do def cancel(self, *args, **kwargs): + #print('[INFO] Action: Skipping cancel (plotdata)') pass # nothing to do def applyOnPlotData(self, x, y, tabID): diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index c9aee60..1f8a38b 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -6,13 +6,25 @@ def _function_name(mainframe, event=None, label='') and call the main function of your package. Your are free to use the signature your want for your package - 2) add a tuple to the variable dataPlugins of the form + 2) add a tuple to the variable DATA_PLUGINS of the form (string, _function_name) where string will be displayed under the data menu of pyDatView. See working examples in this file and this directory. + +NOTE: + - data plugins constructors should return an Action with a GUI Editor class + + - simple data plugins constructors should return an Action + + - tool plugins constructor should return a Panel class + + """ +from collections import OrderedDict + + def _data_mask(label, mainframe): from .data_mask import maskAction return maskAction(label, mainframe) @@ -53,36 +65,32 @@ def _tool_curvefitting(*args, **kwargs): from .tool_curvefitting import CurveFitToolPanel return CurveFitToolPanel(*args, **kwargs) +# --- TODO Action def _tool_radialavg(*args, **kwargs): from .tool_radialavg import RadialToolPanel return RadialToolPanel(*args, **kwargs) +# --- Ordered dictionaries with key="Tool Name", value="Constructor" -dataPlugins=[ - # Name/label , callback , is a Panel - ('Mask' , _data_mask , True ), - ('Remove Outliers' , _data_removeOutliers , True ), - ('Filter' , _data_filter , True ), - ('Resample' , _data_sampler , True ), - ('Bin data' , _data_binning , True ), - ('Standardize Units (SI)', _data_standardizeUnitsSI, False), - ('Standardize Units (WE)', _data_standardizeUnitsWE, False), - ] - - -TOOLS={ - 'LogDec': _tool_logdec, - 'FASTRadialAverage': _tool_radialavg, - 'CurveFitting': _tool_curvefitting, -} - +DATA_PLUGINS_WITH_EDITOR=OrderedDict([ + ('Mask' , _data_mask ), + ('Remove Outliers' , _data_removeOutliers ), + ('Filter' , _data_filter ), + ('Resample' , _data_sampler ), + ('Bin data' , _data_binning ), + ]) +DATA_PLUGINS_SIMPLE=OrderedDict([ + ('Standardize Units (SI)', _data_standardizeUnitsSI), + ('Standardize Units (WE)', _data_standardizeUnitsWE), + ]) +DATA_TOOLS=OrderedDict([ # TODO + ('FAST - Radial Average', _tool_radialavg), + ]) -# --- -def getDataPluginsDict(): - d={} - for toolName, function, isPanel in dataPlugins: - d[toolName]={'callback':function, 'isPanel':isPanel} - return d +TOOLS=OrderedDict([ + ('Damping from decay',_tool_logdec), + ('Curve fitting', _tool_curvefitting), + ]) diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py index 7ab7ab2..f19c615 100644 --- a/pydatview/plugins/base_plugin.py +++ b/pydatview/plugins/base_plugin.py @@ -1,10 +1,7 @@ import wx import numpy as np -from pydatview.common import CHAR, Error, Info, pretty_num_short -from pydatview.common import DummyMainFrame -# from pydatview.plotdata import PlotData -# from pydatview.pipeline import PlotDataAction - +from pydatview.common import CHAR, Error, Info +from pydatview.plotdata import PlotData TOOL_BORDER=15 @@ -50,3 +47,341 @@ def getToggleBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): par.Bind(wx.EVT_TOGGLEBUTTON, callback, bt) return bt +# --------------------------------------------------------------------------------} +# --- Default class for GUI to edit plugin and control an action +# --------------------------------------------------------------------------------{ +class ActionEditor(GUIToolPanel): + """ + Class to edit an action. + Contains: + - the action + - the action data + - a set of function handles to process some triggers and callbacks + """ + def __init__(self, parent, action, buttons=None, tables=True): + GUIToolPanel.__init__(self, parent) + + # --- Data + self.data = action.data + self.action = action + self.applyRightAway = True # This is false for plotData actions. + self.cancelOnRemove = True # This is false for plotData actions. + # --- Unfortunate data to remove/manage + self.addActionHandle = self.pipeLike.append + self.removeActionHandle = self.pipeLike.remove + self.addTablesHandle = action.mainframe.load_dfs + + # Register ourselves to the action to be safe + self.action.guiEditorObj = self + + # --- unfortunate data + @property + def plotPanel(self): return self.action.mainframe.plotPanel # NOTE: we need mainframe if plotPlanel is respawned.. + @property + def redrawHandle(self): return self.action.mainframe.plotPanel.load_and_draw # or action.guiCallback + @property + def pipeLike(self): return self.action.mainframe.plotPanel.pipeLike + @property + def tabList(self): return self.action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate + + # --- External Calls + def cancelAction(self): + """ set the GUI state when the action is cancelled""" + self.data['active'] = False + + def guiActionAppliedState(self): + """ set the GUI state when the action is applied""" + self.data['active'] = True + + # --- Fairly generic + def _GUI2Data(self): + pass + + def _Data2GUI(self): + pass + + def onToggleApply(self, event=None, init=False): + if not init: + self.data['active'] = not self.data['active'] + + # Check if action is already in pipeline + i = self.pipeLike.index(self.action) + + if self.data['active']: + self._GUI2Data() + # We update the GUI + self.guiActionAppliedState() + # Add action to pipeline, apply it right away (if needed), update the GUI (redraw) + if i<0: + self.addActionHandle(self.action, overwrite=True, apply=self.applyRightAway, tabList=self.tabList, updateGUI=True) + else: + if not init: + # Remove action from pipeline, cancel it (if needed), update the GUI (redraw) + self.removeActionHandle(self.action, cancel=self.cancelOnRemove, tabList=self.tabList, updateGUI=True) + else: + self.cancelAction() + + def onAdd(self, event=None): + """ + Apply tableFunction on all selected tables, create new tables, add them to the GUI + """ + #iSel = self.cbTabs.GetSelection() + #icol, colname = self.plotPanel.selPanel.xCol + self._GUI2Data() + + dfs, names, errors = self.action.applyAndAdd(self.tabList) + if len(errors)>0: + raise Exception('Error: The action {} failed on some tables:\n\n'.format(action.name)+'\n'.join(errors)) + + # We stop applying if we were applying it (NOTE: doing this before adding table due to redraw trigger of the whole panel) + if self.data['active']: + self.onToggleApply() + + self.addTablesHandle(dfs, names, bAdd=True, bPlot=False) # Triggers a redraw of the whole panel... + # df, name = self.tabList[iSel-1].applyFiltering(icol, self.data, bAdd=True) + # self.parent.addTables([df], [name], bAdd=True) + #self.updateTabList() + + def onClear(self, event=None): + self.redrawHandle() + + def onHelp(self,event=None): + Info(self, """Dummy help""") + + +# --------------------------------------------------------------------------------} +# --- Default calss for GUI to edit plugin and control a plot data action +# --------------------------------------------------------------------------------{ +class PlotDataActionEditor(ActionEditor): + + def __init__(self, parent, action, buttons=None, tables=True): + """ + """ + ActionEditor.__init__(self, parent, action=action) + + # --- Data + self.data = action.data + self.action = action + # --- Unfortunate data to remove/manage + self.addActionHandle = self.pipeLike.append + self.removeActionHandle = self.pipeLike.remove + self.addTablesHandle = action.mainframe.load_dfs + + # Register ourselves to the action to be safe + self.action.guiEditorObj = self + + # --- GUI elements + if buttons is None: + buttons=['Close', 'Add', 'Help', 'Clear', 'Plot', 'Apply'] + self.btAdd = None + self.btHelp = None + self.btClear = None + self.btPlot = None + nButtons = 0 + self.btClose = self.getBtBitmap(self,'Close','close',self.destroy); nButtons+=1 + if 'Add' in buttons: + self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd); nButtons+=1 + if 'Help' in buttons: + self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp); nButtons+=1 + if 'Clear' in buttons: + self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear); nButtons+=1 + if 'Plot' in buttons: + self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot); nButtons+=1 + self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply); nButtons+=1 + + + if tables: + self.cbTabs= wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better + else: + self.cbTabs = None + + # --- Layout + btSizer = wx.FlexGridSizer(rows=int(nButtons/2), cols=2, hgap=2, vgap=0) + btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if self.btClear is not None: + btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if self.btAdd is not None: + btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if self.btPlot is not None: + btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if self.btHelp is not None: + btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) + # Child sizer will go here + # TODO cbTabs has no sizer + self.SetSizer(self.sizer) + + # --- Events + if tables: + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + + ## --- Init triggers + #self.updateTabList() + + # --- unfortunate data + @property + def plotPanel(self): return self.action.mainframe.plotPanel # NOTE: we need mainframe if plotPlanel is respawned.. + @property + def redrawHandle(self): return self.action.mainframe.plotPanel.load_and_draw # or action.guiCallback + @property + def pipeLike(self): return self.action.mainframe.plotPanel.pipeLike + @property + def tabList(self): return self.action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate + + # --- Bindings for plot triggers on parameters changes + def onParamChange(self, event=None): + if self.data['active']: + self._GUI2Data() + self.redrawHandle() + + def onParamChangeArrow(self, event): + self.onParamChange() + event.Skip() + + def onParamChangeEnter(self, event): + """ Action when the user presses Enter: we activate the action """ + self.onParamChange() + if not self.data['active']: + self.onToggleApply() + event.Skip() + + def onParamChangeChar(self, event): + event.Skip() + code = event.GetKeyCode() + if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: + #self.tParam.SetValue(self.spintxt.Value) # TODO + self.onParamChangeEnter(event) + + # --- Table related + def onTabChange(self,event=None): + #tabList = self.parent.selPanel.tabList + #iSel=self.cbTabs.GetSelection() + pass + + def updateTabList(self,event=None): + if self.cbTabs is not None: + tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() + #try: + iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + #except RuntimeError: + # pass + + # --- External Calls + def cancelAction(self): + """ set the GUI state when the action is cancelled""" + if self.btPlot is not None: + self.btPlot.Enable(True) + if self.btClear is not None: + self.btClear.Enable(True) + self.btApply.SetLabel(CHAR['cloud']+' Apply') + self.btApply.SetValue(False) + self.data['active'] = False + ActionEditor.cancelAction(self) + + def guiActionAppliedState(self): + """ set the GUI state when the action is applied""" + if self.btPlot is not None: + self.btPlot.Enable(False) + if self.btClear is not None: + self.btClear.Enable(False) + self.btApply.SetLabel(CHAR['sun']+' Clear') + self.btApply.SetValue(True) + self.data['active'] = True + ActionEditor.guiActionAppliedState(self) + + # --- Fairly generic + #def _GUI2Data(self): + # ActionEditor._GUI2Data(self) + + #def _Data2GUI(self): + # ActionEditor._Data2GUI(self) + + #def onToggleApply(self, event=None, init=False): + # ActionEditor.onToggleApply(self, event=event, init=init) + + #def onAdd(self, event=None): + # ActionEditor.onAdd(self, event) + + def onPlot(self, event=None): + self._GUI2Data() + # Loop on axes and plotdata of each axes + for ax in self.plotPanel.fig.axes: + for iPD in ax.iPD: + PD = self.plotPanel.plotData[iPD] + # Apply the plotDataFunction + x_new, y_new = self.action.plotDataFunction(PD.x0, PD.y0, self.data) + # Go through plotPanel own pipeline + PD_new = PlotData() + PD_new.fromXY(x_new, y_new) + self.plotPanel.transformPlotData(PD_new) + # Plot + ax.plot(PD_new.x, PD_new.y, '-') + # Simple canvas draw (a redraw would remove what we just added) + self.plotPanel.canvas.draw() + + +def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableFunctionAdd=None): + """ Function to demonstrate behavior of a plotdata plugin""" + from pydatview.pipeline import PlotDataAction + from pydatview.common import DummyMainFrame + from pydatview.Tables import TableList + from pydatview.pipeline import Pipeline + from pydatview.GUIPlotPanel import PlotPanel + from pydatview.GUISelectionPanel import SelectionPanel + if data is None: + data={'active':False} + + if plotDataFunction is None: + plotDataFunction = lambda x,y,opts: (x+0.1, 1.01*y) + + # --- Data + tabList = TableList.createDummy(nTabs=2, n=100, addLabel=False) + app = wx.App(False) + self = wx.Frame(None,-1,"Data Binning GUI") + pipeline = Pipeline() + + # --- Panels + self.selPanel = SelectionPanel(self, tabList, mode='auto') + self.plotPanel = PlotPanel(self, self.selPanel, pipeLike=pipeline) + self.plotPanel.load_and_draw() # <<< Important + self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # Binding the two + + # --- Dummy mainframe and action.. + mainframe = DummyMainFrame(self) + mainframe.plotPanel = self.plotPanel + guiCallback = self.plotPanel.load_and_draw + action = PlotDataAction( + name = 'Dummy Action', + tableFunctionAdd = tableFunctionAdd, + plotDataFunction = plotDataFunction, + guiEditorClass = panelClass, + guiCallback = guiCallback, + data = data, + mainframe = mainframe + ) + + # --- Create main object to be tested + p = panelClass(self.plotPanel, action=action, tables=False) + + # --- Finalize GUI + sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(self.selPanel ,0, wx.EXPAND|wx.ALL, border=5) + sizer.Add(self.plotPanel,1, wx.EXPAND|wx.ALL, border=5) + self.SetSizer(sizer) + self.SetSize((900, 600)) + self.Center() + self.Show() + self.plotPanel.showToolPanel(panel=p) # <<< Show + app.MainLoop() + +if __name__ == '__main__': + + demoPlotDataActionPanel(PlotDataActionEditor) + diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index aecb098..f2f9f9c 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -1,6 +1,6 @@ import wx import numpy as np -from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER +from pydatview.plugins.base_plugin import ActionEditor, TOOL_BORDER from pydatview.common import CHAR, Error, Info, pretty_num_short from pydatview.common import DummyMainFrame from pydatview.pipeline import ReversibleTableAction @@ -32,6 +32,7 @@ def maskAction(label, mainframe, data=None): action = ReversibleTableAction( name=label, + tableFunctionAdd = addTabMask, tableFunctionApply = applyMask, tableFunctionCancel = removeMask, guiEditorClass = MaskToolPanel, @@ -41,40 +42,36 @@ def maskAction(label, mainframe, data=None): ) return action # --------------------------------------------------------------------------------} -# --- Main method +# --- Main methods # --------------------------------------------------------------------------------{ def applyMask(tab, data): # dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=False) dfs, name = tab.applyMaskString(data['maskString'], bAdd=False) def removeMask(tab, data): + print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> CLEAR MASK') tab.clearMask() # tabList.clearCommonMask() +def addTabMask(tab, opts): + """ Apply action on a a table and return a new one with a new name + df_new, name_new = f(t, opts) + """ + df_new, name_new = tab.applyMaskString(opts['maskString'], bAdd=True) + return df_new, name_new + # --------------------------------------------------------------------------------} -# --- GUI to edit plugin and control a plot data action +# --- GUI to edit plugin and control the mask action # --------------------------------------------------------------------------------{ -class MaskToolPanel(GUIToolPanel): +class MaskToolPanel(ActionEditor): def __init__(self, parent, action): - GUIToolPanel.__init__(self, parent) + ActionEditor.__init__(self, parent, action=action) # --- Creating "Fake data" for testing only! if action is None: print('[WARN] Calling GUI without an action! Creating one.') action = maskAction(label='dummyAction', mainframe=DummyMainFrame(parent)) - # --- Data - self.data = action.data - self.action = action - - # --- Unfortunate data to remove/manage - self.addActionHandle = self.pipeLike.append - self.removeActionHandle = self.pipeLike.remove - self.addTablesHandle = action.mainframe.load_dfs - - # Register ourselves to the action to be safe - self.action.guiEditorObj = self - # --- GUI elements self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) self.btAdd = self.getBtBitmap(self, u'Mask (add)','add' , self.onAdd) @@ -120,16 +117,6 @@ def __init__(self, parent, action): self.onToggleApply(init=True) self.updateTabList() - # --- unfortunate data - @property - def plotPanel(self): return self.action.mainframe.plotPanel # NOTE: we need mainframe if plotPlanel is respawned.. - @property - def redrawHandle(self): return self.action.mainframe.plotPanel.load_and_draw # or action.guiCallback - @property - def pipeLike(self): return self.action.mainframe.plotPanel.pipeLike - @property - def tabList(self): return self.action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate - # --- Implementation specific def guessMask(self): cols=self.tabList[0].columns_clean @@ -198,45 +185,3 @@ def _Data2GUI(self): self.data['maskString'] = self.guessMask() # no known mask, we guess one to help the user self.textMask.SetValue(self.data['maskString']) - def onToggleApply(self, event=None, init=False): - if not init: - self.data['active'] = not self.data['active'] - - # Check if action is already in pipeline - i = self.pipeLike.index(self.action) - - if self.data['active']: - self._GUI2Data() - # We update the GUI - self.guiActionAppliedState() - # Add action to pipeline, apply it, update the GUI - self.addActionHandle(self.action, overwrite=True, apply=True, tabList=self.tabList, updateGUI=True) - else: - if not init: - # Remove action from pipeline, cancel it, update the GUI - self.removeActionHandle(self.action, cancel=True, tabList=self.tabList, updateGUI=True) - else: - self.cancelAction() - - def onAdd(self, event=None): - """ - Apply tableFunction on all selected tables, create new tables, add them to the GUI - """ - self._GUI2Data() - iSel = self.cbTabs.GetSelection() - # TODO this should be handled by the action - dfs, names, errors = self.tabList.applyCommonMaskString(self.data['maskString'], bAdd=True) - if len(errors)>0: - raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) - - # We stop applying if we were applying it (NOTE: doing this before adding table due to redraw trigger of the whole panel) - if self.data['active']: - self.onToggleApply() - - self.addTablesHandle(dfs, names, bAdd=True, bPlot=False) # Triggers a redraw of the whole panel... - #if iSel==0: - #else: - # dfs, name = tabList[iSel-1].applyMaskString(self.data['maskString'], bAdd=True) - # self.parent.addTables([dfs],[name], bAdd=True) - #self.updateTabList() - diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index 36d9ae4..277394a 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -1,7 +1,6 @@ import wx import numpy as np -from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER -from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor +from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info, pretty_num_short from pydatview.pipeline import PlotDataAction # --------------------------------------------------------------------------------} @@ -229,6 +228,6 @@ def onHelp(self,event=None): if __name__ == '__main__': - from pydatview.plugins.plotdata_default_plugin import demoPlotDataActionPanel + from pydatview.plugins.base_plugin import demoPlotDataActionPanel demoPlotDataActionPanel(BinningToolPanel, plotDataFunction=bin_plot, data=_DEFAULT_DICT, tableFunctionAdd=binTabAdd) diff --git a/pydatview/plugins/plotdata_default_plugin.py b/pydatview/plugins/plotdata_default_plugin.py deleted file mode 100644 index 9c167e2..0000000 --- a/pydatview/plugins/plotdata_default_plugin.py +++ /dev/null @@ -1,287 +0,0 @@ -import wx -import numpy as np -from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER -from pydatview.common import CHAR, Error, Info, pretty_num_short -from pydatview.common import DummyMainFrame -from pydatview.plotdata import PlotData -from pydatview.pipeline import PlotDataAction - - -# --------------------------------------------------------------------------------} -# --- GUI to edit plugin and control a plot data action -# --------------------------------------------------------------------------------{ -class PlotDataActionEditor(GUIToolPanel): - - def __init__(self, parent, action, buttons=None, tables=True): - """ - """ - GUIToolPanel.__init__(self, parent) - - # --- Data - self.data = action.data - self.action = action - # --- Unfortunate data to remove/manage - self.addActionHandle = self.pipeLike.append - self.removeActionHandle = self.pipeLike.remove - self.addTablesHandle = action.mainframe.load_dfs - - # Register ourselves to the action to be safe - self.action.guiEditorObj = self - - # --- GUI elements - if buttons is None: - buttons=['Close', 'Add', 'Help', 'Clear', 'Plot', 'Apply'] - self.btAdd = None - self.btHelp = None - self.btClear = None - self.btPlot = None - nButtons = 0 - self.btClose = self.getBtBitmap(self,'Close','close',self.destroy); nButtons+=1 - if 'Add' in buttons: - self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd); nButtons+=1 - if 'Help' in buttons: - self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp); nButtons+=1 - if 'Clear' in buttons: - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear); nButtons+=1 - if 'Plot' in buttons: - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot); nButtons+=1 - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply); nButtons+=1 - - - if tables: - self.cbTabs= wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) - self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better - else: - self.cbTabs = None - - # --- Layout - btSizer = wx.FlexGridSizer(rows=int(nButtons/2), cols=2, hgap=2, vgap=0) - btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) - if self.btClear is not None: - btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) - if self.btAdd is not None: - btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) - if self.btPlot is not None: - btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) - if self.btHelp is not None: - btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) - # Child sizer will go here - # TODO cbTabs has no sizer - self.SetSizer(self.sizer) - - # --- Events - if tables: - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) - - ## --- Init triggers - #self.updateTabList() - - # --- unfortunate data - @property - def plotPanel(self): return self.action.mainframe.plotPanel # NOTE: we need mainframe if plotPlanel is respawned.. - @property - def redrawHandle(self): return self.action.mainframe.plotPanel.load_and_draw # or action.guiCallback - @property - def pipeLike(self): return self.action.mainframe.plotPanel.pipeLike - @property - def tabList(self): return self.action.mainframe.plotPanel.selPanel.tabList # a bit unfortunate - - # --- Bindings for plot triggers on parameters changes - def onParamChange(self, event=None): - if self.data['active']: - self._GUI2Data() - self.redrawHandle() - - def onParamChangeArrow(self, event): - self.onParamChange() - event.Skip() - - def onParamChangeEnter(self, event): - """ Action when the user presses Enter: we activate the action """ - self.onParamChange() - if not self.data['active']: - self.onToggleApply() - event.Skip() - - def onParamChangeChar(self, event): - event.Skip() - code = event.GetKeyCode() - if code in [wx.WXK_RETURN, wx.WXK_NUMPAD_ENTER]: - #self.tParam.SetValue(self.spintxt.Value) # TODO - self.onParamChangeEnter(event) - - # --- Table related - def onTabChange(self,event=None): - #tabList = self.parent.selPanel.tabList - #iSel=self.cbTabs.GetSelection() - pass - - def updateTabList(self,event=None): - if self.cbTabs is not None: - tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() - #try: - iSel=np.max([np.min([self.cbTabs.GetSelection(),len(tabListNames)]),0]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - #except RuntimeError: - # pass - - # --- External Calls - def cancelAction(self): - """ set the GUI state when the action is cancelled""" - if self.btPlot is not None: - self.btPlot.Enable(True) - if self.btClear is not None: - self.btClear.Enable(True) - self.btApply.SetLabel(CHAR['cloud']+' Apply') - self.btApply.SetValue(False) - self.data['active'] = False - - def guiActionAppliedState(self): - """ set the GUI state when the action is applied""" - if self.btPlot is not None: - self.btPlot.Enable(False) - if self.btClear is not None: - self.btClear.Enable(False) - self.btApply.SetLabel(CHAR['sun']+' Clear') - self.btApply.SetValue(True) - self.data['active'] = True - - # --- Fairly generic - def _GUI2Data(self): - pass - - def _Data2GUI(self): - pass - - def onToggleApply(self, event=None, init=False): - if not init: - self.data['active'] = not self.data['active'] - - # Check if action is already in pipeline - i = self.pipeLike.index(self.action) - - if self.data['active']: - self._GUI2Data() - # We update the GUI - self.guiActionAppliedState() - # Add action to pipeline, no need to apply it, update the GUI (redraw) - if i<0: - self.addActionHandle(self.action, overwrite=True, apply=False, updateGUI=True) - else: - if not init: - # Remove action from pipeline, no need to cancel it, update the GUI (redraw) - self.removeActionHandle(self.action, cancel=False, updateGUI=True) - else: - self.cancelAction() - - def onAdd(self, event=None): - """ - Apply tableFunction on all selected tables, create new tables, add them to the GUI - """ - #iSel = self.cbTabs.GetSelection() - #icol, colname = self.plotPanel.selPanel.xCol - self._GUI2Data() - - dfs, names, errors = self.action.applyAndAdd(self.tabList) - if len(errors)>0: - raise Exception('Error: The action {} failed on some tables:\n\n'.format(action.name)+'\n'.join(errors)) - - # We stop applying if we were applying it (NOTE: doing this before adding table due to redraw trigger of the whole panel) - if self.data['active']: - self.onToggleApply() - - self.addTablesHandle(dfs, names, bAdd=True, bPlot=False) # Triggers a redraw of the whole panel... - # df, name = self.tabList[iSel-1].applyFiltering(icol, self.data, bAdd=True) - # self.parent.addTables([df], [name], bAdd=True) - #self.updateTabList() - - - def onPlot(self, event=None): - self._GUI2Data() - # Loop on axes and plotdata of each axes - for ax in self.plotPanel.fig.axes: - for iPD in ax.iPD: - PD = self.plotPanel.plotData[iPD] - # Apply the plotDataFunction - x_new, y_new = self.action.plotDataFunction(PD.x0, PD.y0, self.data) - # Go through plotPanel own pipeline - PD_new = PlotData() - PD_new.fromXY(x_new, y_new) - self.plotPanel.transformPlotData(PD_new) - # Plot - ax.plot(PD_new.x, PD_new.y, '-') - # Simple canvas draw (a redraw would remove what we just added) - self.plotPanel.canvas.draw() - - def onClear(self, event=None): - self.redrawHandle() - - def onHelp(self,event=None): - Info(self, """Dummy help""") - - - -def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableFunctionAdd=None): - """ Function to demonstrate behavior of a plotdata plugin""" - from pydatview.Tables import TableList - from pydatview.pipeline import Pipeline - from pydatview.GUIPlotPanel import PlotPanel - from pydatview.GUISelectionPanel import SelectionPanel - if data is None: - data={'active':False} - - if plotDataFunction is None: - plotDataFunction = lambda x,y,opts: (x+0.1, 1.01*y) - - - # --- Data - tabList = TableList.createDummy(nTabs=2, n=100, addLabel=False) - app = wx.App(False) - self = wx.Frame(None,-1,"Data Binning GUI") - pipeline = Pipeline() - - # --- Panels - self.selPanel = SelectionPanel(self, tabList, mode='auto') - self.plotPanel = PlotPanel(self, self.selPanel, pipeLike=pipeline) - self.plotPanel.load_and_draw() # <<< Important - self.selPanel.setRedrawCallback(self.plotPanel.load_and_draw) # Binding the two - - # --- Dummy mainframe and action.. - mainframe = DummyMainFrame(self) - mainframe.plotPanel = self.plotPanel - guiCallback = self.plotPanel.load_and_draw - action = PlotDataAction( - name = 'Dummy Action', - tableFunctionAdd = tableFunctionAdd, - plotDataFunction = plotDataFunction, - guiEditorClass = panelClass, - guiCallback = guiCallback, - data = data, - mainframe = mainframe - ) - - # --- Create main object to be tested - p = panelClass(self.plotPanel, action=action, tables=False) - - # --- Finalize GUI - sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(self.selPanel ,0, wx.EXPAND|wx.ALL, border=5) - sizer.Add(self.plotPanel,1, wx.EXPAND|wx.ALL, border=5) - self.SetSizer(sizer) - self.SetSize((900, 600)) - self.Center() - self.Show() - self.plotPanel.showToolPanel(panel=p) # <<< Show - app.MainLoop() - -if __name__ == '__main__': - - demoPlotDataActionPanel(PlotDataActionEditor) - - diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index 4ebd43a..2019993 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -1,7 +1,6 @@ import wx import numpy as np -from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER -from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor +from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction import platform @@ -215,6 +214,6 @@ def onHelp(self,event=None): if __name__ == '__main__': - from pydatview.plugins.plotdata_default_plugin import demoPlotDataActionPanel + from pydatview.plugins.base_plugin import demoPlotDataActionPanel demoPlotDataActionPanel(FilterToolPanel, plotDataFunction=filterXY, data=_DEFAULT_DICT, tableFunctionAdd=filterTabAdd) diff --git a/pydatview/plugins/plotdata_removeOutliers.py b/pydatview/plugins/plotdata_removeOutliers.py index 52f5d10..d1583e8 100644 --- a/pydatview/plugins/plotdata_removeOutliers.py +++ b/pydatview/plugins/plotdata_removeOutliers.py @@ -1,7 +1,6 @@ import wx import numpy as np -from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER -from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor +from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction import platform @@ -159,6 +158,6 @@ def _Data2GUI(self): # self.cancelAction(redraw= not init) if __name__ == '__main__': - from pydatview.plugins.plotdata_default_plugin import demoPlotDataActionPanel + from pydatview.plugins.base_plugin import demoPlotDataActionPanel demoPlotDataActionPanel(RemoveOutliersToolPanel, plotDataFunction=removeOutliersXY, data=_DEFAULT_DICT) diff --git a/pydatview/plugins/plotdata_sampler.py b/pydatview/plugins/plotdata_sampler.py index fd470dd..28769ad 100644 --- a/pydatview/plugins/plotdata_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -1,7 +1,6 @@ import wx import numpy as np -from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER -from pydatview.plugins.plotdata_default_plugin import PlotDataActionEditor +from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction # --------------------------------------------------------------------------------} @@ -213,6 +212,6 @@ def onHelp(self,event=None): """) if __name__ == '__main__': - from pydatview.plugins.plotdata_default_plugin import demoPlotDataActionPanel + from pydatview.plugins.base_plugin import demoPlotDataActionPanel demoPlotDataActionPanel(SamplerToolPanel, plotDataFunction=samplerXY, data=_DEFAULT_DICT, tableFunctionAdd=samplerTabAdd) diff --git a/pydatview/plugins/tests/test_standardizeUnits.py b/pydatview/plugins/tests/test_standardizeUnits.py index c809d33..599fea2 100644 --- a/pydatview/plugins/tests/test_standardizeUnits.py +++ b/pydatview/plugins/tests/test_standardizeUnits.py @@ -13,7 +13,7 @@ def test_change_units(self): data[:,2] *= 10*np.pi/180 # rad df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) tab=Table(data=df) - changeUnits(tab, flavor='WE') + changeUnits(tab, data={'flavor':'WE'}) np.testing.assert_almost_equal(tab.data.values[:,1],[1]) np.testing.assert_almost_equal(tab.data.values[:,2],[2]) np.testing.assert_almost_equal(tab.data.values[:,3],[10]) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index afa763d..6462865 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -17,18 +17,17 @@ def test_pipeline(self): if __name__ == '__main__': - from pydatview.plugins import getDataPluginsDict + from pydatview.plugins import DATA_PLUGINS_SIMPLE - DPD = getDataPluginsDict() + DPD = DATA_PLUGINS_SIMPLE - - tablist = TableList.createDummyList(1) + tablist = TableList.createDummy(1) print(tablist._tabs[0].data) pipeline = Pipeline() - action = DPD['Standardize Units (SI)']['callback']() - pipeline.append(action) + action = DPD['Standardize Units (SI)'](label='Standardize Units (SI)') + pipeline.append(action, apply=False) print(pipeline) @@ -41,8 +40,8 @@ def test_pipeline(self): - action = DPD['Standardize Units (WE)']['callback']() - pipeline.append(action) + action = DPD['Standardize Units (WE)'](label='Standardize Units (WE)') + pipeline.append(action, apply=False) pipeline.apply(tablist) print(tablist._tabs[0].data) From 0d834d1a2e0e777a73c57609d33d4f8aefdbf7fe Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 28 Dec 2022 15:19:07 +0100 Subject: [PATCH 075/178] Plugins: adding show GUI tests --- pydatview/GUIPlotPanel.py | 2 +- pydatview/Tables.py | 2 +- pydatview/plugins/base_plugin.py | 12 ++++++----- pydatview/plugins/plotdata_binning.py | 10 +++++----- pydatview/plugins/tests/test_binning.py | 18 +++++++++++++++++ pydatview/plugins/tests/test_filter.py | 20 +++++++++++++++++++ .../plugins/tests/test_removeOutliers.py | 19 ++++++++++++++++++ pydatview/plugins/tests/test_sampler.py | 18 +++++++++++++++++ 8 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 pydatview/plugins/tests/test_binning.py create mode 100644 pydatview/plugins/tests/test_filter.py create mode 100644 pydatview/plugins/tests/test_removeOutliers.py create mode 100644 pydatview/plugins/tests/test_sampler.py diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 7d3e470..adbda7c 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -413,7 +413,7 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): if data is not None: self.data = data else: - print('>>> Using default settings for plot panel') + #print('>>> Using default settings for plot panel') self.data = self.defaultData() if self.selPanel is not None: bg=self.selPanel.BackgroundColour diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 95b2fe1..4958c01 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -728,7 +728,7 @@ def getColumn(self, i): c = self.data.iloc[:, i] x = self.data.iloc[:, i].values - isString = c.dtype == np.object and isinstance(c.values[0], str) + isString = c.dtype == object and isinstance(c.values[0], str) if isString: x=x.astype(str) isDate = np.issubdtype(c.dtype, np.datetime64) diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py index f19c615..05bc8dd 100644 --- a/pydatview/plugins/base_plugin.py +++ b/pydatview/plugins/base_plugin.py @@ -327,7 +327,7 @@ def onPlot(self, event=None): self.plotPanel.canvas.draw() -def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableFunctionAdd=None): +def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableFunctionAdd=None, mainLoop=True, title='Demo'): """ Function to demonstrate behavior of a plotdata plugin""" from pydatview.pipeline import PlotDataAction from pydatview.common import DummyMainFrame @@ -344,7 +344,7 @@ def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableF # --- Data tabList = TableList.createDummy(nTabs=2, n=100, addLabel=False) app = wx.App(False) - self = wx.Frame(None,-1,"Data Binning GUI") + self = wx.Frame(None,-1, title) pipeline = Pipeline() # --- Panels @@ -368,7 +368,7 @@ def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableF ) # --- Create main object to be tested - p = panelClass(self.plotPanel, action=action, tables=False) + p = panelClass(self.plotPanel, action=action) # --- Finalize GUI sizer = wx.BoxSizer(wx.HORIZONTAL) @@ -377,9 +377,11 @@ def demoPlotDataActionPanel(panelClass, data=None, plotDataFunction=None, tableF self.SetSizer(sizer) self.SetSize((900, 600)) self.Center() - self.Show() self.plotPanel.showToolPanel(panel=p) # <<< Show - app.MainLoop() + + if mainLoop: + self.Show() + app.MainLoop() if __name__ == '__main__': diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index 277394a..fe3aed9 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -140,10 +140,7 @@ def __init__(self, parent, action, **kwargs): self.textXMax.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter) # --- Init triggers - if self.data['active']: - self.setXRange(x=[self.data['xMin'], self.data['xMax']]) - else: - self.setXRange() + self.setXRange() self._Data2GUI() self.onToggleApply(init=True) @@ -154,7 +151,10 @@ def reset(self, event=None): def setXRange(self, x=None): if x is None: - x= self.plotPanel.plotData[0].x0 + if self.data['active']: + x=[self.data['xMin'], self.data['xMax']] + else: + x= self.plotPanel.plotData[0].x0 xmin, xmax = np.nanmin(x), np.nanmax(x) self.textXMin.SetValue(pretty_num_short(xmin)) self.textXMax.SetValue(pretty_num_short(xmax)) diff --git a/pydatview/plugins/tests/test_binning.py b/pydatview/plugins/tests/test_binning.py new file mode 100644 index 0000000..76bcb65 --- /dev/null +++ b/pydatview/plugins/tests/test_binning.py @@ -0,0 +1,18 @@ +import unittest +import numpy as np + +from pydatview.plugins.base_plugin import demoPlotDataActionPanel +from pydatview.plugins.plotdata_binning import * +from pydatview.plugins.plotdata_binning import _DEFAULT_DICT + + +class TestBinning(unittest.TestCase): + + def test_showGUI(self): + + demoPlotDataActionPanel(BinningToolPanel, plotDataFunction=bin_plot, data=_DEFAULT_DICT, tableFunctionAdd=binTabAdd, mainLoop=False, title='Binning') + + +if __name__ == '__main__': + unittest.main() + diff --git a/pydatview/plugins/tests/test_filter.py b/pydatview/plugins/tests/test_filter.py new file mode 100644 index 0000000..cb7a0f6 --- /dev/null +++ b/pydatview/plugins/tests/test_filter.py @@ -0,0 +1,20 @@ +import unittest +import numpy as np + +from pydatview.plugins.base_plugin import demoPlotDataActionPanel +from pydatview.plugins.plotdata_filter import * +from pydatview.plugins.plotdata_filter import _DEFAULT_DICT + + +class TestFilter(unittest.TestCase): + + def test_showGUI(self): + + demoPlotDataActionPanel(FilterToolPanel, plotDataFunction=filterXY, data=_DEFAULT_DICT, tableFunctionAdd=filterTabAdd, mainLoop=False, title='Filter') + + + +if __name__ == '__main__': + unittest.main() + + diff --git a/pydatview/plugins/tests/test_removeOutliers.py b/pydatview/plugins/tests/test_removeOutliers.py new file mode 100644 index 0000000..257f849 --- /dev/null +++ b/pydatview/plugins/tests/test_removeOutliers.py @@ -0,0 +1,19 @@ +import unittest +import numpy as np + +from pydatview.plugins.base_plugin import demoPlotDataActionPanel +from pydatview.plugins.plotdata_removeOutliers import * +from pydatview.plugins.plotdata_removeOutliers import _DEFAULT_DICT + + +class TestRemoveOutliers(unittest.TestCase): + + def test_showGUI(self): + + demoPlotDataActionPanel(RemoveOutliersToolPanel, plotDataFunction=removeOutliersXY, data=_DEFAULT_DICT, mainLoop=False, title='Remove Outliers') + + +if __name__ == '__main__': + unittest.main() + + diff --git a/pydatview/plugins/tests/test_sampler.py b/pydatview/plugins/tests/test_sampler.py new file mode 100644 index 0000000..14a4c66 --- /dev/null +++ b/pydatview/plugins/tests/test_sampler.py @@ -0,0 +1,18 @@ +import unittest +import numpy as np + +from pydatview.plugins.base_plugin import demoPlotDataActionPanel +from pydatview.plugins.plotdata_sampler import * +from pydatview.plugins.plotdata_sampler import _DEFAULT_DICT + + +class TestSampler(unittest.TestCase): + + def test_showGUI(self): + + demoPlotDataActionPanel(SamplerToolPanel, plotDataFunction=samplerXY, data=_DEFAULT_DICT, tableFunctionAdd=samplerTabAdd, mainLoop=False, title='Sampler') + + +if __name__ == '__main__': + unittest.main() + From cdf1df90531ef1e7b1e1e63c11aac85db505dbe2 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 28 Dec 2022 20:47:52 +0100 Subject: [PATCH 076/178] GH: failed attempt to add wxpython to CI requirements, skipping tests --- .github/workflows/tests.yml | 11 +++++++++-- _tools/travis_requirements.txt | 1 + pydatview/Tables.py | 2 +- pydatview/plugins/base_plugin.py | 10 +++++++++- pydatview/plugins/plotdata_binning.py | 2 +- pydatview/plugins/plotdata_filter.py | 2 +- pydatview/plugins/plotdata_removeOutliers.py | 2 +- pydatview/plugins/plotdata_sampler.py | 2 +- pydatview/plugins/tests/test_binning.py | 9 +++++---- pydatview/plugins/tests/test_filter.py | 9 +++++---- pydatview/plugins/tests/test_removeOutliers.py | 9 +++++---- pydatview/plugins/tests/test_sampler.py | 11 +++++------ 12 files changed, 44 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ef2c60c..af8d54c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ on: jobs: build-and-test: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: python-version: [3.8, 3.9, 3.11] @@ -64,7 +64,14 @@ jobs: - name: Install python dependencies run: | - python -m pip install --upgrade pip + #python -m pip install --upgrade pip + # --- Attempt for wxpython, but we don't have access to X display + #sudo apt-get install libgtk-3-dev + #sudo apt-get install git curl libsdl2-mixer-2.0-0 libsdl2-image-2.0-0 libsdl2-2.0-0 || true + #sudo apt-get install libnotify-dev || true + #sudo apt-get install libnotify4 || true + #pip install -U wxpython || true + #pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-22.04 wxPython || true pip install -r _tools/travis_requirements.txt - name: System info diff --git a/_tools/travis_requirements.txt b/_tools/travis_requirements.txt index 77dfc8e..e8a9a83 100644 --- a/_tools/travis_requirements.txt +++ b/_tools/travis_requirements.txt @@ -5,3 +5,4 @@ pyarrow # for parquet files matplotlib chardet scipy +attrdict3 diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 4958c01..23c4ffe 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -199,7 +199,7 @@ def sort(self, method='byName'): else: raise Exception('Sorting method unknown: `{}`'.format(method)) - def mergeTabs(self, I=None, ICommonColPerTab=None, samplDict=None, extrap='nan'): + def mergeTabs(self, I=None, ICommonColPerTab=None, extrap='nan'): """ Merge table together. TODO: add options for how interpolation/merging is done diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py index 05bc8dd..88c24b3 100644 --- a/pydatview/plugins/base_plugin.py +++ b/pydatview/plugins/base_plugin.py @@ -1,4 +1,12 @@ -import wx +try: + import wx + HAS_WX=True +except: + # Creating a fake wx package just so that this package can be imported without failing + # and plugins can be tested without GUI + print('[FAIL] cannot import wx') + wx=type('wx', (object,), {'Panel':object}) + HAS_WX=False import numpy as np from pydatview.common import CHAR, Error, Info from pydatview.plotdata import PlotData diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index fe3aed9..370890e 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -1,4 +1,3 @@ -import wx import numpy as np from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info, pretty_num_short @@ -87,6 +86,7 @@ def binTabAdd(tab, data): class BinningToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): + import wx PlotDataActionEditor.__init__(self, parent, action, tables=True, **kwargs) # --- GUI elements diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index 2019993..35bdc35 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -1,4 +1,3 @@ -import wx import numpy as np from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info @@ -65,6 +64,7 @@ def filterTabAdd(tab, opts): class FilterToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): + import wx PlotDataActionEditor.__init__(self, parent, action, tables=True, **kwargs) # --- Data diff --git a/pydatview/plugins/plotdata_removeOutliers.py b/pydatview/plugins/plotdata_removeOutliers.py index d1583e8..535fedb 100644 --- a/pydatview/plugins/plotdata_removeOutliers.py +++ b/pydatview/plugins/plotdata_removeOutliers.py @@ -1,4 +1,3 @@ -import wx import numpy as np from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info @@ -54,6 +53,7 @@ def removeOutliersXY(x, y, opts): class RemoveOutliersToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): + import wx PlotDataActionEditor.__init__(self, parent, action, tables=False, buttons=[''], **kwargs) # --- GUI elements diff --git a/pydatview/plugins/plotdata_sampler.py b/pydatview/plugins/plotdata_sampler.py index 28769ad..602f395 100644 --- a/pydatview/plugins/plotdata_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -1,4 +1,3 @@ -import wx import numpy as np from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info @@ -60,6 +59,7 @@ def samplerTabAdd(tab, opts): class SamplerToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): + import wx PlotDataActionEditor.__init__(self, parent, action, tables=True) # --- Data diff --git a/pydatview/plugins/tests/test_binning.py b/pydatview/plugins/tests/test_binning.py index 76bcb65..6badcad 100644 --- a/pydatview/plugins/tests/test_binning.py +++ b/pydatview/plugins/tests/test_binning.py @@ -1,16 +1,17 @@ import unittest import numpy as np -from pydatview.plugins.base_plugin import demoPlotDataActionPanel +from pydatview.plugins.base_plugin import demoPlotDataActionPanel, HAS_WX from pydatview.plugins.plotdata_binning import * from pydatview.plugins.plotdata_binning import _DEFAULT_DICT - class TestBinning(unittest.TestCase): def test_showGUI(self): - - demoPlotDataActionPanel(BinningToolPanel, plotDataFunction=bin_plot, data=_DEFAULT_DICT, tableFunctionAdd=binTabAdd, mainLoop=False, title='Binning') + if HAS_WX: + demoPlotDataActionPanel(BinningToolPanel, plotDataFunction=bin_plot, data=_DEFAULT_DICT, tableFunctionAdd=binTabAdd, mainLoop=False, title='Binning') + else: + print('[WARN] skipping test because wx is not available.') if __name__ == '__main__': diff --git a/pydatview/plugins/tests/test_filter.py b/pydatview/plugins/tests/test_filter.py index cb7a0f6..404659c 100644 --- a/pydatview/plugins/tests/test_filter.py +++ b/pydatview/plugins/tests/test_filter.py @@ -1,16 +1,17 @@ import unittest import numpy as np -from pydatview.plugins.base_plugin import demoPlotDataActionPanel +from pydatview.plugins.base_plugin import demoPlotDataActionPanel, HAS_WX from pydatview.plugins.plotdata_filter import * from pydatview.plugins.plotdata_filter import _DEFAULT_DICT - class TestFilter(unittest.TestCase): def test_showGUI(self): - - demoPlotDataActionPanel(FilterToolPanel, plotDataFunction=filterXY, data=_DEFAULT_DICT, tableFunctionAdd=filterTabAdd, mainLoop=False, title='Filter') + if HAS_WX: + demoPlotDataActionPanel(FilterToolPanel, plotDataFunction=filterXY, data=_DEFAULT_DICT, tableFunctionAdd=filterTabAdd, mainLoop=False, title='Filter') + else: + print('[WARN] skipping test because wx is not available.') diff --git a/pydatview/plugins/tests/test_removeOutliers.py b/pydatview/plugins/tests/test_removeOutliers.py index 257f849..101514c 100644 --- a/pydatview/plugins/tests/test_removeOutliers.py +++ b/pydatview/plugins/tests/test_removeOutliers.py @@ -1,16 +1,17 @@ import unittest import numpy as np -from pydatview.plugins.base_plugin import demoPlotDataActionPanel +from pydatview.plugins.base_plugin import demoPlotDataActionPanel, HAS_WX from pydatview.plugins.plotdata_removeOutliers import * from pydatview.plugins.plotdata_removeOutliers import _DEFAULT_DICT - class TestRemoveOutliers(unittest.TestCase): def test_showGUI(self): - - demoPlotDataActionPanel(RemoveOutliersToolPanel, plotDataFunction=removeOutliersXY, data=_DEFAULT_DICT, mainLoop=False, title='Remove Outliers') + if HAS_WX: + demoPlotDataActionPanel(RemoveOutliersToolPanel, plotDataFunction=removeOutliersXY, data=_DEFAULT_DICT, mainLoop=False, title='Remove Outliers') + else: + print('[WARN] skipping test because wx is not available.') if __name__ == '__main__': diff --git a/pydatview/plugins/tests/test_sampler.py b/pydatview/plugins/tests/test_sampler.py index 14a4c66..cc10807 100644 --- a/pydatview/plugins/tests/test_sampler.py +++ b/pydatview/plugins/tests/test_sampler.py @@ -1,17 +1,16 @@ import unittest import numpy as np - -from pydatview.plugins.base_plugin import demoPlotDataActionPanel +from pydatview.plugins.base_plugin import demoPlotDataActionPanel, HAS_WX from pydatview.plugins.plotdata_sampler import * from pydatview.plugins.plotdata_sampler import _DEFAULT_DICT - class TestSampler(unittest.TestCase): def test_showGUI(self): - - demoPlotDataActionPanel(SamplerToolPanel, plotDataFunction=samplerXY, data=_DEFAULT_DICT, tableFunctionAdd=samplerTabAdd, mainLoop=False, title='Sampler') - + if HAS_WX: + demoPlotDataActionPanel(SamplerToolPanel, plotDataFunction=samplerXY, data=_DEFAULT_DICT, tableFunctionAdd=samplerTabAdd, mainLoop=False, title='Sampler') + else: + print('[WARN] skipping test because wx is not available.') if __name__ == '__main__': unittest.main() From 8915d2380887b8ff70801c82e52ed7087982371f Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 30 Dec 2022 12:43:44 +0100 Subject: [PATCH 077/178] GH: update of workflow set-output --- .github/workflows/tests.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index af8d54c..9611e24 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,9 +53,10 @@ jobs: echo "VERSION_TAG $VERSION_TAG" echo "VERSION_NAME: $VERSION_NAME" echo "FULL_VERSION_NAME: $FULL_VERSION_NAME" - echo "::set-output name=FULL_VERSION_NAME::$FULL_VERSION_NAME" - echo "::set-output name=VERSION_NAME::$VERSION_NAME" - echo "::set-output name=VERSION_TAG::$VERSION_TAG" + # Save variables as github outputs + echo "VERSION_TAG=$VERSION_TAG" >> $GITHUB_OUTPUT + echo "VERSION_NAME=$VERSION_NAME" >> $GITHUB_OUTPUT + echo "FULL_VERSION_NAME=$FULL_VERSION_NAME" >> $GITHUB_OUTPUT - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 @@ -103,7 +104,13 @@ jobs: fi fi echo "DEPLOY : $OK" - echo "::set-output name=GO::$OK" + if [[ $OK == "1" ]]; then + echo ">>> Deployment WILL take place" + else + echo ">>> Deployment WILL NOT take place" + fi + # Save variables as github outputs + echo "GO=$OK" >> $GITHUB_OUTPUT # --- Run Deployments - name: Install system dependencies From 668b5f104e099aee4b5562655aa1aa8162b0b0d9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 30 Dec 2022 12:53:35 +0100 Subject: [PATCH 078/178] GH: deploy on dev and main only --- .github/workflows/tests.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9611e24..b7b98c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,12 +25,17 @@ jobs: - name: Versioning id: versioning run: | - git fetch --unshallow - git fetch --tags + git fetch --unshallow > /dev/null + git fetch --tags > /dev/null export CURRENT_TAG="v0.3" export CURRENT_DEV_TAG="$CURRENT_TAG-dev" + # BRANCH FAILS + export BRANCH1=`git rev-parse --abbrev-ref HEAD` + export BRANCH="${GITHUB_REF#refs/heads/}" echo "GIT DESCRIBE: `git describe`" echo "GITHUB_REF: $GITHUB_REF" + echo "GIT BRANCH1: $BRANCH1" + echo "GIT BRANCH: $BRANCH" echo "Commits: `git rev-list $CURRENT_TAG.. --count`" echo "Commits-dev: `git rev-list $CURRENT_DEV_TAG.. --count`" # Check if current version corresponds to a tagged commit @@ -44,8 +49,13 @@ jobs: export VERSION_TAG=$CURRENT_TAG export VERSION_NAME="$CURRENT_TAG-`git rev-list $CURRENT_TAG.. --count`" export FULL_VERSION_NAME="version $VERSION_NAME" + #elif [[ $BRANCH == "dev" ]]; then + # echo ">>> This is not a tagged version, but on the dev branch" + # export VERSION_TAG=$CURRENT_DEV_TAG ; + # export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` + # export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" else - echo ">>> This is not a tagged version" + echo ">>> This is not a tagged version, but on a special branch" export VERSION_TAG=$CURRENT_DEV_TAG ; export VERSION_NAME=`git describe | sed 's/\(.*\)-.*/\1/'` export FULL_VERSION_NAME="latest dev. version $VERSION_NAME" @@ -54,6 +64,7 @@ jobs: echo "VERSION_NAME: $VERSION_NAME" echo "FULL_VERSION_NAME: $FULL_VERSION_NAME" # Save variables as github outputs + echo "BRANCH=$BRANCH" >> $GITHUB_OUTPUT echo "VERSION_TAG=$VERSION_TAG" >> $GITHUB_OUTPUT echo "VERSION_NAME=$VERSION_NAME" >> $GITHUB_OUTPUT echo "FULL_VERSION_NAME=$FULL_VERSION_NAME" >> $GITHUB_OUTPUT @@ -93,20 +104,25 @@ jobs: env: PY_VERSION: ${{matrix.python-version}} GH_EVENT : ${{github.event_name}} + BRANCH : ${{steps.versioning.outputs.BRANCH}} run: | echo "GH_EVENT : $GH_EVENT" echo "PY_VERSION : $PY_VERSION" + echo "GIT BRANCH : $BRANCH" export OK=0 # Only deploy for push events if [[ $PY_VERSION == "3.9" ]]; then if [[ $GH_EVENT == "push" ]]; then + # BRANCH FAILS + if [[ $BRANCH == "main" ]] || [[ $BRANCH == "dev" ]] || [[ $BRANCH == "HEAD" ]] ; then export OK=1 ; + fi fi fi echo "DEPLOY : $OK" if [[ $OK == "1" ]]; then echo ">>> Deployment WILL take place" - else + else echo ">>> Deployment WILL NOT take place" fi # Save variables as github outputs From cc6a4dfdc35a2f4b8fbf89f220e0d7bddfd9e13f Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 30 Dec 2022 16:12:41 +0100 Subject: [PATCH 079/178] Exe: adding wx.lib.agw back to install package --- installer.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer.cfg b/installer.cfg index 96120a1..d562fc6 100644 --- a/installer.cfg +++ b/installer.cfg @@ -90,7 +90,7 @@ exclude= pkgs/matplotlib/backends/web_backend pkgs/wx/locale pkgs/wx/py - pkgs/wx/lib/agw +# pkgs/wx/lib/agw pkgs/wx/lib/analogclock pkgs/wx/lib/art pkgs/wx/lib/colourchooser From 0e9510f83334655f5dc1eaac4d3d25b73c098153 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 13 Jan 2023 21:05:11 -0700 Subject: [PATCH 080/178] Bug fix: merge table was broken --- pydatview/GUISelectionPanel.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 7f9a9d3..b762226 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -317,7 +317,6 @@ def OnMergeTabs(self, event): nCommonCols = len(IKeepPerTab[0]) commonCol = None ICommonColPerTab = None - samplDict = None if nCommonCols>=2: # NOTE: index will always be a duplicated... # We use the first one # TODO Menu to figure out which column to chose and how to merge (interp?) @@ -330,7 +329,7 @@ def OnMergeTabs(self, event): pass # Merge tables and add it to the list - self.tabList.mergeTabs(self.ISel, ICommonColPerTab, samplDict=samplDict) + self.tabList.mergeTabs(self.ISel, ICommonColPerTab) # Updating tables self.selPanel.update_tabs(self.tabList) # TODO select latest From 4b109aa466937a08882fc1d183a6ad1d95a6cfa9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 18 Jan 2023 10:35:39 -0700 Subject: [PATCH 081/178] Leq: using wrapper function equivalent_load --- pydatview/plotdata.py | 5 +++-- pydatview/tools/fatigue.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 127b8f4..61bd041 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -609,12 +609,13 @@ def xMin(PD): return v,s def leq(PD,m): - from pydatview.tools.fatigue import eq_load + from pydatview.tools.fatigue import equivalent_load if PD.yIsString or PD.yIsDate: return 'NA','NA' else: T,_=PD.xRange() - v=eq_load(PD.y, m=m, neq=T)[0][0] + v=equivalent_load(PD.y, m=m, Teq=T, nBins=46, method='rainflow_windap') + #v=equivalent_load(PD.y, m=m, Teq=T, nBins=46, method='fatpack') return v,pretty_num(v) def Info(PD,var): diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index 8886c1f..24b641f 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -58,7 +58,11 @@ def equivalent_load(signal, m=3, Teq=1, nBins=46, method='rainflow_windap'): elif method=='fatpack': import fatpack # find rainflow ranges - ranges = fatpack.find_rainflow_ranges(signal) + try: + ranges = fatpack.find_rainflow_ranges(signal) + except IndexError: + # Typically fails for constant signal + return np.nan # find range count and bin Nrf, Srf = fatpack.find_range_count(ranges, nBins) # get DEL From b6af6e7d485ac48e7c95955e41afadd1bbcd07a7 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 18 Jan 2023 19:20:25 -0700 Subject: [PATCH 082/178] Measure: rethinking measure behavior and implementation (#134) --- pydatview/GUIInfoPanel.py | 67 +++++---- pydatview/GUIMeasure.py | 306 +++++++++++++++++++++++++++----------- pydatview/GUIPlotPanel.py | 76 ++++++---- pydatview/common.py | 2 + pydatview/plotdata.py | 94 ++++++------ 5 files changed, 349 insertions(+), 196 deletions(-) diff --git a/pydatview/GUIInfoPanel.py b/pydatview/GUIInfoPanel.py index 7017715..0fab7ab 100644 --- a/pydatview/GUIInfoPanel.py +++ b/pydatview/GUIInfoPanel.py @@ -100,8 +100,8 @@ def __init__(self, parent, data=None): self.ColsReg.append({'name':'Abs. Max' , 'al':'R' , 'm':'yAbsMax', 's' :False}) self.ColsReg.append({'name':'Range' , 'al':'R' , 'm':'yRange', 's' :True}) self.ColsReg.append({'name':'dx' , 'al':'R' , 'm':'dx' , 's' :True}) - self.ColsReg.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) - self.ColsReg.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) + self.ColsReg.append({'name':'yMeas 1' , 'al':'R' , 'm':'ymeas1' , 's' :False}) + self.ColsReg.append({'name':'yMeas 2' , 'al':'R' , 'm':'ymeas2' , 's' :False}) self.ColsReg.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) self.ColsReg.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) self.ColsReg.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) @@ -148,8 +148,8 @@ def __init__(self, parent, data=None): self.ColsFFT.append({'name':'nWin(FFT)' , 'al':'R' , 'f':lambda x:x.Info('LWin') , 's' :False}) self.ColsFFT.append({'name':'nFFT(FFT)' , 'al':'R' , 'f':lambda x:x.Info('nFFT') , 's' :False}) self.ColsFFT.append({'name':'n(FFT)' , 'al':'R' , 'm':'ylen' , 's' :True}) - self.ColsFFT.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) - self.ColsFFT.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) + self.ColsFFT.append({'name':'yMeas 1' , 'al':'R' , 'm':'ymeas1' , 's' :False}) + self.ColsFFT.append({'name':'yMeas 2' , 'al':'R' , 'm':'ymeas2' , 's' :False}) self.ColsFFT.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) self.ColsFFT.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) self.ColsFFT.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) @@ -189,8 +189,8 @@ def __init__(self, parent, data=None): self.ColsPDF.append({'name':'Min(PDF)' , 'al':'R' , 'm':'yMin' , 's' :True}) self.ColsPDF.append({'name':'Max(PDF)' , 'al':'R' , 'm':'yMax' , 's' :True}) self.ColsPDF.append({'name':u'\u222By(PDF)' , 'al':'R' , 'm':'inty' , 's' :True}) - self.ColsPDF.append({'name':'Meas 1' , 'al':'R' , 'm':'meas1' , 's' :False}) - self.ColsPDF.append({'name':'Meas 2' , 'al':'R' , 'm':'meas2' , 's' :False}) + self.ColsPDF.append({'name':'yMeas 1' , 'al':'R' , 'm':'ymeas1' , 's' :False}) + self.ColsPDF.append({'name':'yMeas 2' , 'al':'R' , 'm':'ymeas2' , 's' :False}) self.ColsPDF.append({'name':'Mean (Meas)' , 'al':'R' , 'm':'yMeanMeas' , 's' :False}) self.ColsPDF.append({'name':'Min (Meas)' , 'al':'R' , 'm':'yMinMeas' , 's' :False}) self.ColsPDF.append({'name':'Max (Meas)' , 'al':'R' , 'm':'yMaxMeas' , 's' :False}) @@ -231,8 +231,6 @@ def __init__(self, parent, data=None): self.PD=[] self.tab_mode = None self.last_sub = False - self.meas_xy1 = (None, None) - self.meas_xy2 = (None, None) self.tbStats = TestListCtrl(self, size=(-1,100), style=wx.LC_REPORT @@ -274,11 +272,14 @@ def __init__(self, parent, data=None): # --- GUI Data def saveData(self, data): - data['ColumnsRegular'] = [c['name'] for c in self.ColsReg if c['s']] - data['ColumnsFFT'] = [c['name'] for c in self.ColsFFT if c['s']] + data['ColumnsRegular'] = [c['name'] for c in self.ColsReg if c['s']] + data['ColumnsFFT'] = [c['name'] for c in self.ColsFFT if c['s']] data['ColumnsMinMax'] = [c['name'] for c in self.ColsMinMax if c['s']] - data['ColumnsPDF'] = [c['name'] for c in self.ColsPDF if c['s']] - data['ColumnsCmp'] = [c['name'] for c in self.ColsCmp if c['s']] + data['ColumnsPDF'] = [c['name'] for c in self.ColsPDF if c['s']] + data['ColumnsCmp'] = [c['name'] for c in self.ColsCmp if c['s']] + # Not storing meas + data['ColumnsRegular'] = [c for c in data['ColumnsRegular'] if c.find('Meas')<0] + data['ColumnsFFT'] = [c for c in data['ColumnsFFT'] if c.find('Meas')<0] @staticmethod def defaultData(): @@ -352,17 +353,15 @@ def _showStats(self,erase=True): self.tbStats.InsertColumn(i,c['name'], AL[c['al']]) # Inserting items index = self.tbStats.GetItemCount() - for PD in self.PD: + for pd in self.PD: for j,c in enumerate(selCols): - # TODO: could be nicer: + # Calling dedicated function: either a function handle, f(PD), or a method, pd.m. if 'm' in c.keys(): - if c['m'] in ('meas1', 'meas2'): - v,sv=getattr(PD,c['m'])(self.meas_xy1, self.meas_xy2) - else: - v,sv=getattr(PD,c['m'])() + v,sv=getattr(pd,c['m'])() else: - v,sv=c['f'](PD) + v,sv=c['f'](pd) + # Insert items try: if j==0: self.tbStats.InsertItem(index, sv) @@ -489,20 +488,22 @@ def setCol(self, name, value): if c['name'] == name: c['s'] = value - def setMeasurements(self, xy1, xy2): - if xy1 is not None: - self.meas_xy1 = xy1 - self.menu.setItem('Meas 1', True) - self.setCol('Meas 1', True) - if xy2 is not None: - self.meas_xy2 = xy2 - for col in ['Meas 2', 'Mean (Meas)', 'Min (Meas)', 'Max (Meas)']: - self.menu.setItem(col, True) - self.setCol(col, True) - elif xy1 is None: - for col in ['Meas 1', 'Meas 2', 'Mean (Meas)', 'Min (Meas)', 'Max (Meas)']: - self.menu.setItem(col, False) - self.setCol(col, False) + def clearMeasurements(self): + # Clear + for col in ['yeas 1', 'yMeas 2', 'Mean (Meas)', 'Min (Meas)', 'Max (Meas)']: + self.menu.setItem(col, False) + self.setCol(col, False) + self._showStats() + + def showMeasure1(self): + self.menu.setItem('yMeas 1', True) + self.setCol('yMeas 1', True) + self._showStats() + + def showMeasure2(self): + for col in ['yMeas 2', 'Mean (Meas)', 'Min (Meas)', 'Max (Meas)']: + self.menu.setItem(col, True) + self.setCol(col, True) self._showStats() def clean(self): diff --git a/pydatview/GUIMeasure.py b/pydatview/GUIMeasure.py index e75f841..4f556f7 100644 --- a/pydatview/GUIMeasure.py +++ b/pydatview/GUIMeasure.py @@ -3,85 +3,191 @@ class GUIMeasure: def __init__(self, index, color): + # Main data self.index = index self.color = color - self.point = None - self.line = None - self.annotation = None - self.clear() + self.x_target = None # x closest in data where click was done + self.y_target = None # y closest in data where click was done + # Plot data + self.points = [] # Intersection points + self.lines = [] # vertical lines (per ax) + self.annotations = [] def clear(self): - self.axis_idx = -1 - self.x = None - self.y = None + self.x_target=None + self.y_target=None + self.clearPlot() + + def clearPlot(self): + """ Remove points, vertical lines and annotation from plot""" + self.clearPointLines() + self.clearAnnotations() + + def clearAnnotations(self): try: - if self.point is not None: - self.point.remove() - if self.line is not None: - self.line.remove() - if self.annotation is not None: - self.annotation.remove() - except ValueError: + [a.remove() for a in self.annotations] + except (ValueError,TypeError,AttributeError): pass - self.annotation = None - self.point = None - self.line = None + self.annotations = [] + + def clearPointLines(self): + """ Remove points, vertical lines""" + try: + [p.remove() for p in self.points] + except (ValueError,TypeError,AttributeError): + pass + try: + [l.remove() for l in self.lines] + except (ValueError,TypeError,AttributeError): + pass + self.points= [] + self.lines= [] def get_xydata(self): - if self.x is None or self.y is None: - return None + if self.x_target is None or self.y_target is None: + return None, None else: - return (self.x, self.y) + return (self.x_target, self.y_target) - def set(self, axis_idx, x, y): - self.axis_idx = axis_idx - self.x = x - self.y = y + def set(self, axes, ax, x, y, PD): + """ + - x,y : point where the user clicked (will likely be slightly off plotdata) + """ + self.clearPlot() + # Point closest to user click location + x_closest, y_closest, pd_closest = self.find_closest_point(x, y, ax) + self.x_target = x_closest + self.y_target = y_closest + self.pd_closest = pd_closest - def plot(self, ax, ax_idx): - if self.axis_idx == -1 or self.axis_idx != ax_idx: - return - try: - self.point.remove() - self.line.remove() - self.annotation.remove() - except (AttributeError, ValueError): - pass + # Plot measure where the user clicked (only for the axis that the user chose) + # - plot intersection point, vertical line, and annotation + self.plot(axes, PD) - # Hook annotation to closest on signal - x_closest = self.x - y_closest = self.y - rdist_min = 1e9 - for line in ax.get_lines(): - # TODO: check if 'if'can be avoided by using len(PD): - if str(line).startswith('Line2D(_line') is False: - xy = np.array([line.get_xdata(), line.get_ydata()]).transpose() + def compute(self, PD): + for ipd, pd in enumerate(PD): + if pd !=self.pd_closest: + XY = np.array([pd.x, pd.y]).transpose() try: - x, y = find_closest(xy, [self.x, self.y]) - rdist = abs(x - self.x) + abs(y - self.y) - if rdist < rdist_min: - rdist_min = rdist - x_closest = x - y_closest = y - except (TypeError,ValueError): - # Fails when x/y data are dates or strings - pass - self.x = x_closest - self.y = y_closest - - annotation = '{0}: ({1}, {2})'.format(self.index, - formatValue(self.x), - formatValue(self.y)) + xc, yc = find_closestX(XY, self.x_target) + pd.xyMeas[self.index-1] = (xc, yc) + except: + print('[FAIL] GUIMeasure: failed to compute closest point') + xc, yc = np.nan, np.nan + else: + # Already computed + xc, yc = self.x_target, self.y_target + pd.xyMeas[self.index-1] = (xc, yc) + + def plotAnnotation(self, ax, xc, yc): + #self.clearAnnotation() + sAnnotation = '{0}: ({1}, {2})'.format(self.index, formatValue(xc), formatValue(yc)) bbox_args = dict(boxstyle='round', fc='0.9', alpha=0.75) - self.point = ax.plot(self.x, self.y, color=self.color, marker='o', - markersize=1)[0] - self.line = ax.axvline(x=self.x, color=self.color, linewidth=0.5) - self.annotation = ax.annotate(annotation, - (self.x, self.y), - xytext=(5, -2), - textcoords='offset points', - color=self.color, - bbox=bbox_args) + annotation = ax.annotate(sAnnotation, (xc, yc), xytext=(5, -2), textcoords='offset points', color=self.color, bbox=bbox_args) + self.annotations.append(annotation) + + def plotPoint(self, ax, xc, yc, ms=3): + """ plot point at intersection""" + Mark = ['o','d','^','s'] + #i = np.random.randint(0,4) + i=0 + point = ax.plot(xc, yc, color=self.color, marker=Mark[i], markersize=ms)[0] + self.points.append(point) + + def plotLine(self, ax): + """ plot vertical line across axis""" + line = ax.axvline(x=self.x_target, color=self.color, linewidth=0.5) + self.lines.append(line) + + def plot(self, axes, PD): + """ + Given an axis, + - find intersection point + - closest to our "target" x when matchY is False + - or closest to "target" (x,y) point when matchY is True + - plot intersection point, vertical line + """ + if self.x_target is None: + return + if PD is not None: + self.compute(PD) + # Clear data + self.clearAnnotations() # Adapt if not wanted + self.clearPointLines() + + # Find interesction points and plot points + for iax, ax in enumerate(axes): + for pd in ax.PD: + if pd !=self.pd_closest: + xc, yc = pd.xyMeas[self.index-1] + self.plotPoint(ax, xc, yc, ms=3) + self.plotAnnotation(ax, xc, yc) # NOTE Comment if unwanted + else: + #xc, yc = pd.xyMeas[self.index-1] + xc, yc = self.x_target, self.y_target + self.plotPoint(ax, xc, yc, ms=6) + self.plotAnnotation(ax, xc, yc) + + # Plot lines + for iax, ax in enumerate(axes): + self.plotLine(ax) + + # Store as target if there is only one plot and one ax (important for "dx dy") + if PD is not None: + if len(axes)==1 and len(PD)==1: + self.x_target = xc + self.y_target = yc + # self.plotAnnotation(axes[0], xc, yc) + + + def find_closest_point(self, xt, yt, ax): + """ + Find closest point to target across all plotdata in a given ax + """ + # Compute axis diagonal + try: + xlim = ax.get_xlim() + ylim = ax.get_ylim() + rdist_min = np.sqrt((xlim[1]-xlim[0])**2 + (ylim[1]-ylim[0])**2) + except: + print('[FAIL] GUIMeasure: Computing axis diagonal failed') + rdist_min = 1e9 + + # --- Find closest intersection point + x_closest = xt + y_closest = yt + pd_closest= None + for pd in ax.PD: + XY = np.array([pd.x, pd.y]).transpose() + try: + x, y = find_closest(XY, [xt, yt], xlim, ylim) + rdist = abs(x - xt) + abs(y - yt) + if rdist < rdist_min: + rdist_min = rdist + x_closest = x + y_closest = y + pd_closest = pd + except (TypeError,ValueError): + # Fails when x/y data are dates or strings + print('[FAIL] GUIMeasure: find_closest failed on some data') + return x_closest, y_closest, pd_closest + + + def sDeltaX(self, meas2): + try: + dx = self.x_target - meas2.x_target + return 'dx = ' + formatValue(dx) + except: + return '' + + def sDeltaY(self, meas2): + try: + dy = self.y_target - meas2.y_target + return 'dy = ' + formatValue(dy) + except: + return '' + + def formatValue(value): @@ -95,31 +201,51 @@ def formatValue(value): return s -def find_closest(matrix, vector, single=True): - """Return closest point(s). - By default return closest single point. - Set single=False to find up to two y-values on - one x-position, where index needs to have - min. discontinuity of 1% of number of samples - and y-values need to differ at least by 5% of FS. +def find_closestX(XY, x_target): + """ return x,y values closest to a given x value """ + i = np.argmin(np.abs(XY[:,0]-x_target)) + return XY[i,:] + +def find_closest(XY, point, xlim=None, ylim=None): + """Return closest point(s), using norm2 distance + if xlim and ylim is provided, these are used to make the data non dimensional. """ # NOTE: this will fail for datetime - ind = np.argsort(np.abs(matrix - vector), axis=0) - closest = matrix[ind[0, 0]] - N = 5 - closest_Nind = ind[0:N-1] - diff = np.diff(closest_Nind[:, 0]) - discont_ind = [i for i, x in enumerate(diff) if abs(x) > (len(matrix) / 100)] - for di in discont_ind: - y = matrix[closest_Nind[di+1, 0]][1] - if abs(closest[1] - y) > (max(matrix[:, 1]) / 20): - closest = np.vstack([closest, matrix[closest_Nind[di+1, 0]]]) - break - if closest.ndim == 2: - # For multiple y-candidates find closest on y-direction: - ind_y = np.argsort(abs(closest[:, 1] - vector[1])) - closest = closest[ind_y, :] - if closest.ndim == 1 or single is False: - return closest + if xlim is not None: + x_scale = (xlim[1]-xlim[0])**2 + y_scale = (ylim[1]-ylim[0])**2 else: - return closest[0, :] + x_scale = 1 + y_scale = 1 + + norm2 = ((XY[:,0]-point[0])**2)/x_scale + ((XY[:,1]-point[1])**2)/y_scale + ind = np.argmin(norm2, axis=0) + return XY[ind,:] + + # --- Old method + ## By default return closest single point. + ## Set single=False to find up to two y-values on + ## one x-position, where index needs to have + ## min. discontinuity of 1% of number of samples + ## and y-values need to differ at least by 5% of FS. + ##ind = np.argsort(np.abs(XY - point), axis=0) # TODO use norm instead of abs + #ind = np.argsort(norm2, axis=0) + #closest = XY[ind[0, 0]] + + #N = 5 + #closest_Nind = ind[0:N-1] + #diff = np.diff(closest_Nind[:, 0]) + #discont_ind = [i for i, x in enumerate(diff) if abs(x) > (len(XY) / 100)] + #for di in discont_ind: + # y = XY[closest_Nind[di+1, 0]][1] + # if abs(closest[1] - y) > (max(XY[:, 1]) / 20): + # closest = np.vstack([closest, XY[closest_Nind[di+1, 0]]]) + # break + #if closest.ndim == 2: + # # For multiple y-candidates find closest on y-direction: + # ind_y = np.argsort(abs(closest[:, 1] - point[1])) + # closest = closest[ind_y, :] + #if closest.ndim == 1 or single is False: + # return closest + #else: + # return closest[0, :] diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index adbda7c..323bb8e 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -158,6 +158,7 @@ def __init__(self, parent): def onXlimChange(self,event=None): self.parent.redraw_same_data(); + def onSpecCtrlChange(self,event=None): if self.cbAveraging.GetStringSelection()=='None': self.scP2.Enable(False) @@ -499,7 +500,6 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbGrid ) self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbStepPlot ) self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) - self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) # LAYOUT cb_sizer = wx.FlexGridSizer(rows=4, cols=3, hgap=0, vgap=0) cb_sizer.Add(self.cbCurveType , 0, flag=wx.ALL, border=1) @@ -669,7 +669,17 @@ def plot_matrix_select(self, event): def measure_select(self, event): if self.cbMeasure.IsChecked(): - self.cbAutoScale.SetValue(False) + pass + #self.cbAutoScale.SetValue(False) + else: + # We clear + for measure in [self.leftMeasure, self.rightMeasure]: + measure.clear() + if self.infoPanel is not None: + self.infoPanel.clearMeasurements() + self.lbDeltaX.SetLabel('') + self.lbDeltaY.SetLabel('') + self.redraw_same_data() def redraw_event(self, event): @@ -730,35 +740,32 @@ def onMouseClick(self, event): def onMouseRelease(self, event): if self.cbMeasure.GetValue(): - for ax, ax_idx in zip(self.fig.axes, range(len(self.fig.axes))): + # Loop on axes + for iax, ax in enumerate(self.fig.axes): if event.inaxes == ax: x, y = event.xdata, event.ydata if self.clickLocation != (ax, x, y): # Ignore measurements for zoom-actions. Possibly add small tolerance. # Zoom-actions disable autoscale - self.cbAutoScale.SetValue(False) + #self.cbAutoScale.SetValue(False) return if event.button == 1: + # Left click, measure 1 - set values, compute all intersections and plot + self.leftMeasure.set(self.fig.axes, ax, x, y, self.plotData) # Set and plot if self.infoPanel is not None: - self.infoPanel.setMeasurements((x, y), None) - self.leftMeasure.set(ax_idx, x, y) - self.leftMeasure.plot(ax, ax_idx) + self.infoPanel.showMeasure1() elif event.button == 3: + # Left click, measure 2 - set values, compute all intersections and plot + self.rightMeasure.set(self.fig.axes, ax, x, y, self.plotData) # Set and plot if self.infoPanel is not None: - self.infoPanel.setMeasurements(None, (x, y)) - self.rightMeasure.set(ax_idx, x, y) - self.rightMeasure.plot(ax, ax_idx) + self.infoPanel.showMeasure2() else: return if not self.cbAutoScale.IsChecked(): self._restore_limits() - - if self.leftMeasure.axis_idx == self.rightMeasure.axis_idx and self.leftMeasure.axis_idx != -1: - self.lbDeltaX.SetLabel('dx=' + self.formatLabelValue(self.rightMeasure.x - self.leftMeasure.x)) - self.lbDeltaY.SetLabel('dy=' + self.formatLabelValue(self.rightMeasure.y - self.leftMeasure.y)) - else: - self.lbDeltaX.SetLabel('') - self.lbDeltaY.SetLabel('') + # Update label + self.lbDeltaX.SetLabel(self.rightMeasure.sDeltaX(self.leftMeasure)) + self.lbDeltaY.SetLabel(self.rightMeasure.sDeltaY(self.leftMeasure)) return def onDraw(self, event): @@ -999,18 +1006,9 @@ def set_axes_lim(self, PDs, axis): def plot_all(self, keep_limits=True): self.multiCursors=[] - if self.cbMeasure.GetValue() is False: - for measure in [self.leftMeasure, self.rightMeasure]: - measure.clear() - if self.infoPanel is not None: - self.infoPanel.setMeasurements(None, None) - self.lbDeltaX.SetLabel('') - self.lbDeltaY.SetLabel('') - axes=self.fig.axes PD=self.plotData - # --- Plot options bStep = self.cbStepPlot.IsChecked() plot_options = dict() @@ -1065,11 +1063,6 @@ def plot_all(self, keep_limits=True): __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) - if self.infoPanel is not None: - self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) - for measure in [self.leftMeasure, self.rightMeasure]: - measure.plot(ax_left, axis_idx) - # Log Axes if self.cbLogX.IsChecked(): try: @@ -1177,6 +1170,24 @@ def plot_all(self, keep_limits=True): for ax in axes: ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) + # --- End loop on axes + # --- Measure + if self.cbMeasure.IsChecked(): + # Compute and plot them + self.leftMeasure.plot (axes, self.plotData) + self.rightMeasure.plot(axes, self.plotData) + # Update dx,dy label + self.lbDeltaX.SetLabel(self.rightMeasure.sDeltaX(self.leftMeasure)) + self.lbDeltaY.SetLabel(self.rightMeasure.sDeltaY(self.leftMeasure)) + ## Update info panel + #if self.infoPanel is not None: + # self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) + else: + # Update dx,dy label + self.lbDeltaX.SetLabel('') + self.lbDeltaY.SetLabel('') + + # --- xlabel axes[-1].set_xlabel(PD[axes[-1].iPD[0]].sx, **font_options) #print('sy :',[pd.sy for pd in PD]) @@ -1340,6 +1351,9 @@ def distributePlots(self,mode,nSubPlots,spreadBy): axes[i].iPD.append(ipd) else: raise Exception('Wrong spreadby value') + # Use PD + for ax in axes: + ax.PD=[self.plotData[i] for i in ax.iPD] def setLegendLabels(self,mode): """ Set labels for legend """ diff --git a/pydatview/common.py b/pydatview/common.py index 5e7cb6d..1bbccdd 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -341,6 +341,8 @@ def pretty_time(t): return s def pretty_num(x): + if np.isnan(x): + return 'NA' if abs(x)<1000 and abs(x)>1e-4: return "{:9.4f}".format(x) else: diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 61bd041..6051818 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -1,9 +1,8 @@ import os import numpy as np -from .common import no_unit, unit, inverse_unit, has_chinese_char -from .common import isString, isDate, getDt -from .common import unique, pretty_num, pretty_time -from .GUIMeasure import find_closest # Should not depend on wx +from pydatview.common import no_unit, unit, inverse_unit, has_chinese_char +from pydatview.common import isString, isDate, getDt +from pydatview.common import unique, pretty_num, pretty_time class PlotData(): """ @@ -31,6 +30,29 @@ def __init__(PD, x=None, y=None, sx='', sy=''): PD.xIsDate =False # true if dates PD.yIsString=False # true if strings PD.yIsDate =False # true if dates + # Misc data + PD._xMin = None + PD._xMax = None + PD._yMin = None + PD._yMax = None + PD._xAtYMin = None + PD._xAtYMax = None + # Backup data + PD._x0Min = None + PD._x0Max = None + PD._y0Min = None + PD._y0Max = None + PD._x0AtYMin = None + PD._x0AtYMax = None + PD._y0Std = None + PD._y0Mean = None + PD._n0 = None + PD.x0 = None + PD.y0 = None + # Store xyMeas input values so we don't need to recompute xyMeas in case they didn't change + PD.xyMeasInput1 = (None, None) + PD.xyMeasInput2 = (None, None) + PD.xyMeas = [(None,None)]*2 # 2 measures for now if x is not None and y is not None: PD.fromXY(x,y,sx,sy) @@ -101,12 +123,10 @@ def _post_init(PD, pipeline=None): PD._n0 = (n,'{:d}'.format(n)) PD.x0 =PD.x PD.y0 =PD.y - # Store xyMeas input values so we don't need to recompute xyMeas in case they didn't change - PD.xyMeasInput1, PD.xyMeasInput2 = None, None - PD.xyMeas1, PD.xyMeas2 = None, None def __repr__(s): s1='id:{}, it:{}, ix:{}, iy:{}, sx:"{}", sy:"{}", st:{}, syl:{}\n'.format(s.id,s.it,s.ix,s.iy,s.sx,s.sy,s.st,s.syl) + #s1='id:{}, it:{}, sx:"{}", xyMeas:{}\n'.format(s.id,s.it,s.sx,s.xyMeas) return s1 def toPDF(PD, nBins=30, smooth=False): @@ -496,24 +516,25 @@ def intyx2(PD): s=pretty_num(v) return v,s - def meas1(PD, xymeas1, xymeas2): - if PD.xyMeasInput1 is not None and PD.xyMeasInput1 == xymeas1: - yv = PD.xyMeas1[1] + # --------------------------------------------------------------------------------} + # --- Measure - TODO: cleanup + # --------------------------------------------------------------------------------{ + def ymeas1(PD): + # NOTE: calculation happens in GUIMeasure.. + if PD.xyMeas[0][0] is not None: + yv = PD.xyMeas[0][1] s = pretty_num(yv) else: - xv, yv, s = PD._meas(xymeas1) - PD.xyMeas1 = [xv, yv] - PD.xyMeasInput1 = xymeas1 + return np.nan, 'NA' return yv, s - def meas2(PD, xymeas1, xymeas2): - if PD.xyMeasInput2 is not None and PD.xyMeasInput2 == xymeas2: - yv = PD.xyMeas2[1] + def ymeas2(PD): + # NOTE: calculation happens in GUIMeasure.. + if PD.xyMeas[1][0] is not None: + yv = PD.xyMeas[1][1] s = pretty_num(yv) else: - xv, yv, s = PD._meas(xymeas2) - PD.xyMeas2 = [xv, yv] - PD.xyMeasInput2 = xymeas2 + return np.nan, 'NA' return yv, s def yMeanMeas(PD): @@ -531,31 +552,17 @@ def xAtYMinMeas(PD): def xAtYMaxMeas(PD): return PD._measCalc('xmax') - def _meas(PD, xymeas): - try: - xv, yv = 'NA', 'NA' - xy = np.array([PD.x, PD.y]).transpose() - points = find_closest(xy, [xymeas[0], xymeas[1]], False) - if points.ndim == 1: - xv, yv = points[0:2] - s = pretty_num(yv) - else: - xv, yv = points[0, 0], points[0, 1] - s = ' / '.join([str(p) for p in points[:, 1]]) - except (IndexError, TypeError): - xv, yv = 'NA', 'NA' - s='NA' - return xv, yv, s - def _measCalc(PD, mode): - if PD.xyMeas1 is None or PD.xyMeas2 is None: - return 'NA', 'NA' + if PD.xyMeas[0][0] is None or PD.xyMeas[1][0] is None: + return np.nan, 'NA' + if np.isnan(PD.xyMeas[0][0]) or np.isnan(PD.xyMeas[1][0]): + return np.nan, 'NA' try: - v = 'NA' - left_index = np.where(PD.x == PD.xyMeas1[0])[0][0] - right_index = np.where(PD.x == PD.xyMeas2[0])[0][0] + v = np.nan + left_index = np.argmin(np.abs(PD.x - PD.xyMeas[0][0])) + right_index = np.argmin(np.abs(PD.x - PD.xyMeas[1][0])) if left_index == right_index: - raise IndexError + return np.nan, 'Empty' if left_index > right_index: left_index, right_index = right_index, left_index if mode == 'mean': @@ -572,10 +579,13 @@ def _measCalc(PD, mode): raise NotImplementedError('Error: Mode ' + mode + ' not implemented') s = pretty_num(v) except (IndexError, TypeError): - v = 'NA' + v = np.nan s = 'NA' return v, s + # --------------------------------------------------------------------------------} + # --- Other Stats functions + # --------------------------------------------------------------------------------{ def dx(PD): if len(PD.x)<=1: return 'NA','NA' From 194470dc83342b6211bb855ed9e05d576f5b2a40 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 18 Jan 2023 19:38:17 -0700 Subject: [PATCH 083/178] IO: adding pickle file --- pydatview/GUIInfoPanel.py | 2 +- pydatview/io/__init__.py | 2 + pydatview/io/fast_input_file_graph.py | 20 ++-- pydatview/io/fast_summary_file.py | 20 ++-- pydatview/io/pickle_file.py | 166 ++++++++++++++++++++++++++ 5 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 pydatview/io/pickle_file.py diff --git a/pydatview/GUIInfoPanel.py b/pydatview/GUIInfoPanel.py index 0fab7ab..e67f151 100644 --- a/pydatview/GUIInfoPanel.py +++ b/pydatview/GUIInfoPanel.py @@ -490,7 +490,7 @@ def setCol(self, name, value): def clearMeasurements(self): # Clear - for col in ['yeas 1', 'yMeas 2', 'Mean (Meas)', 'Min (Meas)', 'Max (Meas)']: + for col in ['yMeas 1', 'yMeas 2', 'Mean (Meas)', 'Min (Meas)', 'Max (Meas)']: self.menu.setItem(col, False) self.setCol(col, False) self._showStats() diff --git a/pydatview/io/__init__.py b/pydatview/io/__init__.py index a09ca6c..b48f7fe 100644 --- a/pydatview/io/__init__.py +++ b/pydatview/io/__init__.py @@ -56,6 +56,7 @@ def fileFormats(userpath=None, ignoreErrors=False, verbose=False): from .vtk_file import VTKFile from .bladed_out_file import BladedFile from .parquet_file import ParquetFile + from .pickle_file import PickleFile from .cactus_file import CactusFile from .raawmat_file import RAAWMatFile from .rosco_performance_file import ROSCOPerformanceFile @@ -95,6 +96,7 @@ def addFormat(priority, fmt): addFormat(60, FileFormat(VTKFile)) addFormat(60, FileFormat(TDMSFile)) addFormat(60, FileFormat(ParquetFile)) + addFormat(60, FileFormat(PickleFile)) addFormat(70, FileFormat(CactusFile)) addFormat(70, FileFormat(RAAWMatFile)) diff --git a/pydatview/io/fast_input_file_graph.py b/pydatview/io/fast_input_file_graph.py index 3cb55a4..b4a558c 100644 --- a/pydatview/io/fast_input_file_graph.py +++ b/pydatview/io/fast_input_file_graph.py @@ -189,17 +189,18 @@ def type2Color(Pot): Graph.addMiscPropertySet('MemberCoefs') for ip,P in enumerate(hd['MemberProp']): # MemberID MemberCd1 MemberCd2 MemberCdMG1 MemberCdMG2 MemberCa1 MemberCa2 MemberCaMG1 MemberCaMG2 MemberCp1 MemberCp2 MemberCpMG1 MemberCpMG2 MemberAxCd1 MemberAxCd2 MemberAxCdMG1 MemberAxCdMG2 MemberAxCa1 MemberAxCa2 MemberAxCaMG1 MemberAxCaMG2 MemberAxCp1 MemberAxCp2 MemberAxCpMG1 MemberAxCpMG2 - prop = Property(ID=ip+1, MemberID=P[0], Cd1=P[1], Cd2=P[2], CdMG1=P[3], CdMG2=P[4], Ca1=P[5], Ca2=P[6], CaMG1=P[7], CaMG2=P[8], Cp1=P[9], Cp2=P[10], CpMG1=P[11], CpMG2=P[12], AxCd1=P[14], AxCd2=P[15], axCdMG1=P[16], axCdMG2=P[17], AxCa1=P[18], AxCa2=P[19], AxCaMG1=P[20], AxCaMG2=P[21], AxCp1=P[22], AxCp2=P[23]) + prop = ElemProperty(ID=ip+1, MemberID=P[0], Cd1=P[1], Cd2=P[2], CdMG1=P[3], CdMG2=P[4], Ca1=P[5], Ca2=P[6], CaMG1=P[7], CaMG2=P[8], Cp1=P[9], Cp2=P[10], CpMG1=P[11], CpMG2=P[12], AxCd1=P[14], AxCd2=P[15], axCdMG1=P[16], axCdMG2=P[17], AxCa1=P[18], AxCa2=P[19], AxCaMG1=P[20], AxCaMG2=P[21], AxCp1=P[22], AxCp2=P[23]) Graph.addMiscProperty('MemberCoefs',prop) # --- if 'FillGroups' in hd.keys(): # Filled members Graph.addMiscPropertySet('FillGroups') - for ip,P in enumerate(hd['FillGroups']): - # FillNumM FillMList FillFSLoc FillDens - raise NotImplementedError('hydroDynToGraph, Fill List might not be properly set, verify below') - prop = MiscProperty(ID=ip+1, FillNumM=P[0], FillMList=P[1], FillFSLoc=P[2], FillDens=P[3]) - Graph.addMiscProperty('FillGroups',prop) + print('>>> TODO Filled Groups') + #for ip,P in enumerate(hd['FillGroups']): + # # FillNumM FillMList FillFSLoc FillDens + # raise NotImplementedError('hydroDynToGraph, Fill List might not be properly set, verify below') + # prop = MiscProperty(ID=ip+1, FillNumM=P[0], FillMList=P[1], FillFSLoc=P[2], FillDens=P[3]) + # Graph.addMiscProperty('FillGroups',prop) if 'MGProp' in hd.keys(): # Marine Growth @@ -242,7 +243,7 @@ def type2Color(Pot): else: print('>>> TODO type DepthCoefs and MemberCoefs') # NOTE: this is disallowed by default because a same node can have two different diameters in SubDyn (it's by element) - Graph.setElementNodalProp(elem, propset=PropSets[Type-1], propIDs=E[3:5]) + #Graph.setElementNodalProp(elem, propset=PropSets[Type-1], propIDs=E[3:5]) return Graph @@ -250,7 +251,7 @@ def type2Color(Pot): # --------------------------------------------------------------------------------} # --- SubDyn Summary file # --------------------------------------------------------------------------------{ -def subdynSumToGraph(data): +def subdynSumToGraph(data, Graph=None): """ data: dict-like object as returned by weio """ @@ -266,7 +267,8 @@ def subdynSumToGraph(data): DOF2Nodes = data['DOF2Nodes'] nDOF = data['nDOF_red'] - Graph = GraphModel() + if Graph is None: + Graph = GraphModel() # --- Nodes and DOFs Nodes = data['Nodes'] diff --git a/pydatview/io/fast_summary_file.py b/pydatview/io/fast_summary_file.py index 735c702..e435215 100644 --- a/pydatview/io/fast_summary_file.py +++ b/pydatview/io/fast_summary_file.py @@ -149,7 +149,7 @@ def NodesDisp(self, IDOF, UDOF, maxDisp=None, sortDim=None): pos = pos[I,:] return disp, pos, INodes - def getModes(data, maxDisp=None, sortDim=2): + def getModes(data, maxDisp=None, sortDim=None): """ return Guyan and CB modes""" if maxDisp is None: #compute max disp such as it's 10% of maxdimension @@ -181,11 +181,12 @@ def subDynToJson(data, outfile=None): TODO: convert to graph and use graph.toJSON """ + #return data.toGraph().toJSON(outfile) - dispGy, posGy, _, dispCB, posCB, _ = data.getModes() + dispGy, posGy, _, dispCB, posCB, _ = data.getModes(sortDim=None) # Sorting mess things up - Nodes = self['Nodes'] - Elements = self['Elements'] + Nodes = self['Nodes'].copy() + Elements = self['Elements'].copy() Elements[:,0]-=1 Elements[:,1]-=1 Elements[:,2]-=1 @@ -221,7 +222,7 @@ def subDynToJson(data, outfile=None): return d - def subDynToDataFrame(data): + def subDynToDataFrame(data, sortDim=2, removeZero=True): """ Convert to DataFrame containing nodal displacements """ def toDF(pos,disp,preffix=''): disp[np.isnan(disp)]=0 @@ -234,14 +235,15 @@ def toDF(pos,disp,preffix=''): disptot= np.moveaxis(disptot,2,1).reshape(disptot.shape[0],disptot.shape[1]*disptot.shape[2]) disp = np.moveaxis(disp,2,1).reshape(disp.shape[0],disp.shape[1]*disp.shape[2]) df= pd.DataFrame(data = disptot ,columns = columns) - # remove zero dfDisp= pd.DataFrame(data = disp ,columns = columns) - df = df.loc[:, (dfDisp != 0).any(axis=0)] - dfDisp = dfDisp.loc[:, (dfDisp != 0).any(axis=0)] + # remove mode components that are fully zero + if removeZero: + df = df.loc[:, (dfDisp != 0).any(axis=0)] + dfDisp = dfDisp.loc[:, (dfDisp != 0).any(axis=0)] dfDisp.columns = [c.replace('Mode','Disp') for c in dfDisp.columns.values] return df, dfDisp - dispGy, posGy, _, dispCB, posCB, _ = data.getModes() + dispGy, posGy, _, dispCB, posCB, _ = data.getModes(sortDim=sortDim) columns = ['z_[m]','x_[m]','y_[m]'] dataZXY = np.column_stack((posGy[:,2],posGy[:,0],posGy[:,1])) diff --git a/pydatview/io/pickle_file.py b/pydatview/io/pickle_file.py new file mode 100644 index 0000000..36713a2 --- /dev/null +++ b/pydatview/io/pickle_file.py @@ -0,0 +1,166 @@ +""" +Input/output class for the pickle fileformats +""" +import numpy as np +import pandas as pd +import os +import pickle +import builtins + +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File=dict + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + +class PickleFile(File): + """ + Read/write a pickle file. The object behaves as a dictionary. + + Main methods + ------------ + - read, write, toDataFrame, keys + + Examples + -------- + f = PickleFile('file.pkl') + print(f.keys()) + print(f.toDataFrame().columns) + + """ + + @staticmethod + def defaultExtensions(): + """ List of file extensions expected for this fileformat""" + return ['.pkl'] + + @staticmethod + def formatName(): + """ Short string (~100 char) identifying the file format""" + return 'Pickle file' + + @staticmethod + def priority(): return 60 # Priority in weio.read fileformat list between 0=high and 100:low + + + def __init__(self, filename=None, data=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename and not data: + self.read(**kwargs) + if data: + self._setData(data) + if filename: + self.write() + + def _setData(self, data): + if isinstance(data, dict): + for k,v in data.items(): + self[k] = v + else: + self['data'] = data + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # Reads self.filename and stores data into self. Self is (or behaves like) a dictionary + # If pickle data is a dict we store its keys in self, otherwise with store the pickle in the "data" key + d = pickle.load(open(self.filename, 'rb')) + self._setData(d) + + def write(self, filename=None): + """ Rewrite object to file, or write object to `filename` if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + with open(self.filename, 'wb') as fid: + pickle.dump(dict(self), fid) + + def toDataFrame(self): + """ Returns object into one DataFrame, or a dictionary of DataFrames""" + dfs={} + for k,v in self.items(): + if isinstance(v, pd.DataFrame): + dfs[k] = v + elif isinstance(v, np.ndarray): + if len(v.shape)==2: + dfs[k] = pd.DataFrame(data=v, columns=['C{}'.format(i) for i in range(v.shape[1])]) + elif len(v.shape)==1: + dfs[k] = pd.DataFrame(data=v, columns=[k]) + if len(dfs)==1: + dfs=dfs[list(dfs.keys())[0]] + return dfs + + # --- Optional functions + def __repr__(self): + """ String that is written to screen when the user calls `print()` on the object. + Provide short and relevant information to save time for the user. + """ + s='<{} object>:\n'.format(type(self).__name__) + s+='|Main attributes:\n' + s+='| - filename: {}\n'.format(self.filename) + # --- Example printing some relevant information for user + s+='|Main keys:\n' + for k,v in self.items(): + try: + s+='| - {}: type:{} shape:{}\n'.format(k,type(v),v.shape) + except: + try: + s+='| - {}: type:{} len:{}\n'.format(k,type(v), len(v)) + except: + s+='| - {}: type:{}\n'.format(k,type(v)) + s+='|Main methods:\n' + s+='| - read, write, toDataFrame, keys' + return s + + + # --- Functions speficic to filetype + def toGlobal(self, namespace=None, overwrite=True, verbose=False, force=False) : + #def toGlobal(self, **kwargs): + """ + NOTE: very dangerous, mostly works for global, but then might infect everything + + Inject variables (keys of read dict) into namespace (e.g. globals()). + By default, the namespace of the caller is used + To use the global namespace, use namespace=globals() + """ + import inspect + st = inspect.stack() + if len(st)>2: + if not force: + raise Exception('toGlobal is very dangerous, only use in isolated script. use `force=True` if you really know what you are doing') + else: + print('[WARN] toGlobal is very dangerous, only use in isolated script') + if namespace is None: + # Using parent local namespace + namespace = inspect.currentframe().f_back.f_globals + #namespace = inspect.currentframe().f_back.f_locals # could use f_globals + # Using global (difficult, overwriting won't work) It's best if the user sets namespace=globals() + # import builtins as _builtins + # namespace = _builtins # NOTE: globals() is for the package globals only, we need "builtins" + + gl_keys = list(namespace.keys()) + for k,v in self.items(): + if k in gl_keys: + if not overwrite: + print('[INFO] not overwritting variable {}, already present in global namespace'.format(k)) + continue + else: + print('[WARN] overwritting variable {}, already present in global namespace'.format(k)) + + if verbose: + print('[INFO] inserting in namespace: {}'.format(k)) + namespace[k] = v # OR do: builtins.__setattr__(k,v) + From 61aceb53049237fbd84299afa8b2bfd8d4d2f1be Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 18 Jan 2023 19:41:42 -0700 Subject: [PATCH 084/178] Fatigue: now using fatpack for equivalent loads --- installer.cfg | 1 + pydatview/plotdata.py | 9 +++++++-- pydatview/tools/fatigue.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/installer.cfg b/installer.cfg index d562fc6..c6d5941 100644 --- a/installer.cfg +++ b/installer.cfg @@ -34,6 +34,7 @@ pypi_wheels = pyarrow==8.0.0 Pillow==9.1.1 packaging==21.2 + fatpack==0.7.3 # numpy==1.19.3 # wxPython==4.0.3 diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 6051818..4ede5ee 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -624,8 +624,13 @@ def leq(PD,m): return 'NA','NA' else: T,_=PD.xRange() - v=equivalent_load(PD.y, m=m, Teq=T, nBins=46, method='rainflow_windap') - #v=equivalent_load(PD.y, m=m, Teq=T, nBins=46, method='fatpack') + try: + import fatpack + method='fatpack' + except ModuleNotFoundError: + print('[INFO] module fatpack not installed, default to windap method for equivalent load') + method='rainflow_windap' + v=equivalent_load(PD.y, m=m, Teq=T, nBins=100, method=method) return v,pretty_num(v) def Info(PD,var): diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index 24b641f..91649eb 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -31,7 +31,7 @@ __all__ = ['rainflow_astm', 'rainflow_windap','eq_load','eq_load_and_cycles','cycle_matrix','cycle_matrix2'] -def equivalent_load(signal, m=3, Teq=1, nBins=46, method='rainflow_windap'): +def equivalent_load(signal, m=3, Teq=1, nBins=100, method='rainflow_windap'): """Equivalent load calculation Calculate the equivalent loads for a list of Wohler exponent From 5c297eeeca1c766a4de912dad228a8c6b613e416 Mon Sep 17 00:00:00 2001 From: SimonHH <38310787+SimonHH@users.noreply.github.com> Date: Wed, 22 Feb 2023 09:40:29 +0100 Subject: [PATCH 085/178] Update data_mask.py Add example for the use of masks for variables that are text. --- pydatview/plugins/data_mask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index f2f9f9c..e683b39 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -80,7 +80,7 @@ def __init__(self, parent, action): self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better - self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'")""") + self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'" or "['substring' in str(x) for x in {string_variable}]")""") self.textMask = wx.TextCtrl(self, wx.ID_ANY, 'Dummy', style = wx.TE_PROCESS_ENTER) #self.textMask.SetValue('({Time}>100) & ({Time}<400)') #self.textMask.SetValue("{Date} > '2018-10-01'") From 450e90379d17e3b7d90374137cd0edfac01cc0b6 Mon Sep 17 00:00:00 2001 From: SimonHH <38310787+SimonHH@users.noreply.github.com> Date: Thu, 9 Mar 2023 10:07:59 +0100 Subject: [PATCH 086/178] Merging with try and except --- pydatview/Tables.py | 49 +++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 23c4ffe..36f56cb 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -220,28 +220,33 @@ def mergeTabs(self, I=None, ICommonColPerTab=None, extrap='nan'): # Remove duplicated columns #df = df.loc[:,~df.columns.duplicated()].copy() else: - # --- Option 1 - We combine all the x from the common column together - # NOTE: We use unique and sort, which will distrupt the user data (e.g. Airfoil Coords) - # The user should then use other methods (when implemented) - x_new=[] - cols = [] - for it, icol in zip(I, ICommonColPerTab): - xtab = self._tabs[it].data.iloc[:, icol].values - cols.append(self._tabs[it].data.columns[icol]) - x_new = np.concatenate( (x_new, xtab) ) - x_new = np.unique(np.sort(x_new)) - # Create interpolated dataframes based on x_new - dfs_new = [] - for i, (col, df_old) in enumerate(zip(cols, dfs)): - df = interpDF(x_new, col, df_old, extrap=extrap) - if 'Index' in df.columns: - df = df.drop(['Index'], axis=1) - if i>0: - df = df.drop([col], axis=1) - dfs_new.append(df) - df = pd.concat(dfs_new, axis=1) - # Reindex at the end - df.insert(0, 'Index', np.arange(df.shape[0])) + try: + # --- Option 1 - We combine all the x from the common column together + # NOTE: We use unique and sort, which will distrupt the user data (e.g. Airfoil Coords) + # The user should then use other methods (when implemented) + x_new=[] + cols = [] + for it, icol in zip(I, ICommonColPerTab): + xtab = self._tabs[it].data.iloc[:, icol].values + cols.append(self._tabs[it].data.columns[icol]) + x_new = np.concatenate( (x_new, xtab) ) + x_new = np.unique(np.sort(x_new)) + # Create interpolated dataframes based on x_new + dfs_new = [] + for i, (col, df_old) in enumerate(zip(cols, dfs)): + df = interpDF(x_new, col, df_old, extrap=extrap) + if 'Index' in df.columns: + df = df.drop(['Index'], axis=1) + if i>0: + df = df.drop([col], axis=1) + dfs_new.append(df) + df = pd.concat(dfs_new, axis=1) + # Reindex at the end + df.insert(0, 'Index', np.arange(df.shape[0])) + except: + # --- Option 0 - Index concatenation + print('Using dataframe index concatenation...') + df = pd.concat(dfs, axis=1) newName = self._tabs[I[0]].name+'_merged' self.append(Table(data=df, name=newName)) return newName, df From 6f32a65ddc91db7d2aca7514d9ede92ed88dbc29 Mon Sep 17 00:00:00 2001 From: SimonHH <38310787+SimonHH@users.noreply.github.com> Date: Thu, 9 Mar 2023 13:21:37 +0100 Subject: [PATCH 087/178] bladed_out: allow >2 lines for AXITICK We might want to do that for AXIVAL and other keys in a similar way. In this case we can reuse combined_string. --- pydatview/io/bladed_out_file.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pydatview/io/bladed_out_file.py b/pydatview/io/bladed_out_file.py index f08e2d1..f6ac45c 100644 --- a/pydatview/io/bladed_out_file.py +++ b/pydatview/io/bladed_out_file.py @@ -70,13 +70,18 @@ def read_bladed_sensor_file(sensorfile): # sometimes, the info is written on "AXIVAL" # Check next line, we concatenate if doesnt start with AXISLAB (Might need more cases) try: - nextLine=sensorLines[i+1].strip() - if not nextLine.startswith('AXISLAB'): - t_line = t_line.strip()+' '+nextLine + # Combine the strings into one string + combined_string = ''.join(sensorLines) + + # Search everything betwee AXITICK and AXISLAB with a regex pattern + t_line = re.search(r'(?<=AXITICK).+?(?=AXISLAB)', combined_string, flags=re.DOTALL) + t_line=t_line.group(0) + # Replace consecutive whitespace characters with a single space + t_line = re.sub('\s+', ' ', t_line) except: pass - temp = t_line[7:].strip() + temp = t_line.strip() temp = temp.strip('\'').split('\' \'') dat['SectionList'] = np.array(temp, dtype=str) dat['nSections'] = len(dat['SectionList']) From ba40108342017c729ff680eeead41db8049d8125 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 3 Apr 2023 15:40:01 -0600 Subject: [PATCH 088/178] Formula: adding rpm2Hz, radps2Hz and diff --- pydatview/GUISelectionPanel.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index b762226..e284d46 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -41,9 +41,10 @@ def __init__(self, title='', name='', formula='',columns=[],unit='',xcol='',xuni name=self.getDefaultName() self.formula_in=formula + colPreDef=['None','x 1000','/ 1000','deg2rad','rad2deg','rpm2radps','rpm2Hz','radps2rpm','radps2Hz','norm','squared','d/dx','diff'] quick_lbl = wx.StaticText(self, label="Predefined: " ) - self.cbQuick = wx.ComboBox(self, choices=['None','x 1000','/ 1000','deg2rad','rad2deg','rpm2radps','radps2rpm','norm','squared','d/dx'], style=wx.CB_READONLY) + self.cbQuick = wx.ComboBox(self, choices=colPreDef, style=wx.CB_READONLY) self.cbQuick.SetSelection(0) self.cbQuick.Bind(wx.EVT_COMBOBOX ,self.onQuickFormula) @@ -76,6 +77,8 @@ def __init__(self, title='', name='', formula='',columns=[],unit='',xcol='',xuni info+=' - ` {ColA} + {ColB} `\n' info+=' - ` np.sqrt( {ColA}**2/1000 + 1/{ColB}**2 ) `\n' info+=' - ` np.sin ( {ColA}*2*np.pi + {ColB} ) `\n' + info+=' - etc.\n\n' + info+=' You can also use the `Predefined` menu on the right to select a common operation.\n' help_lbl = wx.StaticText(self, label='Help: ') info_lbl = wx.StaticText(self, label=info) help_sizer = wx.BoxSizer(wx.HORIZONTAL) @@ -212,9 +215,15 @@ def onQuickFormula(self, event): elif s=='rpm2radps': self.formula.SetValue(c1+' *2*np.pi/60') self.name.SetValue(n1+'_radps [rad/s]') + elif s=='rpm2Hz': + self.formula.SetValue(c1+'/60') + self.name.SetValue(n1+'_Freq_[Hz]') elif s=='radps2rpm': self.formula.SetValue(c1+' *60/(2*np.pi)') self.name.SetValue(n1+'_rpm [rpm]') + elif s=='radps2Hz': + self.formula.SetValue(c1+' /(2*np.pi)') + self.name.SetValue(n1+'_Freq_[Hz]') elif s=='norm': self.formula.SetValue('np.sqrt( '+'**2 + '.join(self.columns)+'**2 )') self.name.SetValue(n1+'_norm'+self.get_unit()) @@ -246,6 +255,9 @@ def onQuickFormula(self, event): else: n1='d('+n1+')/d('+nx+')' self.name.SetValue(n1+self.get_deriv_unit()) + elif s=='diff': + self.formula.SetValue('np.concatenate( [ [np.nan], np.diff( '+c1+' ) ] )' ) + self.name.SetValue(n1+'_diff'+ self.get_unit()) else: raise Exception('Unknown quick formula {}'.s) From a0ff6b4a869fa9e6948b9e5e8d457625e2bd8f95 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 23 Jun 2023 17:48:00 -0600 Subject: [PATCH 089/178] IO/Tools: updates from welib --- README.md | 15 +- pydatview/Tables.py | 4 +- pydatview/fast/postpro.py | 368 +++++++++++++++++------- pydatview/io/__init__.py | 4 + pydatview/io/csv_file.py | 2 - pydatview/io/fast_input_deck.py | 96 ++++--- pydatview/io/fast_input_file.py | 337 ++++++++++++++++++---- pydatview/io/fast_linearization_file.py | 3 - pydatview/io/fast_output_file.py | 104 ++++--- pydatview/io/file.py | 3 +- pydatview/io/flex_out_file.py | 28 +- pydatview/io/mannbox_file.py | 8 +- pydatview/io/netcdf_file.py | 2 - pydatview/io/parquet_file.py | 3 + pydatview/io/pickle_file.py | 5 +- pydatview/io/rosco_discon_file.py | 265 +++++++++++++++++ pydatview/io/tools/graph.py | 18 ++ pydatview/io/turbsim_file.py | 86 +++++- pydatview/io/wetb/hawc2/htc_file.py | 1 - pydatview/tools/curve_fitting.py | 64 ++++- pydatview/tools/fatigue.py | 25 +- pydatview/tools/signal_analysis.py | 32 ++- pydatview/tools/spectral.py | 12 +- pydatview/tools/stats.py | 155 ++++++++-- 24 files changed, 1324 insertions(+), 316 deletions(-) create mode 100644 pydatview/io/rosco_discon_file.py diff --git a/README.md b/README.md index 8f8ca4c..c53ffcd 100644 --- a/README.md +++ b/README.md @@ -23,19 +23,18 @@ make # will run python pyDatView.py echo "alias pydat='make -C `pwd`'" >> ~/.bashrc ``` -**MacOS** users can use a `brew`, `anaconda` or `virtualenv` version of python and pip, but the final version of python that calls the script needs to have access to the screen (for instance using `pythonw`) (see [details for MacOS](#macos-installation)). We recommend using conda, for which the following commands should work: +**MacOS** users can use a `brew`, `anaconda` or `virtualenv` version of python and pip, but the final version of python that calls the script needs to have access to the screen (for instance using `pythonw`) (see [details for MacOS](#macos-installation)). We recommend using conda, in the base environment, for which the following commands should work: ```bash conda install -c conda-forge wxpython # install wxpython -git clone https://github.com/ebranlard/pyDatView +git clone https://github.com/ebranlard/pyDatView -b dev cd pyDatView python -m pip install --user -r requirements.txt make # will run ./pythonmac pyDatView.py # OR try #pythonw pyDatView.py # NOTE: using pythonw not python -echo "alias pydat='make -C `pwd`'" >> ~/.bashrc +echo "alias pydat='make -C `pwd`'" >> ~/.bashrc # add an alias for quicklaunch ``` - -More information about the download, requirements and installation is provided [further down this page](#installation) +If this fails using the Mac terminal, try the zsh terminal from [VSCode](https://code.visualstudio.com) or [iterm2](https://iterm2.com/downloads.html). More information about the download, requirements and installation is provided [further down this page](#installation) ## Usage @@ -224,6 +223,12 @@ If that still doesn't work, you can try using the `python.app` from anaconda: where `/anaconda3/bin/` is the path that would be returned by the command `which conda`. Note the `.app` at the end. If you don't have `python.app`, try installing it with `conda install -c anaconda python.app` +Note also that several users have been struggling to run pyDatView on the mac Terminal in new macOS systems. If you encounter the same issues, we recommend using the integrated zsh terminal from [VSCode](https://code.visualstudio.com) or using a more advanced terminal like [iterm2](https://iterm2.com/downloads.html) and perform the installation steps there. Also, make sure to stick to the base anaconda environment. + + + + + ### Brew-python version (outside of a virtualenv) If you have `brew` installed, and you installed python with `brew install python`, then the easiest is to use your `python3` version: ``` diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 23c4ffe..fd47969 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -620,7 +620,9 @@ def radialAvg(self,avgMethod, avgParam): fst_in=Files[0] - dfRadED, dfRadAD, dfRadBD= fastlib.spanwisePostPro(fst_in, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df = self.data) + out= fastlib.spanwisePostPro(fst_in, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df = self.data) + dfRadED=out['ED_bld']; dfRadAD = out['AD']; dfRadBD = out['BD'] + dfs_new = [dfRadAD, dfRadED, dfRadBD] names_new=[self.raw_name+'_AD', self.raw_name+'_ED', self.raw_name+'_BD'] return dfs_new, names_new diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 3f6e1be..523e53f 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -14,6 +14,24 @@ # --------------------------------------------------------------------------------} # --- Tools for IO # --------------------------------------------------------------------------------{ +def getEDClass(class_or_filename): + """ + Return ElastoDyn instance of FileCl + INPUT: either + - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) + - a filepath to a ElastoDyn input file + - a filepath to a main OpenFAST input file + """ + if hasattr(class_or_filename,'startswith'): # if string + ED = FASTInputFile(class_or_filename) + if 'EDFile' in ED.keys(): # User provided a .fst file... + parentDir=os.path.dirname(class_or_filename) + EDfilename = os.path.join(parentDir, ED['EDFile'].replace('"','')) + ED = FASTInputFile(EDfilename) + else: + ED = class_or_filename + return ED + def ED_BldStations(ED): """ Returns ElastoDyn Blade Station positions, useful to know where the outputs are. INPUTS: @@ -25,15 +43,14 @@ def ED_BldStations(ED): - bld_fract: fraction of the blade length were stations are defined - r_nodes: spanwise position from the rotor apex of the Blade stations """ - if hasattr(ED,'startswith'): # if string - ED = FASTInputFile(ED) + ED = getEDClass(ED) nBldNodes = ED['BldNodes'] bld_fract = np.arange(1./nBldNodes/2., 1, 1./nBldNodes) r_nodes = bld_fract*(ED['TipRad']-ED['HubRad']) + ED['HubRad'] return bld_fract, r_nodes -def ED_TwrStations(ED): +def ED_TwrStations(ED, addBase=True): """ Returns ElastoDyn Tower Station positions, useful to know where the outputs are. INPUTS: - ED: either: @@ -44,12 +61,13 @@ def ED_TwrStations(ED): - r_fract: fraction of the towet length were stations are defined - h_nodes: height from the *ground* of the stations (not from the Tower base) """ - if hasattr(ED,'startswith'): # if string - ED = FASTInputFile(ED) + ED = getEDClass(ED) nTwrNodes = ED['TwrNodes'] twr_fract = np.arange(1./nTwrNodes/2., 1, 1./nTwrNodes) - h_nodes = twr_fract*(ED['TowerHt']-ED['TowerBsHt']) + ED['TowerBsHt'] + h_nodes = twr_fract*(ED['TowerHt']-ED['TowerBsHt']) + if addBase: + h_nodes += ED['TowerBsHt'] return twr_fract, h_nodes def ED_BldGag(ED): @@ -61,8 +79,7 @@ def ED_BldGag(ED): OUTPUTS: - r_gag: The radial positions of the gages, given from the rotor apex """ - if hasattr(ED,'startswith'): # if string - ED = FASTInputFile(ED) + ED = getEDClass(ED) _,r_nodes= ED_BldStations(ED) # if ED.hasNodal: @@ -77,27 +94,28 @@ def ED_BldGag(ED): r_gag = r_nodes[ Inodes[:nOuts] -1] return r_gag, Inodes -def ED_TwrGag(ED): +def ED_TwrGag(ED, addBase=True): """ Returns the heights of ElastoDyn blade gages INPUTS: - ED: either: - a filename of a ElastoDyn input file - an instance of FileCl, as returned by reading the file, ED = weio.read(ED_filename) + - addBase: if True, TowerBsHt is added to h_gag OUTPUTS: - h_gag: The heights of the gages, given from the ground height (tower base + TowerBsHt) """ - if hasattr(ED,'startswith'): # if string - ED = FASTInputFile(ED) - _,h_nodes= ED_TwrStations(ED) + ED = getEDClass(ED) + + _,h_nodes= ED_TwrStations(ED, addBase=addBase) nOuts = ED['NTwGages'] if nOuts<=0: - return np.array([]) + return np.array([]), None if type(ED['TwrGagNd']) is list: Inodes = np.asarray(ED['TwrGagNd']) else: Inodes = np.array([ED['TwrGagNd']]) h_gag = h_nodes[ Inodes[:nOuts] -1] - return h_gag + return h_gag, Inodes def AD14_BldGag(AD): @@ -109,7 +127,7 @@ def AD14_BldGag(AD): OUTPUTS: - r_gag: The radial positions of the gages, given from the blade root """ - if hasattr(ED,'startswith'): # if string + if hasattr(AD,'startswith'): # if string AD = FASTInputFile(AD) Nodes=AD['BldAeroNodes'] @@ -238,6 +256,18 @@ def BD_BldGag(BD): return r_gag, Inodes, r_nodes # +def SD_MembersNodes(SD): + sd = SubDyn(SD) + return sd.pointsMN + +def SD_MembersJoints(SD): + sd = SubDyn(SD) + return sd.pointsMJ + +def SD_MembersGages(SD): + sd = SubDyn(SD) + return sd.pointsMNout + # # 1, 7, 14, 21, 30, 36, 43, 52, 58 BldGagNd List of blade nodes that have strain gages [1 to BldNodes] (-) [unused if NBlGages=0] @@ -300,7 +330,7 @@ def _HarmonizeSpanwiseData(Name, Columns, vr, R, IR=None) : return dfRad, nrMax, ValidRow -def insert_radial_columns(df, vr=None, R=None, IR=None): +def insert_spanwise_columns(df, vr=None, R=None, IR=None, sspan='r', sspan_bar='r/R'): """ Add some columns to the radial data df: dataframe @@ -321,13 +351,13 @@ def insert_radial_columns(df, vr=None, R=None, IR=None): vr_bar=vr_bar[:nrMax] elif (nrMax)>len(vr_bar): raise Exception('Inconsitent length between radial stations ({:d}) and max index present in output chanels ({:d})'.format(len(vr_bar),nrMax)) - df.insert(0, 'r/R_[-]', vr_bar) + df.insert(0, sspan_bar+'_[-]', vr_bar) if IR is not None: df['Node_[#]']=IR[:nrMax] df['i_[#]']=ids+1 if vr is not None: - df['r_[m]'] = vr[:nrMax] + df[sspan+'_[m]'] = vr[:nrMax] return df def find_matching_columns(Cols, PatternMap): @@ -572,6 +602,35 @@ def spanwiseColED(Cols): EDSpanMap[r'^Spn(\d)MLz'+sB+r'_\[kN-m\]' ]=SB+'MLz_[kN-m]' return find_matching_columns(Cols, EDSpanMap) +def spanwiseColEDTwr(Cols): + """ Return column info, available columns and indices that contain ED spanwise data""" + EDSpanMap=dict() + # All Outs + EDSpanMap[r'^TwHt(\d*)ALxt_\[m/s^2\]'] = 'ALxt_[m/s^2]' + EDSpanMap[r'^TwHt(\d*)ALyt_\[m/s^2\]'] = 'ALyt_[m/s^2]' + EDSpanMap[r'^TwHt(\d*)ALzt_\[m/s^2\]'] = 'ALzt_[m/s^2]' + EDSpanMap[r'^TwHt(\d*)TDxt_\[m\]' ] = 'TDxt_[m]' + EDSpanMap[r'^TwHt(\d*)TDyt_\[m\]' ] = 'TDyt_[m]' + EDSpanMap[r'^TwHt(\d*)TDzt_\[m\]' ] = 'TDzt_[m]' + EDSpanMap[r'^TwHt(\d*)RDxt_\[deg\]' ] = 'RDxt_[deg]' + EDSpanMap[r'^TwHt(\d*)RDyt_\[deg\]' ] = 'RDyt_[deg]' + EDSpanMap[r'^TwHt(\d*)RDzt_\[deg\]' ] = 'RDzt_[deg]' + EDSpanMap[r'^TwHt(\d*)TPxi_\[m\]' ] = 'TPxi_[m]' + EDSpanMap[r'^TwHt(\d*)TPyi_\[m\]' ] = 'TPyi_[m]' + EDSpanMap[r'^TwHt(\d*)TPzi_\[m\]' ] = 'TPzi_[m]' + EDSpanMap[r'^TwHt(\d*)RPxi_\[deg\]' ] = 'RPxi_[deg]' + EDSpanMap[r'^TwHt(\d*)RPyi_\[deg\]' ] = 'RPyi_[deg]' + EDSpanMap[r'^TwHt(\d*)RPzi_\[deg\]' ] = 'RPzi_[deg]' + EDSpanMap[r'^TwHt(\d*)FLxt_\[kN\]' ] = 'FLxt_[kN]' + EDSpanMap[r'^TwHt(\d*)FLyt_\[kN\]' ] = 'FLyt_[kN]' + EDSpanMap[r'^TwHt(\d*)FLzt_\[kN\]' ] = 'FLzt_[kN]' + EDSpanMap[r'^TwHt(\d*)MLxt_\[kN-m\]' ] = 'MLxt_[kN-m]' + EDSpanMap[r'^TwHt(\d*)MLyt_\[kN-m\]' ] = 'MLyt_[kN-m]' + EDSpanMap[r'^TwHt(\d*)MLzt_\[kN-m\]' ] = 'MLzt_[kN-m]' + return find_matching_columns(Cols, EDSpanMap) + + + def spanwiseColAD(Cols): """ Return column info, available columns and indices that contain AD spanwise data""" ADSpanMap=dict() @@ -680,7 +739,8 @@ def insert_extra_columns_AD(dfRad, tsAvg, vr=None, rho=None, R=None, nB=None, ch def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='.outb',df=None): """ - Postprocess FAST radial data. Average the time series, return a dataframe nr x nColumns + Postprocess FAST radial data. + if avgMethod is not None: Average the time series, return a dataframe nr x nColumns INPUTS: - FST_IN: Fast .fst input file @@ -690,20 +750,38 @@ def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='. """ # --- Opens Fast output and performs averaging if df is None: - df = FASTOutputFile(FST_In.replace('.fst',out_ext).replace('.dvr',out_ext)).toDataFrame() + filename =FST_In.replace('.fst',out_ext).replace('.dvr',out_ext) + df = FASTOutputFile(filename).toDataFrame() returnDF=True else: + filename='' returnDF=False # NOTE: spanwise script doest not support duplicate columns df = df.loc[:,~df.columns.duplicated()] - dfAvg = averageDF(df,avgMethod=avgMethod ,avgParam=avgParam) # NOTE: average 5 last seconds + if avgMethod is not None: + dfAvg = averageDF(df,avgMethod=avgMethod ,avgParam=avgParam, filename=filename) # NOTE: average 5 last seconds + else: + dfAvg=df # --- Extract info (e.g. radial positions) from Fast input file # We don't have a .fst input file, so we'll rely on some default values for "r" rho = 1.225 chord = None # --- Extract radial positions of output channels - r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst = FASTRadialOutputs(FST_In, OutputCols=df.columns.values) + d = FASTSpanwiseOutputs(FST_In, OutputCols=df.columns.values) + r_AD = d['r_AD'] + r_ED_bld = d['r_ED_bld'] + r_ED_twr = d['r_ED_twr'] + r_BD = d['r_BD'] + IR_AD = d['IR_AD'] + IR_ED_bld = d['IR_ED_bld'] + IR_ED_twr = d['IR_ED_twr'] + IR_BD = d['IR_BD'] + TwrLen = d['TwrLen'] + R = d['R'] + r_hub = d['r_hub'] + fst = d['fst'] + if R is None: R=1 try: @@ -723,29 +801,47 @@ def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='. #print('I_AD:', IR_AD) #print('I_ED:', IR_ED) #print('I_BD:', IR_BD) + out = {} + if returnDF: + out['df'] = df + out['dfAvg'] = dfAvg # --- Extract radial data and export to csv if needed - dfRad_AD = None - dfRad_ED = None - dfRad_BD = None Cols=dfAvg.columns.values # --- AD ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) dfRad_AD = extract_spanwise_data(ColsInfoAD, nrMaxAD, df=None, ts=dfAvg.iloc[0]) dfRad_AD = insert_extra_columns_AD(dfRad_AD, dfAvg.iloc[0], vr=r_AD, rho=rho, R=R, nB=3, chord=chord) - dfRad_AD = insert_radial_columns(dfRad_AD, r_AD, R=R, IR=IR_AD) - # --- ED + dfRad_AD = insert_spanwise_columns(dfRad_AD, r_AD, R=R, IR=IR_AD) + out['AD'] = dfRad_AD + # --- ED Bld ColsInfoED, nrMaxED = spanwiseColED(Cols) dfRad_ED = extract_spanwise_data(ColsInfoED, nrMaxED, df=None, ts=dfAvg.iloc[0]) - dfRad_ED = insert_radial_columns(dfRad_ED, r_ED, R=R, IR=IR_ED) + dfRad_ED = insert_spanwise_columns(dfRad_ED, r_ED_bld, R=R, IR=IR_ED_bld) + out['ED_bld'] = dfRad_ED + # --- ED Twr + ColsInfoED, nrMaxEDt = spanwiseColEDTwr(Cols) + dfRad_EDt = extract_spanwise_data(ColsInfoED, nrMaxEDt, df=None, ts=dfAvg.iloc[0]) + dfRad_EDt2 = insert_spanwise_columns(dfRad_EDt, r_ED_twr, R=TwrLen, IR=IR_ED_twr, sspan='H',sspan_bar='H/L') + # TODO we could insert TwrBs and TwrTp quantities here... + out['ED_twr'] = dfRad_EDt # --- BD ColsInfoBD, nrMaxBD = spanwiseColBD(Cols) dfRad_BD = extract_spanwise_data(ColsInfoBD, nrMaxBD, df=None, ts=dfAvg.iloc[0]) - dfRad_BD = insert_radial_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) - if returnDF: - return dfRad_ED , dfRad_AD, dfRad_BD, df + dfRad_BD = insert_spanwise_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) + out['BD'] = dfRad_BD + # --- SubDyn + if fst.SD is not None: + sd = SubDyn(fst.SD) + MN = sd.pointsMN + MNout, MJout = sd.memberPostPro(dfAvg) + out['SD_MembersOut'] = MNout + out['SD_JointsOut'] = MJout else: - return dfRad_ED , dfRad_AD, dfRad_BD + out['SD_MembersOut'] = None + out['SD_JointsOut'] = None + # Combine all into a dictionary + return out def spanwisePostProRows(df, FST_In=None): @@ -759,7 +855,19 @@ def spanwisePostProRows(df, FST_In=None): rho = 1.225 chord = None # --- Extract radial positions of output channels - r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst = FASTRadialOutputs(FST_In, OutputCols=df.columns.values) + d = FASTSpanwiseOutputs(FST_In, OutputCols=df.columns.values) + r_AD = d['r_AD'] + r_ED_bld = d['r_ED_bld'] + r_ED_twr = d['r_ED_twr'] + r_BD = d['r_BD'] + IR_AD = d['IR_AD'] + IR_ED_bld = d['IR_ED_bld'] + IR_ED_twr = d['IR_ED_twr'] + IR_BD = d['IR_BD'] + TwrLen = d['TwrLen'] + R = d['R'] + r_hub = d['r_hub'] + fst = d['fst'] #print('r_AD:', r_AD) #print('r_ED:', r_ED) #print('r_BD:', r_BD) @@ -798,21 +906,21 @@ def spanwisePostProRows(df, FST_In=None): if r_AD is not None: dfRad_AD = extract_spanwise_data(ColsInfoAD, nrMaxAD, df=None, ts=df.iloc[i]) dfRad_AD = insert_extra_columns_AD(dfRad_AD, df.iloc[i], vr=r_AD, rho=rho, R=R, nB=3, chord=chord) - dfRad_AD = insert_radial_columns(dfRad_AD, r_AD, R=R, IR=IR_AD) + dfRad_AD = insert_spanwise_columns(dfRad_AD, r_AD, R=R, IR=IR_AD) if i==0: M_AD = np.zeros((len(v), len(dfRad_AD), len(dfRad_AD.columns))) Col_AD=dfRad_AD.columns.values M_AD[i, :, : ] = dfRad_AD.values if r_ED is not None and len(r_ED)>0: dfRad_ED = extract_spanwise_data(ColsInfoED, nrMaxED, df=None, ts=df.iloc[i]) - dfRad_ED = insert_radial_columns(dfRad_ED, r_ED, R=R, IR=IR_ED) + dfRad_ED = insert_spanwise_columns(dfRad_ED, r_ED, R=R, IR=IR_ED) if i==0: M_ED = np.zeros((len(v), len(dfRad_ED), len(dfRad_ED.columns))) Col_ED=dfRad_ED.columns.values M_ED[i, :, : ] = dfRad_ED.values if r_BD is not None and len(r_BD)>0: dfRad_BD = extract_spanwise_data(ColsInfoBD, nrMaxBD, df=None, ts=df.iloc[i]) - dfRad_BD = insert_radial_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) + dfRad_BD = insert_spanwise_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) if i==0: M_BD = np.zeros((len(v), len(dfRad_BD), len(dfRad_BD.columns))) Col_BD=dfRad_BD.columns.values @@ -820,24 +928,28 @@ def spanwisePostProRows(df, FST_In=None): return M_AD, Col_AD, M_ED, Col_ED, M_BD, Col_BD -def FASTRadialOutputs(FST_In, OutputCols=None): - """ Returns radial positions where FAST has outputs +def FASTSpanwiseOutputs(FST_In, OutputCols=None, verbose=False): + """ Returns spanwise positions where OpenFAST has outputs INPUTS: - FST_In: fast input file (.fst) + - FST_In: fast input file (.fst) OUTPUTS: - r_AD: radial positions of FAST Outputs from the rotor center + dictionary with fields: + - r_AD: radial positions of FAST Outputs from the rotor center """ R = None + TwrLen = None r_hub =0 r_AD = None - r_ED = None + r_ED_bld = None + r_ED_twr = None r_BD = None - IR_ED = None + IR_ED_bld = None + IR_ED_twr = None IR_AD = None IR_BD = None fst=None if FST_In is not None: - fst = FASTInputDeck(FST_In, readlist=['AD','ADbld','ED','BD','BDbld']) + fst = FASTInputDeck(FST_In, readlist=['AD','ADbld','ED','BD','BDbld','SD']) # NOTE: all this below should be in FASTInputDeck if fst.version == 'F7': # --- FAST7 @@ -862,16 +974,21 @@ def FASTRadialOutputs(FST_In, OutputCols=None): r_hub = fst.fst['BldHubRad_bl(1_1)'] elif not hasattr(fst,'ED'): - print('[WARN] The Elastodyn file couldn''t be found or read, from main file: '+FST_In) + if verbose: + print('[WARN] The Elastodyn file couldn''t be found or read, from main file: '+FST_In) #raise Exception('The Elastodyn file couldn''t be found or read, from main file: '+FST_In) else: R = fst.ED['TipRad'] r_hub = fst.ED['HubRad'] if fst.ED.hasNodal: - _, r_ED = ED_BldStations(fst.ED) - IR_ED =None + _, r_ED_bld = ED_BldStations(fst.ED) + IR_ED_bld =None else: - r_ED, IR_ED = ED_BldGag(fst.ED) + r_ED_bld, IR_ED_bld = ED_BldGag(fst.ED) + + # No nodal output for elastodyn tower yet + TwrLen = fst.ED['TowerHt'] -fst.ED['TowerBsHt'] + r_ED_twr, IR_ED_twr = ED_TwrGag(fst.ED) # --- BeamDyn if fst.BD is not None: @@ -885,7 +1002,8 @@ def FASTRadialOutputs(FST_In, OutputCols=None): # --- AeroDyn if fst.AD is None: - print('[WARN] The AeroDyn file couldn''t be found or read, from main file: '+FST_In) + if verbose: + print('[WARN] The AeroDyn file couldn''t be found or read, from main file: '+FST_In) #raise Exception('The AeroDyn file couldn''t be found or read, from main file: '+FST_In) else: if fst.ADversion == 'AD15': @@ -910,7 +1028,13 @@ def FASTRadialOutputs(FST_In, OutputCols=None): else: raise Exception('AeroDyn version unknown') - return r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst + # Put everything into a dictionary for convenience + outs = {'r_AD':r_AD, 'IR_AD':IR_AD, 'r_ED_bld':r_ED_bld, 'IR_ED_bld':IR_ED_bld, 'r_ED_twr':r_ED_twr, 'IR_ED_twr':IR_ED_twr, 'r_BD':r_BD, 'IR_BD':IR_BD} + outs['R'] = R + outs['TwrLen']= TwrLen + outs['r_hub'] = r_hub + outs['fst'] = fst + return outs # r_AD, r_ED, r_BD, IR_AD, IR_ED, IR_BD, R, r_hub, fst @@ -930,9 +1054,13 @@ def addToOutlist(OutList, Signals): # --- Generic df # --------------------------------------------------------------------------------{ def remap_df(df, ColMap, bColKeepNewOnly=False, inPlace=False, dataDict=None, verbose=False): - """ Add/rename columns of a dataframe, potentially perform operations between columns + """ + NOTE: see welib.tools.pandalib + + Add/rename columns of a dataframe, potentially perform operations between columns - dataDict: dicitonary of data to be made available as "variable" in the column mapping + dataDict: dictionary of data to be made available as "variable" in the column mapping + 'key' (new) : value (old) Example: @@ -940,6 +1068,7 @@ def remap_df(df, ColMap, bColKeepNewOnly=False, inPlace=False, dataDict=None, ve 'WS_[m/s]' : '{Wind1VelX_[m/s]}' , # create a new column from existing one 'RtTSR_[-]' : '{RtTSR_[-]} * 2 + {RtAeroCt_[-]}' , # change value of column 'RotSpeed_[rad/s]' : '{RotSpeed_[rpm]} * 2*np.pi/60 ', # new column [rpm] -> [rad/s] + 'q_p' : ['Q_P_[rad]', '{PtfmSurge_[deg]}*np.pi/180'] # List of possible matches } # Read df = weio.read('FASTOutBin.outb').toDataFrame() @@ -961,33 +1090,47 @@ def remap_df(df, ColMap, bColKeepNewOnly=False, inPlace=False, dataDict=None, ve # Loop for expressions for k0,v in ColMap.items(): k=k0.strip() - v=v.strip() - if v.find('{')>=0: - search_results = re.finditer(r'\{.*?\}', v) - expr=v - if verbose: - print('Attempt to insert column {:15s} with expr {}'.format(k,v)) - # For more advanced operations, we use an eval - bFail=False - for item in search_results: - col=item.group(0)[1:-1] - if col not in df.columns: - ColMapMiss.append(col) - bFail=True - expr=expr.replace(item.group(0),'df[\''+col+'\']') - #print(k0, '=', expr) - if not bFail: - df[k]=eval(expr) - ColNew.append(k) - else: - print('[WARN] Column not present in dataframe, cannot evaluate: ',expr) + if type(v) is not list: + values = [v] else: - #print(k0,'=',v) - if v not in df.columns: - ColMapMiss.append(v) - print('[WARN] Column not present in dataframe: ',v) + values = v + Found = False + for v in values: + v=v.strip() + if Found: + break # We avoid replacing twice + if v.find('{')>=0: + # --- This is an advanced substitution using formulae + search_results = re.finditer(r'\{.*?\}', v) + expr=v + if verbose: + print('Attempt to insert column {:15s} with expr {}'.format(k,v)) + # For more advanced operations, we use an eval + bFail=False + for item in search_results: + col=item.group(0)[1:-1] + if col not in df.columns: + ColMapMiss.append(col) + bFail=True + expr=expr.replace(item.group(0),'df[\''+col+'\']') + #print(k0, '=', expr) + if not bFail: + df[k]=eval(expr) + ColNew.append(k) + else: + print('[WARN] Column not present in dataframe, cannot evaluate: ',expr) else: - RenameMap[k]=v + #print(k0,'=',v) + if v not in df.columns: + ColMapMiss.append(v) + if verbose: + print('[WARN] Column not present in dataframe: ',v) + else: + if k in RenameMap.keys(): + print('[WARN] Not renaming {} with {} as the key is already present'.format(k,v)) + else: + RenameMap[k]=v + Found=True # Applying renaming only now so that expressions may be applied in any order for k,v in RenameMap.items(): @@ -1064,9 +1207,9 @@ def _zero_crossings(y,x=None,direction=None): raise Exception('Direction should be either `up` or `down`') return xzc, iBef, sign -def find_matching_pattern(List, pattern, sort=False, integers=True): +def find_matching_pattern(List, pattern, sort=False, integers=True, n=1): r""" Return elements of a list of strings that match a pattern - and return the first matching group + and return the n first matching group Example: @@ -1109,7 +1252,7 @@ def extractSpanTS(df, pattern): NOTE: time is not inserted in the output dataframe - To find "r" use FASTRadialOutputs, it is different for AeroDyn/ElastoDyn/BeamDyn/ + To find "r" use FASTSpanwiseOutputs, it is different for AeroDyn/ElastoDyn/BeamDyn/ There is no guarantee that the number of columns matching pattern will exactly corresponds to the number of radial stations. That's the responsability of the OpenFAST user. @@ -1278,7 +1421,7 @@ def azimuthal_average_DF(df, psiBin=None, colPsi='Azimuth_[deg]', tStart=None, c return dfPsi -def averageDF(df,avgMethod='periods',avgParam=None,ColMap=None,ColKeep=None,ColSort=None,stats=['mean']): +def averageDF(df,avgMethod='periods',avgParam=None,ColMap=None,ColKeep=None,ColSort=None,stats=['mean'], filename=''): """ See average PostPro for documentation, same interface, just does it for one dataframe """ @@ -1287,6 +1430,9 @@ def renameCol(x): if x==v: return k return x + # Sanity + if len(filename)>0: + filename=' (File: {})'.format(filename) # Before doing the colomn map we store the time time = df['Time_[s]'].values timenoNA = time[~np.isnan(time)] @@ -1306,14 +1452,14 @@ def renameCol(x): elif avgMethod.lower()=='periods': # --- Using azimuth to find periods if 'Azimuth_[deg]' not in df.columns: - raise Exception('The sensor `Azimuth_[deg]` does not appear to be in the output file. You cannot use the averaging method by `periods`, use `constantwindow` instead.') + raise Exception('The sensor `Azimuth_[deg]` does not appear to be in the dataframe{}. You cannot use the averaging method by `periods`, use `constantwindow` instead.'.format(filename)) # NOTE: potentially we could average over each period and then average psi=df['Azimuth_[deg]'].values _,iBef = _zero_crossings(psi-psi[-2],direction='up') if len(iBef)==0: _,iBef = _zero_crossings(psi-180,direction='up') if len(iBef)==0: - print('[WARN] Not able to find a zero crossing!') + print('[WARN] Not able to find a zero crossing!{}'.format(filename)) tEnd = time[-1] iBef=[0] else: @@ -1324,7 +1470,7 @@ def renameCol(x): else: avgParam=int(avgParam) if len(iBef)-1=tStart) & (time<=tEnd) & (~np.isnan(time)))[0] iEnd = IWindow[-1] iStart = IWindow[0] @@ -1375,10 +1521,15 @@ def renameCol(x): -def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKeep=None,ColSort=None,stats=['mean']): +def averagePostPro(outFiles_or_DFs,avgMethod='periods',avgParam=None, + ColMap=None,ColKeep=None,ColSort=None,stats=['mean'], + skipIfWrongCol=False): """ Opens a list of FAST output files, perform average of its signals and return a panda dataframe For now, the scripts only computes the mean within a time window which may be a constant or a time that is a function of the rotational speed (see `avgMethod`). The script only computes the mean for now. Other stats will be added + INPUTS: + + outFiles_or_DFs: list of fst filenames or dataframes `ColMap` : dictionary where the key is the new column name, and v the old column name. Default: None, output is not sorted @@ -1399,33 +1550,52 @@ def averagePostPro(outFiles,avgMethod='periods',avgParam=None,ColMap=None,ColKee Default: None, full simulation length is used """ result=None - if len(outFiles)==0: - raise Exception('No outFiles provided') + if len(outFiles_or_DFs)==0: + raise Exception('No outFiles or DFs provided') invalidFiles =[] # Loop trough files and populate result - for i,f in enumerate(outFiles): - try: - df=weio.read(f).toDataFrame() - #df=FASTOutputFile(f).toDataFrame()A # For pyFAST - except: - invalidFiles.append(f) - continue - postpro=averageDF(df, avgMethod=avgMethod, avgParam=avgParam, ColMap=ColMap, ColKeep=ColKeep,ColSort=ColSort,stats=stats) + for i,f in enumerate(outFiles_or_DFs): + if isinstance(f, pd.DataFrame): + df = f + else: + try: + df=weio.read(f).toDataFrame() + #df=FASTOutputFile(f).toDataFrame()A # For pyFAST + except: + invalidFiles.append(f) + continue + postpro=averageDF(df, avgMethod=avgMethod, avgParam=avgParam, ColMap=ColMap, ColKeep=ColKeep,ColSort=ColSort,stats=stats, filename=f) MeanValues=postpro # todo if result is None: # We create a dataframe here, now that we know the colums columns = MeanValues.columns - result = pd.DataFrame(np.nan, index=np.arange(len(outFiles)), columns=columns) - result.iloc[i,:] = MeanValues.copy().values + result = pd.DataFrame(np.nan, index=np.arange(len(outFiles_or_DFs)), columns=columns) + if MeanValues.shape[1]!=result.shape[1]: + columns_ref = result.columns + columns_loc = MeanValues.columns + if skipIfWrongCol: + print('[WARN] File {} has {} columns and not {}. Skipping.'.format(f, MeanValues.shape[1], result.shape[1])) + else: + try: + MeanValues=MeanValues[columns_ref] + result.iloc[i,:] = MeanValues.copy().values + print('[WARN] File {} has more columns than other files. Truncating.'.format(f, MeanValues.shape[1], result.shape[1])) + except: + print('[WARN] File {} is missing some columns compared to other files. Skipping.'.format(f)) + else: + result.iloc[i,:] = MeanValues.copy().values - if len(invalidFiles)==len(outFiles): + if len(invalidFiles)==len(outFiles_or_DFs): raise Exception('None of the files can be read (or exist)!. For instance, cannot find: {}'.format(invalidFiles[0])) elif len(invalidFiles)>0: print('[WARN] There were {} missing/invalid files: \n {}'.format(len(invalidFiles),'\n'.join(invalidFiles))) if ColSort is not None: + if not ColSort in result.keys(): + print('[INFO] Columns present: ', result.keys()) + raise Exception('[FAIL] Cannot sort results with column `{}`, column not present in dataframe (see above)'.format(ColSort)) # Sorting result.sort_values([ColSort],inplace=True,ascending=True) result.reset_index(drop=True,inplace=True) diff --git a/pydatview/io/__init__.py b/pydatview/io/__init__.py index b48f7fe..6929fd8 100644 --- a/pydatview/io/__init__.py +++ b/pydatview/io/__init__.py @@ -59,6 +59,7 @@ def fileFormats(userpath=None, ignoreErrors=False, verbose=False): from .pickle_file import PickleFile from .cactus_file import CactusFile from .raawmat_file import RAAWMatFile + from .rosco_discon_file import ROSCODISCONFile from .rosco_performance_file import ROSCOPerformanceFile priorities = [] formats = [] @@ -91,6 +92,7 @@ def addFormat(priority, fmt): addFormat(40, FileFormat(FLEXWaveKinFile)) addFormat(40, FileFormat(FLEXDocFile)) addFormat(50, FileFormat(BModesOutFile)) + addFormat(50, FileFormat(ROSCODISCONFile)) addFormat(50, FileFormat(ROSCOPerformanceFile)) addFormat(60, FileFormat(NetCDFFile)) addFormat(60, FileFormat(VTKFile)) @@ -247,6 +249,8 @@ def detectFormat(filename, **kwargs): def read(filename, fileformat=None, **kwargs): F = None + if not os.path.exists(filename): + raise FileNotFoundError('weio cannot read the following file because it does not exist:\n Inp. path: {}\n Abs. path: {}'.format(filename, os.path.abspath(filename))) # Detecting format if necessary if fileformat is None: fileformat,F = detectFormat(filename, **kwargs) diff --git a/pydatview/io/csv_file.py b/pydatview/io/csv_file.py index 18cdcbf..5621f03 100644 --- a/pydatview/io/csv_file.py +++ b/pydatview/io/csv_file.py @@ -244,8 +244,6 @@ def strIsFloat(s): self.colNames=['C{}'.format(i) for i in range(len(self.data.columns))] self.data.columns = self.colNames; self.data.rename(columns=lambda x: x.strip(),inplace=True) - #import pdb - #pdb.set_trace() def _write(self): # --- Safety diff --git a/pydatview/io/fast_input_deck.py b/pydatview/io/fast_input_deck.py index 56119ad..178d7d4 100644 --- a/pydatview/io/fast_input_deck.py +++ b/pydatview/io/fast_input_deck.py @@ -24,10 +24,13 @@ def __init__(self, fullFstPath='', readlist=['all'], verbose=False): AC: airfoil coordinates (if present) """ + # Sanity if type(verbose) is not bool: raise Exception('`verbose` arguments needs to be a boolean') + # Main Data + self.inputFilesRead = {} self.filename = fullFstPath self.verbose = verbose self.readlist = readlist @@ -38,7 +41,6 @@ def __init__(self, fullFstPath='', readlist=['all'], verbose=False): else: self.readlist = ['Fst']+self.readlist - self.inputfiles = {} # --- Harmonization with AeroElasticSE self.FAST_ver = 'OPENFAST' @@ -74,6 +76,19 @@ def __init__(self, fullFstPath='', readlist=['all'], verbose=False): if len(fullFstPath)>0: self.read() + @property + def ED(self): + ED = self.fst_vt['ElastoDyn'] + if ED is None: + if 'ED' not in self.readlist: + self.readlist.append('ED') + if self.verbose: + print('>>> Reading ED', self.ED_path) + self.fst_vt['ElastoDyn'] = self._read(self.fst_vt['Fst']['EDFile'],'ED') + return self.fst_vt['ElastoDyn'] + else: + return ED + def readAD(self, filename=None, readlist=None, verbose=False, key='AeroDyn15'): """ @@ -145,57 +160,37 @@ def inputFiles(self): files=[] files+=[self.ED_path, self.ED_twr_path, self.ED_bld_path] files+=[self.BD_path, self.BD_bld_path] + files+=[self.SD_path] return [f for f in files if f not in self.unusedNames] - - @property - def ED_relpath(self): - try: - return self.fst_vt['Fst']['EDFile'].replace('"','') - except: - return 'none' - - @property - def ED_twr_relpath(self): - try: - return os.path.join(os.path.dirname(self.fst_vt['Fst']['EDFile']).replace('"',''), self.fst_vt['ElastoDyn']['TwrFile'].replace('"','')) - except: - return 'none' - - @property - def ED_bld_relpath(self): + def _relpath(self, k1, k2=None, k3=None): try: - if 'BldFile(1)' in self.fst_vt['ElastoDyn'].keys(): - return os.path.join(os.path.dirname(self.fst_vt['Fst']['EDFile'].replace('"','')), self.fst_vt['ElastoDyn']['BldFile(1)'].replace('"','')) + if k2 is None: + return self.fst_vt['Fst'][k1].replace('"','') else: - return os.path.join(os.path.dirname(self.fst_vt['Fst']['EDFile'].replace('"','')), self.fst_vt['ElastoDyn']['BldFile1'].replace('"','')) + parent = os.path.dirname(self.fst_vt['Fst'][k1]).replace('"','') + if type(k3)==list: + for k in k3: + if k in self.fst_vt[k2].keys(): + child = self.fst_vt[k2][k].replace('"','') + else: + child = self.fst_vt[k2][k3].replace('"','') + return os.path.join(parent, child) except: return 'none' @property - def BD_relpath(self): - try: - return self.fst_vt['Fst']['BDBldFile(1)'].replace('"','') - except: - return 'none' - + def ED_path(self): return self._fullpath(self._relpath('EDFile')) @property - def BD_bld_relpath(self): - try: - return os.path.join(os.path.dirname(self.fst_vt['Fst']['BDBldFile(1)'].replace('"','')), self.fst_vt['BeamDyn']['BldFile'].replace('"','')) - except: - return 'none' - - @property - def ED_path(self): return self._fullpath(self.ED_relpath) + def SD_path(self): return self._fullpath(self._relpath('SubFile')) @property - def BD_path(self): return self._fullpath(self.BD_relpath) + def BD_path(self): return self._fullpath(self._relpath('BDBldFile(1)')) @property - def BD_bld_path(self): return self._fullpath(self.BD_bld_relpath) + def BD_bld_path(self): return self._fullpath(self._relpath('BDBldFile(1)','BeamDyn','BldFile')) @property - def ED_twr_path(self): return self._fullpath(self.ED_twr_relpath) + def ED_twr_path(self): return self._fullpath(self._relpath('EDFile','ElastoDyn','TwrFile')) @property - def ED_bld_path(self): return self._fullpath(self.ED_bld_relpath) + def ED_bld_path(self): return self._fullpath(self._relpath('EDFile','ElastoDyn',['BldFile(1)','BldFile1'])) @@ -209,10 +204,15 @@ def _fullpath(self, relfilepath): def read(self, filename=None): + """ + Read all OpenFAST inputs files, based on the requested list of modules `readlist` + """ if filename is not None: self.filename = filename # Read main file (.fst, or .drv) and store into key "Fst" + if self.verbose: + print('Reading:', self.FAST_InputFile) self.fst_vt['Fst'] = self._read(self.FAST_InputFile, 'Fst') if self.fst_vt['Fst'] is None: raise Exception('Error reading main file {}'.format(self.filename)) @@ -255,8 +255,8 @@ def read(self, filename=None): if 'EDFile' in self.fst_vt['Fst'].keys(): self.fst_vt['ElastoDyn'] = self._read(self.fst_vt['Fst']['EDFile'],'ED') if self.fst_vt['ElastoDyn'] is not None: - twr_file = self.ED_twr_relpath - bld_file = self.ED_bld_relpath + twr_file = self.ED_twr_path + bld_file = self.ED_bld_path self.fst_vt['ElastoDynTower'] = self._read(twr_file,'EDtwr') self.fst_vt['ElastoDynBlade'] = self._read(bld_file,'EDbld') @@ -280,7 +280,7 @@ def read(self, filename=None): # SubDyn if self.fst_vt['Fst']['CompSub'] == 1: - self.fst_vt['SubDyn'] = self._read(self.fst_vt['Fst']['SubFile'],'HD') + self.fst_vt['SubDyn'] = self._read(self.fst_vt['Fst']['SubFile'], 'SD') # Mooring if self.fst_vt['Fst']['CompMooring']==1: @@ -298,7 +298,7 @@ def read(self, filename=None): # --- Backward compatibility self.fst = self.fst_vt['Fst'] - self.ED = self.fst_vt['ElastoDyn'] + self._ED = self.fst_vt['ElastoDyn'] if not hasattr(self,'AD'): self.AD = None if self.AD is not None: @@ -307,6 +307,7 @@ def read(self, filename=None): self.IW = self.fst_vt['InflowWind'] self.BD = self.fst_vt['BeamDyn'] self.BDbld = self.fst_vt['BeamDynBlade'] + self.SD = self.fst_vt['SubDyn'] @ property def unusedNames(self): @@ -330,12 +331,15 @@ def _read(self, relfilepath, shortkey): return None # Attempt reading - fullpath =os.path.join(self.FAST_directory, relfilepath) + if relfilepath.startswith(self.FAST_directory): + fullpath = relfilepath + else: + fullpath = os.path.join(self.FAST_directory, relfilepath) try: data = FASTInputFile(fullpath) if self.verbose: print('>>> Read: ',fullpath) - self.inputfiles[shortkey] = fullpath + self.inputFilesRead[shortkey] = fullpath return data except FileNotFoundError: print('[WARN] File not found '+fullpath) @@ -460,10 +464,12 @@ def write(self, filename=None, prefix='', suffix='', directory=None): def __repr__(self): s=''+'\n' s+='filename : '+self.filename+'\n' + s+='readlist : {}'.format(self.readlist)+'\n' s+='version : '+self.version+'\n' s+='AD version : '+self.ADversion+'\n' s+='fst_vt : dict{'+','.join([k for k,v in self.fst_vt.items() if v is not None])+'}\n' s+='inputFiles : {}\n'.format(self.inputFiles) + s+='inputFilesRead : {}\n'.format(self.inputFilesRead) s+='\n' return s diff --git a/pydatview/io/fast_input_file.py b/pydatview/io/fast_input_file.py index 1d4061e..14ddb82 100644 --- a/pydatview/io/fast_input_file.py +++ b/pydatview/io/fast_input_file.py @@ -89,9 +89,13 @@ def fixedFormat(self): KEYS = list(self.basefile.keys()) if 'NumBlNds' in KEYS: return ADBladeFile.from_fast_input_file(self.basefile) + elif 'rhoinf' in KEYS: + return BDFile.from_fast_input_file(self.basefile) elif 'NBlInpSt' in KEYS: return EDBladeFile.from_fast_input_file(self.basefile) - elif 'MassMatrix' in KEYS and self.module =='ExtPtfm': + elif 'NTwInpSt' in KEYS: + return EDTowerFile.from_fast_input_file(self.basefile) + elif 'MassMatrix' in KEYS and self.module == 'ExtPtfm': return ExtPtfmFile.from_fast_input_file(self.basefile) elif 'NumCoords' in KEYS and 'InterpOrd' in KEYS: return ADPolarFile.from_fast_input_file(self.basefile) @@ -352,12 +356,19 @@ def _read(self): # --- Tables that can be detected based on the "Label" (second entry on line) # NOTE: MJointID1, used by SubDyn and HydroDyn - NUMTAB_FROM_LAB_DETECT = ['NumAlf' , 'F_X' , 'MemberCd1' , 'MJointID1' , 'NOutLoc' , 'NOutCnt' , 'PropD' ,'Diam' ,'Type' ,'LineType' ] - NUMTAB_FROM_LAB_DIM_VAR = ['NumAlf' , 'NKInpSt' , 'NCoefMembers' , 'NMembers' , 'NMOutputs' , 'NMOutputs' , 'NPropSets' ,'NTypes' ,'NConnects' ,'NLines' ] - NUMTAB_FROM_LAB_VARNAME = ['AFCoeff' , 'TMDspProp' , 'MemberProp' , 'Members' , 'MemberOuts' , 'MemberOuts' , 'SectionProp' ,'LineTypes' ,'ConnectionProp' ,'LineProp' ] - NUMTAB_FROM_LAB_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 ] - NUMTAB_FROM_LAB_NOFFSET = [0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ] - NUMTAB_FROM_LAB_TYPE = ['num' , 'num' , 'num' , 'mix' , 'num' , 'sdout' , 'num' ,'mix' ,'mix' ,'mix' ] + NUMTAB_FROM_LAB_DETECT = ['NumAlf' , 'F_X' , 'MemberCd1' , 'MJointID1' , 'NOutLoc' , 'NOutCnt' , 'PropD' ] + NUMTAB_FROM_LAB_DIM_VAR = ['NumAlf' , 'NKInpSt' , 'NCoefMembers' , 'NMembers' , 'NMOutputs' , 'NMOutputs' , 'NPropSets' ] + NUMTAB_FROM_LAB_VARNAME = ['AFCoeff' , 'TMDspProp' , 'MemberProp' , 'Members' , 'MemberOuts' , 'MemberOuts' , 'SectionProp' ] + NUMTAB_FROM_LAB_NHEADER = [2 , 2 , 2 , 2 , 2 , 2 , 2 ] + NUMTAB_FROM_LAB_NOFFSET = [0 , 0 , 0 , 0 , 0 , 0 , 0 ] + NUMTAB_FROM_LAB_TYPE = ['num' , 'num' , 'num' , 'mix' , 'num' , 'sdout' , 'num' ] + # MoorDyn Version 1 and 2 (with AUTO for LAB_DIM_VAR) + NUMTAB_FROM_LAB_DETECT += ['Diam' ,'Type' ,'LineType' , 'Attachment'] + NUMTAB_FROM_LAB_DIM_VAR += ['NTypes:AUTO','NConnects' ,'NLines:AUTO' , 'AUTO'] + NUMTAB_FROM_LAB_VARNAME += ['LineTypes' ,'ConnectionProp' ,'LineProp' , 'Points'] + NUMTAB_FROM_LAB_NHEADER += [ 2 , 2 , 2 , 2 ] + NUMTAB_FROM_LAB_NOFFSET += [ 0 , 0 , 0 , 0 ] + NUMTAB_FROM_LAB_TYPE += ['mix' ,'mix' ,'mix' , 'mix'] # SubDyn NUMTAB_FROM_LAB_DETECT += ['GuyanDampSize' , 'YoungE' , 'YoungE' , 'EA' , 'MatDens' ] NUMTAB_FROM_LAB_DIM_VAR += [6 , 'NPropSets', 'NXPropSets', 'NCablePropSets' , 'NRigidPropSets'] @@ -480,6 +491,19 @@ def _read(self): i+=1; self.readBeamDynProps(lines,i) return + elif line.upper().find('OUTPUTS')>0: + if 'Points' in self.keys() and 'dtM' in self.keys(): + OutList,i = parseFASTOutList(lines,i+1) + d = getDict() + d['label'] = 'Outlist' + d['descr'] = '' + d['tabType'] = TABTYPE_FIL # TODO + d['value'] = OutList + self.addComment('------------------------ OUTPUTS --------------------------------------------') + self.data.append(d) + self.addComment('END') + self.addComment('------------------------- need this line --------------------------------------') + return # --- Parsing of standard lines: value(s) key comment line = lines[i] @@ -534,11 +558,16 @@ def _read(self): break elif labelRaw=='re': - nAirfoilTab = self['NumTabs'] - iTab +=1 - if nAirfoilTab>1: - labOffset ='_'+str(iTab) - d['label']=labelRaw+labOffset + try: + nAirfoilTab = self['NumTabs'] + iTab +=1 + if nAirfoilTab>1: + labOffset ='_'+str(iTab) + d['label']=labelRaw+labOffset + except: + # Unsteady driver input file... + pass + #print('label>',d['label'],'<',type(d['label'])); #print('value>',d['value'],'<',type(d['value'])); @@ -597,7 +626,6 @@ def _read(self): self.data.append(dd) d['label'] = NUMTAB_FROM_LAB_VARNAME[ii] - d['tabDimVar'] = NUMTAB_FROM_LAB_DIM_VAR[ii] if d['label'].lower()=='afcoeff' : d['tabType'] = TABTYPE_NUM_WITH_HEADERCOM else: @@ -607,10 +635,28 @@ def _read(self): d['tabType'] = TABTYPE_NUM_SUBDYNOUT else: d['tabType'] = TABTYPE_MIX_WITH_HEADER - if isinstance(d['tabDimVar'],int): + # Finding table dimension (number of lines) + tabDimVar = NUMTAB_FROM_LAB_DIM_VAR[ii] + if isinstance(tabDimVar, int): # dimension hardcoded + d['tabDimVar'] = tabDimVar nTabLines = d['tabDimVar'] else: - nTabLines = self[d['tabDimVar']+labOffset] + # We either use a variable name or "AUTO" to find the number of rows + tabDimVars = tabDimVar.split(':') + for tabDimVar in tabDimVars: + d['tabDimVar'] = tabDimVar + if tabDimVar=='AUTO': + # Determine table dimension automatically + nTabLines = findNumberOfTableLines(lines[i+nHeaders:], break_chars=['---','!','#']) + break + else: + try: + nTabLines = self[tabDimVar+labOffset] + break + except KeyError: + #print('Cannot determine table dimension using {}'.format(tabDimVar)) + # Hopefully this table has AUTO as well + pass d['label'] += labOffset #print('Reading table {} Dimension {} (based on {})'.format(d['label'],nTabLines,d['tabDimVar'])); @@ -690,7 +736,7 @@ def toStringVLD(val,lab,descr): val='{:13s}'.format(val) if len(lab)<13: lab='{:13s}'.format(lab) - return val+' '+lab+' - '+descr.strip().strip('-').strip()+'\n' + return val+' '+lab+' - '+descr.strip().lstrip('-').lstrip() def toStringIntFloatStr(x): try: @@ -725,9 +771,9 @@ def mat_tostring(M,fmt='24.16e'): elif d['tabType']==TABTYPE_NOT_A_TAB: if isinstance(d['value'], list): sList=', '.join([str(x) for x in d['value']]) - s+='{} {} {}'.format(sList,d['label'],d['descr']) + s+=toStringVLD(sList, d['label'], d['descr']) else: - s+=toStringVLD(d['value'],d['label'],d['descr']).strip() + s+=toStringVLD(d['value'],d['label'],d['descr']) elif d['tabType']==TABTYPE_NUM_WITH_HEADER: if d['tabColumnNames'] is not None: s+='{}'.format(' '.join(['{:15s}'.format(s) for s in d['tabColumnNames']])) @@ -755,10 +801,13 @@ def mat_tostring(M,fmt='24.16e'): s+='\n'.join('\t'.join('{:15.8e}'.format(x) for x in y) for y in d['value']) elif d['tabType']==TABTYPE_FIL: #f.write('{} {} {}\n'.format(d['value'][0],d['tabDetect'],d['descr'])) + label = d['label'] + if 'kbot' in self.keys(): # Moordyn has no 'OutList' label.. + label='' if len(d['value'])==1: - s+='{} {} {}'.format(d['value'][0],d['label'],d['descr']) # TODO? + s+='{} {} {}'.format(d['value'][0], label, d['descr']) # TODO? else: - s+='{} {} {}\n'.format(d['value'][0],d['label'],d['descr']) # TODO? + s+='{} {} {}\n'.format(d['value'][0], label, d['descr']) # TODO? s+='\n'.join(fil for fil in d['value'][1:]) elif d['tabType']==TABTYPE_NUM_BEAMDYN: # TODO use dedicated sub-class @@ -827,30 +876,16 @@ def _toDataFrame(self): if self.getIDSafe('TwFAM1Sh(2)')>0: # Hack for tower files, we add the modes + # NOTE: we provide interpolated shape function just in case the resolution of the input file is low.. x=Val[:,0] Modes=np.zeros((x.shape[0],4)) - Modes[:,0] = x**2 * self['TwFAM1Sh(2)'] \ - + x**3 * self['TwFAM1Sh(3)'] \ - + x**4 * self['TwFAM1Sh(4)'] \ - + x**5 * self['TwFAM1Sh(5)'] \ - + x**6 * self['TwFAM1Sh(6)'] - Modes[:,1] = x**2 * self['TwFAM2Sh(2)'] \ - + x**3 * self['TwFAM2Sh(3)'] \ - + x**4 * self['TwFAM2Sh(4)'] \ - + x**5 * self['TwFAM2Sh(5)'] \ - + x**6 * self['TwFAM2Sh(6)'] - Modes[:,2] = x**2 * self['TwSSM1Sh(2)'] \ - + x**3 * self['TwSSM1Sh(3)'] \ - + x**4 * self['TwSSM1Sh(4)'] \ - + x**5 * self['TwSSM1Sh(5)'] \ - + x**6 * self['TwSSM1Sh(6)'] - Modes[:,3] = x**2 * self['TwSSM2Sh(2)'] \ - + x**3 * self['TwSSM2Sh(3)'] \ - + x**4 * self['TwSSM2Sh(4)'] \ - + x**5 * self['TwSSM2Sh(5)'] \ - + x**6 * self['TwSSM2Sh(6)'] + Modes[:,0] = x**2 * self['TwFAM1Sh(2)'] + x**3 * self['TwFAM1Sh(3)'] + x**4 * self['TwFAM1Sh(4)'] + x**5 * self['TwFAM1Sh(5)'] + x**6 * self['TwFAM1Sh(6)'] + Modes[:,1] = x**2 * self['TwFAM2Sh(2)'] + x**3 * self['TwFAM2Sh(3)'] + x**4 * self['TwFAM2Sh(4)'] + x**5 * self['TwFAM2Sh(5)'] + x**6 * self['TwFAM2Sh(6)'] + Modes[:,2] = x**2 * self['TwSSM1Sh(2)'] + x**3 * self['TwSSM1Sh(3)'] + x**4 * self['TwSSM1Sh(4)'] + x**5 * self['TwSSM1Sh(5)'] + x**6 * self['TwSSM1Sh(6)'] + Modes[:,3] = x**2 * self['TwSSM2Sh(2)'] + x**3 * self['TwSSM2Sh(3)'] + x**4 * self['TwSSM2Sh(4)'] + x**5 * self['TwSSM2Sh(5)'] + x**6 * self['TwSSM2Sh(6)'] Val = np.hstack((Val,Modes)) - Cols = Cols + ['ShapeForeAft1_[-]','ShapeForeAft2_[-]','ShapeSideSide1_[-]','ShapeSideSide2_[-]'] + ShapeCols = [c+'_[-]' for c in ['ShapeForeAft1','ShapeForeAft2','ShapeSideSide1','ShapeSideSide2']] + Cols = Cols + ShapeCols name=d['label'] @@ -1183,6 +1218,17 @@ def detectUnits(s,nRef): return Units +def findNumberOfTableLines(lines, break_chars): + """ Loop through lines until a one of the "break character is found""" + for i, l in enumerate(lines): + for bc in break_chars: + if l.startswith(bc): + return i + # Not found + print('[FAIL] end of table not found') + return len(lines) + + def parseFASTNumTable(filename,lines,n,iStart,nHeaders=2,tableType='num',nOffset=0, varNumLines=''): """ First lines of data starts at: nHeaders+nOffset @@ -1337,8 +1383,101 @@ def parseFASTFilTable(lines,n,iStart): # --------------------------------------------------------------------------------{ # --------------------------------------------------------------------------------{ + # --------------------------------------------------------------------------------} -# --- AeroDyn Blade +# --- BeamDyn +# --------------------------------------------------------------------------------{ +class BDFile(FASTInputFileBase): + @classmethod + def from_fast_input_file(cls, parent): + self = cls() + self.setData(filename=parent.filename, data=parent.data, hasNodal=parent.hasNodal, module='BD') + return self + + def __init__(self, filename=None, **kwargs): + FASTInputFileBase.__init__(self, filename, **kwargs) + if filename is None: + # Define a prototype for this file format + self.addComment('--------- BEAMDYN with OpenFAST INPUT FILE -------------------------------------------') + self.addComment('BeamDyn input file, written by BDFile') + self.addComment('---------------------- SIMULATION CONTROL --------------------------------------') + self.addValKey(False , 'Echo' , 'Echo input data to ".ech"? (flag)') + self.addValKey(True , 'QuasiStaticInit' , 'Use quasi-static pre-conditioning with centripetal accelerations in initialization? (flag) [dynamic solve only]') + self.addValKey( 0 , 'rhoinf' , 'Numerical damping parameter for generalized-alpha integrator') + self.addValKey( 2 , 'quadrature' , 'Quadrature method: 1=Gaussian; 2=Trapezoidal (switch)') + self.addValKey("DEFAULT" , 'refine' , 'Refinement factor for trapezoidal quadrature (-) [DEFAULT = 1; used only when quadrature=2]') + self.addValKey("DEFAULT" , 'n_fact' , 'Factorization frequency for the Jacobian in N-R iteration(-) [DEFAULT = 5]') + self.addValKey("DEFAULT" , 'DTBeam' , 'Time step size (s)') + self.addValKey("DEFAULT" , 'load_retries' , 'Number of factored load retries before quitting the simulation [DEFAULT = 20]') + self.addValKey("DEFAULT" , 'NRMax' , 'Max number of iterations in Newton-Raphson algorithm (-) [DEFAULT = 10]') + self.addValKey("DEFAULT" , 'stop_tol' , 'Tolerance for stopping criterion (-) [DEFAULT = 1E-5]') + self.addValKey("DEFAULT" , 'tngt_stf_fd' , 'Use finite differenced tangent stiffness matrix? (flag)') + self.addValKey("DEFAULT" , 'tngt_stf_comp' , 'Compare analytical finite differenced tangent stiffness matrix? (flag)') + self.addValKey("DEFAULT" , 'tngt_stf_pert' , 'Perturbation size for finite differencing (-) [DEFAULT = 1E-6]') + self.addValKey("DEFAULT" , 'tngt_stf_difftol', 'Maximum allowable relative difference between analytical and fd tangent stiffness (-); [DEFAULT = 0.1]') + self.addValKey(True , 'RotStates' , 'Orient states in the rotating frame during linearization? (flag) [used only when linearizing] ') + self.addComment('---------------------- GEOMETRY PARAMETER --------------------------------------') + self.addValKey( 1 , 'member_total' , 'Total number of members (-)') + self.addValKey( 0 , 'kp_total' , 'Total number of key points (-) [must be at least 3]') + self.addValKey( [1, 0] , 'kp_per_member' , 'Member number; Number of key points in this member') + self.addTable('MemberGeom', np.zeros((0,4)), tabType=1, tabDimVar='kp_total', + cols=['kp_xr', 'kp_yr', 'kp_zr', 'initial_twist'], + units=['(m)', '(m)', '(m)', '(deg)']) + self.addComment('---------------------- MESH PARAMETER ------------------------------------------') + self.addValKey( 5 , 'order_elem' , 'Order of interpolation (basis) function (-)') + self.addComment('---------------------- MATERIAL PARAMETER --------------------------------------') + self.addValKey('"undefined"', 'BldFile' , 'Name of file containing properties for blade (quoted string)') + self.addComment('---------------------- PITCH ACTUATOR PARAMETERS -------------------------------') + self.addValKey(False , 'UsePitchAct' , 'Whether a pitch actuator should be used (flag)') + self.addValKey( 1 , 'PitchJ' , 'Pitch actuator inertia (kg-m^2) [used only when UsePitchAct is true]') + self.addValKey( 0 , 'PitchK' , 'Pitch actuator stiffness (kg-m^2/s^2) [used only when UsePitchAct is true]') + self.addValKey( 0 , 'PitchC' , 'Pitch actuator damping (kg-m^2/s) [used only when UsePitchAct is true]') + self.addComment('---------------------- OUTPUTS -------------------------------------------------') + self.addValKey(False , 'SumPrint' , 'Print summary data to ".sum" (flag)') + self.addValKey('"ES10.3E2"' , 'OutFmt' , 'Format used for text tabular output, excluding the time channel.') + self.addValKey( 0 , 'NNodeOuts' , 'Number of nodes to output to file [0 - 9] (-)') + self.addValKey( [1] , 'OutNd' , 'Nodes whose values will be output (-)') + self.addValKey( [''] , 'OutList' , 'The next line(s) contains a list of output parameters. See OutListParameters.xlsx, BeamDyn tab for a listing of available output channels, (-)') + self.addComment('END of OutList (the word "END" must appear in the first 3 columns of this last OutList line)') + self.addComment('---------------------- NODE OUTPUTS --------------------------------------------') + self.addValKey( 99 , 'BldNd_BlOutNd' , 'Blade nodes on each blade (currently unused)') + self.addValKey( [''] , 'OutList_Nodal' , 'The next line(s) contains a list of output parameters. See OutListParameters.xlsx, BeamDyn_Nodes tab for a listing of available output channels, (-)') + self.addComment('END of input file (the word "END" must appear in the first 3 columns of this last OutList line)') + self.addComment('--------------------------------------------------------------------------------') + self.hasNodal=True + #"RootFxr, RootFyr, RootFzr" + #"RootMxr, RootMyr, RootMzr" + #"TipTDxr, TipTDyr, TipTDzr" + #"TipRDxr, TipRDyr, TipRDzr" + + else: + # fix some stuff that generic reader fail at + self.data[1] = {'value':self._lines[1], 'label':'', 'isComment':True, 'descr':'', 'tabType':0} + i = self.getID('kp_total') + listval = [int(v) for v in str(self.data[i+1]['value']).split()] + self.data[i+1]['value']=listval + self.data[i+1]['label']='kp_per_member' + self.data[i+1]['isComment']=False + self.module='BD' + + def _writeSanityChecks(self): + """ Sanity checks before write """ + self['kp_total']=self['MemberGeom'].shape[0] + i = self.getID('kp_total') + self.data[i+1]['value']=[1, self['MemberGeom'].shape[0]] # kp_per_member + self.data[i+1]['label']='kp_per_member' + # Could check length of OutNd + + def _toDataFrame(self): + df = FASTInputFileBase._toDataFrame(self) + # TODO add quadrature points based on trapz/gauss + return df + + @property + def _IComment(self): return [1] + +# --------------------------------------------------------------------------------} +# --- ElastoDyn Blade # --------------------------------------------------------------------------------{ class EDBladeFile(FASTInputFileBase): @classmethod @@ -1372,11 +1511,11 @@ def __init__(self, filename=None, **kwargs): self.addValKey( 0.0 , 'BldFl1Sh(4)', ' , coeff of x^4') self.addValKey( 0.0 , 'BldFl1Sh(5)', ' , coeff of x^5') self.addValKey( 0.0 , 'BldFl1Sh(6)', ' , coeff of x^6') - self.addValKey( 1.0 , 'BldFl2Sh(2)', 'Flap mode 2, coeff of x^2') + self.addValKey( 0.0 , 'BldFl2Sh(2)', 'Flap mode 2, coeff of x^2') # NOTE: using something not too bad just incase user uses these as is.. self.addValKey( 0.0 , 'BldFl2Sh(3)', ' , coeff of x^3') - self.addValKey( 0.0 , 'BldFl2Sh(4)', ' , coeff of x^4') - self.addValKey( 0.0 , 'BldFl2Sh(5)', ' , coeff of x^5') - self.addValKey( 0.0 , 'BldFl2Sh(6)', ' , coeff of x^6') + self.addValKey( -13.0 , 'BldFl2Sh(4)', ' , coeff of x^4') + self.addValKey( 27.0 , 'BldFl2Sh(5)', ' , coeff of x^5') + self.addValKey( -13.0 , 'BldFl2Sh(6)', ' , coeff of x^6') self.addValKey( 1.0 , 'BldEdgSh(2)', 'Edge mode 1, coeff of x^2') self.addValKey( 0.0 , 'BldEdgSh(3)', ' , coeff of x^3') self.addValKey( 0.0 , 'BldEdgSh(4)', ' , coeff of x^4') @@ -1384,7 +1523,7 @@ def __init__(self, filename=None, **kwargs): self.addValKey( 0.0 , 'BldEdgSh(6)', ' , coeff of x^6') else: # fix some stuff that generic reader fail at - self.data[1] = self._lines[1] + self.data[1] = {'value':self._lines[1], 'label':'', 'isComment':True, 'descr':'', 'tabType':0} self.module='EDBlade' def _writeSanityChecks(self): @@ -1401,27 +1540,101 @@ def _toDataFrame(self): # We add the shape functions for EDBladeFile x=df['BlFract_[-]'].values Modes=np.zeros((x.shape[0],3)) - Modes[:,0] = x**2 * self['BldFl1Sh(2)'] \ - + x**3 * self['BldFl1Sh(3)'] \ - + x**4 * self['BldFl1Sh(4)'] \ - + x**5 * self['BldFl1Sh(5)'] \ - + x**6 * self['BldFl1Sh(6)'] - Modes[:,1] = x**2 * self['BldFl2Sh(2)'] \ - + x**3 * self['BldFl2Sh(3)'] \ - + x**4 * self['BldFl2Sh(4)'] \ - + x**5 * self['BldFl2Sh(5)'] \ - + x**6 * self['BldFl2Sh(6)'] - Modes[:,2] = x**2 * self['BldEdgSh(2)'] \ - + x**3 * self['BldEdgSh(3)'] \ - + x**4 * self['BldEdgSh(4)'] \ - + x**5 * self['BldEdgSh(5)'] \ - + x**6 * self['BldEdgSh(6)'] + Modes[:,0] = x**2 * self['BldFl1Sh(2)'] + x**3 * self['BldFl1Sh(3)'] + x**4 * self['BldFl1Sh(4)'] + x**5 * self['BldFl1Sh(5)'] + x**6 * self['BldFl1Sh(6)'] + Modes[:,1] = x**2 * self['BldFl2Sh(2)'] + x**3 * self['BldFl2Sh(3)'] + x**4 * self['BldFl2Sh(4)'] + x**5 * self['BldFl2Sh(5)'] + x**6 * self['BldFl2Sh(6)'] + Modes[:,2] = x**2 * self['BldEdgSh(2)'] + x**3 * self['BldEdgSh(3)'] + x**4 * self['BldEdgSh(4)'] + x**5 * self['BldEdgSh(5)'] + x**6 * self['BldEdgSh(6)'] df[['ShapeFlap1_[-]','ShapeFlap2_[-]','ShapeEdge1_[-]']]=Modes return df @property def _IComment(self): return [1] +# --------------------------------------------------------------------------------} +# --- ElastoDyn Tower +# --------------------------------------------------------------------------------{ +class EDTowerFile(FASTInputFileBase): + @classmethod + def from_fast_input_file(cls, parent): + self = cls() + self.setData(filename=parent.filename, data=parent.data, hasNodal=parent.hasNodal, module='EDTower') + return self + + def __init__(self, filename=None, **kwargs): + FASTInputFileBase.__init__(self, filename, **kwargs) + if filename is None: + # Define a prototype for this file format + self.addComment('------- ELASTODYN V1.00.* TOWER INPUT FILE -------------------------------------') + self.addComment('ElastoDyn tower definition, written by EDTowerFile.') + self.addComment('---------------------- TOWER PARAMETERS ----------------------------------------') + self.addValKey( 0 , 'NTwInpSt' , 'Number of blade input stations (-)') + self.addValKey( 1. , 'TwrFADmp(1)' , 'Tower 1st fore-aft mode structural damping ratio (%)') + self.addValKey( 1. , 'TwrFADmp(2)' , 'Tower 2nd fore-aft mode structural damping ratio (%)') + self.addValKey( 1. , 'TwrSSDmp(1)' , 'Tower 1st side-to-side mode structural damping ratio (%)') + self.addValKey( 1. , 'TwrSSDmp(2)' , 'Tower 2nd side-to-side mode structural damping ratio (%)') + self.addComment('---------------------- TOWER ADJUSTMENT FACTORS --------------------------------') + self.addValKey( 1. , 'FAStTunr(1)' , 'Tower fore-aft modal stiffness tuner, 1st mode (-)') + self.addValKey( 1. , 'FAStTunr(2)' , 'Tower fore-aft modal stiffness tuner, 2nd mode (-)') + self.addValKey( 1. , 'SSStTunr(1)' , 'Tower side-to-side stiffness tuner, 1st mode (-)') + self.addValKey( 1. , 'SSStTunr(2)' , 'Tower side-to-side stiffness tuner, 2nd mode (-)') + self.addValKey( 1. , 'AdjTwMa' , 'Factor to adjust tower mass density (-)') + self.addValKey( 1. , 'AdjFASt' , 'Factor to adjust tower fore-aft stiffness (-)') + self.addValKey( 1. , 'AdjSSSt' , 'Factor to adjust tower side-to-side stiffness (-)') + self.addComment('---------------------- DISTRIBUTED TOWER PROPERTIES ----------------------------') + self.addTable('TowProp', np.zeros((0,6)), tabType=1, tabDimVar='NTwInpSt', + cols=['HtFract','TMassDen','TwFAStif','TwSSStif'], + units=['(-)', '(kg/m)', '(Nm^2)', '(Nm^2)']) + self.addComment('---------------------- TOWER FORE-AFT MODE SHAPES ------------------------------') + self.addValKey( 1.0 , 'TwFAM1Sh(2)', 'Mode 1, coefficient of x^2 term') + self.addValKey( 0.0 , 'TwFAM1Sh(3)', ' , coefficient of x^3 term') + self.addValKey( 0.0 , 'TwFAM1Sh(4)', ' , coefficient of x^4 term') + self.addValKey( 0.0 , 'TwFAM1Sh(5)', ' , coefficient of x^5 term') + self.addValKey( 0.0 , 'TwFAM1Sh(6)', ' , coefficient of x^6 term') + self.addValKey( -26. , 'TwFAM2Sh(2)', 'Mode 2, coefficient of x^2 term') # NOTE: using something not too bad just incase user uses these as is.. + self.addValKey( 0.0 , 'TwFAM2Sh(3)', ' , coefficient of x^3 term') + self.addValKey( 27. , 'TwFAM2Sh(4)', ' , coefficient of x^4 term') + self.addValKey( 0.0 , 'TwFAM2Sh(5)', ' , coefficient of x^5 term') + self.addValKey( 0.0 , 'TwFAM2Sh(6)', ' , coefficient of x^6 term') + self.addComment('---------------------- TOWER SIDE-TO-SIDE MODE SHAPES --------------------------') + self.addValKey( 1.0 , 'TwSSM1Sh(2)', 'Mode 1, coefficient of x^2 term') + self.addValKey( 0.0 , 'TwSSM1Sh(3)', ' , coefficient of x^3 term') + self.addValKey( 0.0 , 'TwSSM1Sh(4)', ' , coefficient of x^4 term') + self.addValKey( 0.0 , 'TwSSM1Sh(5)', ' , coefficient of x^5 term') + self.addValKey( 0.0 , 'TwSSM1Sh(6)', ' , coefficient of x^6 term') + self.addValKey( -26. , 'TwSSM2Sh(2)', 'Mode 2, coefficient of x^2 term') # NOTE: using something not too bad just incase user uses these as is.. + self.addValKey( 0.0 , 'TwSSM2Sh(3)', ' , coefficient of x^3 term') + self.addValKey( 27. , 'TwSSM2Sh(4)', ' , coefficient of x^4 term') + self.addValKey( 0.0 , 'TwSSM2Sh(5)', ' , coefficient of x^5 term') + self.addValKey( 0.0 , 'TwSSM2Sh(6)', ' , coefficient of x^6 term') + else: + # fix some stuff that generic reader fail at + self.data[1] = {'value':self._lines[1], 'label':'', 'isComment':True, 'descr':'', 'tabType':0} + self.module='EDTower' + + def _writeSanityChecks(self): + """ Sanity checks before write """ + self['NTwInpSt']=self['TowProp'].shape[0] + # Sum of Coeffs should be 1 + for s in ['TwFAM1Sh','TwFAM2Sh','TwSSM1Sh','TwSSM2Sh']: + sumcoeff=np.sum([self[s+'('+str(i)+')'] for i in [2,3,4,5,6] ]) + if np.abs(sumcoeff-1)>1e-4: + print('[WARN] Sum of coefficients for polynomial {} not equal to 1 ({}). File: {}'.format(s, sumcoeff, self.filename)) + + def _toDataFrame(self): + df = FASTInputFileBase._toDataFrame(self) + # We add the shape functions for EDBladeFile + # NOTE: we provide interpolated shape function just in case the resolution of the input file is low.. + x = df['HtFract_[-]'].values + Modes=np.zeros((x.shape[0],4)) + Modes[:,0] = x**2 * self['TwFAM1Sh(2)'] + x**3 * self['TwFAM1Sh(3)'] + x**4 * self['TwFAM1Sh(4)'] + x**5 * self['TwFAM1Sh(5)'] + x**6 * self['TwFAM1Sh(6)'] + Modes[:,1] = x**2 * self['TwFAM2Sh(2)'] + x**3 * self['TwFAM2Sh(3)'] + x**4 * self['TwFAM2Sh(4)'] + x**5 * self['TwFAM2Sh(5)'] + x**6 * self['TwFAM2Sh(6)'] + Modes[:,2] = x**2 * self['TwSSM1Sh(2)'] + x**3 * self['TwSSM1Sh(3)'] + x**4 * self['TwSSM1Sh(4)'] + x**5 * self['TwSSM1Sh(5)'] + x**6 * self['TwSSM1Sh(6)'] + Modes[:,3] = x**2 * self['TwSSM2Sh(2)'] + x**3 * self['TwSSM2Sh(3)'] + x**4 * self['TwSSM2Sh(4)'] + x**5 * self['TwSSM2Sh(5)'] + x**6 * self['TwSSM2Sh(6)'] + ShapeCols = [c+'_[-]' for c in ['ShapeForeAft1','ShapeForeAft2','ShapeSideSide1','ShapeSideSide2']] + df[ShapeCols]=Modes + return df + + @property + def _IComment(self): return [1] # --------------------------------------------------------------------------------} # --- AeroDyn Blade diff --git a/pydatview/io/fast_linearization_file.py b/pydatview/io/fast_linearization_file.py index af23860..d9f8f8a 100644 --- a/pydatview/io/fast_linearization_file.py +++ b/pydatview/io/fast_linearization_file.py @@ -90,10 +90,7 @@ def readOP(fid, n): def readMat(fid, n, m): vals=[f.readline().strip().split() for i in np.arange(n)] -# try: return np.array(vals).astype(float) -# except ValueError: -# import pdb; pdb.set_trace() # Reading with open(self.filename, 'r', errors="surrogateescape") as f: diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index 820a9ae..0a8de62 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -30,6 +30,13 @@ class EmptyFileError(Exception): pass print('CSVFile not available') + +FileFmtID_WithTime = 1 # File identifiers used in FAST +FileFmtID_WithoutTime = 2 +FileFmtID_NoCompressWithoutTime = 3 +FileFmtID_ChanLen_In = 4 # Channel length included in file + + # --------------------------------------------------------------------------------} # --- OUT FILE # --------------------------------------------------------------------------------{ @@ -279,12 +286,6 @@ def freadRowOrderTableBuffered(fid, n, type_in, nCols, nOff=0, type_out='float64 raise Exception('Read only %d of %d values in file:' % (nIntRead, n, filename)) return data - - FileFmtID_WithTime = 1 # File identifiers used in FAST - FileFmtID_WithoutTime = 2 - FileFmtID_NoCompressWithoutTime = 3 - FileFmtID_ChanLen_In = 4 - with open(filename, 'rb') as fid: #---------------------------- # get the header information @@ -392,40 +393,7 @@ def freadRowOrderTableBuffered(fid, n, type_in, nCols, nOff=0, type_out='float64 return data, info -def writeDataFrame(df, filename, binary=True): - channels = df.values - # attempt to extract units from channel names - chanNames=[] - chanUnits=[] - for c in df.columns: - c = c.strip() - name = c - units = '' - if c[-1]==']': - chars=['[',']'] - elif c[-1]==')': - chars=['(',')'] - else: - chars=[] - if len(chars)>0: - op,cl = chars - iu=c.rfind(op) - if iu>1: - name = c[:iu] - unit = c[iu+1:].replace(cl,'') - if name[-1]=='_': - name=name[:-1] - - chanNames.append(name) - chanUnits.append(unit) - - if binary: - writeBinary(filename, channels, chanNames, chanUnits) - else: - NotImplementedError() - - -def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr=''): +def writeBinary(fileName, channels, chanNames, chanUnits, fileID=4, descStr=''): """ Write an OpenFAST binary file. @@ -479,8 +447,8 @@ def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr=''): ColOff = np.single(int16Min - np.single(mins)*ColScl) #Just available for fileID - if fileID != 2: - print("current version just works with FileID = 2") + if fileID not in [2,4]: + print("current version just works with fileID = 2 or 4") else: with open(fileName,'wb') as fid: @@ -494,6 +462,14 @@ def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr=''): # Write header informations fid.write(struct.pack('@h',fileID)) + if fileID == FileFmtID_ChanLen_In: + maxChanLen = np.max([len(s) for s in chanNames]) + maxUnitLen = np.max([len(s) for s in chanUnits]) + nChar = max(maxChanLen, maxUnitLen) + fid.write(struct.pack('@h',nChar)) + else: + nChar = 10 + fid.write(struct.pack('@i',nChannels)) fid.write(struct.pack('@i',nT)) fid.write(struct.pack('@d',timeStart)) @@ -506,13 +482,15 @@ def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr=''): # Write channel names for chan in chanNames: - ordchan = [ord(char) for char in chan]+ [32]*(10-len(chan)) - fid.write(struct.pack('@10B', *ordchan)) + chan = chan[:nChar] + ordchan = [ord(char) for char in chan] + [32]*(nChar-len(chan)) + fid.write(struct.pack('@'+str(nChar)+'B', *ordchan)) # Write channel units for unit in chanUnits: - ordunit = [ord(char) for char in unit]+ [32]*(10-len(unit)) - fid.write(struct.pack('@10B', *ordunit)) + unit = unit[:nChar] + ordunit = [ord(char) for char in unit] + [32]*(nChar-len(unit)) + fid.write(struct.pack('@'+str(nChar)+'B', *ordunit)) # Pack data packedData=np.zeros((nT, nChannels), dtype=np.int16) @@ -523,6 +501,40 @@ def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr=''): fid.write(struct.pack('@{}h'.format(packedData.size), *packedData.flatten())) fid.close() +def writeDataFrame(df, filename, binary=True): + """ write a DataFrame to OpenFAST output format""" + channels = df.values + # attempt to extract units from channel names + chanNames=[] + chanUnits=[] + for c in df.columns: + c = c.strip() + name = c + unit = '' + if c[-1]==']': + chars=['[',']'] + elif c[-1]==')': + chars=['(',')'] + else: + chars=[] + if len(chars)>0: + op,cl = chars + iu=c.rfind(op) + if iu>1: + name = c[:iu] + unit = c[iu+1:].replace(cl,'') + if name[-1]=='_': + name=name[:-1] + + chanNames.append(name) + chanUnits.append(unit) + + if binary: + writeBinary(filename, channels, chanNames, chanUnits, fileID=FileFmtID_ChanLen_In) + else: + NotImplementedError() + + if __name__ == "__main__": B=FASTOutputFile('tests/example_files/FASTOutBin.outb') diff --git a/pydatview/io/file.py b/pydatview/io/file.py index 9c00f99..f9211bc 100644 --- a/pydatview/io/file.py +++ b/pydatview/io/file.py @@ -1,4 +1,5 @@ import os +from collections import OrderedDict class WrongFormatError(Exception): pass @@ -17,7 +18,7 @@ class BrokenReaderError(Exception): except NameError: # Python2 FileNotFoundError = IOError -class File(dict): +class File(OrderedDict): def __init__(self,filename=None,**kwargs): if filename: self.read(filename, **kwargs) diff --git a/pydatview/io/flex_out_file.py b/pydatview/io/flex_out_file.py index 5352351..ea8eb38 100644 --- a/pydatview/io/flex_out_file.py +++ b/pydatview/io/flex_out_file.py @@ -33,14 +33,30 @@ def _read(self): self.time = np.arange(self.tmin, self.tmin + self.nt * self.dt, self.dt).reshape(self.nt,1).astype(dtype) # --- Then the sensor file - sensor_filename = os.path.join(os.path.dirname(self.filename), "sensor") - if not os.path.isfile(sensor_filename): + parentdir = os.path.dirname(self.filename) + basename = os.path.splitext(os.path.basename(self.filename))[0] + #print(parentdir) + #print(basename) + PossibleFiles=[] + PossibleFiles+=[os.path.join(parentdir, basename+'.Sensor')] + PossibleFiles+=[os.path.join(parentdir, 'Sensor_'+basename)] + PossibleFiles+=[os.path.join(parentdir, 'sensor')] + # We try allow for other files + Found =False + for sf in PossibleFiles: + if os.path.isfile(sf): + self.sensors=read_flex_sensor(sf) + if len(self.sensors['ID'])!=self.nSensors: + Found = False + else: + Found = True + break + if not Found: # we are being nice and create some fake sensors info self.sensors=read_flex_sensor_fake(self.nSensors) - else: - self.sensors=read_flex_sensor(sensor_filename) - if len(self.sensors['ID'])!=self.nSensors: - raise BrokenFormatError('Inconsistent number of sensors: {} (sensor file) {} (out file), for file: {}'.format(len(self.sensors['ID']),self.nSensors,self.filename)) + + if len(self.sensors['ID'])!=self.nSensors: + raise BrokenFormatError('Inconsistent number of sensors: {} (sensor file) {} (out file), for file: {}'.format(len(self.sensors['ID']),self.nSensors,self.filename)) #def _write(self): # TODO # pass diff --git a/pydatview/io/mannbox_file.py b/pydatview/io/mannbox_file.py index 859caef..7af29cc 100644 --- a/pydatview/io/mannbox_file.py +++ b/pydatview/io/mannbox_file.py @@ -170,11 +170,13 @@ def __repr__(self): s+='| min: {}, max: {}, mean: {} \n'.format(np.min(self['field']), np.max(self['field']), np.mean(self['field'])) s+='| - dy, dz: {}, {}\n'.format(self['dy'], self['dz']) s+='| - y0, z0 zMid: {}, {}, {}\n'.format(self['y0'], self['z0'], self['zMid']) - s+='|useful getters: y, z, _iMid, fromTurbSim\n' z=self.z y=self.y - s+='| y: [{} ... {}], dy: {}, n: {} \n'.format(y[0],y[-1],self['dy'],len(y)) - s+='| z: [{} ... {}], dz: {}, n: {} \n'.format(z[0],z[-1],self['dz'],len(z)) + s+='| * y: [{} ... {}], dy: {}, n: {} \n'.format(y[0],y[-1],self['dy'],len(y)) + s+='| * z: [{} ... {}], dz: {}, n: {} \n'.format(z[0],z[-1],self['dz'],len(z)) + s+='|useful functions:\n' + s+='| - t(dx, U)\n' + s+='| - valuesAt(y,z), vertProfile(), fromTurbSim(*), _iMid()\n' return s diff --git a/pydatview/io/netcdf_file.py b/pydatview/io/netcdf_file.py index 5fe5c1a..7d69b45 100644 --- a/pydatview/io/netcdf_file.py +++ b/pydatview/io/netcdf_file.py @@ -36,7 +36,5 @@ def _toDataFrame(self): dfs[k]=pd.DataFrame(data=self.data[k].values) elif len(self.data[k].shape)==1: dfs[k]=pd.DataFrame(data=self.data[k].values) - #import pdb - #pdb.set_trace() return dfs diff --git a/pydatview/io/parquet_file.py b/pydatview/io/parquet_file.py index 9c9feba..7cf8dd1 100644 --- a/pydatview/io/parquet_file.py +++ b/pydatview/io/parquet_file.py @@ -32,6 +32,9 @@ def toDataFrame(self): #just return self.data return self.data + def fromDataFrame(self, df): + #data already in dataframe + self.data = df def toString(self): """ use pandas DataFrame.to_string method to convert to a string """ diff --git a/pydatview/io/pickle_file.py b/pydatview/io/pickle_file.py index 36713a2..3626093 100644 --- a/pydatview/io/pickle_file.py +++ b/pydatview/io/pickle_file.py @@ -60,7 +60,10 @@ def _setData(self, data): for k,v in data.items(): self[k] = v else: - self['data'] = data + if hasattr(data, '__dict__'): + self.update(data.__dict__) + else: + self['data'] = data def read(self, filename=None, **kwargs): """ Reads the file self.filename, or `filename` if provided """ diff --git a/pydatview/io/rosco_discon_file.py b/pydatview/io/rosco_discon_file.py new file mode 100644 index 0000000..b4de969 --- /dev/null +++ b/pydatview/io/rosco_discon_file.py @@ -0,0 +1,265 @@ +""" +Input/output class for the fileformat ROSCO DISCON file +""" +import numpy as np +import pandas as pd +import os +from collections import OrderedDict + +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File=OrderedDict + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + +class ROSCODISCONFile(File): + """ + Read/write a ROSCO DISCON file. The object behaves as a dictionary. + + Main methods + ------------ + - read, write, toDataFrame, keys + + Examples + -------- + f = ROSCODISCONFile('DISCON.IN') + print(f.keys()) + print(f.toDataFrame().columns) + + """ + + @staticmethod + def defaultExtensions(): + """ List of file extensions expected for this fileformat""" + return ['.in'] + + @staticmethod + def formatName(): + """ Short string (~100 char) identifying the file format""" + return 'ROSCO DISCON file' + + @staticmethod + def priority(): return 60 # Priority in weio.read fileformat list between 0=high and 100:low + + + def __init__(self, filename=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # --- Calling (children) function to read + _, comments, lineKeys = read_DISCON(self.filename, self) + self.comments=comments + self.lineKeys=lineKeys + + def write(self, filename=None): + """ Rewrite object to file, or write object to `filename` if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + with open(self.filename, 'w') as f: + f.write(self.toString()) + + + def toDataFrame(self): + """ Returns object into one DataFrame, or a dictionary of DataFrames""" + dfs={} + low_keys = [s.lower() for s in self.keys()] + if 'pc_gs_n' in low_keys: + M = np.column_stack([self['PC_GS_angles']*180/np.pi, self['PC_GS_KP'], self['PC_GS_KI'], self['PC_GS_KD'], self['PC_GS_TF']] ) + cols = ['Pitch_[deg]', 'KP_[-]', 'KI_[s]', 'KD_[1/s]', 'TF_[-]'] + dfs['PitchSchedule'] = pd.DataFrame(data=M, columns=cols) + if 'ps_bldpitchmin_n' in low_keys: + M = np.column_stack([self['PS_WindSpeeds'], self['PS_BldPitchMin']]) + cols = ['WindSpeed_[m/s]', 'Pitch_[deg]'] + dfs['PitchSaturation'] = pd.DataFrame(data=M, columns=cols) + if 'prc_n' in low_keys: + M = np.column_stack([self['PRC_WindSpeeds'], self['PRC_RotorSpeeds']*30/np.pi]) + cols = ['WindSpeed_[m/s]', 'RotorSpeed_[rpm]'] + dfs['PowerTracking'] = pd.DataFrame(data=M, columns=cols) + + return dfs + + # --- Optional functions + def __repr__(self): + """ String that is written to screen when the user calls `print()` on the object. + Provide short and relevant information to save time for the user. + """ + s='<{} object>:\n'.format(type(self).__name__) + s+='|Main attributes:\n' + s+='| - filename: {}\n'.format(self.filename) + # --- Example printing some relevant information for user + #s+='|Main keys:\n' + #s+='| - ID: {}\n'.format(self['ID']) + #s+='| - data : shape {}\n'.format(self['data'].shape) + s+='|Main methods:\n' + s+='| - read, write, toDataFrame, keys' + return s + + def toString(self): + """ """ + maxKeyLengh = np.max([len(k) for k in self.keys()]) + maxKeyLengh = max(maxKeyLengh, 18) + fmtKey = '{:' +str(maxKeyLengh)+'s}' + s='' + for l in self.lineKeys: + if len(l)==0: + s+='\n' + elif l.startswith('!'): + s+=l+'\n' + else: + param = l + comment = self.comments[param] + v = self[param] + sparam = '! '+fmtKey.format(param) + # NOTE: could to "param" specific outputs here + FMTs = {} + FMTs['{:<4.6f}']=['F_NotchBetaNumDen', 'F_FlCornerFreq', 'F_FlpCornerFreq', 'PC_GS_angles', 'PC_GS_KP', 'PC_GS_KI', 'PC_GS_KD', 'PC_GS_TF', 'IPC_Vramp', 'IPC_aziOffset'] + FMTs['{:<4.3e}']=['IPC_KP','IPC_KI'] + FMTs['{:<4.3f}']=['PRC_WindSpeeds', 'PRC_RotorSpeed','PS_WindSpeeds'] + FMTs['{:<4.4f}']=['WE_FOPoles_v'] + FMTs['{:<10.8f}']=['WE_FOPoles'] + FMTs['{:<10.3f}']=['PS_BldPitchMin'] + fmtFloat='{:<014.5f}' + for fmt,keys in FMTs.items(): + if param in keys: + fmtFloat=fmt + break + if type(v) is str: + sval='"{:15s}" '.format(v) + elif hasattr(v, '__len__'): + if isinstance(v[0], (np.floating, float)): + sval=' '.join([fmtFloat.format(vi) for vi in v] )+' ' + else: + sval=' '.join(['{}'.format(vi) for vi in v] )+' ' + elif type(v) is int: + sval='{:<14d} '.format(v) + elif isinstance(v, (np.floating, float)): + sval=fmtFloat.format(v) + ' ' + else: + sval='{} '.format(v) + s+='{}{}{}\n'.format(sval, sparam, comment) + return s + + + + + + +# Some useful constants +pi = np.pi +rad2deg = np.rad2deg(1) +deg2rad = np.deg2rad(1) +rpm2RadSec = 2.0*(np.pi)/60.0 +RadSec2rpm = 60/(2.0 * np.pi) + +def write_DISCON(turbine, controller, param_file='DISCON.IN', txt_filename='Cp_Ct_Cq.txt', rosco_vt = {}): + """ + Print the controller parameters to the DISCON.IN input file for the generic controller + + Parameters: + ----------- + turbine: class + Turbine class containing turbine operation information (ref speeds, etc...) + controller: class + Controller class containing controller operation information (gains, etc...) + param_file: str, optional + filename for parameter input file, should be DISCON.IN + txt_filename: str, optional + filename of rotor performance file + """ + + # Get ROSCO var tree if not provided + if not rosco_vt: + rosco_vt = DISCON_dict(turbine, controller, txt_filename) + + print('Writing new controller parameter file parameter file: %s.' % param_file) + # Should be obvious what's going on here... + file = open(param_file,'w') + + # Write Open loop input + if rosco_vt['OL_Mode'] and hasattr(controller, 'OpenLoop'): + write_ol_control(controller) + +def read_DISCON(DISCON_filename, DISCON_in = None): + ''' + Read the DISCON input file. + Adapted from ROSCO_Toolbox, https:github.com/NREL/ROSCO + + Parameters: + ---------- + DISCON_filename: string + Name of DISCON input file to read + + Returns: + -------- + DISCON_in: Dict + Dictionary containing input parameters from DISCON_in, organized by parameter name + ''' + + if DISCON_in is None: + DISCON_in = OrderedDict() + comments={} + lineKeys=[] + with open(DISCON_filename) as discon: + for line in discon: + line=line.strip() + # empty lines + if len(line)==0: + lineKeys.append('') + continue + # Pure comments + if line[0] == '!': + lineKeys.append(line) + continue + + if (line.split()[1] != '!'): # Array valued entries + sps = line.split() + array_length = sps.index('!') + param = sps[array_length+1] + values = np.array( [float(x) for x in sps[:array_length]] ) + else: # All other entries + param = line.split()[2] + value = line.split()[0] + # Remove printed quotations if string is in quotes + if (value[0] == '"') or (value[0] == "'"): + values = value[1:-1] + else: + if value.find('.')>0: + values = float(value) + else: + values = int(value) + DISCON_in[param] = values + lineKeys.append(param) + + sp = line.split('!') + comment = sp[1].strip() + comment = comment[len(param):].strip() + comments [param] = comment + + return DISCON_in, comments, lineKeys + + +if __name__ == '__main__': + filename = 'DISCON.in' + rd = ROSCODISCONFile(filename) + #print(rd.keys()) +# print(rd.toString()) + rd.write(filename+'_WEIO') + print(rd.toDataFrame()) diff --git a/pydatview/io/tools/graph.py b/pydatview/io/tools/graph.py index 04d7eb3..7f8c7de 100644 --- a/pydatview/io/tools/graph.py +++ b/pydatview/io/tools/graph.py @@ -127,6 +127,13 @@ def length(self): n2=self.nodes[1] return np.sqrt((n1.x-n2.x)**2+(n1.y-n2.y)**2+(n1.z-n2.z)**2) + def setData(self, data_dict): + """ set or add data""" + for k,v in data_dict.items(): + #if k in self.data.keys(): + # print('Warning overriding key {} for node {}'.format(k,self.ID)) + self.data[k]=v + def __repr__(self): s=' NodeIDs: {} {}'.format(self.ID, self.nodeIDs, self.data) if self.propIDs is not None: @@ -299,6 +306,17 @@ def setElementNodalProp(self, elem, propset, propIDs): for node, pID in zip(elem.nodes, propIDs): node.setData(self.getNodeProperty(propset, pID).data) + def setElementNodalPropToElem(self, elem): + """ + Set Element Properties to an element + TODO: this seems to be a hack. It should be automatic I think... + """ + propset=elem.propset + propIDs=elem.propIDs + # USING PROPID 0!!! + elem.setData(self.getNodeProperty(propset, propIDs[0]).data) + # TODO average the two maybe.. + def setNodeNodalProp(self, node, propset, propID): """ Set Nodal Properties to a node diff --git a/pydatview/io/turbsim_file.py b/pydatview/io/turbsim_file.py index 99f33d0..2e004c6 100644 --- a/pydatview/io/turbsim_file.py +++ b/pydatview/io/turbsim_file.py @@ -592,7 +592,11 @@ def __repr__(self): s+=' ux: min: {}, max: {}, mean: {} \n'.format(np.min(ux), np.max(ux), np.mean(ux)) s+=' uy: min: {}, max: {}, mean: {} \n'.format(np.min(uy), np.max(uy), np.mean(uy)) s+=' uz: min: {}, max: {}, mean: {} \n'.format(np.min(uz), np.max(uz), np.mean(uz)) - + s += ' Useful methods:\n' + s += ' - read, write, toDataFrame, keys\n' + s += ' - valuesAt, vertProfile, horizontalPlane, verticalPlane, closestPoint\n' + s += ' - fitPowerLaw\n' + s += ' - makePeriodic, checkPeriodic\n' return s def toDataFrame(self): @@ -677,8 +681,88 @@ def toDataFrame(self): # pass return dfs + def toDataset(self): + """ + Convert the data that was read in into a xarray Dataset + """ + from xarray import IndexVariable, DataArray, Dataset + + y = IndexVariable("y", self.y, attrs={"description":"lateral coordinate","units":"m"}) + zround = np.asarray([np.round(zz,6) for zz in self.z]) #the open function here returns something like *.0000000001 which is annoying + z = IndexVariable("z", zround, attrs={"description":"vertical coordinate","units":"m"}) + time = IndexVariable("time", self.t, attrs={"description":"time since start of simulation","units":"s"}) + + da = {} + for component,direction,velname in zip([0,1,2],["x","y","z"],["u","v","w"]): + # the dataset produced here has y/z axes swapped relative to data stored in original object + velocity = np.swapaxes(self["u"][component,...],1,2) + da[velname] = DataArray(velocity, + coords={"time":time,"y":y,"z":z}, + dims=["time","y","z"], + name="velocity", + attrs={"description":"velocity along {0}".format(direction),"units":"m/s"}) + + return Dataset(data_vars=da, coords={"time":time,"y":y,"z":z}) # Useful converters + def fromAMRWind_PD(self, filename, timestep, output_frequency, sampling_identifier, verbose=1, fileout=None, zref=None, xloc=None): + """ + Reads a AMRWind netcdf file, grabs a group of sampling planes (e.g. p_slice), + + Parameters + ---------- + filename : str, + full path to netcdf file generated by amrwind + timestep : float, + amr-wind code timestep (time.fixed_dt) + output_frequency : int, + frequency chosen for sampling output in amrwind input file (sampling.output_frequency) + sampling_identifier : str, + identifier of the sampling being requested (an entry of sampling.labels in amrwind input file) + zref : float, + height to be written to turbsim as the reference height. if none is given, it is taken as the vertical centerpoint of the slice + """ + try: + from weio.amrwind_file import AMRWind + except: + try: + from .amrwind_file import AMRWind + except: + from amrwind_file import AMRWind + + obj = AMRWind(filename,timestep,output_frequency) + obj.read(sampling_identifier) + + self["u"] = np.ndarray((3,obj.nt,obj.ny,obj.nz)) + + xloc = float(obj.data.x[0]) if xloc is None else xloc + if verbose: + print("Grabbing the slice at x={0} m".format(xloc)) + self['u'][0,:,:,:] = np.swapaxes(obj.data.u.sel(x=xloc).values,1,2) + self['u'][1,:,:,:] = np.swapaxes(obj.data.v.sel(x=xloc).values,1,2) + self['u'][2,:,:,:] = np.swapaxes(obj.data.w.sel(x=xloc).values,1,2) + self['t'] = obj.data.t.values + + self['y'] = obj.data.y.values + self['z'] = obj.data.z.values + self['dt'] = obj.output_dt + + self['ID'] = 7 + ltime = time.strftime('%d-%b-%Y at %H:%M:%S', time.localtime()) + self['info'] = 'Converted from AMRWind output file {0} {1:s}.'.format(filename,ltime) + + iz = int(obj.nz/2) + self['zRef'] = float(obj.data.z[iz]) if zref is None else zref + if verbose: + print("Setting the TurbSim file reference height to z={0} m".format(self["zRef"])) + + self['uRef'] = float(obj.data.u.sel(x=xloc).sel(y=0).sel(z=self["zRef"]).mean().values) + self['zRef'], self['uRef'], bHub = self.hubValues() + + fileout = filename.replace(".nc",".bts") if fileout is None else fileout + print("===> {0}".format(fileout)) + self.write(fileout) + def fromAMRWind(self, filename, dt, nt): """ Convert current TurbSim file into one generated from AMR-Wind LES sampling data in .nc format diff --git a/pydatview/io/wetb/hawc2/htc_file.py b/pydatview/io/wetb/hawc2/htc_file.py index 7648ac7..6b6bf3a 100644 --- a/pydatview/io/wetb/hawc2/htc_file.py +++ b/pydatview/io/wetb/hawc2/htc_file.py @@ -591,7 +591,6 @@ def unix_path(self, filename): if "__main__" == __name__: f = HTCFile(r"C:/Work/BAR-Local/Hawc2ToBeamDyn/sim.htc", ".") print(f.input_files()) - import pdb; pdb.set_trace() # f.save(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT_power_curve.htc") # # f = HTCFile(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT.htc", "../") diff --git a/pydatview/tools/curve_fitting.py b/pydatview/tools/curve_fitting.py index cf99636..64c71ee 100644 --- a/pydatview/tools/curve_fitting.py +++ b/pydatview/tools/curve_fitting.py @@ -192,6 +192,68 @@ def fit_powerlaw_u_alpha(x, y, z_ref=100, p0=(10,0.1)): return y_fit, pfit, {'coeffs':coeffs_dict,'formula':formula,'fitted_function':fitted_fun} +def polyfit2d(x, y, z, kx=3, ky=3, order=None): + ''' + Two dimensional polynomial fitting by least squares. + Fits the functional form f(x,y) = z. + + Notes + ----- + Resultant fit can be plotted with: + np.polynomial.polynomial.polygrid2d(x, y, soln.reshape((kx+1, ky+1))) + + Parameters + ---------- + x, y: array-like, 1d + x and y coordinates. + z: np.ndarray, 2d + Surface to fit. + kx, ky: int, default is 3 + Polynomial order in x and y, respectively. + order: int or None, default is None + If None, all coefficients up to maxiumum kx, ky, ie. up to and including x^kx*y^ky, are considered. + If int, coefficients up to a maximum of kx+ky <= order are considered. + + Returns + ------- + Return paramters from np.linalg.lstsq. + + soln: np.ndarray + Array of polynomial coefficients. + residuals: np.ndarray + rank: int + s: np.ndarray + + # The resultant fit can be visualised with: + # + # fitted_surf = np.polynomial.polynomial.polyval2d(x, y, soln.reshape((kx+1,ky+1))) + # plt.matshow(fitted_surf + + + ''' + + # grid coords + x, y = np.meshgrid(x, y) + # coefficient array, up to x^kx, y^ky + coeffs = np.ones((kx+1, ky+1)) + + # solve array + a = np.zeros((coeffs.size, x.size)) + + # for each coefficient produce array x^i, y^j + for index, (j, i) in enumerate(np.ndindex(coeffs.shape)): # TODO should it be i,j + # do not include powers greater than order + if order is not None and i + j > order: + arr = np.zeros_like(x) + else: + arr = coeffs[i, j] * x**i * y**j + a[index] = arr.ravel() + + # do leastsq fitting and return leastsq result + return np.linalg.lstsq(a.T, np.ravel(z), rcond=None) + + + # --------------------------------------------------------------------------------} # --- Predifined functions NOTE: they need to be registered in variable `MODELS` # --------------------------------------------------------------------------------{ @@ -1019,7 +1081,7 @@ def fit_data(self, x, y, p0=None, bounds=None): def fitted_function(xx): y=np.zeros(xx.shape) for i,(e,c) in enumerate(zip(self.exponents,pfit)): - y += c*x**e + y += c*xx**e return y self.model['fitted_function']=fitted_function diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index 91649eb..0b55043 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -31,16 +31,17 @@ __all__ = ['rainflow_astm', 'rainflow_windap','eq_load','eq_load_and_cycles','cycle_matrix','cycle_matrix2'] -def equivalent_load(signal, m=3, Teq=1, nBins=100, method='rainflow_windap'): +def equivalent_load(time, signal, m=3, Teq=1, nBins=100, method='rainflow_windap'): """Equivalent load calculation Calculate the equivalent loads for a list of Wohler exponent Parameters ---------- - signals : array-like, the signal + time : array-like, the time values corresponding to the signal (s) + signals : array-like, the load signal m : Wohler exponent (default is 3) - Teq : The equivalent number of load cycles (default is 1, but normally the time duration in seconds is used) + Teq : The equivalent period (Default 1Hz) nBins : Number of bins in rainflow count histogram method: 'rainflow_windap, rainflow_astm, fatpack @@ -48,12 +49,22 @@ def equivalent_load(signal, m=3, Teq=1, nBins=100, method='rainflow_windap'): ------- Leq : the equivalent load for given m and Tea """ + time = np.asarray(time) signal = np.asarray(signal) + # Remove nan, might not be the cleanest + b = ~np.isnan(signal) + signal = signal[b] + time = time[b] + + T = time[-1]-time[0] # time length of signal (s) + + neq = T/Teq # number of equivalent periods + rainflow_func_dict = {'rainflow_windap':rainflow_windap, 'rainflow_astm':rainflow_astm} if method in rainflow_func_dict.keys(): # Call wetb function for one m - Leq = eq_load(signal, m=[m], neq=Teq, no_bins=nBins, rainflow_func=rainflow_func_dict[method])[0][0] + Leq = eq_load(signal, m=[m], neq=neq, no_bins=nBins, rainflow_func=rainflow_func_dict[method])[0][0] elif method=='fatpack': import fatpack @@ -61,12 +72,12 @@ def equivalent_load(signal, m=3, Teq=1, nBins=100, method='rainflow_windap'): try: ranges = fatpack.find_rainflow_ranges(signal) except IndexError: - # Typically fails for constant signal + # Currently fails for constant signal return np.nan # find range count and bin Nrf, Srf = fatpack.find_range_count(ranges, nBins) - # get DEL - DELs = Srf**m * Nrf / Teq + # get DEL + DELs = Srf**m * Nrf / neq Leq = DELs.sum() ** (1/m) else: diff --git a/pydatview/tools/signal_analysis.py b/pydatview/tools/signal_analysis.py index bcf1d24..1e7a169 100644 --- a/pydatview/tools/signal_analysis.py +++ b/pydatview/tools/signal_analysis.py @@ -355,11 +355,12 @@ def applyFilterDF(df_old, x_col, options): # --------------------------------------------------------------------------------} # --- # --------------------------------------------------------------------------------{ -def zero_crossings(y,x=None,direction=None): +def zero_crossings(y, x=None, direction=None, bouncingZero=False): """ Find zero-crossing points in a discrete vector, using linear interpolation. direction: 'up' or 'down', to select only up-crossings or down-crossings + bouncingZero: also returns zeros that are exactly zero and do not change sign returns: x values xzc such that y(yzc)==0 @@ -386,7 +387,8 @@ def zero_crossings(y,x=None,direction=None): # Selecting points that are exactly 0 and where neighbor change sign iZero = np.where(y == 0.0)[0] iZero = iZero[np.where((iZero > 0) & (iZero < x.size-1))] - iZero = iZero[np.where(y[iZero-1]*y[iZero+1] < 0.0)] + if not bouncingZero: + iZero = iZero[np.where(y[iZero-1]*y[iZero+1] < 0.0)] # we only accept zeros that change signs # Concatenate xzc = np.concatenate((xzc, x[iZero])) @@ -405,7 +407,7 @@ def zero_crossings(y,x=None,direction=None): I= np.where(sign==-1)[0] return xzc[I],iBef[I] elif direction is not None: - raise Exception('Direction should be either `up` or `down`') + raise Exception('Direction should be either `up` or `down` or `None`') return xzc, iBef, sign @@ -416,15 +418,32 @@ def correlation(x, nMax=80, dt=1, method='manual'): """ Compute auto correlation of a signal """ + + def acf(x, nMax=20): + return np.array([1]+[np.corrcoef(x[:-i], x[i:])[0,1] for i in range(1, nMax)]) + + nvec = np.arange(0,nMax) sigma2 = np.var(x) R = np.zeros(nMax) - R[0] =1 - for i,nDelay in enumerate(nvec[1:]): - R[i+1] = np.mean( x[0:-nDelay] * x[nDelay:] ) / sigma2 + #R[0] =1 + #for i,nDelay in enumerate(nvec[1:]): + # R[i+1] = np.mean( x[0:-nDelay] * x[nDelay:] ) / sigma2 + # R[i+1] = np.corrcoef(x[:-nDelay], x[nDelay:])[0,1] + + R= acf(x, nMax=nMax) tau = nvec*dt return R, tau +# Auto-correlation comes in two versions: statistical and convolution. They both do the same, except for a little detail: The statistical version is normalized to be on the interval [-1,1]. Here is an example of how you do the statistical one: +# +# +# def autocorr(x): +# result = numpy.correlate(x, x, mode='full') +# return result[result.size/2:] + + + def correlated_signal(coeff, n=1000, seed=None): @@ -521,7 +540,6 @@ def amplitude(x, t=None, T = None, mask=None, debug=False): return A # split signals into subsets - import pdb; pdb.set_trace() else: return (np.max(x)-np.min(x))/2 diff --git a/pydatview/tools/spectral.py b/pydatview/tools/spectral.py index 245cb7a..9334b17 100644 --- a/pydatview/tools/spectral.py +++ b/pydatview/tools/spectral.py @@ -49,15 +49,21 @@ def fft_wrap(t,y,dt=None, output_type='amplitude',averaging='None',averaging_win averaging = averaging.lower() averaging_window = averaging_window.lower() y = np.asarray(y) + n0 = len(y) + nt = len(t) + if len(t)!=len(y): + raise Exception('t and y should have the same length') y = y[~np.isnan(y)] n = len(y) if dt is None: dtDelta0 = t[1]-t[0] # Hack to use a constant dt - dt = (np.max(t)-np.min(t))/(n-1) - if dtDelta0 !=dt: - print('[WARN] dt from tmax-tmin different from dt from t2-t1' ) + dt = (np.max(t)-np.min(t))/(n0-1) + relDiff = abs(dtDelta0-dt)/dt*100 + #if dtDelta0 !=dt: + if relDiff>0.01: + print('[WARN] dt from tmax-tmin different from dt from t2-t1 {} {}'.format(dt, dtDelta0) ) Fs = 1/dt if averaging =='none': frq, PSD, Info = psd(y, fs=Fs, detrend=detrend, return_onesided=True) diff --git a/pydatview/tools/stats.py b/pydatview/tools/stats.py index c5492d7..e546c34 100644 --- a/pydatview/tools/stats.py +++ b/pydatview/tools/stats.py @@ -11,7 +11,64 @@ # --------------------------------------------------------------------------------} # --- Stats measures # --------------------------------------------------------------------------------{ -def rsquare(y,f, c = True): +def comparison_stats(t1, y1, t2, y2, stats='sigRatio,eps,R2', method='mean', absVal=True): + """ + y1: ref + y2: other + + """ + from welib.tools.fatigue import equivalent_load + + sp=stats.split(',') + stats = {} + sStats=[] + + t1=np.asarray(t1).astype(float) + y1=np.asarray(y1).astype(float) + t2=np.asarray(t2).astype(float) + y2=np.asarray(y2).astype(float) + + # Loop on statistics requested + for s in sp: + s= s.strip().lower() + if s=='sigratio': + # Ratio of standard deviation: + sig_ref = float(np.nanstd(y1)) + sig_est = float(np.nanstd(y2)) + try: + r_sig = sig_est/sig_ref + except: + r_sig = np.nan + stats = {'sigRatio':r_sig} + sStats+= [r'$\sigma_\mathrm{est}/\sigma_\mathrm{ref} = $'+r'{:.3f}'.format(r_sig)] + + elif s=='eps': + # Mean relative error + eps = float(mean_rel_err(t1, y1, t2, y2, method=method, absVal=absVal)) + stats['eps'] = eps + sStats+=['$\epsilon=$'+r'{:.1f}%'.format(eps)] + + elif s=='r2': + # Rsquare + R2 = float(rsquare(y2, y1)[0]) + stats['R2'] = R2 + sStats+=[r'$R^2=$'+r'{:.3f}'.format(R2)] + + elif s=='epsleq': + Leq1 = equivalent_load(t1, y1, m=5, nBins=100, method='fatpack') + Leq2 = equivalent_load(t2, y2, m=5, nBins=100, method='fatpack') + epsLeq = (Leq2-Leq1)/Leq1*100 + stats['epsLeq'] = epsLeq + sStats+=[r'$\epsilon L_{eq}=$'+r'{:.1f}%'.format(epsLeq)] + + else: + raise NotImplementedError(s) + sStats=' - '.join(sStats) + return stats, sStats + + + +def rsquare(y, f, c = True): """ Compute coefficient of determination of data fit model and RMSE [r2 rmse] = rsquare(y,f) [r2 rmse] = rsquare(y,f,c) @@ -35,9 +92,11 @@ def rsquare(y,f, c = True): # OUTPUT R2 : Coefficient of determination RMSE : Root mean squared error """ - # Compare inputs + # Sanity if not np.all(y.shape == f.shape) : raise Exception('Y and F must be the same size') + y = np.asarray(y).astype(float) + f = np.asarray(f).astype(float) # Check for NaN tmp = np.logical_not(np.logical_or(np.isnan(y),np.isnan(f))) y = y[tmp] @@ -53,7 +112,7 @@ def rsquare(y,f, c = True): rmse = np.sqrt(np.mean((y - f) ** 2)) return r2,rmse -def mean_rel_err(t1=None, y1=None, t2=None, y2=None, method='mean', verbose=False, varname=''): +def mean_rel_err(t1=None, y1=None, t2=None, y2=None, method='meanabs', verbose=False, varname='', absVal=True): """ return mean relative error in % @@ -64,6 +123,13 @@ def mean_rel_err(t1=None, y1=None, t2=None, y2=None, method='mean', verbose=Fals |y1s-y2s|/|y1| '0-2': signals are scalled between 0 & 2 """ + def myabs(y): + if absVal: + return np.abs(y) + else: + return y + + if t1 is None and t2 is None: pass else: @@ -71,27 +137,27 @@ def mean_rel_err(t1=None, y1=None, t2=None, y2=None, method='mean', verbose=Fals y2=np.interp(t1,t2,y2) if method=='mean': # Method 1 relative to mean - ref_val = np.mean(y1) - meanrelerr = np.mean(np.abs(y2-y1)/ref_val)*100 + ref_val = np.nanmean(y1) + meanrelerr = np.nanmean(myabs(y2-y1)/ref_val)*100 elif method=='meanabs': - ref_val = np.mean(np.abs(y1)) - meanrelerr = np.mean(np.abs(y2-y1)/ref_val)*100 + ref_val = np.nanmean(abs(y1)) + meanrelerr = np.nanmean(myabs(y2-y1)/ref_val)*100 elif method=='loc': - meanrelerr = np.mean(np.abs(y2-y1)/abs(y1))*100 + meanrelerr = np.nanmean(myabs(y2-y1)/abs(y1))*100 elif method=='minmax': # Method 2 scaling signals - Min=min(np.min(y1), np.min(y2)) - Max=max(np.max(y1), np.max(y2)) + Min=min(np.nanmin(y1), np.nanmin(y2)) + Max=max(np.nanmax(y1), np.nanmax(y2)) y1=(y1-Min)/(Max-Min)+0.5 y2=(y2-Min)/(Max-Min)+0.5 - meanrelerr = np.mean(np.abs(y2-y1)/np.abs(y1))*100 + meanrelerr = np.nanmean(myabs(y2-y1)/np.abs(y1))*100 elif method=='1-2': # transform values from 1 to 2 - Min=min(np.min(y1), np.min(y2)) - Max=max(np.max(y1), np.max(y2)) + Min=min(np.nanmin(y1), np.nanmin(y2)) + Max=max(np.nanmax(y1), np.nanmax(y2)) y1 = (y1-Min)/(Max-Min)+1 y2 = (y2-Min)/(Max-Min)+1 - meanrelerr = np.mean(np.abs(y2-y1)/np.abs(y1))*100 + meanrelerr = np.nanmean(myabs(y2-y1)/np.abs(y1))*100 else: raise Exception('Unknown method',method) @@ -174,7 +240,7 @@ def bin_DF(df, xbins, colBin, stats='mean'): Perform bin averaging of a dataframe INPUTS: - df : pandas dataframe - - xBins: end points delimiting the bins, array of ascending x values) + - xBins: end points delimiting the bins, array of ascending x values - colBin: column name (string) of the dataframe, used for binning OUTPUTS: binned dataframe, with additional columns 'Counts' for the number @@ -198,13 +264,13 @@ def bin_DF(df, xbins, colBin, stats='mean'): def bin_signal(x, y, xbins=None, stats='mean', nBins=None): """ - Perform bin averaging of a dataframe + Perform bin averaging of a signal INPUTS: - - df : pandas dataframe - - xBins: end points delimiting the bins, array of ascending x values) - - colBin: column name (string) of the dataframe, used for binning + - x: x-values + - y: y-values, signal values + - xBins: end points delimiting the bins, array of ascending x values OUTPUTS: - binned dataframe, with additional columns 'Counts' for the number + - xBinned, yBinned """ if xbins is None: @@ -217,6 +283,55 @@ def bin_signal(x, y, xbins=None, stats='mean', nBins=None): +def bin2d_signal(x, y, z, xbins=None, ybins=None, nXBins=None, nYBins=None): + """ + Bin signal z based on x and y values using xbins and ybins + + """ + if xbins is None: + xmin, xmax = np.min(x), np.max(x) + dx = (xmax-xmin)/nXBins + xbins=np.arange(xmin, xmax+dx/2, dx) + if ybins is None: + ymin, ymax = np.min(y), np.max(y) + dy = (ymax-ymin)/nYBins + ybins=np.arange(ymin, ymax+dy/2, dy) + + x = np.asarray(x).flatten() + y = np.asarray(y).flatten() + z = np.asarray(z).flatten() + + Counts = np.zeros((len(xbins)-1, len(ybins)-1)) + XMean = np.zeros((len(xbins)-1, len(ybins)-1))*np.nan + YMean = np.zeros((len(xbins)-1, len(ybins)-1))*np.nan + ZMean = np.zeros((len(xbins)-1, len(ybins)-1))*np.nan + ZStd = np.zeros((len(xbins)-1, len(ybins)-1))*np.nan + + xmid = xbins[:-1] + np.diff(xbins)/2 + ymid = ybins[:-1] + np.diff(ybins)/2 + YMid, XMid = np.meshgrid(ymid, xmid) + + for ixb, xb in enumerate(xbins[:-1]): + print(ixb) + bX = np.logical_and(x >= xb, x <= xbins[ixb+1]) # TODO decide on bounds + for iyb, yb in enumerate(ybins[:-1]): + bY = np.logical_and(y >= yb, y <= ybins[iyb+1]) # TODO decide on bounds + + bXY = np.logical_and(bX, bY) + Counts[ixb, iyb] = sum(bXY) + if Counts[ixb,iyb]>0: + ZMean [ixb, iyb] = np.mean(z[bXY]) + ZStd [ixb, iyb] = np.std( z[bXY]) + XMean [ixb, iyb] = np.mean(x[bXY]) + YMean [ixb, iyb] = np.mean(y[bXY]) + + return XMean, YMean, ZMean, ZStd, Counts, XMid, YMid + + + + + + def azimuthal_average_DF(df, psiBin=np.arange(0,360+1,10), colPsi='Azimuth_[deg]', tStart=None, colTime='Time_[s]'): """ Average a dataframe based on azimuthal value From 48fc21d72961e47df71bf2149299e639525b8e10 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 26 Jun 2023 19:22:41 -0600 Subject: [PATCH 090/178] Plot: fix xlim/ylim for large values and small delta (Fixes #157) --- pydatview/GUIPlotPanel.py | 243 +++++++++++++++++++------------------- 1 file changed, 124 insertions(+), 119 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 323bb8e..9af81cf 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -975,13 +975,14 @@ def set_axes_lim(self, PDs, axis): try: xMin=np.min([PDs[i]._xMin[0] for i in axis.iPD]) xMax=np.max([PDs[i]._xMax[0] for i in axis.iPD]) - if np.isclose(xMin,xMax): - delta=1 if np.isclose(xMax,0) else 0.1*xMax + delta = xMax-xMin + if delta==0: + delta=1 else: - if tight: - delta=0 - else: - delta = (xMax-xMin)*pyplot_rc['axes.xmargin'] + # if tight: + # delta=0 + # else: + delta = delta*pyplot_rc['axes.xmargin'] axis.set_xlim(xMin-delta,xMax+delta) axis.autoscale(False, axis='x', tight=False) except: @@ -990,14 +991,18 @@ def set_axes_lim(self, PDs, axis): try: yMin=np.min([PDs[i]._yMin[0] for i in axis.iPD]) yMax=np.max([PDs[i]._yMax[0] for i in axis.iPD]) - delta = (yMax-yMin)*pyplot_rc['axes.ymargin'] - if np.isclose(yMin,yMax): - delta=1 if np.isclose(yMax,0) else 0.1*yMax + delta = (yMax-yMin) + # Note: uncomment and figure something out for small fluctuations + if delta==0: + delta=1 else: - if tight: - delta=0 - else: - delta = (yMax-yMin)*pyplot_rc['axes.xmargin'] + #if np.isclose(yMin,yMax): + # delta=1 if np.isclose(yMax,0) else 0.1*delta + #else: + # if tight: + # delta=0 + # else: + delta = delta*pyplot_rc['axes.xmargin'] axis.set_ylim(yMin-delta,yMax+delta) axis.autoscale(False, axis='y', tight=False) except: @@ -1063,112 +1068,112 @@ def plot_all(self, keep_limits=True): __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) - # Log Axes - if self.cbLogX.IsChecked(): - try: - ax_left.set_xscale("log", nonpositive='clip') # latest - except: - ax_left.set_xscale("log", nonposx='clip') # legacy - - if self.cbLogY.IsChecked(): - if bAllNegLeft is False: - try: - ax_left.set_yscale("log", nonpositive='clip') # latest - except: - ax_left.set_yscale("log", nonposy='clip') - if bAllNegRight is False and ax_right is not None: - try: - ax_right.set_yscale("log", nonpositive='clip') # latest - except: - ax_left.set_yscale("log", nonposy='clip') # legacy - - # XLIM - TODO FFT ONLY NASTY - if self.pltTypePanel.cbFFT.GetValue(): - try: - if self.cbAutoScale.IsChecked(): - xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) - if xlim>0: - ax_left.set_xlim([0,xlim]) - pd=PD[ax_left.iPD[0]] - I=pd.x 0 and len(yleft_labels) <= 3: - ax_left.set_ylabel(' and '.join(yleft_labels), **font_options) - elif ax_left is not None: - ax_left.set_ylabel('') - if len(yright_labels) > 0 and len(yright_labels) <= 3: - ax_right.set_ylabel(' and '.join(yright_labels), **font_options) - elif ax_right is not None: - ax_right.set_ylabel('') - - # Legends - lgdLoc = self.esthPanel.cbLegend.Value.lower() - if (self.pltTypePanel.cbCompare.GetValue() or - ((len(yleft_legends) + len(yright_legends)) > 1)): - if lgdLoc !='none': - if len(yleft_legends) > 0: - ax_left.legend(fancybox=False, loc=lgdLoc, **font_options_legd) - if ax_right is not None and len(yright_legends) > 0: - ax_right.legend(fancybox=False, loc=4, **font_options_legd) - elif len(axes)>1 and len(axes)==len(PD): - # TODO: can this be removed? If there is only one unique signal - # per subplot, normally only ylabel is displayed and no legend. - # Special case when we have subplots and all plots have the same label - if lgdLoc !='none': - usy = unique([pd.sy for pd in PD]) - if len(usy)==1: - for ax in axes: - ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) +# # Log Axes +# if self.cbLogX.IsChecked(): +# try: +# ax_left.set_xscale("log", nonpositive='clip') # latest +# except: +# ax_left.set_xscale("log", nonposx='clip') # legacy +# +# if self.cbLogY.IsChecked(): +# if bAllNegLeft is False: +# try: +# ax_left.set_yscale("log", nonpositive='clip') # latest +# except: +# ax_left.set_yscale("log", nonposy='clip') +# if bAllNegRight is False and ax_right is not None: +# try: +# ax_right.set_yscale("log", nonpositive='clip') # latest +# except: +# ax_left.set_yscale("log", nonposy='clip') # legacy +# +# # XLIM - TODO FFT ONLY NASTY +# if self.pltTypePanel.cbFFT.GetValue(): +# try: +# if self.cbAutoScale.IsChecked(): +# xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) +# if xlim>0: +# ax_left.set_xlim([0,xlim]) +# pd=PD[ax_left.iPD[0]] +# I=pd.x 0 and len(yleft_labels) <= 3: +# ax_left.set_ylabel(' and '.join(yleft_labels), **font_options) +# elif ax_left is not None: +# ax_left.set_ylabel('') +# if len(yright_labels) > 0 and len(yright_labels) <= 3: +# ax_right.set_ylabel(' and '.join(yright_labels), **font_options) +# elif ax_right is not None: +# ax_right.set_ylabel('') +# +# # Legends +# lgdLoc = self.esthPanel.cbLegend.Value.lower() +# if (self.pltTypePanel.cbCompare.GetValue() or +# ((len(yleft_legends) + len(yright_legends)) > 1)): +# if lgdLoc !='none': +# if len(yleft_legends) > 0: +# ax_left.legend(fancybox=False, loc=lgdLoc, **font_options_legd) +# if ax_right is not None and len(yright_legends) > 0: +# ax_right.legend(fancybox=False, loc=4, **font_options_legd) +# elif len(axes)>1 and len(axes)==len(PD): +# # TODO: can this be removed? If there is only one unique signal +# # per subplot, normally only ylabel is displayed and no legend. +# # Special case when we have subplots and all plots have the same label +# if lgdLoc !='none': +# usy = unique([pd.sy for pd in PD]) +# if len(usy)==1: +# for ax in axes: +# ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) # --- End loop on axes # --- Measure From 7ce7c5797ad127ea2b471e2b71d4aa14ad4da969 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 27 Jun 2023 16:10:41 -0600 Subject: [PATCH 091/178] Plot: reintroducing code commented by mistake --- pydatview/GUIPlotPanel.py | 212 +++++++++++++++++++------------------- 1 file changed, 106 insertions(+), 106 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 9af81cf..6ee1f87 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -1068,112 +1068,112 @@ def plot_all(self, keep_limits=True): __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) -# # Log Axes -# if self.cbLogX.IsChecked(): -# try: -# ax_left.set_xscale("log", nonpositive='clip') # latest -# except: -# ax_left.set_xscale("log", nonposx='clip') # legacy -# -# if self.cbLogY.IsChecked(): -# if bAllNegLeft is False: -# try: -# ax_left.set_yscale("log", nonpositive='clip') # latest -# except: -# ax_left.set_yscale("log", nonposy='clip') -# if bAllNegRight is False and ax_right is not None: -# try: -# ax_right.set_yscale("log", nonpositive='clip') # latest -# except: -# ax_left.set_yscale("log", nonposy='clip') # legacy -# -# # XLIM - TODO FFT ONLY NASTY -# if self.pltTypePanel.cbFFT.GetValue(): -# try: -# if self.cbAutoScale.IsChecked(): -# xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) -# if xlim>0: -# ax_left.set_xlim([0,xlim]) -# pd=PD[ax_left.iPD[0]] -# I=pd.x 0 and len(yleft_labels) <= 3: -# ax_left.set_ylabel(' and '.join(yleft_labels), **font_options) -# elif ax_left is not None: -# ax_left.set_ylabel('') -# if len(yright_labels) > 0 and len(yright_labels) <= 3: -# ax_right.set_ylabel(' and '.join(yright_labels), **font_options) -# elif ax_right is not None: -# ax_right.set_ylabel('') -# -# # Legends -# lgdLoc = self.esthPanel.cbLegend.Value.lower() -# if (self.pltTypePanel.cbCompare.GetValue() or -# ((len(yleft_legends) + len(yright_legends)) > 1)): -# if lgdLoc !='none': -# if len(yleft_legends) > 0: -# ax_left.legend(fancybox=False, loc=lgdLoc, **font_options_legd) -# if ax_right is not None and len(yright_legends) > 0: -# ax_right.legend(fancybox=False, loc=4, **font_options_legd) -# elif len(axes)>1 and len(axes)==len(PD): -# # TODO: can this be removed? If there is only one unique signal -# # per subplot, normally only ylabel is displayed and no legend. -# # Special case when we have subplots and all plots have the same label -# if lgdLoc !='none': -# usy = unique([pd.sy for pd in PD]) -# if len(usy)==1: -# for ax in axes: -# ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) + # Log Axes + if self.cbLogX.IsChecked(): + try: + ax_left.set_xscale("log", nonpositive='clip') # latest + except: + ax_left.set_xscale("log", nonposx='clip') # legacy + + if self.cbLogY.IsChecked(): + if bAllNegLeft is False: + try: + ax_left.set_yscale("log", nonpositive='clip') # latest + except: + ax_left.set_yscale("log", nonposy='clip') + if bAllNegRight is False and ax_right is not None: + try: + ax_right.set_yscale("log", nonpositive='clip') # latest + except: + ax_left.set_yscale("log", nonposy='clip') # legacy + + # XLIM - TODO FFT ONLY NASTY + if self.pltTypePanel.cbFFT.GetValue(): + try: + if self.cbAutoScale.IsChecked(): + xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) + if xlim>0: + ax_left.set_xlim([0,xlim]) + pd=PD[ax_left.iPD[0]] + I=pd.x 0 and len(yleft_labels) <= 3: + ax_left.set_ylabel(' and '.join(yleft_labels), **font_options) + elif ax_left is not None: + ax_left.set_ylabel('') + if len(yright_labels) > 0 and len(yright_labels) <= 3: + ax_right.set_ylabel(' and '.join(yright_labels), **font_options) + elif ax_right is not None: + ax_right.set_ylabel('') + + # Legends + lgdLoc = self.esthPanel.cbLegend.Value.lower() + if (self.pltTypePanel.cbCompare.GetValue() or + ((len(yleft_legends) + len(yright_legends)) > 1)): + if lgdLoc !='none': + if len(yleft_legends) > 0: + ax_left.legend(fancybox=False, loc=lgdLoc, **font_options_legd) + if ax_right is not None and len(yright_legends) > 0: + ax_right.legend(fancybox=False, loc=4, **font_options_legd) + elif len(axes)>1 and len(axes)==len(PD): + # TODO: can this be removed? If there is only one unique signal + # per subplot, normally only ylabel is displayed and no legend. + # Special case when we have subplots and all plots have the same label + if lgdLoc !='none': + usy = unique([pd.sy for pd in PD]) + if len(usy)==1: + for ax in axes: + ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) # --- End loop on axes # --- Measure From 0a832bf622d10437b76d6afe11c371cd8b702a78 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 27 Jun 2023 18:16:04 -0600 Subject: [PATCH 092/178] Tab: vertical stacking of tables (Closes #154) --- pydatview/GUISelectionPanel.py | 20 ++++++++++++++++++-- pydatview/Tables.py | 29 +++++++++++++++++++++++++++++ tests/test_Tables.py | 12 ++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index e284d46..e2665c4 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -294,10 +294,15 @@ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=Fal self.Bind(wx.EVT_MENU, self.mainframe.onAdd, item) if len(self.ISel)>1: - item = wx.MenuItem(self, -1, "Merge") + item = wx.MenuItem(self, -1, "Merge (horizontally)") self.Append(item) self.Bind(wx.EVT_MENU, self.OnMergeTabs, item) + if len(self.ISel)>1: + item = wx.MenuItem(self, -1, "Concatenate (vertically)") + self.Append(item) + self.Bind(wx.EVT_MENU, self.OnVStackTabs, item) + if len(self.ISel)>0: item = wx.MenuItem(self, -1, "Delete") self.Append(item) @@ -344,7 +349,18 @@ def OnMergeTabs(self, event): self.tabList.mergeTabs(self.ISel, ICommonColPerTab) # Updating tables self.selPanel.update_tabs(self.tabList) - # TODO select latest + # Select the newly created table + self.selPanel.tabPanel.lbTab.SetSelection(-1) # Empty selection + self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)-1) # Select new/last table + # Trigger a replot + self.selPanel.onTabSelectionChange() + + def OnVStackTabs(self, event): + # --- Figure out the common columns + # Merge tables and add it to the list + self.tabList.vstack(self.ISel, commonOnly=True) + # Updating tables + self.selPanel.update_tabs(self.tabList) # Select the newly created table self.selPanel.tabPanel.lbTab.SetSelection(-1) # Empty selection self.selPanel.tabPanel.lbTab.SetSelection(len(self.tabList)-1) # Select new/last table diff --git a/pydatview/Tables.py b/pydatview/Tables.py index c3c9769..ca687b9 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -251,6 +251,35 @@ def mergeTabs(self, I=None, ICommonColPerTab=None, extrap='nan'): self.append(Table(data=df, name=newName)) return newName, df + def vstack(self, I=None, commonOnly=False): + """ + Vertical stacking of tables + + I: index of tables to stack, if None: all tables are stacked + commonOnly: if True, keep only the common columns. + Otherwise, NaN will be introduced for missing columns + """ + if I is None: + I = range(len(self._tabs)) + dfs = [self._tabs[i].data for i in I] + + if commonOnly: + # --- Concatenate all but keep only common columns + df = pd.concat(dfs, join='inner', ignore_index=True) + else: + # --- Concatenate all, not worrying about common columns + df = pd.concat(dfs, ignore_index=True) + # Set unique index + if 'Index' in df.columns: + df = df.drop(['Index'], axis=1) + df.insert(0, 'Index', np.arange(df.shape[0])) + # Add to table list + newName = self._tabs[I[0]].name+'_concat' + self.append(Table(data=df, name=newName)) + return newName, df + + + def deleteTabs(self, I): self._tabs = [t for i,t in enumerate(self._tabs) if i not in I] diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 0d386cc..7fa7a4e 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -52,6 +52,18 @@ def test_merge(self): np.testing.assert_almost_equal(df['ColB'] , [np.nan , np.nan , 11 , 11.5 , 12 , 12.5 , 13.0] ) np.testing.assert_almost_equal(df['Index'], [0,1,2,3,4,5,6]) + def test_vstack(self): + # Vertical stack + tab1=Table(data=pd.DataFrame(data={'ID0': np.arange(0,3,0.5),'ColA': [10,10.5,11,11.5,12,12.5]})) + tab2=Table(data=pd.DataFrame(data={'ID': np.arange(1,4),'ColA': [11,12,13]})) + tablist = TableList([tab1,tab2]) + + # Concatenate keep only the common columns + name, df = tablist.vstack(commonOnly=True) + np.testing.assert_almost_equal(df['Index'], [0,1,2,3,4,5,6,7,8]) + np.testing.assert_almost_equal(df['ColA'], np.concatenate((tab1.data['ColA'], tab2.data['ColA'], ))) + np.testing.assert_equal(df.columns.values, ['Index','ColA']) + def test_resample(self): tab1=Table(data=pd.DataFrame(data={'BlSpn': [0,1,2],'Chord': [1,2,1]})) From f18c58507daf38e067084923de91923baa630a4d Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 30 Jun 2023 13:35:30 -0600 Subject: [PATCH 093/178] Postpro: fix subdyn radial outputs when fst not present --- pydatview/fast/postpro.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 523e53f..e20ded1 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -830,13 +830,14 @@ def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='. dfRad_BD = insert_spanwise_columns(dfRad_BD, r_BD, R=R, IR=IR_BD) out['BD'] = dfRad_BD # --- SubDyn - if fst.SD is not None: + try: + # NOTE: fst might be None sd = SubDyn(fst.SD) - MN = sd.pointsMN + #MN = sd.pointsMN MNout, MJout = sd.memberPostPro(dfAvg) out['SD_MembersOut'] = MNout out['SD_JointsOut'] = MJout - else: + except: out['SD_MembersOut'] = None out['SD_JointsOut'] = None From 32e97332bd1a8b06b9cc19c9f8a649978f3572e0 Mon Sep 17 00:00:00 2001 From: SimonHH <38310787+SimonHH@users.noreply.github.com> Date: Tue, 4 Jul 2023 12:04:03 +0200 Subject: [PATCH 094/178] fix plotdata.py for updates in fatigue.py --- pydatview/plotdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 4ede5ee..39807e3 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -630,7 +630,7 @@ def leq(PD,m): except ModuleNotFoundError: print('[INFO] module fatpack not installed, default to windap method for equivalent load') method='rainflow_windap' - v=equivalent_load(PD.y, m=m, Teq=T, nBins=100, method=method) + v=equivalent_load(PD.x, PD.y, m=m, Teq=T, nBins=100, method=method) return v,pretty_num(v) def Info(PD,var): From 85a398d23d373db526e2e5070159debf0b3e77fd Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 4 Jul 2023 12:33:00 -0600 Subject: [PATCH 095/178] PlotData: adding test for fatigue --- pydatview/plotdata.py | 15 ++++++++------- tests/test_plotdata.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 39807e3..a2c7cb0 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -618,18 +618,19 @@ def xMin(PD): s=pretty_num(v) return v,s - def leq(PD,m): + def leq(PD, m, method=None): from pydatview.tools.fatigue import equivalent_load if PD.yIsString or PD.yIsDate: return 'NA','NA' else: T,_=PD.xRange() - try: - import fatpack - method='fatpack' - except ModuleNotFoundError: - print('[INFO] module fatpack not installed, default to windap method for equivalent load') - method='rainflow_windap' + if method is None: + try: + import fatpack + method='fatpack' + except ModuleNotFoundError: + print('[INFO] module fatpack not installed, default to windap method for equivalent load') + method='rainflow_windap' v=equivalent_load(PD.x, PD.y, m=m, Teq=T, nBins=100, method=method) return v,pretty_num(v) diff --git a/tests/test_plotdata.py b/tests/test_plotdata.py index 71a77c2..c9ce78d 100644 --- a/tests/test_plotdata.py +++ b/tests/test_plotdata.py @@ -60,5 +60,19 @@ def test_PDF(self): #plt.plot(PD.x,fitter.model['fitted_function'](PD.x),'k--') #plt.show() + def test_fatigue(self): + dt = 0.1 + f0 = 1 ; + A = 5 ; + t=np.arange(0,10,dt); + y=A*np.sin(2*np.pi*f0*t) + + PD = PlotData(t,y) + v, s = PD.leq(m=10, method='rainflow_windap') + np.testing.assert_almost_equal(v, 11.91189, 3) + + + + if __name__ == '__main__': unittest.main() From a02cef84816ef25797811691ba76d667e9213d4d Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 6 Jul 2023 20:19:14 -0600 Subject: [PATCH 096/178] PlotData: fix Teq to use 1Hz equivalent loads --- pydatview/plotdata.py | 3 +-- tests/test_plotdata.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index a2c7cb0..fdd9bc7 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -623,7 +623,6 @@ def leq(PD, m, method=None): if PD.yIsString or PD.yIsDate: return 'NA','NA' else: - T,_=PD.xRange() if method is None: try: import fatpack @@ -631,7 +630,7 @@ def leq(PD, m, method=None): except ModuleNotFoundError: print('[INFO] module fatpack not installed, default to windap method for equivalent load') method='rainflow_windap' - v=equivalent_load(PD.x, PD.y, m=m, Teq=T, nBins=100, method=method) + v=equivalent_load(PD.x, PD.y, m=m, Teq=1, nBins=100, method=method) return v,pretty_num(v) def Info(PD,var): diff --git a/tests/test_plotdata.py b/tests/test_plotdata.py index c9ce78d..bb0bc9c 100644 --- a/tests/test_plotdata.py +++ b/tests/test_plotdata.py @@ -69,7 +69,7 @@ def test_fatigue(self): PD = PlotData(t,y) v, s = PD.leq(m=10, method='rainflow_windap') - np.testing.assert_almost_equal(v, 11.91189, 3) + np.testing.assert_almost_equal(v, 9.4714702, 3) From 64979da2fab5d5909b032493ca7b86f8aa21f43f Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sat, 8 Jul 2023 16:25:24 -0600 Subject: [PATCH 097/178] Data: Adding rename plugin for fld/aero NOTE: might need to make this a bit more hidden as it is very OpenFAST specific --- pydatview/Tables.py | 57 ++++++++++++++++++++++++++--------- pydatview/io/__init__.py | 6 ++++ pydatview/pipeline.py | 17 +++++++++++ pydatview/plugins/__init__.py | 6 ++++ tests/test_Tables.py | 4 +++ 5 files changed, 75 insertions(+), 15 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index ca687b9..7cbceea 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -456,7 +456,7 @@ class Table(object): """ Main attributes: - data - - columns + - columns # TODO get rid of me - name - raw_name # Should be unique and can be used for identification - ID # Should be unique and can be used for identification @@ -710,6 +710,20 @@ def renameColumn(self,iCol,newName): self.columns[iCol]=newName self.data.columns.values[iCol]=newName + def renameColumns(self, strReplDict=None): + """ Rename all the columns of given table + - strReplDict: a string replacement dictionary of the form: {'new':'old'} + """ + if strReplDict is not None: + cols = self.data.columns + newcols = [] + for c in cols: + for new,old in strReplDict.items(): + c = c.replace(old,new) + newcols.append(c) + self.data.columns = newcols + self.columns = newcols + def deleteColumns(self, ICol): """ Delete columns by index, not column names which can have duplicates""" IKeep =[i for i in np.arange(self.data.shape[1]) if i not in ICol] @@ -860,20 +874,33 @@ def nRows(self): return len(self.data.iloc[:,0]) # TODO if not panda @staticmethod - def createDummy(n, label=''): - """ create a dummy table of length n""" - t = np.linspace(0, 4*np.pi, n) - x = np.sin(t)+10 - alpha_d = np.linspace(0, 360, n) - P = np.random.normal(0,100,n)+5000 - RPM = np.random.normal(-0.2,0.2,n) + 12. - - d={'Time_[s]':t, - 'x{}_[m]'.format(label): x, - 'alpha{}_[deg]'.format(label):alpha_d, - 'P{}_[W]'.format(label):P, - 'RotSpeed{}_[rpm]'.format(label):RPM} - df = pd.DataFrame(data=d) + def createDummy(n, label='', columns=None, nCols=None): + """ create a dummy table of length n + If columns or nCols are provided, they are used for the + """ + # + if nCols is None and columns is None: + t = np.linspace(0, 4*np.pi, n) + x = np.sin(t)+10 + alpha_d = np.linspace(0, 360, n) + P = np.random.normal(0,100,n)+5000 + RPM = np.random.normal(-0.2,0.2,n) + 12. + d={'Time_[s]':t, + 'x{}_[m]'.format(label): x, + 'alpha{}_[deg]'.format(label):alpha_d, + 'P{}_[W]'.format(label):P, + 'RotSpeed{}_[rpm]'.format(label):RPM} + else: + units=['m','m/s','kn','rad','w','deg'] + if columns is None: + columns = ['C{}{}_[{}]'.format(i,label, units[np.mod(i, len(units))] ) for i in range(nCols)] + else: + nCols=len(columns) + + d = np.zeros((n, nCols)) + for i in range(nCols): + d[:,i] = np.random.normal(0, 1, n) + i + df = pd.DataFrame(data=d, columns= columns) return Table(data=df, name='Dummy '+label) diff --git a/pydatview/io/__init__.py b/pydatview/io/__init__.py index 6929fd8..740c706 100644 --- a/pydatview/io/__init__.py +++ b/pydatview/io/__init__.py @@ -17,6 +17,12 @@ def fileFormats(userpath=None, ignoreErrors=False, verbose=False): """ return list of fileformats supported by the library If userpath is provided, + OUTPUTS: + if ignoreErrors is True: + formats, errors + else: + formats + """ global _FORMATS errors=[] diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index 91a792e..e967e4b 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -1,5 +1,22 @@ """ pipelines and actions + + +An action has: + - data: some data (dict) + + - A set of callbacks that manipulates tables: + + - tableFunctionAdd (Table, data=dict()) # applies to a full table, return a new table + - tableFunctionApply (Table, data=dict()) # applies to a full table (inplace) + - tableFunctionCancel(Table, data=dict()) # cancel action on a full table (inplace) + + - A set of callbacks that plot additional data: + + - plotDataFunction (x, y , data=dict()) # applies to x,y arrays only + + - guiEditorClass: to Edit the data of the action + """ import numpy as np diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 1f8a38b..1356ffd 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -55,6 +55,11 @@ def _data_standardizeUnitsWE(label, mainframe=None): from .data_standardizeUnits import standardizeUnitsAction return standardizeUnitsAction(label, mainframe, flavor='WE') +# --- Reversible actions +def _data_renameFldAero(label, mainframe=None): + from .data_renameFldAero import renameFldAeroAction + return renameFldAeroAction(label, mainframe) + # --- Tools def _tool_logdec(*args, **kwargs): @@ -83,6 +88,7 @@ def _tool_radialavg(*args, **kwargs): DATA_PLUGINS_SIMPLE=OrderedDict([ ('Standardize Units (SI)', _data_standardizeUnitsSI), ('Standardize Units (WE)', _data_standardizeUnitsWE), + ('Rename "Fld" > "Aero' , _data_renameFldAero), ]) diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 7fa7a4e..91fc4d3 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -114,6 +114,10 @@ def test_change_units(self): np.testing.assert_almost_equal(tab.data.values[:,3],[10]) self.assertEqual(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) + def test_renameColumns(self): + tab = Table.createDummy(n=3, columns=['RtFldCp [-]','B1FldFx [N]', 'angle [rad]']) + tab.renameColumns(strReplDict={'Aero':'Fld'}) + self.assertEqual(tab.columns, ['Index','RtAeroCp [-]', 'B1AeroFx [N]', 'angle [rad]']) if __name__ == '__main__': # TestTable.setUpClass() From aec7e2efcfe6cd52f3e80c0704f0bcc920a5aaa9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 10 Jul 2023 15:46:22 -0600 Subject: [PATCH 098/178] Table: columns are now readonly, taken from dataframe --- pydatview/Tables.py | 88 ++++++++++++---------- pydatview/plugins/data_standardizeUnits.py | 7 +- 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 7cbceea..d18c76a 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -383,50 +383,59 @@ def applyCommonMaskString(self,maskString,bAdd=True): # --- Resampling TODO MOVE THIS OUT OF HERE OR UNIFY def applyResampling(self,iCol,sampDict,bAdd=True): + """ Apply resampling on table list + TODO Make this part of the action + """ dfs_new = [] names_new = [] errors=[] for i,t in enumerate(self._tabs): -# try: - df_new, name_new = t.applyResampling(iCol, sampDict, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) -# except: -# errors.append('Resampling failed for table: '+t.active_name) # TODO + try: + df_new, name_new = t.applyResampling(iCol, sampDict, bAdd=bAdd) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) + except: + errors.append('Resampling failed for table: '+t.active_name) # TODO return dfs_new, names_new, errors # --- Filtering TODO MOVE THIS OUT OF HERE OR UNIFY def applyFiltering(self,iCol,options,bAdd=True): + """ Apply filtering on table list + TODO Make this part of the action + """ dfs_new = [] names_new = [] errors=[] for i,t in enumerate(self._tabs): -# try: - df_new, name_new = t.applyFiltering(iCol, options, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) -# except: -# errors.append('Resampling failed for table: '+t.active_name) # TODO + try: + df_new, name_new = t.applyFiltering(iCol, options, bAdd=bAdd) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) + except: + errors.append('Filtering failed for table: '+t.active_name) # TODO return dfs_new, names_new, errors - - - # --- Radial average related def radialAvg(self,avgMethod,avgParam): + """ Apply radial average on table list + TODO Make this part of the action + """ dfs_new = [] names_new = [] errors=[] for i,t in enumerate(self._tabs): - dfs, names = t.radialAvg(avgMethod,avgParam) - for df,n in zip(dfs,names): - if df is not None: - dfs_new.append(df) - names_new.append(n) + try: + dfs, names = t.radialAvg(avgMethod,avgParam) + for df,n in zip(dfs,names): + if df is not None: + dfs_new.append(df) + names_new.append(n) + except: + errors.append('Radial averaging failed for table: '+t.active_name) # TODO return dfs_new, names_new, errors # --- Element--related functions @@ -475,7 +484,7 @@ class Table(object): # active_name : # raw_name : # filename : - def __init__(self,data=None,name='',filename='',columns=[], fileformat=None, dayfirst=False): + def __init__(self,data=None, name='',filename='', fileformat=None, dayfirst=False): # Default init self.maskString='' self.mask=None @@ -499,7 +508,6 @@ def __init__(self,data=None,name='',filename='',columns=[], fileformat=None, day pass else: data.insert(0, 'Index', np.arange(self.data.shape[0])) - self.columns = self.columnsFromDF(data) # --- Trying to figure out how to name this table if name is None or len(str(name))==0: if data.columns.name is not None: @@ -534,7 +542,6 @@ def setupName(self,name=''): self.name=name self.active_name=self.name - def __repr__(self): s='Table object:\n' s+=' - name: {}\n'.format(self.name) @@ -548,8 +555,6 @@ def __repr__(self): s+=' - maskString: {}\n'.format(self.maskString) return s - def columnsFromDF(self,df): - return [s.replace('_',' ') for s in df.columns.values.astype(str)] # --- Mask def clearMask(self): @@ -619,6 +624,7 @@ def applyFiltering(self, iCol, options, bAdd=True): def radialAvg(self,avgMethod, avgParam): + # TODO make this a pluggin import pydatview.fast.postpro as fastlib import pydatview.fast.fastfarm as fastfarm df = self.data @@ -707,7 +713,6 @@ def convertTimeColumns(self, dayfirst=False): # --- Column manipulations def renameColumn(self,iCol,newName): - self.columns[iCol]=newName self.data.columns.values[iCol]=newName def renameColumns(self, strReplDict=None): @@ -722,14 +727,12 @@ def renameColumns(self, strReplDict=None): c = c.replace(old,new) newcols.append(c) self.data.columns = newcols - self.columns = newcols def deleteColumns(self, ICol): """ Delete columns by index, not column names which can have duplicates""" IKeep =[i for i in np.arange(self.data.shape[1]) if i not in ICol] self.data = self.data.iloc[:, IKeep] # Drop won't work for duplicates for i in sorted(ICol, reverse=True): - del(self.columns[i]) for f in self.formulas: if f['pos'] == (i + 1): self.formulas.remove(f) @@ -748,7 +751,6 @@ def addColumn(self, sNewName, NewCol, i=-1, sFormula=''): elif i>self.data.shape[1]+1: i=self.data.shape[1] self.data.insert(int(i+1),sNewName,NewCol) - self.columns=self.columnsFromDF(self.data) for f in self.formulas: if f['pos'] > i: f['pos'] = f['pos'] + 1 @@ -759,7 +761,6 @@ def setColumn(self,sNewName,NewCol,i,sFormula=''): raise ValueError('Cannot set column at position ' + str(i)) self.data = self.data.drop(columns=self.data.columns[i]) self.data.insert(int(i),sNewName,NewCol) - self.columns=self.columnsFromDF(self.data) for f in self.formulas: if f['pos'] == i: f['name'] = sNewName @@ -844,6 +845,14 @@ def shapestring(self): def shape(self): return (self.nRows, self.nCols) + @property + def columns(self): + return [s.replace('_',' ') for s in self.data.columns.values.astype(str)] + + @columns.setter + def columns(self, cols): + raise Exception('Columns is read only') + @property def columns_clean(self): return [no_unit(s) for s in self.columns] @@ -894,12 +903,11 @@ def createDummy(n, label='', columns=None, nCols=None): units=['m','m/s','kn','rad','w','deg'] if columns is None: columns = ['C{}{}_[{}]'.format(i,label, units[np.mod(i, len(units))] ) for i in range(nCols)] - else: - nCols=len(columns) - - d = np.zeros((n, nCols)) - for i in range(nCols): - d[:,i] = np.random.normal(0, 1, n) + i + nCols=len(columns) + + d = np.zeros((n, nCols)) + for i in range(nCols): + d[:,i] = np.random.normal(0, 1, n) + i df = pd.DataFrame(data=d, columns= columns) return Table(data=df, name='Dummy '+label) diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py index 3edadb4..dc64107 100644 --- a/pydatview/plugins/data_standardizeUnits.py +++ b/pydatview/plugins/data_standardizeUnits.py @@ -45,17 +45,14 @@ def changeUnits(tab, data): if data['flavor']=='WE': for i, colname in enumerate(tab.columns): colname, tab.data.iloc[:,i] = change_units_to_WE(colname, tab.data.iloc[:,i]) - tab.columns[i] = colname # TODO, use a dataframe everywhere.. - tab.data.columns = tab.columns + tab.data.columns.values[i] = colname elif data['flavor']=='SI': for i, colname in enumerate(tab.columns): colname, tab.data.iloc[:,i] = change_units_to_SI(colname, tab.data.iloc[:,i]) - tab.columns[i] = colname # TODO, use a dataframe everywhere.. - tab.data.columns = tab.columns + tab.data.columns.values[i] = colname else: raise NotImplementedError(data['flavor']) - def change_units_to_WE(s, c): """ Change units to wind energy units From 9eeb688454b1bf83cb9741e1fab5ac682d58e33b Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 10 Jul 2023 15:47:42 -0600 Subject: [PATCH 099/178] OpenFAST plugins: adding rename FldAero and radial Concat. Separate menu --- pydatview/GUIPlotPanel.py | 6 +-- pydatview/GUISelectionPanel.py | 2 - pydatview/fast/postpro.py | 64 ++++++++++++++++++++++++- pydatview/main.py | 25 ++++++++-- pydatview/pipeline.py | 27 +++++++++++ pydatview/plugins/__init__.py | 32 +++++++++---- pydatview/plugins/data_radialConcat.py | 47 ++++++++++++++++++ pydatview/plugins/data_renameFldAero.py | 55 +++++++++++++++++++++ pydatview/plugins/tool_radialavg.py | 2 +- 9 files changed, 239 insertions(+), 21 deletions(-) create mode 100644 pydatview/plugins/data_radialConcat.py create mode 100644 pydatview/plugins/data_renameFldAero.py diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 6ee1f87..9905184 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -790,11 +790,11 @@ def removeTools(self, event=None, Layout=True): def showTool(self, toolName=''): from pydatview.plugins import TOOLS - from pydatview.plugins import DATA_TOOLS # TODO remove me + from pydatview.plugins import OF_DATA_TOOLS # TODO remove me if toolName in TOOLS.keys(): self.showToolPanel(panelClass=TOOLS[toolName]) - elif toolName in DATA_TOOLS.keys(): - self.showToolPanel(panelClass=DATA_TOOLS[toolName]) + elif toolName in OF_DATA_TOOLS.keys(): + self.showToolPanel(panelClass=OF_DATA_TOOLS[toolName]) else: raise Exception('Unknown tool {}'.format(toolName)) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index e2665c4..3b75497 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -1258,11 +1258,9 @@ def setColForSimTab(self,ISel): ColInfo.append('----------------------------------') bFirst=True for it,t in enumerate(tabs): - print('IKeep',IKeepPerTab[it]) if it==0: continue INotOrdered=[ii for i,ii in enumerate(IKeepPerTab[it]) if ii!=IKeepPerTab[0][i]] - print('INot',INotOrdered) if len(INotOrdered)>0: im=INotOrdered[0] if bFirst: diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index e20ded1..29f3f01 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -639,6 +639,12 @@ def spanwiseColAD(Cols): ADSpanMap['^[A]*'+sB+r'N(\d*)AOA_\[deg\]' ]=sB+'Alpha_[deg]' # DBGOuts ADSpanMap['^[A]*'+sB+r'N(\d*)AxInd_\[-\]' ]=sB+'AxInd_[-]' ADSpanMap['^[A]*'+sB+r'N(\d*)TnInd_\[-\]' ]=sB+'TnInd_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)AxInd_qs_\[-\]' ]=sB+'AxInd_qs_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)TnInd_qs_\[-\]' ]=sB+'TnInd_qs_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)BEM_k_\[-\]' ]=sB+'BEM_k_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)BEM_kp_\[-\]' ]=sB+'BEM_kp_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)BEM_F_\[-\]' ]=sB+'BEM_F_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)BEM_CT_qs_\[-\]' ]=sB+'BEM_CT_qs_[-]' ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[deg\]' ]=sB+'AxInd_[-]' # DBGOuts NOTE BUG Unit ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[deg\]' ]=sB+'TnInd_[-]' # DBGOuts NOTE BUG Unit ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[-\]' ]=sB+'AxInd_[-]' # DBGOuts @@ -1039,6 +1045,58 @@ def FASTSpanwiseOutputs(FST_In, OutputCols=None, verbose=False): +def spanwiseConcat(df): + """ + Perform time-concatenation of all the spanwise data (AeroDyn only for now) + + For instance if df is: + + Time B1N001Alpha B1N002Alpha B1N003Alpha + t a1 a2 a3 + + with t, a1, a2, a3, arrays or length nt + + The concatenated dataframe will be: + Time i Alpha + t 1 a1 + t 2 a2 + t 3 a3 + + INPUTS: + - df: a dataframe, typically returned by FASTOutputFile (nt x (nc*nr + nother) ) + + OUTPUTS: + - dfCat: the time-concatenated dataframe (nt*nr x (2 + nc) ) + + """ + Cols = df.columns + ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) + nChan = len(ColsInfoAD) + imin = np.min( [np.min(ColsInfoAD[i]['Idx']) for i in range(nChan)] ) + imax = np.max( [np.max(ColsInfoAD[i]['Idx']) for i in range(nChan)] ) + time = df['Time_[s]'] + nt = len(time) + # We add two channels one for time, one for ispan + data = np.zeros((nt*nrMaxAD, nChan+2))*np.nan + # Loop on Channels and radial positions.. + for ic in range(nChan): + for ir in range(nrMaxAD): + data[ir*nt:(ir+1)*nt, 0] = time + data[ir*nt:(ir+1)*nt, 1] = ir+1 + IdxAvailableForThisChannel = ColsInfoAD[ic]['Idx'] + chanName = ColsInfoAD[ic]['name'] + colName = ColsInfoAD[ic]['cols'][ir] + #print('Channel {}: colName {}'.format(chanName, colName)) + if ir+1 in IdxAvailableForThisChannel: + data[ir*nt:(ir+1)*nt, ic+2] = df[colName].values + #else: + # raise Exception('Channel {}: Index missing {}'.format(chanName, ic+1)) + columns = ['Time_[s]'] + ['i_[-]'] + [ColsInfoAD[i]['name'] for i in range(nChan)] + dfCat = pd.DataFrame(data=data, columns=columns) + + return dfCat + + def addToOutlist(OutList, Signals): if not isinstance(Signals,list): raise Exception('Signals must be a list') @@ -1657,4 +1715,8 @@ def integrateMomentTS(r, F): return M if __name__ == '__main__': - main() + + import welib.weio as weio + df = weio.read('ad_driver_yaw.6.outb').toDataFrame() + dfCat = spanwiseConcat(df) + print(dfCat) diff --git a/pydatview/main.py b/pydatview/main.py index 9c8ab80..c7ab3e2 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -32,7 +32,8 @@ from .GUICommon import * import pydatview.io as weio # File Formats and File Readers # Pluggins -from .plugins import DATA_PLUGINS_WITH_EDITOR, DATA_PLUGINS_SIMPLE, DATA_TOOLS, TOOLS +from .plugins import DATA_PLUGINS_WITH_EDITOR, DATA_PLUGINS_SIMPLE, TOOLS +from .plugins import OF_DATA_TOOLS, OF_DATA_PLUGINS_SIMPLE from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData # --------------------------------------------------------------------------------} @@ -144,9 +145,6 @@ def __init__(self, data=None): # NOTE: very important, need "s_loc" otherwise the lambda function take the last toolName dataMenu = wx.Menu() menuBar.Append(dataMenu, "&Data") - for toolName in DATA_TOOLS.keys(): # TODO remove me, should be an action - self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onShowTool(e, s_loc), dataMenu.Append(wx.ID_ANY, toolName)) - for toolName in DATA_PLUGINS_WITH_EDITOR.keys(): self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onDataPlugin(e, s_loc), dataMenu.Append(wx.ID_ANY, toolName)) @@ -159,6 +157,17 @@ def __init__(self, data=None): for toolName in TOOLS.keys(): self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onShowTool(e, s_loc), toolMenu.Append(wx.ID_ANY, toolName)) + # --- OpenFAST Plugins + ofMenu = wx.Menu() + menuBar.Append(ofMenu, "&OpenFAST") + for toolName in OF_DATA_TOOLS.keys(): # TODO remove me, should be an action + self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onShowTool(e, s_loc), ofMenu.Append(wx.ID_ANY, toolName)) + + for toolName in OF_DATA_PLUGINS_SIMPLE.keys(): + self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onDataPlugin(e, s_loc), ofMenu.Append(wx.ID_ANY, toolName)) + + + # --- Help Menu helpMenu = wx.Menu() aboutMenuItem = helpMenu.Append(wx.NewId(), 'About', 'About') resetMenuItem = helpMenu.Append(wx.NewId(), 'Reset options', 'Rest options') @@ -490,12 +499,18 @@ def onDataPlugin(self, event=None, toolName=''): self.plotPanel.showToolAction(action) # The panel will have the responsibility to apply/delete the action, updateGUI, etc elif toolName in DATA_PLUGINS_SIMPLE.keys(): - print('>>> toolName') function = DATA_PLUGINS_SIMPLE[toolName] action = function(label=toolName, mainframe=self) # calling the data function # Here we apply the action directly # We can't overwrite, so we'll delete by name.. self.addAction(action, overwrite=False, apply=True, tabList=self.tabList, updateGUI=True) + + elif toolName in OF_DATA_PLUGINS_SIMPLE.keys(): # TODO merge with DATA_PLUGINS_SIMPLE + function = OF_DATA_PLUGINS_SIMPLE[toolName] + action = function(label=toolName, mainframe=self) # calling the data function + # Here we apply the action directly + # We can't overwrite, so we'll delete by name.. + self.addAction(action, overwrite=False, apply=True, tabList=self.tabList, updateGUI=True) else: raise NotImplementedError('Tool: ',toolName) diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index e967e4b..f6c23c4 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -201,6 +201,33 @@ def __repr__(self): s=''.format(self.name, self.applied) return s +class AdderAction(Action): + + def __init__(self, name, tableFunctionAdd, **kwargs): + Action.__init__(self, name, tableFunctionAdd=tableFunctionAdd, **kwargs) + + def apply(self, tabList, force=False, applyToAll=False): + """ The apply of an Adder Action is to Add a Panel """ + if force: + self.applied = False + if self.applied: + print('>>> Action: Skipping Adder action', self.name) + return + # Call parent function applyAndAdd + dfs_new, names_new, errors = Action.applyAndAdd(self, tabList) + # Update GUI + if self.mainframe is not None: + addTablesHandle = self.mainframe.load_dfs + addTablesHandle(dfs_new, names_new, bAdd=True, bPlot=False) + + def cancel(self, *args, **kwargs): + pass + # print('>>> Action: Cancel: skipping irreversible action', self.name) + # pass + + def __repr__(self): + s=''.format(self.name, self.applied) + return s # --------------------------------------------------------------------------------} # --- Pipeline diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 1356ffd..72c5623 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -14,11 +14,11 @@ def _function_name(mainframe, event=None, label='') See working examples in this file and this directory. NOTE: - - data plugins constructors should return an Action with a GUI Editor class + - DATA_PLUGINS_WITH_EDITOR: plugins constructors should return an Action with a GUI Editor class - - simple data plugins constructors should return an Action + - DATA_PLUGINS_SIMPLE: simple data plugins constructors should return an Action - - tool plugins constructor should return a Panel class + - TOOLS: tool plugins constructor should return a Panel class """ @@ -60,6 +60,11 @@ def _data_renameFldAero(label, mainframe=None): from .data_renameFldAero import renameFldAeroAction return renameFldAeroAction(label, mainframe) +# --- Adding actions +def _data_radialConcat(label, mainframe=None): + from .data_radialConcat import radialConcatAction + return radialConcatAction(label, mainframe) + # --- Tools def _tool_logdec(*args, **kwargs): @@ -77,6 +82,7 @@ def _tool_radialavg(*args, **kwargs): # --- Ordered dictionaries with key="Tool Name", value="Constructor" +# DATA_PLUGINS constructors should return an Action with a GUI Editor class DATA_PLUGINS_WITH_EDITOR=OrderedDict([ ('Mask' , _data_mask ), ('Remove Outliers' , _data_removeOutliers ), @@ -85,18 +91,26 @@ def _tool_radialavg(*args, **kwargs): ('Bin data' , _data_binning ), ]) +# DATA_PLUGINS_SIMPLE: simple data plugins constructors should return an Action DATA_PLUGINS_SIMPLE=OrderedDict([ ('Standardize Units (SI)', _data_standardizeUnitsSI), ('Standardize Units (WE)', _data_standardizeUnitsWE), - ('Rename "Fld" > "Aero' , _data_renameFldAero), - ]) - - -DATA_TOOLS=OrderedDict([ # TODO - ('FAST - Radial Average', _tool_radialavg), ]) +# TOOLS: tool plugins constructor should return a Panel class TOOLS=OrderedDict([ ('Damping from decay',_tool_logdec), ('Curve fitting', _tool_curvefitting), ]) + + +# --- OpenFAST +# TOOLS: tool plugins constructor should return a Panel class +OF_DATA_TOOLS=OrderedDict([ # TODO + ('Radial Average', _tool_radialavg), + ]) +# DATA_PLUGINS_SIMPLE: simple data plugins constructors should return an Action +OF_DATA_PLUGINS_SIMPLE=OrderedDict([ + ('Radial Time Concatenation' , _data_radialConcat), + ('Rename "Fld" > "Aero' , _data_renameFldAero), + ]) diff --git a/pydatview/plugins/data_radialConcat.py b/pydatview/plugins/data_radialConcat.py new file mode 100644 index 0000000..5a79277 --- /dev/null +++ b/pydatview/plugins/data_radialConcat.py @@ -0,0 +1,47 @@ +import unittest +import numpy as np +from pydatview.common import splitunit +from pydatview.pipeline import AdderAction + +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def radialConcatAction(label, mainframe=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + """ + guiCallback=None + data={} + if mainframe is not None: + # TODO TODO TODO Clean this up + def guiCallback(): + if hasattr(mainframe,'selPanel'): + mainframe.selPanel.colPanel1.setColumns() + mainframe.selPanel.colPanel2.setColumns() + mainframe.selPanel.colPanel3.setColumns() + mainframe.onTabSelectionChange() # trigger replot + if hasattr(mainframe,'pipePanel'): + pass + # Function that will be applied to all tables + + action = AdderAction( + name=label, + tableFunctionAdd = radialConcat, + #tableFunctionApply = renameFldAero, + #tableFunctionCancel= renameAeroFld, + guiCallback=guiCallback, + mainframe=mainframe, # shouldnt be needed + data = data + ) + + return action + +def radialConcat(tab, data=None): + from pydatview.fast.postpro import spanwiseConcat + df_new = spanwiseConcat(tab.data) + name_new = tab.name+'_concat' + return df_new, name_new + + +if __name__ == '__main__': + unittest.main() diff --git a/pydatview/plugins/data_renameFldAero.py b/pydatview/plugins/data_renameFldAero.py new file mode 100644 index 0000000..ab9b514 --- /dev/null +++ b/pydatview/plugins/data_renameFldAero.py @@ -0,0 +1,55 @@ +import unittest +import numpy as np +from pydatview.common import splitunit +from pydatview.pipeline import ReversibleTableAction + +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def renameFldAeroAction(label, mainframe=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + """ + guiCallback=None + data={} + if mainframe is not None: + # TODO TODO TODO Clean this up + def guiCallback(): + if hasattr(mainframe,'selPanel'): + mainframe.selPanel.colPanel1.setColumns() + mainframe.selPanel.colPanel2.setColumns() + mainframe.selPanel.colPanel3.setColumns() + mainframe.onTabSelectionChange() # trigger replot + if hasattr(mainframe,'pipePanel'): + pass + # Function that will be applied to all tables + + action = ReversibleTableAction( + name=label, + tableFunctionApply = renameFldAero, + tableFunctionCancel= renameAeroFld, + guiCallback=guiCallback, + mainframe=mainframe, # shouldnt be needed + data = data + ) + + return action + +def renameFldAero(tab, data=None): + tab.renameColumns(strReplDict={'Aero':'Fld'}) # New:Old + +def renameAeroFld(tab, data=None): + tab.renameColumns( strReplDict={'Fld':'Aero'}) # New:Old + + +class TestRenameFldAero(unittest.TestCase): + + def test_change_units(self): + from pydatview.Tables import Table + tab = Table.createDummy(n=10, columns=['RtFldCp [-]','B1FldFx [N]', 'angle [rad]']) + renameFldAero(tab) + self.assertEqual(tab.columns, ['Index','RtAeroCp [-]', 'B1AeroFx [N]', 'angle [rad]']) + + +if __name__ == '__main__': + unittest.main() diff --git a/pydatview/plugins/tool_radialavg.py b/pydatview/plugins/tool_radialavg.py index f8a4494..5b06d2b 100644 --- a/pydatview/plugins/tool_radialavg.py +++ b/pydatview/plugins/tool_radialavg.py @@ -64,7 +64,7 @@ def onApply(self,event=None): dfs, names, errors = tabList.radialAvg(avgMethod,avgParam) self.parent.addTables(dfs,names,bAdd=True) if len(errors)>0: - raise Exception('Error: The mask failed on some tables:\n\n'+'\n'.join(errors)) + raise Exception('Error: The filtering failed on some tables:\n\n'+'\n'.join(errors)) else: dfs, names = tabList[iSel-1].radialAvg(avgMethod,avgParam) self.parent.addTables([dfs],[names], bAdd=True) From 9707d4a33cd913fc2fb5f22f02abe558f376e5cd Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 10 Jul 2023 16:03:39 -0600 Subject: [PATCH 100/178] Plot: set ylim when delta-y is small (see #157) --- pydatview/GUIPlotPanel.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 9905184..a720633 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -992,16 +992,21 @@ def set_axes_lim(self, PDs, axis): yMin=np.min([PDs[i]._yMin[0] for i in axis.iPD]) yMax=np.max([PDs[i]._yMax[0] for i in axis.iPD]) delta = (yMax-yMin) - # Note: uncomment and figure something out for small fluctuations if delta==0: - delta=1 + # If delta is zero, we extend the bounds to "readable" values + yMean = (yMax+yMin)/2 + if abs(yMean)>1e-6: + delta = 0.05*yMean + else: + delta = 1 + elif abs(yMin)>1e-6: + delta_rel = delta/abs(yMin) + if delta_rel<1e-5: + # we set a delta such that the numerical fluctuations are visible but + # it's obvious that it's still a "constant" signal + delta = 100*delta else: - #if np.isclose(yMin,yMax): - # delta=1 if np.isclose(yMax,0) else 0.1*delta - #else: - # if tight: - # delta=0 - # else: + # Note: uncomment and figure something out for small fluctuations delta = delta*pyplot_rc['axes.xmargin'] axis.set_ylim(yMin-delta,yMax+delta) axis.autoscale(False, axis='y', tight=False) From a2c779aa93ef1a9d628e91e1cf1e35de14d7fd60 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 10 Jul 2023 17:11:11 -0600 Subject: [PATCH 101/178] Common: fix unit for varaible of length 1 --- pydatview/common.py | 6 +++--- tests/test_common.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pydatview/common.py b/pydatview/common.py index 1bbccdd..7c9dd92 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -237,21 +237,21 @@ def cleanCol(s): def no_unit(s): s=s.replace('_[',' [') iu=s.rfind(' [') - if iu>1: + if iu>0: return s[:iu] else: return s def unit(s): iu=s.rfind('[') - if iu>1: + if iu>0: return s[iu+1:].replace(']','') else: return '' def splitunit(s): iu=s.rfind('[') - if iu>1: + if iu>0: return s[:iu], s[iu+1:].replace(']','') else: return s, '' diff --git a/tests/test_common.py b/tests/test_common.py index 17f61ff..ce31a12 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -17,6 +17,8 @@ def test_unit(self): self.assertEqual(unit ('speed [m/s]'),'m/s' ) self.assertEqual(unit ('speed [m/s' ),'m/s' ) # ... self.assertEqual(no_unit('speed [m/s]'),'speed') + self.assertEqual(no_unit('i [-]'),'i') + self.assertEqual(unit ('i [-]'),'-') def test_splitunit(self): self.assertEqual(splitunit ('speed [m/s]'),('speed ','m/s' )) From 0f6efd729bba220471d379a2e8b03d8121718316 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 10 Jul 2023 17:11:43 -0600 Subject: [PATCH 102/178] Pipeline: popup when errorlist >= numTabs --- pydatview/Tables.py | 24 ++++++++++------- pydatview/main.py | 13 ++++++++++ pydatview/pipeline.py | 44 ++++++++++++++++++-------------- pydatview/plugins/base_plugin.py | 10 +++++--- 4 files changed, 60 insertions(+), 31 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index d18c76a..8a46821 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -377,7 +377,7 @@ def applyCommonMaskString(self,maskString,bAdd=True): dfs_new.append(df_new) names_new.append(name_new) except: - errors.append('Mask failed for table: '+t.active_name) # TODO + errors.append('Mask failed for table: '+t.nickname) # TODO return dfs_new, names_new, errors @@ -397,7 +397,7 @@ def applyResampling(self,iCol,sampDict,bAdd=True): dfs_new.append(df_new) names_new.append(name_new) except: - errors.append('Resampling failed for table: '+t.active_name) # TODO + errors.append('Resampling failed for table: '+t.nickname) # TODO return dfs_new, names_new, errors # --- Filtering TODO MOVE THIS OUT OF HERE OR UNIFY @@ -416,7 +416,7 @@ def applyFiltering(self,iCol,options,bAdd=True): dfs_new.append(df_new) names_new.append(name_new) except: - errors.append('Filtering failed for table: '+t.active_name) # TODO + errors.append('Filtering failed for table: '+t.nickname) # TODO return dfs_new, names_new, errors # --- Radial average related @@ -435,7 +435,7 @@ def radialAvg(self,avgMethod,avgParam): dfs_new.append(df) names_new.append(n) except: - errors.append('Radial averaging failed for table: '+t.active_name) # TODO + errors.append('Radial averaging failed for table: '+t.nickname) # TODO return dfs_new, names_new, errors # --- Element--related functions @@ -564,9 +564,9 @@ def clearMask(self): def applyMaskString(self, sMask, bAdd=True): # Apply mask on Table df = self.data - for i,c in enumerate(self.columns): - c_no_unit = no_unit(c).strip() - c_in_df = df.columns[i] + # TODO Loop on {VAR} instead.. + for i,(c_in_df,c_user) in enumerate(zip(self.data.columns, self.columns)): + c_no_unit = no_unit(c_user).strip() # TODO sort out the mess with asarray (introduced to have and/or # as array won't work with date comparison # NOTE: using iloc to avoid duplicates column issue @@ -586,10 +586,11 @@ def applyMaskString(self, sMask, bAdd=True): self.mask=mask self.maskString=sMask except: - raise Exception('Error: The mask failed for table: '+self.name) + # TODO come up with better error messages + raise Exception('Error: The mask failed to evaluate for table: '+self.nickname) if sum(mask)==0: self.clearMask() - raise Exception('Error: The mask returned no value for table: '+self.name) + raise Exception('Error: The mask returned no value for table: '+self.nickname) return df_new, name_new # --- Important manipulation TODO MOVE THIS OUT OF HERE OR UNIFY @@ -874,6 +875,11 @@ def raw_name(self): def name(self,new_name): self.__name=new_name + @property + def nickname(self): + sp = self.name.split('|') + return sp[-1] + @property def nCols(self): return len(self.columns) diff --git a/pydatview/main.py b/pydatview/main.py index c7ab3e2..df6a9ba 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -521,6 +521,17 @@ def removeAction(self, action, **kwargs): self.pipePanel.remove(action, **kwargs) def applyPipeline(self, *args, **kwargs): self.pipePanel.apply(*args, **kwargs) + def checkErrors(self): + # TODO this should be done at a given point in the GUI + nErr = len(self.pipePanel.errorList) + if nErr>0: + if not self.pipePanel.user_warned: + sErr = '\n\nCheck `Errors` in the bottom right side of the window.' + if nErr>=len(self.tabList): + Warn(self, 'Errors occured on all tables.'+sErr) + #elif nErr>> Applying action', self.name, 'to', t.name) + print('>>> Applying action', self.name, 'to', t.nickname) try: + # TODO TODO TODO Collect errors here self.tableFunctionApply(t, data=self.data) - except: - err = 'Failed to apply action {} to table {}.'.format(self.name, t.name) + except Exception as e: + err = 'Failed to apply action {} to table {}.\nException: {}\n\n'.format(self.name, t.nickname, e.args[0]) self.errorList.append(err) self.applied = True @@ -94,16 +95,20 @@ def applyAndAdd(self, tabList): dfs_new = [] names_new = [] + self.errorList=[] errors=[] for i,t in enumerate(tabList): -# try: - df_new, name_new = self.tableFunctionAdd(t, self.data) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) -# except: -# errors.append('Resampling failed for table: '+t.active_name) # TODO + try: + df_new, name_new = self.tableFunctionAdd(t, self.data) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) + except: + err = 'Failed to apply action and add for table: '+t.nickname + self.errorList.append(err) + errors.append(err) + return dfs_new, names_new, errors @@ -112,10 +117,10 @@ def applyAndAdd(self, tabList): def updateGUI(self): """ Typically called by a callee after append""" if self.guiCallback is not None: - try: - self.guiCallback() - except: - print('[FAIL] Action: failed to call GUI callback, action', self.name) +# try: + self.guiCallback() +# except: +# print('[FAIL] Action: failed to call GUI callback, action', self.name) def __repr__(self): s=''.format(self.name) @@ -191,11 +196,11 @@ def cancel(self, tabList): print('[WARN] Cannot cancel action {} on None tablist'.format(self)) return for t in tabList: - print('>>> Action: Cancel: ', self, 'to', t.name) + print('>>> Action: Cancel: ', self, 'to', t.nickname) try: self.tableFunctionCancel(t, data=self.data) - except: - self.errorList.append('Failed to apply action {} to table {}.'.format(self.name, t.name)) + except Exception as e: + self.errorList.append('Failed to cancel action {} to table {}.\nException: {}\n\n'.format(self.name, t.nickname, e.args[0])) def __repr__(self): s=''.format(self.name, self.applied) @@ -238,6 +243,7 @@ def __init__(self, data=[]): self.actionsData = [] self.actionsPlotFilters = [] self.errorList = [] + self.user_warned = False # Has the user been warn that errors are present self.plotFiltersData=[] # list of data for plot data filters, that plotData.py will use @property @@ -258,7 +264,6 @@ def apply(self, tabList, force=False, applyToAll=False): # self.collectErrors() - def applyOnPlotData(self, x, y, tabID): x = np.copy(x) y = np.copy(y) @@ -270,6 +275,7 @@ def collectErrors(self): self.errorList=[] for action in self.actions: self.errorList+= action.errorList + self.user_warned = False # --- Behave like a list.. def append(self, action, overwrite=False, apply=True, updateGUI=True, tabList=None): diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py index 88c24b3..c775adc 100644 --- a/pydatview/plugins/base_plugin.py +++ b/pydatview/plugins/base_plugin.py @@ -8,7 +8,7 @@ wx=type('wx', (object,), {'Panel':object}) HAS_WX=False import numpy as np -from pydatview.common import CHAR, Error, Info +from pydatview.common import CHAR, Error, Info, Warn from pydatview.plotdata import PlotData TOOL_BORDER=15 @@ -138,8 +138,6 @@ def onAdd(self, event=None): self._GUI2Data() dfs, names, errors = self.action.applyAndAdd(self.tabList) - if len(errors)>0: - raise Exception('Error: The action {} failed on some tables:\n\n'.format(action.name)+'\n'.join(errors)) # We stop applying if we were applying it (NOTE: doing this before adding table due to redraw trigger of the whole panel) if self.data['active']: @@ -150,6 +148,12 @@ def onAdd(self, event=None): # self.parent.addTables([df], [name], bAdd=True) #self.updateTabList() + if len(errors)>0: + if len(errors)>=len(self.tabList): + Error(self, 'Error: The action {} failed on all tables:\n\n'.format(self.action.name)+'\n'.join(errors)) + #elif len(errors) Date: Tue, 11 Jul 2023 10:23:42 -0600 Subject: [PATCH 103/178] Plot: set ylim in regular case (see #157) --- pydatview/GUIPlotPanel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index a720633..ba02a2d 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -1005,8 +1005,9 @@ def set_axes_lim(self, PDs, axis): # we set a delta such that the numerical fluctuations are visible but # it's obvious that it's still a "constant" signal delta = 100*delta + else: + delta = delta*pyplot_rc['axes.xmargin'] else: - # Note: uncomment and figure something out for small fluctuations delta = delta*pyplot_rc['axes.xmargin'] axis.set_ylim(yMin-delta,yMax+delta) axis.autoscale(False, axis='y', tight=False) From 4ab309c90d939525d2a482ec7f0833837ae635f8 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 11 Jul 2023 15:52:26 -0600 Subject: [PATCH 104/178] Tables: using dataframe columns directly,'_' removed at init --- Makefile | 176 +++++++++--------- _tools/NewRelease.md | 22 +++ pydatview/GUISelectionPanel.py | 14 +- pydatview/Tables.py | 39 ++-- pydatview/main.py | 2 + pydatview/plugins/data_standardizeUnits.py | 26 +-- .../plugins/tests/test_standardizeUnits.py | 2 +- tests/prof_all.py | 65 ++++--- tests/test_Tables.py | 5 +- 9 files changed, 189 insertions(+), 162 deletions(-) diff --git a/Makefile b/Makefile index 3804fb4..90f839d 100644 --- a/Makefile +++ b/Makefile @@ -1,87 +1,89 @@ -# --- Detecting OS -ifeq '$(findstring ;,$(PATH))' ';' - detected_OS := Windows -else - detected_OS := $(shell uname 2>/dev/null || echo Unknown) - detected_OS := $(patsubst CYGWIN%,Cygwin,$(detected_OS)) - detected_OS := $(patsubst MSYS%,MSYS,$(detected_OS)) - detected_OS := $(patsubst MINGW%,MSYS,$(detected_OS)) -endif - -testfile= example_files/FASTIn_arf_coords.txt -all: -ifeq ($(detected_OS),Darwin) # Mac OS X - ./pythonmac pyDatView.py $(testfile) -else - python pyDatView.py $(testfile) -endif - - - - -deb: - python DEBUG.py - -install: - python setup.py install - -dep: - python -m pip install -r requirements.txt - -pull: - git pull --recurse-submodules -update:pull - - -help: - @echo "Available rules:" - @echo " all run the standalone program" - @echo " install install the python package in the system" - @echo " dep download the dependencies " - @echo " pull download the latest version " - @echo " test run the unit tests " - -test: -ifeq ($(detected_OS),Darwin) # Mac OS X - ./pythonmac -m unittest discover -v tests - ./pythonmac -m unittest discover -v pydatview/plugins/tests -else - python -m unittest discover -v tests - python -m unittest discover -v pydatview/plugins/tests -endif - -prof: - python -m cProfile -o tests/prof_all.prof tests/prof_all.py - python -m pyprof2calltree -i tests/prof_all.prof -o tests/callgrind.prof_all.prof - snakeviz tests/prof_all.prof - - -exe: - python -m nuitka --follow-imports --include-plugin-directory --include-plugin-files --show-progress --show-modules --output-dir=build-nuitka pyDatView.py - -exestd: - python -m nuitka --python-flag=no_site --assume-yes-for-downloads --standalone --follow-imports --include-plugin-directory --include-plugin-files --show-progress --show-modules --output-dir=build-nuitka-std pyDatView.py - -clean: - rm -rf __pycache__ - rm -rf *.egg-info - rm -rf *.spec - rm -rf build* - rm -rf dist - - -pyexe: - pyinstaller --onedir pyDatView.py - -version: -ifeq ($(OS),Windows_NT) - @echo "Doing nothing" -else - @sh _tools/setVersion.sh -endif - -installer: - python -m nsist installer.cfg - - - +# --- Detecting OS +ifeq '$(findstring ;,$(PATH))' ';' + detected_OS := Windows +else + detected_OS := $(shell uname 2>/dev/null || echo Unknown) + detected_OS := $(patsubst CYGWIN%,Cygwin,$(detected_OS)) + detected_OS := $(patsubst MSYS%,MSYS,$(detected_OS)) + detected_OS := $(patsubst MINGW%,MSYS,$(detected_OS)) +endif + +testfile= example_files/FASTIn_arf_coords.txt +all: +ifeq ($(detected_OS),Darwin) # Mac OS X + ./pythonmac pyDatView.py $(testfile) +else + python pyDatView.py $(testfile) +endif + + + + +deb: + python DEBUG.py + +install: + python setup.py install + +dep: + python -m pip install -r requirements.txt + +pull: + git pull --recurse-submodules +update:pull + + +help: + @echo "Available rules:" + @echo " all run the standalone program" + @echo " install install the python package in the system" + @echo " dep download the dependencies " + @echo " pull download the latest version " + @echo " test run the unit tests " + +test: +ifeq ($(detected_OS),Darwin) # Mac OS X + ./pythonmac -m unittest discover -v tests + ./pythonmac -m unittest discover -v pydatview/plugins/tests +else + python -m unittest discover -v tests + python -m unittest discover -v pydatview/plugins/tests +endif + +prof: + python -m cProfile -o tests/prof_all.prof tests/prof_all.py + python -m pyprof2calltree -i tests/prof_all.prof -o tests/callgrind.prof_all.prof + snakeviz tests/prof_all.prof + #viztracer .\tests\prof_all.py + #vizviewer.exe .\result.json + + +exe: + python -m nuitka --follow-imports --include-plugin-directory --include-plugin-files --show-progress --show-modules --output-dir=build-nuitka pyDatView.py + +exestd: + python -m nuitka --python-flag=no_site --assume-yes-for-downloads --standalone --follow-imports --include-plugin-directory --include-plugin-files --show-progress --show-modules --output-dir=build-nuitka-std pyDatView.py + +clean: + rm -rf __pycache__ + rm -rf *.egg-info + rm -rf *.spec + rm -rf build* + rm -rf dist + + +pyexe: + pyinstaller --onedir pyDatView.py + +version: +ifeq ($(OS),Windows_NT) + @echo "Doing nothing" +else + @sh _tools/setVersion.sh +endif + +installer: + python -m nsist installer.cfg + + + diff --git a/_tools/NewRelease.md b/_tools/NewRelease.md index 13a10e5..a8256d0 100644 --- a/_tools/NewRelease.md +++ b/_tools/NewRelease.md @@ -41,3 +41,25 @@ git tag new old git tag -d old git push origin new :old + + + +# Profiling +Dependencies: +``` +pip install snakeviz pyprof2calltree +#or +pip install viztracer +``` + +Profile: +``` +make prof +``` + +Runs the following: +``` +viztracer .\tests\prof_all.py +vizviewer.exe .\result.json +``` + diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 3b75497..3b8f813 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -1244,6 +1244,10 @@ def setColForSimTab(self,ISel): LenKeep = np.array([len(I) for I in IKeepPerTab]) LenDupl = np.array([len(I) for I in IDuplPerTab]) + + # Store columns + columnsPerTab = [ t.columns for t in tabs] + ColInfo = ['Sim. table mode '] ColInfo += [''] if self.tabList.haveSameColumns(ISel): @@ -1265,10 +1269,10 @@ def setColForSimTab(self,ISel): im=INotOrdered[0] if bFirst: ColInfo.append('{}:'.format(tabs[0].active_name)) - ColInfo.append('{:03d} {:s}'.format(im, tabs[0].columns[im])) + ColInfo.append('{:03d} {:s}'.format(im, columnsPerTab[0][im])) bFirst=False ColInfo.append('{}:'.format(t.active_name)) - ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) + ColInfo.append('{:03d} {:s}'.format(im, columnsPerTab[it][im])) ColInfo.append('----------------------------------') else: @@ -1281,7 +1285,7 @@ def setColForSimTab(self,ISel): if len(IMissPerTab[it])==0: ColInfo.append(' (None) ') for im in IMissPerTab[it]: - ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) + ColInfo.append('{:03d} {:s}'.format(im, columnsPerTab[it][im])) ColInfo.append('----------------------------------') if (np.any(np.array(LenDupl)>0)): @@ -1295,12 +1299,12 @@ def setColForSimTab(self,ISel): if len(IDuplPerTab[it])==0: ColInfo.append(' (None) ') for im in IDuplPerTab[it]: - ColInfo.append('{:03d} {:s}'.format(im, t.columns[im])) + ColInfo.append('{:03d} {:s}'.format(im, columnsPerTab[it][im])) ColInfo.append('----------------------------------') - colNames = [tabs[0].columns[i] for i in IKeepPerTab[0]] + colNames = [columnsPerTab[0][i] for i in IKeepPerTab[0]] # restore selection xSel = -1 diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 8a46821..8f24471 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -70,6 +70,7 @@ def append(self, t): # --- Main high level methods def from_dataframes(self, dataframes=[], names=[], bAdd=False): + assert(len(dataframes)==len(names)) if not bAdd: self.clean() # TODO figure it out # Returning a list of tables @@ -161,6 +162,7 @@ def _load_file_tabs(self, filename, fileformat=None, bReload=False): if len(warn)>0: return tabs, warn + # --- Creating list of tables here if dfs is None: pass elif not isinstance(dfs,dict): @@ -177,7 +179,7 @@ def _load_file_tabs(self, filename, fileformat=None, bReload=False): def haveSameColumns(self,I=None): if I is None: I=list(range(len(self._tabs))) - A=[len(self._tabs[i].columns)==len(self._tabs[I[0]].columns) for i in I ] + A=[len(self._tabs[i].data.columns)==len(self._tabs[I[0]].data.columns) for i in I ] if all(A): B=[self._tabs[i].columns_clean==self._tabs[I[0]].columns_clean for i in I] #list comparison return all(B) @@ -498,23 +500,24 @@ def __init__(self,data=None, name='',filename='', fileformat=None, dayfirst=Fals self.formulas = [] if not isinstance(data,pd.DataFrame): - # ndarray?? raise NotImplementedError('Tables that are not dataframe not implemented.') + # --- Pandas DataFrame + self.data = data + # Adding index + if data.columns[0].lower().find('index')>=0: + pass else: - # --- Pandas DataFrame - self.data = data - # Adding index - if data.columns[0].lower().find('index')>=0: - pass - else: - data.insert(0, 'Index', np.arange(self.data.shape[0])) - # --- Trying to figure out how to name this table - if name is None or len(str(name))==0: - if data.columns.name is not None: - name=data.columns.name + data.insert(0, 'Index', np.arange(self.data.shape[0])) + + # Clean columns only once + data.columns = [s.replace('_',' ') for s in self.data.columns.values.astype(str)] + + # --- Trying to figure out how to name this table + if name is None or len(str(name))==0: + if data.columns.name is not None: + name=data.columns.name self.setupName(name=str(name)) - self.convertTimeColumns(dayfirst=dayfirst) @@ -565,8 +568,8 @@ def applyMaskString(self, sMask, bAdd=True): # Apply mask on Table df = self.data # TODO Loop on {VAR} instead.. - for i,(c_in_df,c_user) in enumerate(zip(self.data.columns, self.columns)): - c_no_unit = no_unit(c_user).strip() + for i, c_in_df in enumerate(self.data.columns): + c_no_unit = no_unit(c_in_df).strip() # TODO sort out the mess with asarray (introduced to have and/or # as array won't work with date comparison # NOTE: using iloc to avoid duplicates column issue @@ -848,7 +851,7 @@ def shape(self): @property def columns(self): - return [s.replace('_',' ') for s in self.data.columns.values.astype(str)] + return self.data.columns.values #.astype(str) @columns.setter def columns(self, cols): @@ -856,7 +859,7 @@ def columns(self, cols): @property def columns_clean(self): - return [no_unit(s) for s in self.columns] + return [no_unit(s) for s in self.data.columns.values.astype(str)] @property def name(self): diff --git a/pydatview/main.py b/pydatview/main.py index df6a9ba..cd7768f 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -336,6 +336,8 @@ def load_dfs(self, dfs, names=None, bAdd=False, bPlot=True): # if not isinstance(dfs,list): dfs=[dfs] + if names is None: + names = ['tab{}'.format(i) for i in range(len(dfs))] if not isinstance(names,list): names=[names] self.tabList.from_dataframes(dataframes=dfs, names=names, bAdd=bAdd) diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py index dc64107..686e4f9 100644 --- a/pydatview/plugins/data_standardizeUnits.py +++ b/pydatview/plugins/data_standardizeUnits.py @@ -1,4 +1,3 @@ -import unittest import numpy as np from pydatview.common import splitunit from pydatview.pipeline import IrreversibleTableAction @@ -99,27 +98,6 @@ def change_units_to_SI(s, c): return s, c - - - -class TestChangeUnits(unittest.TestCase): - - def test_change_units(self): - import pandas as pd - from pydatview.Tables import Table - data = np.ones((1,3)) - data[:,0] *= 2*np.pi/60 # rad/s - data[:,1] *= 2000 # N - data[:,2] *= 10*np.pi/180 # rad - df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) - tab=Table(data=df) - changeUnits(tab, {'flavor':'WE'}) - np.testing.assert_almost_equal(tab.data.values[:,0],[1]) - np.testing.assert_almost_equal(tab.data.values[:,1],[2]) - np.testing.assert_almost_equal(tab.data.values[:,2],[10]) - self.assertEqual(tab.columns, ['om [rpm]', 'F [kN]', 'angle [deg]']) - raise Exception('>>>>>>>>>>>>') - - if __name__ == '__main__': - unittest.main() + pass + #unittest.main() diff --git a/pydatview/plugins/tests/test_standardizeUnits.py b/pydatview/plugins/tests/test_standardizeUnits.py index 599fea2..051d634 100644 --- a/pydatview/plugins/tests/test_standardizeUnits.py +++ b/pydatview/plugins/tests/test_standardizeUnits.py @@ -17,7 +17,7 @@ def test_change_units(self): np.testing.assert_almost_equal(tab.data.values[:,1],[1]) np.testing.assert_almost_equal(tab.data.values[:,2],[2]) np.testing.assert_almost_equal(tab.data.values[:,3],[10]) - self.assertEqual(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) + np.testing.assert_equal(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) if __name__ == '__main__': diff --git a/tests/prof_all.py b/tests/prof_all.py index 4503e95..75d2210 100644 --- a/tests/prof_all.py +++ b/tests/prof_all.py @@ -1,15 +1,16 @@ +import os +import time +import pydatview +import pandas as pd +import numpy as np +import wx +from pydatview.perfmon import PerfMon, Timer +from pydatview.main import MainFrame +import pydatview.io as weio +import gc +scriptDir = os.path.dirname(__file__) def test_heavy(): - import time - import sys - import pydatview - import pandas as pd - import numpy as np - import wx - from pydatview.perfmon import PerfMon, Timer - from pydatview.pydatview import MainFrame - from pydatview.GUISelectionPanel import ellude_common - import gc dt = 0 with Timer('Test'): # --- Test df @@ -34,18 +35,18 @@ def test_heavy(): with PerfMon('Redraw 1'): frame.selPanel.colPanel1.lbColumns.SetSelection(-1) frame.selPanel.colPanel1.lbColumns.SetSelection(2) - frame.plotPanel.redraw() + frame.redraw() time.sleep(dt) with PerfMon('Redraw 1 (igen)'): frame.selPanel.colPanel1.lbColumns.SetSelection(-1) frame.selPanel.colPanel1.lbColumns.SetSelection(2) - frame.plotPanel.redraw() + frame.redraw() time.sleep(dt) with PerfMon('FFT 1'): frame.plotPanel.pltTypePanel.cbFFT.SetValue(True) #frame.plotPanel.cbLogX.SetValue(True) #frame.plotPanel.cbLogY.SetValue(True) - frame.plotPanel.redraw() + frame.redraw() frame.plotPanel.pltTypePanel.cbFFT.SetValue(False) time.sleep(dt) with PerfMon('Plot 3'): @@ -54,26 +55,40 @@ def test_heavy(): frame.onColSelectionChange() time.sleep(dt) with PerfMon('Redraw 3'): - frame.plotPanel.redraw() + frame.redraw() time.sleep(dt) with PerfMon('FFT 3'): frame.plotPanel.pltTypePanel.cbFFT.SetValue(True) - frame.plotPanel.redraw() + frame.redraw() frame.plotPanel.pltTypePanel.cbFFT.SetValue(False) +def test_debug(show=False): + dt = 0 + with Timer('Test'): + with Timer('read'): + df1 =weio.read(os.path.join(scriptDir,'../ad_driver_m50.1.outb')).toDataFrame() + df2 =weio.read(os.path.join(scriptDir,'../ad_driver_p50.csv')).toDataFrame() + + time.sleep(dt) + with PerfMon('Plot 1'): + app = wx.App(False) + frame = MainFrame() + frame.load_dfs([df1,df2]) + frame.selPanel.tabPanel.lbTab.SetSelection(0) + frame.selPanel.tabPanel.lbTab.SetSelection(1) + frame.onTabSelectionChange() + #frame.redraw() + if show: + app.MainLoop() + + +def test_files(filenames): + pydatview.test(filenames=filenames) + if __name__ == '__main__': - import sys - import os - root_dir=os.getcwd() - script_dir=os.path.dirname(os.path.realpath(__file__)) - sys.path.append(root_dir) -# print(root_dir) - #filenames=['../_TODO/DLC120_ws13_yeNEG_s2_r3_PIT.SFunc.outb','../_TODO/DLC120_ws13_ye000_s1_r1.SFunc.outb'] -# filenames=['../example_files/CSVComma.csv'] # filenames =[os.path.join(script_dir,f) for f in filenames] - - #pydatview.test(filenames=filenames) test_heavy() + #test_debug(False) diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 91fc4d3..b0c7cad 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -112,12 +112,13 @@ def test_change_units(self): np.testing.assert_almost_equal(tab.data.values[:,1],[1]) np.testing.assert_almost_equal(tab.data.values[:,2],[2]) np.testing.assert_almost_equal(tab.data.values[:,3],[10]) - self.assertEqual(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) + print('>>>> tab.columns', tab.columns) + np.testing.assert_equal(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) def test_renameColumns(self): tab = Table.createDummy(n=3, columns=['RtFldCp [-]','B1FldFx [N]', 'angle [rad]']) tab.renameColumns(strReplDict={'Aero':'Fld'}) - self.assertEqual(tab.columns, ['Index','RtAeroCp [-]', 'B1AeroFx [N]', 'angle [rad]']) + np.testing.assert_equal(tab.columns, ['Index','RtAeroCp [-]', 'B1AeroFx [N]', 'angle [rad]']) if __name__ == '__main__': # TestTable.setUpClass() From 7f6e4cc64923664f9d57c77eaedf70ebde386789 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 13 Jul 2023 04:48:58 -0600 Subject: [PATCH 105/178] Tables: reintroducing underscore in column names --- pydatview/Tables.py | 1853 ++++++++--------- pydatview/plotdata.py | 8 +- .../plugins/tests/test_standardizeUnits.py | 2 +- pydatview/plugins/tool_radialavg.py | 2 +- tests/test_Tables.py | 3 +- 5 files changed, 932 insertions(+), 936 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 8f24471..aeac019 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -1,928 +1,925 @@ -import numpy as np -import os.path -from dateutil import parser -import pandas as pd -try: - from .common import no_unit, ellude_common, getDt -except: - from common import no_unit, ellude_common, getDt -import pydatview.io as weio # File Formats and File Readers - - - -# --------------------------------------------------------------------------------} -# --- TabList -# --------------------------------------------------------------------------------{ -class TableList(object): # todo inherit list - - def __init__(self, tabs=None, options=None): - if tabs is None: - tabs =[] - self._tabs = tabs - - self.options = self.defaultOptions() if options is None else options - - # --- Options - def saveOptions(self, optionts): - options['naming'] = self.options['naming'] - options['dayfirst'] = self.options['dayfirst'] - - @staticmethod - def defaultOptions(): - options={} - options['naming'] = 'Ellude' - options['dayfirst'] = False - return options - - # --- behaves like a list... - #def __delitem__(self, key): - # self.__delattr__(key) - - def __getitem__(self, key): - return self._tabs[key] - - def __setitem__(self, key, value): - raise Exception('Setting not allowed') - self._tabs[key] = value - - def __iter__(self): - self.__n = 0 - return self - - def __next__(self): - if self.__n < len(self._tabs): - self.__n += 1 - return self._tabs[self.__n-1] - else: - raise StopIteration - - def __len__(self): - return len(self._tabs) - - def len(self): - return len(self._tabs) - - def append(self, t): - if isinstance(t,list): - self._tabs += t - else: - self._tabs += [t] - - # --- Main high level methods - def from_dataframes(self, dataframes=[], names=[], bAdd=False): - assert(len(dataframes)==len(names)) - if not bAdd: - self.clean() # TODO figure it out - # Returning a list of tables - for df,name in zip(dataframes, names): - if df is not None: - self.append(Table(data=df, name=name, dayfirst=self.options['dayfirst'])) - - def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bReload=False, statusFunction=None): - """ load multiple files into table list""" - if not bAdd: - self.clean() # TODO figure it out - - if fileformats is None: - fileformats=[None]*len(filenames) - assert type(fileformats) ==list, 'fileformats must be a list' - - # Loop through files, appending tables within files - warnList=[] - newTabs=[] - for i, (f,ff) in enumerate(zip(filenames, fileformats)): - if statusFunction is not None: - statusFunction(i) - if f in self.unique_filenames: - warnList.append('Warn: Cannot add a file already opened ' + f) - elif len(f)==0: - pass - # warn+= 'Warn: an empty filename was skipped' +'\n' - else: - tabs, warnloc = self._load_file_tabs(f,fileformat=ff, bReload=bReload) - if len(warnloc)>0: - warnList.append(warnloc) - self.append(tabs) - newTabs +=tabs - - return newTabs, warnList - - def _load_file_tabs(self, filename, fileformat=None, bReload=False): - """ load a single file, returns a list (often of size one) of tables """ - # Returning a list of tables - tabs=[] - warn='' - if not os.path.isfile(filename): - warn = 'Error: File not found: `'+filename+'`\n' - return tabs, warn - - fileformatAllowedToFailOnReload = (fileformat is not None) and bReload - if fileformatAllowedToFailOnReload: - try: - F = fileformat.constructor(filename=filename) - dfs = F.toDataFrame() - except: - warnLoc = 'Failed to read file:\n\n {}\n\nwith fileformat: {}\n\nIf you see this message, the reader tried again and succeeded with "auto"-fileformat.\n\n'.format(filename, fileformat.name) - tabs,warn = self._load_file_tabs(filename, fileformat=None, bReload=False) - return tabs, warnLoc+warn - - else: - - try: - #F = weio.read(filename, fileformat = fileformat) - # --- Expanded version of weio.read - F = None - if fileformat is None: - fileformat, F = weio.detectFormat(filename) - # Reading the file with the appropriate class if necessary - if not isinstance(F, fileformat.constructor): - F=fileformat.constructor(filename=filename) - dfs = F.toDataFrame() - except weio.FileNotFoundError as e: - warn = 'Error: A file was not found!\n\n While opening:\n\n {}\n\n the following file was not found:\n\n {}\n'.format(filename, e.filename) - except IOError: - warn = 'Error: IO Error thrown while opening file: '+filename+'\n' - except MemoryError: - warn='Error: Insufficient memory!\n\nFile: '+filename+'\n\nTry closing and reopening the program, or use a 64 bit version of this program (i.e. of python).\n' - except weio.EmptyFileError: - warn='Error: File empty!\n\nFile is empty: '+filename+'\n\nOpen a different file.\n' - except weio.FormatNotDetectedError: - warn='Error: File format not detected!\n\nFile: '+filename+'\n\nUse an explicit file-format from the list\n' - except weio.WrongFormatError as e: - warn='Error: Wrong file format!\n\nFile: '+filename+'\n\n' \ - 'The file parser for the selected format failed to open the file.\n\n'+ \ - 'The reported error was:\n'+e.args[0]+'\n\n' + \ - 'Double-check your file format and report this error if you think it''s a bug.\n' - except weio.BrokenFormatError as e: - warn = 'Error: Inconsistency in the file format!\n\nFile: '+filename+'\n\n' \ - 'The reported error was:\n\n'+e.args[0]+'\n\n' + \ - 'Double-check your file format and report this error if you think it''s a bug.' - except: - raise - if len(warn)>0: - return tabs, warn - - # --- Creating list of tables here - if dfs is None: - pass - elif not isinstance(dfs,dict): - if len(dfs)>0: - tabs=[Table(data=dfs, filename=filename, fileformat=fileformat, dayfirst=self.options['dayfirst'])] - else: - for k in list(dfs.keys()): - if len(dfs[k])>0: - tabs.append(Table(data=dfs[k], name=str(k), filename=filename, fileformat=fileformat, dayfirst=self.options['dayfirst'])) - if len(tabs)<=0: - warn='Warn: No dataframe found in file: '+filename+'\n' - return tabs, warn - - def haveSameColumns(self,I=None): - if I is None: - I=list(range(len(self._tabs))) - A=[len(self._tabs[i].data.columns)==len(self._tabs[I[0]].data.columns) for i in I ] - if all(A): - B=[self._tabs[i].columns_clean==self._tabs[I[0]].columns_clean for i in I] #list comparison - return all(B) - else: - return False - - def renameTable(self, iTab, newName): - oldName = self._tabs[iTab].name - if newName in [t.name for t in self._tabs]: - raise Exception('Error: This table already exist, choose a different name.') - # Renaming table - self._tabs[iTab].rename(newName) - return oldName - - def sort(self, method='byName'): - if method=='byName': - tabnames_display=self.getDisplayTabNames() - self._tabs = [t for _,t in sorted(zip(tabnames_display,self._tabs))] - else: - raise Exception('Sorting method unknown: `{}`'.format(method)) - - def mergeTabs(self, I=None, ICommonColPerTab=None, extrap='nan'): - """ - Merge table together. - TODO: add options for how interpolation/merging is done - - I: index of tables to merge, if None: all tables are merged - """ - from pydatview.tools.signal_analysis import interpDF - #from pydatview.tools.signal_analysis import applySampler - #df_new, name_new = t.applyResampling(iCol,sampDict, bAdd=bAdd) - if I is None: - I = range(len(self._tabs)) - - dfs = [self._tabs[i].data for i in I] - if ICommonColPerTab is None: - # --- Option 0 - Index concatenation - print('Using dataframe index concatenation...') - df = pd.concat(dfs, axis=1) - # Remove duplicated columns - #df = df.loc[:,~df.columns.duplicated()].copy() - else: - try: - # --- Option 1 - We combine all the x from the common column together - # NOTE: We use unique and sort, which will distrupt the user data (e.g. Airfoil Coords) - # The user should then use other methods (when implemented) - x_new=[] - cols = [] - for it, icol in zip(I, ICommonColPerTab): - xtab = self._tabs[it].data.iloc[:, icol].values - cols.append(self._tabs[it].data.columns[icol]) - x_new = np.concatenate( (x_new, xtab) ) - x_new = np.unique(np.sort(x_new)) - # Create interpolated dataframes based on x_new - dfs_new = [] - for i, (col, df_old) in enumerate(zip(cols, dfs)): - df = interpDF(x_new, col, df_old, extrap=extrap) - if 'Index' in df.columns: - df = df.drop(['Index'], axis=1) - if i>0: - df = df.drop([col], axis=1) - dfs_new.append(df) - df = pd.concat(dfs_new, axis=1) - # Reindex at the end - df.insert(0, 'Index', np.arange(df.shape[0])) - except: - # --- Option 0 - Index concatenation - print('Using dataframe index concatenation...') - df = pd.concat(dfs, axis=1) - newName = self._tabs[I[0]].name+'_merged' - self.append(Table(data=df, name=newName)) - return newName, df - - def vstack(self, I=None, commonOnly=False): - """ - Vertical stacking of tables - - I: index of tables to stack, if None: all tables are stacked - commonOnly: if True, keep only the common columns. - Otherwise, NaN will be introduced for missing columns - """ - if I is None: - I = range(len(self._tabs)) - dfs = [self._tabs[i].data for i in I] - - if commonOnly: - # --- Concatenate all but keep only common columns - df = pd.concat(dfs, join='inner', ignore_index=True) - else: - # --- Concatenate all, not worrying about common columns - df = pd.concat(dfs, ignore_index=True) - # Set unique index - if 'Index' in df.columns: - df = df.drop(['Index'], axis=1) - df.insert(0, 'Index', np.arange(df.shape[0])) - # Add to table list - newName = self._tabs[I[0]].name+'_concat' - self.append(Table(data=df, name=newName)) - return newName, df - - - - def deleteTabs(self, I): - self._tabs = [t for i,t in enumerate(self._tabs) if i not in I] - - def setActiveNames(self,names): - for t,tn in zip(self._tabs,names): - t.active_name=tn - - def getDisplayTabNames(self): - if self.options['naming']=='Ellude': - # Temporary hack, using last names if all last names are unique - names = [t.raw_name for t in self._tabs] - last_names=[n.split('|')[-1] for n in names] - if len(np.unique(last_names)) == len(names): - return ellude_common(last_names) - else: - return ellude_common(names) - elif self.options['naming']=='FileNames': - return [os.path.splitext(os.path.basename(t.filename))[0] for t in self._tabs] - else: - raise Exception('Table naming unknown: {}'.format(self.options['naming'])) - - # --- Properties - @property - def tabNames(self): - return [t.name for t in self._tabs] - - @property - def filenames(self): - return [t.filename for t in self._tabs] - - @property - def fileformats(self): - return [t.fileformat for t in self._tabs] - - @property - def unique_filenames(self): - return list(set([t.filename for t in self._tabs])) - - @property - def filenames_and_formats(self): - """ return unique list of filenames with associated fileformats """ - filenames = [] - fileformats = [] - for t in self._tabs: - if t.filename not in filenames: - filenames.append(t.filename) - fileformats.append(t.fileformat) - return filenames, fileformats - - @property - def naming(self): - return self.options['naming'] - - @naming.setter - def naming(self, naming): - if naming not in ['FileNames', 'Ellude']: - raise NotImplementedError('Naming',naming) - self.options['naming']=naming - - - def clean(self): - del self._tabs - self._tabs=[] - - def __repr__(self): - return '\n'.join([t.__repr__() for t in self._tabs]) - - # --- Mask related - @property - def maskStrings(self): - return [t.maskString for t in self._tabs] - - @property - def commonMaskString(self): - maskStrings=set(self.maskStrings) - if len(maskStrings) == 1: - return next(iter(maskStrings)) - else: - return '' - - def clearCommonMask(self): - for t in self._tabs: - t.clearMask() - - def applyCommonMaskString(self,maskString,bAdd=True): - # Apply mask on tablist - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - df_new, name_new = t.applyMaskString(maskString, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) - except: - errors.append('Mask failed for table: '+t.nickname) # TODO - - return dfs_new, names_new, errors - - # --- Resampling TODO MOVE THIS OUT OF HERE OR UNIFY - def applyResampling(self,iCol,sampDict,bAdd=True): - """ Apply resampling on table list - TODO Make this part of the action - """ - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - df_new, name_new = t.applyResampling(iCol, sampDict, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) - except: - errors.append('Resampling failed for table: '+t.nickname) # TODO - return dfs_new, names_new, errors - - # --- Filtering TODO MOVE THIS OUT OF HERE OR UNIFY - def applyFiltering(self,iCol,options,bAdd=True): - """ Apply filtering on table list - TODO Make this part of the action - """ - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - df_new, name_new = t.applyFiltering(iCol, options, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) - except: - errors.append('Filtering failed for table: '+t.nickname) # TODO - return dfs_new, names_new, errors - - # --- Radial average related - def radialAvg(self,avgMethod,avgParam): - """ Apply radial average on table list - TODO Make this part of the action - """ - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - dfs, names = t.radialAvg(avgMethod,avgParam) - for df,n in zip(dfs,names): - if df is not None: - dfs_new.append(df) - names_new.append(n) - except: - errors.append('Radial averaging failed for table: '+t.nickname) # TODO - return dfs_new, names_new, errors - - # --- Element--related functions - def get(self,i): - print('.>> GET') - return self._tabs[i] - - - - @staticmethod - def createDummy(nTabs=3, n=30, addLabel=True): - tabs=[] - label='' - for iTab in range(nTabs): - if addLabel: - label='_'+str(iTab) - tabs.append( Table.createDummy(n=n, label=label)) - tablist = TableList(tabs) - return tablist - - -# --------------------------------------------------------------------------------} -# --- Table -# --------------------------------------------------------------------------------{ -# -class Table(object): - """ - Main attributes: - - data - - columns # TODO get rid of me - - name - - raw_name # Should be unique and can be used for identification - - ID # Should be unique and can be used for identification - - active_name - - filename - - fileformat - - fileformat_name - - nCols x nRows - - mask - - maskString - - formulas - """ - # TODO sort out the naming - # Main naming concepts: - # name : - # active_name : - # raw_name : - # filename : - def __init__(self,data=None, name='',filename='', fileformat=None, dayfirst=False): - # Default init - self.maskString='' - self.mask=None - - self.filename = filename - self.fileformat = fileformat - if fileformat is not None: - self.fileformat_name = fileformat.name - else: - self.fileformat_name = '' - self.formulas = [] - - if not isinstance(data,pd.DataFrame): - raise NotImplementedError('Tables that are not dataframe not implemented.') - # --- Pandas DataFrame - self.data = data - # Adding index - if data.columns[0].lower().find('index')>=0: - pass - else: - data.insert(0, 'Index', np.arange(self.data.shape[0])) - - # Clean columns only once - data.columns = [s.replace('_',' ') for s in self.data.columns.values.astype(str)] - - # --- Trying to figure out how to name this table - if name is None or len(str(name))==0: - if data.columns.name is not None: - name=data.columns.name - - self.setupName(name=str(name)) - self.convertTimeColumns(dayfirst=dayfirst) - - - def setupName(self,name=''): - # Creates a "codename": path | basename | name | ext - splits=[] - ext='' - if len(self.filename)>0: - base_dir = os.path.dirname(self.filename) - if len(base_dir)==0: - base_dir= os.getcwd() - self.filename=os.path.join(base_dir,self.filename) - splits.append(base_dir.replace('/','|').replace('\\','|')) - basename,ext=os.path.splitext(os.path.basename(self.filename)) - if len(basename)>0: - splits.append(basename) - if name is not None and len(name)>0: - splits.append(name) - #if len(ext)>0: - # splits.append(ext) - self.extension=ext - name='|'.join(splits) - if len(name)==0: - name='default' - self.name=name - self.active_name=self.name - - def __repr__(self): - s='Table object:\n' - s+=' - name: {}\n'.format(self.name) - s+=' - raw_name : {}\n'.format(self.raw_name) - s+=' - active_name: {}\n'.format(self.raw_name) - s+=' - filename : {}\n'.format(self.filename) - s+=' - fileformat : {}\n'.format(self.fileformat) - s+=' - fileformat_name : {}\n'.format(self.fileformat_name) - s+=' - columns : {}\n'.format(self.columns) - s+=' - nCols x nRows: {}x{}\n'.format(self.nCols, self.nRows) - s+=' - maskString: {}\n'.format(self.maskString) - return s - - - # --- Mask - def clearMask(self): - self.maskString='' - self.mask=None - - def applyMaskString(self, sMask, bAdd=True): - # Apply mask on Table - df = self.data - # TODO Loop on {VAR} instead.. - for i, c_in_df in enumerate(self.data.columns): - c_no_unit = no_unit(c_in_df).strip() - # TODO sort out the mess with asarray (introduced to have and/or - # as array won't work with date comparison - # NOTE: using iloc to avoid duplicates column issue - if isinstance(df.iloc[0,i], pd._libs.tslibs.timestamps.Timestamp): - sMask=sMask.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') - else: - sMask=sMask.replace('{'+c_no_unit+'}','np.asarray(df[\''+c_in_df+'\'])') - df_new = None - name_new = None - if len(sMask.strip())>0 and sMask.strip().lower()!='no mask': - try: - mask = np.asarray(eval(sMask)) - if bAdd: - df_new = df[mask] - name_new=self.raw_name+'_masked' - else: - self.mask=mask - self.maskString=sMask - except: - # TODO come up with better error messages - raise Exception('Error: The mask failed to evaluate for table: '+self.nickname) - if sum(mask)==0: - self.clearMask() - raise Exception('Error: The mask returned no value for table: '+self.nickname) - return df_new, name_new - - # --- Important manipulation TODO MOVE THIS OUT OF HERE OR UNIFY - def applyResampling(self, iCol, sampDict, bAdd=True): - # Resample Table - from pydatview.tools.signal_analysis import applySamplerDF - colName=self.data.columns[iCol] - df_new =applySamplerDF(self.data, colName, sampDict=sampDict) - # Reindex afterwards - if df_new.columns[0]=='Index': - df_new['Index'] = np.arange(0,len(df_new)) - if bAdd: - name_new=self.raw_name+'_resampled' - else: - name_new=None - self.data=df_new - return df_new, name_new - - def applyFiltering(self, iCol, options, bAdd=True): - from pydatview.tools.signal_analysis import applyFilterDF - colName=self.data.columns[iCol] - df_new =applyFilterDF(self.data, colName, options) - # Reindex afterwards - if df_new.columns[0]=='Index': - df_new['Index'] = np.arange(0,len(df_new)) - if bAdd: - name_new=self.raw_name+'_filtered' - else: - name_new=None - self.data=df_new - return df_new, name_new - - - def radialAvg(self,avgMethod, avgParam): - # TODO make this a pluggin - import pydatview.fast.postpro as fastlib - import pydatview.fast.fastfarm as fastfarm - df = self.data - base,out_ext = os.path.splitext(self.filename) - - # --- Detect if it's a FAST Farm file - sCols = ''.join(df.columns) - if sCols.find('WkDf')>1 or sCols.find('CtT')>0: - # --- FAST FARM files - Files=[base+ext for ext in ['.fstf','.FSTF','.Fstf','.fmas','.FMAS','.Fmas'] if os.path.exists(base+ext)] - if len(Files)==0: - fst_in=None - #raise Exception('Error: No .fstf file found with name: '+base+'.fstf') - else: - fst_in=Files[0] - - dfRad,_,dfDiam = fastfarm.spanwisePostProFF(fst_in,avgMethod=avgMethod,avgParam=avgParam,D=1,df=df) - dfs_new = [dfRad,dfDiam] - names_new=[self.raw_name+'_rad',self.raw_name+'_diam'] - else: - # --- FAST files - - # HACK for AD file to find the right .fst file - iDotAD=base.lower().find('.ad') - if iDotAD>1: - base=base[:iDotAD] - # - Files=[base+ext for ext in ['.fst','.FST','.Fst','.dvr','.Dvr','.DVR'] if os.path.exists(base+ext)] - if len(Files)==0: - fst_in=None - #raise Exception('Error: No .fst file found with name: '+base+'.fst') - else: - fst_in=Files[0] - - - out= fastlib.spanwisePostPro(fst_in, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df = self.data) - dfRadED=out['ED_bld']; dfRadAD = out['AD']; dfRadBD = out['BD'] - - dfs_new = [dfRadAD, dfRadED, dfRadBD] - names_new=[self.raw_name+'_AD', self.raw_name+'_ED', self.raw_name+'_BD'] - return dfs_new, names_new - - def changeUnits(self, data=None): - """ Change units of the table """ - if data is None: - data={'flavor':'WE'} - # NOTE: moved to a plugin, but interface kept - from pydatview.plugins.data_standardizeUnits import changeUnits - changeUnits(self, data=data) - - def convertTimeColumns(self, dayfirst=False): - if len(self.data)>0: - for i,c in enumerate(self.data.columns.values): - y = self.data.iloc[:,i] - if y.dtype == object: - if isinstance(y.values[0], str): - # tring to convert to date - try: - vals = parser.parse(y.values[0]) - isDate=True - except: - if y.values[0]=='NaT': - isDate=True - else: - isDate=False - if isDate: - try: - self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst).to_pydatetime() - print('Column {} converted to datetime, dayfirst: {}'.format(c, dayfirst)) - except: - # Happens if values are e.g. "Monday, Tuesday" - print('Conversion to datetime failed, column {} inferred as string'.format(c)) - else: - print('Column {} inferred as string'.format(c)) - elif isinstance(y.values[0], (float, int)): - try: - self.data[c]=self.data[c].astype(float) - print('Column {} converted to float (likely nan)'.format(c)) - except: - self.data[c]=self.data[c].astype(str) - print('Column {} inferred and converted to string'.format(c)) - else : - print('>> Unknown type:',type(y.values[0])) - #print(self.data.dtypes) - - - # --- Column manipulations - def renameColumn(self,iCol,newName): - self.data.columns.values[iCol]=newName - - def renameColumns(self, strReplDict=None): - """ Rename all the columns of given table - - strReplDict: a string replacement dictionary of the form: {'new':'old'} - """ - if strReplDict is not None: - cols = self.data.columns - newcols = [] - for c in cols: - for new,old in strReplDict.items(): - c = c.replace(old,new) - newcols.append(c) - self.data.columns = newcols - - def deleteColumns(self, ICol): - """ Delete columns by index, not column names which can have duplicates""" - IKeep =[i for i in np.arange(self.data.shape[1]) if i not in ICol] - self.data = self.data.iloc[:, IKeep] # Drop won't work for duplicates - for i in sorted(ICol, reverse=True): - for f in self.formulas: - if f['pos'] == (i + 1): - self.formulas.remove(f) - break - for f in self.formulas: - if f['pos'] > (i + 1): - f['pos'] = f['pos'] - 1 - - def rename(self,new_name): - self.name='>'+new_name - - def addColumn(self, sNewName, NewCol, i=-1, sFormula=''): - print('>>> adding Column') - if i<0: - i=self.data.shape[1] - elif i>self.data.shape[1]+1: - i=self.data.shape[1] - self.data.insert(int(i+1),sNewName,NewCol) - for f in self.formulas: - if f['pos'] > i: - f['pos'] = f['pos'] + 1 - self.formulas.append({'pos': i+1, 'formula': sFormula, 'name': sNewName}) - - def setColumn(self,sNewName,NewCol,i,sFormula=''): - if i<1: - raise ValueError('Cannot set column at position ' + str(i)) - self.data = self.data.drop(columns=self.data.columns[i]) - self.data.insert(int(i),sNewName,NewCol) - for f in self.formulas: - if f['pos'] == i: - f['name'] = sNewName - f['formula'] = sFormula - - def getColumn(self, i): - """ Return column of data - If a mask exist, the mask is applied - - TODO TODO TODO get rid of this! - """ - if self.mask is not None: - c = self.data.iloc[self.mask, i] - x = self.data.iloc[self.mask, i].values - else: - c = self.data.iloc[:, i] - x = self.data.iloc[:, i].values - - isString = c.dtype == object and isinstance(c.values[0], str) - if isString: - x=x.astype(str) - isDate = np.issubdtype(c.dtype, np.datetime64) - if isDate: - dt=getDt(x) - if dt>1: - x=x.astype('datetime64[s]') - else: - x=x.astype('datetime64') - return x,isString,isDate,c - - - def evalFormula(self,sFormula): - df = self.data - for i,c in enumerate(self.columns): - c_no_unit = no_unit(c).strip() - c_in_df = df.columns[i] - sFormula=sFormula.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') - try: - NewCol=eval(sFormula) - return NewCol - except: - return None - - def addColumnByFormula(self,sNewName,sFormula,i=-1): - NewCol=self.evalFormula(sFormula) - if NewCol is None: - return False - else: - self.addColumn(sNewName,NewCol,i,sFormula) - return True - - def setColumnByFormula(self,sNewName,sFormula,i=-1): - NewCol=self.evalFormula(sFormula) - if NewCol is None: - return False - else: - self.setColumn(sNewName,NewCol,i,sFormula) - return True - - - def export(self, path): - if isinstance(self.data, pd.DataFrame): - df = self.data - if 'Index' in df.columns.values: - df = df.drop(['Index'], axis=1) - df.to_csv(path, sep=',', index=False) - else: - raise NotImplementedError('Export of data that is not a dataframe') - - - - # --- Properties - @property - def basename(self): - return os.path.splitext(os.path.basename(self.filename))[0] - - @property - def shapestring(self): - return '{}x{}'.format(self.nCols, self.nRows) - - @property - def shape(self): - return (self.nRows, self.nCols) - - @property - def columns(self): - return self.data.columns.values #.astype(str) - - @columns.setter - def columns(self, cols): - raise Exception('Columns is read only') - - @property - def columns_clean(self): - return [no_unit(s) for s in self.data.columns.values.astype(str)] - - @property - def name(self): - if len(self.__name)<=0: - return '' - if self.__name[0]=='>': - return self.__name[1:] - else: - return self.__name - - @property - def raw_name(self): - return self.__name - - @name.setter - def name(self,new_name): - self.__name=new_name - - @property - def nickname(self): - sp = self.name.split('|') - return sp[-1] - - @property - def nCols(self): - return len(self.columns) - - @property - def nRows(self): - return len(self.data.iloc[:,0]) # TODO if not panda - - @staticmethod - def createDummy(n, label='', columns=None, nCols=None): - """ create a dummy table of length n - If columns or nCols are provided, they are used for the - """ - # - if nCols is None and columns is None: - t = np.linspace(0, 4*np.pi, n) - x = np.sin(t)+10 - alpha_d = np.linspace(0, 360, n) - P = np.random.normal(0,100,n)+5000 - RPM = np.random.normal(-0.2,0.2,n) + 12. - d={'Time_[s]':t, - 'x{}_[m]'.format(label): x, - 'alpha{}_[deg]'.format(label):alpha_d, - 'P{}_[W]'.format(label):P, - 'RotSpeed{}_[rpm]'.format(label):RPM} - else: - units=['m','m/s','kn','rad','w','deg'] - if columns is None: - columns = ['C{}{}_[{}]'.format(i,label, units[np.mod(i, len(units))] ) for i in range(nCols)] - nCols=len(columns) - - d = np.zeros((n, nCols)) - for i in range(nCols): - d[:,i] = np.random.normal(0, 1, n) + i - df = pd.DataFrame(data=d, columns= columns) - return Table(data=df, name='Dummy '+label) - - -if __name__ == '__main__': - import pandas as pd; - from Tables import Table - import numpy as np - +import numpy as np +import os.path +from dateutil import parser +import pandas as pd +try: + from .common import no_unit, ellude_common, getDt +except: + from common import no_unit, ellude_common, getDt +import pydatview.io as weio # File Formats and File Readers + +# --------------------------------------------------------------------------------} +# --- TabList +# --------------------------------------------------------------------------------{ +class TableList(object): # todo inherit list + + def __init__(self, tabs=None, options=None): + if tabs is None: + tabs =[] + self._tabs = tabs + + self.options = self.defaultOptions() if options is None else options + + # --- Options + def saveOptions(self, optionts): + options['naming'] = self.options['naming'] + options['dayfirst'] = self.options['dayfirst'] + + @staticmethod + def defaultOptions(): + options={} + options['naming'] = 'Ellude' + options['dayfirst'] = False + return options + + # --- behaves like a list... + #def __delitem__(self, key): + # self.__delattr__(key) + + def __getitem__(self, key): + return self._tabs[key] + + def __setitem__(self, key, value): + raise Exception('Setting not allowed') + self._tabs[key] = value + + def __iter__(self): + self.__n = 0 + return self + + def __next__(self): + if self.__n < len(self._tabs): + self.__n += 1 + return self._tabs[self.__n-1] + else: + raise StopIteration + + def __len__(self): + return len(self._tabs) + + def len(self): + return len(self._tabs) + + def append(self, t): + if isinstance(t,list): + self._tabs += t + else: + self._tabs += [t] + + # --- Main high level methods + def from_dataframes(self, dataframes=[], names=[], bAdd=False): + assert(len(dataframes)==len(names)) + if not bAdd: + self.clean() # TODO figure it out + # Returning a list of tables + for df,name in zip(dataframes, names): + if df is not None: + self.append(Table(data=df, name=name, dayfirst=self.options['dayfirst'])) + + def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bReload=False, statusFunction=None): + """ load multiple files into table list""" + if not bAdd: + self.clean() # TODO figure it out + + if fileformats is None: + fileformats=[None]*len(filenames) + assert type(fileformats) ==list, 'fileformats must be a list' + + # Loop through files, appending tables within files + warnList=[] + newTabs=[] + for i, (f,ff) in enumerate(zip(filenames, fileformats)): + if statusFunction is not None: + statusFunction(i) + if f in self.unique_filenames: + warnList.append('Warn: Cannot add a file already opened ' + f) + elif len(f)==0: + pass + # warn+= 'Warn: an empty filename was skipped' +'\n' + else: + tabs, warnloc = self._load_file_tabs(f,fileformat=ff, bReload=bReload) + if len(warnloc)>0: + warnList.append(warnloc) + self.append(tabs) + newTabs +=tabs + + return newTabs, warnList + + def _load_file_tabs(self, filename, fileformat=None, bReload=False): + """ load a single file, returns a list (often of size one) of tables """ + # Returning a list of tables + tabs=[] + warn='' + if not os.path.isfile(filename): + warn = 'Error: File not found: `'+filename+'`\n' + return tabs, warn + + fileformatAllowedToFailOnReload = (fileformat is not None) and bReload + if fileformatAllowedToFailOnReload: + try: + F = fileformat.constructor(filename=filename) + dfs = F.toDataFrame() + except: + warnLoc = 'Failed to read file:\n\n {}\n\nwith fileformat: {}\n\nIf you see this message, the reader tried again and succeeded with "auto"-fileformat.\n\n'.format(filename, fileformat.name) + tabs,warn = self._load_file_tabs(filename, fileformat=None, bReload=False) + return tabs, warnLoc+warn + + else: + + try: + #F = weio.read(filename, fileformat = fileformat) + # --- Expanded version of weio.read + F = None + if fileformat is None: + fileformat, F = weio.detectFormat(filename) + # Reading the file with the appropriate class if necessary + if not isinstance(F, fileformat.constructor): + F=fileformat.constructor(filename=filename) + dfs = F.toDataFrame() + except weio.FileNotFoundError as e: + warn = 'Error: A file was not found!\n\n While opening:\n\n {}\n\n the following file was not found:\n\n {}\n'.format(filename, e.filename) + except IOError: + warn = 'Error: IO Error thrown while opening file: '+filename+'\n' + except MemoryError: + warn='Error: Insufficient memory!\n\nFile: '+filename+'\n\nTry closing and reopening the program, or use a 64 bit version of this program (i.e. of python).\n' + except weio.EmptyFileError: + warn='Error: File empty!\n\nFile is empty: '+filename+'\n\nOpen a different file.\n' + except weio.FormatNotDetectedError: + warn='Error: File format not detected!\n\nFile: '+filename+'\n\nUse an explicit file-format from the list\n' + except weio.WrongFormatError as e: + warn='Error: Wrong file format!\n\nFile: '+filename+'\n\n' \ + 'The file parser for the selected format failed to open the file.\n\n'+ \ + 'The reported error was:\n'+e.args[0]+'\n\n' + \ + 'Double-check your file format and report this error if you think it''s a bug.\n' + except weio.BrokenFormatError as e: + warn = 'Error: Inconsistency in the file format!\n\nFile: '+filename+'\n\n' \ + 'The reported error was:\n\n'+e.args[0]+'\n\n' + \ + 'Double-check your file format and report this error if you think it''s a bug.' + except: + raise + if len(warn)>0: + return tabs, warn + + # --- Creating list of tables here + if dfs is None: + pass + elif not isinstance(dfs,dict): + if len(dfs)>0: + tabs=[Table(data=dfs, filename=filename, fileformat=fileformat, dayfirst=self.options['dayfirst'])] + else: + for k in list(dfs.keys()): + if len(dfs[k])>0: + tabs.append(Table(data=dfs[k], name=str(k), filename=filename, fileformat=fileformat, dayfirst=self.options['dayfirst'])) + if len(tabs)<=0: + warn='Warn: No dataframe found in file: '+filename+'\n' + return tabs, warn + + def haveSameColumns(self,I=None): + if I is None: + I=list(range(len(self._tabs))) + A=[len(self._tabs[i].data.columns)==len(self._tabs[I[0]].data.columns) for i in I ] + if all(A): + B=[self._tabs[i].columns_clean==self._tabs[I[0]].columns_clean for i in I] #list comparison + return all(B) + else: + return False + + def renameTable(self, iTab, newName): + oldName = self._tabs[iTab].name + if newName in [t.name for t in self._tabs]: + raise Exception('Error: This table already exist, choose a different name.') + # Renaming table + self._tabs[iTab].rename(newName) + return oldName + + def sort(self, method='byName'): + if method=='byName': + tabnames_display=self.getDisplayTabNames() + self._tabs = [t for _,t in sorted(zip(tabnames_display,self._tabs))] + else: + raise Exception('Sorting method unknown: `{}`'.format(method)) + + def mergeTabs(self, I=None, ICommonColPerTab=None, extrap='nan'): + """ + Merge table together. + TODO: add options for how interpolation/merging is done + + I: index of tables to merge, if None: all tables are merged + """ + from pydatview.tools.signal_analysis import interpDF + #from pydatview.tools.signal_analysis import applySampler + #df_new, name_new = t.applyResampling(iCol,sampDict, bAdd=bAdd) + if I is None: + I = range(len(self._tabs)) + + dfs = [self._tabs[i].data for i in I] + if ICommonColPerTab is None: + # --- Option 0 - Index concatenation + print('Using dataframe index concatenation...') + df = pd.concat(dfs, axis=1) + # Remove duplicated columns + #df = df.loc[:,~df.columns.duplicated()].copy() + else: + try: + # --- Option 1 - We combine all the x from the common column together + # NOTE: We use unique and sort, which will distrupt the user data (e.g. Airfoil Coords) + # The user should then use other methods (when implemented) + x_new=[] + cols = [] + for it, icol in zip(I, ICommonColPerTab): + xtab = self._tabs[it].data.iloc[:, icol].values + cols.append(self._tabs[it].data.columns[icol]) + x_new = np.concatenate( (x_new, xtab) ) + x_new = np.unique(np.sort(x_new)) + # Create interpolated dataframes based on x_new + dfs_new = [] + for i, (col, df_old) in enumerate(zip(cols, dfs)): + df = interpDF(x_new, col, df_old, extrap=extrap) + if 'Index' in df.columns: + df = df.drop(['Index'], axis=1) + if i>0: + df = df.drop([col], axis=1) + dfs_new.append(df) + df = pd.concat(dfs_new, axis=1) + # Reindex at the end + df.insert(0, 'Index', np.arange(df.shape[0])) + except: + # --- Option 0 - Index concatenation + print('Using dataframe index concatenation...') + df = pd.concat(dfs, axis=1) + newName = self._tabs[I[0]].name+'_merged' + self.append(Table(data=df, name=newName)) + return newName, df + + def vstack(self, I=None, commonOnly=False): + """ + Vertical stacking of tables + + I: index of tables to stack, if None: all tables are stacked + commonOnly: if True, keep only the common columns. + Otherwise, NaN will be introduced for missing columns + """ + if I is None: + I = range(len(self._tabs)) + dfs = [self._tabs[i].data for i in I] + + if commonOnly: + # --- Concatenate all but keep only common columns + df = pd.concat(dfs, join='inner', ignore_index=True) + else: + # --- Concatenate all, not worrying about common columns + df = pd.concat(dfs, ignore_index=True) + # Set unique index + if 'Index' in df.columns: + df = df.drop(['Index'], axis=1) + df.insert(0, 'Index', np.arange(df.shape[0])) + # Add to table list + newName = self._tabs[I[0]].name+'_concat' + self.append(Table(data=df, name=newName)) + return newName, df + + + + def deleteTabs(self, I): + self._tabs = [t for i,t in enumerate(self._tabs) if i not in I] + + def setActiveNames(self,names): + for t,tn in zip(self._tabs,names): + t.active_name=tn + + def getDisplayTabNames(self): + if self.options['naming']=='Ellude': + # Temporary hack, using last names if all last names are unique + names = [t.raw_name for t in self._tabs] + last_names=[n.split('|')[-1] for n in names] + if len(np.unique(last_names)) == len(names): + return ellude_common(last_names) + else: + return ellude_common(names) + elif self.options['naming']=='FileNames': + return [os.path.splitext(os.path.basename(t.filename))[0] for t in self._tabs] + else: + raise Exception('Table naming unknown: {}'.format(self.options['naming'])) + + # --- Properties + @property + def tabNames(self): + return [t.name for t in self._tabs] + + @property + def filenames(self): + return [t.filename for t in self._tabs] + + @property + def fileformats(self): + return [t.fileformat for t in self._tabs] + + @property + def unique_filenames(self): + return list(set([t.filename for t in self._tabs])) + + @property + def filenames_and_formats(self): + """ return unique list of filenames with associated fileformats """ + filenames = [] + fileformats = [] + for t in self._tabs: + if t.filename not in filenames: + filenames.append(t.filename) + fileformats.append(t.fileformat) + return filenames, fileformats + + @property + def naming(self): + return self.options['naming'] + + @naming.setter + def naming(self, naming): + if naming not in ['FileNames', 'Ellude']: + raise NotImplementedError('Naming',naming) + self.options['naming']=naming + + + def clean(self): + del self._tabs + self._tabs=[] + + def __repr__(self): + return '\n'.join([t.__repr__() for t in self._tabs]) + + # --- Mask related + @property + def maskStrings(self): + return [t.maskString for t in self._tabs] + + @property + def commonMaskString(self): + maskStrings=set(self.maskStrings) + if len(maskStrings) == 1: + return next(iter(maskStrings)) + else: + return '' + + def clearCommonMask(self): + for t in self._tabs: + t.clearMask() + + def applyCommonMaskString(self,maskString,bAdd=True): + # Apply mask on tablist + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(self._tabs): + try: + df_new, name_new = t.applyMaskString(maskString, bAdd=bAdd) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) + except: + errors.append('Mask failed for table: '+t.nickname) # TODO + + return dfs_new, names_new, errors + + # --- Resampling TODO MOVE THIS OUT OF HERE OR UNIFY + def applyResampling(self,iCol,sampDict,bAdd=True): + """ Apply resampling on table list + TODO Make this part of the action + """ + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(self._tabs): + try: + df_new, name_new = t.applyResampling(iCol, sampDict, bAdd=bAdd) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) + except: + errors.append('Resampling failed for table: '+t.nickname) # TODO + return dfs_new, names_new, errors + + # --- Filtering TODO MOVE THIS OUT OF HERE OR UNIFY + def applyFiltering(self,iCol,options,bAdd=True): + """ Apply filtering on table list + TODO Make this part of the action + """ + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(self._tabs): + try: + df_new, name_new = t.applyFiltering(iCol, options, bAdd=bAdd) + if df_new is not None: + # we don't append when string is empty + dfs_new.append(df_new) + names_new.append(name_new) + except: + errors.append('Filtering failed for table: '+t.nickname) # TODO + return dfs_new, names_new, errors + + # --- Radial average related + def radialAvg(self,avgMethod,avgParam): + """ Apply radial average on table list + TODO Make this part of the action + """ + dfs_new = [] + names_new = [] + errors=[] + for i,t in enumerate(self._tabs): + try: + dfs, names = t.radialAvg(avgMethod,avgParam) + for df,n in zip(dfs,names): + if df is not None: + dfs_new.append(df) + names_new.append(n) + except: + errors.append('Radial averaging failed for table: '+t.nickname) # TODO + return dfs_new, names_new, errors + + # --- Element--related functions + def get(self,i): + print('.>> GET') + return self._tabs[i] + + + + @staticmethod + def createDummy(nTabs=3, n=30, addLabel=True): + tabs=[] + label='' + for iTab in range(nTabs): + if addLabel: + label='_'+str(iTab) + tabs.append( Table.createDummy(n=n, label=label)) + tablist = TableList(tabs) + return tablist + + +# --------------------------------------------------------------------------------} +# --- Table +# --------------------------------------------------------------------------------{ +# +class Table(object): + """ + Main attributes: + - data + - columns # TODO get rid of me + - name + - raw_name # Should be unique and can be used for identification + - ID # Should be unique and can be used for identification + - active_name + - filename + - fileformat + - fileformat_name + - nCols x nRows + - mask + - maskString + - formulas + """ + # TODO sort out the naming + # Main naming concepts: + # name : + # active_name : + # raw_name : + # filename : + def __init__(self,data=None, name='',filename='', fileformat=None, dayfirst=False): + # Default init + self.maskString='' + self.mask=None + + self.filename = filename + self.fileformat = fileformat + if fileformat is not None: + self.fileformat_name = fileformat.name + else: + self.fileformat_name = '' + self.formulas = [] + + if not isinstance(data,pd.DataFrame): + raise NotImplementedError('Tables that are not dataframe not implemented.') + # --- Pandas DataFrame + self.data = data + # Adding index + if data.columns[0].lower().find('index')>=0: + pass + else: + data.insert(0, 'Index', np.arange(self.data.shape[0])) + + # Clean columns only once + #data.columns = [s.replace('_',' ') for s in self.data.columns.values.astype(str)] + + # --- Trying to figure out how to name this table + if name is None or len(str(name))==0: + if data.columns.name is not None: + name=data.columns.name + + self.setupName(name=str(name)) + self.convertTimeColumns(dayfirst=dayfirst) + + + def setupName(self,name=''): + # Creates a "codename": path | basename | name | ext + splits=[] + ext='' + if len(self.filename)>0: + base_dir = os.path.dirname(self.filename) + if len(base_dir)==0: + base_dir= os.getcwd() + self.filename=os.path.join(base_dir,self.filename) + splits.append(base_dir.replace('/','|').replace('\\','|')) + basename,ext=os.path.splitext(os.path.basename(self.filename)) + if len(basename)>0: + splits.append(basename) + if name is not None and len(name)>0: + splits.append(name) + #if len(ext)>0: + # splits.append(ext) + self.extension=ext + name='|'.join(splits) + if len(name)==0: + name='default' + self.name=name + self.active_name=self.name + + def __repr__(self): + s='Table object:\n' + s+=' - name: {}\n'.format(self.name) + s+=' - raw_name : {}\n'.format(self.raw_name) + s+=' - active_name: {}\n'.format(self.raw_name) + s+=' - filename : {}\n'.format(self.filename) + s+=' - fileformat : {}\n'.format(self.fileformat) + s+=' - fileformat_name : {}\n'.format(self.fileformat_name) + s+=' - columns : {}\n'.format(self.columns) + s+=' - nCols x nRows: {}x{}\n'.format(self.nCols, self.nRows) + s+=' - maskString: {}\n'.format(self.maskString) + return s + + + # --- Mask + def clearMask(self): + self.maskString='' + self.mask=None + + def applyMaskString(self, sMask, bAdd=True): + # Apply mask on Table + df = self.data + # TODO Loop on {VAR} instead.. + for i, c_in_df in enumerate(self.data.columns): + c_no_unit = no_unit(c_in_df).strip() + # TODO sort out the mess with asarray (introduced to have and/or + # as array won't work with date comparison + # NOTE: using iloc to avoid duplicates column issue + if isinstance(df.iloc[0,i], pd._libs.tslibs.timestamps.Timestamp): + sMask=sMask.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') + else: + sMask=sMask.replace('{'+c_no_unit+'}','np.asarray(df[\''+c_in_df+'\'])') + df_new = None + name_new = None + if len(sMask.strip())>0 and sMask.strip().lower()!='no mask': + try: + mask = np.asarray(eval(sMask)) + if bAdd: + df_new = df[mask] + name_new=self.raw_name+'_masked' + else: + self.mask=mask + self.maskString=sMask + except: + # TODO come up with better error messages + raise Exception('Error: The mask failed to evaluate for table: '+self.nickname) + if sum(mask)==0: + self.clearMask() + raise Exception('Error: The mask returned no value for table: '+self.nickname) + return df_new, name_new + + # --- Important manipulation TODO MOVE THIS OUT OF HERE OR UNIFY + def applyResampling(self, iCol, sampDict, bAdd=True): + # Resample Table + from pydatview.tools.signal_analysis import applySamplerDF + colName=self.data.columns[iCol] + df_new =applySamplerDF(self.data, colName, sampDict=sampDict) + # Reindex afterwards + if df_new.columns[0]=='Index': + df_new['Index'] = np.arange(0,len(df_new)) + if bAdd: + name_new=self.raw_name+'_resampled' + else: + name_new=None + self.data=df_new + return df_new, name_new + + def applyFiltering(self, iCol, options, bAdd=True): + from pydatview.tools.signal_analysis import applyFilterDF + colName=self.data.columns[iCol] + df_new =applyFilterDF(self.data, colName, options) + # Reindex afterwards + if df_new.columns[0]=='Index': + df_new['Index'] = np.arange(0,len(df_new)) + if bAdd: + name_new=self.raw_name+'_filtered' + else: + name_new=None + self.data=df_new + return df_new, name_new + + + def radialAvg(self,avgMethod, avgParam): + # TODO make this a pluggin + import pydatview.fast.postpro as fastlib + import pydatview.fast.fastfarm as fastfarm + df = self.data + base,out_ext = os.path.splitext(self.filename) + + # --- Detect if it's a FAST Farm file + sCols = ''.join(df.columns) + if sCols.find('WkDf')>1 or sCols.find('CtT')>0: + # --- FAST FARM files + Files=[base+ext for ext in ['.fstf','.FSTF','.Fstf','.fmas','.FMAS','.Fmas'] if os.path.exists(base+ext)] + if len(Files)==0: + fst_in=None + #raise Exception('Error: No .fstf file found with name: '+base+'.fstf') + else: + fst_in=Files[0] + + dfRad,_,dfDiam = fastfarm.spanwisePostProFF(fst_in,avgMethod=avgMethod,avgParam=avgParam,D=1,df=df) + dfs_new = [dfRad,dfDiam] + names_new=[self.raw_name+'_rad',self.raw_name+'_diam'] + else: + # --- FAST files + + # HACK for AD file to find the right .fst file + iDotAD=base.lower().find('.ad') + if iDotAD>1: + base=base[:iDotAD] + # + Files=[base+ext for ext in ['.fst','.FST','.Fst','.dvr','.Dvr','.DVR'] if os.path.exists(base+ext)] + if len(Files)==0: + fst_in=None + #raise Exception('Error: No .fst file found with name: '+base+'.fst') + else: + fst_in=Files[0] + + out= fastlib.spanwisePostPro(fst_in, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df = self.data) + dfRadED=out['ED_bld']; dfRadAD = out['AD']; dfRadBD = out['BD'] + + dfs_new = [dfRadAD, dfRadED, dfRadBD] + names_new=[self.raw_name+'_AD', self.raw_name+'_ED', self.raw_name+'_BD'] + return dfs_new, names_new + + def changeUnits(self, data=None): + """ Change units of the table """ + if data is None: + data={'flavor':'WE'} + # NOTE: moved to a plugin, but interface kept + from pydatview.plugins.data_standardizeUnits import changeUnits + changeUnits(self, data=data) + + def convertTimeColumns(self, dayfirst=False): + if len(self.data)>0: + for i,c in enumerate(self.data.columns.values): + y = self.data.iloc[:,i] + if y.dtype == object: + if isinstance(y.values[0], str): + # tring to convert to date + try: + vals = parser.parse(y.values[0]) + isDate=True + except: + if y.values[0]=='NaT': + isDate=True + else: + isDate=False + if isDate: + try: + self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst).to_pydatetime() + print('Column {} converted to datetime, dayfirst: {}'.format(c, dayfirst)) + except: + # Happens if values are e.g. "Monday, Tuesday" + print('Conversion to datetime failed, column {} inferred as string'.format(c)) + else: + print('Column {} inferred as string'.format(c)) + elif isinstance(y.values[0], (float, int)): + try: + self.data[c]=self.data[c].astype(float) + print('Column {} converted to float (likely nan)'.format(c)) + except: + self.data[c]=self.data[c].astype(str) + print('Column {} inferred and converted to string'.format(c)) + else : + print('>> Unknown type:',type(y.values[0])) + #print(self.data.dtypes) + + + # --- Column manipulations + def renameColumn(self,iCol,newName): + self.data.columns.values[iCol]=newName + + def renameColumns(self, strReplDict=None): + """ Rename all the columns of given table + - strReplDict: a string replacement dictionary of the form: {'new':'old'} + """ + if strReplDict is not None: + cols = self.data.columns + newcols = [] + for c in cols: + for new,old in strReplDict.items(): + c = c.replace(old,new) + newcols.append(c) + self.data.columns = newcols + + def deleteColumns(self, ICol): + """ Delete columns by index, not column names which can have duplicates""" + IKeep =[i for i in np.arange(self.data.shape[1]) if i not in ICol] + self.data = self.data.iloc[:, IKeep] # Drop won't work for duplicates + for i in sorted(ICol, reverse=True): + for f in self.formulas: + if f['pos'] == (i + 1): + self.formulas.remove(f) + break + for f in self.formulas: + if f['pos'] > (i + 1): + f['pos'] = f['pos'] - 1 + + def rename(self,new_name): + self.name='>'+new_name + + def addColumn(self, sNewName, NewCol, i=-1, sFormula=''): + print('>>> adding Column') + if i<0: + i=self.data.shape[1] + elif i>self.data.shape[1]+1: + i=self.data.shape[1] + self.data.insert(int(i+1),sNewName,NewCol) + for f in self.formulas: + if f['pos'] > i: + f['pos'] = f['pos'] + 1 + self.formulas.append({'pos': i+1, 'formula': sFormula, 'name': sNewName}) + + def setColumn(self,sNewName,NewCol,i,sFormula=''): + if i<1: + raise ValueError('Cannot set column at position ' + str(i)) + self.data = self.data.drop(columns=self.data.columns[i]) + self.data.insert(int(i),sNewName,NewCol) + for f in self.formulas: + if f['pos'] == i: + f['name'] = sNewName + f['formula'] = sFormula + + def getColumn(self, i): + """ Return column of data + If a mask exist, the mask is applied + + TODO TODO TODO get rid of this! + """ + if self.mask is not None: + c = self.data.iloc[self.mask, i] + x = self.data.iloc[self.mask, i].values + else: + c = self.data.iloc[:, i] + x = self.data.iloc[:, i].values + + isString = c.dtype == object and isinstance(c.values[0], str) + if isString: + x=x.astype(str) + isDate = np.issubdtype(c.dtype, np.datetime64) + if isDate: + dt=getDt(x) + if dt>1: + x=x.astype('datetime64[s]') + else: + x=x.astype('datetime64') + return x,isString,isDate,c + + + def evalFormula(self,sFormula): + df = self.data + for i,c in enumerate(self.columns): + c_no_unit = no_unit(c).strip() + c_in_df = df.columns[i] + sFormula=sFormula.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') + try: + NewCol=eval(sFormula) + return NewCol + except: + return None + + def addColumnByFormula(self,sNewName,sFormula,i=-1): + NewCol=self.evalFormula(sFormula) + if NewCol is None: + return False + else: + self.addColumn(sNewName,NewCol,i,sFormula) + return True + + def setColumnByFormula(self,sNewName,sFormula,i=-1): + NewCol=self.evalFormula(sFormula) + if NewCol is None: + return False + else: + self.setColumn(sNewName,NewCol,i,sFormula) + return True + + + def export(self, path): + if isinstance(self.data, pd.DataFrame): + df = self.data + if 'Index' in df.columns.values: + df = df.drop(['Index'], axis=1) + df.to_csv(path, sep=',', index=False) + else: + raise NotImplementedError('Export of data that is not a dataframe') + + + + # --- Properties + @property + def basename(self): + return os.path.splitext(os.path.basename(self.filename))[0] + + @property + def shapestring(self): + return '{}x{}'.format(self.nCols, self.nRows) + + @property + def shape(self): + return (self.nRows, self.nCols) + + @property + def columns(self): + return self.data.columns.values #.astype(str) + + @columns.setter + def columns(self, cols): + raise Exception('Columns is read only') + + @property + def columns_clean(self): + return [no_unit(s) for s in self.data.columns.values.astype(str)] + + @property + def name(self): + if len(self.__name)<=0: + return '' + if self.__name[0]=='>': + return self.__name[1:] + else: + return self.__name + + @property + def raw_name(self): + return self.__name + + @name.setter + def name(self,new_name): + self.__name=new_name + + @property + def nickname(self): + sp = self.name.split('|') + return sp[-1] + + @property + def nCols(self): + return len(self.columns) + + @property + def nRows(self): + return len(self.data.iloc[:,0]) # TODO if not panda + + @staticmethod + def createDummy(n, label='', columns=None, nCols=None): + """ create a dummy table of length n + If columns or nCols are provided, they are used for the + """ + # + if nCols is None and columns is None: + t = np.linspace(0, 4*np.pi, n) + x = np.sin(t)+10 + alpha_d = np.linspace(0, 360, n) + P = np.random.normal(0,100,n)+5000 + RPM = np.random.normal(-0.2,0.2,n) + 12. + d={'Time_[s]':t, + 'x{}_[m]'.format(label): x, + 'alpha{}_[deg]'.format(label):alpha_d, + 'P{}_[W]'.format(label):P, + 'RotSpeed{}_[rpm]'.format(label):RPM} + else: + units=['m','m/s','kn','rad','w','deg'] + if columns is None: + columns = ['C{}{}_[{}]'.format(i,label, units[np.mod(i, len(units))] ) for i in range(nCols)] + nCols=len(columns) + + d = np.zeros((n, nCols)) + for i in range(nCols): + d[:,i] = np.random.normal(0, 1, n) + i + df = pd.DataFrame(data=d, columns= columns) + return Table(data=df, name='Dummy '+label) + + +if __name__ == '__main__': + import pandas as pd; + from Tables import Table + import numpy as np + diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index fdd9bc7..c425c03 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -63,8 +63,8 @@ def fromIDs(PD, tabs, i, idx, SameCol, pipeline=None): PD.it = idx[0] # table index PD.ix = idx[1] # x index PD.iy = idx[2] # y index - PD.sx = idx[3] # x label - PD.sy = idx[4] # y label + PD.sx = idx[3].replace('_',' ') # x label + PD.sy = idx[4].replace('_',' ') # y label PD.syl = '' # y label for legend PD.st = idx[5] # table label PD.filename = tabs[PD.it].filename @@ -81,8 +81,8 @@ def fromXY(PD, x, y, sx='', sy=''): PD.x = x PD.y = y PD.c = y - PD.sx = sx - PD.sy = sy + PD.sx = sx.replace('_',' ') + PD.sy = sy.replace('_',' ') PD.xIsString = isString(x) PD.yIsString = isString(y) PD.xIsDate = isDate (x) diff --git a/pydatview/plugins/tests/test_standardizeUnits.py b/pydatview/plugins/tests/test_standardizeUnits.py index 051d634..841ee3b 100644 --- a/pydatview/plugins/tests/test_standardizeUnits.py +++ b/pydatview/plugins/tests/test_standardizeUnits.py @@ -17,7 +17,7 @@ def test_change_units(self): np.testing.assert_almost_equal(tab.data.values[:,1],[1]) np.testing.assert_almost_equal(tab.data.values[:,2],[2]) np.testing.assert_almost_equal(tab.data.values[:,3],[10]) - np.testing.assert_equal(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) + np.testing.assert_equal(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle_[deg]']) if __name__ == '__main__': diff --git a/pydatview/plugins/tool_radialavg.py b/pydatview/plugins/tool_radialavg.py index 5b06d2b..76b2d76 100644 --- a/pydatview/plugins/tool_radialavg.py +++ b/pydatview/plugins/tool_radialavg.py @@ -64,7 +64,7 @@ def onApply(self,event=None): dfs, names, errors = tabList.radialAvg(avgMethod,avgParam) self.parent.addTables(dfs,names,bAdd=True) if len(errors)>0: - raise Exception('Error: The filtering failed on some tables:\n\n'+'\n'.join(errors)) + raise Exception('Error: The radial averaging failed on some tables:\n\n'+'\n'.join(errors)) else: dfs, names = tabList[iSel-1].radialAvg(avgMethod,avgParam) self.parent.addTables([dfs],[names], bAdd=True) diff --git a/tests/test_Tables.py b/tests/test_Tables.py index b0c7cad..2e4a474 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -112,8 +112,7 @@ def test_change_units(self): np.testing.assert_almost_equal(tab.data.values[:,1],[1]) np.testing.assert_almost_equal(tab.data.values[:,2],[2]) np.testing.assert_almost_equal(tab.data.values[:,3],[10]) - print('>>>> tab.columns', tab.columns) - np.testing.assert_equal(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle [deg]']) + np.testing.assert_equal(tab.columns, ['Index','om [rpm]', 'F [kN]', 'angle_[deg]']) def test_renameColumns(self): tab = Table.createDummy(n=3, columns=['RtFldCp [-]','B1FldFx [N]', 'angle [rad]']) From 5850a9c73f50698663b4749833e79ce90bde574a Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 13 Jul 2023 04:50:47 -0600 Subject: [PATCH 106/178] Common: improved error handling, introducing PyDataView exception --- pydatview/GUICommon.py | 9 ++ pydatview/Tables.py | 34 +++---- pydatview/common.py | 58 +++++++++++- pydatview/fast/postpro.py | 183 +++++++++++++++++++++++++------------- pydatview/main.py | 8 +- pydatview/pipeline.py | 9 +- 6 files changed, 214 insertions(+), 87 deletions(-) diff --git a/pydatview/GUICommon.py b/pydatview/GUICommon.py index 623da7d..5e0aae5 100644 --- a/pydatview/GUICommon.py +++ b/pydatview/GUICommon.py @@ -67,6 +67,13 @@ def __init__(self, parent, title, message): self.Destroy() MessageBox(parent, 'About', message) +class MessageBox(wx.Dialog): + def __init__(self, parent, title, message): + wx.Dialog.__init__(self, parent, title=title, style=wx.CAPTION|wx.CLOSE_BOX) + text = wx.TextCtrl(self, style=wx.TE_READONLY|wx.BORDER_NONE|wx.TE_MULTILINE|wx.TE_AUTO_URL) + text.SetValue(message) + text.SetBackgroundColour(wx.SystemSettings.GetColour(4)) + def YesNo(parent, question, caption = 'Yes or no?'): dlg = wx.MessageDialog(parent, question, caption, wx.YES_NO | wx.ICON_QUESTION) result = dlg.ShowModal() == wx.ID_YES @@ -81,6 +88,8 @@ def Warn(parent, message, caption = 'Warning!'): dlg.ShowModal() dlg.Destroy() def Error(parent, message, caption = 'Error!'): + # For large errors we might want to use something else + #dlg = MessageBox(parent, title=caption, message=message) dlg = wx.MessageDialog(parent, message, caption, wx.OK | wx.ICON_ERROR) dlg.ShowModal() dlg.Destroy() diff --git a/pydatview/Tables.py b/pydatview/Tables.py index aeac019..ca483de 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -3,9 +3,9 @@ from dateutil import parser import pandas as pd try: - from .common import no_unit, ellude_common, getDt + from .common import no_unit, ellude_common, getDt, exception2string, PyDatViewException except: - from common import no_unit, ellude_common, getDt + from common import no_unit, ellude_common, getDt, exception2string, PyDatViewException import pydatview.io as weio # File Formats and File Readers # --------------------------------------------------------------------------------} @@ -187,7 +187,7 @@ def haveSameColumns(self,I=None): def renameTable(self, iTab, newName): oldName = self._tabs[iTab].name if newName in [t.name for t in self._tabs]: - raise Exception('Error: This table already exist, choose a different name.') + raise PyDatViewException('Error: This table already exist, choose a different name.') # Renaming table self._tabs[iTab].rename(newName) return oldName @@ -197,7 +197,7 @@ def sort(self, method='byName'): tabnames_display=self.getDisplayTabNames() self._tabs = [t for _,t in sorted(zip(tabnames_display,self._tabs))] else: - raise Exception('Sorting method unknown: `{}`'.format(method)) + raise PyDatViewException('Sorting method unknown: `{}`'.format(method)) def mergeTabs(self, I=None, ICommonColPerTab=None, extrap='nan'): """ @@ -299,7 +299,7 @@ def getDisplayTabNames(self): elif self.options['naming']=='FileNames': return [os.path.splitext(os.path.basename(t.filename))[0] for t in self._tabs] else: - raise Exception('Table naming unknown: {}'.format(self.options['naming'])) + raise PyDatViewException('Table naming unknown: {}'.format(self.options['naming'])) # --- Properties @property @@ -376,8 +376,8 @@ def applyCommonMaskString(self,maskString,bAdd=True): # we don't append when string is empty dfs_new.append(df_new) names_new.append(name_new) - except: - errors.append('Mask failed for table: '+t.nickname) # TODO + except Exception as e: + errors.append('Mask failed for table: '+t.nickname+'\n'+exception2string(e)) return dfs_new, names_new, errors @@ -396,8 +396,8 @@ def applyResampling(self,iCol,sampDict,bAdd=True): # we don't append when string is empty dfs_new.append(df_new) names_new.append(name_new) - except: - errors.append('Resampling failed for table: '+t.nickname) # TODO + except Exception as e: + errors.append('Resampling failed for table: '+t.nickname+'\n'+exception2string(e)) return dfs_new, names_new, errors # --- Filtering TODO MOVE THIS OUT OF HERE OR UNIFY @@ -415,8 +415,8 @@ def applyFiltering(self,iCol,options,bAdd=True): # we don't append when string is empty dfs_new.append(df_new) names_new.append(name_new) - except: - errors.append('Filtering failed for table: '+t.nickname) # TODO + except Exception as e: + errors.append('Filtering failed for table: '+t.nickname+'\n'+exception2string(e)) return dfs_new, names_new, errors # --- Radial average related @@ -434,16 +434,10 @@ def radialAvg(self,avgMethod,avgParam): if df is not None: dfs_new.append(df) names_new.append(n) - except: - errors.append('Radial averaging failed for table: '+t.nickname) # TODO + except Exception as e: + errors.append('Radial averaging failed for table: '+t.nickname+'\n'+exception2string(e)) return dfs_new, names_new, errors - # --- Element--related functions - def get(self,i): - print('.>> GET') - return self._tabs[i] - - @staticmethod def createDummy(nTabs=3, n=30, addLabel=True): @@ -591,7 +585,7 @@ def applyMaskString(self, sMask, bAdd=True): raise Exception('Error: The mask failed to evaluate for table: '+self.nickname) if sum(mask)==0: self.clearMask() - raise Exception('Error: The mask returned no value for table: '+self.nickname) + raise PyDatViewException('Error: The mask returned no value for table: '+self.nickname) return df_new, name_new # --- Important manipulation TODO MOVE THIS OUT OF HERE OR UNIFY diff --git a/pydatview/common.py b/pydatview/common.py index 7c9dd92..1e7374e 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -5,6 +5,12 @@ import datetime import re import inspect +import traceback + + +class PyDatViewException(Exception): + pass + CHAR={ 'menu' : u'\u2630', @@ -428,12 +434,53 @@ def Error(parent, message, caption = 'Error!'): dlg.ShowModal() dlg.Destroy() +def exception2string(excp, iMax=40, prefix=' | '): + if isinstance(excp, PyDatViewException): + return prefix + excp.args[0] + else: + stack = traceback.extract_stack()[:-3] + traceback.extract_tb(excp.__traceback__) + stacklist = traceback.format_list(stack) + # --- Parse stacktrace for file/ line / content + traceback_dicts=[] + for i, line in enumerate(stacklist): + element = line.split(',') + d = {} + filename = element[0].strip().lstrip('File').strip(' "') + # We identify "local content" and strip the path + if i==0: + basePath = os.path.dirname(filename) + isLoc = filename.find(basePath)==0 + filename = filename.replace(basePath,'') + if filename[0] == '\\': + filename=filename[1:] + filename = filename[:iMax] + (filename[iMax:] and '..') + d['File'] = filename + d['isLocal'] = isLoc + d['line'] = int(element[1].strip().lstrip('line').strip()) + content=element[2].strip().lstrip('in').strip().split('\n') + d['mod'] = content[0] + if len(content)>1: + d['content'] = content[1].strip() + else: + d['content'] = '' + traceback_dicts.append(d) + # --- + string='' + for d in traceback_dicts: + if d['content'].find('MainLoop')>0: + continue + if d['isLocal']: + string+=prefix+'{:40s}|{:<6d}| {:s}\n'.format(d['File'],d['line'],d['content']) + else: + #string+='|(backtrace continues)' + break + string += prefix+'{} {}'.format(excp.__class__,excp) + return string # --------------------------------------------------------------------------------} # --- # --------------------------------------------------------------------------------{ - def isString(x): b = x.dtype == object and isinstance(x.values[0], str) return b @@ -452,3 +499,12 @@ def removeAction (self, *args, **kwargs): Info(self.parent, 'This is dum def load_dfs (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) def mainFrameUpdateLayout(self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) def redraw (self, *args, **kwargs): Info(self.parent, 'This is dummy '+inspect.stack()[0][3]) + + +if __name__ == '__main__': +# from welib.tools.clean_exceptions import * + + try: + raise Exception('Hello') + except Exception as excp: + s= exception2string(excp) diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 29f3f01..78621a2 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -6,9 +6,10 @@ # --- fast libraries import pydatview.io as weio -from pydatview.io.fast_input_file import FASTInputFile -from pydatview.io.fast_output_file import FASTOutputFile -from pydatview.io.fast_input_deck import FASTInputDeck +from pydatview.common import PyDatViewException as WELIBException +from pydatview.io.fast_input_file import FASTInputFile +from pydatview.io.fast_output_file import FASTOutputFile +from pydatview.io.fast_input_deck import FASTInputDeck # --------------------------------------------------------------------------------} @@ -350,7 +351,7 @@ def insert_spanwise_columns(df, vr=None, R=None, IR=None, sspan='r', sspan_bar=' if (nrMax)<=len(vr_bar): vr_bar=vr_bar[:nrMax] elif (nrMax)>len(vr_bar): - raise Exception('Inconsitent length between radial stations ({:d}) and max index present in output chanels ({:d})'.format(len(vr_bar),nrMax)) + raise Exception('Inconsistent length between radial stations ({:d}) and max index present in output chanels ({:d})'.format(len(vr_bar),nrMax)) df.insert(0, sspan_bar+'_[-]', vr_bar) if IR is not None: @@ -635,60 +636,91 @@ def spanwiseColAD(Cols): """ Return column info, available columns and indices that contain AD spanwise data""" ADSpanMap=dict() for sB in ['B1','B2','B3']: - ADSpanMap['^[A]*'+sB+r'N(\d*)Alpha_\[deg\]']=sB+'Alpha_[deg]' - ADSpanMap['^[A]*'+sB+r'N(\d*)AOA_\[deg\]' ]=sB+'Alpha_[deg]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)AxInd_\[-\]' ]=sB+'AxInd_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)TnInd_\[-\]' ]=sB+'TnInd_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Alpha_\[deg\]'] =sB+'Alpha_[deg]' + ADSpanMap['^[A]*'+sB+r'N(\d*)AxInd_\[-\]' ] =sB+'AxInd_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)TnInd_\[-\]' ] =sB+'TnInd_[-]' ADSpanMap['^[A]*'+sB+r'N(\d*)AxInd_qs_\[-\]' ]=sB+'AxInd_qs_[-]' ADSpanMap['^[A]*'+sB+r'N(\d*)TnInd_qs_\[-\]' ]=sB+'TnInd_qs_[-]' ADSpanMap['^[A]*'+sB+r'N(\d*)BEM_k_\[-\]' ]=sB+'BEM_k_[-]' ADSpanMap['^[A]*'+sB+r'N(\d*)BEM_kp_\[-\]' ]=sB+'BEM_kp_[-]' ADSpanMap['^[A]*'+sB+r'N(\d*)BEM_F_\[-\]' ]=sB+'BEM_F_[-]' ADSpanMap['^[A]*'+sB+r'N(\d*)BEM_CT_qs_\[-\]' ]=sB+'BEM_CT_qs_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[deg\]' ]=sB+'AxInd_[-]' # DBGOuts NOTE BUG Unit - ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[deg\]' ]=sB+'TnInd_[-]' # DBGOuts NOTE BUG Unit - ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[-\]' ]=sB+'AxInd_[-]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[-\]' ]=sB+'TnInd_[-]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Uin_\[m/s\]' ]=sB+'Uin_[m/s]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Uit_\[m/s\]' ]=sB+'Uit_[m/s]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Uir_\[m/s\]' ]=sB+'Uir_[m/s]' # DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Cl_\[-\]' ]=sB+'Cl_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cd_\[-\]' ]=sB+'Cd_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cm_\[-\]' ]=sB+'Cm_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cx_\[-\]' ]=sB+'Cx_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cy_\[-\]' ]=sB+'Cy_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Cn_\[-\]' ]=sB+'Cn_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Ct_\[-\]' ]=sB+'Ct_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Re_\[-\]' ]=sB+'Re_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vrel_\[m/s\]' ]=sB+'Vrel_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Theta_\[deg\]']=sB+'Theta_[deg]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Phi_\[deg\]' ]=sB+'Phi_[deg]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Twst_\[deg\]' ]=sB+'Twst_[deg]' #DBGOuts - ADSpanMap['^[A]*'+sB+r'N(\d*)Curve_\[deg\]']=sB+'Curve_[deg]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vindx_\[m/s\]']=sB+'Vindx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vindy_\[m/s\]']=sB+'Vindy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fx_\[N/m\]' ]=sB+'Fx_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fy_\[N/m\]' ]=sB+'Fy_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fl_\[N/m\]' ]=sB+'Fl_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fd_\[N/m\]' ]=sB+'Fd_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Fn_\[N/m\]' ]=sB+'Fn_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Ft_\[N/m\]' ]=sB+'Ft_[N/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VUndx_\[m/s\]']=sB+'VUndx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VUndy_\[m/s\]']=sB+'VUndy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VUndz_\[m/s\]']=sB+'VUndz_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VDisx_\[m/s\]']=sB+'VDisx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VDisy_\[m/s\]']=sB+'VDisy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)VDisz_\[m/s\]']=sB+'VDisz_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)STVx_\[m/s\]' ]=sB+'STVx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)STVy_\[m/s\]' ]=sB+'STVy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)STVz_\[m/s\]' ]=sB+'STVz_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vx_\[m/s\]' ]=sB+'Vx_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vy_\[m/s\]' ]=sB+'Vy_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Vz_\[m/s\]' ]=sB+'Vz_[m/s]' - ADSpanMap['^[A]*'+sB+r'N(\d*)DynP_\[Pa\]' ]=sB+'DynP_[Pa]' - ADSpanMap['^[A]*'+sB+r'N(\d*)M_\[-\]' ]=sB+'M_[-]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Mm_\[N-m/m\]' ]=sB+'Mm_[N-m/m]' - ADSpanMap['^[A]*'+sB+r'N(\d*)Gam_\[' ]=sB+'Gam_[m^2/s]' #DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Cl_\[-\]' ] =sB+'Cl_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cd_\[-\]' ] =sB+'Cd_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cm_\[-\]' ] =sB+'Cm_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cx_\[-\]' ] =sB+'Cx_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cy_\[-\]' ] =sB+'Cy_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Cn_\[-\]' ] =sB+'Cn_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Ct_\[-\]' ] =sB+'Ct_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Re_\[-\]' ] =sB+'Re_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vrel_\[m/s\]' ] =sB+'Vrel_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Theta_\[deg\]'] =sB+'Theta_[deg]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Phi_\[deg\]' ] =sB+'Phi_[deg]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Curve_\[deg\]'] =sB+'Curve_[deg]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindx_\[m/s\]'] =sB+'Vindx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindy_\[m/s\]'] =sB+'Vindy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindxi_\[m/s\]'] =sB+'Vindxi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindyi_\[m/s\]'] =sB+'Vindyi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindzi_\[m/s\]'] =sB+'Vindzi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindxh_\[m/s\]'] =sB+'Vindxh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindyh_\[m/s\]'] =sB+'Vindyh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindzh_\[m/s\]'] =sB+'Vindzh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindxp_\[m/s\]'] =sB+'Vindxp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindyp_\[m/s\]'] =sB+'Vindyp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vindzp_\[m/s\]'] =sB+'Vindzp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fx_\[N/m\]' ] =sB+'Fx_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fy_\[N/m\]' ] =sB+'Fy_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fl_\[N/m\]' ] =sB+'Fl_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fd_\[N/m\]' ] =sB+'Fd_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fn_\[N/m\]' ] =sB+'Fn_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Ft_\[N/m\]' ] =sB+'Ft_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndx_\[m/s\]'] =sB+'VUndx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndy_\[m/s\]'] =sB+'VUndy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndz_\[m/s\]'] =sB+'VUndz_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndxi_\[m/s\]'] =sB+'VUndxi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndyi_\[m/s\]'] =sB+'VUndyi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VUndzi_\[m/s\]'] =sB+'VUndzi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisx_\[m/s\]'] =sB+'VDisx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisy_\[m/s\]'] =sB+'VDisy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisz_\[m/s\]'] =sB+'VDisz_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisxi_\[m/s\]'] =sB+'VDisxi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisyi_\[m/s\]'] =sB+'VDisyi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDiszi_\[m/s\]'] =sB+'VDiszi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisxh_\[m/s\]'] =sB+'VDisxh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisyh_\[m/s\]'] =sB+'VDisyh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDiszh_\[m/s\]'] =sB+'VDiszh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisxp_\[m/s\]'] =sB+'VDisxp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDisyp_\[m/s\]'] =sB+'VDisyp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)VDiszp_\[m/s\]'] =sB+'VDiszp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVx_\[m/s\]' ] =sB+'STVx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVy_\[m/s\]' ] =sB+'STVy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVz_\[m/s\]' ] =sB+'STVz_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVxi_\[m/s\]' ] =sB+'STVxi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVyi_\[m/s\]' ] =sB+'STVyi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVzi_\[m/s\]' ] =sB+'STVzi_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVxh_\[m/s\]' ] =sB+'STVxh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVyh_\[m/s\]' ] =sB+'STVyh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVzh_\[m/s\]' ] =sB+'STVzh_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVxp_\[m/s\]' ] =sB+'STVxp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVyp_\[m/s\]' ] =sB+'STVyp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)STVzp_\[m/s\]' ] =sB+'STVzp_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vx_\[m/s\]' ] =sB+'Vx_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vy_\[m/s\]' ] =sB+'Vy_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Vz_\[m/s\]' ] =sB+'Vz_[m/s]' + ADSpanMap['^[A]*'+sB+r'N(\d*)DynP_\[Pa\]' ] =sB+'DynP_[Pa]' + ADSpanMap['^[A]*'+sB+r'N(\d*)M_\[-\]' ] =sB+'M_[-]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Mm_\[N-m/m\]' ] =sB+'Mm_[N-m/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Gam_\[' ] =sB+'Gam_[m^2/s]' #DBGOuts + # DEPRECIATED + ADSpanMap['^[A]*'+sB+r'N(\d*)AOA_\[deg\]' ] =sB+'Alpha_[deg]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[deg\]' ] =sB+'AxInd_[-]' # DBGOuts NOTE BUG Unit + ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[deg\]' ] =sB+'TnInd_[-]' # DBGOuts NOTE BUG Unit + ADSpanMap['^[A]*'+sB+r'N(\d*)AIn_\[-\]' ] =sB+'AxInd_[-]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)ApI_\[-\]' ] =sB+'TnInd_[-]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Uin_\[m/s\]' ] =sB+'Uin_[m/s]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Uit_\[m/s\]' ] =sB+'Uit_[m/s]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Uir_\[m/s\]' ] =sB+'Uir_[m/s]' # DBGOuts + ADSpanMap['^[A]*'+sB+r'N(\d*)Twst_\[deg\]' ] =sB+'Twst_[deg]' #DBGOuts # --- AD 14 ADSpanMap[r'^Alpha(\d*)_\[deg\]' ]='Alpha_[deg]' ADSpanMap[r'^DynPres(\d*)_\[Pa\]' ]='DynPres_[Pa]' @@ -718,6 +750,25 @@ def insert_extra_columns_AD(dfRad, tsAvg, vr=None, rho=None, R=None, nB=None, ch if vr is not None: chord =chord[0:len(dfRad)] for sB in ['B1','B2','B3']: + for coord in ['i','p','h']: + for comp in ['x','y','z']: + s=comp+coord + try: + dfRad[sB+'Vflw{}_[m/s]'.format(s)] = dfRad[sB+'VDis{}_[m/s]'.format(s)] - dfRad[sB+'STV{}_[m/s]'.format(s)] + except: + pass + for coord in ['i','p','h']: + for comp in ['x','y','z']: + s=comp+coord + try: + dfRad[sB+'Vrel{}_[m/s]'.format(s)] = dfRad[sB+'VDis{}_[m/s]'.format(s)] - dfRad[sB+'STV{}_[m/s]'.format(s)] + dfRad[sB+'Vind{}_[m/s]'.format(s)] + except: + pass + try: + s='p' + dfRad[sB+'phi_{}_[def]'.format(s)] = np.arctan2(dfRad[sB+'Vrelx{}_[m/s]'.format(s)], dfRad[sB+'Vrely{}_[m/s]'.format(s)])*180/np.pi + except: + pass try: vr_bar=vr/R Fx = dfRad[sB+'Fx_[N/m]'] @@ -768,13 +819,14 @@ def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='. dfAvg = averageDF(df,avgMethod=avgMethod ,avgParam=avgParam, filename=filename) # NOTE: average 5 last seconds else: dfAvg=df - + # --- The script assume '_' between units and colnames + Cols= dfAvg.columns # --- Extract info (e.g. radial positions) from Fast input file # We don't have a .fst input file, so we'll rely on some default values for "r" rho = 1.225 chord = None # --- Extract radial positions of output channels - d = FASTSpanwiseOutputs(FST_In, OutputCols=df.columns.values) + d = FASTSpanwiseOutputs(FST_In, OutputCols=Cols) r_AD = d['r_AD'] r_ED_bld = d['r_ED_bld'] r_ED_twr = d['r_ED_twr'] @@ -812,7 +864,6 @@ def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='. out['df'] = df out['dfAvg'] = dfAvg # --- Extract radial data and export to csv if needed - Cols=dfAvg.columns.values # --- AD ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) dfRad_AD = extract_spanwise_data(ColsInfoAD, nrMaxAD, df=None, ts=dfAvg.iloc[0]) @@ -1072,8 +1123,12 @@ def spanwiseConcat(df): Cols = df.columns ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) nChan = len(ColsInfoAD) + if nChan==0: + raise WELIBException('Cannot perform spanwise concatenation, no AeroDyn spanwise data was detected in the dataframe (e.g. columns of the form "AB1N001Cl_[-]"). ') imin = np.min( [np.min(ColsInfoAD[i]['Idx']) for i in range(nChan)] ) imax = np.max( [np.max(ColsInfoAD[i]['Idx']) for i in range(nChan)] ) + if 'Time_[s]' not in df.columns: + raise WELIBException('Cannot perform spanwise concatenation, the column `Time_[s]` is not present in the dataframe.') time = df['Time_[s]'] nt = len(time) # We add two channels one for time, one for ispan @@ -1492,8 +1547,14 @@ def renameCol(x): # Sanity if len(filename)>0: filename=' (File: {})'.format(filename) + + sTAllowed = ['Time_[s]','Time [s]'] + sT = [s for s in sTAllowed if s in df.columns] + if len(sT)==0: + raise WELIBException('The dataframe must contain one of the following column: {}'.format(','.join(sTAllowed))) + # Before doing the colomn map we store the time - time = df['Time_[s]'].values + time = df[sT[0]].values timenoNA = time[~np.isnan(time)] # Column mapping if ColMap is not None: @@ -1510,10 +1571,12 @@ def renameCol(x): tStart =tEnd-avgParam elif avgMethod.lower()=='periods': # --- Using azimuth to find periods - if 'Azimuth_[deg]' not in df.columns: - raise Exception('The sensor `Azimuth_[deg]` does not appear to be in the dataframe{}. You cannot use the averaging method by `periods`, use `constantwindow` instead.'.format(filename)) + sAAllowed = ['Azimuth_[deg]','Azimuth [deg]'] + sA = [s for s in sAAllowed if s in df.columns] + if len(sA)==0: + raise WELIBException('The dataframe must contain one of the following columns: {}.\nYou cannot use the averaging method by `periods`, use `constantwindow` instead.\n{}'.format(','.join(sAAllowed),filename)) # NOTE: potentially we could average over each period and then average - psi=df['Azimuth_[deg]'].values + psi=df[sA[0]].values _,iBef = _zero_crossings(psi-psi[-2],direction='up') if len(iBef)==0: _,iBef = _zero_crossings(psi-180,direction='up') @@ -1539,7 +1602,7 @@ def renameCol(x): elif avgMethod.lower()=='periods_omega': # --- Using average omega to find periods if 'RotSpeed_[rpm]' not in df.columns: - raise Exception('The sensor `RotSpeed_[rpm]` does not appear to be in the dataframe{}. You cannot use the averaging method by `periods_omega`, use `periods` or `constantwindow` instead.'.format(filename)) + raise WELIBException('The sensor `RotSpeed_[rpm]` does not appear to be in the dataframe{}. You cannot use the averaging method by `periods_omega`, use `periods` or `constantwindow` instead.'.format(filename)) Omega=df['RotSpeed_[rpm]'].mean()/60*2*np.pi Period = 2*np.pi/Omega if avgParam is None: diff --git a/pydatview/main.py b/pydatview/main.py index cd7768f..bc08785 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -528,9 +528,13 @@ def checkErrors(self): nErr = len(self.pipePanel.errorList) if nErr>0: if not self.pipePanel.user_warned: - sErr = '\n\nCheck `Errors` in the bottom right side of the window.' if nErr>=len(self.tabList): - Warn(self, 'Errors occured on all tables.'+sErr) + if nErr==1: + sErr='\n'+'\n'.join(self.pipePanel.errorList) + Warn(self, message=sErr, caption = 'The following error occured when applying the pipeline actions:') + else: + sErr = '\n\nCheck `Errors` in the bottom right side of the window.' + Warn(self, 'Errors occured on all tables.'+sErr) #elif nErr Date: Fri, 14 Jul 2023 22:53:57 -0600 Subject: [PATCH 107/178] Plot: not forcing autoscale back, and reverting ylim behavior (see #157) --- pydatview/GUIPlotPanel.py | 105 +++++++++++++++++++++++--------------- pydatview/GUIToolBox.py | 2 +- 2 files changed, 65 insertions(+), 42 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index ba02a2d..dacd38a 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -992,29 +992,46 @@ def set_axes_lim(self, PDs, axis): yMin=np.min([PDs[i]._yMin[0] for i in axis.iPD]) yMax=np.max([PDs[i]._yMax[0] for i in axis.iPD]) delta = (yMax-yMin) - if delta==0: - # If delta is zero, we extend the bounds to "readable" values - yMean = (yMax+yMin)/2 - if abs(yMean)>1e-6: - delta = 0.05*yMean - else: - delta = 1 - elif abs(yMin)>1e-6: - delta_rel = delta/abs(yMin) - if delta_rel<1e-5: - # we set a delta such that the numerical fluctuations are visible but - # it's obvious that it's still a "constant" signal - delta = 100*delta - else: - delta = delta*pyplot_rc['axes.xmargin'] + # Old behavior + if np.isclose(yMin,yMax): + # NOTE: by using 10% of yMax we usually avoid having the "mean" written at + # the top of the script + delta=1 if np.isclose(yMax,0) else 0.1*yMax else: delta = delta*pyplot_rc['axes.xmargin'] +# if delta==0: +# # If delta is zero, we extend the bounds to "readable" values +# yMean = (yMax+yMin)/2 +# if abs(yMean)>1e-6: +# delta = 0.05*yMean +# else: +# delta = 1 +# elif abs(yMin)>1e-6: +# delta_rel = delta/abs(yMin) +# if delta<1e-5: +# delta = 0.1 +# elif delta_rel<1e-5: +# # we set a delta such that the numerical fluctuations are visible but +# # it's obvious that it's still a "constant" signal +# delta = 100*delta +# else: +# delta = delta*pyplot_rc['axes.xmargin'] +# else: +# if delta<1e-5: +# delta = 1 +# else: +# delta = delta*pyplot_rc['axes.xmargin'] axis.set_ylim(yMin-delta,yMax+delta) axis.autoscale(False, axis='y', tight=False) except: pass - def plot_all(self, keep_limits=True): + def plot_all(self, autoscale=True): + """ + autoscale: if True, find the limits based on the data. + Otherwise, the limits are restored using: + self._restore_limits and the variables: self.xlim_prev, self.ylim_prev + """ self.multiCursors=[] axes=self.fig.axes @@ -1093,25 +1110,23 @@ def plot_all(self, keep_limits=True): except: ax_left.set_yscale("log", nonposy='clip') # legacy - # XLIM - TODO FFT ONLY NASTY - if self.pltTypePanel.cbFFT.GetValue(): + if not autoscale: + # We force the limits to be the same as before + self._restore_limits() + elif self.pltTypePanel.cbFFT.GetValue(): + # XLIM - TODO FFT ONLY NASTY try: - if self.cbAutoScale.IsChecked(): - xlim=float(self.spcPanel.tMaxFreq.GetLineText(0)) - if xlim>0: - ax_left.set_xlim([0,xlim]) - pd=PD[ax_left.iPD[0]] - I=pd.x0: + ax_left.set_xlim([0,xlim]) + pd=PD[ax_left.iPD[0]] + I=pd.x Date: Sun, 16 Jul 2023 19:41:34 -0600 Subject: [PATCH 108/178] Make OpenFAST Radial Averaging an action and part of the pipeline (Closes #159) --- pydatview/GUIPlotPanel.py | 5 +- pydatview/Tables.py | 4 +- pydatview/fast/postpro.py | 6 + pydatview/main.py | 20 ++- pydatview/pipeline.py | 18 ++- pydatview/plugins/__init__.py | 13 +- pydatview/plugins/base_plugin.py | 87 +++++++++-- pydatview/plugins/data_mask.py | 6 +- pydatview/plugins/data_radialavg.py | 169 ++++++++++++++++++++++ pydatview/plugins/plotdata_filter.py | 2 +- pydatview/plugins/tests/test_radialAvg.py | 17 +++ pydatview/plugins/tool_radialavg.py | 81 ----------- 12 files changed, 306 insertions(+), 122 deletions(-) create mode 100644 pydatview/plugins/data_radialavg.py create mode 100644 pydatview/plugins/tests/test_radialAvg.py delete mode 100644 pydatview/plugins/tool_radialavg.py diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index dacd38a..31799fb 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -581,7 +581,7 @@ def addTables(self, *args, **kwargs): if self.addTablesCallback is not None: self.addTablesCallback(*args, **kwargs) else: - print('[WARN] callback to add tables to parent was not set.') + print('[WARN] callback to add tables to parent was not set. (call setAddTablesCallback)') # --- GUI DATA @@ -790,11 +790,8 @@ def removeTools(self, event=None, Layout=True): def showTool(self, toolName=''): from pydatview.plugins import TOOLS - from pydatview.plugins import OF_DATA_TOOLS # TODO remove me if toolName in TOOLS.keys(): self.showToolPanel(panelClass=TOOLS[toolName]) - elif toolName in OF_DATA_TOOLS.keys(): - self.showToolPanel(panelClass=OF_DATA_TOOLS[toolName]) else: raise Exception('Unknown tool {}'.format(toolName)) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index ca483de..6081c24 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -619,7 +619,7 @@ def applyFiltering(self, iCol, options, bAdd=True): return df_new, name_new - def radialAvg(self,avgMethod, avgParam): + def radialAvg(self, avgMethod, avgParam): # TODO make this a pluggin import pydatview.fast.postpro as fastlib import pydatview.fast.fastfarm as fastfarm @@ -660,6 +660,8 @@ def radialAvg(self,avgMethod, avgParam): dfs_new = [dfRadAD, dfRadED, dfRadBD] names_new=[self.raw_name+'_AD', self.raw_name+'_ED', self.raw_name+'_BD'] + if all(df is None for df in dfs_new): + raise PyDatViewException('No OpenFAST radial data found for table: '+self.nickname) return dfs_new, names_new def changeUnits(self, data=None): diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 78621a2..3dfaaa9 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -670,6 +670,12 @@ def spanwiseColAD(Cols): ADSpanMap['^[A]*'+sB+r'N(\d*)Vindzp_\[m/s\]'] =sB+'Vindzp_[m/s]' ADSpanMap['^[A]*'+sB+r'N(\d*)Fx_\[N/m\]' ] =sB+'Fx_[N/m]' ADSpanMap['^[A]*'+sB+r'N(\d*)Fy_\[N/m\]' ] =sB+'Fy_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fxi_\[N/m\]' ] =sB+'Fxi_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fyi_\[N/m\]' ] =sB+'Fyi_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Fzi_\[N/m\]' ] =sB+'Fzi_[N/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Mxi_\[N-m/m\]' ] =sB+'Mxi_[N-m/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Myi_\[N-m/m\]' ] =sB+'Myi_[N-m/m]' + ADSpanMap['^[A]*'+sB+r'N(\d*)Mzi_\[N-m/m\]' ] =sB+'Mzi_[N-m/m]' ADSpanMap['^[A]*'+sB+r'N(\d*)Fl_\[N/m\]' ] =sB+'Fl_[N/m]' ADSpanMap['^[A]*'+sB+r'N(\d*)Fd_\[N/m\]' ] =sB+'Fd_[N/m]' ADSpanMap['^[A]*'+sB+r'N(\d*)Fn_\[N/m\]' ] =sB+'Fn_[N/m]' diff --git a/pydatview/main.py b/pydatview/main.py index bc08785..7251121 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -33,7 +33,7 @@ import pydatview.io as weio # File Formats and File Readers # Pluggins from .plugins import DATA_PLUGINS_WITH_EDITOR, DATA_PLUGINS_SIMPLE, TOOLS -from .plugins import OF_DATA_TOOLS, OF_DATA_PLUGINS_SIMPLE +from .plugins import OF_DATA_PLUGINS_WITH_EDITOR, OF_DATA_PLUGINS_SIMPLE from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData # --------------------------------------------------------------------------------} @@ -160,8 +160,10 @@ def __init__(self, data=None): # --- OpenFAST Plugins ofMenu = wx.Menu() menuBar.Append(ofMenu, "&OpenFAST") - for toolName in OF_DATA_TOOLS.keys(): # TODO remove me, should be an action - self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onShowTool(e, s_loc), ofMenu.Append(wx.ID_ANY, toolName)) + #for toolName in OF_DATA_TOOLS.keys(): # TODO remove me, should be an action + # self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onShowTool(e, s_loc), ofMenu.Append(wx.ID_ANY, toolName)) + for toolName in OF_DATA_PLUGINS_WITH_EDITOR.keys(): + self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onDataPlugin(e, s_loc), ofMenu.Append(wx.ID_ANY, toolName)) for toolName in OF_DATA_PLUGINS_SIMPLE.keys(): self.Bind(wx.EVT_MENU, lambda e, s_loc=toolName: self.onDataPlugin(e, s_loc), ofMenu.Append(wx.ID_ANY, toolName)) @@ -500,6 +502,18 @@ def onDataPlugin(self, event=None, toolName=''): print('>>> The action already exists, we use it for the GUI') self.plotPanel.showToolAction(action) # The panel will have the responsibility to apply/delete the action, updateGUI, etc + + elif toolName in OF_DATA_PLUGINS_WITH_EDITOR.keys(): + # Check to see if the pipeline already contains this action + action = self.pipePanel.find(toolName) # old action to edit + if action is None: + function = OF_DATA_PLUGINS_WITH_EDITOR[toolName] + action = function(label=toolName, mainframe=self) # getting brand new action + else: + print('>>> The action already exists, we use it for the GUI') + self.plotPanel.showToolAction(action) + # The panel will have the responsibility to apply/delete the action, updateGUI, etc + elif toolName in DATA_PLUGINS_SIMPLE.keys(): function = DATA_PLUGINS_SIMPLE[toolName] action = function(label=toolName, mainframe=self) # calling the data function diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index 86c0f21..cd5149d 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -7,7 +7,7 @@ - A set of callbacks that manipulates tables: - - tableFunctionAdd (Table, data=dict()) # applies to a full table, return a new table + - df(s)_new, name(s)_new = tableFunctionAdd (Table, data=dict()) # applies to a full table - tableFunctionApply (Table, data=dict()) # applies to a full table (inplace) - tableFunctionCancel(Table, data=dict()) # cancel action on a full table (inplace) @@ -100,11 +100,12 @@ def applyAndAdd(self, tabList): errors=[] for i,t in enumerate(tabList): try: - df_new, name_new = self.tableFunctionAdd(t, self.data) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) + dfs_new_, names_new_ = self.tableFunctionAdd(t, self.data) + if not isinstance(dfs_new_, list): + dfs_new_ = [dfs_new_] + names_new_ = [names_new_] + dfs_new += dfs_new_ + names_new += names_new_ except Exception as e: err = 'Failed to apply action "{}" to table "{}" and creating a new table.\n{}\n\n'.format(self.name, t.nickname, exception2string(e)) self.errorList.append(err) @@ -113,8 +114,6 @@ def applyAndAdd(self, tabList): return dfs_new, names_new, errors - - def updateGUI(self): """ Typically called by a callee after append""" if self.guiCallback is not None: @@ -246,6 +245,7 @@ def __init__(self, data=[]): self.errorList = [] self.user_warned = False # Has the user been warn that errors are present self.plotFiltersData=[] # list of data for plot data filters, that plotData.py will use + self.verbose=False @property def actions(self): @@ -280,6 +280,8 @@ def collectErrors(self): # --- Behave like a list.. def append(self, action, overwrite=False, apply=True, updateGUI=True, tabList=None): + if self.verbose: + print('[Pipe] calling `append` for action `{}`. apply: {}, updateGUI: {}'.format(action.name, apply, updateGUI)) i = self.index(action) if i>=0 and overwrite: print('[Pipe] Not adding action, its already present') diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 72c5623..1a3991e 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -65,6 +65,10 @@ def _data_radialConcat(label, mainframe=None): from .data_radialConcat import radialConcatAction return radialConcatAction(label, mainframe) +def _data_radialavg(label, mainframe=None): + from .data_radialavg import radialAvgAction + return radialAvgAction(label, mainframe) + # --- Tools def _tool_logdec(*args, **kwargs): @@ -75,10 +79,6 @@ def _tool_curvefitting(*args, **kwargs): from .tool_curvefitting import CurveFitToolPanel return CurveFitToolPanel(*args, **kwargs) -# --- TODO Action -def _tool_radialavg(*args, **kwargs): - from .tool_radialavg import RadialToolPanel - return RadialToolPanel(*args, **kwargs) # --- Ordered dictionaries with key="Tool Name", value="Constructor" @@ -106,8 +106,9 @@ def _tool_radialavg(*args, **kwargs): # --- OpenFAST # TOOLS: tool plugins constructor should return a Panel class -OF_DATA_TOOLS=OrderedDict([ # TODO - ('Radial Average', _tool_radialavg), +# OF_DATA_TOOLS={} +OF_DATA_PLUGINS_WITH_EDITOR=OrderedDict([ # TODO + ('Radial Average', _data_radialavg), ]) # DATA_PLUGINS_SIMPLE: simple data plugins constructors should return an Action OF_DATA_PLUGINS_SIMPLE=OrderedDict([ diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py index c775adc..d8f8777 100644 --- a/pydatview/plugins/base_plugin.py +++ b/pydatview/plugins/base_plugin.py @@ -10,6 +10,7 @@ import numpy as np from pydatview.common import CHAR, Error, Info, Warn from pydatview.plotdata import PlotData +from pydatview.pipeline import AdderAction TOOL_BORDER=15 @@ -137,22 +138,27 @@ def onAdd(self, event=None): #icol, colname = self.plotPanel.selPanel.xCol self._GUI2Data() - dfs, names, errors = self.action.applyAndAdd(self.tabList) + if issubclass(AdderAction, type(self.action)): + self.addActionHandle(self.action, overwrite=True, apply=True, tabList=self.tabList, updateGUI=True) + #self.addActionHandle(self.action, overwrite=True, apply=self.applyRightAway, tabList=self.tabList, updateGUI=True) + else: + dfs, names, errors = self.action.applyAndAdd(self.tabList) - # We stop applying if we were applying it (NOTE: doing this before adding table due to redraw trigger of the whole panel) - if self.data['active']: - self.onToggleApply() + # We stop applying if we were applying it (NOTE: doing this before adding table due to redraw trigger of the whole panel) + if self.data['active']: + self.onToggleApply() - self.addTablesHandle(dfs, names, bAdd=True, bPlot=False) # Triggers a redraw of the whole panel... - # df, name = self.tabList[iSel-1].applyFiltering(icol, self.data, bAdd=True) - # self.parent.addTables([df], [name], bAdd=True) - #self.updateTabList() + self.addTablesHandle(dfs, names, bAdd=True, bPlot=False) # Triggers a redraw of the whole panel... + # df, name = self.tabList[iSel-1].applyFiltering(icol, self.data, bAdd=True) + # self.parent.addTables([df], [name], bAdd=True) + #self.updateTabList() + + if len(errors)>0: + if len(errors)>=len(self.tabList): + Error(self, 'Error: The action {} failed on all tables:\n\n'.format(self.action.name)+'\n'.join(errors)) + #elif len(errors)0: - if len(errors)>=len(self.tabList): - Error(self, 'Error: The action {} failed on all tables:\n\n'.format(self.action.name)+'\n'.join(errors)) - #elif len(errors)>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> CLEAR MASK') tab.clearMask() # tabList.clearCommonMask() @@ -64,7 +61,8 @@ def addTabMask(tab, opts): # --- GUI to edit plugin and control the mask action # --------------------------------------------------------------------------------{ class MaskToolPanel(ActionEditor): - def __init__(self, parent, action): + def __init__(self, parent, action=None): + import wx # avoided at module level for unittests ActionEditor.__init__(self, parent, action=action) # --- Creating "Fake data" for testing only! diff --git a/pydatview/plugins/data_radialavg.py b/pydatview/plugins/data_radialavg.py new file mode 100644 index 0000000..45b1cff --- /dev/null +++ b/pydatview/plugins/data_radialavg.py @@ -0,0 +1,169 @@ +import numpy as np +from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER +from pydatview.plugins.base_plugin import ActionEditor + +# from pydatview.common import CHAR, Error, Info, pretty_num_short +# from pydatview.common import DummyMainFrame +from pydatview.pipeline import AdderAction + + +# --------------------------------------------------------------------------------} +# --- Radial +# --------------------------------------------------------------------------------{ +sAVG_METHODS = ['Last `n` seconds','Last `n` periods'] +AVG_METHODS = ['constantwindow','periods'] + +# --------------------------------------------------------------------------------} +# --- Data +# --------------------------------------------------------------------------------{ +_DEFAULT_DICT={ + 'active':False, + 'avgMethod':'constantwindow', + 'avgParam': '2' +} + +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def radialAvgAction(label, mainframe, data=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + The action is also edited and created by the GUI Editor + """ + if data is None: + # NOTE: if we don't do copy below, we will end up remembering even after the action was deleted + # its not a bad feature, but we might want to think it through + # One issue is that "active" is kept in memory + data=_DEFAULT_DICT + data['active'] = False #<<< Important + + #guiCallback = mainframe.redraw + if mainframe is not None: + # TODO TODO TODO Clean this up + def guiCallback(): + if hasattr(mainframe,'selPanel'): + mainframe.selPanel.colPanel1.setColumns() + mainframe.selPanel.colPanel2.setColumns() + mainframe.selPanel.colPanel3.setColumns() + mainframe.onTabSelectionChange() # trigger replot + if hasattr(mainframe,'pipePanel'): + pass + + action = AdderAction( + name=label, + tableFunctionAdd = radialAvg, +# tableFunctionApply = applyMask, +# tableFunctionCancel = removeMask, + guiEditorClass = RadialToolPanel, + guiCallback = guiCallback, + data = data, + mainframe=mainframe + ) + return action + + +# --------------------------------------------------------------------------------} +# --- Main methods +# --------------------------------------------------------------------------------{ +# add method +def radialAvg(tab, data=None): + """ NOTE: radial average may return several dataframe""" + #print('>>> RadialAvg',data) + dfs_new, names_new = tab.radialAvg(data['avgMethod'],data['avgParam']) + return dfs_new, names_new + +# --------------------------------------------------------------------------------} +# --- GUI to edit plugin and control the action +# --------------------------------------------------------------------------------{ +class RadialToolPanel(ActionEditor): + def __init__(self, parent, action=None): + import wx # avoided at module level for unittests + super(RadialToolPanel,self).__init__(parent, action=action) + + # --- Creating "Fake data" for testing only! + if action is None: + print('[WARN] Calling GUI without an action! Creating one.') + action = maskAction(label='dummyAction', mainframe=DummyMainFrame(parent)) + + # --- GUI elements + self.btClose = self.getBtBitmap(self,'Close' ,'close' , self.destroy) + self.btAdd = self.getBtBitmap(self,'Average','compute', self.onAdd) # ART_PLUS + + self.lb = wx.StaticText( self, -1, """Select tables, averaging method and average parameter (`Period` methods uses the `azimuth` signal) """) + self.cbTabs = wx.ComboBox(self, choices=[], style=wx.CB_READONLY) + self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better + + self.cbMethod = wx.ComboBox(self, choices=sAVG_METHODS, style=wx.CB_READONLY) + + self.textAverageParam = wx.TextCtrl(self, wx.ID_ANY, '', size = (36,-1), style=wx.TE_PROCESS_ENTER) + + # --- Layout + btSizer = wx.FlexGridSizer(rows=2, cols=1, hgap=0, vgap=0) + #btSizer = wx.BoxSizer(wx.VERTICAL) + btSizer.Add(self.btClose ,0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btAdd ,0, flag = wx.ALL|wx.EXPAND, border = 1) + + row_sizer = wx.BoxSizer(wx.HORIZONTAL) + row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) + row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Method:'), 0, wx.CENTER|wx.LEFT, 5) + row_sizer.Add(self.cbMethod , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Param:') , 0, wx.CENTER|wx.LEFT, 5) + row_sizer.Add(self.textAverageParam , 0, wx.CENTER|wx.LEFT|wx.RIGHT| wx.EXPAND, 2) + + vert_sizer = wx.BoxSizer(wx.VERTICAL) + vert_sizer.Add(self.lb ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) + vert_sizer.Add(row_sizer ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) + + self.sizer = wx.BoxSizer(wx.HORIZONTAL) + self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) + self.sizer.Add(vert_sizer ,0, flag = wx.LEFT|wx.EXPAND,border = TOOL_BORDER) + self.SetSizer(self.sizer) + + # --- Events + # NOTE: getBtBitmap and getToggleBtBitmap already specify the binding + self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + + # --- Init triggers + self._Data2GUI() + #self.onToggleApply(init=True) + self.updateTabList() + + # --- Implementation specific + + + # --- Table related + def onTabChange(self,event=None): + tabList = self.parent.selPanel.tabList + + def updateTabList(self,event=None): + tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() + #try: + iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) + self.cbTabs.Clear() + [self.cbTabs.Append(tn) for tn in tabListNames] + self.cbTabs.SetSelection(iSel) + #except RuntimeError: + # pass + + # --- Fairly generic + def _GUI2Data(self): + try: + self.data['avgParam'] = float(self.textAverageParam.GetLineText(0)) + except: + raise Exception('Error: the averaging parameter needs to be an integer or a float') + self.data['avgMethod'] = AVG_METHODS[self.cbMethod.GetSelection()] + #iSel = self.cbTabs.GetSelection() + #iSel = self.cbTabs.GetSelection() + #tabList = self.parent.selPanel.tabList + + def _Data2GUI(self): + iSel = AVG_METHODS.index(self.data['avgMethod']) + self.cbMethod.SetSelection(iSel) + self.textAverageParam.SetValue(str(self.data['avgParam'])) + + + +if __name__ == '__main__': + from pydatview.plugins.base_plugin import demoGUIPlugin + demoGUIPlugin(RadialToolPanel, actionCreator=radialAvgAction, mainLoop=False, title='Radial Avg') diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index 35bdc35..edff7ef 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -64,7 +64,7 @@ def filterTabAdd(tab, opts): class FilterToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): - import wx + import wx # avoided at module level for unittests PlotDataActionEditor.__init__(self, parent, action, tables=True, **kwargs) # --- Data diff --git a/pydatview/plugins/tests/test_radialAvg.py b/pydatview/plugins/tests/test_radialAvg.py new file mode 100644 index 0000000..df2fbfe --- /dev/null +++ b/pydatview/plugins/tests/test_radialAvg.py @@ -0,0 +1,17 @@ +import unittest +import numpy as np +from pydatview.plugins.base_plugin import demoGUIPlugin, HAS_WX +from pydatview.plugins.data_radialavg import * +from pydatview.plugins.data_radialavg import _DEFAULT_DICT + +class TestRadialAvg(unittest.TestCase): + + def test_showGUI(self): + if HAS_WX: + demoGUIPlugin(RadialToolPanel, actionCreator=radialAvgAction, mainLoop=False, title='Radial Avg') + else: + print('[WARN] skipping test because wx is not available.') + +if __name__ == '__main__': + unittest.main() + diff --git a/pydatview/plugins/tool_radialavg.py b/pydatview/plugins/tool_radialavg.py deleted file mode 100644 index 76b2d76..0000000 --- a/pydatview/plugins/tool_radialavg.py +++ /dev/null @@ -1,81 +0,0 @@ -import wx -import numpy as np -from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER -# --------------------------------------------------------------------------------} -# --- Radial -# --------------------------------------------------------------------------------{ -sAVG_METHODS = ['Last `n` seconds','Last `n` periods'] -AVG_METHODS = ['constantwindow','periods'] - -class RadialToolPanel(GUIToolPanel): - def __init__(self, parent): - super(RadialToolPanel,self).__init__(parent) - - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - - # --- GUI - btClose = self.getBtBitmap(self,'Close' ,'close' , self.destroy) - btComp = self.getBtBitmap(self,'Average','compute', self.onApply) # ART_PLUS - - self.lb = wx.StaticText( self, -1, """Select tables, averaging method and average parameter (`Period` methods uses the `azimuth` signal) """) - self.cbTabs = wx.ComboBox(self, choices=tabListNames, style=wx.CB_READONLY) - self.cbMethod = wx.ComboBox(self, choices=sAVG_METHODS, style=wx.CB_READONLY) - self.cbTabs.SetSelection(0) - self.cbMethod.SetSelection(0) - - self.textAverageParam = wx.TextCtrl(self, wx.ID_ANY, '2',size = (36,-1), style=wx.TE_PROCESS_ENTER) - - btSizer = wx.FlexGridSizer(rows=2, cols=1, hgap=0, vgap=0) - #btSizer = wx.BoxSizer(wx.VERTICAL) - btSizer.Add(btClose ,0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(btComp ,0, flag = wx.ALL|wx.EXPAND, border = 1) - - row_sizer = wx.BoxSizer(wx.HORIZONTAL) - row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) - row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) - row_sizer.Add(wx.StaticText(self, -1, 'Method:'), 0, wx.CENTER|wx.LEFT, 5) - row_sizer.Add(self.cbMethod , 0, wx.CENTER|wx.LEFT, 2) - row_sizer.Add(wx.StaticText(self, -1, 'Param:') , 0, wx.CENTER|wx.LEFT, 5) - row_sizer.Add(self.textAverageParam , 0, wx.CENTER|wx.LEFT|wx.RIGHT| wx.EXPAND, 2) - - vert_sizer = wx.BoxSizer(wx.VERTICAL) - vert_sizer.Add(self.lb ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) - vert_sizer.Add(row_sizer ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) - - self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) - self.sizer.Add(vert_sizer ,0, flag = wx.LEFT|wx.EXPAND,border = TOOL_BORDER) - self.SetSizer(self.sizer) - self.Bind(wx.EVT_COMBOBOX, self.onTabChange, self.cbTabs ) - - def onTabChange(self,event=None): - tabList = self.parent.selPanel.tabList - - def onApply(self,event=None): - try: - avgParam = float(self.textAverageParam.GetLineText(0)) - except: - raise Exception('Error: the averaging parameter needs to be an integer or a float') - iSel = self.cbTabs.GetSelection() - avgMethod = AVG_METHODS[self.cbMethod.GetSelection()] - tabList = self.parent.selPanel.tabList - if iSel==0: - dfs, names, errors = tabList.radialAvg(avgMethod,avgParam) - self.parent.addTables(dfs,names,bAdd=True) - if len(errors)>0: - raise Exception('Error: The radial averaging failed on some tables:\n\n'+'\n'.join(errors)) - else: - dfs, names = tabList[iSel-1].radialAvg(avgMethod,avgParam) - self.parent.addTables([dfs],[names], bAdd=True) - - self.updateTabList() - - def updateTabList(self,event=None): - tabList = self.parent.selPanel.tabList - tabListNames = ['All opened tables']+tabList.getDisplayTabNames() - iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) - From a0b4c1d26630eb330f8eb12ef17a03be5999428a Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 16 Jul 2023 20:32:02 -0600 Subject: [PATCH 109/178] IO/tools: updates from weio/welib --- pydatview/fast/postpro.py | 2 +- pydatview/fast/subdyn.py | 908 +++++++++++++++++++++++++++++++ pydatview/io/__init__.py | 2 +- pydatview/io/converters.py | 64 +++ pydatview/io/fast_output_file.py | 198 +++++-- pydatview/io/file.py | 50 +- pydatview/io/matlabmat_file.py | 112 ++++ pydatview/io/tdms_file.py | 5 +- pydatview/plotdata.py | 2 +- pydatview/tools/fatigue.py | 271 ++++++++- pydatview/tools/spectral.py | 3 +- pydatview/tools/stats.py | 4 +- pydatview/tools/tictoc.py | 73 +++ 13 files changed, 1617 insertions(+), 77 deletions(-) create mode 100644 pydatview/fast/subdyn.py create mode 100644 pydatview/io/converters.py create mode 100644 pydatview/io/matlabmat_file.py create mode 100644 pydatview/tools/tictoc.py diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 3dfaaa9..3777cae 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -10,7 +10,7 @@ from pydatview.io.fast_input_file import FASTInputFile from pydatview.io.fast_output_file import FASTOutputFile from pydatview.io.fast_input_deck import FASTInputDeck - +from pydatview.fast.subdyn import SubDyn # --------------------------------------------------------------------------------} # --- Tools for IO diff --git a/pydatview/fast/subdyn.py b/pydatview/fast/subdyn.py new file mode 100644 index 0000000..25ff4ff --- /dev/null +++ b/pydatview/fast/subdyn.py @@ -0,0 +1,908 @@ +""" +Tools for SubDyn + +- Setup a FEM model, compute Guyan and CB modes +- Get a dataframe with properties +- More todo + +""" + + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import copy +import re +# Local +from pydatview.io.fast_input_file import FASTInputFile +from pydatview.tools.tictoc import Timer + +idGuyanDamp_None = 0 +idGuyanDamp_Rayleigh = 1 +idGuyanDamp_66 = 2 + +class SubDyn: + + def __init__(self, sdFilename_or_data=None): + """ + Initialize a SubDyn object either with: + - sdFilename: a subdyn input file name + - sdData: an instance of FASTInputFile + """ + + self._graph=None + self.File=None + + # Read SubDyn file + if sdFilename_or_data is not None: + if hasattr(sdFilename_or_data,'startswith'): # if string + self.File = FASTInputFile(sdFilename_or_data) + else: + self.File = sdFilename_or_data + + self.M_tip=None + + # Internal + self._graph=None + self._mgraph=None # Member graph + self._FEM=None + + def __repr__(self): + s='<{} object>:\n'.format(type(self).__name__) + s+='|properties:\n' + s+='|- File: (input file data)\n' + s+='|* graph: (Nodes/Elements/Members)\n' + s+='|* pointsMJ, pointsMN, pointsMNout\n' + s+='|methods:\n' + s+='|- memberPostPro\n' + s+='|- setTopMass\n' + s+='|- beamDataFrame, beamFEM, beamModes\n' + s+='|- toYAMSData\n' + return s + + # --------------------------------------------------------------------------------} + # --- Functions for general FEM model (jacket, flexible floaters) + # --------------------------------------------------------------------------------{ + def init(self, TP=(0,0,0), gravity = 9.81): + """ + Initialize SubDyn FEM model + + TP: position of transition point + gravity: position of transition point + """ + import welib.FEM.fem_beam as femb + import welib.FEM.fem_model as femm + BC = 'clamped-free' # TODO Boundary condition: free-free or clamped-free + element = 'frame3d' # Type of element used in FEM + + FEMMod = self.File['FEMMod'] + if FEMMod==1: + mainElementType='frame3d' + elif FEMMod==2: + mainElementType='frame3dlin' + elif FEMMod==3: + mainElementType='timoshenko' + else: + raise NotImplementedError() + # Get graph + graph = self.graph + #print('>>> graph\n',graph) + #graph.toJSON('_GRAPH.json') + # Convert to FEM model + with Timer('From graph'): + FEM = femm.FEMModel.from_graph(self.graph, mainElementType=mainElementType, refPoint=TP, gravity=gravity) + #model.toJSON('_MODEL.json') + with Timer('Assembly'): + FEM.assembly() + with Timer('Internal constraints'): + FEM.applyInternalConstraints() + FEM.partition() + with Timer('BC'): + FEM.applyFixedBC() + with Timer('EIG'): + Q, freq = FEM.eig(normQ='byMax') + with Timer('CB'): + FEM.CraigBampton(nModesCB = self.File['Nmodes']) + + with Timer('Modes'): + FEM.setModes(nModesFEM=30, nModesCB=self.File['Nmodes']) +# FEM.nodesDisp(Q) + + # --- SubDyn partition/notations + FEM.MBB = FEM.MM_CB[np.ix_(FEM.DOF_Leader_CB , FEM.DOF_Leader_CB)] + FEM.KBB = FEM.KK_CB[np.ix_(FEM.DOF_Leader_CB , FEM.DOF_Leader_CB)] + FEM.MBM = FEM.MM_CB[np.ix_(FEM.DOF_Leader_CB , FEM.DOF_Follower_CB)] + FEM.KMM = FEM.KK_CB[np.ix_(FEM.DOF_Follower_CB, FEM.DOF_Follower_CB)] + zeta =self.File['JDampings']/100 + if not hasattr(zeta,'__len__'): + zeta = [zeta]*FEM.nModesCB + FEM.CMM = 2*np.array(zeta) * FEM.f_CB * 2 * np.pi + + # --- Matrices wrt TP point + TI=FEM.T_refPoint + MBBt = TI.T.dot(FEM.MBB).dot(TI) + KBBt = TI.T.dot(FEM.KBB).dot(TI) + MBBt[np.abs(MBBt)<1e-4] =0 + KBBt[np.abs(KBBt)<1e-4] =0 + FEM.MBBt = MBBt + FEM.KBBt = KBBt + + # --- Set Damping + dampMod = self.File['GuyanDampMod'] + alpha_Rayleigh, beta_Rayleigh = None, None + # 6x6 Guyan Damping matrix + CC_CB_G = None + if dampMod == idGuyanDamp_None: + FEM.CBBt = np.zeros((6,6)) + elif dampMod == idGuyanDamp_Rayleigh: + # Rayleigh Damping + alpha_Rayleigh, beta_Rayleigh = self.File['RayleighDamp'] + FEM.CBBt = alpha_Rayleigh * FEM.MBBt + beta_Rayleigh * FEM.KBBt + elif dampMod == idGuyanDamp_66: + FEM.CBBt = self.File['GuyanDampMatrix'] + else: + raise Exception() + + # --- Compute rigid body equivalent + FEM.rigidBodyEquivalent() + self._FEM = FEM + + return FEM + + + def setTopMass(self): + # TODO + # Add an optional top mass and ineria + if TopMass: + # NOTE: you can use welib.yams.windturbine to compute RNA mass and inertia + Mtop = 50000 # Top mass [kg] + M_tip= rigidBodyMassMatrixAtP(m=Mtop, J_G=None, Ref2COG=None) + else: + M_tip=None + + + def getGraph(self, nDiv=1): + # See welib.weio.fast_input_file_graph.py to see how SubDyn files are converted to graph + # See welib.fem.graph.py for Graph interface + _graph = self.File.toGraph().divideElements(nDiv, + excludeDataKey='Type', excludeDataList=['Cable','Rigid'], method='insert', + keysNotToCopy=['IBC','RBC','addedMassMatrix'] # Very important + ) + + if len(_graph.Members)==0: + raise Exception('Problem in graph subdivisions, no members found.') + # Sanitization + #idBC_Fixed = 0 # Fixed BC + #idBC_Internal = 10 # Free/Internal BC + #idBC_Leader = 20 # Leader DOF + MapIBC={0:0, 1:20} # 1 in the input file is leader + MapRBC={0:10, 1:0} # 1 in the input file is fixed + for n in _graph.Nodes: + #print(n) + if 'IBC' in n.data.keys(): + IBC = n.data['IBC'].copy() + n.data['IBC'] = [MapIBC[i] for i in IBC[:6]] + if 'RBC' in n.data.keys(): + RBC = n.data['RBC'].copy() + n.data['RBC'] = [MapRBC[i] for i in RBC[:6]] + if any(RBC[:6])==0: + print('RBC ',RBC) + print('n.data[RBC]',n.data['RBC'] ) + print('n ',n ) + raise NotImplementedError('SSI') + + return _graph + + @property + def graph(self): + if self._graph is None: + self._graph = self.getGraph(nDiv = self.File['NDiv']) + return copy.deepcopy(self._graph) + + + @property + def pointsMJ(self): + """ return a dataframe with the coordinates of all members and joints + The index corresponds to the SubDyn outputs "M_J_XXX" + """ + Joints=[] + labels =[] + graph = self.graph + for ie,M in enumerate(graph.Members): # loop on members + Nodes = M.getNodes(graph) + for iN,N in enumerate([Nodes[0], Nodes[-1]]): + s='M{}J{}'.format(ie+1, iN+1) + Joints.append([N.x,N.y,N.z]) + labels.append(s) + df =pd.DataFrame(data=np.asarray(Joints), index=labels, columns=['x','y','z']) + return df + + @property + def pointsMN(self): + """ return a dataframe with the coordinates of all members and nodes + The index would correspond to the SubDyn outputs "M_N_XXX *prior* to the user selection" + """ + Nodes=[] + labels =[] + graph = self.graph + for im,M in enumerate(graph.Members): # loop on members + nodes = M.getNodes(graph) + for iN,N in enumerate(nodes): # Loop on member nodes + s='M{}N{}'.format(im+1, iN+1) + Nodes.append([N.x,N.y,N.z]) + labels.append(s) + df =pd.DataFrame(data=np.asarray(Nodes), index=labels, columns=['x','y','z']) + return df + + @property + def pointsMNout(self): + """ return a dataframe with the coordinates of members and nodes requested by user + The index corresponds to the SubDyn outputs "M_N_XXX selected by the user" + """ + Nodes=[] + labels =[] + graph = self.graph + for im, out in enumerate(self.File['MemberOuts']): + mID = out[0] # Member ID + iNodes = np.array(out[2:])-1 # Node positions within member (-1 for python indexing) + nodes = graph.getMemberNodes(mID) + nodes = np.array(nodes)[iNodes] + for iN,N in enumerate(nodes): # Loop on selected nodes + s='M{}N{}'.format(im+1, iN+1) + Nodes.append([N.x,N.y,N.z]) + labels.append(s) + df =pd.DataFrame(data=np.asarray(Nodes), index=labels, columns=['x','y','z']) + return df + + + def memberPostPro(self, dfAvg): + """ + Convert a dataframe of SubDyn/OpenFAST outputs (time-averaged) + with columns such as: M_N_* and M_J_* + into a dataframe that is organized by main channel name and nodal coordinates. + The scripts taken into account with member ID and node the user requested as outputs channels. + Discretization (nDIV) is also taken into account. + + For instance: + dfAvg with columns = ['M1N1MKye_[N*m]', 'M1N2MKye_[N*m]', 'M1N1TDxss_[m]'] + returns: + MNout with columns ['x', 'y', 'z', 'MKye_[Nm]', 'TDxss_[m]'] + and index ['M1N1', 'M1N2'] + with x,y,z the undisplaced nodal positions (accounting for discretization) + + INPUTS: + - dfAvg: a dataframe of time-averaged SubDyn/OpenFAST outputs, for instance obtained as: + df = FASTInputFile(filename).toDataFrame() + dfAvg = postpro.averageDF(df, avgMethod=avgMethod ,avgParam=avgParam) + OUTPUTS + - MNout: dataframe of members outputs (requested by the user) + - MJout: dataframe of joints outputs + """ + import welib.fast.postpro as postpro # Import done here to avoid circular dependency + + # --- Get Points where output are requested + MJ = self.pointsMJ + MNo= self.pointsMNout + MJ.columns = ['x_[m]','y_[m]', 'z_[m]'] + MNo.columns = ['x_[m]','y_[m]', 'z_[m]'] + + # --- Requested Member Outputs + Map={} + Map['^'+r'M(\d*)N(\d*)TDxss_\[m\]'] = 'TDxss_[m]' + Map['^'+r'M(\d*)N(\d*)TDyss_\[m\]'] = 'TDyss_[m]' + Map['^'+r'M(\d*)N(\d*)TDzss_\[m\]'] = 'TDzss_[m]' + Map['^'+r'M(\d*)N(\d*)RDxe_\[rad\]'] = 'RDxe_[deg]' # NOTE rescale needed + Map['^'+r'M(\d*)N(\d*)RDye_\[rad\]'] = 'RDye_[deg]' # NOTE rescale needed + Map['^'+r'M(\d*)N(\d*)RDze_\[rad\]'] = 'RDze_[deg]' # NOTE rescale needed + Map['^'+r'M(\d*)N(\d*)FKxe_\[N\]'] = 'FKxe_[N]' + Map['^'+r'M(\d*)N(\d*)FKye_\[N\]'] = 'FKye_[N]' + Map['^'+r'M(\d*)N(\d*)FKze_\[N\]'] = 'FKze_[N]' + Map['^'+r'M(\d*)N(\d*)MKxe_\[N\*m\]'] = 'MKxe_[Nm]' + Map['^'+r'M(\d*)N(\d*)MKye_\[N\*m\]'] = 'MKye_[Nm]' + Map['^'+r'M(\d*)N(\d*)MKze_\[N\*m\]'] = 'MKze_[Nm]' + ColsInfo, _ = postpro.find_matching_columns(dfAvg.columns, Map) + nCols = len(ColsInfo) + if nCols>0: + newCols=[c['name'] for c in ColsInfo ] + ValuesM = pd.DataFrame(index=MNo.index, columns=newCols) + for ic,c in enumerate(ColsInfo): + Idx, cols, colname = c['Idx'], c['cols'], c['name'] + labels = [re.match(r'(^M\d*N\d*)', s)[0] for s in cols] + ValuesM.loc[labels,colname] = dfAvg[cols].values.flatten() + if 'deg' in colname and 'rad' in cols[0]: + ValuesM[colname] *= 180/np.pi + # We remove lines that are all NaN + Values = ValuesM.dropna(axis = 0, how = 'all') + MNo2 = MNo.loc[Values.index] + MNout = pd.concat((MNo2, Values), axis=1) + else: + MNout = None + + # --- Joint Outputs + Map={} + Map['^'+r'M(\d*)J(\d*)FKxe_\[N\]'] ='FKxe_[N]' + Map['^'+r'M(\d*)J(\d*)FKye_\[N\]'] ='FKye_[N]' + Map['^'+r'M(\d*)J(\d*)FKze_\[N\]'] ='FKze_[N]' + Map['^'+r'M(\d*)J(\d*)MKxe_\[N\*m\]']='MKxe_[Nm]' + Map['^'+r'M(\d*)J(\d*)MKye_\[N\*m\]']='MKye_[Nm]' + Map['^'+r'M(\d*)J(\d*)MKze_\[N\*m\]']='MKze_[Nm]' + Map['^'+r'M(\d*)J(\d*)FMxe_\[N\]'] ='FMxe_[N]' + Map['^'+r'M(\d*)J(\d*)FMye_\[N\]'] ='FMye_[N]' + Map['^'+r'M(\d*)J(\d*)FMze_\[N\]'] ='FMze_[N]' + Map['^'+r'M(\d*)J(\d*)MMxe_\[N\*m\]']='MMxe_[Nm]' + Map['^'+r'M(\d*)J(\d*)MMye_\[N\*m\]']='MMye_[Nm]' + Map['^'+r'M(\d*)J(\d*)MMze_\[N\*m\]']='MMze_[Nm]' + ColsInfo, _ = postpro.find_matching_columns(dfAvg.columns, Map) + nCols = len(ColsInfo) + if nCols>0: + newCols=[c['name'] for c in ColsInfo ] + ValuesJ = pd.DataFrame(index=MJ.index, columns=newCols) + for ic,c in enumerate(ColsInfo): + Idx, cols, colname = c['Idx'], c['cols'], c['name'] + labels = [re.match(r'(^M\d*J\d*)', s)[0] for s in cols] + ValuesJ.loc[labels,colname] = dfAvg[cols].values.flatten() + # We remove lines that are all NaN + Values = ValuesJ.dropna(axis = 0, how = 'all') + MJ2 = MJ.loc[Values.index] + MJout = pd.concat((MJ2, Values), axis=1) + else: + MJout = None + return MNout, MJout + + + + + # --------------------------------------------------------------------------------} + # --- Functions for beam-like structure (Spar, Monopile) + # --------------------------------------------------------------------------------{ + def beamDataFrame(self, equispacing=False): + """ """ + # --- Parameters + UseSubDynModel=True + TopMass = False + + + # Convert to "welib.fem.Graph" class to easily handle the model (overkill for a monopile) + locgraph = self.graph.sortNodesBy('z') + # Add nodal properties from propsets (NOTE: Not done anymore by SubDyn because a same node can have different diameters...) + for e in locgraph.Elements: + locgraph.setElementNodalProp(e, propset=e.propset, propIDs=e.propIDs) + df = locgraph.nodalDataFrame() + + if equispacing: + from welib.tools.pandalib import pd_interp1 + # Interpolate dataframe to equispaced values + xOld = df['z'] # NOTE: FEM uses "x" as main axis + nSpan = len(xOld) + x = np.linspace(np.min(xOld),np.max(xOld), nSpan) + df = pd_interp1(x, 'z', df) + + x = df['z'] # NOTE: FEM uses "x" as main axis + D = df['D'] # Diameter [m] + t = df['t'] # thickness [m] + # Derive section properties for a hollow cylinder based on diameter and thickness + A = np.pi*( (D/2)**2 - (D/2-t)**2) # Area for annulus [m^2] + I = np.pi/64*(D**4-(D-2*t)**4) # Second moment of area for annulus (m^4) + Kt = I # Torsion constant, same as I for annulus [m^4] + Ip = 2*I # Polar second moment of area [m^4] + df['A'] = A + df['I'] = I + df['Kt'] = Kt + df['Ip'] = Ip + df['m'] = df['rho'].values*A + + return df + + def beamFEM(self, df=None): + """ return FEM model for beam-like structures, like Spar/Monopile""" + import welib.FEM.fem_beam as femb + + BC = 'clamped-free' # TODO Boundary condition: free-free or clamped-free + element = 'frame3d' # Type of element used in FEM + + if df is None: + df = self.beamDataFrame() + x = df['z'] # NOTE: FEM uses "x" as main axis + E = df['E'] # Young modules [N/m^2] + G = df['G'] # Shear modules [N/m^2] + rho = df['rho'] # material density [kg/m^3] + Ip = df['Ip'] + I = df['I'] + A = df['A'] + Kt = df['Kt'] + + # --- Compute FEM model and mode shapes + with Timer('Setting up FEM model'): + FEM=femb.cbeam(x,m=rho*A,EIx=E*Ip,EIy=E*I,EIz=E*I,EA=E*A,A=A,E=E,G=G,Kt=Kt, + element=element, BC=BC, M_tip=self.M_tip) + return FEM + + def beamModes(self, nCB=8, FEM = None): + """ Returns mode shapes for beam-like structures, like Spar/Monopile """ + import welib.FEM.fem_beam as femb + element = 'frame3d' # Type of element used in FEM + if FEM is None: + FEM = self.beamFEM() + # --- Perform Craig-Bampton reduction, fixing the top node of the beam + with Timer('FEM eigenvalue analysis'): + Q_G,_Q_CB, df_G, df_CB, Modes_G, Modes_CB, CB = femb.CB_topNode(FEM, nCB=nCB, element=element, main_axis='x') + # df_CB.to_csv('_CB.csv',index=False) + # df_G.to_csv('_Guyan.csv',index=False) + return Q_G,_Q_CB, df_G, df_CB, Modes_G, Modes_CB, CB + + def beamModesPlot(self): + """ """ + # TODO + nModesPlot=8 + # --- Show frequencies to screen + print('Mode Frequency Label ') + for i in np.arange(8): + print('{:4d} {:10.3f} {:s}'.format(i+1,FEM['freq'][i],FEM['modeNames'][i])) + + # --- Plot mode components for first few modes + print(x.shape) + #Q=FEM['Q'] ; modeNames = FEM['modeNames'] + #Q=Q_CB ;modeNames = names_CB + Modes=Modes_CB + nModesPlot=min(len(Modes),nModesPlot) + + fig,axes = plt.subplots(1, nModesPlot, sharey=False, figsize=(12.4,2.5)) + fig.subplots_adjust(left=0.04, right=0.98, top=0.91, bottom=0.11, hspace=0.40, wspace=0.30) + for i in np.arange(nModesPlot): + key= list(Modes.keys())[i] + + axes[i].plot(x, Modes[key]['comp'][:,0] ,'-' , label='ux') + axes[i].plot(x, Modes[key]['comp'][:,1] ,'-' , label='uy') + axes[i].plot(x, Modes[key]['comp'][:,2] ,'-' , label='uz') + axes[i].plot(x, Modes[key]['comp'][:,3] ,':' , label='vx') + axes[i].plot(x, Modes[key]['comp'][:,4] ,':' , label='vy') + axes[i].plot(x, Modes[key]['comp'][:,5] ,':' , label='vz') + axes[i].set_xlabel('') + axes[i].set_ylabel('') + axes[i].set_title(Modes[key]['label']) + if i==0: + axes[i].legend() + + + + + # --------------------------------------------------------------------------------} + # --- IO/Converters + # --------------------------------------------------------------------------------{ + def toYAML(self, filename): + if self._FEM is None: + raise Exception('Call `initFEM()` before calling `toYAML`') + subdyntoYAMLSum(self._FEM, filename, more = self.File['OutAll']) + + + def toYAMSData(self, shapes=[0,4], main_axis='z'): + """ + Convert to Data needed to setup a Beam Model in YAMS (see bodies.py in yams) + """ + from welib.mesh.gradient import gradient_regular + + # --- Perform Craig-Bampton reduction, fixing the top node of the beam + # Get beam data frame + df = self.beamDataFrame(equispacing=True) + if np.any(df['y']!=0): + raise NotImplementedError('FASTBeamBody for substructure only support monopile, structure not fully vertical in file: {}'.format(self.File.filename)) + if np.any(df['x']!=0): + raise NotImplementedError('FASTBeamBody for substructure only support monopile, structure not fully vertical in file: {}'.format(self.File.filename)) + + FEM = self.beamFEM(df) + Q_G,_Q_CB, df_G, df_CB, Modes_G, Modes_CB, CB = self.beamModes(nCB=0, FEM=FEM) + + x = df['z'].values + nSpan = len(x) + + # TODO TODO finda way to use these matrices instead of the ones computed with flexibility + #print('CB MM\n',CB['MM']) + #print('CB KK\n',CB['KK']) + + # --- Setup shape functions + if main_axis=='x': + raise NotImplementedError('') + else: + pass + # we need to swap the CB modes + nShapes=len(shapes) + PhiU = np.zeros((nShapes,3,nSpan)) # Shape + PhiV = np.zeros((nShapes,3,nSpan)) # Shape + PhiK = np.zeros((nShapes,3,nSpan)) # Shape + dx=np.unique(np.around(np.diff(x),4)) + if len(dx)>1: + print(x) + print(dx) + raise NotImplementedError() + for iShape, idShape in enumerate(shapes): + if idShape==0: + # shape 0 "ux" (uz in FEM) + PhiU[iShape][0,:] = df_G['G3_uz'].values + PhiV[iShape][0,:] =-df_G['G3_ty'].values + PhiK[iShape][0,:] = gradient_regular(PhiV[iShape][0,:],dx=dx[0],order=4) + elif idShape==1: + # shape 1, "uy" + PhiU[iShape][1,:] = df_G['G2_uy'].values + PhiV[iShape][1,:] = df_G['G2_tz'].values + PhiK[iShape][1,:] = gradient_regular(PhiV[iShape][1,:],dx=dx[0],order=4) + elif idShape==4: + # shape 4, "vy" (vz in FEM) + PhiU[iShape][0,:] = df_G['G6_uy'].values + PhiV[iShape][0,:] = df_G['G6_tz'].values + PhiK[iShape][0,:] = gradient_regular(PhiV[iShape][0,:],dx=dx[0],order=4) + else: + raise NotImplementedError() + + # --- Dictionary structure for YAMS + p=dict() + p['s_span']=x-np.min(x) + p['s_P0']=np.zeros((3,nSpan)) + if main_axis=='z': + p['s_P0'][2,:]=x-np.min(x) + p['r_O'] = (df['x'].values[0], df['y'].values[0], df['z'].values[0]) + p['R_b2g'] = np.eye(3) + p['m'] = df['m'].values + p['EI'] = np.zeros((3,nSpan)) + if main_axis=='z': + p['EI'][0,:]=df['E'].values*df['I'].values + p['EI'][1,:]=df['E'].values*df['I'].values + p['jxxG'] = df['rho']*df['Ip'] # TODO verify + p['s_min'] = p['s_span'][0] + p['s_max'] = p['s_span'][-1] + p['PhiU'] = PhiU + p['PhiV'] = PhiV + p['PhiK'] = PhiK + + # --- Damping + damp_zeta = None + RayleighCoeff = None + DampMat = None + if self.File['GuyanDampMod']==1: + # Rayleigh Damping + RayleighCoeff=self.File['RayleighDamp'] + #if RayleighCoeff[0]==0: + # damp_zeta=omega*RayleighCoeff[1]/2. + elif self.File['GuyanDampMod']==2: + # Full matrix + DampMat = self.File['GuyanDampMatrix'] + DampMat=DampMat[np.ix_(shapes,shapes)] + + return p, damp_zeta, RayleighCoeff, DampMat + + +# --------------------------------------------------------------------------------} +# --- Export of summary file and Misc FEM variables used by SubDyn +# --------------------------------------------------------------------------------{ +def yaml_array(var, M, Fmt='{:15.6e}', comment=''): + M = np.atleast_2d(M) + if len(comment)>0: + s='{}: # {} x {} {}\n'.format(var, M.shape[0], M.shape[1], comment) + else: + s='{}: # {} x {}\n'.format(var, M.shape[0], M.shape[1]) + + if M.shape[0]==1: + if M.shape[1]==0: + s+= ' - [ ]\n' + else: + for l in M: + s+= ' - [' + ','.join([Fmt.format(le) for le in l]) + ',]\n' + else: + for l in M: + s+= ' - [' + ','.join([Fmt.format(le) for le in l]) + ']\n' + s = s.replace('e+','E+').replace('e-','E-') + return s + + + +def subdynPartitionVars(model): + from welib.FEM.fem_elements import idDOF_Leader, idDOF_Fixed, idDOF_Internal + # --- Count nodes per types + nNodes = len(model.Nodes) + nNodes_I = len(model.interfaceNodes) + nNodes_C = len(model.reactionNodes) + nNodes_L = len(model.internalNodes) + + # --- Partition Nodes: Nodes_L = IAll - NodesR + Nodes_I = [n.ID for n in model.interfaceNodes] + Nodes_C = [n.ID for n in model.reactionNodes] + Nodes_R = Nodes_I + Nodes_C + Nodes_L = [n.ID for n in model.Nodes if n.ID not in Nodes_R] + + # --- Count DOFs - NOTE: we count node by node + nDOF___ = sum([len(n.data['DOFs_c']) for n in model.Nodes]) + # Interface DOFs + nDOFI__ = sum([len(n.data['DOFs_c']) for n in model.interfaceNodes]) + nDOFI_B = sum([sum(np.array(n.data['IBC'])==idDOF_Leader) for n in model.interfaceNodes]) + nDOFI_F = sum([sum(np.array(n.data['IBC'])==idDOF_Fixed ) for n in model.interfaceNodes]) + if nDOFI__!=nDOFI_B+nDOFI_F: raise Exception('Wrong distribution of interface DOFs') + # DOFs of reaction nodes + nDOFC__ = sum([len(n.data['DOFs_c']) for n in model.reactionNodes]) + nDOFC_B = sum([sum(np.array(n.data['RBC'])==idDOF_Leader) for n in model.reactionNodes]) + nDOFC_F = sum([sum(np.array(n.data['RBC'])==idDOF_Fixed) for n in model.reactionNodes]) + nDOFC_L = sum([sum(np.array(n.data['RBC'])==idDOF_Internal) for n in model.reactionNodes]) + if nDOFC__!=nDOFC_B+nDOFC_F+nDOFC_L: raise Exception('Wrong distribution of reaction DOFs') + # DOFs of reaction + interface nodes + nDOFR__ = nDOFI__ + nDOFC__ # Total number, used to be called "nDOFR" + # DOFs of internal nodes + nDOFL_L = sum([len(n.data['DOFs_c']) for n in model.internalNodes]) + if nDOFL_L!=nDOF___-nDOFR__: raise Exception('Wrong distribution of internal DOF') + # Total number of DOFs in each category: + nDOF__B = nDOFC_B + nDOFI_B + nDOF__F = nDOFC_F + nDOFI_F + nDOF__L = nDOFC_L + nDOFL_L + + # --- Distibutes the I, L, C nodal DOFs into B, F, L sub-categories + # NOTE: order is importatn for compatibility with SubDyn + IDI__ = [] + IDI_B = [] + IDI_F = [] + for n in model.interfaceNodes: + IDI__ += n.data['DOFs_c'] # NOTE: respects order + IDI_B += [dof for i,dof in enumerate(n.data['DOFs_c']) if n.data['IBC'][i]==idDOF_Leader] + IDI_F += [dof for i,dof in enumerate(n.data['DOFs_c']) if n.data['IBC'][i]==idDOF_Fixed ] + IDI__ = IDI_B+IDI_F + IDC__ = [] + IDC_B = [] + IDC_L = [] + IDC_F = [] + for n in model.reactionNodes: + IDC__ += n.data['DOFs_c'] # NOTE: respects order + IDC_B += [dof for i,dof in enumerate(n.data['DOFs_c']) if n.data['RBC'][i]==idDOF_Leader ] + IDC_L += [dof for i,dof in enumerate(n.data['DOFs_c']) if n.data['RBC'][i]==idDOF_Internal] + IDC_F += [dof for i,dof in enumerate(n.data['DOFs_c']) if n.data['RBC'][i]==idDOF_Fixed ] + IDR__=IDC__+IDI__ + IDL_L = [] + for n in model.internalNodes: + IDL_L += n.data['DOFs_c'] + + # Storing variables similar to SubDyn + SD_Vars={} + SD_Vars['nDOF___']=nDOF___; + SD_Vars['nDOFI__']=nDOFI__; SD_Vars['nDOFI_B']=nDOFI_B; SD_Vars['nDOFI_F']=nDOFI_F; + SD_Vars['nDOFC__']=nDOFC__; SD_Vars['nDOFC_B']=nDOFC_B; SD_Vars['nDOFC_F']=nDOFC_F; SD_Vars['nDOFC_L']=nDOFC_L; + SD_Vars['nDOFR__']=nDOFR__; SD_Vars['nDOFL_L']=nDOFL_L; + SD_Vars['nDOF__B']=nDOF__B; SD_Vars['nDOF__F']=nDOF__F; SD_Vars['nDOF__L']=nDOF__L; + SD_Vars['IDC__']=IDC__; + SD_Vars['IDC_B']=IDC_B; + SD_Vars['IDC_F']=IDC_F; + SD_Vars['IDC_L']=IDC_L; + SD_Vars['IDI__']=IDI__; + SD_Vars['IDR__']=IDR__; + SD_Vars['IDI_B']=IDI_B; + SD_Vars['IDI_F']=IDI_F; + SD_Vars['IDL_L']=IDL_L; + SD_Vars['ID__B']=model.DOFc_Leader + SD_Vars['ID__F']=model.DOFc_Fixed + SD_Vars['ID__L']=model.DOFc_Follower + return SD_Vars + +def subdyntoYAMLSum(model, filename, more=False): + """ + Write a YAML summary file, similar to SubDyn + """ + # --- Helper functions + def nodeID(nodeID): + if hasattr(nodeID,'__len__'): + return [model.Nodes.index(model.getNode(n))+1 for n in nodeID] + else: + return model.Nodes.index(model.getNode(nodeID))+1 + + def elemID(elemID): + #e=model.getElement(elemID) + for ie,e in enumerate(model.Elements): + if e.ID==elemID: + return ie+1 + def elemType(elemType): + from welib.FEM.fem_elements import idMemberBeam, idMemberCable, idMemberRigid + return {'SubDynBeam3d':idMemberBeam, 'SubDynFrame3d':idMemberBeam, 'Beam':idMemberBeam, 'Frame3d':idMemberBeam, + 'SubDynTimoshenko3d':idMemberBeam, + 'SubDynCable3d':idMemberCable, 'Cable':idMemberCable, + 'Rigid':idMemberRigid, + 'SubDynRigid3d':idMemberRigid}[elemType] + + def propID(propID, propset): + prop = model.NodePropertySets[propset] + for ip, p in enumerate(prop): + if p.ID == propID: + return ip+1 + + SD_Vars = subdynPartitionVars(model) + + # --- Helper functions + s='' + s += '#____________________________________________________________________________________________________\n' + s += '# RIGID BODY EQUIVALENT DATA\n' + s += '#____________________________________________________________________________________________________\n' + s0 = 'Mass: {:15.6e} # Total Mass\n'.format(model.M_O[0,0]) + s += s0.replace('e+','E+').replace('e-','E-') + s0 = 'CM_point: [{:15.6e},{:15.6e},{:15.6e},] # Center of mass coordinates (Xcm,Ycm,Zcm)\n'.format(model.center_of_mass[0],model.center_of_mass[1],model.center_of_mass[2]) + s += s0.replace('e+','E+').replace('e-','E-') + s0 = 'TP_point: [{:15.6e},{:15.6e},{:15.6e},] # Transition piece reference point\n'.format(model.refPoint[0],model.refPoint[1],model.refPoint[2]) + s += s0.replace('e+','E+').replace('e-','E-') + s += yaml_array('MRB', model.M_O, comment = 'Rigid Body Equivalent Mass Matrix w.r.t. (0,0,0).') + s += yaml_array('M_P' , model.M_ref,comment = 'Rigid Body Equivalent Mass Matrix w.r.t. TP Ref point') + s += yaml_array('M_G' , model.M_G, comment = 'Rigid Body Equivalent Mass Matrix w.r.t. CM (Xcm,Ycm,Zcm).') + s += '#____________________________________________________________________________________________________\n' + s += '# GUYAN MATRICES at the TP reference point\n' + s += '#____________________________________________________________________________________________________\n' + s += yaml_array('KBBt' , model.KBBt, comment = '') + s += yaml_array('MBBt' , model.MBBt, comment = '') + s += yaml_array('CBBt' , model.CBBt, comment = '(user Guyan Damping + potential joint damping from CB-reduction)') + s += '#____________________________________________________________________________________________________\n' + s += '# SYSTEM FREQUENCIES\n' + s += '#____________________________________________________________________________________________________\n' + s += '#Eigenfrequencies [Hz] for full system, with reaction constraints (+ Soil K/M + SoilDyn K0) \n' + s += yaml_array('Full_frequencies', model.freq) + s += '#Frequencies of Guyan modes [Hz]\n' + s += yaml_array('GY_frequencies', model.f_G) + s += '#Frequencies of Craig-Bampton modes [Hz]\n' + s += yaml_array('CB_frequencies', model.f_CB) + s += '#____________________________________________________________________________________________________\n' + s += '# Internal FEM representation\n' + s += '#____________________________________________________________________________________________________\n' + s += 'nNodes_I: {:7d} # Number of Nodes: "interface" (I)\n'.format(len(model.interfaceNodes)) + s += 'nNodes_C: {:7d} # Number of Nodes: "reactions" (C)\n'.format(len(model.reactionNodes)) + s += 'nNodes_L: {:7d} # Number of Nodes: "internal" (L)\n'.format(len(model.internalNodes)) + s += 'nNodes : {:7d} # Number of Nodes: total (I+C+L)\n'.format(len(model.Nodes)) + if more: + s += 'nDOFI__ : {:7d} # Number of DOFs: "interface" (I__)\n'.format(len(SD_Vars['IDI__'])) + s += 'nDOFI_B : {:7d} # Number of DOFs: "interface" retained (I_B)\n'.format(len(SD_Vars['IDI_B'])) + s += 'nDOFI_F : {:7d} # Number of DOFs: "interface" fixed (I_F)\n'.format(len(SD_Vars['IDI_F'])) + s += 'nDOFC__ : {:7d} # Number of DOFs: "reactions" (C__)\n'.format(len(SD_Vars['IDC__'])) + s += 'nDOFC_B : {:7d} # Number of DOFs: "reactions" retained (C_B)\n'.format(len(SD_Vars['IDC_B'])) + s += 'nDOFC_L : {:7d} # Number of DOFs: "reactions" internal (C_L)\n'.format(len(SD_Vars['IDC_L'])) + s += 'nDOFC_F : {:7d} # Number of DOFs: "reactions" fixed (C_F)\n'.format(len(SD_Vars['IDC_F'])) + s += 'nDOFR__ : {:7d} # Number of DOFs: "intf+react" (__R)\n'.format(len(SD_Vars['IDR__'])) + s += 'nDOFL_L : {:7d} # Number of DOFs: "internal" internal (L_L)\n'.format(len(SD_Vars['IDL_L'])) + s += 'nDOF__B : {:7d} # Number of DOFs: retained (__B)\n'.format(SD_Vars['nDOF__B']) + s += 'nDOF__L : {:7d} # Number of DOFs: internal (__L)\n'.format(SD_Vars['nDOF__L']) + s += 'nDOF__F : {:7d} # Number of DOFs: fixed (__F)\n'.format(SD_Vars['nDOF__F']) + s += 'nDOF_red: {:7d} # Number of DOFs: total\n' .format(SD_Vars['nDOF___']) + s += yaml_array('Nodes_I', nodeID([n.ID for n in model.interfaceNodes]), Fmt='{:7d}', comment='"interface" nodes"'); + s += yaml_array('Nodes_C', nodeID([n.ID for n in model.reactionNodes ]), Fmt='{:7d}', comment='"reaction" nodes"'); + s += yaml_array('Nodes_L', nodeID([n.ID for n in model.internalNodes ]), Fmt='{:7d}', comment='"internal" nodes"'); + if more: + s += yaml_array('DOF_I__', np.array(SD_Vars['IDI__'])+1, Fmt='{:7d}', comment = '"interface" DOFs"') + s += yaml_array('DOF_I_B', np.array(SD_Vars['IDI_B'])+1, Fmt='{:7d}', comment = '"interface" retained DOFs') + s += yaml_array('DOF_I_F', np.array(SD_Vars['IDI_F'])+1, Fmt='{:7d}', comment = '"interface" fixed DOFs') + s += yaml_array('DOF_C__', np.array(SD_Vars['IDC__'])+1, Fmt='{:7d}', comment = '"reaction" DOFs"') + s += yaml_array('DOF_C_B', np.array(SD_Vars['IDC_B'])+1, Fmt='{:7d}', comment = '"reaction" retained DOFs') + s += yaml_array('DOF_C_L', np.array(SD_Vars['IDC_L'])+1, Fmt='{:7d}', comment = '"reaction" internal DOFs') + s += yaml_array('DOF_C_F', np.array(SD_Vars['IDC_F'])+1, Fmt='{:7d}', comment = '"reaction" fixed DOFs') + s += yaml_array('DOF_L_L', np.array(SD_Vars['IDL_L'])+1, Fmt='{:7d}', comment = '"internal" internal DOFs') + s += yaml_array('DOF_R_' , np.array(SD_Vars['IDR__'])+1, Fmt='{:7d}', comment = '"interface&reaction" DOFs') + s += yaml_array('DOF___B', np.array(model.DOFc_Leader )+1, Fmt='{:7d}', comment='all retained DOFs'); + s += yaml_array('DOF___F', np.array(model.DOFc_Fixed )+1, Fmt='{:7d}', comment='all fixed DOFs'); + s += yaml_array('DOF___L', np.array(model.DOFc_Follower)+1, Fmt='{:7d}', comment='all internal DOFs'); + s += '\n' + s += '#Index map from DOF to nodes\n' + s += '# Node No., DOF/Node, NodalDOF\n' + s += 'DOF2Nodes: # {} x 3 (nDOFRed x 3, for each constrained DOF, col1: node index, col2: number of DOF, col3: DOF starting from 1)\n'.format(model.nDOFc) + DOFc2Nodes = model.DOFc2Nodes + for l in DOFc2Nodes: + s +=' - [{:7d},{:7d},{:7d}] # {}\n'.format(l[1]+1, l[2], l[3], l[0]+1 ) + s += '# Node_[#] X_[m] Y_[m] Z_[m] JType_[-] JDirX_[-] JDirY_[-] JDirZ_[-] JStff_[Nm/rad]\n' + s += 'Nodes: # {} x 9\n'.format(len(model.Nodes)) + for n in model.Nodes: + s += ' - [{:7d}.,{:15.3f},{:15.3f},{:15.3f},{:14d}., 0.000000E+00, 0.000000E+00, 0.000000E+00, 0.000000E+00]\n'.format(nodeID(n.ID), n.x, n.y, n.z, int(n.data['Type']) ) + s += '# Elem_[#] Node_1 Node_2 Prop_1 Prop_2 Type Length_[m] Area_[m^2] Dens._[kg/m^3] E_[N/m2] G_[N/m2] shear_[-] Ixx_[m^4] Iyy_[m^4] Jzz_[m^4] T0_[N]\n' + s += 'Elements: # {} x 16\n'.format(len(model.Elements)) + for e in model.Elements: + I = e.inertias + s0=' - [{:7d}.,{:7d}.,{:7d}.,{:7d}.,{:7d}.,{:7d}.,{:15.3f},{:15.3f},{:15.3f},{:15.6e},{:15.6e},{:15.6e},{:15.6e},{:15.6e},{:15.6e},{:15.6e}]\n'.format( + elemID(e.ID), nodeID(e.nodeIDs[0]), nodeID(e.nodeIDs[1]), propID(e.propIDs[0], e.propset), propID(e.propIDs[1], e.propset), elemType(e.data['Type']), + e.length, e.area, e.rho, e.E, e.G, e.kappa, I[0], I[1], I[2], e.T0) + s += s0.replace('e+','E+').replace('e-','E-') + s += '#____________________________________________________________________________________________________\n' + s += '#User inputs\n' + s += '\n' + s += '#Number of properties (NProps):{:6d}\n'.format(len(model.NodePropertySets['Beam'])) + s += '#Prop No YoungE ShearG MatDens XsecD XsecT\n' + for ip,p in enumerate(model.NodePropertySets['Beam']): + s0='#{:8d}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}\n'.format(p.ID, p['E'],p['G'],p['rho'],p['D'],p['t']) + s += s0.replace('e+','E+').replace('e-','E-') + s +='\n' + s += '#No. of Reaction DOFs:{:6d}\n'.format(len(SD_Vars['IDC__']) ) + s += '#React. DOF_ID BC\n' + s += '\n'.join(['#{:10d}{:10s}'.format(idof+1,' Fixed' ) for idof in SD_Vars['IDC_F']]) + s += '\n'.join(['#{:10d}{:10s}'.format(idof+1,' Free' ) for idof in SD_Vars['IDC_L']]) + s += '\n'.join(['#{:10d}{:10s}'.format(idof+1,' Leader') for idof in SD_Vars['IDC_B']]) + s += '\n\n' + s += '#No. of Interface DOFs:{:6d}\n'.format(len(SD_Vars['IDI__'])) + s += '#Interf. DOF_ID BC\n' + s += '\n'.join(['#{:10d}{:10s}'.format(idof+1,' Fixed' ) for idof in SD_Vars['IDI_F']]) + s += '\n'.join(['#{:10d}{:10s}'.format(idof+1,' Leader') for idof in SD_Vars['IDI_B']]) + s += '\n\n' + CM = [] + from welib.yams.utils import identifyRigidBodyMM + for n in model.Nodes: + if 'addedMassMatrix' in n.data: + mass, J_G, ref2COG = identifyRigidBodyMM(n.data['addedMassMatrix']) + CM.append( (n.ID, mass, J_G, ref2COG) ) + s += '#Number of concentrated masses (NCMass):{:6d}\n'.format(len(CM)) + s += '#JointCMas Mass JXX JYY JZZ JXY JXZ JYZ MCGX MCGY MCGZ\n' + for cm in CM: + s0 = '# {:9.0f}.{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}\n'.format( nodeID(cm[0]), cm[1], cm[2][0,0], cm[2][1,1], cm[2][2,2], cm[2][0,1], cm[2][0,2], cm[2][1,2],cm[3][0],cm[3][1],cm[3][2] ) + s += s0.replace('e+','E+').replace('e-','E-') + s += '\n' + #s += '#Number of members 18\n' + #s += '#Number of nodes per member: 2\n' + #s += '#Member I Joint1_ID Joint2_ID Prop_I Prop_J Mass Length Node IDs...\n' + #s += '# 77 61 60 11 11 1.045888E+04 2.700000E+00 19 18\n' + #s += '#____________________________________________________________________________________________________\n' + #s += '#Direction Cosine Matrices for all Members: GLOBAL-2-LOCAL. No. of 3x3 matrices= 18\n' + #s += '#Member I DC(1,1) DC(1,2) DC(1,3) DC(2,1) DC(2,2) DC(2,3) DC(3,1) DC(3,2) DC(3,3)\n' + #s += '# 77 1.000E+00 0.000E+00 0.000E+00 0.000E+00 -1.000E+00 0.000E+00 0.000E+00 0.000E+00 -1.000E+00\n' + s += '#____________________________________________________________________________________________________\n' + s += '#FEM Eigenvectors ({} x {}) [m or rad], full system with reaction constraints (+ Soil K/M + SoilDyn K0)\n'.format(*model.Q.shape) + s += yaml_array('Full_Modes', model.Q) + s += '#____________________________________________________________________________________________________\n' + s += '#CB Matrices (PhiM,PhiR) (reaction constraints applied)\n' + s += yaml_array('PhiM', model.Phi_CB[:,:model.nModesCB] ,comment='(CB modes)') + s += yaml_array('PhiR', model.Phi_G, comment='(Guyan modes)') + s += '\n' + if more: + s += '#____________________________________________________________________________________________________\n' + s += '# ADDITIONAL DEBUGGING INFORMATION\n' + s += '#____________________________________________________________________________________________________\n' + s += '' + e = model.Elements[0] + rho=e.rho + A = e.area + L = e.length + t= rho*A*L + s0 = '{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}{:15.6e}\n'.format(model.gravity,e.area, e.length, e.inertias[0], e.inertias[1], e.inertias[2], e.kappa, e.E, e.G, e.rho, t) + s0 = s0.replace('e+','E+').replace('e-','E-') + s += s0 + s += yaml_array('KeLocal' +str(), model.Elements[0].Ke(local=True)) + + for ie,e in enumerate(model.Elements): + s += yaml_array('DC' +str(ie+1), e.DCM.transpose()) + s += yaml_array('Ke' +str(ie+1), e.Ke()) + s += yaml_array('Me' +str(ie+1), e.Me()) + s += yaml_array('FGe'+str(ie+1), e.Fe_g(model.gravity)) + s += yaml_array('FCe'+str(ie+1), e.Fe_o()) + + s += yaml_array('KeLocal' +str(ie+1), e.Ke(local=True)) + s += yaml_array('MeLocal' +str(ie+1), e.Me(local=True)) + s += yaml_array('FGeLocal'+str(ie+1), e.Fe_g(model.gravity, local=True)) + s += yaml_array('FCeLocal'+str(ie+1), e.Fe_o(local=True)) + + s += '#____________________________________________________________________________________________________\n' + e = model.Elements[0] + s += yaml_array('Ke', e.Ke(local=True), comment='First element stiffness matrix'); # TODO not in local + s += yaml_array('Me', e.Me(local=True), comment='First element mass matrix'); + s += yaml_array('FGe', e.Fe_g(model.gravity,local=True), comment='First element gravity vector'); + s += yaml_array('FCe', e.Fe_o(local=True), comment='First element cable pretension'); + s += '#____________________________________________________________________________________________________\n' + s += '#FULL FEM K and M matrices. TOTAL FEM TDOFs: {}\n'.format(model.nDOF); # NOTE: wrong in SubDyn, should be nDOFc + s += yaml_array('K', model.KK, comment='Stiffness matrix'); + s += yaml_array('M', model.MM, comment='Mass matrix'); + s += '#____________________________________________________________________________________________________\n' + s += '#Gravity and cable loads applied at each node of the system (before DOF elimination with T matrix)\n' + s += yaml_array('FG', model.FF_init, comment=' '); + s += '#____________________________________________________________________________________________________\n' + s += '#Additional CB Matrices (MBB,MBM,KBB) (constraint applied)\n' + s += yaml_array('MBB' , model.MBB, comment=''); + s += yaml_array('MBM' , model.MBM[:,:model.nModesCB], comment=''); + s += yaml_array('CMMdiag', model.CMM, comment='(2 Zeta OmegaM)'); + s += yaml_array('KBB' , model.KBB, comment=''); + s += yaml_array('KMM' , np.diag(model.KMM), comment='(diagonal components, OmegaL^2)'); + s += yaml_array('KMMdiag', np.diag(model.KMM)[:model.nModesCB], comment='(diagonal components, OmegaL^2)'); + s += yaml_array('PhiL' , model.Phi_CB, comment=''); + s += 'PhiLOm2-1: # 18 x 18 \n' + s += 'KLL^-1: # 18 x 18 \n' + s += '#____________________________________________________________________________________________________\n' + s += yaml_array('T_red', model.T_c, Fmt = '{:9.2e}', comment='(Constraint elimination matrix)'); + s += 'AA: # 16 x 16 (State matrix dXdx)\n' + s += 'BB: # 16 x 48 (State matrix dXdu)\n' + s += 'CC: # 6 x 16 (State matrix dYdx)\n' + s += 'DD: # 6 x 48 (State matrix dYdu)\n' + s += '#____________________________________________________________________________________________________\n' + s += yaml_array('TI', model.T_refPoint, Fmt = '{:9.2e}',comment='(TP refpoint Transformation Matrix TI)'); + if filename is not None: + with open(filename, 'w') as f: + f.write(s) + + + diff --git a/pydatview/io/__init__.py b/pydatview/io/__init__.py index 740c706..5eed528 100644 --- a/pydatview/io/__init__.py +++ b/pydatview/io/__init__.py @@ -1,4 +1,4 @@ -from .file import File, WrongFormatError, BrokenFormatError, FileNotFoundError, EmptyFileError +from .file import File, WrongFormatError, BrokenFormatError, FileNotFoundError, EmptyFileError, OptionalImportError from .file_formats import FileFormat, isRightFormat import sys import os diff --git a/pydatview/io/converters.py b/pydatview/io/converters.py new file mode 100644 index 0000000..990d1e4 --- /dev/null +++ b/pydatview/io/converters.py @@ -0,0 +1,64 @@ +import os + + +# -------------------------------------------------------------------------------- +# --- Writing pandas DataFrame to different formats +# -------------------------------------------------------------------------------- +# The + +def writeFileDataFrames(fileObject, writer, extension='.conv', filename=None, **kwargs): + """ + From a fileObejct, extract dataframes and write them to disk. + + - fileObject: object inheriting from weio.File with at least + - the attributes .filename + - the method .toDataFrame() + - writer: function with the interface: writer ( dataframe, filename, **kwargs ) + """ + if filename is None: + base, _ = os.path.splitext(fileObject.filename) + filename = base + extension + else: + base, ext = os.path.splitext(filename) + if len(ext)!=0: + extension = ext + if filename == fileObject.filename: + raise Exception('Not overwritting {}. Specify a filename or an extension.'.format(filename)) + + dfs = fileObject.toDataFrame() + if isinstance(dfs, dict): + for name,df in dfs.items(): + filename = base + name + extension + if filename == fileObject.filename: + raise Exception('Not overwritting {}. Specify a filename or an extension.'.format(filename)) + writeDataFrame(df=df, writer=writer, filename=filename, **kwargs) + else: + writeDataFrame(df=dfs, writer=writer, filename=filename, **kwargs) + + +def writeDataFrame(df, writer, filename, **kwargs): + """ + Write a dataframe to disk based on a "writer" function. + - df: pandas dataframe + - writer: function with the interface: writer ( dataframe, filename, **kwargs ) + - filename: filename + """ + writer(df, filename, **kwargs) + +# --- Low level writers +def dataFrameToCSV(df, filename, sep=',', index=False, **kwargs): + base, ext = os.path.splitext(filename) + if len(ext)==0: + filename = base='.csv' + df.to_csv(filename, sep=sep, index=index, **kwargs) + +def dataFrameToOUTB(df, filename, **kwargs): + from .fast_output_file import writeDataFrame as writeDataFrameToOUTB + base, ext = os.path.splitext(filename) + if len(ext)==0: + filename = base='.outb' + writeDataFrameToOUTB(df, filename, binary=True) + +def dataFrameToParquet(df, filename, **kwargs): + df.to_parquet(path=filename, **kwargs) + diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index 0a8de62..1c3005b 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -9,11 +9,21 @@ - data, info = def load_binary_output(filename, use_buffer=True) - def writeDataFrame(df, filename, binary=True) - def writeBinary(fileName, channels, chanNames, chanUnits, fileID=2, descStr='') + +NOTE: + - load_binary and writeBinary are not "fully reversible" for now. + Some small numerical errors are introduced in the conversion. + Some of the error is likely due to the fact that Python converts to "int" and "float" (double). + Maybe all the operations should be done in single. I tried but failed. + I simply wonder if the operation is perfectly reversible. + + """ from itertools import takewhile import numpy as np import pandas as pd import struct +import ctypes import os import re try: @@ -115,7 +125,12 @@ def readline(iLine): self.info['attribute_units']=readline(3).replace('sec','s').split() self.info['attribute_names']=self.data.columns.values else: - self.data, self.info = load_output(self.filename) + if isBinary(self.filename): + self.data, self.info = load_binary_output(self.filename) + self['binary']=True + else: + self.data, self.info = load_ascii_output(self.filename) + self['binary']=False except MemoryError as e: raise BrokenReaderError('FAST Out File {}: Memory error encountered\n{}'.format(self.filename,e)) except Exception as e: @@ -127,13 +142,13 @@ def readline(iLine): self.info['attribute_units'] = [re.sub(r'[()\[\]]','',u) for u in self.info['attribute_units']] - def _write(self): - if self['binary']: - channels = self.data - chanNames = self.info['attribute_names'] - chanUnits = self.info['attribute_units'] - descStr = self.info['description'] - writeBinary(self.filename, channels, chanNames, chanUnits, fileID=2, descStr=descStr) + def _write(self, binary=None, fileID=4): + if binary is None: + binary = self['binary'] + + if binary: + # NOTE: user provide a filename, we allow overwrite + self.toOUTB(filename=self.filename, fileID=fileID, noOverWrite=False) else: # ascii output with open(self.filename,'w') as f: @@ -174,38 +189,59 @@ def __repr__(self): s+='and keys: {}\n'.format(self.keys()) return s + # -------------------------------------------------------------------------------- + # --- Converters + # -------------------------------------------------------------------------------- + def toOUTB(self, filename=None, extension='.outb', fileID=4, noOverWrite=True, **kwargs): + #NOTE: we override the File class here + if filename is None: + base, _ = os.path.splitext(self.filename) + filename = base + extension + else: + base, ext = os.path.splitext(filename) + if len(ext)!=0: + extension = ext + if (filename==self.filename) and noOverWrite: + raise Exception('Not overwritting {}. Specify a filename or an extension.'.format(filename)) + + # NOTE: fileID=2 will chop the channels name of long channels use fileID4 instead + channels = self.data + chanNames = self.info['attribute_names'] + chanUnits = self.info['attribute_units'] + descStr = self.info['description'] + if isinstance(descStr, list): + descStr=(''.join(descStr[:2])).replace('\n','') + writeBinary(filename, channels, chanNames, chanUnits, fileID=fileID, descStr=descStr) + + # -------------------------------------------------------------------------------- # --- Helper low level functions # -------------------------------------------------------------------------------- -def load_output(filename): - """Load a FAST binary or ascii output file - - Parameters - ---------- - filename : str - filename - - Returns - ------- - data : ndarray - data values - info : dict - info containing: - - name: filename - - description: description of dataset - - attribute_names: list of attribute names - - attribute_units: list of attribute units - """ - - assert os.path.isfile(filename), "File, %s, does not exists" % filename +def isBinary(filename): with open(filename, 'r') as f: try: - f.readline() + # first try to read as string + l = f.readline() + # then look for weird characters + for c in l: + code = ord(c) + if code<10 or (code>14 and code<31): + return True + return False except UnicodeDecodeError: - return load_binary_output(filename) - return load_ascii_output(filename) + return True + + + + + +def load_ascii_output(filename, method='numpy'): + + + if method in ['forLoop','pandas']: + from .file import numberOfLines + nLines = numberOfLines(filename, method=2) -def load_ascii_output(filename): with open(filename) as f: info = {} info['name'] = os.path.splitext(os.path.basename(filename))[0] @@ -224,12 +260,42 @@ def load_ascii_output(filename): info['description'] = header info['attribute_names'] = l.split() info['attribute_units'] = [unit[1:-1] for unit in f.readline().split()] - # --- - # Data, up to end of file or empty line (potential comment line at the end) -# data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(np.float) - # --- - data = np.loadtxt(f, comments=('This')) # Adding "This" for the Hydro Out files.. - return data, info + + nHeader = len(header)+1 + nCols = len(info['attribute_names']) + + if method=='numpy': + # The most efficient, and will remove empty lines and the lines that starts with "This" + # ("This" is found at the end of some Hydro Out files..) + data = np.loadtxt(f, comments=('This')) + + elif method =='pandas': + # Could probably be made more efficient, but + f.close() + nRows = nLines-nHeader + sep=r'\s+' + cols= ['C{}'.format(i) for i in range(nCols)] + df = pd.read_csv(filename, sep=sep, header=0, skiprows=nHeader, names=cols, dtype=float, na_filter=False, nrows=nRows, engine='pyarrow'); print(df) + data=df.values + + elif method == 'forLoop': + # The most inefficient + nRows = nLines-nHeader + sep=r'\s+' + data = np.zeros((nRows, nCols)) + for i in range(nRows): + l = f.readline().strip() + sp = np.array(l.split()).astype(np.float) + data[i,:] = sp[:nCols] + + elif method == 'listCompr': + # --- Method 4 - List comprehension + # Data, up to end of file or empty line (potential comment line at the end) + data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(np.float) + else: + raise NotImplementedError() + + return data, info def load_binary_output(filename, use_buffer=True): @@ -244,8 +310,15 @@ def load_binary_output(filename, use_buffer=True): % % Edited for FAST v7.02.00b-bjj 22-Oct-2012 """ - def fread(fid, n, type): - fmt, nbytes = {'uint8': ('B', 1), 'int16':('h', 2), 'int32':('i', 4), 'float32':('f', 4), 'float64':('d', 8)}[type] + StructDict = { + 'uint8': ('B', 1, np.uint8), + 'int16':('h', 2, np.int16), + 'int32':('i', 4, np.int32), + 'float32':('f', 4, np.float32), + 'float64':('d', 8, np.float64)} + def fread(fid, n, dtype): + fmt, nbytes, npdtype = StructDict[dtype] + #return np.array(struct.unpack(fmt * n, fid.read(nbytes * n)), dtype=npdtype) return struct.unpack(fmt * n, fid.read(nbytes * n)) def freadRowOrderTableBuffered(fid, n, type_in, nCols, nOff=0, type_out='float64'): @@ -305,11 +378,11 @@ def freadRowOrderTableBuffered(fid, n, type_in, nCols, nOff=0, type_out='float64 NT = fread(fid, 1, 'int32')[0] #; % The number of time steps, INT(4) if FileID == FileFmtID_WithTime: - TimeScl = fread(fid, 1, 'float64') #; % The time slopes for scaling, REAL(8) - TimeOff = fread(fid, 1, 'float64') #; % The time offsets for scaling, REAL(8) + TimeScl = fread(fid, 1, 'float64')[0] # The time slopes for scaling, REAL(8) + TimeOff = fread(fid, 1, 'float64')[0] # The time offsets for scaling, REAL(8) else: - TimeOut1 = fread(fid, 1, 'float64') #; % The first time in the time series, REAL(8) - TimeIncr = fread(fid, 1, 'float64') #; % The time increment, REAL(8) + TimeOut1 = fread(fid, 1, 'float64')[0] # The first time in the time series, REAL(8) + TimeIncr = fread(fid, 1, 'float64')[0] # The time increment, REAL(8) if FileID == FileFmtID_NoCompressWithoutTime: ColScl = np.ones ((NumOutChans, 1)) # The channel slopes for scaling, REAL(4) @@ -428,7 +501,7 @@ def writeBinary(fileName, channels, chanNames, chanUnits, fileID=4, descStr=''): time = channels[:,iTime] timeStart = time[0] - timeIncr = time[1]-time[0] + timeIncr = (time[-1]-time[0])/(nT-1) dataWithoutTime = channels[:,1:] # Compute data range, scaling and offsets to convert to int16 @@ -492,13 +565,33 @@ def writeBinary(fileName, channels, chanNames, chanUnits, fileID=4, descStr=''): ordunit = [ord(char) for char in unit] + [32]*(nChar-len(unit)) fid.write(struct.pack('@'+str(nChar)+'B', *ordunit)) - # Pack data - packedData=np.zeros((nT, nChannels), dtype=np.int16) + # --- Pack and write data + # Method 1 + #packedData=np.zeros((nT, nChannels), dtype=np.int16) + #for iChan in range(nChannels): + # packedData[:,iChan] = np.clip( ColScl[iChan]*dataWithoutTime[:,iChan]+ColOff[iChan], int16Min, int16Max) + #packedData = packedData.ravel() + ## NOTE: the *packedData converts to a tuple before passing to struct.pack + ## which is inefficient + #fid.write(struct.pack('@{}h'.format(packedData.size), *packedData)) + + # --- Method 2 + #packedData=np.zeros((nT, nChannels), dtype=np.int16) + #for iChan in range(nChannels): + # packedData[:,iChan] = np.clip( ColScl[iChan]*dataWithoutTime[:,iChan]+ColOff[iChan], int16Min, int16Max) + #packedData = packedData.ravel() + ## Here we use slice assignment + #buf = (ctypes.c_int16 * len(packedData))() + #buf[:] = packedData + #fid.write(buf) + + # --- Method 3 use packedData as slice directly + packedData = (ctypes.c_int16 * (nT*nChannels))() for iChan in range(nChannels): - packedData[:,iChan] = np.clip( ColScl[iChan]*dataWithoutTime[:,iChan]+ColOff[iChan], int16Min, int16Max) + packedData[iChan::nChannels] = np.clip( ColScl[iChan]*dataWithoutTime[:,iChan]+ColOff[iChan], int16Min, int16Max).astype(np.int16) + fid.write(packedData) + - # Write data - fid.write(struct.pack('@{}h'.format(packedData.size), *packedData.flatten())) fid.close() def writeDataFrame(df, filename, binary=True): @@ -540,5 +633,8 @@ def writeDataFrame(df, filename, binary=True): B=FASTOutputFile('tests/example_files/FASTOutBin.outb') df=B.toDataFrame() B.writeDataFrame(df, 'tests/example_files/FASTOutBin_OUT.outb') + B.toOUTB(extension='.dat.outb') + B.toParquet() + B.toCSV() diff --git a/pydatview/io/file.py b/pydatview/io/file.py index f9211bc..b942a78 100644 --- a/pydatview/io/file.py +++ b/pydatview/io/file.py @@ -13,6 +13,9 @@ class BrokenFormatError(Exception): class BrokenReaderError(Exception): pass +class OptionalImportError(Exception): + pass + try: #Python3 FileNotFoundError=FileNotFoundError except NameError: # Python2 @@ -37,13 +40,13 @@ def read(self, filename=None, **kwargs): # Calling children function self._read(**kwargs) - def write(self, filename=None): + def write(self, filename=None, **kwargs): if filename: self.filename = filename if not self.filename: raise Exception('No filename provided') # Calling children function - self._write() + self._write(**kwargs) def toDataFrame(self): return self._toDataFrame() @@ -74,8 +77,32 @@ def encoding(self): # --------------------------------------------------------------------------------} - # --- Helper methods + # --- Conversions # --------------------------------------------------------------------------------{ + def toCSV(self, filename=None, extension='.csv', **kwargs): + """ By default, writes dataframes to CSV + Keywords arguments are the same as pandas DataFrame to_csv: + - sep: separator + - index: write index or not + - etc. + """ + from .converters import writeFileDataFrames + from .converters import dataFrameToCSV + writeFileDataFrames(self, dataFrameToCSV, filename=filename, extension=extension, **kwargs) + + def toOUTB(self, filename=None, extension='.outb', **kwargs): + """ write to OUTB""" + from .converters import writeFileDataFrames + from .converters import dataFrameToOUTB + writeFileDataFrames(self, dataFrameToOUTB, filename=filename, extension=extension, **kwargs) + + + def toParquet(self, filename=None, extension='.parquet', **kwargs): + """ write to OUTB""" + from .converters import writeFileDataFrames + from .converters import dataFrameToParquet + writeFileDataFrames(self, dataFrameToParquet, filename=filename, extension=extension, **kwargs) + # -------------------------------------------------------------------------------- # --- Sub class methods @@ -169,6 +196,23 @@ def isBinary(filename): except UnicodeDecodeError: return True + +def numberOfLines(filename, method=1): + + if method==1: + return sum(1 for i in open(filename, 'rb')) + + elif method==2: + def blocks(files, size=65536): + while True: + b = files.read(size) + if not b: break + yield b + with open(filename, "r",encoding="utf-8",errors='ignore') as f: + return sum(bl.count("\n") for bl in blocks(f)) + else: + raise NotImplementedError() + def ascii_comp(file1,file2,bDelete=False): """ Compares two ascii files line by line. Comparison is done ignoring multiple white spaces for now""" diff --git a/pydatview/io/matlabmat_file.py b/pydatview/io/matlabmat_file.py new file mode 100644 index 0000000..e1aff75 --- /dev/null +++ b/pydatview/io/matlabmat_file.py @@ -0,0 +1,112 @@ +""" +Input/output class for the matlab .mat fileformat +""" +import numpy as np +import pandas as pd +import os +import scipy.io + +try: + from .file import File, WrongFormatError, BrokenFormatError +except: + File=dict + EmptyFileError = type('EmptyFileError', (Exception,),{}) + WrongFormatError = type('WrongFormatError', (Exception,),{}) + BrokenFormatError = type('BrokenFormatError', (Exception,),{}) + +class MatlabMatFile(File): + """ + Read/write a mat file. The object behaves as a dictionary. + + Main methods + ------------ + - read, write, toDataFrame, keys + + Examples + -------- + f = MatlabMatFile('file.mat') + print(f.keys()) + print(f.toDataFrame().columns) + + """ + + @staticmethod + def defaultExtensions(): + """ List of file extensions expected for this fileformat""" + return ['.mat'] + + @staticmethod + def formatName(): + """ Short string (~100 char) identifying the file format""" + return 'Matlab mat file' + + @staticmethod + def priority(): return 60 # Priority in weio.read fileformat list between 0=high and 100:low + + def __init__(self, filename=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + + mfile = scipy.io.loadmat(self.filename) + import pdb; pdb.set_trace() + + def write(self, filename=None): + """ Rewrite object to file, or write object to `filename` if provided """ + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + #with open(self.filename,'w') as f: + # f.write(self.toString) + raise NotImplementedError() + + def toDataFrame(self): + """ Returns object into one DataFrame, or a dictionary of DataFrames""" + # --- Example (returning one DataFrame): + # return pd.DataFrame(data=np.zeros((10,2)),columns=['Col1','Col2']) + # --- Example (returning dict of DataFrames): + #dfs={} + #cols=['Alpha_[deg]','Cl_[-]','Cd_[-]','Cm_[-]'] + #dfs['Polar1'] = pd.DataFrame(data=..., columns=cols) + #dfs['Polar1'] = pd.DataFrame(data=..., columns=cols) + # return dfs + raise NotImplementedError() + + # --- Optional functions + def __repr__(self): + """ String that is written to screen when the user calls `print()` on the object. + Provide short and relevant information to save time for the user. + """ + s='<{} object>:\n'.format(type(self).__name__) + s+='|Main attributes:\n' + s+='| - filename: {}\n'.format(self.filename) + # --- Example printing some relevant information for user + #s+='|Main keys:\n' + #s+='| - ID: {}\n'.format(self['ID']) + #s+='| - data : shape {}\n'.format(self['data'].shape) + s+='|Main methods:\n' + s+='| - read, write, toDataFrame, keys' + return s + + def toString(self): + """ """ + s='' + return s + + + diff --git a/pydatview/io/tdms_file.py b/pydatview/io/tdms_file.py index 1d4cae1..8bffff9 100644 --- a/pydatview/io/tdms_file.py +++ b/pydatview/io/tdms_file.py @@ -3,11 +3,12 @@ import os try: - from .file import File, WrongFormatError, BrokenFormatError + from .file import File, WrongFormatError, BrokenFormatError, OptionalImportError except: File = dict class WrongFormatError(Exception): pass class BrokenFormatError(Exception): pass + class OptionalImportError(Exception): pass class TDMSFile(File): @@ -40,7 +41,7 @@ def read(self, filename=None, **kwargs): try: from nptdms import TdmsFile except: - raise Exception('Install the library nptdms to read this file') + raise OptionalImportError('Install the library nptdms to read this file') fh = TdmsFile(self.filename, read_metadata_only=False) # --- OLD, using some kind of old version of tdms and probably specific to one file diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index c425c03..7000520 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -630,7 +630,7 @@ def leq(PD, m, method=None): except ModuleNotFoundError: print('[INFO] module fatpack not installed, default to windap method for equivalent load') method='rainflow_windap' - v=equivalent_load(PD.x, PD.y, m=m, Teq=1, nBins=100, method=method) + v=equivalent_load(PD.x, PD.y, m=m, Teq=1, bins=100, method=method) return v,pretty_num(v) def Info(PD,var): diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index 0b55043..a7a2b1f 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -31,7 +31,7 @@ __all__ = ['rainflow_astm', 'rainflow_windap','eq_load','eq_load_and_cycles','cycle_matrix','cycle_matrix2'] -def equivalent_load(time, signal, m=3, Teq=1, nBins=100, method='rainflow_windap'): +def equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap', meanBin=True, binStartAt0=False): """Equivalent load calculation Calculate the equivalent loads for a list of Wohler exponent @@ -41,9 +41,14 @@ def equivalent_load(time, signal, m=3, Teq=1, nBins=100, method='rainflow_windap time : array-like, the time values corresponding to the signal (s) signals : array-like, the load signal m : Wohler exponent (default is 3) - Teq : The equivalent period (Default 1Hz) - nBins : Number of bins in rainflow count histogram + Teq : The equivalent period (Default 1, for 1Hz) + bins : Number of bins in rainflow count histogram method: 'rainflow_windap, rainflow_astm, fatpack + meanBin: if True, use the mean of the ranges within a bin (recommended) + otherwise use the middle of the bin (not recommended). + binStartAt0: if True bins start at zero. Otherwise, start a lowest range + + Returns ------- @@ -61,11 +66,10 @@ def equivalent_load(time, signal, m=3, Teq=1, nBins=100, method='rainflow_windap neq = T/Teq # number of equivalent periods - rainflow_func_dict = {'rainflow_windap':rainflow_windap, 'rainflow_astm':rainflow_astm} if method in rainflow_func_dict.keys(): # Call wetb function for one m - Leq = eq_load(signal, m=[m], neq=neq, no_bins=nBins, rainflow_func=rainflow_func_dict[method])[0][0] - + #Leq = eq_load(signal, m=[m], neq=neq, no_bins=bins, rainflow_func=rainflow_func_dict[method])[0][0] + N, S = find_range_count(signal, bins=bins, method=method, meanBin=meanBin) elif method=='fatpack': import fatpack # find rainflow ranges @@ -74,18 +78,108 @@ def equivalent_load(time, signal, m=3, Teq=1, nBins=100, method='rainflow_windap except IndexError: # Currently fails for constant signal return np.nan - # find range count and bin - Nrf, Srf = fatpack.find_range_count(ranges, nBins) - # get DEL - DELs = Srf**m * Nrf / neq - Leq = DELs.sum() ** (1/m) + + # --- Legacy fatpack + # if (not binStartAt0) and (not meanBin): + # N, S = fatpack.find_range_count(ranges, bins) + # --- Setup bins + # If binStartAt0 is True, the same bins as WINDAP are used + bins = create_bins(ranges, bins, binStartAt0=binStartAt0) + # --- Using bin_count to get value at center of bins + N, S = bin_count(ranges, bins, meanBin=meanBin) else: raise NotImplementedError(method) + # get DEL + DELs = S**m * N / neq + Leq = DELs.sum() ** (1/m) return Leq +def find_range_count(signal, bins, method='rainflow_windap', meanBin=True, binStartAt0=True): + """ + Returns number of cycles `N` for each range range `S` + Equidistant bins are setup based on the min/max of the signal. + INPUTS: + - signal: array + - bins : 1d-array, int + If bins is a sequence, left edges (and the rightmost edge) of the bins. + If bins is an int, a sequence is created dividing the range `min`--`max` of signal into `bins` number of equally sized bins. + OUTPUTS: + - N: number of cycles for each bin + - S: Ranges for each bin + S is either the center of the bin (meanBin=False) + or + S is the mean of the ranges within this bin (meanBin=True) + """ + + rainflow_func = rainflow_func_dict[method] + N, S, S_bin_edges, _, _ = cycle_matrix(signal, ampl_bins=bins, mean_bins=1, rainflow_func=rainflow_func, binStartAt0=binStartAt0) + S_bin_edges = S_bin_edges.flatten() + N = N.flatten() + S = S.flatten() + S_mid = (S_bin_edges[:-1] + S_bin_edges[1:]) / 2 + if not meanBin: + S=S_mid + # Remove NaN + b = np.isnan(S) + S[b] = 0 + N[b] = 0 + return N, S + +def create_bins(x, bins, binStartAt0=False): + """ + Equidistant bins are setup based on the min/max of the x, unless the user provided the bins as a sequence. + INPUTS: + - x: array + - bins : 1d-array, int + If bins is a sequence, left edges (and the rightmost edge) of the bins. + If bins is an int, a sequence is created dividing the range `min`--`max` of x into `bins` number of equally sized bins. + OUTPUTS: + - bins: + """ + if isinstance(bins, int): + xmax = np.max(x) + xmin, xmax = np.min(x), np.max(x) + if binStartAt0: + xmin = 0 + else: + xmin = np.min(x) + if xmin==xmax: + # I belive that's what's done by histogram. double check + xmin=xmin-0.5 + xmax=xmax+0.5 + bins = np.linspace(xmin, xmax, num=bins + 1) + return bins + + +def bin_count(x, bins, meanBin=True): + """ + Return counts of x within bins + """ + if not meanBin: + # Use the middle of the bin + N, bns = np.histogram(x, bins=bins) + S = bns[:-1] + np.diff(bns) / 2. + else: + bins = create_bins(x, bins, binStartAt0=False) + import pandas as pd + df = pd.DataFrame(data=x, columns=['x']) + xmid = (bins[:-1]+bins[1:])/2 + df['x_mid']= pd.cut(df['x'], bins= bins, labels = xmid ) # Adding a column that has bin attribute + df2 = df.groupby('x_mid').mean() # Average by bin + df['N'] = 1 + dfCount = df[['N','x_mid']].groupby('x_mid').sum() + df2['N'] = dfCount['N'] + # Just in case some bins are missing (will be nan) + df2 = df2.reindex(xmid) + df2 = df2.fillna(0) + S = df2['x'].values + N = df2['N'].values + return N, S + + def check_signal(signal): @@ -293,11 +387,12 @@ def eq_load_and_cycles(signals, no_bins=46, m=[3, 4, 6, 8, 10, 12], neq=[10 ** 6 cycles, ampl_bin_mean = cycles.flatten(), ampl_bin_mean.flatten() with warnings.catch_warnings(): warnings.simplefilter("ignore") + #DEL = [[( (cycles * ampl_bin_mean ** _m) / _neq) for _m in np.atleast_1d(m)] for _neq in np.atleast_1d(neq)] eq_loads = [[((np.nansum(cycles * ampl_bin_mean ** _m) / _neq) ** (1. / _m)) for _m in np.atleast_1d(m)] for _neq in np.atleast_1d(neq)] return eq_loads, cycles, ampl_bin_mean, ampl_bin_edges -def cycle_matrix(signals, ampl_bins=10, mean_bins=10, rainflow_func=rainflow_windap): +def cycle_matrix(signals, ampl_bins=10, mean_bins=10, rainflow_func=rainflow_windap, binStartAt0=True): """Markow load cycle matrix Calculate the Markow load cycle matrix @@ -315,6 +410,8 @@ def cycle_matrix(signals, ampl_bins=10, mean_bins=10, rainflow_func=rainflow_win if array-like, the bin edges for mea rainflow_func : {rainflow_windap, rainflow_astm}, optional The rainflow counting function to use (default is rainflow_windap) + binStartAt0 : boolean + Start the bins at 0. Otherwise, start at the min of ranges Returns ------- @@ -343,7 +440,7 @@ def cycle_matrix(signals, ampl_bins=10, mean_bins=10, rainflow_func=rainflow_win ampls, means = rainflow_func(signals[:]) weights = np.ones_like(ampls) if isinstance(ampl_bins, int): - ampl_bins = np.linspace(0, 1, num=ampl_bins + 1) * ampls[weights>0].max() + ampl_bins = create_bins(ampls[weights>0], ampl_bins, binStartAt0=binStartAt0) cycles, ampl_edges, mean_edges = np.histogram2d(ampls, means, [ampl_bins, mean_bins], weights=weights) with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -797,6 +894,7 @@ def pair_range_amplitude_mean(x): # cpdef pair_range(np.ndarray[long,ndim=1] x return ampl_mean +rainflow_func_dict = {'rainflow_windap':rainflow_windap, 'rainflow_astm':rainflow_astm} # --------------------------------------------------------------------------------} @@ -823,7 +921,7 @@ def test_leq_1hz(self): # mean value of the signal shouldn't matter signal = amplitude * np.sin(time) + 5 r_eq_1hz = eq_load(signal, no_bins=1, m=m, neq=neq)[0] - r_eq_1hz_expected = ((2*nr_periods*amplitude**m)/neq)**(1/m) + r_eq_1hz_expected = 2*((nr_periods*amplitude**m)/neq)**(1/m) np.testing.assert_allclose(r_eq_1hz, r_eq_1hz_expected) # sine signal with 20 periods (40 peaks) @@ -833,7 +931,7 @@ def test_leq_1hz(self): # mean value of the signal shouldn't matter signal = amplitude * np.sin(time) + 9 r_eq_1hz2 = eq_load(signal, no_bins=1, m=m, neq=neq)[0] - r_eq_1hz_expected2 = ((2*nr_periods*amplitude**m)/neq)**(1/m) + r_eq_1hz_expected2 = 2*((nr_periods*amplitude**m)/neq)**(1/m) np.testing.assert_allclose(r_eq_1hz2, r_eq_1hz_expected2) # 1hz equivalent should be independent of the length of the signal @@ -902,6 +1000,149 @@ def test_eq_load_basic(self): # print (cycle_matrix([(.5, signal1), (.5, signal2)], 4, 8, rainflow_func=rainflow_astm)) + def test_equivalent_load(self): + """ Higher level interface """ + try: + import fatpack + hasFatpack=True + except: + hasFatpack=False + dt = 0.1 + f0 = 1 ; + A = 5 ; + t=np.arange(0,10,dt); + y=A*np.sin(2*np.pi*f0*t) + + Leq = equivalent_load(t, y, m=10, bins=100, method='rainflow_windap') + np.testing.assert_almost_equal(Leq, 9.4714702, 3) + + Leq = equivalent_load(t, y, m=1, bins=100, method='rainflow_windap') + np.testing.assert_almost_equal(Leq, 9.4625320, 3) + + Leq = equivalent_load(t, y, m=4, bins=10, method='rainflow_windap') + np.testing.assert_almost_equal(Leq, 9.420937, 3) + + + if hasFatpack: + Leq = equivalent_load(t, y, m=4, bins=10, method='fatpack', binStartAt0=False, meanBin=False) + np.testing.assert_almost_equal(Leq, 9.584617089, 3) + + Leq = equivalent_load(t, y, m=4, bins=1, method='fatpack', binStartAt0=False, meanBin=False) + np.testing.assert_almost_equal(Leq, 9.534491302, 3) + + + + def test_equivalent_load_sines(self): + # Check analytical formulae for sine of various frequencies + # See welib.tools.examples.Example_Fatigue.py + try: + import fatpack + hasFatpack=True + except: + hasFatpack=False + + # --- Dependency on frequency + m = 2 # Wohler slope + A = 3 # Amplitude + nT = 100 # Number of periods + nPerT = 100 # Number of points per period + Teq = 1 # Equivalent period [s] + nBins = 10 # Number of bins + + vf =np.linspace(0.1,10,21) + vT = 1/vf + T_max=np.max(vT*nT) + vomega =vf*2*np.pi + Leq1 = np.zeros_like(vomega) + Leq2 = np.zeros_like(vomega) + Leq_ref = np.zeros_like(vomega) + for it, (T,omega) in enumerate(zip(vT,vomega)): + # --- Option 1 - Same number of periods + time = np.linspace(0, nT*T, nPerT*nT+1) + signal = A * np.sin(omega*time) # Mean does not matter + T_all=time[-1] + Leq1[it] = equivalent_load(time, signal, m=m, Teq=Teq, bins=nBins, method='rainflow_windap') + if hasFatpack: + Leq2[it] = equivalent_load(time, signal, m=m, Teq=Teq, bins=nBins, method='fatpack', binStartAt0=False) + Leq_ref = 2*A*(vf*Teq)**(1/m) + np.testing.assert_array_almost_equal( Leq1/A, Leq_ref/A, 2) + if hasFatpack: + np.testing.assert_array_almost_equal(Leq2/A, Leq_ref/A, 2) + #import matplotlib.pyplot as plt + #fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) + #fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) + #ax.plot(vf, Leq_ref/A, 'kd' , label ='Theory') + #ax.plot(vf, Leq1 /A, 'o' , label ='Windap m={}'.format(m)) + #if hasFatpack: + # ax.plot(vf, Leq2/A, 'k.' , label ='Fatpack') + #ax.legend() + #plt.show() + + def test_equivalent_load_sines_sum(self): + # --- Sum of two sinusoids + try: + import fatpack + hasFatpack=True + except: + hasFatpack=False + bs0 = True # Bin Start at 0 + m = 2 # Wohler slope + nPerT = 100 # Number of points per period + Teq = 1 # Equivalent period [s] + nBins = 10 # Number of bins + nT1 = 10 # Number of periods + nT2 = 20 # Number of periods + T1 = 10 + T2 = 5 + A1 = 3 # Amplitude + A2 = 5 # Amplitude + # --- Signals + time1 = np.linspace(0, nT1*T1, nPerT*nT1+1) + time2 = np.linspace(0, nT2*T2, nPerT*nT2+1) + signal1 = A1 * np.sin(2*np.pi/T1*time1) + signal2 = A2 * np.sin(2*np.pi/T2*time2) + # --- Individual Leq + #print('----------------- SIGNAL 1') + DEL1 = (2*A1)**m * nT1/time1[-1] + Leq_th = (DEL1)**(1/m) + Leq1 = equivalent_load(time1, signal1, m=m, Teq=Teq, bins=nBins, method='rainflow_windap', binStartAt0=bs0) + if hasFatpack: + Leq2 = equivalent_load(time1, signal1, m=m, Teq=Teq, bins=nBins, method='fatpack', binStartAt0=bs0) + np.testing.assert_array_almost_equal(Leq2, Leq_th, 3) + np.testing.assert_array_almost_equal(Leq1, Leq_th, 1) + #print('>>> Leq1 ',Leq1) + #print('>>> Leq2 ',Leq2) + #print('>>> Leq TH ',Leq_th) + #print('----------------- SIGNAL 2') + DEL2 = (2*A2)**m * nT2/time2[-1] + Leq_th = (DEL2)**(1/m) + Leq1 = equivalent_load(time2, signal2, m=m, Teq=Teq, bins=nBins, method='rainflow_windap', binStartAt0=bs0) + if hasFatpack: + Leq2 = equivalent_load(time2, signal2, m=m, Teq=Teq, bins=nBins, method='fatpack', binStartAt0=bs0) + np.testing.assert_array_almost_equal(Leq2, Leq_th, 3) + np.testing.assert_array_almost_equal(Leq1, Leq_th, 1) + #print('>>> Leq1 ',Leq1) + #print('>>> Leq2 ',Leq2) + #print('>>> Leq TH ',Leq_th) + # --- Concatenation + #print('----------------- CONCATENATION') + signal = np.concatenate((signal1, signal2)) + time = np.concatenate((time1, time2+time1[-1])) + T_all=time[-1] + DEL1 = (2*A1)**m * nT1/T_all + DEL2 = (2*A2)**m * nT2/T_all + Leq_th = (DEL1+DEL2)**(1/m) + Leq1 = equivalent_load(time, signal, m=m, Teq=Teq, bins=nBins, method='rainflow_windap', binStartAt0=bs0) + if hasFatpack: + Leq2 = equivalent_load(time, signal, m=m, Teq=Teq, bins=nBins, method='fatpack', binStartAt0=bs0) + np.testing.assert_array_almost_equal(Leq2, Leq_th, 1) + np.testing.assert_array_almost_equal(Leq1, Leq_th, 1) + #print('>>> Leq1 ',Leq1) + #print('>>> Leq2 ',Leq2) + #print('>>> Leq TH ',Leq_th) + + + diff --git a/pydatview/tools/spectral.py b/pydatview/tools/spectral.py index 9334b17..028ed10 100644 --- a/pydatview/tools/spectral.py +++ b/pydatview/tools/spectral.py @@ -59,7 +59,8 @@ def fft_wrap(t,y,dt=None, output_type='amplitude',averaging='None',averaging_win if dt is None: dtDelta0 = t[1]-t[0] # Hack to use a constant dt - dt = (np.max(t)-np.min(t))/(n0-1) + #dt = (np.max(t)-np.min(t))/(n0-1) + dt = (t[-1]-t[0])/(n0-1) relDiff = abs(dtDelta0-dt)/dt*100 #if dtDelta0 !=dt: if relDiff>0.01: diff --git a/pydatview/tools/stats.py b/pydatview/tools/stats.py index e546c34..2ba3cf3 100644 --- a/pydatview/tools/stats.py +++ b/pydatview/tools/stats.py @@ -55,8 +55,8 @@ def comparison_stats(t1, y1, t2, y2, stats='sigRatio,eps,R2', method='mean', abs sStats+=[r'$R^2=$'+r'{:.3f}'.format(R2)] elif s=='epsleq': - Leq1 = equivalent_load(t1, y1, m=5, nBins=100, method='fatpack') - Leq2 = equivalent_load(t2, y2, m=5, nBins=100, method='fatpack') + Leq1 = equivalent_load(t1, y1, m=5, bins=100, method='fatpack') + Leq2 = equivalent_load(t2, y2, m=5, bins=100, method='fatpack') epsLeq = (Leq2-Leq1)/Leq1*100 stats['epsLeq'] = epsLeq sStats+=[r'$\epsilon L_{eq}=$'+r'{:.1f}%'.format(epsLeq)] diff --git a/pydatview/tools/tictoc.py b/pydatview/tools/tictoc.py new file mode 100644 index 0000000..6df9e2d --- /dev/null +++ b/pydatview/tools/tictoc.py @@ -0,0 +1,73 @@ +import numpy as np +import time + +def pretty_time(t): + # fPrettyTime: returns a 6-characters string corresponding to the input time in seconds. + # fPrettyTime(612)=='10m12s' + # AUTHOR: E. Branlard + if(t<0): + s='------'; + elif (t<1) : + c=np.floor(t*100); + s='{:2d}.{:02d}s'.format(0,int(c)) + elif(t<60) : + s=np.floor(t); + c=np.floor((t-s)*100); + s='{:2d}.{:02d}s'.format(int(s),int(c)) + elif(t<3600) : + m=np.floor(t/60); + s=np.mod( np.floor(t), 60); + s='{:2d}m{:02d}s'.format(int(m),int(s)) + elif(t<86400) : + h=np.floor(t/3600); + m=np.floor(( np.mod( np.floor(t) , 3600))/60); + s='{:2d}h{:02d}m'.format(int(h),int(m)) + elif(t<8553600) : #below 3month + d=np.floor(t/86400); + h=np.floor( np.mod(np.floor(t), 86400)/3600); + s='{:2d}d{:02d}h'.format(int(d),int(h)) + elif(t<31536000): + m=t/(3600*24*30.5); + s='{:4.1f}mo'.format(m) + #s='+3mon.'; + else: + y=t/(3600*24*365.25); + s='{:.1f}y'.format(y) + return s + + +class Timer(object): + """ Time a set of commands, as a context manager + usage: + + with Timer('A name'): + cmd1 + cmd2 + """ + def __init__(self, name=None, writeBefore=False, silent=False): + self.name = name + self.writeBefore = writeBefore + self.silent=silent + + def ref_str(self): + s='[TIME] ' + if self.name: + s+='{:31s}'.format(self.name[:30]) + return s + + def __enter__(self): + if self.silent: + return + self.tstart = time.time() + if self.writeBefore: + s=self.ref_str() + print(s,end='') + + def __exit__(self, type, value, traceback): + if self.silent: + return + if self.writeBefore: + print('Elapsed: {:6s}'.format(pretty_time(time.time() - self.tstart))) + else: + s=self.ref_str() + print(s+'Elapsed: {:6s}'.format(pretty_time(time.time() - self.tstart))) From 65d7f49f9d9045a33445341bb5567a31bddf2456 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 16 Jul 2023 21:12:21 -0600 Subject: [PATCH 110/178] Export to other fileformats than CSV (see #161) --- pydatview/Tables.py | 16 +++++++------ pydatview/io/converters.py | 46 +++++++++++++++++++++++++++++++------- pydatview/main.py | 20 +++++++++++++---- 3 files changed, 63 insertions(+), 19 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 6081c24..81e4a34 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -818,14 +818,16 @@ def setColumnByFormula(self,sNewName,sFormula,i=-1): return True - def export(self, path): - if isinstance(self.data, pd.DataFrame): - df = self.data - if 'Index' in df.columns.values: - df = df.drop(['Index'], axis=1) - df.to_csv(path, sep=',', index=False) + def export(self, path, fformat='auto'): + from pydatview.io.converters import writeDataFrameAutoFormat, writeDataFrameToFormat + df = self.data + base, ext = os.path.splitext(path) + if 'Index' in df.columns.values: + df = df.drop(['Index'], axis=1) + if fformat=='auto': + writeDataFrameAutoFormat(df, path) else: - raise NotImplementedError('Export of data that is not a dataframe') + writeDataFrameToFormat(df, path, fformat=fformat) diff --git a/pydatview/io/converters.py b/pydatview/io/converters.py index 990d1e4..99dc276 100644 --- a/pydatview/io/converters.py +++ b/pydatview/io/converters.py @@ -4,7 +4,44 @@ # -------------------------------------------------------------------------------- # --- Writing pandas DataFrame to different formats # -------------------------------------------------------------------------------- -# The +def writeDataFrameToFormat(df, filename, fformat): + """ + Write a dataframe to disk based on user-specified fileformat + - df: pandas dataframe + - filename: filename + - fformat: fileformat in: ['csv', 'outb', 'parquet'] + """ + + if fformat=='outb': + dataFrameToOUTB(df, filename) + elif fformat=='parquet': + dataFrameToParquet(df, filename) + elif fformat=='csv': + dataFrameToCSV(df, filename, sep=',', index=False) + else: + raise Exception('File format not supported for dataframe export `{}`'.format(fformat)) + +def writeDataFrameAutoFormat(df, filename, fformat=None): + """ + Write a dataframe to disk based on extension + - df: pandas dataframe + - filename: filename + """ + if fformat is not None: + raise Exception() + base, ext = os.path.splitext(filename) + ext = ext.lower() + if ext in ['.outb']: + fformat = 'outb' + elif ext in ['.parquet']: + fformat = 'parquet' + elif ext in ['.csv']: + fformat = 'csv' + else: + print('[WARN] defaulting to csv, extension unknown: `{}`'.format(ext)) + fformat = 'csv' + + writeDataFrameToFormat(df, filename, fformat) def writeFileDataFrames(fileObject, writer, extension='.conv', filename=None, **kwargs): """ @@ -35,7 +72,6 @@ def writeFileDataFrames(fileObject, writer, extension='.conv', filename=None, ** else: writeDataFrame(df=dfs, writer=writer, filename=filename, **kwargs) - def writeDataFrame(df, writer, filename, **kwargs): """ Write a dataframe to disk based on a "writer" function. @@ -47,16 +83,10 @@ def writeDataFrame(df, writer, filename, **kwargs): # --- Low level writers def dataFrameToCSV(df, filename, sep=',', index=False, **kwargs): - base, ext = os.path.splitext(filename) - if len(ext)==0: - filename = base='.csv' df.to_csv(filename, sep=sep, index=index, **kwargs) def dataFrameToOUTB(df, filename, **kwargs): from .fast_output_file import writeDataFrame as writeDataFrameToOUTB - base, ext = os.path.splitext(filename) - if len(ext)==0: - filename = base='.outb' writeDataFrameToOUTB(df, filename, binary=True) def dataFrameToParquet(df, filename, **kwargs): diff --git a/pydatview/main.py b/pydatview/main.py index 7251121..bb54e97 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -463,13 +463,25 @@ def deleteTabs(self, I): def exportTab(self, iTab): tab=self.tabList[iTab] default_filename=tab.basename +'.csv' - with wx.FileDialog(self, "Save to CSV file",defaultFile=default_filename, - style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as dlg: - #, wildcard="CSV files (*.csv)|*.csv", + + # --- Set list of allowed formats + # NOTE: this needs to be in harmony with io.converters + fformat= ['auto' ]; wildcard ='auto (based on extension, default to CSV) (.*)|*.*|' + fformat+=['csv' ]; wildcard+='CSV file (.csv,.txt)|*.csv;*.txt|' + fformat+=['outb' ]; wildcard+='FAST output file (.outb)|*.outb|' + fformat+=['parquet']; wildcard+='Parquet file (.parquet)|*.parquet' + #fformat= ['excel' ];wildcard+='Excel file (.xls,.xlsx)|*.xls;*.xlsx|' + #fformat+=['pkl' ];wildcard+='Pickle file (.pkl)|*.pkl|' + #fformat+=['tecplot'];wildcard+='Tecplot ASCII file (.dat)|*.dat|' + + with wx.FileDialog(self, "Save to CSV file", defaultFile=default_filename, + style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT, wildcard=wildcard) as dlg: dlg.CentreOnParent() if dlg.ShowModal() == wx.ID_CANCEL: return # the user changed their mind - tab.export(dlg.GetPath()) + path = dlg.GetPath() + fformat = fformat[dlg.GetFilterIndex()] + tab.export(path=path, fformat=fformat) def onShowTool(self, event=None, toolName=''): """ From dea3447ac4e2115493ff38ab79a4c4132584d3c6 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 16 Jul 2023 21:43:27 -0600 Subject: [PATCH 111/178] Possibility to change/swap order of tables via a right click menu (see #136) --- pydatview/GUISelectionPanel.py | 29 +++++++++++++++++++++++++++++ pydatview/Tables.py | 4 ++++ 2 files changed, 33 insertions(+) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 3b8f813..4f875cf 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -293,6 +293,16 @@ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=Fal self.Append(item) self.Bind(wx.EVT_MENU, self.mainframe.onAdd, item) + if len(self.ISel)==1 and self.ISel[0]!=0: + item = wx.MenuItem(self, -1, "Move Up") + self.Append(item) + self.Bind(wx.EVT_MENU, self.OnMoveTabUp, item) + + if len(self.ISel)==1 and self.ISel[0]1: item = wx.MenuItem(self, -1, "Merge (horizontally)") self.Append(item) @@ -327,6 +337,25 @@ def OnNaming(self, event=None): self.tabPanel.updateTabNames() + def OnMoveTabUp(self, event=None): + iOld = self.ISel[0] + iNew = self.ISel[0]-1 + self.tabList.swap(iOld, iNew) + # Updating tables + self.selPanel.update_tabs(self.tabList) + # Trigger a replot + self.mainframe.onTabSelectionChange() + + + def OnMoveTabDown(self, event=None): + iOld = self.ISel[0] + iNew = self.ISel[0]+1 + self.tabList.swap(iOld, iNew) + # Updating tables + self.selPanel.update_tabs(self.tabList) + # Trigger a replot + self.mainframe.onTabSelectionChange() + def OnMergeTabs(self, event): # --- Figure out the common columns tabs = [self.tabList[i] for i in self.ISel] diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 81e4a34..fd8ee02 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -192,6 +192,10 @@ def renameTable(self, iTab, newName): self._tabs[iTab].rename(newName) return oldName + def swap(self, i1, i2): + """ Swap two elements of the list""" + self._tabs[i1], self._tabs[i2] = self._tabs[i2], self._tabs[i1] + def sort(self, method='byName'): if method=='byName': tabnames_display=self.getDisplayTabNames() From 66a2632a2b79f7de09417b87a519c569c36788a0 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 16 Jul 2023 22:16:03 -0600 Subject: [PATCH 112/178] Bug Fix: deleted user formulas are properly deleted (see #148) --- pydatview/Tables.py | 28 +++++++++++++++++++++++++--- pydatview/main.py | 13 +++---------- pydatview/plotdata.py | 2 ++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index fd8ee02..ca64dc0 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -385,6 +385,25 @@ def applyCommonMaskString(self,maskString,bAdd=True): return dfs_new, names_new, errors + + # --- Formulas + def storeFormulas(self): + formulas = {} + for tab in self._tabs: + f = tab.formulas # list of dict('pos','formula','name') + f = sorted(f, key=lambda k: k['pos']) # Sort formulae by position in list of formua + formulas[tab.raw_name]=f # we use raw_name as key + return formulas + + # --- Formulas + def applyFormulas(self, formulas): + """ formuals: dict as returned by storeFormulas""" + for tab in self._tabs: + if tab.raw_name in formulas.keys(): + for f in formulas[tab.raw_name]: + tab.addColumnByFormula(f['name'], f['formula'], f['pos']-1) + + # --- Resampling TODO MOVE THIS OUT OF HERE OR UNIFY def applyResampling(self,iCol,sampDict,bAdd=True): """ Apply resampling on table list @@ -733,25 +752,28 @@ def deleteColumns(self, ICol): """ Delete columns by index, not column names which can have duplicates""" IKeep =[i for i in np.arange(self.data.shape[1]) if i not in ICol] self.data = self.data.iloc[:, IKeep] # Drop won't work for duplicates + # TODO find a way to add a "formula" attribute to a column of a dataframe to avoid dealing with "pos". for i in sorted(ICol, reverse=True): + # Remove formulae if these are part of the columns deleted for f in self.formulas: - if f['pos'] == (i + 1): + if f['pos'] == i: self.formulas.remove(f) break + # Shift formulae locations due to column being removed for f in self.formulas: - if f['pos'] > (i + 1): + if f['pos'] > i: f['pos'] = f['pos'] - 1 def rename(self,new_name): self.name='>'+new_name def addColumn(self, sNewName, NewCol, i=-1, sFormula=''): - print('>>> adding Column') if i<0: i=self.data.shape[1] elif i>self.data.shape[1]+1: i=self.data.shape[1] self.data.insert(int(i+1),sNewName,NewCol) + # Due to new column, formulas position needs to be incremented. for f in self.formulas: if f['pos'] > i: f['pos'] = f['pos'] + 1 diff --git a/pydatview/main.py b/pydatview/main.py index bb54e97..40ce29b 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -321,11 +321,8 @@ def load_files(self, filenames=[], fileformats=None, bReload=False, bAdd=False, if bReload: # Restore formulas that were previously added - for tab in self.tabList: - if tab.raw_name in self.restore_formulas.keys(): - for f in self.restore_formulas[tab.raw_name]: - tab.addColumnByFormula(f['name'], f['formula'], f['pos']-1) - self.restore_formulas = {} + self.tabList.applyFormulas(self.formulas_backup) + self.formulas_backup = {} # Display warnings for warn in warnList: Warn(self,warn) @@ -666,11 +663,7 @@ def onReload(self, event=None): fileformats = [self.FILE_FORMATS[iFormat-1]] # Save formulas to restore them after reload with sorted tabs - self.restore_formulas = {} - for tab in self.tabList._tabs: - f = tab.formulas # list of dict('pos','formula','name') - f = sorted(f, key=lambda k: k['pos']) # Sort formulae by position in list of formua - self.restore_formulas[tab.raw_name]=f # we use raw_name as key + self.formulas_backup = self.tabList.storeFormulas() # Actually load files (read and add in GUI) self.load_files(filenames, fileformats=fileformats, bReload=True, bAdd=False, bPlot=True) else: diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 7000520..b871c0f 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -438,6 +438,8 @@ def y0Var(PD): return v,s def y0TI(PD): + if PD._y0Mean[0]==0: + return np.nan,'NA' v=PD._y0Std[0]/PD._y0Mean[0] s=pretty_num(v) return v,s From a0e877c7850dba18a5a825bff716ee987ad407a6 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 16 Jul 2023 23:13:48 -0600 Subject: [PATCH 113/178] Explored an option to store subplots parameters (see #147) --- pydatview/GUIPlotPanel.py | 13 ++++++++++++- pydatview/GUIToolBox.py | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 31799fb..0a6dbd4 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -630,7 +630,14 @@ def set_subplot_spacing(self, init=False): - this will override the user settings """ #self.fig.set_tight_layout(True) # NOTE: works almost fine, but problem with FFT multiple - # TODO this is definitely not generic, but tight fails.. + ## TODO this is definitely not generic, but tight fails.. + #return + #if hasattr(self.fig, '_subplotsPar'): + # # See GUIToolBox.py configure_toolbar + # self.fig.subplots_adjust(**self.fig._subplotsPar) + # return + + if init: # NOTE: at init size is (20,20) because sizer is not initialized yet bottom = 0.12 @@ -662,6 +669,10 @@ def set_subplot_spacing(self, init=False): else: self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98) + def get_subplot_spacing(self, init=False): + pass + + def plot_matrix_select(self, event): if self.infoPanel is not None: self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) diff --git a/pydatview/GUIToolBox.py b/pydatview/GUIToolBox.py index 1e9f072..c591816 100644 --- a/pydatview/GUIToolBox.py +++ b/pydatview/GUIToolBox.py @@ -309,3 +309,27 @@ def home(self, *args): def set_message(self, s): pass + +# def configure_subplots(self, *args): +# NavigationToolbar2Wx.configure_subplots(self, *args) +# +# def close_event(e): +# # We delete the suplot_tool (default behavior) +# try: +# delattr(self, 'subplot_tool') +# except: +# pass +# # Then we introduce a hook to store the new subplot +# params = self.canvas.figure.subplotpars +# paramsD= {} +# for key in ["left", "bottom", "right", "top", "wspace", "hspace"]: +# paramsD[key]=getattr(params, key) +# self.canvas.figure._subplotsPar = paramsD +# +# # We change the default hook for the close event +# tool_fig = self.subplot_tool.figure +# tool_fig.canvas.mpl_connect("close_event", close_event) +# +# return self.subplot_tool + + From 3dabfc3095c65aee0654bae97d1e61d457de67c5 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 13 Sep 2023 19:01:09 -0600 Subject: [PATCH 114/178] Tools: Freq and Damp estimation (See #164) --- pydatview/GUIPlotPanel.py | 33 ++-- pydatview/plotdata.py | 9 +- pydatview/plugins/__init__.py | 4 +- pydatview/plugins/tool_logdec.py | 199 +++++++++++++++++++-- pydatview/tools/damping.py | 298 ++++++++++++++++++++++++------- 5 files changed, 445 insertions(+), 98 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 0a6dbd4..7dea4e0 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -368,6 +368,10 @@ def __init__(self, parent, data): self.Bind(wx.EVT_COMBOBOX ,self.onAnyEsthOptionChange) self.cbFont.Bind(wx.EVT_COMBOBOX ,self.onFontOptionChange) + # Store data + self.data={} + self._GUI2Data() + def onAnyEsthOptionChange(self,event=None): self.parent.redraw_same_data() @@ -375,6 +379,15 @@ def onFontOptionChange(self,event=None): matplotlib_rc('font', **{'size':int(self.cbFont.Value) }) # affect all (including ticks) self.onAnyEsthOptionChange() + def _GUI2Data(self): + """ data['plotStyle'] """ + self.data['Font'] = int(self.cbFont.GetValue()) + self.data['LegendFont'] = int(self.cbLgdFont.GetValue()) + self.data['LegendPosition'] = self.cbLegend.GetValue() + self.data['LineWidth'] = float(self.cbLW.GetValue()) + self.data['MarkerSize'] = float(self.cbMS.GetValue()) + return self.data + class PlotPanel(wx.Panel): def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): @@ -588,11 +601,8 @@ def addTables(self, *args, **kwargs): def saveData(self, data): data['Grid'] = self.cbGrid.IsChecked() data['CrossHair'] = self.cbXHair.IsChecked() - data['plotStyle']['Font'] = self.esthPanel.cbFont.GetValue() - data['plotStyle']['LegendFont'] = self.esthPanel.cbLgdFont.GetValue() - data['plotStyle']['LegendPosition'] = self.esthPanel.cbLegend.GetValue() - data['plotStyle']['LineWidth'] = self.esthPanel.cbLW.GetValue() - data['plotStyle']['MarkerSize'] = self.esthPanel.cbMS.GetValue() + self.esthPanel._GUI2Data() + data['plotStyle']= self.esthPanel.data @staticmethod def defaultData(): @@ -1045,11 +1055,14 @@ def plot_all(self, autoscale=True): axes=self.fig.axes PD=self.plotData + # --- PlotStyles + plotStyle = self.esthPanel._GUI2Data() + # --- Plot options bStep = self.cbStepPlot.IsChecked() plot_options = dict() - plot_options['lw']=float(self.esthPanel.cbLW.Value) - plot_options['ms']=float(self.esthPanel.cbMS.Value) + plot_options['lw']=plotStyle['LineWidth'] + plot_options['ms']=plotStyle['MarkerSize'] if self.cbCurveType.Value=='Plain': plot_options['LineStyles'] = ['-'] plot_options['Markers'] = [''] @@ -1072,8 +1085,8 @@ def plot_all(self, autoscale=True): # --- Font options font_options = dict() font_options_legd = dict() - font_options['size'] = int(self.esthPanel.cbFont.Value) # affect labels - font_options_legd['fontsize'] = int(self.esthPanel.cbLgdFont.Value) + font_options['size'] = plotStyle['Font'] + font_options_legd['fontsize'] = plotStyle['LegendFont'] needChineseFont = any([pd.needChineseFont for pd in PD]) if needChineseFont and self.specialFont is not None: font_options['fontproperties']= self.specialFont @@ -1186,7 +1199,7 @@ def plot_all(self, autoscale=True): ax_right.set_ylabel('') # Legends - lgdLoc = self.esthPanel.cbLegend.Value.lower() + lgdLoc = plotStyle['LegendPosition'].lower() if (self.pltTypePanel.cbCompare.GetValue() or ((len(yleft_legends) + len(yright_legends)) > 1)): if lgdLoc !='none': diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index b871c0f..65d3bf8 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -440,9 +440,12 @@ def y0Var(PD): def y0TI(PD): if PD._y0Mean[0]==0: return np.nan,'NA' - v=PD._y0Std[0]/PD._y0Mean[0] - s=pretty_num(v) - return v,s + try: + v=PD._y0Std[0]/PD._y0Mean[0] + s=pretty_num(v) + return v,s + except: + return np.nan,'NA' def yRange(PD): diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 1a3991e..87ab7ba 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -99,8 +99,8 @@ def _tool_curvefitting(*args, **kwargs): # TOOLS: tool plugins constructor should return a Panel class TOOLS=OrderedDict([ - ('Damping from decay',_tool_logdec), - ('Curve fitting', _tool_curvefitting), + ('Estimate Freq. and Damping', _tool_logdec), + ('Curve fitting' , _tool_curvefitting), ]) diff --git a/pydatview/plugins/tool_logdec.py b/pydatview/plugins/tool_logdec.py index 815cf64..830d25f 100644 --- a/pydatview/plugins/tool_logdec.py +++ b/pydatview/plugins/tool_logdec.py @@ -1,39 +1,202 @@ import wx +import numpy as np +from pydatview.common import Error, Info, Warn from pydatview.plugins.base_plugin import GUIToolPanel -from pydatview.tools.damping import logDecFromDecay +from pydatview.tools.damping import freqDampEstimator, zetaEnvelop +_HELP = """Damping and frequency estimation. + +This tool attemps to estimate the natural frequency and damping ratio of the signal, +assuming it has a dominant frequency and an exponential decay or growth. +The damping ratio is referred to as "zeta". + +NOTE: you can zoom on the figure first (and deselect "AutoScale"), to perform the estimation + on the zoomed subset of the plot. + +- Click on "Compute and Plot" to perform the estimate. + +- Algorithm options: None for now. + See freqDampEstimator in damping.py for ways to extend this. + +- Plotting options: + These options do no affect the algorithm used to compute the frequency and zeta. + - Envelop: If selected, the exponential envelop corresponding to a given zeta is shown. + - Ref point: the reference point used to plot the envelope. The envelop will pass by this + point, either the first, last or middle point of the plot. + - Peaks: If selected, the upper and lower peaks that are detected by the algorithm are + shown on the plot + - More Env.: Additional guess for zeta are used and plotted. This gives an idea of the min + and max value that zeta might take. + +- Results: Display values for + - Natural frequency "F_n" (in [Hz] if x in [s]) + - Damping ratio "zeta" [-] + - Logarithmic decrements "logdec" [-] + - Damped frequency "F_d" (in [Hz] if x in [s]) + - Damped period "T_d" (in [s] if x in [s]) +""" # --------------------------------------------------------------------------------} # --- Log Dec # --------------------------------------------------------------------------------{ class LogDecToolPanel(GUIToolPanel): def __init__(self, parent): super(LogDecToolPanel,self).__init__(parent) + self.data = {} + # --- GUI btClose = self.getBtBitmap(self,'Close' ,'close' ,self.destroy ) - btComp = self.getBtBitmap(self,'Compute','compute',self.onCompute) - self.lb = wx.StaticText( self, -1, ' ') + btComp = self.getBtBitmap(self,'Compute and Plot','compute',self.onCompute) + btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) + self.cbRefPoint = wx.ComboBox(self, -1, choices=['start','mid','end'], style=wx.CB_READONLY) + self.cbRefPoint.SetSelection(1) + self.cbPeaks = wx.CheckBox(self, -1, 'Peaks',(10,10)) + self.cbEnv = wx.CheckBox(self, -1, 'Envelop',(10,10)) + self.cbMinMax = wx.CheckBox(self, -1, 'More Envelops',(10,10)) + self.cbMinMax.SetValue(False) + self.cbPeaks .SetValue(False) + self.cbEnv .SetValue(True) + self.results = wx.TextCtrl(self, wx.ID_ANY, '', style=wx.TE_READONLY) + self.results.SetValue('Click on "Compute and Plot" to show results here ') + + boldFont = self.GetFont().Bold() + lbAlgOpts = wx.StaticText(self, -1, 'Algo. Options') + lbPltOpts = wx.StaticText(self, -1, 'Plot Options') + lbResults = wx.StaticText(self, -1, 'Results') + lbAlgOpts.SetFont(boldFont) + lbPltOpts.SetFont(boldFont) + lbResults.SetFont(boldFont) + + # --- Layout + btSizer = wx.FlexGridSizer(rows=3, cols=1, hgap=2, vgap=0) + btSizer.Add(btClose ,0, flag = wx.LEFT|wx.CENTER,border = 1) + btSizer.Add(btComp ,0, flag = wx.LEFT|wx.CENTER,border = 1) + btSizer.Add(btHelp ,0, flag = wx.LEFT|wx.CENTER,border = 1) + + gridSizer = wx.FlexGridSizer(rows=3, cols=2, hgap=2, vgap=0) + + sizerOpts = wx.BoxSizer(wx.HORIZONTAL) + sizerOpts.Add(wx.StaticText( self, -1, 'Ref. point: ' ) , 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT,border = 1) + sizerOpts.Add(self.cbRefPoint , 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT,border = 1) + sizerOpts.Add(self.cbEnv , 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT,border = 10) + sizerOpts.Add(self.cbPeaks , 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT,border = 1) + sizerOpts.Add(self.cbMinMax , 0, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT,border = 1) + + sizerAlg = wx.BoxSizer(wx.HORIZONTAL) + sizerAlg.Add(wx.StaticText( self, -1, 'No options for now, using "peaks" and log dec.'), 1, flag = wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT,border = 1) + + sizerRes = wx.BoxSizer(wx.HORIZONTAL) + sizerRes.Add(self.results , 1, flag = wx.EXPAND|wx.ALIGN_LEFT|wx.LEFT,border = 1) + + gridSizer.Add(lbAlgOpts, 0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + gridSizer.Add(sizerAlg, 1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + gridSizer.Add(lbPltOpts, 0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + gridSizer.Add(sizerOpts, 1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + gridSizer.Add(lbResults, 0, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM,border = 1) + gridSizer.Add(sizerRes , 1, flag=wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND,border = 1) + gridSizer.AddGrowableCol(1,0.5) + self.sizer = wx.BoxSizer(wx.HORIZONTAL) - self.sizer.Add(btClose ,0, flag = wx.LEFT|wx.CENTER,border = 1) - self.sizer.Add(btComp ,0, flag = wx.LEFT|wx.CENTER,border = 5) - self.sizer.Add(self.lb ,0, flag = wx.LEFT|wx.CENTER,border = 5) + self.sizer.Add(btSizer , 0, flag = wx.LEFT ,border = 5) + self.sizer.Add(gridSizer , 1, flag = wx.LEFT|wx.EXPAND,border = 10) self.SetSizer(self.sizer) + # --- Binding + #self.cbRefPoint.Bind(wx.EVT_COMBOBOX, self.onRefPointChange) + + def onClear(self): + #.mainframe.plotPanel.load_and_draw # or action.guiCallback + return self.parent.load_and_draw() + + def onHelp(self,event=None): + Info(self, _HELP) + + # --- Fairly generic + def _GUI2Data(self): + self.data['method'] = 'fromPeaks' + self.data['refPoint'] = self.cbRefPoint.GetValue() + self.data['minMax'] = self.cbMinMax.IsChecked() + self.data['env'] = self.cbEnv .IsChecked() + self.data['peaks'] = self.cbPeaks .IsChecked() + def onCompute(self,event=None): if len(self.parent.plotData)!=1: Error(self,'Log Dec tool only works with a single plot.') return - pd =self.parent.plotData[0] + + # --- Clean plot..., but store limits first + ax = self.parent.fig.axes[0] + xlim = ax.get_xlim() + self.onClear() # NOTE: this will create different axes.. + + + # --- GUI2Data + self._GUI2Data() + + # --- PlotStyles + plotStyle = self.parent.esthPanel._GUI2Data() + lgdLoc = plotStyle['LegendPosition'].lower() + legd_opts=dict() + legd_opts['fontsize'] = plotStyle['LegendFont'] + legd_opts['loc'] = lgdLoc if lgdLoc != 'none' else 4 + legd_opts['fancybox'] = False + + # --- Data to work with + PD =self.parent.plotData[0] + ax =self.parent.fig.axes[0] + # Restricting data to axes visible bounds on the x axis + b=np.logical_and(PD.x>=xlim[0], PD.x<=xlim[1]) + t = PD.x[b] + x = PD.y[b] + try: - logdec,DampingRatio,T,fn,fd,IPos,INeg,epos,eneg=logDecFromDecay(pd.y,pd.x) - lab='LogDec.: {:.4f} - Damping ratio: {:.4f} - F_n: {:.4f} - F_d: {:.4f} - T:{:.3f}'.format(logdec,DampingRatio,fn,fd,T) - self.lb.SetLabel(lab) - self.sizer.Layout() - ax=self.parent.fig.axes[0] - ax.plot(pd.x[IPos],pd.y[IPos],'o') - ax.plot(pd.x[INeg],pd.y[INeg],'o') - ax.plot(pd.x ,epos,'k--') - ax.plot(pd.x ,eneg,'k--') - self.parent.canvas.draw() +# if True: + fn, zeta, info = freqDampEstimator(x, t, opts=self.data) except: - self.lb.SetLabel('Failed. The signal needs to look like the decay of a first order system.') + self.results.SeValue('Failed. The signal needs to look like the decay of a first order system.') + event.Skip() + return + + # --- Handling returned data + logdec = 2*np.pi*zeta /np.sqrt(1-zeta**2) + omega0 = 2*np.pi*fn + fd = fn*np.sqrt(1-zeta**2) + Td = 1/fd + IPos = info['IPos'] + if self.data['refPoint']=='mid': + iRef = IPos[int(len(IPos)/2)] + elif self.data['refPoint']=='start': + iRef = IPos[0] + elif self.data['refPoint']=='end': + iRef = IPos[-1] + else: + raise Exception('Wrong value for ref Point') + + lab='F_n={:.4f} , DampingRatio={:.4f} , LogDec.={:.4f}, F_d={:.4f} , T_d={:.3f}'.format(fn, zeta, logdec, fd, Td) + self.results.SetValue(lab) + #self.sizer.Layout() + + + # --- Plot Min and Max points + if self.data['peaks']: + ax.plot(t[info['IPos']],x[info['IPos']],'o') + ax.plot(t[info['INeg']],x[info['INeg']],'o') + + + # --- Plot envelop + def plotEnv(zeta, sty, c): + epos, eneg = zetaEnvelop(x, t, omega0, zeta, iRef=iRef) + ax.plot(t, epos, sty, color=c, label=r'$\zeta={:.2f}$%'.format(zeta*100)) + ax.plot(t, eneg, sty, color=c) + if self.data['env']: + plotEnv(zeta, '--', 'k') + + # --- Plot more envelops + if self.data['minMax']: + # Could also do +/- 10 % of zeta + plotEnv(info['zetaMax'] , ':', 'b') + plotEnv(info['zetaMin'] , ':', 'r') + plotEnv(info['zetaMean'], ':', 'g') + + ax.legend(**legd_opts) + self.parent.canvas.draw() #self.parent.load_and_draw(); # DATA HAS CHANGED diff --git a/pydatview/tools/damping.py b/pydatview/tools/damping.py index f1340f1..43ba737 100644 --- a/pydatview/tools/damping.py +++ b/pydatview/tools/damping.py @@ -1,6 +1,33 @@ +""" +Context: + +Logarithmic decrement: + + delta = 1/N log [ x(t) / x(t+N T_d)] = 2 pi zeta / sqrt(1-zeta^2) + +Damping ratio: + + zeta = delta / sqrt( 4 pi^2 + delta^2 ) + +Damped period, frequency: + + Td = 2pi / omega_d + + omegad = omega_0 sqrt(1-zeta**2) + +Damping exponent: + + + alpha = zeta omega_0 = delta/ T_d + + +""" + import numpy as np -__all__ = ['logDecFromDecay'] +__all__ = ['freqDampEstimator'] +__all__ += ['freqDampFromPeaks'] +__all__ += ['zetaEnvelop'] __all__ += ['TestDamping'] def indexes(y, thres=0.3, min_dist=1, thres_abs=False): @@ -96,12 +123,13 @@ def indexes(y, thres=0.3, min_dist=1, thres_abs=False): #indexes =indexes(x, thres=0.02/max(x), min_dist=1, thres_abs=true) -def logDecFromThreshold(x,threshold=None,bothSides=False): - """ Detect maxima in a signal, computes the log deg based on it """ +def logDecFromThreshold(x, threshold=None, bothSides=False, decay=True): + """ Detect maxima in a signal, computes the log deg based on it + """ if bothSides: - ldPos,iTPos,stdPos,IPos = logDecFromThreshold( x, threshold=threshold) - ldNeg,iTNeg,stdNeg,INeg = logDecFromThreshold(-x, threshold=threshold) - return (ldPos+ldNeg)/2, (iTPos+iTNeg)/2, (stdPos+stdNeg)/2, (IPos,INeg) + ldPos,iTPos,stdPos,IPos,vldPos = logDecFromThreshold( x, threshold=threshold, decay=decay) + ldNeg,iTNeg,stdNeg,INeg,vldNeg = logDecFromThreshold(-x, threshold=threshold, decay=decay) + return (ldPos+ldNeg)/2, (iTPos+iTNeg)/2, (stdPos+stdNeg)/2, (IPos,INeg), (vldPos, vldNeg) if threshold is None: threshold = np.mean(abs(x-np.mean(x)))/3; @@ -110,58 +138,179 @@ def logDecFromThreshold(x,threshold=None,bothSides=False): iT = round(np.median(np.diff(I))); vn=np.arange(0,len(I)-1)+1 # Quick And Dirty Way using one as ref and assuming all periods were found - vDamping = 1/vn*np.log( x[I[0]]/x[I[1:]] ) # Damping Ratios - vLogDec = 1/np.sqrt(1+ (2*np.pi/vDamping)**2 ) + if decay: + # For a decay we take the first peak as a reference + vLogDec = 1/vn*np.log( x[I[0]]/x[I[1:]] ) # Logarithmic decrement + else: + # For negative damping we take the last peak as a reference + vLogDec = 1/vn*np.log( x[I[-2::-1]]/x[I[-1]] ) # Logarithmic decrement logdec = np.mean(vLogDec); std_logdec = np.std(vLogDec) ; - return logdec,iT,std_logdec,I + return logdec, iT, std_logdec, I, vLogDec + +def logDecTwoTimes(x, t, i1, i2, Td): + t1, t2 = t[i1], t[i2] + x1, x2 = x[i1], x[i2] + N = (t2-t1)/Td + logdec = 1/N * np.log(x1/x2) + return logdec + +def zetaTwoTimes(x, t, i1, i2, Td): + logdec = logDecTwoTimes(x, t, i1, i2, Td) + zeta = logdec/np.sqrt(4*np.pi**2 + logdec**2) # damping ratio + return zeta + +def zetaRange(x, t, IPos, INeg, Td, decay): + """ + Compute naive zeta based on different peak values (first, mid, last) + """ + def naivezeta(i1, i2): + zetaP = zetaTwoTimes( x, t, IPos[i1], IPos[i2], Td) + zetaN = zetaTwoTimes( -x, t, INeg[i1], INeg[i2], Td) + return [zetaP, zetaN] + zetas = [] + # --- Computing naive log dec from first and last peaks + zetas += naivezeta(0, -1) + # --- Computing naive log dec from one peak and the middle one + if len(IPos)>3 and len(INeg)>3: + if decay: + i1, i2 = 0, int(len(IPos)/2) + else: + i1, i2 = -int(len(IPos)/2), -1 + zetas += naivezeta(i1, i2) + zetaSup = np.max(zetas) + zetaInf = np.min(zetas) + zetaMean = np.mean(zetas) + return zetaSup, zetaInf, zetaMean + +def zetaEnvelop(x, t, omega0, zeta, iRef, moff=0): + """ NOTE: x is assumed to be centered on 0""" + m = np.mean(x) + tref = t[iRef] + Aref = x[iRef]-m + epos = Aref*np.exp(-zeta*omega0*(t-tref))+m+moff + eneg = -Aref*np.exp(-zeta*omega0*(t-tref))+m+moff + return epos, eneg -def logDecFromDecay(x,t,threshold=None): +def freqDampFromPeaks(x, t, threshold=None, plot=False, refPoint='mid'): + """ + Use Upper and lower peaks to compute log decrements between neighboring peaks + Previously called logDecFromDecay. + """ + info = {} + x = np.array(x).copy() m = np.mean(x) + x = x-m # we remove the mean once and for all if threshold is None: - threshold = np.mean(abs(x-m))/3; - + threshold = np.mean(abs(x))/3 dt = t[1]-t[0] # todo signal with dt not uniform - # Computing log decs from positive and negative side and taking the mean - logdec,iT,std,(IPos,INeg) = logDecFromThreshold( x-m, threshold=threshold, bothSides=True) - DampingRatio=2*np.pi*logdec/np.sqrt(1-logdec**2) # damping ratio + # Is it a decay or an exloding signal + xstart, xend = np.array_split(np.abs(x),2) + decay= np.mean(xstart)> np.mean(xend) + + # --- Computing log decs from positive and negative side and taking the mean + logdec,iT,std,(IPos,INeg), (vLogDecPos, vLogDecNeg) = logDecFromThreshold( x, threshold=threshold, bothSides=True, decay=decay) + + # --- Finding damped period + Td = iT*dt # Period of damped oscillations. Badly estimated due to dt resolution + # % Better estimate of period + # [T,~,iCross]=fGetPeriodFromZeroCrossing(x(1:IPos(end)),dt); + fd = 1/Td + # --- Naive ranges + zetaMax, zetaMin, zetaMean = zetaRange(x, t, IPos, INeg, Td, decay) - # Going to time space - T = iT*dt # Period of damped oscillations. Badly estimated due to dt resolution -# % Better estimate of period -# [T,~,iCross]=fGetPeriodFromZeroCrossing(x(1:IPos(end)),dt); - fd = 1/T - fn = fd/np.sqrt(1-logdec**2) + zeta = logdec/np.sqrt(4*np.pi**2 + logdec**2 ) # Damping Ratio + fn = fd/np.sqrt(1-zeta**2) + T0 = 1/fn + omega0=2*np.pi*fn # --- Model # Estimating signal params - delta = DampingRatio - alpha = delta/T + alpha = logdec/Td omega = 2*np.pi*fd - # Values at a peak half way through - i1 = IPos[int(len(IPos)/2)] - A1 = x[i1] ; - t1 = dt*i1 ; - XX=x[i1:]-m - ineg=i1+np.where(XX<0)[0][0] - ipos=ineg-1 - xcross=[x[ipos],x[ineg]]-m - icross=[ipos,ineg] - i0 = np.interp(0,xcross,icross) - # For phase determination, we use a precise 0-up-crossing - - t0=dt*i0 - phi0 = np.mod(2*np.pi- omega*t0+np.pi/2,2*np.pi) ; - A = (A1-m)/(np.exp(-alpha*t1)*np.cos(omega*t1+phi0)); + # Find amplitude at a reference peak + # (we chose the middle peak of the time series to minimize period drift before and after) + # We will adjust for phase and time offset later + i1 = IPos[int(len(IPos)/2)] + A1 = x[i1] + t1 = dt*i1 + # --- Find a zero up-crossing around our value of reference for phase determination + XX=x[i1:] + ineg = i1+np.where(XX<0)[0][0] + ipos = ineg-1 + xcross = [x[ipos],x[ineg]] + icross = [ipos,ineg] + i0 = np.interp(0,xcross,icross) # precise 0-up-crossing + t0 = dt*i0 + phi0 = np.mod(2*np.pi- omega*t0+np.pi/2,2*np.pi); + # --- Model + A = A1/(np.exp(-alpha*t1)*np.cos(omega*t1+phi0)); # Adjust for phase and time offset x_model = A*np.exp(-alpha*t)*np.cos(omega*t+phi0)+m; - epos = A*np.exp(-alpha*t)+m ; - eneg = -A*np.exp(-alpha*t)+m ; - - return logdec,DampingRatio,T,fn,fd,IPos,INeg,epos,eneg - - + epos = A*np.exp(-alpha*t)+m + eneg = -A*np.exp(-alpha*t)+m + + if plot: + if refPoint=='mid': + iRef = i1 + elif refPoint=='start': + iRef = IPos[0] + else: + iRef = IPos[-1] + import matplotlib.pyplot as plt + print('LogDec.: {:.4f} - Damping ratio: {:.4f} - F_n: {:.4f} - F_d: {:.4f} - T_d:{:.3f} - T_n:{:.3f}'.format(logdec, zeta, fn, fd, Td,T0)) + fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8)) # (6.4,4.8) + fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20) + ax.plot(t, x+m) + ax.plot(t[IPos],x[IPos]+m,'o') + ax.plot(t[INeg],x[INeg]+m,'o') + epos, eneg = zetaEnvelop(x, t, omega0, zeta, iRef=iRef, moff=m) + ax.plot(t ,epos, 'k--', label=r'$\zeta={:.4f}$'.format(zeta)) + ax.plot(t ,eneg, 'k--') + epos, eneg = zetaEnvelop(x, t, omega0, zetaMax, iRef=iRef, moff=m) + ax.plot(t ,epos, 'b:', label=r'$\zeta={:.4f}$'.format(zetaMax)) + ax.plot(t ,eneg, 'b:') + epos, eneg = zetaEnvelop(x, t, omega0, zetaMin, iRef=iRef, moff=m) + ax.plot(t ,epos, 'r:', label=r'$\zeta={:.4f}$'.format(zetaMin)) + ax.plot(t ,eneg, 'r:') + #ax.plot(t ,x_model,'k:') + #ax.legend() + dx = np.max(abs(x-m)) + ax.set_ylim([m-dx*1.1 , m+dx*1.1]) + + # We return a dictionary + info['zeta'] = zeta + info['fd'] = fd + info['Td'] = Td + info['fn'] = fn + info['omega0'] = omega0 + info['IPos'] = IPos + info['INeg'] = INeg + # TODO + info['x_model'] = x_model + info['epos'] = epos + info['eneg'] = eneg + # + info['zeta'] = zeta + info['zetaMin'] = zetaMin + info['zetaMax'] = zetaMax + info['zetaMean'] = zetaMean + + return fn, zeta, info + + +def freqDampEstimator(x, t, opts): + """ + Estimate natural frequency and damping ratio. + Wrapper function to use different methods. + """ + if opts['method']=='fromPeaks': + fn, zeta, info = freqDampFromPeaks(x, t) + else: + raise NotImplementedError() + return fn, zeta, info @@ -172,33 +321,52 @@ def logDecFromDecay(x,t,threshold=None): class TestDamping(unittest.TestCase): - def test_logdec_from_decay(self): - T=10; - logdec=0.1; # log decrements (<1) logdec = 1./sqrt(1+ (2*pi./delta).^2 ) ; - delta=2*np.pi*logdec/np.sqrt(1-logdec**2); # damping ratio; - alpha=delta/T - t=np.linspace(0,30*T,1000) - x=np.cos(2*np.pi/T*t)*np.exp(-alpha*t)+10; - logdec_out,delta_out,T,fn,fd,IPos,INeg,epos,eneg=logDecFromDecay(x,t) - alpha_out=delta_out/T - self.assertAlmostEqual(logdec,logdec_out,3) - self.assertAlmostEqual(delta,delta_out,3) + def test_logdec_from_peaks(self): + plot = (__name__ == '__main__') + + for zeta in [0.1, -0.01]: + T0 = 10 + Td = T0 / np.sqrt(1-zeta**2) + delta = 2*np.pi*zeta/np.sqrt(1-zeta**2) # logdec + alpha = delta/Td + t = np.linspace(0,30*Td,2000) + x = np.cos(2*np.pi/Td*t)*np.exp(-alpha*t)+10; + fn, zeta_out, info = freqDampFromPeaks(x, t, plot=plot) + self.assertAlmostEqual(zeta , zeta_out,4) + self.assertAlmostEqual(1/T0 , fn ,2) + if __name__ == '__main__': import matplotlib.pyplot as plt - print('logdec in: ',logdec) - print('logdec out: ',logdec_out) - print('alpha in : ',alpha) - print('alpha out : ',delta_out/T) - plt.plot(t,x) - plt.plot(t[IPos],x[IPos],'o') - plt.plot(t[INeg],x[INeg],'o') - plt.plot(t,epos,'k--') - plt.plot(t,eneg,'k--') plt.show() - - if __name__ == '__main__': unittest.main() +# import matplotlib.pyplot as plt +# import pydatview.io as weio +# df= weio.read('DampingExplodingExample2.csv').toDataFrame() +# M = df.values +# x= M[:,1] +# t= M[:,0] +# #for zeta in [-0.01, 0.1]: +# # T0 = 30 +# # Td = T0 / np.sqrt(1-zeta**2) +# # delta = 2*np.pi*zeta/np.sqrt(1-zeta**2) # logdec +# # alpha = delta/Td +# # x = np.cos(2*np.pi/Td*t)*np.exp(-alpha*t)+10; +# # df.insert(1,'PureDecay{}'.format(zeta), x) +# +# #df.to_csv('DECAY_Example.csv',index=False, sep=',') +# +# # Td = 10 +# # zeta = -0.01 # damping ratio (<1) +# # delta = 2*np.pi*zeta/np.sqrt(1-zeta**2) # logdec +# # alpha = delta/Td +# # t = np.linspace(0,30*Td,1000) +# # x = np.cos(2*np.pi/Td*t)*np.exp(-alpha*t)+10; +# # +# # fn, zeta, info = freqDampFromPeaks(x, t, plot=True, refPoint='mid') +# plt.show() + + From c3cd8739772182284e8e1a831244dcffa659dfa7 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 13 Sep 2023 19:03:48 -0600 Subject: [PATCH 115/178] Tools: misc updates from welib --- pydatview/tools/fatigue.py | 181 ++++++++------ pydatview/tools/signal_analysis.py | 22 +- pydatview/tools/spectral.py | 380 +++++++++++++++++++++++++++++ pydatview/tools/stats.py | 16 ++ pydatview/tools/tictoc.py | 6 +- 5 files changed, 515 insertions(+), 90 deletions(-) diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index a7a2b1f..c725e5d 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -1,58 +1,71 @@ -# --------------------------------------------------------------------------------} -# --- Info -# --------------------------------------------------------------------------------{ -# Tools for fatigue analysis -# -# Taken from: -# repository: wetb -# package: wetb.fatigue_tools, -# institution: DTU wind energy, Denmark -# main author: mmpe -''' -Created on 04/03/2013 -@author: mmpe +""" +Tools for fatigue analysis -'eq_load' calculate equivalent loads using one of the two rain flow counting methods -'cycle_matrix' calculates a matrix of cycles (binned on amplitude and mean value) -'eq_load_and_cycles' is used to calculate eq_loads of multiple time series (e.g. life time equivalent load) +Main functions: +- equivalent_load: calculate damage equivalent load for a given signal +- find_range_count: returns range and number of cycles for a given signal -The methods uses the rainflow counting routines (See documentation in top of methods): -- 'rainflow_windap': (Described in "Recommended Practices for Wind Turbine Testing - 3. Fatigue Loads", - 2. edition 1990, Appendix A) -or -- 'rainflow_astm' (based on the c-implementation by Adam Nieslony found at the MATLAB Central File Exchange - http://www.mathworks.com/matlabcentral/fileexchange/3026) -''' -import warnings -import numpy as np +Subfunctions: +- eq_load: calculate equivalent loads using one of the two rain flow counting methods +- cycle_matrix: calculates a matrix of cycles (binned on amplitude and mean value) +- eq_load_and_cycles: calculate eq_loads of multiple time series (e.g. life time equivalent load) -__all__ = ['rainflow_astm', 'rainflow_windap','eq_load','eq_load_and_cycles','cycle_matrix','cycle_matrix2'] +Main aglorithms for rain flow counting: +- rainflow_windap: taken from [2], based on [3] +- rainflow_astm: taken from [2], based [4] +- fatpack: using [5] -def equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap', meanBin=True, binStartAt0=False): - """Equivalent load calculation +References: + [1] Hayman (2012) MLife theory manual for Version 1.00 + [2] Wind energy toolbox, wetb.fatigue_tools, DTU wind energy, Denmark + [3] "Recommended Practices for Wind Turbine Testing - 3. Fatigue Loads", 2. edition 1990, Appendix A + [4] Adam Nieslony - Rainflow Counting Algorithm, MATLAB Central File Exchange + http://www.mathworks.com/matlabcentral/fileexchange/3026) + [5] Fatpack - Python package + https://github.com/Gunnstein/fatpack - Calculate the equivalent loads for a list of Wohler exponent - Parameters - ---------- - time : array-like, the time values corresponding to the signal (s) - signals : array-like, the load signal - m : Wohler exponent (default is 3) - Teq : The equivalent period (Default 1, for 1Hz) - bins : Number of bins in rainflow count histogram - method: 'rainflow_windap, rainflow_astm, fatpack - meanBin: if True, use the mean of the ranges within a bin (recommended) - otherwise use the middle of the bin (not recommended). - binStartAt0: if True bins start at zero. Otherwise, start a lowest range +""" +import warnings +import numpy as np +__all__ = ['equivalent_load', 'find_range_count'] +__all__ += ['rainflow_astm', 'rainflow_windap','eq_load','eq_load_and_cycles','cycle_matrix','cycle_matrix2'] - Returns - ------- - Leq : the equivalent load for given m and Tea + +def equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap', + meanBin=True, binStartAt0=False, + outputMore=False, debug=False): + """Equivalent load calculation + + Calculate the damage equivalent load for a given signal and a given Wohler exponent + + INPUTS + - time : array-like, the time values corresponding to the signal (s) + - signals : array-like, the load signal + - m : Wohler exponent (default is 3) + - Teq : The equivalent period (Default 1, for 1Hz) + - bins : Number of bins in rainflow count histogram + - method: rain flow counting algorithm: 'rainflow_windap', 'rainflow_astm' or 'fatpack' + - meanBin: if True, use the mean of the ranges within a bin (recommended) + otherwise use the middle of the bin (not recommended). + - binStartAt0: if True bins start at zero. Otherwise, start a lowest range + - outputMore: if True, returns range, cycles and bins as well + + OUTPUTS + - Leq : the equivalent load for given m and Teq + + or (if outputMore is True ) + + - Leq, S, N, bins, DELi: + - S: ranges + - N: cycles + - bins: bin edges + - DELi: component 'i' of the DEL (for cycle i) """ time = np.asarray(time) signal = np.asarray(signal) @@ -64,37 +77,23 @@ def equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap' T = time[-1]-time[0] # time length of signal (s) - neq = T/Teq # number of equivalent periods + neq = T/Teq # number of equivalent periods, see Eq. (26) of [1] - if method in rainflow_func_dict.keys(): - # Call wetb function for one m - #Leq = eq_load(signal, m=[m], neq=neq, no_bins=bins, rainflow_func=rainflow_func_dict[method])[0][0] - N, S = find_range_count(signal, bins=bins, method=method, meanBin=meanBin) - elif method=='fatpack': - import fatpack - # find rainflow ranges - try: - ranges = fatpack.find_rainflow_ranges(signal) - except IndexError: - # Currently fails for constant signal - return np.nan + # --- Range (S) and counts (N) + N, S, bins = find_range_count(signal, bins=bins, method=method, meanBin=meanBin, binStartAt0=binStartAt0) - # --- Legacy fatpack - # if (not binStartAt0) and (not meanBin): - # N, S = fatpack.find_range_count(ranges, bins) - # --- Setup bins - # If binStartAt0 is True, the same bins as WINDAP are used - bins = create_bins(ranges, bins, binStartAt0=binStartAt0) - # --- Using bin_count to get value at center of bins - N, S = bin_count(ranges, bins, meanBin=meanBin) + # --- get DEL + DELi = S**m * N / neq + Leq = DELi.sum() ** (1/m) # See e.g. eq. (30) of [1] + if debug: + for i,(b,n,s,DEL) in enumerate(zip(bins, N, S, DELi)): + if n>0: + print('Bin {:3d}: [{:6.1f}-{:6.1f}] Mid:{:6.1f} - Mean:{:6.1f} Counts:{:4.1f} DEL:{:8.1f} Fraction:{:3.0f}%'.format(i,b,bins[i+1],(b+bins[i+1])/2,s,n,DEL,DEL/Leq**m*100)) + if outputMore: + return Leq, S, N, bins, DELi else: - raise NotImplementedError(method) - - # get DEL - DELs = S**m * N / neq - Leq = DELs.sum() ** (1/m) - return Leq + return Leq def find_range_count(signal, bins, method='rainflow_windap', meanBin=True, binStartAt0=True): @@ -112,21 +111,45 @@ def find_range_count(signal, bins, method='rainflow_windap', meanBin=True, binSt S is either the center of the bin (meanBin=False) or S is the mean of the ranges within this bin (meanBin=True) + - S_bin_edges: edges of the bins """ - rainflow_func = rainflow_func_dict[method] - N, S, S_bin_edges, _, _ = cycle_matrix(signal, ampl_bins=bins, mean_bins=1, rainflow_func=rainflow_func, binStartAt0=binStartAt0) - S_bin_edges = S_bin_edges.flatten() - N = N.flatten() - S = S.flatten() - S_mid = (S_bin_edges[:-1] + S_bin_edges[1:]) / 2 - if not meanBin: - S=S_mid + if method in rainflow_func_dict.keys(): + rainflow_func = rainflow_func_dict[method] + N, S, S_bin_edges, _, _ = cycle_matrix(signal, ampl_bins=bins, mean_bins=1, rainflow_func=rainflow_func, binStartAt0=binStartAt0) + S_bin_edges = S_bin_edges.flatten() + N = N.flatten() + S = S.flatten() + S_mid = (S_bin_edges[:-1] + S_bin_edges[1:]) / 2 + if not meanBin: + S=S_mid + + elif method=='fatpack': + import fatpack + # find rainflow ranges + try: + ranges = fatpack.find_rainflow_ranges(signal) + except IndexError: + # Currently fails for constant signal + return np.nan, np.nan, np.nan + # --- Legacy fatpack + # if (not binStartAt0) and (not meanBin): + # N, S = fatpack.find_range_count(ranges, bins) + # --- Setup bins + # If binStartAt0 is True, the same bins as WINDAP are used + S_bin_edges = create_bins(ranges, bins, binStartAt0=binStartAt0) + # --- Using bin_count to get value at center of bins + N, S = bin_count(ranges, S_bin_edges, meanBin=meanBin) + + else: + raise NotImplementedError('Rain flow algorithm {}'.format(method)) + # Remove NaN b = np.isnan(S) S[b] = 0 N[b] = 0 - return N, S + + return N, S, S_bin_edges def create_bins(x, bins, binStartAt0=False): """ diff --git a/pydatview/tools/signal_analysis.py b/pydatview/tools/signal_analysis.py index 1e7a169..253a9f1 100644 --- a/pydatview/tools/signal_analysis.py +++ b/pydatview/tools/signal_analysis.py @@ -414,7 +414,7 @@ def zero_crossings(y, x=None, direction=None, bouncingZero=False): # --------------------------------------------------------------------------------} # --- Correlation # --------------------------------------------------------------------------------{ -def correlation(x, nMax=80, dt=1, method='manual'): +def correlation(x, nMax=80, dt=1, method='numpy'): """ Compute auto correlation of a signal """ @@ -424,14 +424,18 @@ def acf(x, nMax=20): nvec = np.arange(0,nMax) - sigma2 = np.var(x) - R = np.zeros(nMax) - #R[0] =1 - #for i,nDelay in enumerate(nvec[1:]): - # R[i+1] = np.mean( x[0:-nDelay] * x[nDelay:] ) / sigma2 - # R[i+1] = np.corrcoef(x[:-nDelay], x[nDelay:])[0,1] - - R= acf(x, nMax=nMax) + if method=='manual': + sigma2 = np.var(x) + R = np.zeros(nMax) + R[0] =1 + for i,nDelay in enumerate(nvec[1:]): + R[i+1] = np.mean( x[0:-nDelay] * x[nDelay:] ) / sigma2 + #R[i+1] = np.corrcoef(x[:-nDelay], x[nDelay:])[0,1] + + elif method=='numpy': + R= acf(x, nMax=nMax) + else: + raise NotImplementedError() tau = nvec*dt return R, tau diff --git a/pydatview/tools/spectral.py b/pydatview/tools/spectral.py index 028ed10..0a1cb9a 100644 --- a/pydatview/tools/spectral.py +++ b/pydatview/tools/spectral.py @@ -1013,6 +1013,321 @@ def _triage_segments(window, nperseg,input_length): +# -------------------------------------------------------------------------------- +# --- Simple implementations to figure out the math +# -------------------------------------------------------------------------------- +def DFT(x, method='vectorized'): + """ + Calculate the Discrete Fourier Transform (DFT) of real signal x + + Definition: + for k in 0..N-1 (but defined for k in ZZ, see below for index) + + X_k = sum_n=0^{N-1} x_n e^{-i 2 pi k n /N} + + = sum_n=0^{N-1} x_n [ cos( 2 pi k n /N) - i sin( 2 pi k n /N) + + Xk are complex numbers. The amplitude/phase are: + A = |X_k|/N + phi = atan2(Im(Xk) / Re(Xk) + + Indices: + The DFT creates a periodic signal of period N + X[k] = X[k+N] + X[-k] = X[N-k] + + Therefore, any set of successive indices could be used. + For instance, with N=4: [0,1,2,3], [-1,0,1,2], [-2,-1,0,1] (canonical one) + [0,..N/2-1], [-N/2, ..N/2-1] + + If N is even + 0 is the 0th frequency (mean) + 1.....N/2-1 terms corresponds to positive frequencies + N/2.....N-1 terms corresponds to negative frequencies + If N is odd + 0 is the 0th frequency (mean) + 1.....(N-1)/2 terms corresponds to positive frequencies + (N+1)/2..N-1 terms corresponds to negative frequencies + + Frequencies convention: (see np.fft.fftfreq and DFT_freq) + f = [0, 1, ..., n/2-1, -n/2, ..., -1] / (dt*n) if n is even + f = [0, 1, ..., (n-1)/2, -(n-1)/2, ..., -1] / (dt*n) if n is odd + + NOTE: when n is even you could chose to go to +n/2 and start at -n/2+1 + The Python convention goes to n/2-1 and start at -n/2. + + Properties: + - if x is a real signal + X[-k] = X*[k] (=X[N-k]) + - Parseval's theorem: sum |x_n|^2 = sum |X_k|^2 (energy conservation) + + + """ + N = len(x) + + if method=='naive': + X = np.zeros_like(x, dtype=complex) + for k in np.arange(N): + for n in np.arange(N): + X[k] += x[n] * np.exp(-1j * 2*np.pi * k * n / N) + + elif method=='vectorized': + n = np.arange(N) + k = n.reshape((N, 1)) # k*n will be of shape (N x N) + e = np.exp(-2j * np.pi * k * n / N) + X = np.dot(e, x) + elif method=='fft': + X = np.fft.fft(x) + elif method=='fft_py': + X = recursive_fft(x) + else: + raise NotImplementedError() + + return X + +def IDFT(X, method='vectorized'): + """ + Calculate the Inverse Discrete Fourier Transform (IDFT) of complex coefficients X + + The transformation DFT->IDFT is fully reversible + + Definition: + for n in 0..N-1: + + x_n = 1/N sum_k=0^{N-1} X_k e^{i 2 pi k n/N} + = 1/N sum_k=0^{N-1} X_k [ cos( 2 pi k n/N) + i sin( 2 pi k n/N) + = 1/N sum_k=0^{N-1} A_k [ cos( 2 pi k n/N + phi_k) + i sin( 2 pi k n/N + phi_k) + + Xk are complex numbers, that can be written X[k] = A[k] e^{j phi[k]} therefore. + + Properties: + - if the "X" given as input come from a DFT, then the coefficients are periodic with period N + Therefore + X[-k] = X[N-k], + X[k] = X[N+k] + and therefore (see the discussion in the documentation of DFT), the summation from + k=0 to N-1 can be interpreted as a summation over any other indices set of length N. + - if "X" comes for the DFT of x, where x is a real signal, then: + X[-k] = X*[k] (=X[N-k]) + + - a converse is that, if X has conjugate symmetry (X[k]=X*[N-k]), then the IDFT will be real: + + x_n = 1/N sum_k=0^{N-1} A_k cos( 2 pi k n/N + phi_k) + 1/N sum_k={-(N-1)/2}^{(N-1)/2} A_k cos( 2 pi k n/N + phi_k) + + But remember that the A_k and phi_k need to satisfy the conjugate symmetry, so they are not + fully independent. + If we want x to be the sum over "N0" independent components, then we need to do the IDFT + of a spectrum "X" of length 2N0-1. + + Indices and frequency (python convention): + (see np.fft.fftfreq and DFT_freq) + f = [0, 1, ..., n/2-1, -n/2, ..., -1] / (dt*n) if n is even + f = [0, 1, ..., (n-1)/2, -(n-1)/2, ..., -1] / (dt*n) if n is odd + + When n is even, we lack symmetry of frequency, so it can potentially + make sense to enforce that the X[-n/2] component is 0 when generating + a signal with IDFT + + + + """ + N = len(X) + + if method in ['naive', 'manual', 'sum']: + x = np.zeros_like(X, dtype=complex) + for k in np.arange(N): + for n in np.arange(N): + x[k] += X[n] * np.exp(1j * 2*np.pi * k * n / N) + x = x/N + + elif method=='vectorized': + n = np.arange(N) + k = n.reshape((N, 1)) # k*n will be of shape (N x N) + e = np.exp(2j * np.pi * k * n / N) + x = np.dot(e, X) / N + + elif method=='ifft': + x = np.fft.ifft(X) + + #elif method=='ifft_py': + # x = IFFT(X) + else: + raise NotImplementedError('IDFT: Method {}'.format(method)) + + x = np.real_if_close(x) + + return x + +def DFT_freq(time=None, N=None, T=None, doublesided=True): + """ Returns the frequencies corresponding to a time vector `time`. + The signal "x" and "time" are assumed to have the same length + INPUTS: + - time: 1d array of time + OR + - N: number of time values + - T: time length of signal + """ + if time is not None: + N = len(time) + T = time[-1]-time[0] + dt = T/(N-1) + df = 1/(dt*N) + nhalf_pos, nhalf_neg = nhalf_fft(N) + if doublesided: + freq_pos = np.arange(nhalf_pos+1)*df + freq_neg = np.arange(nhalf_neg,0)*df + freq = np.concatenate((freq_pos, freq_neg)) + assert(len(freq) == N) + else: + # single sided + fMax = nhalf_pos * df + #freq = np.arange(0, fMax+df/2, df) + freq = np.arange(nhalf_pos+1)*df + return freq + +def IDFT_time(freq=None, doublesided=True): + """ Returns the time vector corresponding to a frequency vector `freq`. + + If doublesided is True + The signal "x" , "time" and freq are assumed to have the same length + Note: might lead to some inaccuracies, just use for double checking! + + INPUTS: + - freq: 1d array of time + """ + if doublesided: + N = len(freq) + time = freq*0 + if np.mod(N,2)==0: + nhalf=int(N/2)-1 + else: + nhalf=int((N-1)/2) + fMax = freq[nhalf] + df = (fMax-0)/(nhalf) + dt = 1/(df*N) + tMax= (N-1)*dt + #time = np.arange(0,(N-1)*dt+dt/2, dt) + time = np.linspace(0,tMax, N) + else: + raise NotImplementedError() + return time + +def recursive_fft(x): + """ + A recursive implementation of the 1D Cooley-Tukey FFT + + Returns the same as DFT (see documentation) + + Input should have a length of power of 2. + Reference: Kong, Siauw, Bayen - Python Numerical Methods + """ + N = len(x) + if not is_power_of_two(N): + raise Exception('Recursive FFT requires a power of 2') + + + if N == 1: + return x + else: + X_even = recursive_fft(x[::2]) + X_odd = recursive_fft(x[1::2]) + factor = np.exp(-2j*np.pi*np.arange(N)/ N) + X = np.concatenate([X_even+factor[:int(N/2)]*X_odd, X_even+factor[int(N/2):]*X_odd]) + return X + +def nhalf_fft(N): + """ + Follows the convention of fftfreq + fmax = f[nhalf_pos] = nhalf_pos*df (fftfreq convention) + + fpos = f[:nhalf_pos+1] + fneg = f[nhalf_pos+1:] + + """ + if N%2 ==0: + nhalf_pos = int(N/2)-1 + nhalf_neg = -int(N/2) + else: + nhalf_pos = int((N-1)/2) + nhalf_neg = -nhalf_pos + return nhalf_pos, nhalf_neg + + +def check_DFT_real(X): + """ Check that signal X is the DFT of a real signal + and that therefore IDFT(X) will return a real signal. + For this to be the case, we need conjugate symmetry: + X[k] = X[N-k]* + """ + from welib.tools.spectral import nhalf_fft + N = len(X) + nh, _ = nhalf_fft(N) + Xpos = X[1:nh+1] # we dont take the DC component [0] + Xneg = np.flipud(X[nh+1:]) # might contain one more frequency than the pos part + + if np.mod(N,2)==0: + # We have one extra negative frequency, we check that X is zero there and remove the value. + if Xneg[-1]!=0: + raise Exception('check_DFT_real: Component {} (first negative frequency) is {} instead of zero, but it should be zero if N is even.'.format(nh+1, X[nh+1])) + Xneg = Xneg[:-1] + + notConjugate = Xpos-np.conjugate(Xneg)!=0 + if np.any(notConjugate): + nNotConjugate=sum(notConjugate) + I = np.where(notConjugate)[0][:3] + 1 # +1 for DC component that was removed + raise Exception('check_DFT_real: {}/{} values of the spectrum are not complex conjugate of there symmetric frequency counterpart. See for instance indices: {}'.format(nNotConjugate, nh, I)) + +def double_sided_DFT_real(X1, N=None): + """ + Take a single sided part of a DFT (X1) and make it double sided signal X, of length N, + ensuring that the IDFT of X will be real. + This is done by ensuring conjugate symmetry: + X[k] = X[N-k]* + For N even, the first negative frequency component is set to 0 because it has no positive counterpart. + + Calling check_DFT_real(X) should return no Exception. + + INPUTS: + - X1: array of complex values of length N1 + - N: required length of the output array (2N1-1 or 2N1) + OUTPUTS: + - X: double sided spectrum: + [X1 flip(X1*[1:]) ] + or + [X1 [0] flip(X1*[1:]) ] + """ + if N is None: + N=2*len(X1)-1 # we make it an odd number to ensure symmetry of frequency + else: + if N not in [2*len(X1)-1, 2*len(X1), 2*len(X1)-2]: + raise Exception('N should be twice the length of the single sided spectrum, or one less.') + + if N % 2 ==0: + # Even number + if N == 2*len(X1)-2: + # rfftfreq + # TODO, there look into irfft to see the convention + X = np.concatenate((X1[:-1], [0], np.flipud(np.conjugate(X1[1:-1])))) + else: + X = np.concatenate((X1, [0], np.flipud(np.conjugate(X1[1:])))) + else: + X = np.concatenate((X1, np.flipud(np.conjugate(X1[1:])))) + return X + + +# --------------------------------------------------------------------------------} +# --- Helper functions +# --------------------------------------------------------------------------------{ +def is_power_of_two(n): + """ Uses bit manipulation to figure out if an integer is a power of two""" + return (n != 0) and (n & (n-1) == 0) + +def sinesum(time, As, freqs): + x =np.zeros_like(time) + for ai,fi in zip(As, freqs): + x += ai*np.sin(2*np.pi*fi*time) + return x # --------------------------------------------------------------------------------} @@ -1022,6 +1337,68 @@ def _triage_segments(window, nperseg,input_length): class TestSpectral(unittest.TestCase): + def default_signal(self, time, mean=0): + freqs=[1,4,7 ] # [Hz] + As =[3,1,1/2] # [misc] + x = sinesum(time, As, freqs) + mean + return x + + def compare_with_npfft(self, time, x): + # Compare lowlevels functions with npfft + # Useful to make sure the basic math is correct + N = len(time) + dt = (time[-1]-time[0])/(N-1) + tMax = time[-1] + + # --- Test frequency, dt/df/N-relationships + f_ref = np.fft.fftfreq(N, dt) + nhalf_pos, nhalf_neg = nhalf_fft(N) + fhalf = DFT_freq(time, doublesided = False) + freq = DFT_freq(time, doublesided = True) + df = freq[1]-freq[0] + fmax = fhalf[-1] + + np.testing.assert_almost_equal(fhalf , f_ref[:nhalf_pos+1], 10) + np.testing.assert_almost_equal(fhalf[-1] , np.max(f_ref), 10) + np.testing.assert_almost_equal( 1/(dt*df), N) + np.testing.assert_almost_equal(freq , f_ref, 10) + if N%2 == 0: + np.testing.assert_almost_equal(2*fmax/df, N-2 , 10) + else: + np.testing.assert_almost_equal(2*fmax/df, N-1 , 10) + + # --- Test DFT methods + X0 = DFT(x, method='fft') + X1 = DFT(x, method='naive') + X2 = DFT(x, method='vectorized') + + np.testing.assert_almost_equal(X1, X0, 10) + np.testing.assert_almost_equal(X2, X0, 10) + if is_power_of_two(N): + X3 = DFT(x, method='fft_py') + np.testing.assert_almost_equal(X3, X0, 10) + + # --- Test IDFT methods + x_back0 = IDFT(X0, method='ifft') + x_back1 = IDFT(X0, method='naive') + x_back2 = IDFT(X0, method='vectorized') + np.testing.assert_almost_equal(x_back1, x_back0, 10) + np.testing.assert_almost_equal(x_back2, x_back0, 10) + + np.testing.assert_almost_equal(x_back0, x, 10) + + def test_lowlevel_fft_even(self): + # Test lowlevel functions + time = np.linspace(0,10,16) # NOTE: need a power of two for fft_py + x = self.default_signal(time, mean=0) + self.compare_with_npfft(time, x) + + def test_lowlevel_fft_odd(self): + # Test lowlevel functions + time = np.linspace(0,10,17) + x = self.default_signal(time, mean=0) + self.compare_with_npfft(time, x) + def test_fft_amplitude(self): dt=0.1 t=np.arange(0,10,dt); @@ -1057,5 +1434,8 @@ def test_fft_binning(self): if __name__ == '__main__': #TestSpectral().test_fft_binning() + #TestSpectral().test_ifft() + #TestSpectral().test_lowlevel_fft_even() + #TestSpectral().test_lowlevel_fft_odd() unittest.main() diff --git a/pydatview/tools/stats.py b/pydatview/tools/stats.py index 2ba3cf3..ff6c794 100644 --- a/pydatview/tools/stats.py +++ b/pydatview/tools/stats.py @@ -172,6 +172,22 @@ def myabs(y): # --------------------------------------------------------------------------------} # --- PDF # --------------------------------------------------------------------------------{ +def pdf(y, method='histogram', n=50, **kwargs): + """ + Compute the probability density function. + Wrapper over the different methods present in this package + """ + if method =='sns': + xh, yh = pdf_sns(y, nBins=n, **kwargs) + elif method =='gaussian_kde': + xh, yh = pdf_gaussian_kde(y, nOut=n, **kwargs) + elif method =='histogram': + xh, yh = pdf_histogram(y, nBins=n, **kwargs) + else: + raise NotImplementedError(f'pdf method: {method}') + return xh, yh + + def pdf_histogram(y,nBins=50, norm=True, count=False): yh, xh = np.histogram(y[~np.isnan(y)], bins=nBins) dx = xh[1] - xh[0] diff --git a/pydatview/tools/tictoc.py b/pydatview/tools/tictoc.py index 6df9e2d..2c1bebe 100644 --- a/pydatview/tools/tictoc.py +++ b/pydatview/tools/tictoc.py @@ -44,15 +44,17 @@ class Timer(object): cmd1 cmd2 """ - def __init__(self, name=None, writeBefore=False, silent=False): + def __init__(self, name=None, writeBefore=False, silent=False, nChar=40): self.name = name self.writeBefore = writeBefore self.silent=silent + self.nChar=nChar + self.sFmt='{:'+str(nChar+1)+'s}' def ref_str(self): s='[TIME] ' if self.name: - s+='{:31s}'.format(self.name[:30]) + s+=self.sFmt.format(self.name[:self.nChar]) return s def __enter__(self): From 5c31f4cf311c1d1adc9bc213ce24d024db6dc724 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 13 Sep 2023 19:04:11 -0600 Subject: [PATCH 116/178] IO: updates from welib --- pydatview/fast/fastfarm.py | 59 ++++++++----- pydatview/io/__init__.py | 12 +-- pydatview/io/fast_linearization_file.py | 108 ++++++++++++++++-------- pydatview/io/fast_output_file.py | 19 ++++- pydatview/io/mannbox_file.py | 10 +-- pydatview/io/turbsim_file.py | 88 +++++++++++++++---- 6 files changed, 203 insertions(+), 93 deletions(-) diff --git a/pydatview/fast/fastfarm.py b/pydatview/fast/fastfarm.py index 1c48511..663f4cf 100644 --- a/pydatview/fast/fastfarm.py +++ b/pydatview/fast/fastfarm.py @@ -12,7 +12,7 @@ # --------------------------------------------------------------------------------} # --- Small helper functions # --------------------------------------------------------------------------------{ -def insertTN(s,i,nWT=1000): +def insertTN(s,i,nWT=1000, noLeadingZero=False): """ insert turbine number in name """ if nWT<10: fmt='{:d}' @@ -20,8 +20,15 @@ def insertTN(s,i,nWT=1000): fmt='{:02d}' else: fmt='{:03d}' + + if noLeadingZero: + fmt='{:d}' + if s.find('T1')>=0: s=s.replace('T1','T'+fmt.format(i)) + elif s.find('T0')>=0: + print('this should not be printed') + s=s.replace('T0','T'+fmt.format(i)) else: sp=os.path.splitext(s) s=sp[0]+'_T'+fmt.format(i)+sp[1] @@ -251,7 +258,7 @@ def fastFarmBoxExtent(yBox, zBox, tBox, meanU, hubHeight, D, xWT, yWT, nX_Low = int(np.ceil(LX_Low/dX_Low)) nY_Low = int(np.ceil(LY_Low/dY_Low)) nZ_Low = int(np.ceil(LZ_Low/dZ_Low)) - # Make sure we don't exceed box in Y and Z + # Make sure we don't exceed box in Y and Z #rt: this essentially gives us 1 less grid point than what is on the inp/bst files if (nY_Low*dY_Low>LY_Box): nY_Low=nY_Low-1 if (nZ_Low*dZ_Low>LZ_Box): nZ_Low=nZ_Low-1 @@ -309,14 +316,19 @@ def fastFarmBoxExtent(yBox, zBox, tBox, meanU, hubHeight, D, xWT, yWT, dY = Y_rel - np.round(Y_rel) # Should be close to zero if any(abs(dX)>1e-3): print('Deltas:',dX) - raise Exception('Some X0_High are not on an integer multiple of the high-res grid') + print('Exception has been raise. I put this print statement instead. Check with EB.') + print('Exception: Some X0_High are not on an integer multiple of the high-res grid') + #raise Exception('Some X0_High are not on an integer multiple of the high-res grid') if any(abs(dY)>1e-3): print('Deltas:',dY) - raise Exception('Some Y0_High are not on an integer multiple of the high-res grid') + print('Exception has been raise. I put this print statement instead. Check with EB.') + print('Exception: Some Y0_High are not on an integer multiple of the high-res grid') + #raise Exception('Some Y0_High are not on an integer multiple of the high-res grid') return d -def writeFastFarm(outputFile, templateFile, xWT, yWT, zWT, FFTS=None, OutListT1=None): + +def writeFastFarm(outputFile, templateFile, xWT, yWT, zWT, FFTS=None, OutListT1=None, noLeadingZero=False): """ Write FastFarm input file based on a template, a TurbSimFile and the Layout outputFile: .fstf file to be written @@ -350,7 +362,7 @@ def writeFastFarm(outputFile, templateFile, xWT, yWT, zWT, FFTS=None, OutListT1= WT[iWT,0]=x WT[iWT,1]=y WT[iWT,2]=z - WT[iWT,3]=insertTN(ref_path,iWT+1,nWT) + WT[iWT,3]=insertTN(ref_path,iWT+1,nWT,noLeadingZero=noLeadingZero) if FFTS is not None: WT[iWT,4]=FFTS['X0_High'][iWT] WT[iWT,5]=FFTS['Y0_High'][iWT] @@ -371,15 +383,19 @@ def setFastFarmOutputs(fastFarmFile, OutListT1): OutList=[''] for s in OutListT1: s=s.strip('"') - if s.find('T1'): + if 'T1' in s: OutList+=['"'+s.replace('T1','T{:d}'.format(iWT+1))+'"' for iWT in np.arange(nWTOut) ] + elif 'W1VAmb' in s: # special case for ambient wind + OutList+=['"'+s.replace('1','{:d}'.format(iWT+1))+'"' for iWT in np.arange(nWTOut) ] + elif 'W1VDis' in s: # special case for disturbed wind + OutList+=['"'+s.replace('1','{:d}'.format(iWT+1))+'"' for iWT in np.arange(nWTOut) ] else: OutList+='"'+s+'"' fst['OutList']=OutList fst.write(fastFarmFile) -def plotFastFarmSetup(fastFarmFile, grid=True, fig=None, D=None, plane='XY', hubHeight=None): +def plotFastFarmSetup(fastFarmFile, grid=True, fig=None, D=None, plane='XY', hubHeight=None, showLegend=True): """ """ import matplotlib.pyplot as plt @@ -476,7 +492,8 @@ def boundingBox(x, y): ax.plot(x, y, '-', lw=2, c=col(wt)) #plt.legend(bbox_to_anchor=(1.05,1.015),frameon=False) - ax.legend() + if showLegend: + ax.legend() if plane=='XY': ax.set_xlabel("x [m]") ax.set_ylabel("y [m]") @@ -589,6 +606,7 @@ def spanwisePostProFF(fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1, vr=None vD=None D=0 + main is None else: main=FASTInputFile(fastfarm_input) iOut = main['OutRadii'] @@ -614,20 +632,21 @@ def spanwisePostProFF(fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1, ColsInfo, nrMax = spanwiseColFastFarm(df.columns.values, nWT=nWT, nD=nD) dfRad = fastlib.extract_spanwise_data(ColsInfo, nrMax, df=None, ts=dfAvg.iloc[0]) #dfRad = fastlib.insert_radial_columns(dfRad, vr) - if vr is None: - dfRad.insert(0, 'i_[#]', np.arange(nrMax)+1) - else: - dfRad.insert(0, 'r_[m]', vr[:nrMax]) - dfRad['i/n_[-]']=np.arange(nrMax)/nrMax + if dfRad is not None: + if vr is None: + dfRad.insert(0, 'i_[#]', np.arange(nrMax)+1) + else: + dfRad.insert(0, 'r_[m]', vr[:nrMax]) + dfRad['i/n_[-]']=np.arange(nrMax)/nrMax # --- Extract downstream data ColsInfo, nDMax = diameterwiseColFastFarm(df.columns.values, nWT=nWT) dfDiam = fastlib.extract_spanwise_data(ColsInfo, nDMax, df=None, ts=dfAvg.iloc[0]) - #dfDiam = fastlib.insert_radial_columns(dfDiam) - if vD is None: - dfDiam.insert(0, 'i_[#]', np.arange(nDMax)+1) - else: - dfDiam.insert(0, 'x_[m]', vD[:nDMax]) - dfDiam['i/n_[-]'] = np.arange(nDMax)/nDMax + if dfDiam is not None: + if vD is None: + dfDiam.insert(0, 'i_[#]', np.arange(nDMax)+1) + else: + dfDiam.insert(0, 'x_[m]', vD[:nDMax]) + dfDiam['i/n_[-]'] = np.arange(nDMax)/nDMax return dfRad, dfRadialTime, dfDiam diff --git a/pydatview/io/__init__.py b/pydatview/io/__init__.py index 5eed528..c132bbf 100644 --- a/pydatview/io/__init__.py +++ b/pydatview/io/__init__.py @@ -1,3 +1,4 @@ +# --- Generic reader / fileformat detection from .file import File, WrongFormatError, BrokenFormatError, FileNotFoundError, EmptyFileError, OptionalImportError from .file_formats import FileFormat, isRightFormat import sys @@ -266,15 +267,4 @@ def read(filename, fileformat=None, **kwargs): return F -# --- For legacy code -def FASTInputFile(*args,**kwargs): - from .fast_input_file import FASTInputFile as fi - return fi(*args,**kwargs) -def FASTOutputFile(*args,**kwargs): - from .fast_output_file import FASTOutputFile as fo - return fo(*args,**kwargs) -def CSVFile(*args,**kwargs): - from .csv_file import CSVFile as csv - return csv(*args,**kwargs) - diff --git a/pydatview/io/fast_linearization_file.py b/pydatview/io/fast_linearization_file.py index d9f8f8a..28e1ace 100644 --- a/pydatview/io/fast_linearization_file.py +++ b/pydatview/io/fast_linearization_file.py @@ -1,11 +1,10 @@ +import os import numpy as np -import pandas as pd import re try: from .file import File, WrongFormatError, BrokenFormatError except: File = dict - class WrongFormatError(Exception): pass class BrokenFormatError(Exception): pass class FASTLinearizationFile(File): @@ -42,6 +41,27 @@ def defaultExtensions(): def formatName(): return 'FAST linearization output' + def __init__(self, filename=None, **kwargs): + """ Class constructor. If a `filename` is given, the file is read. """ + self.filename = filename + if filename: + self.read(**kwargs) + + def read(self, filename=None, **kwargs): + """ Reads the file self.filename, or `filename` if provided """ + + # --- Standard tests and exceptions (generic code) + if filename: + self.filename = filename + if not self.filename: + raise Exception('No filename provided') + if not os.path.isfile(self.filename): + raise OSError(2,'File not found:',self.filename) + if os.stat(self.filename).st_size == 0: + raise EmptyFileError('File is empty:',self.filename) + # --- Calling (children) function to read + self._read(**kwargs) + def _read(self, *args, **kwargs): self['header']=[] @@ -61,7 +81,7 @@ def readToMarker(fid, marker, nMax): lines.append(line.strip()) return lines, line - def readOP(fid, n): + def readOP(fid, n, name=''): OP=[] Var = {'RotatingFrame': [], 'DerivativeOrder': [], 'Description': []} colNames=fid.readline().strip() @@ -86,11 +106,30 @@ def readOP(fid, n): Var['Description'].append(' '.join(sp[iRot+1:]).strip()) if i>=n-1: break + OP=np.asarray(OP) return OP, Var - def readMat(fid, n, m): - vals=[f.readline().strip().split() for i in np.arange(n)] - return np.array(vals).astype(float) + def readMat(fid, n, m, name=''): + pattern = re.compile(r"[\*]+") + vals=[pattern.sub(' inf ', fid.readline().strip() ).split() for i in np.arange(n)] + vals = np.array(vals) + try: + vals = np.array(vals).astype(float) # This could potentially fail + except: + raise Exception('Failed to convert into an array of float the matrix `{}`\n\tin linfile: {}'.format(name, self.filename)) + if vals.shape[0]!=n or vals.shape[1]!=m: + shape1 = vals.shape + shape2 = (n,m) + raise Exception('Shape of matrix `{}` has wrong dimension ({} instead of {})\n\tin linfile: {}'.format(name, shape1, shape2, name, self.filename)) + + nNaN = sum(np.isnan(vals.ravel())) + nInf = sum(np.isinf(vals.ravel())) + if nInf>0: + raise Exception('Some ill-formated/infinite values (e.g. `*******`) were found in the matrix `{}`\n\tin linflile: {}'.format(name, self.filename)) + if nNaN>0: + raise Exception('Some NaN values were found in the matrix `{}`\n\tin linfile: `{}`.'.format(name, self.filename)) + return vals + # Reading with open(self.filename, 'r', errors="surrogateescape") as f: @@ -116,35 +155,35 @@ def readMat(fid, n, m): except: self['WindSpeed'] = None - KEYS=['Order of','A:','B:','C:','D:','ED M:', 'dUdu','dUdy'] - for i, line in enumerate(f): line = line.strip() - KeyFound=any([line.find(k)>=0 for k in KEYS]) - if KeyFound: - if line.find('Order of continuous states:')>=0: - self['x'], self['x_info'] = readOP(f, nx) - elif line.find('Order of continuous state derivatives:')>=0: - self['xdot'], self['xdot_info'] = readOP(f, nx) - elif line.find('Order of inputs')>=0: - self['u'], self['u_info'] = readOP(f, nu) - elif line.find('Order of outputs')>=0: - self['y'], self['y_info'] = readOP(f, ny) - elif line.find('A:')>=0: - self['A'] = readMat(f, nx, nx) - elif line.find('B:')>=0: - self['B'] = readMat(f, nx, nu) - elif line.find('C:')>=0: - self['C'] = readMat(f, ny, nx) - elif line.find('D:')>=0: - self['D'] = readMat(f, ny, nu) - elif line.find('dUdu:')>=0: - self['dUdu'] = readMat(f, nu, nu) - elif line.find('dUdy:')>=0: - self['dUdy'] = readMat(f, nu, ny) - elif line.find('ED M:')>=0: - self['EDDOF'] = line[5:].split() - self['M'] = readMat(f, 24, 24) + if line.find('Order of continuous states:')>=0: + self['x'], self['x_info'] = readOP(f, nx, 'x') + elif line.find('Order of continuous state derivatives:')>=0: + self['xdot'], self['xdot_info'] = readOP(f, nx, 'xdot') + elif line.find('Order of inputs')>=0: + self['u'], self['u_info'] = readOP(f, nu, 'u') + elif line.find('Order of outputs')>=0: + self['y'], self['y_info'] = readOP(f, ny, 'y') + elif line.find('A:')>=0: + self['A'] = readMat(f, nx, nx, 'A') + elif line.find('B:')>=0: + self['B'] = readMat(f, nx, nu, 'B') + elif line.find('C:')>=0: + self['C'] = readMat(f, ny, nx, 'C') + elif line.find('D:')>=0: + self['D'] = readMat(f, ny, nu, 'D') + elif line.find('dUdu:')>=0: + self['dUdu'] = readMat(f, nu, nu,'dUdu') + elif line.find('dUdy:')>=0: + self['dUdy'] = readMat(f, nu, ny,'dUdy') + elif line.find('StateRotation:')>=0: + pass + # TODO + #StateRotation: + elif line.find('ED M:')>=0: + self['EDDOF'] = line[5:].split() + self['M'] = readMat(f, 24, 24,'M') def toString(self): s='' @@ -316,7 +355,8 @@ def udescr(self): else: return [] - def _toDataFrame(self): + def toDataFrame(self): + import pandas as pd dfs={} xdescr_short = self.xdescr() diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index 1c3005b..5c00bbc 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -235,23 +235,30 @@ def isBinary(filename): -def load_ascii_output(filename, method='numpy'): +def load_ascii_output(filename, method='numpy', encoding='ascii'): if method in ['forLoop','pandas']: from .file import numberOfLines nLines = numberOfLines(filename, method=2) - with open(filename) as f: + with open(filename, encoding=encoding, errors='ignore') as f: info = {} info['name'] = os.path.splitext(os.path.basename(filename))[0] # Header is whatever is before the keyword `time` - in_header = True header = [] - while in_header: + maxHeaderLines=35 + headerRead = False + for i in range(maxHeaderLines): l = f.readline() if not l: raise Exception('Error finding the end of FAST out file header. Keyword Time missing.') + # Check for utf-16 + if l[:3] == '\x00 \x00': + f.close() + encoding='' + print('[WARN] Attempt to re-read the file with encoding utf-16') + return load_ascii_output(filename=filename, method=method, encoding='utf-16') first_word = (l+' dummy').lower().split()[0] in_header= (first_word != 'time') and (first_word != 'alpha') if in_header: @@ -260,6 +267,10 @@ def load_ascii_output(filename, method='numpy'): info['description'] = header info['attribute_names'] = l.split() info['attribute_units'] = [unit[1:-1] for unit in f.readline().split()] + headerRead=True + break + if not headerRead: + raise WrongFormatError('Could not find the keyword "Time" or "Alpha" in the first {} lines of the file'.format(maxHeaderLines)) nHeader = len(header)+1 nCols = len(info['attribute_names']) diff --git a/pydatview/io/mannbox_file.py b/pydatview/io/mannbox_file.py index 7af29cc..d8dd6c4 100644 --- a/pydatview/io/mannbox_file.py +++ b/pydatview/io/mannbox_file.py @@ -47,8 +47,8 @@ class MannBoxFile(File): print(mb['field'].shape) # Use methods to extract relevant values - u,v,w = my.valuesAt(y=10.5, z=90) - z, means, stds = mb.vertProfile() + u = mb.valuesAt(y=10.5, z=90) + z, means, stds = mb.vertProfile # Write to a new file mb.write('Output_1024x16x16.u') @@ -144,10 +144,6 @@ def _read_nonbuffered(): self['y0']=y0 self['z0']=z0 self['zMid']=zMid -# print('1',self['field'][:,-1,0]) -# print('2',self['field'][0,-1::-1,0]) -# print('3',self['field'][0,-1,:]) - def write(self, filename=None): """ Write mann box """ @@ -176,7 +172,7 @@ def __repr__(self): s+='| * z: [{} ... {}], dz: {}, n: {} \n'.format(z[0],z[-1],self['dz'],len(z)) s+='|useful functions:\n' s+='| - t(dx, U)\n' - s+='| - valuesAt(y,z), vertProfile(), fromTurbSim(*), _iMid()\n' + s+='| - valuesAt(y,z), vertProfile, fromTurbSim(*), _iMid()\n' return s diff --git a/pydatview/io/turbsim_file.py b/pydatview/io/turbsim_file.py index 2e004c6..9d3cd7b 100644 --- a/pydatview/io/turbsim_file.py +++ b/pydatview/io/turbsim_file.py @@ -58,7 +58,7 @@ def __init__(self, filename=None, **kwargs): if filename: self.read(filename, **kwargs) - def read(self, filename=None, header_only=False): + def read(self, filename=None, header_only=False, tdecimals=8): """ read BTS file, with field: u (3 x nt x ny x nz) uTwr (3 x nt x nTwr) @@ -98,11 +98,11 @@ def read(self, filename=None, header_only=False): self['uTwr'] = uTwr self['info'] = info self['ID'] = ID - self['dt'] = dt + self['dt'] = np.round(dt, tdecimals) # dt is stored in single precision in the TurbSim output self['y'] = np.arange(ny)*dy self['y'] -= np.mean(self['y']) # y always centered on 0 self['z'] = np.arange(nz)*dz +zBottom - self['t'] = np.arange(nt)*dt + self['t'] = np.round(np.arange(nt)*dt, tdecimals) self['zTwr'] =-np.arange(nTwr)*dz + zBottom self['zRef'] = zHub self['uRef'] = uHub @@ -684,8 +684,12 @@ def toDataFrame(self): def toDataset(self): """ Convert the data that was read in into a xarray Dataset + + # TODO SORT OUT THE DIFFERENCE WITH toDataSet """ from xarray import IndexVariable, DataArray, Dataset + + print('[TODO] pyFAST.input_output.turbsim_file.toDataset: merge with function toDataSet') y = IndexVariable("y", self.y, attrs={"description":"lateral coordinate","units":"m"}) zround = np.asarray([np.round(zz,6) for zz in self.z]) #the open function here returns something like *.0000000001 which is annoying @@ -704,11 +708,55 @@ def toDataset(self): return Dataset(data_vars=da, coords={"time":time,"y":y,"z":z}) - # Useful converters - def fromAMRWind_PD(self, filename, timestep, output_frequency, sampling_identifier, verbose=1, fileout=None, zref=None, xloc=None): + def toDataSet(self, datetime=False): + """ + Convert the data that was read in into a xarray Dataset + + # TODO SORT OUT THE DIFFERENCE WITH toDataset """ - Reads a AMRWind netcdf file, grabs a group of sampling planes (e.g. p_slice), + import xarray as xr + print('[TODO] pyFAST.input_output.turbsim_file.toDataSet: should be discontinued') + print('[TODO] pyFAST.input_output.turbsim_file.toDataSet: merge with function toDataset') + + if datetime: + timearray = pd.to_datetime(self['t'], unit='s', origin=pd.to_datetime('2000-01-01 00:00:00')) + timestr = 'datetime' + else: + timearray = self['t'] + timestr = 'time' + + ds = xr.Dataset( + data_vars=dict( + u=([timestr,'y','z'], self['u'][0,:,:,:]), + v=([timestr,'y','z'], self['u'][1,:,:,:]), + w=([timestr,'y','z'], self['u'][2,:,:,:]), + ), + coords={ + timestr : timearray, + 'y' : self['y'], + 'z' : self['z'], + }, + ) + + # Add mean computations + ds['up'] = ds['u'] - ds['u'].mean(dim=timestr) + ds['vp'] = ds['v'] - ds['v'].mean(dim=timestr) + ds['wp'] = ds['w'] - ds['w'].mean(dim=timestr) + + if datetime: + # Add time (in s) to the variable list + ds['time'] = (('datetime'), self['t']) + + return ds + + # Useful converters + def fromAMRWind(self, filename, timestep, output_frequency, sampling_identifier, verbose=1, fileout=None, zref=None, xloc=None): + """ + Reads a AMRWind netcdf file, grabs a group of sampling planes (e.g. p_slice), + return an instance of TurbSimFile, optionally write turbsim file to disk + + Parameters ---------- filename : str, @@ -723,15 +771,14 @@ def fromAMRWind_PD(self, filename, timestep, output_frequency, sampling_identifi height to be written to turbsim as the reference height. if none is given, it is taken as the vertical centerpoint of the slice """ try: - from weio.amrwind_file import AMRWind + from pyFAST.input_output.amrwind_file import AMRWindFile except: try: - from .amrwind_file import AMRWind + from .amrwind_file import AMRWindFile except: - from amrwind_file import AMRWind + from amrwind_file import AMRWindFile - obj = AMRWind(filename,timestep,output_frequency) - obj.read(sampling_identifier) + obj = AMRWindFile(filename,timestep,output_frequency, group_name=sampling_identifier) self["u"] = np.ndarray((3,obj.nt,obj.ny,obj.nz)) @@ -759,11 +806,14 @@ def fromAMRWind_PD(self, filename, timestep, output_frequency, sampling_identifi self['uRef'] = float(obj.data.u.sel(x=xloc).sel(y=0).sel(z=self["zRef"]).mean().values) self['zRef'], self['uRef'], bHub = self.hubValues() - fileout = filename.replace(".nc",".bts") if fileout is None else fileout - print("===> {0}".format(fileout)) - self.write(fileout) + if fileout is not None: + filebase = os.path.splitext(filename)[1] + fileout = filebase+".bts" + if verbose: + print("===> {0}".format(fileout)) + self.write(fileout) - def fromAMRWind(self, filename, dt, nt): + def fromAMRWind_legacy(self, filename, dt, nt, y, z, sampling_identifier='p_sw2'): """ Convert current TurbSim file into one generated from AMR-Wind LES sampling data in .nc format Assumes: @@ -772,20 +822,24 @@ def fromAMRWind(self, filename, dt, nt): INPUTS: - filename: (string) full path to .nc sampling data file - - plane_label: (string) name of sampling plane group from .inp file (e.g. "p_sw2") + - sampling_identifier: (string) name of sampling plane group from .inp file (e.g. "p_sw2") - dt: timestep size [s] - nt: number of timesteps (sequential) you want to read in, starting at the first timestep available + INPUTS: TODO - y: user-defined vector of coordinate positions in y - z: user-defined vector of coordinate positions in z - uref: (float) reference mean velocity (e.g. 8.0 hub height mean velocity from input file) - zref: (float) hub height (e.t. 150.0) """ import xarray as xr + + print('[TODO] fromAMRWind_legacy: function might be unfinished. Merge with fromAMRWind') + print('[TODO] fromAMRWind_legacy: figure out y, and z from data (see fromAMRWind)') # read in sampling data plane ds = xr.open_dataset(filename, engine='netcdf4', - group=plane_label) + group=sampling_identifier) ny, nz, _ = ds.attrs['ijk_dims'] noffsets = len(ds.attrs['offsets']) t = np.arange(0, dt*(nt-0.5), dt) From 3a57a470889948f97b0909f9c328238b667a7ea1 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 13 Sep 2023 20:01:15 -0600 Subject: [PATCH 117/178] Misc updates for python 11 --- pydatview/Tables.py | 17 +++++++++++------ pydatview/common.py | 4 ++-- pydatview/io/fast_output_file.py | 4 ++-- pydatview/io/file_formats.py | 2 ++ pydatview/io/wetb/hawc2/htc_file.py | 4 ++-- pydatview/plotdata.py | 8 ++++---- pydatview/plugins/base_plugin.py | 2 +- pydatview/plugins/plotdata_filter.py | 4 ++-- pydatview/plugins/tool_curvefitting.py | 6 +++--- pydatview/plugins/tool_logdec.py | 2 +- pydatview/tools/fatigue.py | 20 ++++++++++++++++++-- 11 files changed, 48 insertions(+), 25 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index ca64dc0..bbc3ee8 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -516,22 +516,27 @@ def __init__(self,data=None, name='',filename='', fileformat=None, dayfirst=Fals if not isinstance(data,pd.DataFrame): raise NotImplementedError('Tables that are not dataframe not implemented.') - # --- Pandas DataFrame - self.data = data + # --- Modify input DataFrame # Adding index if data.columns[0].lower().find('index')>=0: pass else: - data.insert(0, 'Index', np.arange(self.data.shape[0])) - - # Clean columns only once - #data.columns = [s.replace('_',' ') for s in self.data.columns.values.astype(str)] + data.insert(0, 'Index', np.arange(data.shape[0])) + # Delete empty columns at the end (e.g. csv files) + while True: + if data.columns[-1]=='' and data.iloc[:,-1].isnull().all(): + print('[Info] Removing last column because all NaN') + data=data.iloc[:,:-1] + else: + break # --- Trying to figure out how to name this table if name is None or len(str(name))==0: if data.columns.name is not None: name=data.columns.name + # --- Store in object + self.data = data self.setupName(name=str(name)) self.convertTimeColumns(dayfirst=dayfirst) diff --git a/pydatview/common.py b/pydatview/common.py index 1e7374e..12f0a1d 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -142,7 +142,7 @@ def extract_key_tuples(text): all=(0.1,-2),b=(inf,0), c=(-inf,0.3e+10) """ regex = re.compile(r'(?P[\w\-]+)=\((?P[0-9+epinf.-]*?),(?P[0-9+epinf.-]*?)\)($|,)') - return {match.group("key"): (np.float(match.group("value1")),np.float(match.group("value2"))) for match in regex.finditer(text.replace(' ',''))} + return {match.group("key"): (float(match.group("value1")),float(match.group("value2"))) for match in regex.finditer(text.replace(' ',''))} def extract_key_num(text): @@ -150,7 +150,7 @@ def extract_key_num(text): all=0.1, b=inf, c=-0.3e+10 """ regex = re.compile(r'(?P[\w\-]+)=(?P[0-9+epinf.-]*?)($|,)') - return {match.group("key"): np.float(match.group("value")) for match in regex.finditer(text.replace(' ',''))} + return {match.group("key"): float(match.group("value")) for match in regex.finditer(text.replace(' ',''))} def getDt(x): """ returns dt in s """ diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index 5c00bbc..b5e7a53 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -296,13 +296,13 @@ def load_ascii_output(filename, method='numpy', encoding='ascii'): data = np.zeros((nRows, nCols)) for i in range(nRows): l = f.readline().strip() - sp = np.array(l.split()).astype(np.float) + sp = np.array(l.split()).astype(float) data[i,:] = sp[:nCols] elif method == 'listCompr': # --- Method 4 - List comprehension # Data, up to end of file or empty line (potential comment line at the end) - data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(np.float) + data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(float) else: raise NotImplementedError() diff --git a/pydatview/io/file_formats.py b/pydatview/io/file_formats.py index daa5400..595d545 100644 --- a/pydatview/io/file_formats.py +++ b/pydatview/io/file_formats.py @@ -11,6 +11,8 @@ def isRightFormat(fileformat, filename, **kwargs): except WrongFormatError: return False,None except: + # We Raise the Exception. + # It's the responsability of all the file classes to Return a WrongFormatError raise class FileFormat(): diff --git a/pydatview/io/wetb/hawc2/htc_file.py b/pydatview/io/wetb/hawc2/htc_file.py index 6b6bf3a..ca3237c 100644 --- a/pydatview/io/wetb/hawc2/htc_file.py +++ b/pydatview/io/wetb/hawc2/htc_file.py @@ -157,8 +157,8 @@ def isfile_case_insensitive(f): relpath = "../" * np.argmax(found) return abspath(pjoin(os.path.dirname(self.filename), relpath)) else: - raise ValueError( - "Modelpath cannot be autodetected for '%s'.\nInput files not found near htc file" % self.filename) + print("Modelpath cannot be autodetected for '%s'.\nInput files not found near htc file" % self.filename) + return 'unknown' def load(self): self.contents = OrderedDict() diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 65d3bf8..d9d7cc6 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -452,8 +452,8 @@ def yRange(PD): if PD.yIsString: return 'NA','NA' elif PD.yIsDate: - dtAll=getDt([PD.x[-1]-PD.x[0]]) - return '',pretty_time(dtAll) + dtAll=getDt([PD.x[0],PD.x[-1]]) + return np.nan,pretty_time(dtAll) else: v=np.nanmax(PD.y)-np.nanmin(PD.y) s=pretty_num(v) @@ -472,8 +472,8 @@ def xRange(PD): if PD.xIsString: return 'NA','NA' elif PD.xIsDate: - dtAll=getDt([PD.x[-1]-PD.x[0]]) - return '',pretty_time(dtAll) + dtAll=getDt([PD.x[0],PD.x[-1]]) + return np.nan,pretty_time(dtAll) else: v=np.nanmax(PD.x)-np.nanmin(PD.x) s=pretty_num(v) diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py index d8f8777..ac9ca70 100644 --- a/pydatview/plugins/base_plugin.py +++ b/pydatview/plugins/base_plugin.py @@ -204,7 +204,7 @@ def __init__(self, parent, action, buttons=None, tables=True): if 'Clear' in buttons: self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear); nButtons+=1 if 'Plot' in buttons: - self.btPlot = self.getBtBitmap(self, 'Plot' ,'chart' , self.onPlot); nButtons+=1 + self.btPlot = self.getBtBitmap(self, 'Plot ' ,'chart' , self.onPlot); nButtons+=1 self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply); nButtons+=1 diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index edff7ef..60e126d 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -156,10 +156,10 @@ def _GUI2Data(self): iFilt = self.cbFilters.GetSelection() opt = self._FILTERS_USER[iFilt] try: - opt['param']=np.float(self.spintxt.Value) + opt['param']=float(self.spintxt.Value) except: print('[WARN] pyDatView: Issue on Mac: plotdata_filter.py/_GUI2Data. Help needed.') - opt['param']=np.float(self.tParam.Value) + opt['param']=float(self.tParam.Value) if opt['param'] 1: raise TypeError('signal must have one column only, not: ' + str(signal.shape[1])) if np.min(signal) == np.max(signal): - raise TypeError("Signal contains no variation") + raise SignalConstantError("Signal is constant, cannot compute DLC and range") def rainflow_windap(signal, levels=255., thresshold=(255 / 50)): From c751e49719c8309626e57ed666ccf107e8f263e6 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 13 Sep 2023 21:46:05 -0600 Subject: [PATCH 118/178] Datetime: increase support for measure (see #162) --- pydatview/GUIMeasure.py | 115 +++++++++++++++++++++---------------- pydatview/GUIPlotPanel.py | 14 +++-- pydatview/common.py | 11 ++++ pydatview/plotdata.py | 65 ++++++++++++++++----- pydatview/tools/fatigue.py | 29 +++++----- 5 files changed, 154 insertions(+), 80 deletions(-) diff --git a/pydatview/GUIMeasure.py b/pydatview/GUIMeasure.py index 4f556f7..63c4259 100644 --- a/pydatview/GUIMeasure.py +++ b/pydatview/GUIMeasure.py @@ -1,4 +1,5 @@ import numpy as np +from pydatview.common import isString, isDate, getDt, pretty_time, pretty_num, pretty_date, isDateScalar class GUIMeasure: @@ -6,16 +7,16 @@ def __init__(self, index, color): # Main data self.index = index self.color = color - self.x_target = None # x closest in data where click was done - self.y_target = None # y closest in data where click was done + self.P_target_raw = None # closest x-y point stored in "raw" form (including datetime) + self.P_target_num = None # closest x-y point stored in "num" form (internal matplotlib xy) # Plot data self.points = [] # Intersection points self.lines = [] # vertical lines (per ax) self.annotations = [] def clear(self): - self.x_target=None - self.y_target=None + self.P_target_raw = None + self.P_target_num = None self.clearPlot() def clearPlot(self): @@ -43,21 +44,16 @@ def clearPointLines(self): self.points= [] self.lines= [] - def get_xydata(self): - if self.x_target is None or self.y_target is None: - return None, None - else: - return (self.x_target, self.y_target) - def set(self, axes, ax, x, y, PD): """ - x,y : point where the user clicked (will likely be slightly off plotdata) """ self.clearPlot() # Point closest to user click location - x_closest, y_closest, pd_closest = self.find_closest_point(x, y, ax) - self.x_target = x_closest - self.y_target = y_closest + #x_closest, y_closest, pd_closest = self.find_closest_point(x, y, ax) + P_raw, P_num, pd_closest = self.find_closest_point(x, y, ax) + self.P_target_raw = P_raw + self.P_target_num = P_num self.pd_closest = pd_closest # Plot measure where the user clicked (only for the axis that the user chose) @@ -67,21 +63,23 @@ def set(self, axes, ax, x, y, PD): def compute(self, PD): for ipd, pd in enumerate(PD): if pd !=self.pd_closest: - XY = np.array([pd.x, pd.y]).transpose() + # Get XY array + # convert dates to num to be consistent with (xt,yt) click event from matpotlib canvas + XY = pd.toXY_date2num() try: - xc, yc = find_closestX(XY, self.x_target) - pd.xyMeas[self.index-1] = (xc, yc) + (xc, yc), i = find_closestX(XY, self.P_target_num[0]) + xc, yc = pd.x[i], pd.y[i] # We store raw values except: print('[FAIL] GUIMeasure: failed to compute closest point') xc, yc = np.nan, np.nan else: # Already computed - xc, yc = self.x_target, self.y_target - pd.xyMeas[self.index-1] = (xc, yc) + xc, yc = self.P_target_raw + pd.xyMeas[self.index-1] = (xc, yc) def plotAnnotation(self, ax, xc, yc): #self.clearAnnotation() - sAnnotation = '{0}: ({1}, {2})'.format(self.index, formatValue(xc), formatValue(yc)) + sAnnotation = '{}: ({}, {})'.format(self.index, formatValue(xc), formatValue(yc)) bbox_args = dict(boxstyle='round', fc='0.9', alpha=0.75) annotation = ax.annotate(sAnnotation, (xc, yc), xytext=(5, -2), textcoords='offset points', color=self.color, bbox=bbox_args) self.annotations.append(annotation) @@ -96,7 +94,7 @@ def plotPoint(self, ax, xc, yc, ms=3): def plotLine(self, ax): """ plot vertical line across axis""" - line = ax.axvline(x=self.x_target, color=self.color, linewidth=0.5) + line = ax.axvline(x=self.P_target_raw[0], color=self.color, linewidth=0.5) self.lines.append(line) def plot(self, axes, PD): @@ -107,7 +105,7 @@ def plot(self, axes, PD): - or closest to "target" (x,y) point when matchY is True - plot intersection point, vertical line """ - if self.x_target is None: + if self.P_target_raw is None: return if PD is not None: self.compute(PD) @@ -124,7 +122,7 @@ def plot(self, axes, PD): self.plotAnnotation(ax, xc, yc) # NOTE Comment if unwanted else: #xc, yc = pd.xyMeas[self.index-1] - xc, yc = self.x_target, self.y_target + xc, yc = self.P_target_raw self.plotPoint(ax, xc, yc, ms=6) self.plotAnnotation(ax, xc, yc) @@ -135,14 +133,13 @@ def plot(self, axes, PD): # Store as target if there is only one plot and one ax (important for "dx dy") if PD is not None: if len(axes)==1 and len(PD)==1: - self.x_target = xc - self.y_target = yc + self.P_target_raw = (xc,yc) # self.plotAnnotation(axes[0], xc, yc) def find_closest_point(self, xt, yt, ax): """ - Find closest point to target across all plotdata in a given ax + Find closest point to target (xt, yt) across all plotdata in a given ax """ # Compute axis diagonal try: @@ -152,38 +149,45 @@ def find_closest_point(self, xt, yt, ax): except: print('[FAIL] GUIMeasure: Computing axis diagonal failed') rdist_min = 1e9 - # --- Find closest intersection point - x_closest = xt - y_closest = yt + P_closest_num = (xt,yt) + P_closest_raw = (None,None) pd_closest= None for pd in ax.PD: - XY = np.array([pd.x, pd.y]).transpose() try: - x, y = find_closest(XY, [xt, yt], xlim, ylim) - rdist = abs(x - xt) + abs(y - yt) + P_num, P_raw, ind = find_closest(pd, [xt, yt], xlim, ylim) + rdist = abs(P_num[0] - xt) + abs(P_num[1] - yt) if rdist < rdist_min: - rdist_min = rdist - x_closest = x - y_closest = y - pd_closest = pd + rdist_min = rdist + P_closest_num = P_num + P_closest_raw = P_raw + pd_closest = pd + #ind_closest = ind except (TypeError,ValueError): # Fails when x/y data are dates or strings print('[FAIL] GUIMeasure: find_closest failed on some data') - return x_closest, y_closest, pd_closest + return P_closest_raw, P_closest_num, pd_closest def sDeltaX(self, meas2): try: - dx = self.x_target - meas2.x_target - return 'dx = ' + formatValue(dx) + if isDateScalar(self.P_target_raw[0]): + dt = getDt([meas2.P_target_raw[0] , self.P_target_raw[0]]) + return 'dx = ' + pretty_time(dt) + else: + dx = self.P_target_raw[0] - meas2.P_target_raw[0] + return 'dx = ' + formatValue(dx) except: return '' def sDeltaY(self, meas2): try: - dy = self.y_target - meas2.y_target - return 'dy = ' + formatValue(dy) + if isDateScalar(self.P_target_raw[1]): + dt = getDt([meas2.P_target_raw[1] , self.P_target_raw[1]]) + return 'dx = ' + pretty_time(dt) + else: + dy = self.P_target_raw[1] - meas2.P_target_raw[1] + return 'dy = ' + formatValue(dy) except: return '' @@ -192,21 +196,23 @@ def sDeltaY(self, meas2): def formatValue(value): try: - if abs(value) < 1000 and abs(value) > 1e-4: - s = '{:.4f}'.format(value) + if isDateScalar(value): + # TODO could be improved + return pretty_date(value) + elif isString(value): + return value else: - s = '{:.3e}'.format(value) + return pretty_num(value) except TypeError: - s = '' - return s + return '' def find_closestX(XY, x_target): """ return x,y values closest to a given x value """ i = np.argmin(np.abs(XY[:,0]-x_target)) - return XY[i,:] + return XY[i,:], i -def find_closest(XY, point, xlim=None, ylim=None): +def find_closest_i(XY, point, xlim=None, ylim=None): """Return closest point(s), using norm2 distance if xlim and ylim is provided, these are used to make the data non dimensional. """ @@ -220,7 +226,20 @@ def find_closest(XY, point, xlim=None, ylim=None): norm2 = ((XY[:,0]-point[0])**2)/x_scale + ((XY[:,1]-point[1])**2)/y_scale ind = np.argmin(norm2, axis=0) - return XY[ind,:] + return ind + +def find_closest(pd, point, xlim=None, ylim=None): + """Return closest point(s), using norm2 distance + if xlim and ylim is provided, these are used to make the data non dimensional. + """ + # Get XY array + # convert dates to num to be consistent with (xt,yt) click event from matpotlib canvas + XY = pd.toXY_date2num() + ind = find_closest_i(XY, point, xlim=xlim, ylim=ylim) + x_num, y_num = XY[ind,:] + x_raw, y_raw = pd.x[ind], pd.y[ind] + return (x_num, y_num), (x_raw, y_raw), ind + # --- Old method ## By default return closest single point. diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 7dea4e0..d2446fc 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -5,6 +5,7 @@ import dateutil # required by matplotlib #from matplotlib import pyplot as plt import matplotlib +import matplotlib.dates as mdates # Backends: # ['GTK3Agg', 'GTK3Cairo', 'GTK4Agg', 'GTK4Cairo', 'MacOSX', 'nbAgg', 'QtAgg', 'QtCairo', 'Qt5Agg', 'Qt5Cairo', 'TkAgg', 'TkCairo', 'WebAgg', 'WX', 'WXAgg', 'WXCairo', 'agg', 'cairo', 'pdf', 'pgf', 'ps', 'svg', 'template'] matplotlib.use('WX') # Important for Windows version of installer. NOTE: changed from Agg to wxAgg, then to WX @@ -38,7 +39,7 @@ import gc -from pydatview.common import * # unique, CHAR +from pydatview.common import * # unique, CHAR, pretty_date from pydatview.plotdata import PlotData, compareMultiplePD from pydatview.GUICommon import * from pydatview.GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool @@ -753,8 +754,9 @@ def set_subplots(self,nPlots): def onMouseMove(self, event): if event.inaxes: x, y = event.xdata, event.ydata - self.lbCrossHairX.SetLabel('x =' + self.formatLabelValue(x)) - self.lbCrossHairY.SetLabel('y =' + self.formatLabelValue(y)) + + self.lbCrossHairX.SetLabel('x =' + self.formatLabelValue(x,self.plotData[0].xIsDate)) + self.lbCrossHairY.SetLabel('y =' + self.formatLabelValue(y,self.plotData[0].yIsDate)) def onMouseClick(self, event): self.clickLocation = (event.inaxes, event.xdata, event.ydata) @@ -792,9 +794,11 @@ def onMouseRelease(self, event): def onDraw(self, event): self._store_limits() - def formatLabelValue(self, value): + def formatLabelValue(self, value, isdate): try: - if abs(value)<1000 and abs(value)>1e-4: + if isdate: + s = pretty_date(mdates.num2date(value)) + elif abs(value)<1000 and abs(value)>1e-4: s = '{:10.5f}'.format(value) else: s = '{:10.3e}'.format(value) diff --git a/pydatview/common.py b/pydatview/common.py index 12f0a1d..da5ff35 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -346,6 +346,14 @@ def pretty_time(t): s='{:.1f}y'.format(y) return s + +def pretty_date(d, timespan=None): + """ + TODO, placeholder for pretty date based on a given timespan + """ + s ='{}'.format(d) + return s + def pretty_num(x): if np.isnan(x): return 'NA' @@ -488,6 +496,9 @@ def isString(x): def isDate(x): return np.issubdtype(x.dtype, np.datetime64) +def isDateScalar(x): + return np.issubdtype(x, np.datetime64) + # Create a Dummy Main Frame Class for testing purposes (e.g. of plugins) diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index d9d7cc6..1772fc6 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -2,7 +2,8 @@ import numpy as np from pydatview.common import no_unit, unit, inverse_unit, has_chinese_char from pydatview.common import isString, isDate, getDt -from pydatview.common import unique, pretty_num, pretty_time +from pydatview.common import unique, pretty_num, pretty_time, pretty_date +import matplotlib.dates as mdates class PlotData(): """ @@ -129,6 +130,20 @@ def __repr__(s): #s1='id:{}, it:{}, sx:"{}", xyMeas:{}\n'.format(s.id,s.it,s.sx,s.xyMeas) return s1 + + def toXY_date2num(self): + """ return a XY array, converting dates to num if necessary""" + if self.xIsDate : + X = mdates.date2num(self.x) + else: + X = self.x + if self.yIsDate : + Y = mdates.date2num(self.y) + else: + Y = self.y + XY = np.array([X, Y]).transpose() + return XY + def toPDF(PD, nBins=30, smooth=False): """ Convert y-data to Probability density function (PDF) as function of x Uses "stats" library (from welib/pybra) @@ -528,19 +543,27 @@ def ymeas1(PD): # NOTE: calculation happens in GUIMeasure.. if PD.xyMeas[0][0] is not None: yv = PD.xyMeas[0][1] - s = pretty_num(yv) + if PD.yIsString: + return yv, yb + elif PD.yIsDate: + return yv, pretty_date(yv) + else: + return yv, pretty_num(yv) else: return np.nan, 'NA' - return yv, s def ymeas2(PD): # NOTE: calculation happens in GUIMeasure.. if PD.xyMeas[1][0] is not None: yv = PD.xyMeas[1][1] - s = pretty_num(yv) + if PD.yIsString: + return yv, yb + elif PD.yIsDate: + return yv, pretty_date(yv) + else: + return yv, pretty_num(yv) else: return np.nan, 'NA' - return yv, s def yMeanMeas(PD): return PD._measCalc('mean') @@ -558,10 +581,15 @@ def xAtYMaxMeas(PD): return PD._measCalc('xmax') def _measCalc(PD, mode): + # TODO clean up if PD.xyMeas[0][0] is None or PD.xyMeas[1][0] is None: return np.nan, 'NA' if np.isnan(PD.xyMeas[0][0]) or np.isnan(PD.xyMeas[1][0]): return np.nan, 'NA' + # We only support the case where y values are numeric + if PD.yIsDate or PD.yIsString: + return np.nan, 'NA' + try: v = np.nan left_index = np.argmin(np.abs(PD.x - PD.xyMeas[0][0])) @@ -570,22 +598,33 @@ def _measCalc(PD, mode): return np.nan, 'Empty' if left_index > right_index: left_index, right_index = right_index, left_index + except (IndexError, TypeError): + return np.nan, 'NA' + + try: + yValues = PD.y[left_index:right_index] if mode == 'mean': - v = np.nanmean(PD.y[left_index:right_index]) + v = np.nanmean(yValues) + s = pretty_num(v) elif mode == 'min': - v = np.nanmin(PD.y[left_index:right_index]) + v = np.nanmin(yValues) + s = pretty_num(v) elif mode == 'max': - v = np.nanmax(PD.y[left_index:right_index]) + v = np.nanmax(yValues) + s = pretty_num(v) elif mode == 'xmin': - v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmin(PD.y[left_index:right_index]))[0][0]] + v = PD.x[left_index + np.where(yValues == np.nanmin(yValues))[0][0]] + if PD.xIsDate: + s = pretty_date(v) elif mode == 'xmax': - v = PD.x[left_index + np.where(PD.y[left_index:right_index] == np.nanmax(PD.y[left_index:right_index]))[0][0]] + v = PD.x[left_index + np.where(yValues == np.nanmax(yValues))[0][0]] + if PD.xIsDate: + s = pretty_date(v) else: raise NotImplementedError('Error: Mode ' + mode + ' not implemented') - s = pretty_num(v) except (IndexError, TypeError): - v = np.nan - s = 'NA' + return np.nan, 'NA' + return v, s # --------------------------------------------------------------------------------} diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index acf7afc..b2b30f8 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -79,24 +79,25 @@ def equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap' signal = signal[b] time = time[b] - if len(time)<=1: - if outputMore: - return Leq, S, N, bins, DELi - else: - return np.nan + try: + T = time[-1]-time[0] # time length of signal (s). Will fail for signal of length 1 + if type(time[0]) is np.datetime64: + T = T/np.timedelta64(1,'s') # or T.item().total_seconds() - T = time[-1]-time[0] # time length of signal (s) - if type(time[0]) is np.datetime64: - T= T.item().total_seconds() + neq = T/Teq # number of equivalent periods, see Eq. (26) of [1] - neq = T/Teq # number of equivalent periods, see Eq. (26) of [1] + # --- Range (S) and counts (N) + N, S, bins = find_range_count(signal, bins=bins, method=method, meanBin=meanBin, binStartAt0=binStartAt0) - # --- Range (S) and counts (N) - N, S, bins = find_range_count(signal, bins=bins, method=method, meanBin=meanBin, binStartAt0=binStartAt0) + # --- get DEL + DELi = S**m * N / neq + Leq = DELi.sum() ** (1/m) # See e.g. eq. (30) of [1] - # --- get DEL - DELi = S**m * N / neq - Leq = DELi.sum() ** (1/m) # See e.g. eq. (30) of [1] + except: + if outputMore: + return np.nan, np.nan, np.nan, np.nan, np.nan + else: + return np.nan if debug: for i,(b,n,s,DEL) in enumerate(zip(bins, N, S, DELi)): From d754b5f7bc32fc7d1d65fbb7a45990c4b5d237f9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 13 Sep 2023 22:30:03 -0600 Subject: [PATCH 119/178] Table: Attempt to speedup datetime conversion (see #162 #132) --- pydatview/Tables.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index bbc3ee8..0894e13 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -716,12 +716,19 @@ def convertTimeColumns(self, dayfirst=False): else: isDate=False if isDate: + print('[INFO] Converting column {} to datetime, dayfirst: {}. May take a while...'.format(c, dayfirst)) try: - self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst).to_pydatetime() - print('Column {} converted to datetime, dayfirst: {}'.format(c, dayfirst)) + # TODO THIS CAN BE VERY SLOW... + self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst, infer_datetime_format=True).to_pydatetime() + print(' Done.') except: - # Happens if values are e.g. "Monday, Tuesday" - print('Conversion to datetime failed, column {} inferred as string'.format(c)) + try: + print('[FAIL] Attempting without infer datetime. May take a while...') + self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst, infer_datetime_format=False).to_pydatetime() + print(' Done.') + except: + # Happens if values are e.g. "Monday, Tuesday" + print('[FAIL] Inferring column as string instead') else: print('Column {} inferred as string'.format(c)) elif isinstance(y.values[0], (float, int)): From 42faf75dceff881470990e76ceb0f6643924f5f3 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 13 Sep 2023 22:52:31 -0600 Subject: [PATCH 120/178] Main: adding a live plot button for large data mode (see #11) --- pydatview/GUIPlotPanel.py | 3 +-- pydatview/main.py | 52 +++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index d2446fc..25d0ed4 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -752,9 +752,8 @@ def set_subplots(self,nPlots): #self.fig.add_subplot(1,nPlots,i+1) def onMouseMove(self, event): - if event.inaxes: + if event.inaxes and len(self.plotData)>0: x, y = event.xdata, event.ydata - self.lbCrossHairX.SetLabel('x =' + self.formatLabelValue(x,self.plotData[0].xIsDate)) self.lbCrossHairY.SetLabel('y =' + self.formatLabelValue(y,self.plotData[0].yIsDate)) diff --git a/pydatview/main.py b/pydatview/main.py index 40ce29b..e7470a1 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -191,24 +191,24 @@ def __init__(self, data=None): # --- ToolBar tb = self.CreateToolBar(wx.TB_HORIZONTAL|wx.TB_TEXT|wx.TB_HORZ_LAYOUT) - self.toolBar = tb - self.comboFormats = wx.ComboBox(tb, choices = self.FILE_FORMATS_NAMEXT, style=wx.CB_READONLY) - self.comboFormats.SetSelection(0) + tb.AddSeparator() self.comboMode = wx.ComboBox(tb, choices = SEL_MODES, style=wx.CB_READONLY) self.comboMode.SetSelection(0) - self.Bind(wx.EVT_COMBOBOX, self.onModeChange, self.comboMode ) - self.Bind(wx.EVT_COMBOBOX, self.onFormatChange, self.comboFormats ) - tb.AddSeparator() + #tb.AddStretchableSpace() tb.AddControl( wx.StaticText(tb, -1, 'Mode: ' ) ) tb.AddControl( self.comboMode ) + self.cbLivePlot = wx.CheckBox(tb, -1, 'Live Plot') #,(10,10)) + self.cbLivePlot.SetValue(True) + tb.AddControl( self.cbLivePlot ) tb.AddStretchableSpace() tb.AddControl( wx.StaticText(tb, -1, 'Format: ' ) ) + self.comboFormats = wx.ComboBox(tb, choices = self.FILE_FORMATS_NAMEXT, style=wx.CB_READONLY) + self.comboFormats.SetSelection(0) tb.AddControl(self.comboFormats ) # Menu for loader options self.btLoaderMenu = wx.Button(tb, wx.ID_ANY, CHAR['menu'], style=wx.BU_EXACTFIT) tb.AddControl(self.btLoaderMenu) self.loaderMenu = LoaderMenuPopup(tb, self.data['loaderOptions']) - tb.Bind(wx.EVT_BUTTON, self.onShowLoaderMenu, self.btLoaderMenu) tb.AddSeparator() TBAddTool(tb, "Open" , 'ART_FILE_OPEN', self.onLoad) TBAddTool(tb, "Reload", 'ART_REDO' , self.onReload) @@ -217,6 +217,12 @@ def __init__(self, data=None): #self.AddTBBitmapTool(tb,"Debug" ,wx.ArtProvider.GetBitmap(wx.ART_ERROR),self.onDEBUG) tb.AddStretchableSpace() tb.Realize() + self.toolBar = tb + # Bind Toolbox Events + self.Bind(wx.EVT_COMBOBOX, self.onModeChange, self.comboMode ) + self.Bind(wx.EVT_COMBOBOX, self.onFormatChange, self.comboFormats ) + tb.Bind(wx.EVT_BUTTON, self.onShowLoaderMenu, self.btLoaderMenu) + tb.Bind(wx.EVT_CHECKBOX, self.onLivePlotChange, self.cbLivePlot) # --- Status bar self.statusbar=self.CreateStatusBar(3, style=0) @@ -369,6 +375,7 @@ def load_tabs_into_GUI(self, bReload=False, bAdd=False, bPlot=True): #self.tSplitter.SetMinimumPaneSize(20) self.infoPanel = InfoPanel(self.tSplitter, data=self.data['infoPanel']) self.plotPanel = PlotPanel(self.tSplitter, self.selPanel, infoPanel=self.infoPanel, pipeLike=self.pipePanel, data=self.data['plotPanel']) + self.livePlotFreezeUnfreeze() # Dont enable panels if livePlot is not allowed self.tSplitter.SetSashGravity(0.9) self.tSplitter.SplitHorizontally(self.plotPanel, self.infoPanel) self.tSplitter.SetMinimumPaneSize(BOT_PANL) @@ -594,9 +601,37 @@ def onTabSelectionChangeTrigger(self, event=None): def onColSelectionChangeTrigger(self, event=None): pass + def onLivePlotChange(self, event=None): + if self.cbLivePlot.IsChecked(): + if hasattr(self,'plotPanel'): + print('[INFO] Reenabling live plot') + self.plotPanel.Enable(True) + self.infoPanel.Enable(True) + self.redrawCallback() + else: + if hasattr(self,'plotPanel'): + print('[INFO] Disabling live plot') + self.plotPanel.Enable(False) + self.infoPanel.Enable(False) + + def livePlotFreezeUnfreeze(self): + if self.cbLivePlot.IsChecked(): + if hasattr(self,'plotPanel'): + print('[INFO] Enabling live plot') + self.plotPanel.Enable(True) + self.infoPanel.Enable(True) + else: + if hasattr(self,'plotPanel'): + print('[INFO] Disabling live plot') + self.plotPanel.Enable(False) + self.infoPanel.Enable(False) + def redrawCallback(self): if hasattr(self,'plotPanel'): - self.plotPanel.load_and_draw() + if self.cbLivePlot.IsChecked(): + self.plotPanel.load_and_draw() + else: + print('[INFO] Drawing event skipped, live plot is not checked.') # def showStats(self): # self.infoPanel.showStats(self.plotPanel.plotData,self.plotPanel.pltTypePanel.plotType()) @@ -728,6 +763,7 @@ def onFormatChange(self, event=None): # ISel=self.selPanel.tabPanel.lbTab.GetSelections() pass + def onShowLoaderMenu(self, event=None): #pos = (self.btLoaderMenu.GetPosition()[0], self.btLoaderMenu.GetPosition()[1] + self.btLoaderMenu.GetSize()[1]) self.PopupMenu(self.loaderMenu) #, pos) From 0341a1b2cae53fd9d81d65575f6fc0ba540b3bdb Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sat, 23 Sep 2023 17:30:43 -0600 Subject: [PATCH 121/178] Tools: misc updates --- pydatview/fast/fastfarm.py | 14 ++++++-------- pydatview/main.py | 14 +++++++++----- pydatview/tools/fatigue.py | 32 +++++++++++++++++++++++++++++++- tests/prof_all.py | 18 ++++++++++++------ 4 files changed, 58 insertions(+), 20 deletions(-) diff --git a/pydatview/fast/fastfarm.py b/pydatview/fast/fastfarm.py index 663f4cf..0eb3ac4 100644 --- a/pydatview/fast/fastfarm.py +++ b/pydatview/fast/fastfarm.py @@ -606,7 +606,7 @@ def spanwisePostProFF(fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1, vr=None vD=None D=0 - main is None + main = None else: main=FASTInputFile(fastfarm_input) iOut = main['OutRadii'] @@ -633,19 +633,17 @@ def spanwisePostProFF(fastfarm_input,avgMethod='constantwindow',avgParam=30,D=1, dfRad = fastlib.extract_spanwise_data(ColsInfo, nrMax, df=None, ts=dfAvg.iloc[0]) #dfRad = fastlib.insert_radial_columns(dfRad, vr) if dfRad is not None: - if vr is None: - dfRad.insert(0, 'i_[#]', np.arange(nrMax)+1) - else: - dfRad.insert(0, 'r_[m]', vr[:nrMax]) + dfRad.insert(0, 'i_[#]', np.arange(nrMax)+1) # For all, to ease comparison + if vr is not None: + dfRad.insert(0, 'r_[m]', vr[:nrMax]) # give priority to r_[m] when available dfRad['i/n_[-]']=np.arange(nrMax)/nrMax # --- Extract downstream data ColsInfo, nDMax = diameterwiseColFastFarm(df.columns.values, nWT=nWT) dfDiam = fastlib.extract_spanwise_data(ColsInfo, nDMax, df=None, ts=dfAvg.iloc[0]) if dfDiam is not None: - if vD is None: - dfDiam.insert(0, 'i_[#]', np.arange(nDMax)+1) - else: + dfDiam.insert(0, 'i_[#]', np.arange(nDMax)+1) # For all, to ease comparison + if vD is not None: dfDiam.insert(0, 'x_[m]', vD[:nDMax]) dfDiam['i/n_[-]'] = np.arange(nDMax)/nDMax return dfRad, dfRadialTime, dfDiam diff --git a/pydatview/main.py b/pydatview/main.py index e7470a1..d163070 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -604,25 +604,28 @@ def onColSelectionChangeTrigger(self, event=None): def onLivePlotChange(self, event=None): if self.cbLivePlot.IsChecked(): if hasattr(self,'plotPanel'): - print('[INFO] Reenabling live plot') + #print('[INFO] Reenabling live plot') self.plotPanel.Enable(True) self.infoPanel.Enable(True) self.redrawCallback() else: if hasattr(self,'plotPanel'): - print('[INFO] Disabling live plot') + #print('[INFO] Disabling live plot') + for ax in self.plotPanel.fig.axes: + ax.annotate('Live Plot Disabled', xy=(0.5, 0.5), size=20, xycoords='axes fraction', ha='center', va='center',) + self.plotPanel.canvas.draw() self.plotPanel.Enable(False) self.infoPanel.Enable(False) def livePlotFreezeUnfreeze(self): if self.cbLivePlot.IsChecked(): if hasattr(self,'plotPanel'): - print('[INFO] Enabling live plot') + #print('[INFO] Enabling live plot') self.plotPanel.Enable(True) self.infoPanel.Enable(True) else: if hasattr(self,'plotPanel'): - print('[INFO] Disabling live plot') + #print('[INFO] Disabling live plot') self.plotPanel.Enable(False) self.infoPanel.Enable(False) @@ -631,7 +634,8 @@ def redrawCallback(self): if self.cbLivePlot.IsChecked(): self.plotPanel.load_and_draw() else: - print('[INFO] Drawing event skipped, live plot is not checked.') + pass + #print('[INFO] Drawing event skipped, live plot is not checked.') # def showStats(self): # self.infoPanel.showStats(self.plotPanel.plotData,self.plotPanel.pltTypePanel.plotType()) diff --git a/pydatview/tools/fatigue.py b/pydatview/tools/fatigue.py index b2b30f8..e88936d 100644 --- a/pydatview/tools/fatigue.py +++ b/pydatview/tools/fatigue.py @@ -80,9 +80,14 @@ def equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap' time = time[b] try: - T = time[-1]-time[0] # time length of signal (s). Will fail for signal of length 1 + if len(time)<=1: + raise Exception() if type(time[0]) is np.datetime64: T = T/np.timedelta64(1,'s') # or T.item().total_seconds() + else: + T = time[-1]-time[0] # time length of signal (s). Will fail for signal of length 1 + if T==0: + raise Exception() neq = T/Teq # number of equivalent periods, see Eq. (26) of [1] @@ -1183,6 +1188,31 @@ def test_equivalent_load_sines_sum(self): + def test_eqload_cornercases(self): + try: + import fatpack + hasFatpack=True + except: + hasFatpack=False + # Signal of length 1 + time=[0]; signal=[0] + Leq= equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap') + np.testing.assert_equal(Leq, np.nan) + + # Datetime + time= [np.datetime64('2023-10-01'), np.datetime64('2023-10-02')] + signal= [0,1] + Leq= equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap') + np.testing.assert_equal(Leq, np.nan) + + # Constant signal + time =[0,1] + signal =[1,1] + Leq= equivalent_load(time, signal, m=3, Teq=1, bins=100, method='rainflow_windap') + np.testing.assert_equal(Leq, np.nan) + if hasFatpack: + Leq= equivalent_load(time, signal, m=3, Teq=1, bins=100, method='fatpack') + np.testing.assert_equal(Leq, np.nan) diff --git a/tests/prof_all.py b/tests/prof_all.py index 75d2210..51da191 100644 --- a/tests/prof_all.py +++ b/tests/prof_all.py @@ -67,17 +67,22 @@ def test_heavy(): def test_debug(show=False): dt = 0 with Timer('Test'): - with Timer('read'): - df1 =weio.read(os.path.join(scriptDir,'../ad_driver_m50.1.outb')).toDataFrame() - df2 =weio.read(os.path.join(scriptDir,'../ad_driver_p50.csv')).toDataFrame() + with PerfMon('read'): + df1 =weio.read(os.path.join(scriptDir,'../Processed.Data_bkp.csv')).toDataFrame() + #df2=None + #df1 =weio.read(os.path.join(scriptDir,'../ad_driver_m50.1.outb')).toDataFrame() + #df2 =weio.read(os.path.join(scriptDir,'../ad_driver_p50.csv')).toDataFrame() time.sleep(dt) - with PerfMon('Plot 1'): + with PerfMon('AppStart 1'): app = wx.App(False) frame = MainFrame() - frame.load_dfs([df1,df2]) + with PerfMon('Load dfs'): + #frame.load_dfs([df1]) + #frame.load_dfs([df1, df2]) + with PerfMon('Plot 1'): frame.selPanel.tabPanel.lbTab.SetSelection(0) - frame.selPanel.tabPanel.lbTab.SetSelection(1) + #frame.selPanel.tabPanel.lbTab.SetSelection(1) frame.onTabSelectionChange() #frame.redraw() if show: @@ -92,3 +97,4 @@ def test_files(filenames): test_heavy() #test_debug(False) + #test_debug(False) From 624c385c60bc8bef1a77a486699dc3bd06827d8e Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sat, 23 Sep 2023 20:33:14 -0600 Subject: [PATCH 122/178] Preliminary implementation of scripter (#13) --- pydatview/GUIScripter.py | 78 ++++++++++ pydatview/main.py | 26 ++++ pydatview/pipeline.py | 69 +++++++++ pydatview/scripter.py | 305 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 478 insertions(+) create mode 100644 pydatview/GUIScripter.py create mode 100644 pydatview/scripter.py diff --git a/pydatview/GUIScripter.py b/pydatview/GUIScripter.py new file mode 100644 index 0000000..6ceea83 --- /dev/null +++ b/pydatview/GUIScripter.py @@ -0,0 +1,78 @@ +import wx + +class GUIScripterFrame(wx.Frame): + def __init__(self, parent, mainframe, pipeLike, title): + super(GUIScripterFrame, self).__init__(parent, title=title, size=(800, 600)) + + # --- Data + self.mainframe = mainframe + self.pipeline = pipeLike + + # --- GUI + self.panel = wx.Panel(self) + self.text_ctrl = wx.TextCtrl(self.panel, style=wx.TE_MULTILINE) + self.btGen = wx.Button(self.panel, label="Regenerate") + self.btRun = wx.Button(self.panel, label="Run Script") + self.btSave = wx.Button(self.panel, label="Save to File") + self.flavors = ["welib", "pydatview", "pyFAST"] + self.cbFlavors = wx.Choice(self.panel, choices=self.flavors) + self.cbFlavors.SetSelection(1) + + # --- Layout + vbox = wx.BoxSizer(wx.VERTICAL) + hbox = wx.BoxSizer(wx.HORIZONTAL) + + vbox.Add(self.text_ctrl, proportion=1, flag=wx.EXPAND | wx.ALL, border=10) + + hbox.Add(self.cbFlavors, proportion=1, flag=wx.EXPAND | wx.ALL, border=10) + hbox.Add(self.btGen , flag=wx.ALL, border=10) + hbox.Add(self.btRun , flag=wx.ALL, border=10) + hbox.Add(self.btSave , flag=wx.ALL, border=10) + vbox.Add(hbox, flag=wx.EXPAND) + + self.panel.SetSizerAndFit(vbox) + + # -- Binding + self.btSave.Bind(wx.EVT_BUTTON, self.onSave) + self.btRun.Bind(wx.EVT_BUTTON, self.onRun) + self.btGen.Bind(wx.EVT_BUTTON, self.generateScript) + self.cbFlavors.Bind(wx.EVT_CHOICE, self.onFlavorChange) + + self.generateScript() + + + def generateScript(self, *args, **kwargs): + # GUI2Data + flavorDict={} + flavorDict['libFlavor'] = self.cbFlavors.GetStringSelection() + + try: + ID,SameCol,selMode=self.mainframe.selPanel.getPlotDataSelection() + except: + ID is None + s = self.pipeline.script(self.mainframe.tabList, flavorDict, ID) + self.text_ctrl.SetValue(s) + + def onFlavorChange(self, event): + flavor = self.cbFlavors.GetStringSelection() + # You can add code here to handle the selected format change event + self.generateScript() + + + def onRun(self, event): + self.pipeline.scripter.run() + + + + def onSave(self, event): + file_extension = "py" + + dialog = wx.FileDialog(self, "Save Script to File", wildcard=f"(*.{file_extension})|*.{file_extension}", style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) + + if dialog.ShowModal() == wx.ID_OK: + file_path = dialog.GetPath() + with open(file_path, "w") as file: + file.write(self.text_ctrl.GetValue()) + + dialog.Destroy() + diff --git a/pydatview/main.py b/pydatview/main.py index d163070..9855162 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -132,12 +132,14 @@ def __init__(self, data=None): fileMenu = wx.Menu() loadMenuItem = fileMenu.Append(wx.ID_NEW,"Open file" ,"Open file" ) + scrpMenuItem = fileMenu.Append(-1 ,"Export script" ,"Export script" ) exptMenuItem = fileMenu.Append(-1 ,"Export table" ,"Export table" ) saveMenuItem = fileMenu.Append(wx.ID_SAVE,"Save figure" ,"Save figure" ) exitMenuItem = fileMenu.Append(wx.ID_EXIT, 'Quit', 'Quit application') menuBar.Append(fileMenu, "&File") self.Bind(wx.EVT_MENU,self.onExit ,exitMenuItem) self.Bind(wx.EVT_MENU,self.onLoad ,loadMenuItem) + self.Bind(wx.EVT_MENU,self.onScript,scrpMenuItem) self.Bind(wx.EVT_MENU,self.onExport,exptMenuItem) self.Bind(wx.EVT_MENU,self.onSave ,saveMenuItem) @@ -724,6 +726,26 @@ def onExport(self, event=None): else: Error(self,'Open a file and select a table first.') + def onScript(self, event=None): + from pydatview.GUIScripter import GUIScripterFrame + GUIScripterFrame + pop = GUIScripterFrame(parent=None, mainframe=self, pipeLike=self.pipePanel, title="pyDatView - Script export") + pop.Show() +# if hasattr(self,'selPanel') and hasattr(self,'plotPanel'): +# script = pythonScript(self.tabList, self.selPanel, self.plotPanel) +# else: +# Error(self,'Open a file and generate a plot before exporting.') +# tab=self.tabList.get(iTab) +# default_filename=tab.basename +'.csv' +# with wx.FileDialog(self, "Save to CSV file",defaultFile=default_filename, +# style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as dlg: +# #, wildcard="CSV files (*.csv)|*.csv", +# dlg.CentreOnParent() +# if dlg.ShowModal() == wx.ID_CANCEL: +# return # the user changed their mind +# tab.export(dlg.GetPath()) + + def onLoad(self, event=None): self.selectFile(bAdd=False) @@ -948,8 +970,12 @@ def showApp(firstArg=None, dataframes=None, filenames=[], names=None): frame.load_dfs(dataframes, names) elif len(filenames)>0: frame.load_files(filenames, fileformats=None, bPlot=True) + +# frame.onScript() + app.MainLoop() + def cmdline(): if len(sys.argv)>1: pydatview(filename=sys.argv[1]) diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index cd5149d..e55a324 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -265,6 +265,75 @@ def apply(self, tabList, force=False, applyToAll=False): # self.collectErrors() + def script(self, tabList, flavorDict, ID=None): + from pydatview.scripter import PythonScripter + print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> PIPELINE SCRIPT') +# print(type(tabList)) +# print(tabList) + scripter = PythonScripter() + scripter.setFiles(tabList.filenames) + print('Flavor',flavorDict) + scripter.setFlavor(**flavorDict) + + if ID is not None: + for i,idx in enumerate(ID): + print('>>>> PIPELINE ', idx) + it = idx[0] + it = idx[0] # table index + ix = idx[1] # x index + iy = idx[2] # y index + kx = tabList[it].columns[ix] + ky = tabList[it].columns[iy] + scripter.select_data(it, kx, ky) + # Initialize each plotdata based on selected table and selected id channels + #pd.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeLike) + #PD.id = i + #PD.it = idx[0] # table index + #PD.ix = idx[1] # x index + #PD.iy = idx[2] # y index + #PD.sx = idx[3].replace('_',' ') # x label + #PD.sy = idx[4].replace('_',' ') # y label + #PD.syl = '' # y label for legend + #PD.st = idx[5] # table label + #PD.filename = tabs[PD.it].filename + #PD.tabname = tabs[PD.it].active_name + #PD.tabID = -1 # TODO + #PD.SameCol = SameCol + #PD.x, PD.xIsString, PD.xIsDate,_ = tabs[PD.it].getColumn(PD.ix) # actual x data, with info + #PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info + #PD.c =c # raw values, used by PDF + + + + + +# import_statements = ["import numpy as np", "import scipy.stats as stats"] +# # action_code = """df = np.mean(x) +# # p_value = stats.ttest_1samp(y, 0)[1] +# # """ +# action_code = """df = df""" +# scripter.add_action('filter', action_code, import_statements) +# scripter.add_preplot_action("x = x * 2") +# scripter.add_preplot_action("y = y + 10") +# +# plot_params = { +# 'figsize': (8, 6), +# 'xlabel': 'X-Axis', +# 'ylabel': 'Y-Axis', +# 'title': 'Sample Plot', +# 'label': 'Data Series', +# } +# +# scripter.set_plot_parameters(plot_params) +# + + + script = scripter.generate_script() + self.scripter = scripter + return script + + + def applyOnPlotData(self, x, y, tabID): x = np.copy(x) y = np.copy(y) diff --git a/pydatview/scripter.py b/pydatview/scripter.py new file mode 100644 index 0000000..7312d74 --- /dev/null +++ b/pydatview/scripter.py @@ -0,0 +1,305 @@ +import numpy as np +from collections import OrderedDict + +# From pydatview imports to WELIB +_WELIB={ + 'pydatview.io':'welib.weio', + 'pydatview.tools':'welib.tools', + } +_PYFAST={ + 'pydatview.io':'pyFAST.input_output', + 'pydatview.tools.tictoc':'pyFAST.tools.tictoc', + } + +_flavorReplaceDict={ + 'welib':_WELIB, + 'pyFAST':_PYFAST, + 'pydatview':{}, + } + +class PythonScripter: + def __init__(self, libFlavor='pydatview', dfsFlavor='dict', oneTabPerFile=False): + + + self.reset() + self.setFlavor(libFlavor=libFlavor, dfsFlavor=dfsFlavor) + + # Imports that we'll know we'll need + self.add_import('import pydatview.io as weio') + self.add_import('from pydatview.tools.tictoc import Timer') + self.add_import('import matplotlib.pyplot as plt') + + def reset(self): + self.indent=' ' + self.libFlavor = 'pydatview' + self.oneTabPerFile = False + self.dfsRepresentation = 'dict' + self.import_statements = set() + self.actions = OrderedDict() + self.preplot_actions = [] + self.filenames = [] + self.df_selections = [] # List of tuples (df_index, column_x, column_y) + self.dfs = [] + self.plot_params = {} # Dictionary for plotting parameters + + def setFlavor(self, libFlavor=None, dfsFlavor=None, oneTabPerFile=None): + if libFlavor is not None: + if libFlavor in _flavorReplaceDict.keys(): + self.libFlavor=libFlavor + else: + raise Exception('libFlavor not supported' + libFlavor) + if dfsFlavor is not None: + if dfsFlavor in ['dict', 'list', 'enumeration']: + self.dfsRepresentation = dfsFlavor + else: + raise Exception('dfsFlavor not supported' + dfsFlavor) + if oneTabPerFile is not None: + self.oneTabPerFile = oneTabPerFile + + + def add_import(self, import_statement): + self.import_statements.add(import_statement) + + def add_action(self, action_name, code, import_statements=None): + + for imports in import_statements: + self.add_import(imports) + + self.actions[action_name] = code + + def add_preplot_action(self, code): + self.preplot_actions.append(code) + + def setFiles(self, filenames): + self.filenames = [f.replace('\\','/') for f in filenames] + + def select_data(self, df_index, column_x, column_y): + self.df_selections.append((df_index, column_x, column_y)) + + def set_plot_parameters(self, params): + self.plot_params = params + + def generate_script(self): + + script = [] + indent1= self.indent + indent2= self.indent+self.indent + indent3= self.indent+self.indent+self.indent + # --- Disclaimer + script.append('""" Script generated by pyDataView - The script will likely need to be adapted."""') + + # --- Add import statements, different for different flavor + replaceDict=_flavorReplaceDict[self.libFlavor] + # pydatview imports will be last + imports = [ 'zzzzz'+ii if ii.find('pydatview')>0 else ii for ii in self.import_statements] + imports.sort() + imports = [ ii.replace('zzzzz', '') for ii in imports] + for statement in imports: + for k,v, in replaceDict.items(): + statement = statement.replace(k,v) + script.append(statement) + + # --- List of files + script.append("\n# --- Script parameters") + script.append("filenames = []") + for filename in self.filenames: + script.append(f"filenames += ['{filename}']") + + # --- List of Dataframes + script.append("\n# --- Open and convert files to DataFrames") + if self.dfsRepresentation == 'dict': + script.append("dfs = {}") + script.append("for iFile, filename in enumerate(filenames):") + if self.oneTabPerFile: + script.append(indent1 + "dfs[iFile] = weio.read(filename).toDataFrame()") + else: + script.append(indent1 + "dfs_or_df = weio.read(filename).toDataFrame()") + script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") + script.append(indent1 + "if isinstance(dfs_or_df, dict):") + script.append(indent2 + "for k,df in dfs_or_df.items():") + script.append(indent3 + "dfs[k+f'{iFile}'] = df") + script.append(indent1 + "else:") + script.append(indent2 + "dfs[f'tab{iFile}'] = dfs_or_df") + elif self.dfsRepresentation == 'list': + script.append("dfs = []") + script.append("for iFile, filename in enumerate(filenames):") + if self.oneTabPerFile: + script.append(indent1 + "df = weio.read(filenames[iFile]).toDataFrame()") + script.append(indent1 + "dfs.append(df)") + else: + script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") + script.append(indent1 + "dfs = weio.read(filenames[iFile]).toDataFrame()") + script.append(indent1 + "if isinstance(dfs_or_df, dict):") + script.append(indent2 + "dfs+= list(dfs_or_df.values() # NOTE: user will need to adapt this.") + script.append(indent1 + "else:") + script.append(indent2 + "dfs.append(dfs_or_df)") + + elif self.dfsRepresentation == 'enumeration': + for iFile, filename in enumerate(self.filenames): + iFile1 = iFile+1 + if self.oneTabPerFile: + script.append(f"df{iFile1} = weio.read(filenames[{iFile}]).toDataFrame()") + else: + script.append("# NOTE: we need a different action if the file contains multiple dataframes") + script.append(f"dfs_or_df = weio.read('{filename}').toDataFrame()") + script.append("if isinstance(dfs_or_df, dict):") + script.append(indent1 + f"df{iFile1} = dfs_or_df.items()[0][1] # NOTE: user will need to adapt this.") + script.append("else:") + script.append(indent1 + f"df{iFile1} = dfs_or_df") + + + if len(self.actions)>0: + + def addActionCode(actioname, actioncode, ind): + script.append(ind+ "# Apply action {}".format(actioname)) + lines = actioncode.split("\n") + indented_lines = [ind + line for line in lines] + script.append("\n".join(indented_lines)) + + script.append("\n# --- Apply actions to dataframes") + if self.dfsRepresentation=='dict': + script.append("for k, df in dfs.items():") + for actionname, actioncode in self.actions.items(): + addActionCode(actionname, actioncode, indent1) + + elif self.dfsRepresentation=='list': + script.append("for df in dfs):") + for actionname, actioncode in self.actions.items(): + addActionCode(actionname, actioncode, indent1) + + elif self.dfsRepresentation=='enumeration': + for iTab in range(len(self.filenames)): + script.append('df = df{}'.format(iTab+1)) + addActionCode(actionname, actioncode, indent2) + +# + + + # --- Plot Styling + script.append("\n# --- Generate the plot") + # Plot Styling + script.append("# Plot styling") + script.append("stys=['-','-',':','.-'] * len(dfs)") + script.append("cols=['r', 'g', 'b'] * len(dfs)") + if self.dfsRepresentation=='dict': + script.append("tabNames = list(dfs.keys())") + script.append("# Subplots") + script.append("fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8))") + script.append("fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20)") + + for df_index, column_x, column_y in self.df_selections: + script.append("\n# Selecting data for df{}".format(df_index+1)) + if self.dfsRepresentation=='dict': + script.append("x = dfs[tabNames[{}]]['{}']".format(df_index, column_x)) + script.append("y = dfs[tabNames[{}]]['{}']".format(df_index, column_y)) + elif self.dfsRepresentation=='list': + script.append("x = dfs[{}]['{}']".format(df_index, column_x)) + script.append("y = dfs[{}]['{}']".format(df_index, column_y)) + elif self.dfsRepresentation=='enumerate': + script.append("x = df{}['{}']".format(df_index+1, column_x)) + script.append("y = df{}['{}']".format(df_index+1, column_y)) + pass + if len(self.preplot_actions)>0: + script.append("# Applying preplot action for df{}".format(df_index+1)) + for preplot_action in self.preplot_actions: + script.append(preplot_action) + + script.append("# Plotting for df{}".format(df_index+1)) + script.append("ax.plot(x, y, '-', label='')") + + + script.append("ax.set_xlabel('')") + script.append("ax.set_ylabel('')") + #script.append("ax.legend()") + script.append("plt.show()") +# plot_code = f""" +# plt.figure(figsize=({self.plot_params['figsize']})) +# plt.xlabel('{self.plot_params['xlabel']}') +# plt.ylabel('{self.plot_params['ylabel']}') +# plt.title('{self.plot_params['title']}') +# plt.plot(x, y, label='{self.plot_params['label']}') +# plt.legend() +# plt.show() +# """ +# script.append(plot_code) + + return "\n".join(script) + + + def run(self, method='subprocess'): + script = self.generate_script() + import tempfile + import subprocess + import os + errors=[] + + if method=='subprocess': + try: + # --- Create a temporary file + #temp_dir = tempfile.TemporaryDirectory() + #script_file_path = os.path.join(temp_dir.name, "temp_script.py") + script_file_path ='pydatview_temp_script.py' + with open(script_file_path, "w") as script_file: + script_file.write(script) + + + # Run the script as a system call + result = subprocess.run(["python", script_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + # Print the output and errors (if any) + #errors.append("Script Output:") + #errors.append(result.stdout) + if len(result.stderr)>0: + errors.append(result.stderr) + except Exception as e: + error.append("An error occurred: {e}") + finally: + # Clean up by deleting the temporary directory and its contents + #temp_dir.cleanup() + pass +# os.remove(script_file_path) + else: + raise NotImplementedError() + if len(errors)>0: + raise Exception('\n'.join(errors)) + + + + +if __name__ == '__main__': + # Example usage: + scripter = PythonScripter() + scripter.setFiles(['../DampingExplodingExample.csv']) + + import_statements = ["import numpy as np", "import scipy.stats as stats"] +# action_code = """df = np.mean(x) +# p_value = stats.ttest_1samp(y, 0)[1] +# """ + action_code = """df = df""" + scripter.add_action('filter', action_code, import_statements) + scripter.select_data(0, "Time", "TTDspFA") + scripter.add_preplot_action("x = x * 2") + scripter.add_preplot_action("y = y + 10") + + plot_params = { + 'figsize': (8, 6), + 'xlabel': 'X-Axis', + 'ylabel': 'Y-Axis', + 'title': 'Sample Plot', + 'label': 'Data Series', + } + + scripter.set_plot_parameters(plot_params) + + +# scripter.setFlavor(libFlavor='welib', dfsFlavor='dict') + scripter.setFlavor(libFlavor='pydatview', dfsFlavor='dict', oneTabPerFile=False) +# scripter.setFlavor(libFlavor='welib', dfsFlavor='list', oneTabPerFile=True) +# scripter.setFlavor(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=True) +# scripter.setFlavor(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=False) + script = scripter.generate_script() + print(script) + scripter.run() + import matplotlib.pyplot as plt + plt.show() + From f201fbc6b4c79b9763ed615ba6b516efab67f952 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 24 Sep 2023 20:47:13 -0600 Subject: [PATCH 123/178] Scripter: action data, plotdata action (see #13) --- example_files/CSVComma.csv | 2 +- pydatview/GUIScripter.py | 25 +-- pydatview/pipeline.py | 75 ++++--- pydatview/plugins/data_standardizeUnits.py | 34 +++- pydatview/plugins/plotdata_binning.py | 14 +- pydatview/plugins/plotdata_filter.py | 13 +- pydatview/plugins/plotdata_removeOutliers.py | 13 +- pydatview/plugins/plotdata_sampler.py | 13 +- pydatview/scripter.py | 194 +++++++++++-------- pydatview/tools/spectral.py | 2 +- pydatview/tools/stats.py | 6 +- tests/test_pipeline.py | 133 +++++++++---- 12 files changed, 348 insertions(+), 176 deletions(-) diff --git a/example_files/CSVComma.csv b/example_files/CSVComma.csv index 35ea236..6e0d921 100644 --- a/example_files/CSVComma.csv +++ b/example_files/CSVComma.csv @@ -1,4 +1,4 @@ -ColA,ColB +ColA,ColB_[rpm] 1,4 2,9 3,6 diff --git a/pydatview/GUIScripter.py b/pydatview/GUIScripter.py index 6ceea83..7801420 100644 --- a/pydatview/GUIScripter.py +++ b/pydatview/GUIScripter.py @@ -14,9 +14,11 @@ def __init__(self, parent, mainframe, pipeLike, title): self.btGen = wx.Button(self.panel, label="Regenerate") self.btRun = wx.Button(self.panel, label="Run Script") self.btSave = wx.Button(self.panel, label="Save to File") - self.flavors = ["welib", "pydatview", "pyFAST"] - self.cbFlavors = wx.Choice(self.panel, choices=self.flavors) + libflavors = ["welib", "pydatview", "pyFAST"] + self.cbFlavors = wx.Choice(self.panel, choices=libflavors) self.cbFlavors.SetSelection(1) + mono_font = wx.Font(10, wx.FONTFAMILY_TELETYPE, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) + self.text_ctrl.SetFont(mono_font) # --- Layout vbox = wx.BoxSizer(wx.VERTICAL) @@ -40,31 +42,30 @@ def __init__(self, parent, mainframe, pipeLike, title): self.generateScript() - - def generateScript(self, *args, **kwargs): + def _GUI2Data(self, *args, **kwargs): # GUI2Data - flavorDict={} - flavorDict['libFlavor'] = self.cbFlavors.GetStringSelection() + data={} + data['libFlavor'] = self.cbFlavors.GetStringSelection() + return data + def generateScript(self, *args, **kwargs): + data = self._GUI2Data() try: ID,SameCol,selMode=self.mainframe.selPanel.getPlotDataSelection() except: ID is None - s = self.pipeline.script(self.mainframe.tabList, flavorDict, ID) + s = self.pipeline.script(self.mainframe.tabList, data, ID) self.text_ctrl.SetValue(s) def onFlavorChange(self, event): - flavor = self.cbFlavors.GetStringSelection() - # You can add code here to handle the selected format change event self.generateScript() - def onRun(self, event): + """ Run the script in user terminal """ self.pipeline.scripter.run() - - def onSave(self, event): + """ Save script to file """ file_extension = "py" dialog = wx.FileDialog(self, "Save Script to File", wildcard=f"(*.{file_extension})|*.{file_extension}", style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index e55a324..eee244e 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -17,6 +17,9 @@ - guiEditorClass: to Edit the data of the action + - code: a python script to manipulate a dataframe "df" + - imports: import statements needed for the script + """ import numpy as np from pydatview.common import exception2string, PyDatViewException @@ -32,6 +35,9 @@ def __init__(self, name, guiEditorClass=None, data=None, mainframe=None, + imports=None, + data_var=None, + code=None, onPlotData=False, unique=True, removeNeedReload=False): """ tableFunction: signature: f(tab) # TODO that's inplace @@ -51,6 +57,10 @@ def __init__(self, name, # TODO remove me, replace with generic "redraw", "update tab list" self.guiEditorClass = guiEditorClass # Class that can be used to edit this action self.guiEditorObj = None # Instance of guiEditorClass that can be used to edit this action + # For source code generation + self.code=code + self.imports=imports + self.data_var=data_var self.mainframe=mainframe # If possible, dont use that... @@ -122,6 +132,18 @@ def updateGUI(self): # except: # print('[FAIL] Action: failed to call GUI callback, action', self.name) + def getScript(self): + code_init = '' + if self.data is not None and len(self.data)>0 and self.data_var is not None: + code_init = self.data_var + " = {}\n" + for k,v, in self.data.items(): + #print('>>> type ', type(v), k, v) + if type(v) is str: + code_init += "{}['{}'] = '{}'\n".format(self.data_var,k,v) + else: + code_init += "{}['{}'] = {}\n".format(self.data_var,k,v) + return self.code, self.imports, code_init + def __repr__(self): s=''.format(self.name) return s @@ -265,26 +287,41 @@ def apply(self, tabList, force=False, applyToAll=False): # self.collectErrors() - def script(self, tabList, flavorDict, ID=None): + def script(self, tabList, scripterOptions=None, ID=None): from pydatview.scripter import PythonScripter - print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> PIPELINE SCRIPT') -# print(type(tabList)) -# print(tabList) scripter = PythonScripter() - scripter.setFiles(tabList.filenames) - print('Flavor',flavorDict) - scripter.setFlavor(**flavorDict) + scripter.setFiles([t for t in tabList.filenames if len(t)>0]) + #print('Flavor',scripterOptions) + if scripterOptions is not None: + scripter.setOptions(**scripterOptions) + # --- Data and preplot actions + for action in self.actionsData: + action_code, imports, code_init = action.getScript() + if action_code is None: + print('[WARN] No scripting routine for action {}'.format(action.name)) + else: + #print('[INFO] Scripting routine for action {}'.format(action.name)) + scripter.addAction(action.name, action_code, imports, code_init) + + for action in self.actionsPlotFilters: + action_code, imports, code_init = action.getScript() + if action_code is None: + print('[WARN] No scripting routine for action {}'.format(action.name)) + else: + #print('[INFO] Scripting routine for plot action {}'.format(action.name)) + scripter.addPreplotAction(action.name, action_code, imports, code_init) + + # --- Selecting data if ID is not None: for i,idx in enumerate(ID): - print('>>>> PIPELINE ', idx) - it = idx[0] + #print('>>>> PIPELINE ', idx) it = idx[0] # table index ix = idx[1] # x index iy = idx[2] # y index kx = tabList[it].columns[ix] ky = tabList[it].columns[iy] - scripter.select_data(it, kx, ky) + scripter.selectData(it, kx, ky) # Initialize each plotdata based on selected table and selected id channels #pd.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeLike) #PD.id = i @@ -303,19 +340,6 @@ def script(self, tabList, flavorDict, ID=None): #PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info #PD.c =c # raw values, used by PDF - - - - -# import_statements = ["import numpy as np", "import scipy.stats as stats"] -# # action_code = """df = np.mean(x) -# # p_value = stats.ttest_1samp(y, 0)[1] -# # """ -# action_code = """df = df""" -# scripter.add_action('filter', action_code, import_statements) -# scripter.add_preplot_action("x = x * 2") -# scripter.add_preplot_action("y = y + 10") -# # plot_params = { # 'figsize': (8, 6), # 'xlabel': 'X-Axis', @@ -323,12 +347,9 @@ def script(self, tabList, flavorDict, ID=None): # 'title': 'Sample Plot', # 'label': 'Data Series', # } -# # scripter.set_plot_parameters(plot_params) -# - - script = scripter.generate_script() + script = scripter.generate() self.scripter = scripter return script diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py index 686e4f9..7a69b84 100644 --- a/pydatview/plugins/data_standardizeUnits.py +++ b/pydatview/plugins/data_standardizeUnits.py @@ -1,3 +1,4 @@ +import pandas as pd import numpy as np from pydatview.common import splitunit from pydatview.pipeline import IrreversibleTableAction @@ -5,7 +6,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def standardizeUnitsAction(label, mainframe=None, flavor='SI'): +def standardizeUnitsAction(label='stdUnits', mainframe=None, flavor='SI'): """ Return an "action" for the current plugin, to be used in the pipeline. """ @@ -29,7 +30,10 @@ def guiCallback(): tableFunctionApply=changeUnits, guiCallback=guiCallback, mainframe=mainframe, # shouldnt be needed - data = data + data = data , + imports = _imports, + data_var = _data_var, + code = _code ) return action @@ -37,18 +41,32 @@ def guiCallback(): # --------------------------------------------------------------------------------} # --- Main method # --------------------------------------------------------------------------------{ +_imports=['from pydatview.plugins.data_standardizeUnits import changeUnits'] +# _imports+=['from pydatview.Tables import Table'] +_data_var='changeUnitsData' +_code="""changeUnits(df, changeUnitsData)""" + def changeUnits(tab, data): """ Change units of a table NOTE: it relies on the Table class, which may change interface in the future.. """ + if not isinstance(tab, pd.DataFrame): + df = tab.data + else: + df = tab + if data['flavor']=='WE': - for i, colname in enumerate(tab.columns): - colname, tab.data.iloc[:,i] = change_units_to_WE(colname, tab.data.iloc[:,i]) - tab.data.columns.values[i] = colname + cols = [] + for i, colname in enumerate(df.columns): + colname_new, df.iloc[:,i] = change_units_to_WE(colname, df.iloc[:,i]) + cols.append(colname_new) + df.columns = cols elif data['flavor']=='SI': - for i, colname in enumerate(tab.columns): - colname, tab.data.iloc[:,i] = change_units_to_SI(colname, tab.data.iloc[:,i]) - tab.data.columns.values[i] = colname + cols = [] + for i, colname in enumerate(df.columns): + colname_new, df.iloc[:,i] = change_units_to_SI(colname, df.iloc[:,i]) + cols.append(colname_new) + df.columns = cols else: raise NotImplementedError(data['flavor']) diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index 370890e..beb76d0 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -14,7 +14,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def binningAction(label, mainframe=None, data=None): +def binningAction(label='binning', mainframe=None, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -26,7 +26,7 @@ def binningAction(label, mainframe=None, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw if mainframe is not None else None action = PlotDataAction( name=label, @@ -35,12 +35,20 @@ def binningAction(label, mainframe=None, data=None): guiEditorClass = BinningToolPanel, guiCallback = guiCallback, data = data, - mainframe=mainframe + mainframe = mainframe, + imports = _imports, + data_var = _data_var, + code = _code ) return action # --------------------------------------------------------------------------------} # --- Main method # --------------------------------------------------------------------------------{ +_imports =['from pydatview.tools.stats import bin_signal'] +_imports+=['import numpy as np'] +_data_var='binData' +_code="""x, y = bin_signal(x, y, xbins=np.linspace(binData['xMin'], binData['xMax'], binData['nBins']+1))""" + def bin_plot(x, y, opts): from pydatview.tools.stats import bin_signal xBins = np.linspace(opts['xMin'], opts['xMax'], opts['nBins']+1) diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index 60e126d..468f0ef 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -19,7 +19,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def filterAction(label, mainframe, data=None): +def filterAction(label='filter', mainframe=None, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -31,7 +31,7 @@ def filterAction(label, mainframe, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw if mainframe is not None else None action = PlotDataAction( name = label, @@ -40,12 +40,19 @@ def filterAction(label, mainframe, data=None): guiEditorClass = FilterToolPanel, guiCallback = guiCallback, data = data, - mainframe = mainframe + mainframe = mainframe, + imports = _imports, + data_var = _data_var, + code = _code ) return action # --------------------------------------------------------------------------------} # --- Main method # --------------------------------------------------------------------------------{ +_imports=['from pydatview.tools.signal_analysis import applyFilter'] +_data_var='filterData' +_code="""y = applyFilter(x, y, filterData)""" + def filterXY(x, y, opts): """ Apply action on a x and y array """ from pydatview.tools.signal_analysis import applyFilter diff --git a/pydatview/plugins/plotdata_removeOutliers.py b/pydatview/plugins/plotdata_removeOutliers.py index 535fedb..1cae4f2 100644 --- a/pydatview/plugins/plotdata_removeOutliers.py +++ b/pydatview/plugins/plotdata_removeOutliers.py @@ -13,7 +13,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def removeOutliersAction(label, mainframe, data=None): +def removeOutliersAction(label='removeOutlier', mainframe=None, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -25,7 +25,7 @@ def removeOutliersAction(label, mainframe, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw if mainframe is not None else None action = PlotDataAction( name = label, @@ -33,12 +33,19 @@ def removeOutliersAction(label, mainframe, data=None): guiEditorClass = RemoveOutliersToolPanel, guiCallback = guiCallback, data = data, - mainframe = mainframe + mainframe = mainframe, + imports = _imports, + data_var = _data_var, + code = _code ) return action # --------------------------------------------------------------------------------} # --- Main method # --------------------------------------------------------------------------------{ +_imports=['from pydatview.tools.signal_analysis import reject_outliers'] +_data_var='outliersData' +_code="""x, y = reject_outliers(x, y, m=outliersData['medianDeviation'])""" + def removeOutliersXY(x, y, opts): from pydatview.tools.signal_analysis import reject_outliers try: diff --git a/pydatview/plugins/plotdata_sampler.py b/pydatview/plugins/plotdata_sampler.py index 602f395..4c5cc26 100644 --- a/pydatview/plugins/plotdata_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -15,7 +15,7 @@ # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def samplerAction(label, mainframe, data=None): +def samplerAction(label='sampler', mainframe=None, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -27,7 +27,7 @@ def samplerAction(label, mainframe, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw if mainframe is not None else None action = PlotDataAction( name = label, @@ -36,12 +36,19 @@ def samplerAction(label, mainframe, data=None): guiEditorClass = SamplerToolPanel, guiCallback = guiCallback, data = data, - mainframe = mainframe + mainframe = mainframe, + imports = _imports, + data_var = _data_var, + code = _code ) return action # --------------------------------------------------------------------------------} # --- Main method # --------------------------------------------------------------------------------{ +_imports=['from pydatview.tools.signal_analysis import applySampler'] +_data_var='samplerData' +_code="""x, y = applySampler(x, y, samplerData)""" + def samplerXY(x, y, opts): from pydatview.tools.signal_analysis import applySampler x_new, y_new = applySampler(x, y, opts) diff --git a/pydatview/scripter.py b/pydatview/scripter.py index 7312d74..541bc96 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -17,79 +17,78 @@ 'pydatview':{}, } -class PythonScripter: - def __init__(self, libFlavor='pydatview', dfsFlavor='dict', oneTabPerFile=False): +_defaultOpts={ + 'libFlavor':'pydatview', + 'dfsFlavor':'dict', + 'oneTabPerFile':False, + 'indent':' ', + } +class PythonScripter: + def __init__(self, **opts): self.reset() - self.setFlavor(libFlavor=libFlavor, dfsFlavor=dfsFlavor) + self.setOptions(**opts) - # Imports that we'll know we'll need - self.add_import('import pydatview.io as weio') - self.add_import('from pydatview.tools.tictoc import Timer') - self.add_import('import matplotlib.pyplot as plt') + # Imports that we know we'll need + self.addImport('import numpy as np') + self.addImport('import pydatview.io as weio') + self.addImport('import matplotlib.pyplot as plt') def reset(self): - self.indent=' ' - self.libFlavor = 'pydatview' - self.oneTabPerFile = False - self.dfsRepresentation = 'dict' + self.opts = _defaultOpts self.import_statements = set() self.actions = OrderedDict() - self.preplot_actions = [] + self.preplot_actions = OrderedDict() self.filenames = [] self.df_selections = [] # List of tuples (df_index, column_x, column_y) self.dfs = [] self.plot_params = {} # Dictionary for plotting parameters - def setFlavor(self, libFlavor=None, dfsFlavor=None, oneTabPerFile=None): - if libFlavor is not None: - if libFlavor in _flavorReplaceDict.keys(): - self.libFlavor=libFlavor - else: - raise Exception('libFlavor not supported' + libFlavor) - if dfsFlavor is not None: - if dfsFlavor in ['dict', 'list', 'enumeration']: - self.dfsRepresentation = dfsFlavor - else: - raise Exception('dfsFlavor not supported' + dfsFlavor) - if oneTabPerFile is not None: - self.oneTabPerFile = oneTabPerFile - - - def add_import(self, import_statement): + def setOptions(self, **opts): + for k,v in opts.items(): + if k not in _defaultOpts.keys(): + raise Exception('Unsupported option for scripter {}'.format(v)) + if k =='libFlavor' and v not in _flavorReplaceDict.keys(): + raise Exception('libFlavor not supported' + v) + if k =='dfsFlavor' and v not in ['dict', 'list', 'enumeration']: + raise Exception('dfsFlavor not supported' + v) + self.opts[k] = v + + def addImport(self, import_statement): self.import_statements.add(import_statement) - def add_action(self, action_name, code, import_statements=None): - - for imports in import_statements: - self.add_import(imports) - - self.actions[action_name] = code + def addAction(self, action_name, code, imports=None, init_code=None): + for imp in imports: + self.addImport(imp) + self.actions[action_name] = (init_code, code) - def add_preplot_action(self, code): - self.preplot_actions.append(code) + def addPreplotAction(self, action_name, code, imports=None, init_code=None): + if imports is not None: + for imp in imports: + self.addImport(imp) + self.preplot_actions[action_name] = (init_code, code) def setFiles(self, filenames): self.filenames = [f.replace('\\','/') for f in filenames] - def select_data(self, df_index, column_x, column_y): + def selectData(self, df_index, column_x, column_y): self.df_selections.append((df_index, column_x, column_y)) - def set_plot_parameters(self, params): + def setPlotParameters(self, params): self.plot_params = params - def generate_script(self): + def generate(self, pltshow=True): script = [] - indent1= self.indent - indent2= self.indent+self.indent - indent3= self.indent+self.indent+self.indent + indent1= self.opts['indent'] + indent2= indent1 + indent1 + indent3= indent2 + indent1 # --- Disclaimer - script.append('""" Script generated by pyDataView - The script will likely need to be adapted."""') + script.append('""" Script generated by pyDatView - The script will likely need to be adapted."""') # --- Add import statements, different for different flavor - replaceDict=_flavorReplaceDict[self.libFlavor] + replaceDict=_flavorReplaceDict[self.opts['libFlavor']] # pydatview imports will be last imports = [ 'zzzzz'+ii if ii.find('pydatview')>0 else ii for ii in self.import_statements] imports.sort() @@ -105,12 +104,30 @@ def generate_script(self): for filename in self.filenames: script.append(f"filenames += ['{filename}']") + # --- Init data actions + if len(self.actions)>0: + script.append("\n# --- Data for actions") + for actionname, actioncode in self.actions.items(): + if actioncode[0] is not None: + if len(actioncode[0].strip())>0: + script.append("# Data for action {}".format(actionname)) + script.append(actioncode[0].strip()) + + if len(self.preplot_actions)>0: + script.append("\n# --- Data for preplot actions") + for actionname, actioncode in self.preplot_actions.items(): + if actioncode[0] is not None: + if len(actioncode[0].strip())>0: + script.append("# Data for preplot action {}".format(actionname)) + script.append(actioncode[0].strip()) + + # --- List of Dataframes script.append("\n# --- Open and convert files to DataFrames") - if self.dfsRepresentation == 'dict': + if self.opts['dfsFlavor'] == 'dict': script.append("dfs = {}") script.append("for iFile, filename in enumerate(filenames):") - if self.oneTabPerFile: + if self.opts['oneTabPerFile']: script.append(indent1 + "dfs[iFile] = weio.read(filename).toDataFrame()") else: script.append(indent1 + "dfs_or_df = weio.read(filename).toDataFrame()") @@ -120,10 +137,10 @@ def generate_script(self): script.append(indent3 + "dfs[k+f'{iFile}'] = df") script.append(indent1 + "else:") script.append(indent2 + "dfs[f'tab{iFile}'] = dfs_or_df") - elif self.dfsRepresentation == 'list': + elif self.opts['dfsFlavor'] == 'list': script.append("dfs = []") script.append("for iFile, filename in enumerate(filenames):") - if self.oneTabPerFile: + if self.opts['oneTabPerFile']: script.append(indent1 + "df = weio.read(filenames[iFile]).toDataFrame()") script.append(indent1 + "dfs.append(df)") else: @@ -134,10 +151,10 @@ def generate_script(self): script.append(indent1 + "else:") script.append(indent2 + "dfs.append(dfs_or_df)") - elif self.dfsRepresentation == 'enumeration': + elif self.opts['dfsFlavor'] == 'enumeration': for iFile, filename in enumerate(self.filenames): iFile1 = iFile+1 - if self.oneTabPerFile: + if self.opts['oneTabPerFile']: script.append(f"df{iFile1} = weio.read(filenames[{iFile}]).toDataFrame()") else: script.append("# NOTE: we need a different action if the file contains multiple dataframes") @@ -147,7 +164,21 @@ def generate_script(self): script.append("else:") script.append(indent1 + f"df{iFile1} = dfs_or_df") + # --- Insert index for convenience + script.append("\n# --- Insert columns") + if self.opts['dfsFlavor'] == 'dict': + script.append("for k, df in dfs.items():") + script.append(indent1 + "df.insert(0, 'Index', np.arange(df.shape[0]))") + + elif self.opts['dfsFlavor'] == 'list': + script.append("for df in dfs):") + script.append(indent1 + "df.insert(0, 'Index', np.arange(df.shape[0]))") + elif self.opts['dfsFlavor'] == 'enumeration': + for iTab in range(len(self.filenames)): + script.append("df{}.insert(0, 'Index', np.arange(df.shape[0]))".format(iTab+1)) + + # --- Data Actions if len(self.actions)>0: def addActionCode(actioname, actioncode, ind): @@ -157,23 +188,24 @@ def addActionCode(actioname, actioncode, ind): script.append("\n".join(indented_lines)) script.append("\n# --- Apply actions to dataframes") - if self.dfsRepresentation=='dict': + if self.opts['dfsFlavor'] == 'dict': script.append("for k, df in dfs.items():") for actionname, actioncode in self.actions.items(): - addActionCode(actionname, actioncode, indent1) + addActionCode(actionname, actioncode[1], indent1) + script.append(indent1 + "dfs[k] = df") - elif self.dfsRepresentation=='list': + elif self.opts['dfsFlavor'] == 'list': script.append("for df in dfs):") for actionname, actioncode in self.actions.items(): - addActionCode(actionname, actioncode, indent1) + addActionCode(actionname, actioncode[1], indent1) + script.append(indent1 + "dfs[k] = df") - elif self.dfsRepresentation=='enumeration': + elif self.opts['dfsFlavor'] == 'enumeration': for iTab in range(len(self.filenames)): script.append('df = df{}'.format(iTab+1)) - addActionCode(actionname, actioncode, indent2) - -# - + for actionname, actioncode in self.actions.items(): + addActionCode(actionname, actioncode[1], indent2) + script.append('df{} = df'.format(iTab+1)) # --- Plot Styling script.append("\n# --- Generate the plot") @@ -181,7 +213,7 @@ def addActionCode(actioname, actioncode, ind): script.append("# Plot styling") script.append("stys=['-','-',':','.-'] * len(dfs)") script.append("cols=['r', 'g', 'b'] * len(dfs)") - if self.dfsRepresentation=='dict': + if self.opts['dfsFlavor'] == 'dict': script.append("tabNames = list(dfs.keys())") script.append("# Subplots") script.append("fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8))") @@ -189,29 +221,30 @@ def addActionCode(actioname, actioncode, ind): for df_index, column_x, column_y in self.df_selections: script.append("\n# Selecting data for df{}".format(df_index+1)) - if self.dfsRepresentation=='dict': + if self.opts['dfsFlavor'] == 'dict': script.append("x = dfs[tabNames[{}]]['{}']".format(df_index, column_x)) script.append("y = dfs[tabNames[{}]]['{}']".format(df_index, column_y)) - elif self.dfsRepresentation=='list': + elif self.opts['dfsFlavor'] == 'list': script.append("x = dfs[{}]['{}']".format(df_index, column_x)) script.append("y = dfs[{}]['{}']".format(df_index, column_y)) - elif self.dfsRepresentation=='enumerate': + elif self.opts['dfsFlavor'] == 'enumeration': script.append("x = df{}['{}']".format(df_index+1, column_x)) script.append("y = df{}['{}']".format(df_index+1, column_y)) pass if len(self.preplot_actions)>0: script.append("# Applying preplot action for df{}".format(df_index+1)) - for preplot_action in self.preplot_actions: - script.append(preplot_action) + for actionname, actioncode in self.preplot_actions.items(): + script.append(actioncode[1]) script.append("# Plotting for df{}".format(df_index+1)) - script.append("ax.plot(x, y, '-', label='')") + script.append("ax.plot(x, y, 'o', label='')") script.append("ax.set_xlabel('')") script.append("ax.set_ylabel('')") #script.append("ax.legend()") - script.append("plt.show()") + if pltshow: + script.append("plt.show()") # plot_code = f""" # plt.figure(figsize=({self.plot_params['figsize']})) # plt.xlabel('{self.plot_params['xlabel']}') @@ -226,8 +259,8 @@ def addActionCode(actioname, actioncode, ind): return "\n".join(script) - def run(self, method='subprocess'): - script = self.generate_script() + def run(self, method='subprocess', pltshow=True): + script = self.generate(pltshow=pltshow) import tempfile import subprocess import os @@ -276,10 +309,13 @@ def run(self, method='subprocess'): # p_value = stats.ttest_1samp(y, 0)[1] # """ action_code = """df = df""" - scripter.add_action('filter', action_code, import_statements) - scripter.select_data(0, "Time", "TTDspFA") - scripter.add_preplot_action("x = x * 2") - scripter.add_preplot_action("y = y + 10") + init_code="""data['Misc']=2 # That's a parameter +data['Misc2']=3 # That's another parameter""" + scripter.addAction('filter', action_code, import_statements, init_code) + scripter.selectData(0, "Time", "TTDspFA") + + action_code="""x = x * 1\ny = y * 1""" + scripter.addPreplotAction('Scaling', action_code) plot_params = { 'figsize': (8, 6), @@ -289,15 +325,15 @@ def run(self, method='subprocess'): 'label': 'Data Series', } - scripter.set_plot_parameters(plot_params) + scripter.setPlotParameters(plot_params) # scripter.setFlavor(libFlavor='welib', dfsFlavor='dict') - scripter.setFlavor(libFlavor='pydatview', dfsFlavor='dict', oneTabPerFile=False) -# scripter.setFlavor(libFlavor='welib', dfsFlavor='list', oneTabPerFile=True) -# scripter.setFlavor(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=True) -# scripter.setFlavor(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=False) - script = scripter.generate_script() + scripter.setOptions(libFlavor='pydatview', dfsFlavor='dict', oneTabPerFile=False) +# scripter.setOptions(libFlavor='welib', dfsFlavor='list', oneTabPerFile=True) +# scripter.setOptions(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=True) +# scripter.setOptions(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=False) + script = scripter.generate() print(script) scripter.run() import matplotlib.pyplot as plt diff --git a/pydatview/tools/spectral.py b/pydatview/tools/spectral.py index 0a1cb9a..f999562 100644 --- a/pydatview/tools/spectral.py +++ b/pydatview/tools/spectral.py @@ -148,7 +148,7 @@ def psd_binned(y, fs=1.0, nPerDecade=10, detrend ='constant', return_onesided=Tr df = pd.DataFrame(data=np.column_stack((log_f,PSD)), columns=['x','y']) xmid = (xbins[:-1]+xbins[1:])/2 df['Bin'] = pd.cut(df['x'], bins=xbins, labels=xmid ) # Adding a column that has bin attribute - df2 = df.groupby('Bin').mean() # Average by bin + df2 = df.groupby('Bin', observed=False).mean() # Average by bin df2 = df2.reindex(xmid) log_f_bin = df2['x'].values PSD_bin = df2['y'].values diff --git a/pydatview/tools/stats.py b/pydatview/tools/stats.py index ff6c794..d8b448c 100644 --- a/pydatview/tools/stats.py +++ b/pydatview/tools/stats.py @@ -267,12 +267,12 @@ def bin_DF(df, xbins, colBin, stats='mean'): xmid = (xbins[:-1]+xbins[1:])/2 df['Bin'] = pd.cut(df[colBin], bins=xbins, labels=xmid ) # Adding a column that has bin attribute if stats=='mean': - df2 = df.groupby('Bin').mean() # Average by bin + df2 = df.groupby('Bin', observed=False).mean() # Average by bin elif stats=='std': - df2 = df.groupby('Bin').std() # std by bin + df2 = df.groupby('Bin', observed=False).std() # std by bin # also counting df['Counts'] = 1 - dfCount=df[['Counts','Bin']].groupby('Bin').sum() + dfCount=df[['Counts','Bin']].groupby('Bin', observed=False).sum() df2['Counts'] = dfCount['Counts'] # Just in case some bins are missing (will be nan) df2 = df2.reindex(xmid) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 6462865..d9d496f 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -8,44 +8,111 @@ from pydatview.Tables import TableList, Table from pydatview.pipeline import Pipeline, Action from pydatview.plotdata import PlotData +from pydatview.plugins import DATA_PLUGINS_SIMPLE -class TestPipeline(unittest.TestCase): +scriptDir = os.path.dirname(__file__) - def test_pipeline(self): - pass +class TestPipeline(unittest.TestCase): + def test_pipeline_std_units(self): + # Add two actions to the pipeline and verify that "apply" works as intended + + # --- Create Tabe List + tablist = TableList.createDummy(nTabs=1, n=5) + df = tablist._tabs[0].data.copy() + self.assertTrue('RotSpeed_0_[rpm]' in tablist._tabs[0].data.keys()) + + # --- Create a Pipeline with a Standardize units actions + DPD = DATA_PLUGINS_SIMPLE + pipeline = Pipeline() + + # --- Add and apply Standardize Units SI + action = DPD['Standardize Units (SI)'](label='Standardize Units (SI)') + pipeline.append(action, apply=False) + pipeline.apply(tablist) + self.assertTrue('RotSpeed_0_[rad/s]' in tablist._tabs[0].data.keys()) + + #print(pipeline) + #print('>>> Data') + #print(tablist._tabs[0].data) + + # --- Add and apply Standardize Units WE + action = DPD['Standardize Units (WE)'](label='Standardize Units (WE)') + pipeline.append(action, apply=False) + pipeline.apply(tablist) + self.assertTrue('RotSpeed_0_[rpm]' in tablist._tabs[0].data.keys()) + + np.testing.assert_array_almost_equal(tablist._tabs[0].data['RotSpeed_0_[rpm]'],df['RotSpeed_0_[rpm]'], 6) + #print('>>> Data 3') + #print(tablist._tabs[0].data) + #print('>>>> PIPELINE') + #print(pipeline) + + + def test_pipeline_script_squareIt(self): + # Add one action to the pipeline and verify that the script generation works + # And that the script is actually runable! + tablist = TableList() + tablist.load_tables_from_files(filenames=[os.path.join(scriptDir,'../example_files/CSVComma.csv')]) + + # Dummy action + imports=['import pandas as pd'] + code="""df = df**2""" + action = Action(name='squareIt!', code=code, imports=imports) + + pipeline = Pipeline() + pipeline.append(action, apply=False) + ID= [[0,1,2]] + s = pipeline.script(tablist, ID=ID, scripterOptions=None) + #print(s) + pipeline.scripter.run(pltshow=False) + + def test_pipeline_script_plotdata_plugins(self): + # Add a Plot Data action to the pipeline and verify that the script generation works + # And that the script is actually runable! + from pydatview.plugins.plotdata_sampler import samplerAction + from pydatview.plugins.plotdata_binning import binningAction + from pydatview.plugins.plotdata_filter import filterAction + from pydatview.plugins.plotdata_removeOutliers import removeOutliersAction + tablist = TableList() + tablist.load_tables_from_files(filenames=[os.path.join(scriptDir,'../example_files/FASTIn_arf_coords.txt')]) + + pipeline = Pipeline() + # action + action = samplerAction() + action.data['name']='Every n' + action.data['param']=3 + + pipeline.append(binningAction(data={'xMin':0, 'xMax':1, 'nBins':100}), apply=False) + pipeline.append(action, apply=False) + pipeline.append(filterAction(), apply=False) + pipeline.append(removeOutliersAction(), apply=False) + ID= [[0,1,2]] + s = pipeline.script(tablist, ID=ID, scripterOptions=None) + #print(s) + pipeline.scripter.run(pltshow=False) + + def test_pipeline_script_data_plugins(self): + # Add a Data action to the pipeline and verify that the script generation works + # And that the script is actually runable! + from pydatview.plugins.data_standardizeUnits import standardizeUnitsAction + tablist = TableList() + tablist.load_tables_from_files(filenames=[os.path.join(scriptDir,'../example_files/CSVComma.csv')]) + + pipeline = Pipeline() + pipeline.append(standardizeUnitsAction(), apply=False) + pipeline.apply(tablist) # Apply will change unit + ID= [[0,1,2]] + s = pipeline.script(tablist, ID=ID, scripterOptions=None) + #print(s) + self.assertTrue(s.find('ColB_[rad/s]')>10) # Make sure units have been changed in script + pipeline.scripter.run(pltshow=False) if __name__ == '__main__': + unittest.main() +# TestPipeline().test_pipeline_script_squareIt() +# TestPipeline().test_pipeline_script_plotdata_plugins() +# TestPipeline().test_pipeline_script_data_plugins() - from pydatview.plugins import DATA_PLUGINS_SIMPLE - - DPD = DATA_PLUGINS_SIMPLE - - tablist = TableList.createDummy(1) - print(tablist._tabs[0].data) - - pipeline = Pipeline() - - action = DPD['Standardize Units (SI)'](label='Standardize Units (SI)') - pipeline.append(action, apply=False) - - print(pipeline) - - pipeline.apply(tablist) - - print(tablist._tabs[0].data) - - pipeline.apply(tablist) - print(tablist._tabs[0].data) - - - - action = DPD['Standardize Units (WE)'](label='Standardize Units (WE)') - pipeline.append(action, apply=False) - - pipeline.apply(tablist) - print(tablist._tabs[0].data) - pipeline.apply(tablist, force=True) - print(tablist._tabs[0].data) From 56363eeecc43d868e61427d9e733c1479c8e87b7 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Sun, 24 Sep 2023 23:44:27 -0600 Subject: [PATCH 124/178] Scripter: formulae, index, adder actions (#13) --- pydatview/Tables.py | 223 +++++++++++-------------- pydatview/fast/postpro.py | 9 +- pydatview/formulae.py | 21 +++ pydatview/pipeline.py | 84 +++++++--- pydatview/plugins/data_mask.py | 40 ++++- pydatview/plugins/data_radialConcat.py | 14 +- pydatview/plugins/data_radialavg.py | 18 +- pydatview/scripter.py | 217 ++++++++++++++++++------ tests/test_Tables.py | 11 +- tests/test_pipeline.py | 2 + 10 files changed, 427 insertions(+), 212 deletions(-) create mode 100644 pydatview/formulae.py diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 0894e13..b0f6238 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -2,11 +2,9 @@ import os.path from dateutil import parser import pandas as pd -try: - from .common import no_unit, ellude_common, getDt, exception2string, PyDatViewException -except: - from common import no_unit, ellude_common, getDt, exception2string, PyDatViewException +from pydatview.common import no_unit, ellude_common, getDt, exception2string, PyDatViewException import pydatview.io as weio # File Formats and File Readers +from pydatview.formulae import evalFormula # --------------------------------------------------------------------------------} # --- TabList @@ -352,38 +350,38 @@ def __repr__(self): return '\n'.join([t.__repr__() for t in self._tabs]) # --- Mask related - @property - def maskStrings(self): - return [t.maskString for t in self._tabs] - - @property - def commonMaskString(self): - maskStrings=set(self.maskStrings) - if len(maskStrings) == 1: - return next(iter(maskStrings)) - else: - return '' - - def clearCommonMask(self): - for t in self._tabs: - t.clearMask() - - def applyCommonMaskString(self,maskString,bAdd=True): - # Apply mask on tablist - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - df_new, name_new = t.applyMaskString(maskString, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) - except Exception as e: - errors.append('Mask failed for table: '+t.nickname+'\n'+exception2string(e)) - - return dfs_new, names_new, errors +# @property +# def maskStrings(self): +# return [t.maskString for t in self._tabs] +# +# @property +# def commonMaskString(self): +# maskStrings=set(self.maskStrings) +# if len(maskStrings) == 1: +# return next(iter(maskStrings)) +# else: +# return '' +# +# def clearCommonMask(self): +# for t in self._tabs: +# t.clearMask() +# +# def applyCommonMaskString(self,maskString,bAdd=True): +# # Apply mask on tablist +# dfs_new = [] +# names_new = [] +# errors=[] +# for i,t in enumerate(self._tabs): +# try: +# df_new, name_new = t.applyMaskString(maskString, bAdd=bAdd) +# if df_new is not None: +# # we don't append when string is empty +# dfs_new.append(df_new) +# names_new.append(name_new) +# except Exception as e: +# errors.append('Mask failed for table: '+t.nickname+'\n'+exception2string(e)) +# +# return dfs_new, names_new, errors # --- Formulas @@ -404,62 +402,62 @@ def applyFormulas(self, formulas): tab.addColumnByFormula(f['name'], f['formula'], f['pos']-1) - # --- Resampling TODO MOVE THIS OUT OF HERE OR UNIFY - def applyResampling(self,iCol,sampDict,bAdd=True): - """ Apply resampling on table list - TODO Make this part of the action - """ - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - df_new, name_new = t.applyResampling(iCol, sampDict, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) - except Exception as e: - errors.append('Resampling failed for table: '+t.nickname+'\n'+exception2string(e)) - return dfs_new, names_new, errors - - # --- Filtering TODO MOVE THIS OUT OF HERE OR UNIFY - def applyFiltering(self,iCol,options,bAdd=True): - """ Apply filtering on table list - TODO Make this part of the action - """ - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - df_new, name_new = t.applyFiltering(iCol, options, bAdd=bAdd) - if df_new is not None: - # we don't append when string is empty - dfs_new.append(df_new) - names_new.append(name_new) - except Exception as e: - errors.append('Filtering failed for table: '+t.nickname+'\n'+exception2string(e)) - return dfs_new, names_new, errors - - # --- Radial average related - def radialAvg(self,avgMethod,avgParam): - """ Apply radial average on table list - TODO Make this part of the action - """ - dfs_new = [] - names_new = [] - errors=[] - for i,t in enumerate(self._tabs): - try: - dfs, names = t.radialAvg(avgMethod,avgParam) - for df,n in zip(dfs,names): - if df is not None: - dfs_new.append(df) - names_new.append(n) - except Exception as e: - errors.append('Radial averaging failed for table: '+t.nickname+'\n'+exception2string(e)) - return dfs_new, names_new, errors +# # --- Resampling TODO MOVE THIS OUT OF HERE OR UNIFY +# def applyResampling(self,iCol,sampDict,bAdd=True): +# """ Apply resampling on table list +# TODO Make this part of the action +# """ +# dfs_new = [] +# names_new = [] +# errors=[] +# for i,t in enumerate(self._tabs): +# try: +# df_new, name_new = t.applyResampling(iCol, sampDict, bAdd=bAdd) +# if df_new is not None: +# # we don't append when string is empty +# dfs_new.append(df_new) +# names_new.append(name_new) +# except Exception as e: +# errors.append('Resampling failed for table: '+t.nickname+'\n'+exception2string(e)) +# return dfs_new, names_new, errors +# +# # --- Filtering TODO MOVE THIS OUT OF HERE OR UNIFY +# def applyFiltering(self,iCol,options,bAdd=True): +# """ Apply filtering on table list +# TODO Make this part of the action +# """ +# dfs_new = [] +# names_new = [] +# errors=[] +# for i,t in enumerate(self._tabs): +# try: +# df_new, name_new = t.applyFiltering(iCol, options, bAdd=bAdd) +# if df_new is not None: +# # we don't append when string is empty +# dfs_new.append(df_new) +# names_new.append(name_new) +# except Exception as e: +# errors.append('Filtering failed for table: '+t.nickname+'\n'+exception2string(e)) +# return dfs_new, names_new, errors +# +# # --- Radial average related +# def radialAvg(self,avgMethod,avgParam): +# """ Apply radial average on table list +# TODO Make this part of the action +# """ +# dfs_new = [] +# names_new = [] +# errors=[] +# for i,t in enumerate(self._tabs): +# try: +# dfs, names = t.radialAvg(avgMethod,avgParam) +# for df,n in zip(dfs,names): +# if df is not None: +# dfs_new.append(df) +# names_new.append(n) +# except Exception as e: +# errors.append('Radial averaging failed for table: '+t.nickname+'\n'+exception2string(e)) +# return dfs_new, names_new, errors @staticmethod @@ -587,16 +585,6 @@ def clearMask(self): def applyMaskString(self, sMask, bAdd=True): # Apply mask on Table df = self.data - # TODO Loop on {VAR} instead.. - for i, c_in_df in enumerate(self.data.columns): - c_no_unit = no_unit(c_in_df).strip() - # TODO sort out the mess with asarray (introduced to have and/or - # as array won't work with date comparison - # NOTE: using iloc to avoid duplicates column issue - if isinstance(df.iloc[0,i], pd._libs.tslibs.timestamps.Timestamp): - sMask=sMask.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') - else: - sMask=sMask.replace('{'+c_no_unit+'}','np.asarray(df[\''+c_in_df+'\'])') df_new = None name_new = None if len(sMask.strip())>0 and sMask.strip().lower()!='no mask': @@ -826,33 +814,20 @@ def getColumn(self, i): x=x.astype('datetime64') return x,isString,isDate,c - - def evalFormula(self,sFormula): - df = self.data - for i,c in enumerate(self.columns): - c_no_unit = no_unit(c).strip() - c_in_df = df.columns[i] - sFormula=sFormula.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') - try: - NewCol=eval(sFormula) - return NewCol - except: - return None - - def addColumnByFormula(self,sNewName,sFormula,i=-1): - NewCol=self.evalFormula(sFormula) + def addColumnByFormula(self, sNewName, sFormulaRaw, i=-1): + NewCol=evalFormula(self.data, sFormulaRaw) if NewCol is None: return False else: - self.addColumn(sNewName,NewCol,i,sFormula) + self.addColumn(sNewName,NewCol,i,sFormulaRaw) return True - def setColumnByFormula(self,sNewName,sFormula,i=-1): - NewCol=self.evalFormula(sFormula) + def setColumnByFormula(self, sNewName, sFormulaRaw, i=-1): + NewCol=evalFormula(self.data, sFormulaRaw) if NewCol is None: return False else: - self.setColumn(sNewName,NewCol,i,sFormula) + self.setColumn(sNewName,NewCol,i,sFormulaRaw) return True @@ -954,8 +929,14 @@ def createDummy(n, label='', columns=None, nCols=None): return Table(data=df, name='Dummy '+label) + + if __name__ == '__main__': import pandas as pd; from Tables import Table import numpy as np + + + + diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 3777cae..02d8054 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -1147,9 +1147,12 @@ def spanwiseConcat(df): IdxAvailableForThisChannel = ColsInfoAD[ic]['Idx'] chanName = ColsInfoAD[ic]['name'] colName = ColsInfoAD[ic]['cols'][ir] - #print('Channel {}: colName {}'.format(chanName, colName)) - if ir+1 in IdxAvailableForThisChannel: - data[ir*nt:(ir+1)*nt, ic+2] = df[colName].values + print('Channel {}: colName {}'.format(chanName, colName)) + try: + if ir+1 in IdxAvailableForThisChannel: + data[ir*nt:(ir+1)*nt, ic+2] = df[colName].values + except: + pass #else: # raise Exception('Channel {}: Index missing {}'.format(chanName, ic+1)) columns = ['Time_[s]'] + ['i_[-]'] + [ColsInfoAD[i]['name'] for i in range(nChan)] diff --git a/pydatview/formulae.py b/pydatview/formulae.py new file mode 100644 index 0000000..bf591b7 --- /dev/null +++ b/pydatview/formulae.py @@ -0,0 +1,21 @@ +from pydatview.common import no_unit +# --------------------------------------------------------------------------------} +# --- Formula +# --------------------------------------------------------------------------------{ + +def formatFormula(df, sFormulaRaw): + sFormula = sFormulaRaw + for i,c in enumerate(df.columns): + c_no_unit = no_unit(c).strip() + c_in_df = df.columns[i] + sFormula=sFormula.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') + return sFormula + +def evalFormula(df, sFormulaRaw): + sFormula=formatFormula(df, sFormulaRaw) + try: + NewCol=eval(sFormula) + return NewCol + except: + return None + diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index eee244e..df2591a 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -135,13 +135,27 @@ def updateGUI(self): def getScript(self): code_init = '' if self.data is not None and len(self.data)>0 and self.data_var is not None: - code_init = self.data_var + " = {}\n" - for k,v, in self.data.items(): - #print('>>> type ', type(v), k, v) - if type(v) is str: - code_init += "{}['{}'] = '{}'\n".format(self.data_var,k,v) - else: - code_init += "{}['{}'] = {}\n".format(self.data_var,k,v) + if len(self.data)<5: + # One Liner + key_val=[] + for k,v, in self.data.items(): + if k=='active': + continue + if type(v) is str: + key_val.append( "'{}':\"{}\"".format(k, v) ) + else: + key_val.append( "'{}': {}".format(k, v) ) + code_init = self.data_var + " = {"+', '.join(key_val) + '}' + else: + code_init = self.data_var + " = {}\n" + for k,v, in self.data.items(): + if k=='active': + continue + #print('>>> type ', type(v), k, v) + if type(v) is str: + code_init += "{}['{}'] = \"{}\"\n".format(self.data_var,k,v) + else: + code_init += "{}['{}'] = {}\n".format(self.data_var,k,v) return self.code, self.imports, code_init def __repr__(self): @@ -289,20 +303,42 @@ def apply(self, tabList, force=False, applyToAll=False): def script(self, tabList, scripterOptions=None, ID=None): from pydatview.scripter import PythonScripter + if scripterOptions is None: + scripterOptions={} + + # --- Files and number of tables + fileNames = tabList.filenames + fileNamesTrue = [t for t in tabList.filenames if len(t)>0] + if len(fileNames)==len(fileNamesTrue): + # No tables were added + if len(fileNamesTrue) == len(tabList): + # Each file returned one table only + scripterOptions['oneTabPerFile'] = True + scripter = PythonScripter() - scripter.setFiles([t for t in tabList.filenames if len(t)>0]) + scripter.setFiles(fileNamesTrue) #print('Flavor',scripterOptions) - if scripterOptions is not None: - scripter.setOptions(**scripterOptions) + scripter.setOptions(**scripterOptions) - # --- Data and preplot actions + # --- Adder Actions for action in self.actionsData: - action_code, imports, code_init = action.getScript() - if action_code is None: - print('[WARN] No scripting routine for action {}'.format(action.name)) - else: - #print('[INFO] Scripting routine for action {}'.format(action.name)) - scripter.addAction(action.name, action_code, imports, code_init) + if isinstance(action, AdderAction): + action_code, imports, code_init = action.getScript() + if action_code is None: + print('[WARN] No scripting routine for action {}'.format(action.name)) + else: + scripter.addAdderAction(action.name, action_code, imports, code_init) + + + # --- Data Actions + for action in self.actionsData: + if not isinstance(action, AdderAction): + action_code, imports, code_init = action.getScript() + if action_code is None: + print('[WARN] No scripting routine for action {}'.format(action.name)) + else: + #print('[INFO] Scripting routine for action {}'.format(action.name)) + scripter.addAction(action.name, action_code, imports, code_init) for action in self.actionsPlotFilters: action_code, imports, code_init = action.getScript() @@ -312,6 +348,13 @@ def script(self, tabList, scripterOptions=None, ID=None): #print('[INFO] Scripting routine for plot action {}'.format(action.name)) scripter.addPreplotAction(action.name, action_code, imports, code_init) + # --- Formulae + from pydatview.formulae import formatFormula + for it, tab in enumerate(tabList): + for formulaDict in tab.formulas: + formula = formatFormula(tab.data, formulaDict['formula']) + scripter.addFormula(it, name=formulaDict['name'], formula=formula) + # --- Selecting data if ID is not None: for i,idx in enumerate(ID): @@ -325,17 +368,10 @@ def script(self, tabList, scripterOptions=None, ID=None): # Initialize each plotdata based on selected table and selected id channels #pd.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeLike) #PD.id = i - #PD.it = idx[0] # table index - #PD.ix = idx[1] # x index - #PD.iy = idx[2] # y index #PD.sx = idx[3].replace('_',' ') # x label #PD.sy = idx[4].replace('_',' ') # y label #PD.syl = '' # y label for legend #PD.st = idx[5] # table label - #PD.filename = tabs[PD.it].filename - #PD.tabname = tabs[PD.it].active_name - #PD.tabID = -1 # TODO - #PD.SameCol = SameCol #PD.x, PD.xIsString, PD.xIsDate,_ = tabs[PD.it].getColumn(PD.ix) # actual x data, with info #PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info #PD.c =c # raw values, used by PDF diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index 37866ae..545bff5 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd from pydatview.plugins.base_plugin import ActionEditor, TOOL_BORDER from pydatview.common import CHAR, Error, Info, pretty_num_short from pydatview.common import DummyMainFrame @@ -8,13 +9,14 @@ # --------------------------------------------------------------------------------{ _DEFAULT_DICT={ 'active':False, - 'maskString': '' + 'maskString': '', + 'formattedMaskString': '' } # --------------------------------------------------------------------------------} # --- Action # --------------------------------------------------------------------------------{ -def maskAction(label, mainframe, data=None): +def maskAction(label='mask', mainframe=None, data=None): """ Return an "action" for the current plugin, to be used in the pipeline. The action is also edited and created by the GUI Editor @@ -26,7 +28,7 @@ def maskAction(label, mainframe, data=None): data=_DEFAULT_DICT data['active'] = False #<<< Important - guiCallback = mainframe.redraw + guiCallback = mainframe.redraw if mainframe is not None else None action = ReversibleTableAction( name=label, @@ -36,15 +38,23 @@ def maskAction(label, mainframe, data=None): guiEditorClass = MaskToolPanel, guiCallback = guiCallback, data = data, - mainframe=mainframe + mainframe=mainframe, + imports = _imports, + data_var = _data_var, + code = _code ) return action # --------------------------------------------------------------------------------} # --- Main methods # --------------------------------------------------------------------------------{ +_imports=[] +_data_var='maskData' +_code="""df = df[eval(maskData['formattedMaskString'])]""" + def applyMask(tab, data): # dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=False) - dfs, name = tab.applyMaskString(data['maskString'], bAdd=False) + data['formattedMaskString'] = formatMaskString(tab.data, data['maskString']) + dfs, name = tab.applyMaskString(data['formattedMaskString'], bAdd=False) def removeMask(tab, data): tab.clearMask() @@ -54,9 +64,27 @@ def addTabMask(tab, opts): """ Apply action on a a table and return a new one with a new name df_new, name_new = f(t, opts) """ - df_new, name_new = tab.applyMaskString(opts['maskString'], bAdd=True) + opts['formattedMaskString'] = formatMaskString(tab.data, opts['maskString']) + df_new, name_new = tab.applyMaskString(opts['formattedMaskString'], bAdd=True) return df_new, name_new + +def formatMaskString(df, sMask): + """ """ + from pydatview.common import no_unit + # TODO Loop on {VAR} instead.. + for i, c_in_df in enumerate(df.columns): + c_no_unit = no_unit(c_in_df).strip() + # TODO sort out the mess with asarray (introduced to have and/or + # as array won't work with date comparison + # NOTE: using iloc to avoid duplicates column issue + if isinstance(df.iloc[0,i], pd._libs.tslibs.timestamps.Timestamp): + sMask=sMask.replace('{'+c_no_unit+'}','df[\''+c_in_df+'\']') + else: + sMask=sMask.replace('{'+c_no_unit+'}','np.asarray(df[\''+c_in_df+'\'])') + return sMask + + # --------------------------------------------------------------------------------} # --- GUI to edit plugin and control the mask action # --------------------------------------------------------------------------------{ diff --git a/pydatview/plugins/data_radialConcat.py b/pydatview/plugins/data_radialConcat.py index 5a79277..4ec476b 100644 --- a/pydatview/plugins/data_radialConcat.py +++ b/pydatview/plugins/data_radialConcat.py @@ -31,11 +31,23 @@ def guiCallback(): #tableFunctionCancel= renameAeroFld, guiCallback=guiCallback, mainframe=mainframe, # shouldnt be needed - data = data + data = data , + imports = _imports, + data_var = _data_var, + code = _code ) return action +# --------------------------------------------------------------------------------} +# --- Main methods +# --------------------------------------------------------------------------------{ +_imports = ["from pydatview.fast.postpro import spanwiseConcat"] +_data_var=None, +_code = """dfs_new = [spanwiseConcat(df)] +names_new = ['concat'] +""" + def radialConcat(tab, data=None): from pydatview.fast.postpro import spanwiseConcat df_new = spanwiseConcat(tab.data) diff --git a/pydatview/plugins/data_radialavg.py b/pydatview/plugins/data_radialavg.py index 45b1cff..2326bd3 100644 --- a/pydatview/plugins/data_radialavg.py +++ b/pydatview/plugins/data_radialavg.py @@ -19,7 +19,7 @@ _DEFAULT_DICT={ 'active':False, 'avgMethod':'constantwindow', - 'avgParam': '2' + 'avgParam': 2 } # --------------------------------------------------------------------------------} @@ -57,7 +57,10 @@ def guiCallback(): guiEditorClass = RadialToolPanel, guiCallback = guiCallback, data = data, - mainframe=mainframe + mainframe=mainframe, + imports = _imports, + data_var = _data_var, + code = _code ) return action @@ -65,6 +68,17 @@ def guiCallback(): # --------------------------------------------------------------------------------} # --- Main methods # --------------------------------------------------------------------------------{ +_imports = ["from pydatview.plugins.data_radialavg import radialAvg"] +_imports += ["from pydatview.Tables import Table"] +_data_var='dataRadialAvg' +_code = """# NOTE: this code relies on pydatview for now. It will be adapted for welib/pyFAST later. +# For the most part, the underlying functions are: +# dfRad,_,dfDiam = fastfarm.spanwisePostProFF(filename, avgMethod=avgMethod,avgParam=avgParam, D=1, df=df) +# out = fastlib.spanwisePostPro(filename, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df=df) +tab=Table(data=df, filename=filename) +dfs_new, names_new = radialAvg(tab, dataRadialAvg) +""" + # add method def radialAvg(tab, data=None): """ NOTE: radial average may return several dataframe""" diff --git a/pydatview/scripter.py b/pydatview/scripter.py index 541bc96..2db59f3 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -5,10 +5,12 @@ _WELIB={ 'pydatview.io':'welib.weio', 'pydatview.tools':'welib.tools', + 'pydatview.fast.postpro':'welib.fast.postpro', } _PYFAST={ 'pydatview.io':'pyFAST.input_output', 'pydatview.tools.tictoc':'pyFAST.tools.tictoc', + 'pydatview.fast.postpro':'pyFAST.postpro', # I think... } _flavorReplaceDict={ @@ -39,9 +41,11 @@ def reset(self): self.opts = _defaultOpts self.import_statements = set() self.actions = OrderedDict() + self.adder_actions = OrderedDict() self.preplot_actions = OrderedDict() self.filenames = [] self.df_selections = [] # List of tuples (df_index, column_x, column_y) + self.df_formulae = [] # List of tuples (df_index, name, formula) self.dfs = [] self.plot_params = {} # Dictionary for plotting parameters @@ -58,29 +62,57 @@ def setOptions(self, **opts): def addImport(self, import_statement): self.import_statements.add(import_statement) - def addAction(self, action_name, code, imports=None, init_code=None): + def addAction(self, action_name, code, imports=None, code_init=None): for imp in imports: self.addImport(imp) - self.actions[action_name] = (init_code, code) + self.actions[action_name] = (code_init, code.strip()) - def addPreplotAction(self, action_name, code, imports=None, init_code=None): + def addAdderAction(self, action_name, code, imports=None, code_init=None): + for imp in imports: + self.addImport(imp) + self.adder_actions[action_name] = (code_init, code.strip()) + + def addPreplotAction(self, action_name, code, imports=None, code_init=None): if imports is not None: for imp in imports: self.addImport(imp) - self.preplot_actions[action_name] = (init_code, code) + self.preplot_actions[action_name] = (code_init, code.strip()) - def setFiles(self, filenames): - self.filenames = [f.replace('\\','/') for f in filenames] + def addFormula(self, df_index, name, formula): + self.df_formulae.append((df_index, name, formula)) def selectData(self, df_index, column_x, column_y): self.df_selections.append((df_index, column_x, column_y)) + def setFiles(self, filenames): + self.filenames = [f.replace('\\','/') for f in filenames] + + def setPlotParameters(self, params): self.plot_params = params + @property + def needIndex(self): + for df_index, column_x, column_y in self.df_selections: + if column_x=='Index' or column_y=='Index': + return True + + @property + def needFormulae(self): + return len(self.df_formulae)>0 + def generate(self, pltshow=True): + def forLoopOnDFs(): + if self.opts['dfsFlavor'] == 'dict': + script.append("for k, df in dfs.items():") + elif self.opts['dfsFlavor'] == 'list': + script.append("for df in dfs:") + + + script = [] + indent0= '' indent1= self.opts['indent'] indent2= indent1 + indent1 indent3= indent2 + indent1 @@ -104,22 +136,27 @@ def generate(self, pltshow=True): for filename in self.filenames: script.append(f"filenames += ['{filename}']") - # --- Init data actions + # --- Init data/preplot/adder actions if len(self.actions)>0: script.append("\n# --- Data for actions") for actionname, actioncode in self.actions.items(): - if actioncode[0] is not None: - if len(actioncode[0].strip())>0: - script.append("# Data for action {}".format(actionname)) - script.append(actioncode[0].strip()) + if actioncode[0] is not None and len(actioncode[0].strip())>0: + script.append("# Data for action {}".format(actionname)) + script.append(actioncode[0].strip()) if len(self.preplot_actions)>0: script.append("\n# --- Data for preplot actions") for actionname, actioncode in self.preplot_actions.items(): - if actioncode[0] is not None: - if len(actioncode[0].strip())>0: - script.append("# Data for preplot action {}".format(actionname)) - script.append(actioncode[0].strip()) + if actioncode[0] is not None and len(actioncode[0].strip())>0: + script.append("# Data for preplot action {}".format(actionname)) + script.append(actioncode[0].strip()) + + if len(self.adder_actions)>0: + script.append("\n# --- Applying actions that add new dataframes") + for actionname, actioncode in self.adder_actions.items(): + if actioncode[0] is not None and len(actioncode[0].strip())>0: + script.append("# Data for preplot action {}".format(actionname)) + script.append(actioncode[0].strip()) # --- List of Dataframes @@ -164,19 +201,82 @@ def generate(self, pltshow=True): script.append("else:") script.append(indent1 + f"df{iFile1} = dfs_or_df") - # --- Insert index for convenience - script.append("\n# --- Insert columns") - if self.opts['dfsFlavor'] == 'dict': - script.append("for k, df in dfs.items():") - script.append(indent1 + "df.insert(0, 'Index', np.arange(df.shape[0]))") + # --- Adder actions + if len(self.adder_actions)>0: + def addActionCode(actioname, actioncode, ind): + script.append(ind+ "# Apply action {}".format(actioname)) + lines = actioncode.split("\n") + indented_lines = [ind + line for line in lines] + script.append("\n".join(indented_lines)) - elif self.opts['dfsFlavor'] == 'list': - script.append("for df in dfs):") - script.append(indent1 + "df.insert(0, 'Index', np.arange(df.shape[0]))") + script.append("\n# --- Apply adder actions to dataframes") + script.append("dfs_add = [] ; names_add =[]") + if self.opts['dfsFlavor'] == 'dict': + script.append("for k, df in dfs.items():") + script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], indent1) + script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") + script.append("for name_new, df_new in zip(names_add, dfs_new):") + script.append(indent1+"if df_new is not None:") + script.append(indent2+"dfs[name_new] = df_new") + + elif self.opts['dfsFlavor'] == 'list': + script.append("for df in dfs):") + script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], indent1) + script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") + script.append("for name_new, df_new in zip(names_add, dfs_new):") + script.append(indent1+"if df_new is not None:") + script.append(indent2+"dfs+ = [df_new]") + + elif self.opts['dfsFlavor'] == 'enumeration': + nTabs = len(self.filenames) # Approximate + for iTab in range(nTabs): + script.append("filename = filenames[{}] # NOTE: this is approximate..".format(iTab)) + script.append('df = df{}'.format(iTab+1)) + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], '') + script.append("df{} = dfs_new[0] # NOTE: we only keep the first table here..".format(nTabs+iTab+1)) + nTabs += nTabs + + + # --- Insert index and formulae + if self.needIndex or self.needFormulae: + script.append("\n# --- Insert columns") + if self.opts['dfsFlavor'] in ['dict' or 'list']: + forLoopOnDFs() + if self.needIndex: + script.append(indent1 + "if not 'Index' in df.columns:") + script.append(indent2 + "df.insert(0, 'Index', np.arange(df.shape[0]))") + if self.needFormulae: + script.append(indent1 + "# Adding formulae: NOTE adjust to apply to a subset of dfs") + # TODO potentially sort on df_index and use an if statement on k + for df_index, name, formula in self.df_formulae: + script.append(indent1 + "try:") + script.append(indent2 + "df['{}'] = {}".format(name, formula)) + #df.insert(int(i+1), name, formula) + script.append(indent1 + "except:") + script.append(indent2 + "print('[WARN] Cannot add column {} to dataframe)".format(name)) + + elif self.opts['dfsFlavor'] == 'enumeration': + for iTab in range(nTabs): + if self.needIndex: + script.append("if not 'Index' in df.columns:") + script.append(indent1+"df{}.insert(0, 'Index', np.arange(df.shape[0]))".format(iTab+1)) + dfName = 'df{}'.format(iTab+1) + if self.needFormulae: + script.append("# Adding formulae: NOTE adjust to apply to a subset of dfs") + # TODO potentially sort on df_index and use an if statement on k + for df_index, name, formula in self.df_formulae: + formula = formula.replace('df', dfName) + script.append(indent0 + "try:") + script.append(indent1 + "{}['{}'] = {}".format(dfName, name, formula)) + #df.insert(int(i+1), name, formula) + script.append(indent0 + "except:") + script.append(indent1 + "print('[WARN] Cannot add column {} to dataframe')".format(name)) - elif self.opts['dfsFlavor'] == 'enumeration': - for iTab in range(len(self.filenames)): - script.append("df{}.insert(0, 'Index', np.arange(df.shape[0]))".format(iTab+1)) # --- Data Actions if len(self.actions)>0: @@ -204,15 +304,16 @@ def addActionCode(actioname, actioncode, ind): for iTab in range(len(self.filenames)): script.append('df = df{}'.format(iTab+1)) for actionname, actioncode in self.actions.items(): - addActionCode(actionname, actioncode[1], indent2) + addActionCode(actionname, actioncode[1], '') script.append('df{} = df'.format(iTab+1)) # --- Plot Styling script.append("\n# --- Generate the plot") # Plot Styling script.append("# Plot styling") - script.append("stys=['-','-',':','.-'] * len(dfs)") - script.append("cols=['r', 'g', 'b'] * len(dfs)") + # NOTE: dfs not known for enumerate + #script.append("stys=['-','-',':','.-'] * len(dfs)") + #script.append("cols=['r', 'g', 'b'] * len(dfs)") if self.opts['dfsFlavor'] == 'dict': script.append("tabNames = list(dfs.keys())") script.append("# Subplots") @@ -230,7 +331,6 @@ def addActionCode(actioname, actioncode, ind): elif self.opts['dfsFlavor'] == 'enumeration': script.append("x = df{}['{}']".format(df_index+1, column_x)) script.append("y = df{}['{}']".format(df_index+1, column_y)) - pass if len(self.preplot_actions)>0: script.append("# Applying preplot action for df{}".format(df_index+1)) for actionname, actioncode in self.preplot_actions.items(): @@ -259,7 +359,7 @@ def addActionCode(actioname, actioncode, ind): return "\n".join(script) - def run(self, method='subprocess', pltshow=True): + def run(self, method='subprocess', pltshow=True, scriptName='_pydatview_temp_script.py'): script = self.generate(pltshow=pltshow) import tempfile import subprocess @@ -271,7 +371,7 @@ def run(self, method='subprocess', pltshow=True): # --- Create a temporary file #temp_dir = tempfile.TemporaryDirectory() #script_file_path = os.path.join(temp_dir.name, "temp_script.py") - script_file_path ='pydatview_temp_script.py' + script_file_path = scriptName with open(script_file_path, "w") as script_file: script_file.write(script) @@ -288,9 +388,9 @@ def run(self, method='subprocess', pltshow=True): error.append("An error occurred: {e}") finally: # Clean up by deleting the temporary directory and its contents - #temp_dir.cleanup() - pass -# os.remove(script_file_path) + if len(errors)==0: + #temp_dir.cleanup() + os.remove(script_file_path) else: raise NotImplementedError() if len(errors)>0: @@ -301,21 +401,42 @@ def run(self, method='subprocess', pltshow=True): if __name__ == '__main__': # Example usage: + import os + scriptDir =os.path.dirname(__file__) scripter = PythonScripter() - scripter.setFiles(['../DampingExplodingExample.csv']) - - import_statements = ["import numpy as np", "import scipy.stats as stats"] -# action_code = """df = np.mean(x) -# p_value = stats.ttest_1samp(y, 0)[1] -# """ - action_code = """df = df""" - init_code="""data['Misc']=2 # That's a parameter +# scripter.setFiles([os.path.join(scriptDir, '../DampingExplodingExample.csv')]) + scripter.setFiles([os.path.join(scriptDir, '../_TestFiles/CT_1.10.outb')]) + + # --- Data Action + imports = ["import numpy as np", "import scipy.stats as stats"] + _code = """df = df""" + code_init="""data={}; data['Misc']=2 # That's a parameter data['Misc2']=3 # That's another parameter""" - scripter.addAction('filter', action_code, import_statements, init_code) - scripter.selectData(0, "Time", "TTDspFA") + scripter.addAction('filter', _code, imports, code_init) + + # --- PrePlot Action + _code="""x = x * 1\ny = y * 1""" + scripter.addPreplotAction('Scaling', _code) + + # --- Adder Action + imports = ["from pydatview.plugins.data_radialavg import radialAvg"] + imports += ["from pydatview.Tables import Table"] + _code = """ +tab=Table(data=df) +dfs_new, names_new = radialAvg(tab, dataRadial) +""" + code_init="""dataRadial={'avgMethod':'constantwindow', 'avgParam': 2}""" + scripter.addAdderAction('radialAvg', _code, imports, code_init) + + + # --- Formula + scripter.addFormula(0, name='Time2', formula="df['Time_[s]']*20") + - action_code="""x = x * 1\ny = y * 1""" - scripter.addPreplotAction('Scaling', action_code) + #scripter.selectData(0, "Time", "TTDspFA") + #scripter.selectData(0, "Time2", "TTDspFA") +# scripter.selectData(0, "Time2", "Wind1VelX_[m/s]") + scripter.selectData(1, "i/n_[-]", "B1Cl_[-]") plot_params = { 'figsize': (8, 6), @@ -332,7 +453,7 @@ def run(self, method='subprocess', pltshow=True): scripter.setOptions(libFlavor='pydatview', dfsFlavor='dict', oneTabPerFile=False) # scripter.setOptions(libFlavor='welib', dfsFlavor='list', oneTabPerFile=True) # scripter.setOptions(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=True) -# scripter.setOptions(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=False) + scripter.setOptions(libFlavor='welib', dfsFlavor='enumeration', oneTabPerFile=False) script = scripter.generate() print(script) scripter.run() diff --git a/tests/test_Tables.py b/tests/test_Tables.py index 2e4a474..5e5c0eb 100644 --- a/tests/test_Tables.py +++ b/tests/test_Tables.py @@ -67,17 +67,14 @@ def test_vstack(self): def test_resample(self): tab1=Table(data=pd.DataFrame(data={'BlSpn': [0,1,2],'Chord': [1,2,1]})) - tablist = TableList([tab1]) - #print(tablist) - #print(tab1.data) # Test Insertion of new values into table icol=1 opt = {'name': 'Insert', 'param': np.array([0.5, 1.5])} - dfs, names, errors = tablist.applyResampling(icol, opt, bAdd=True) - np.testing.assert_almost_equal(dfs[0]['Index'], [0,1,2,3,4]) - np.testing.assert_almost_equal(dfs[0]['BlSpn'], [0,0.5,1.0,1.5,2.0]) - np.testing.assert_almost_equal(dfs[0]['Chord'], [1,1.5,2.0,1.5,1.0]) + df, name_new = tab1.applyResampling(icol, opt, bAdd=True) + np.testing.assert_almost_equal(df['Index'], [0,1,2,3,4]) + np.testing.assert_almost_equal(df['BlSpn'], [0,0.5,1.0,1.5,2.0]) + np.testing.assert_almost_equal(df['Chord'], [1,1.5,2.0,1.5,1.0]) def test_load_files_misc_formats(self): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index d9d496f..767f621 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -96,11 +96,13 @@ def test_pipeline_script_data_plugins(self): # Add a Data action to the pipeline and verify that the script generation works # And that the script is actually runable! from pydatview.plugins.data_standardizeUnits import standardizeUnitsAction + from pydatview.plugins.data_mask import maskAction tablist = TableList() tablist.load_tables_from_files(filenames=[os.path.join(scriptDir,'../example_files/CSVComma.csv')]) pipeline = Pipeline() pipeline.append(standardizeUnitsAction(), apply=False) + pipeline.append(maskAction(data={'formattedMaskString':"df['ColA']>3"}), apply=False) pipeline.apply(tablist) # Apply will change unit ID= [[0,1,2]] s = pipeline.script(tablist, ID=ID, scripterOptions=None) From 18710ba908cc4fd00ba606edf98a695766f75864 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 25 Sep 2023 02:38:40 -0600 Subject: [PATCH 125/178] Scripter: plotStyles, subplots, labels, less lines, GUIOptions (#13) --- pydatview/GUIPlotPanel.py | 70 ++++++++------ pydatview/GUIScripter.py | 109 ++++++++++++++++++---- pydatview/main.py | 11 ++- pydatview/pipeline.py | 35 +++---- pydatview/plotdata.py | 10 ++ pydatview/scripter.py | 191 +++++++++++++++++++++++++++++--------- 6 files changed, 308 insertions(+), 118 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 25d0ed4..5614629 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -40,7 +40,8 @@ import gc from pydatview.common import * # unique, CHAR, pretty_date -from pydatview.plotdata import PlotData, compareMultiplePD +from pydatview.plotdata import PlotData, compareMultiplePD +from pydatview.plotdata import PDL_xlabel from pydatview.GUICommon import * from pydatview.GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool from pydatview.GUIMeasure import GUIMeasure @@ -1047,23 +1048,17 @@ def set_axes_lim(self, PDs, axis): except: pass - def plot_all(self, autoscale=True): - """ - autoscale: if True, find the limits based on the data. - Otherwise, the limits are restored using: - self._restore_limits and the variables: self.xlim_prev, self.ylim_prev - """ - self.multiCursors=[] - - axes=self.fig.axes - PD=self.plotData - + def getPlotOptions(self, PD=None): # --- PlotStyles plotStyle = self.esthPanel._GUI2Data() # --- Plot options - bStep = self.cbStepPlot.IsChecked() plot_options = dict() + plot_options['step'] = self.cbStepPlot.IsChecked() + plot_options['logX'] = self.cbLogX.IsChecked() + plot_options['logY'] = self.cbLogY.IsChecked() + plot_options['grid'] = self.cbGrid.IsChecked() + plot_options['lw']=plotStyle['LineWidth'] plot_options['ms']=plotStyle['MarkerSize'] if self.cbCurveType.Value=='Plain': @@ -1083,17 +1078,35 @@ def plot_all(self, autoscale=True): # But at that stage, if the user really want this, then we can implement an option to set styles per plot. Not high priority. raise Exception('Not implemented') - - # --- Font options font_options = dict() font_options_legd = dict() font_options['size'] = plotStyle['Font'] font_options_legd['fontsize'] = plotStyle['LegendFont'] - needChineseFont = any([pd.needChineseFont for pd in PD]) - if needChineseFont and self.specialFont is not None: - font_options['fontproperties']= self.specialFont - font_options_legd['prop'] = self.specialFont + if PD is not None: + needChineseFont = any([pd.needChineseFont for pd in PD]) + if needChineseFont and self.specialFont is not None: + font_options['fontproperties']= self.specialFont + font_options_legd['prop'] = self.specialFont + + return plotStyle, plot_options, font_options, font_options_legd + + + + def plot_all(self, autoscale=True): + """ + autoscale: if True, find the limits based on the data. + Otherwise, the limits are restored using: + self._restore_limits and the variables: self.xlim_prev, self.ylim_prev + """ + self.multiCursors=[] + + axes=self.fig.axes + PD=self.plotData + + # --- PlotStyles + plotStyle, plot_options, font_options, font_options_legd = self.getPlotOptions() + # --- Loop on axes. Either use ax.iPD to chose the plot data, or rely on plotmatrix for axis_idx, ax_left in enumerate(axes): @@ -1112,17 +1125,17 @@ def plot_all(self, autoscale=True): pm = self.infoPanel.getPlotMatrix(PD, self.cbSub.IsChecked()) else: pm = None - __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, bStep, plot_options) - ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, bStep, plot_options) + __, bAllNegLeft = self.plotSignals(ax_left, axis_idx, PD, pm, 1, plot_options) + ax_right, bAllNegRight = self.plotSignals(ax_left, axis_idx, PD, pm, 2, plot_options) # Log Axes - if self.cbLogX.IsChecked(): + if plot_options['logX']: try: ax_left.set_xscale("log", nonpositive='clip') # latest except: ax_left.set_xscale("log", nonposx='clip') # legacy - if self.cbLogY.IsChecked(): + if plot_options['logY']: if bAllNegLeft is False: try: ax_left.set_yscale("log", nonpositive='clip') # latest @@ -1152,7 +1165,7 @@ def plot_all(self, autoscale=True): except: pass - ax_left.grid(self.cbGrid.IsChecked()) + ax_left.grid(plot_options['grid']) if ax_right is not None: l = ax_left.get_ylim() l2 = ax_right.get_ylim() @@ -1162,7 +1175,7 @@ def plot_all(self, autoscale=True): if len(ax_left.lines) == 0: ax_left.set_yticks(ax_right.get_yticks()) ax_left.yaxis.set_visible(False) - ax_right.grid(self.cbGrid.IsChecked()) + ax_right.grid(plot_options['grid']) # Special Grids if self.pltTypePanel.cbCompare.GetValue(): @@ -1238,7 +1251,8 @@ def plot_all(self, autoscale=True): self.lbDeltaY.SetLabel('') # --- xlabel - axes[-1].set_xlabel(PD[axes[-1].iPD[0]].sx, **font_options) + #axes[-1].set_xlabel(PD[axes[-1].iPD[0]].sx, **font_options) + axes[-1].set_xlabel(PDL_xlabel(PD), **font_options) #print('sy :',[pd.sy for pd in PD]) #print('syl:',[pd.syl for pd in PD]) @@ -1251,7 +1265,7 @@ def plot_all(self, autoscale=True): bXHair = self.cbXHair.GetValue() self.multiCursors = MyMultiCursor(self.canvas, tuple(self.fig.axes), useblit=True, horizOn=bXHair, vertOn=bXHair, color='gray', linewidth=0.5, linestyle=':') - def plotSignals(self, ax, axis_idx, PD, pm, left_right, is_step, opts): + def plotSignals(self, ax, axis_idx, PD, pm, left_right, opts): axis = None bAllNeg = True if pm is None: @@ -1282,7 +1296,7 @@ def plotSignals(self, ax, axis_idx, PD, pm, left_right, is_step, opts): # TODO allow PlotData to override for "per plot" options in the future marker = opts['Markers'][np.mod(iPlot,len(opts['Markers']))] ls = opts['LineStyles'][np.mod(iPlot,len(opts['LineStyles']))] - if is_step: + if opts['step']: plot = axis.step else: plot = axis.plot diff --git a/pydatview/GUIScripter.py b/pydatview/GUIScripter.py index 7801420..2ddc709 100644 --- a/pydatview/GUIScripter.py +++ b/pydatview/GUIScripter.py @@ -1,4 +1,5 @@ import wx +import wx.stc as stc class GUIScripterFrame(wx.Frame): def __init__(self, parent, mainframe, pipeLike, title): @@ -10,26 +11,46 @@ def __init__(self, parent, mainframe, pipeLike, title): # --- GUI self.panel = wx.Panel(self) - self.text_ctrl = wx.TextCtrl(self.panel, style=wx.TE_MULTILINE) - self.btGen = wx.Button(self.panel, label="Regenerate") + #self.text_ctrl = wx.TextCtrl(self.panel, style=wx.TE_MULTILINE) + self.text_ctrl = stc.StyledTextCtrl(self.panel, style=wx.TE_MULTILINE) + self.setup_syntax_highlighting() + + + self.btGen = wx.Button(self.panel, label="Update") self.btRun = wx.Button(self.panel, label="Run Script") self.btSave = wx.Button(self.panel, label="Save to File") + + txtLib = wx.StaticText(self.panel, -1, 'Library:') libflavors = ["welib", "pydatview", "pyFAST"] - self.cbFlavors = wx.Choice(self.panel, choices=libflavors) - self.cbFlavors.SetSelection(1) - mono_font = wx.Font(10, wx.FONTFAMILY_TELETYPE, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) - self.text_ctrl.SetFont(mono_font) + self.cbLib = wx.Choice(self.panel, choices=libflavors) + self.cbLib.SetSelection(1) + + txtDFS= wx.StaticText(self.panel, -1, 'DF storage:') + DFSflavors = ["dict", "list", "enumeration"] + self.cbDFS = wx.Choice(self.panel, choices=DFSflavors) + self.cbDFS.SetSelection(0) + + txtCom= wx.StaticText(self.panel, -1, 'Comment level:') + ComLevels = ["1", "2"] + self.cbCom = wx.Choice(self.panel, choices=ComLevels) + self.cbCom.SetSelection(0) + # --- Layout vbox = wx.BoxSizer(wx.VERTICAL) hbox = wx.BoxSizer(wx.HORIZONTAL) - vbox.Add(self.text_ctrl, proportion=1, flag=wx.EXPAND | wx.ALL, border=10) + vbox.Add(self.text_ctrl, proportion=1, flag=wx.EXPAND | wx.ALL, border=2) - hbox.Add(self.cbFlavors, proportion=1, flag=wx.EXPAND | wx.ALL, border=10) - hbox.Add(self.btGen , flag=wx.ALL, border=10) - hbox.Add(self.btRun , flag=wx.ALL, border=10) - hbox.Add(self.btSave , flag=wx.ALL, border=10) + hbox.Add( txtLib, flag=wx.LEFT|wx.ALIGN_CENTER_VERTICAL, border=5) + hbox.Add(self.cbLib, proportion=1, flag=wx.EXPAND|wx.ALL, border=5) + hbox.Add( txtDFS, flag=wx.LEFT|wx.ALIGN_CENTER_VERTICAL, border=5) + hbox.Add(self.cbDFS, proportion=1, flag=wx.EXPAND|wx.ALL, border=5) + hbox.Add( txtCom, flag=wx.LEFT|wx.ALIGN_CENTER_VERTICAL, border=5) + hbox.Add(self.cbCom, flag=wx.EXPAND|wx.ALL, border=5) + hbox.Add(self.btGen , flag=wx.ALL|wx.ALIGN_CENTER_VERTICAL, border=5) + hbox.Add(self.btRun , flag=wx.ALL|wx.ALIGN_CENTER_VERTICAL, border=5) + hbox.Add(self.btSave , flag=wx.ALL|wx.ALIGN_CENTER_VERTICAL, border=5) vbox.Add(hbox, flag=wx.EXPAND) self.panel.SetSizerAndFit(vbox) @@ -38,31 +59,59 @@ def __init__(self, parent, mainframe, pipeLike, title): self.btSave.Bind(wx.EVT_BUTTON, self.onSave) self.btRun.Bind(wx.EVT_BUTTON, self.onRun) self.btGen.Bind(wx.EVT_BUTTON, self.generateScript) - self.cbFlavors.Bind(wx.EVT_CHOICE, self.onFlavorChange) + self.cbLib.Bind(wx.EVT_CHOICE, self.generateScript) + self.cbDFS.Bind(wx.EVT_CHOICE, self.generateScript) + self.cbCom.Bind(wx.EVT_CHOICE, self.generateScript) self.generateScript() def _GUI2Data(self, *args, **kwargs): # GUI2Data data={} - data['libFlavor'] = self.cbFlavors.GetStringSelection() + data['libFlavor'] = self.cbLib.GetStringSelection() + data['dfsFlavor'] = self.cbDFS.GetStringSelection() + data['verboseCommentLevel'] = int(self.cbCom.GetStringSelection()) return data def generateScript(self, *args, **kwargs): - data = self._GUI2Data() + # --- GUI 2 data + scripterOptions = self._GUI2Data() + + # --- Mainframe GUI 2 data try: ID,SameCol,selMode=self.mainframe.selPanel.getPlotDataSelection() except: - ID is None - s = self.pipeline.script(self.mainframe.tabList, data, ID) - self.text_ctrl.SetValue(s) + ID = None + try: + fig = self.mainframe.plotPanel.canvas.figure + gs = fig.axes[0].get_gridspec() + x_labels = [] + y_labels = [] + IPD = [] + hasLegend = [] + for i, ax in enumerate(fig.axes): + x_labels.append(ax.get_xlabel()) + y_labels.append(ax.get_ylabel()) + IPD.append(ax.iPD) + hasLegend.append(ax.get_legend() is not None) + subPlots={'i':gs.nrows, 'j':gs.ncols, 'x_labels':x_labels, 'y_labels':y_labels, 'IPD':IPD, 'hasLegend':hasLegend} + except: + print('[WARN] GUIScripter - No Subplot Data') + subPlots = None + try: + plotStyle, plot_options, font_options, font_options_legd = self.mainframe.plotPanel.getPlotOptions() + plotStyle.update(plot_options) + except: + plotStyle=None + print('[WARN] GUIScripter - No Plot Options') - def onFlavorChange(self, event): - self.generateScript() + # --- Use Pipeline on tablist to generate the script + s = self.pipeline.script(self.mainframe.tabList, scripterOptions, ID, subPlots, plotStyle) + self.text_ctrl.SetValue(s) def onRun(self, event): """ Run the script in user terminal """ - self.pipeline.scripter.run() + self.pipeline.scripter.run(script=self.text_ctrl.GetValue()) def onSave(self, event): """ Save script to file """ @@ -77,3 +126,23 @@ def onSave(self, event): dialog.Destroy() + def setup_syntax_highlighting(self): + # --- Basic + mono_font = wx.Font(10, wx.FONTFAMILY_TELETYPE, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) + #self.text_ctrl.SetFont(mono_font) + + # --- Advanced + self.text_ctrl.StyleSetForeground(stc.STC_P_COMMENTLINE, wx.Colour(0, 128, 0)) # Comments (green) + self.text_ctrl.StyleSetForeground(stc.STC_P_NUMBER, wx.Colour(123, 0, 0)) # Numbers (red) + self.text_ctrl.StyleSetForeground(stc.STC_P_STRING , wx.Colour(165, 32, 247)) # Strings + self.text_ctrl.StyleSetForeground(stc.STC_P_CHARACTER, wx.Colour(165, 32, 247)) # Characters + self.text_ctrl.StyleSetForeground(stc.STC_P_WORD, wx.Colour(0, 0, 128)) # Keywords (dark blue) + self.text_ctrl.StyleSetBold(stc.STC_P_WORD, True) # Make keywords bold + self.text_ctrl.SetLexer(stc.STC_LEX_PYTHON) # Set the lexer for Python + self.text_ctrl.StyleSetForeground(stc.STC_P_DEFAULT, wx.Colour(0, 0, 0)) # Default text color (black) + self.text_ctrl.StyleSetBackground(stc.STC_P_DEFAULT, wx.Colour(255, 255, 255)) # Default background color (white) + self.text_ctrl.StyleSetFont(stc.STC_STYLE_DEFAULT, mono_font) + self.text_ctrl.SetUseHorizontalScrollBar(False) + # Remove the left margin (line number margin) + self.text_ctrl.SetMarginWidth(1, 0) # Set the width of margin 1 (line number margin) to 0 + diff --git a/pydatview/main.py b/pydatview/main.py index 9855162..7e169f7 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -491,8 +491,8 @@ def exportTab(self, iTab): def onShowTool(self, event=None, toolName=''): """ - Show tool - tool in 'Outlier', 'Filter', 'LogDec','FASTRadialAverage', 'Mask', 'CurveFitting' + Show tool See pydatview.plugins.__init__.py + tool in 'Outlier', 'Filter', 'LogDec','Radial Average', 'Mask', 'CurveFitting' """ if not hasattr(self,'plotPanel'): Error(self,'Plot some data first') @@ -505,6 +505,8 @@ def onDataPlugin(self, event=None, toolName=''): - simple plugins are directly exectued - plugins that are panels are sent over to plotPanel to show them TODO merge with onShowTool + + See pydatview.plugins.__init__.py for list of toolNames """ if not hasattr(self,'plotPanel'): Error(self,'Plot some data first') @@ -971,7 +973,10 @@ def showApp(firstArg=None, dataframes=None, filenames=[], names=None): elif len(filenames)>0: frame.load_files(filenames, fileformats=None, bPlot=True) -# frame.onScript() + #frame.onShowTool(toolName='Radial Average') + #frame.onDataPlugin(toolName='Radial Average') + #frame.onDataPlugin(toolName='Resample') + #frame.onScript() app.MainLoop() diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index df2591a..ec9e9ce 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -301,19 +301,21 @@ def apply(self, tabList, force=False, applyToAll=False): # self.collectErrors() - def script(self, tabList, scripterOptions=None, ID=None): + def script(self, tabList, scripterOptions=None, ID=None, subPlots=None, plotStyle=None): from pydatview.scripter import PythonScripter if scripterOptions is None: scripterOptions={} # --- Files and number of tables - fileNames = tabList.filenames - fileNamesTrue = [t for t in tabList.filenames if len(t)>0] + fileNames = np.unique(tabList.filenames) + fileNamesTrue = [t for t in fileNames if len(t)>0] if len(fileNames)==len(fileNamesTrue): # No tables were added if len(fileNamesTrue) == len(tabList): # Each file returned one table only scripterOptions['oneTabPerFile'] = True + else: + scripterOptions['oneTabPerFile'] = False scripter = PythonScripter() scripter.setFiles(fileNamesTrue) @@ -329,7 +331,6 @@ def script(self, tabList, scripterOptions=None, ID=None): else: scripter.addAdderAction(action.name, action_code, imports, code_init) - # --- Data Actions for action in self.actionsData: if not isinstance(action, AdderAction): @@ -365,26 +366,14 @@ def script(self, tabList, scripterOptions=None, ID=None): kx = tabList[it].columns[ix] ky = tabList[it].columns[iy] scripter.selectData(it, kx, ky) - # Initialize each plotdata based on selected table and selected id channels - #pd.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeLike) - #PD.id = i - #PD.sx = idx[3].replace('_',' ') # x label - #PD.sy = idx[4].replace('_',' ') # y label - #PD.syl = '' # y label for legend - #PD.st = idx[5] # table label - #PD.x, PD.xIsString, PD.xIsDate,_ = tabs[PD.it].getColumn(PD.ix) # actual x data, with info - #PD.y, PD.yIsString, PD.yIsDate,c = tabs[PD.it].getColumn(PD.iy) # actual y data, with info - #PD.c =c # raw values, used by PDF - -# plot_params = { -# 'figsize': (8, 6), -# 'xlabel': 'X-Axis', -# 'ylabel': 'Y-Axis', -# 'title': 'Sample Plot', -# 'label': 'Data Series', -# } -# scripter.set_plot_parameters(plot_params) + if subPlots is not None: + scripter.setSubPlots(**subPlots) + + if plotStyle is not None: + scripter.setPlotParameters(**plotStyle) + + # --- Do generate the script and store instance script = scripter.generate() self.scripter = scripter return script diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 1772fc6..7dce907 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -5,6 +5,16 @@ from pydatview.common import unique, pretty_num, pretty_time, pretty_date import matplotlib.dates as mdates +# --------------------------------------------------------------------------------} +# --- PlotDataList functions +# --------------------------------------------------------------------------------{ +def PDL_xlabel(PDL): + #PD[axes[-1].iPD[0]].sx, **font_options) + return PDL[-1].sx + +# --------------------------------------------------------------------------------} +# --- PlotData +# --------------------------------------------------------------------------------{ class PlotData(): """ Class for plot data diff --git a/pydatview/scripter.py b/pydatview/scripter.py index 2db59f3..f17b45c 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -24,7 +24,16 @@ 'dfsFlavor':'dict', 'oneTabPerFile':False, 'indent':' ', + 'verboseCommentLevel':1, } +_defaultPlotStyle={ + 'grid':False, 'logX':False, 'logY':False, + 'LineStyles':['-','--','-.',':'], + 'Markers':[''], + 'LegendPosition':'best', + 'LineWidth':1.5, + 'ms':2.0, +} class PythonScripter: def __init__(self, **opts): @@ -47,7 +56,8 @@ def reset(self): self.df_selections = [] # List of tuples (df_index, column_x, column_y) self.df_formulae = [] # List of tuples (df_index, name, formula) self.dfs = [] - self.plot_params = {} # Dictionary for plotting parameters + self.subPlots = {'i':1, 'j':1, 'x_labels':['x'], 'y_labels':['y'], 'IPD':None, 'hasLegend':[True]} + self.plotStyle = _defaultPlotStyle def setOptions(self, **opts): for k,v in opts.items(): @@ -87,9 +97,11 @@ def selectData(self, df_index, column_x, column_y): def setFiles(self, filenames): self.filenames = [f.replace('\\','/') for f in filenames] + def setSubPlots(self, **kwargs): + self.subPlots=kwargs - def setPlotParameters(self, params): - self.plot_params = params + def setPlotParameters(self, **params): + self.plotStyle = params @property def needIndex(self): @@ -112,10 +124,15 @@ def forLoopOnDFs(): script = [] + + verboseCommentLevel = self.opts['verboseCommentLevel'] indent0= '' indent1= self.opts['indent'] indent2= indent1 + indent1 indent3= indent2 + indent1 + plotStyle = self.plotStyle + + # --- Disclaimer script.append('""" Script generated by pyDatView - The script will likely need to be adapted."""') @@ -141,21 +158,24 @@ def forLoopOnDFs(): script.append("\n# --- Data for actions") for actionname, actioncode in self.actions.items(): if actioncode[0] is not None and len(actioncode[0].strip())>0: - script.append("# Data for action {}".format(actionname)) + if verboseCommentLevel>=2: + script.append("# Data for action {}".format(actionname)) script.append(actioncode[0].strip()) if len(self.preplot_actions)>0: script.append("\n# --- Data for preplot actions") for actionname, actioncode in self.preplot_actions.items(): if actioncode[0] is not None and len(actioncode[0].strip())>0: - script.append("# Data for preplot action {}".format(actionname)) + if verboseCommentLevel>=2: + script.append("# Data for preplot action {}".format(actionname)) script.append(actioncode[0].strip()) if len(self.adder_actions)>0: - script.append("\n# --- Applying actions that add new dataframes") + script.append("\n# --- Data for actions that add new dataframes") for actionname, actioncode in self.adder_actions.items(): if actioncode[0] is not None and len(actioncode[0].strip())>0: - script.append("# Data for preplot action {}".format(actionname)) + if verboseCommentLevel>=2: + script.append("# Data for adder action {}".format(actionname)) script.append(actioncode[0].strip()) @@ -194,7 +214,8 @@ def forLoopOnDFs(): if self.opts['oneTabPerFile']: script.append(f"df{iFile1} = weio.read(filenames[{iFile}]).toDataFrame()") else: - script.append("# NOTE: we need a different action if the file contains multiple dataframes") + if verboseCommentLevel>=1: + script.append("# NOTE: we need a different action if the file contains multiple dataframes") script.append(f"dfs_or_df = weio.read('{filename}').toDataFrame()") script.append("if isinstance(dfs_or_df, dict):") script.append(indent1 + f"df{iFile1} = dfs_or_df.items()[0][1] # NOTE: user will need to adapt this.") @@ -308,59 +329,141 @@ def addActionCode(actioname, actioncode, ind): script.append('df{} = df'.format(iTab+1)) # --- Plot Styling - script.append("\n# --- Generate the plot") + script.append("\n# --- Plot") # Plot Styling - script.append("# Plot styling") + if verboseCommentLevel>=2: + script.append("# Plot styling") # NOTE: dfs not known for enumerate - #script.append("stys=['-','-',':','.-'] * len(dfs)") - #script.append("cols=['r', 'g', 'b'] * len(dfs)") + script.append("lw = {} ".format(plotStyle['LineWidth'])) + script.append("stys = {} * 100".format(plotStyle['LineStyles'])) + if len(plotStyle['Markers'])>1: + script.append("mrks = {} * 100".format(plotStyle['Markers'])) + script.append("ms = {} ".format(plotStyle['MarkerSize'])) + #script.append("cols=['r', 'g', 'b'] * 100") if self.opts['dfsFlavor'] == 'dict': script.append("tabNames = list(dfs.keys())") - script.append("# Subplots") - script.append("fig,ax = plt.subplots(1, 1, sharey=False, figsize=(6.4,4.8))") - script.append("fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20)") + if verboseCommentLevel>=2: + script.append("# Subplots") - for df_index, column_x, column_y in self.df_selections: - script.append("\n# Selecting data for df{}".format(df_index+1)) - if self.opts['dfsFlavor'] == 'dict': - script.append("x = dfs[tabNames[{}]]['{}']".format(df_index, column_x)) - script.append("y = dfs[tabNames[{}]]['{}']".format(df_index, column_y)) - elif self.opts['dfsFlavor'] == 'list': - script.append("x = dfs[{}]['{}']".format(df_index, column_x)) - script.append("y = dfs[{}]['{}']".format(df_index, column_y)) + if self.subPlots['i']==1 and self.subPlots['j']==1: + noSubplot=True + else: + noSubplot=False + + if noSubplot: + script.append("fig,ax = plt.subplots({}, {}, sharex=True, figsize=(6.4,4.8))".format(self.subPlots['i'],self.subPlots['j'])) + script.append("fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20)") + else: + script.append("fig,axes = plt.subplots({}, {}, sharex=True, figsize=(6.4,4.8))".format(self.subPlots['i'],self.subPlots['j'])) + script.append("fig.subplots_adjust(left=0.12, right=0.95, top=0.95, bottom=0.11, hspace=0.20, wspace=0.20)") + script.append("axes = np.reshape(axes, ({},{}))".format(self.subPlots['i'],self.subPlots['j'])) + + def getAxesString(i,j): + if noSubplot: + return 'ax' + else: + return 'axes[{},{}]'.format(i, j) + + sAxes='' + sDF='' + for iPD, (df_index, column_x, column_y) in enumerate(self.df_selections): + # --- Find relationship between axis and plotdata + iaxSel=0 + jaxSel=0 # TODO + iPDPos = 0 + if self.subPlots['IPD'] is not None: + for iax, axIPD in enumerate(self.subPlots['IPD']): + if iPD in axIPD: + iaxSel= iax + iPDPos = axIPD.index(iPD) + break + + if not noSubplot: + sAxes_new = 'ax = '+getAxesString(iaxSel, jaxSel) + if sAxes_new != sAxes: + sAxes=sAxes_new + script.append(sAxes_new) + if verboseCommentLevel>=2: + script.append("\n# Selecting data for df{}".format(df_index+1)) + if len(self.preplot_actions)>0: + sPlotXY='x, y, ' + if self.opts['dfsFlavor'] in ['dict', 'list']: + if self.opts['dfsFlavor'] == 'dict': + sDF_new = "dfs[tabNames[{}]]".format(df_index) + else: + sDF_new = "dfs[{}]".format(df_index) + if sDF_new != sDF: + sDF = sDF_new + script.append("df = "+sDF) + if len(self.preplot_actions)>0: + script.append("x = df['{}']".format(column_x)) + script.append("y = df['{}']".format(column_y)) + else: + sPlotXY ="df['{}'], df['{}'], ".format(column_x, column_y) elif self.opts['dfsFlavor'] == 'enumeration': - script.append("x = df{}['{}']".format(df_index+1, column_x)) - script.append("y = df{}['{}']".format(df_index+1, column_y)) + if len(self.preplot_actions)>0: + script.append("x = df{}['{}']".format(df_index+1, column_x)) + script.append("y = df{}['{}']".format(df_index+1, column_y)) + else: + sPlotXY ="df{}['{}'], df{}['{}'], ".format(df_index+1, column_x, df_index+1, column_y) + if len(self.preplot_actions)>0: - script.append("# Applying preplot action for df{}".format(df_index+1)) + if verboseCommentLevel>=2: + script.append("# Applying preplot action for df{}".format(df_index+1)) for actionname, actioncode in self.preplot_actions.items(): script.append(actioncode[1]) - script.append("# Plotting for df{}".format(df_index+1)) - script.append("ax.plot(x, y, 'o', label='')") - - - script.append("ax.set_xlabel('')") - script.append("ax.set_ylabel('')") + # --- Plot + if verboseCommentLevel>=2: + script.append("# Plotting for df{}".format(df_index+1)) + label =column_y.replace('_',' ') # TODO for table comparison + plotLine = "ax.plot(" + plotLine += sPlotXY + #if len(plotStyle['LineStyles'])>0: + plotLine += "ls=stys[{}], ".format(iPDPos) + plotLine += "lw=lw, " + if len(plotStyle['Markers'])==1 and len(plotStyle['Markers'][0])>0: + plotLine += "ms=ms, marker={}, ".format(plotStyle['Markers'][0]) + elif len(plotStyle['Markers'])>1: + plotLine += "ms=ms, marker=mrks[{}], ".format(iPDPos) + plotLine += "label='{}')".format(label) + script.append(plotLine) + + k=0 + for i in range(self.subPlots['i']): + for j in range(self.subPlots['j']): + xlab = self.subPlots['x_labels'][k] + ylab = self.subPlots['y_labels'][k] + if len(xlab.strip())>0: + script.append(getAxesString(i,j)+".set_xlabel('{}')".format(xlab)) + if len(ylab.strip())>0: + script.append(getAxesString(i,j)+".set_ylabel('{}')".format(ylab)) + if self.subPlots['hasLegend'][k]: + script.append(getAxesString(i,j)+".legend(loc='{}')".format(plotStyle['LegendPosition'].lower())) + k+=1 + + # --- Parameters common to all axes + if noSubplot: + indent = indent0 + else: + script.append(indent0 + "for ax in axes.flatten():") + indent = indent1 + if plotStyle['grid']: + script.append(indent + "ax.grid()") + if plotStyle['logX']: + script.append(indent + "ax.set_xscale('log', nonpositive='clip')") + if plotStyle['logY']: + script.append(indent + "ax.set_yscale('log', nonpositive='clip')") #script.append("ax.legend()") if pltshow: script.append("plt.show()") -# plot_code = f""" -# plt.figure(figsize=({self.plot_params['figsize']})) -# plt.xlabel('{self.plot_params['xlabel']}') -# plt.ylabel('{self.plot_params['ylabel']}') -# plt.title('{self.plot_params['title']}') -# plt.plot(x, y, label='{self.plot_params['label']}') -# plt.legend() -# plt.show() -# """ -# script.append(plot_code) return "\n".join(script) - def run(self, method='subprocess', pltshow=True, scriptName='_pydatview_temp_script.py'): - script = self.generate(pltshow=pltshow) + def run(self, script=None, method='subprocess', pltshow=True, scriptName='_pydatview_temp_script.py'): + if script is None: + script = self.generate(pltshow=pltshow) import tempfile import subprocess import os From b7acbb0d4fe35ecc87a973fd22d9bbfd5277c355 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 25 Sep 2023 22:58:17 -0600 Subject: [PATCH 126/178] Scripter: PDF, FFT, MinMax implemented (#13) --- pydatview/GUIPlotPanel.py | 51 ++++++---- pydatview/GUIScripter.py | 26 ++++- pydatview/main.py | 2 +- pydatview/pipeline.py | 6 +- pydatview/scripter.py | 185 ++++++++++++++++++++++++++---------- pydatview/tools/spectral.py | 1 + 6 files changed, 198 insertions(+), 73 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 5614629..f5b2706 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -73,6 +73,11 @@ def __init__(self, parent): def onPDFOptionChange(self,event=None): self.parent.load_and_draw(); # DATA HAS CHANGED + def _GUI2Data(self): + data = {'nBins': self.scBins.GetValue(), + 'smooth': self.cbSmooth.GetValue()} + return data + class MinMaxPanel(wx.Panel): def __init__(self, parent): super(MinMaxPanel,self).__init__(parent) @@ -91,6 +96,11 @@ def __init__(self, parent): def onMinMaxChange(self,event=None): self.parent.load_and_draw(); # DATA HAS CHANGED + def _GUI2Data(self): + data={'yScale':self.cbyMinMax.IsChecked(), + 'xScale':self.cbxMinMax.IsChecked()} + return data + class CompCtrlPanel(wx.Panel): def __init__(self, parent): super(CompCtrlPanel,self).__init__(parent) @@ -107,6 +117,10 @@ def __init__(self, parent): def onTypeChange(self,e): self.parent.load_and_draw(); # DATA HAS CHANGED + def _GUI2Data(self): + data = {'nBins': self.scBins.GetValue(), + 'bSmooth':self.cbSmooth.GetValue()} + return data class SpectralCtrlPanel(wx.Panel): def __init__(self, parent): @@ -198,6 +212,17 @@ def updateP2(self,P2): self.lbWinLength.SetLabel("({})".format(2**P2)) + def _GUI2Data(self): + data = {} + data['xType'] = self.cbTypeX.GetStringSelection() + data['yType'] = self.cbType.GetStringSelection() + data['avgMethod'] = self.cbAveraging.GetStringSelection() + data['avgWindow'] = self.cbAveragingMethod.GetStringSelection() + data['bDetrend'] = self.cbDetrend.IsChecked() + data['nExp'] = self.scP2.GetValue() + data['nPerDecade'] = self.scP2.GetValue() + return data + class PlotTypePanel(wx.Panel): @@ -845,36 +870,28 @@ def showToolPanel(self, panelClass=None, panel=None, action=None): def setPD_PDF(self,PD,c): """ Convert plot data to PDF data based on GUI options""" # ---PDF - nBins = self.pdfPanel.scBins.GetValue() - bSmooth = self.pdfPanel.cbSmooth.GetValue() - nBins_out= PD.toPDF(nBins,bSmooth) - if nBins_out!=nBins: - self.pdfPanel.scBins.SetValue(nBins) + data = self.pdfPanel._GUI2Data() + nBins_out= PD.toPDF(**data) + if nBins_out != data['nBins']: + self.pdfPanel.scBins.SetValue(data['nBins']) def setPD_MinMax(self,PD): """ Convert plot data to MinMax data based on GUI options""" - yScale=self.mmxPanel.cbyMinMax.IsChecked() - xScale=self.mmxPanel.cbxMinMax.IsChecked() + data = self.mmxPanel._GUI2Data() try: - PD.toMinMax(xScale,yScale) + PD.toMinMax(**data) except Exception as e: self.mmxPanel.cbxMinMax.SetValue(False) raise e # Used to be Warn def setPD_FFT(self,pd): """ Convert plot data to FFT data based on GUI options""" - yType = self.spcPanel.cbType.GetStringSelection() - xType = self.spcPanel.cbTypeX.GetStringSelection() - avgMethod = self.spcPanel.cbAveraging.GetStringSelection() - avgWindow = self.spcPanel.cbAveragingMethod.GetStringSelection() - bDetrend = self.spcPanel.cbDetrend.IsChecked() - nExp = self.spcPanel.scP2.GetValue() - nPerDecade = self.spcPanel.scP2.GetValue() + data = self.spcPanel._GUI2Data() # Convert plotdata to FFT data try: - Info = pd.toFFT(yType=yType, xType=xType, avgMethod=avgMethod, avgWindow=avgWindow, bDetrend=bDetrend, nExp=nExp, nPerDecade=nPerDecade) + Info = pd.toFFT(**data) # Trigger - if hasattr(Info,'nExp') and Info.nExp!=nExp: + if hasattr(Info,'nExp') and Info.nExp!=data['nExp']: self.spcPanel.scP2.SetValue(Info.nExp) self.spcPanel.updateP2(Info.nExp) except Exception as e: diff --git a/pydatview/GUIScripter.py b/pydatview/GUIScripter.py index 2ddc709..6282941 100644 --- a/pydatview/GUIScripter.py +++ b/pydatview/GUIScripter.py @@ -96,17 +96,37 @@ def generateScript(self, *args, **kwargs): hasLegend.append(ax.get_legend() is not None) subPlots={'i':gs.nrows, 'j':gs.ncols, 'x_labels':x_labels, 'y_labels':y_labels, 'IPD':IPD, 'hasLegend':hasLegend} except: - print('[WARN] GUIScripter - No Subplot Data') + print('[WARN] GUIScripter - Failed to retrieve Subplot Data') subPlots = None try: plotStyle, plot_options, font_options, font_options_legd = self.mainframe.plotPanel.getPlotOptions() plotStyle.update(plot_options) except: plotStyle=None - print('[WARN] GUIScripter - No Plot Options') + print('[WARN] GUIScripter - Failed to retrieve Plot Options') + try: + plotPanel = self.mainframe.plotPanel + pltTypePanel = self.mainframe.plotPanel.pltTypePanel + plotType = pltTypePanel.plotType() + if plotType=='Regular': + plotTypeData=None + elif plotType=='PDF': + plotTypeData = plotPanel.pdfPanel._GUI2Data() + elif plotType=='FFT': + plotTypeData = plotPanel.spcPanel._GUI2Data() + pass + elif plotType=='MinMax': + plotTypeData = plotPanel.mmxPanel._GUI2Data() + pass + elif plotType=='Compare': + plotTypeData = plotPanel.cmpPanel._GUI2Data() + except: + plotType=None + plotTypeData=None + print('[WARN] GUIScripter - Failed to retrieve plotType and plotTypeData') # --- Use Pipeline on tablist to generate the script - s = self.pipeline.script(self.mainframe.tabList, scripterOptions, ID, subPlots, plotStyle) + s = self.pipeline.script(self.mainframe.tabList, scripterOptions, ID, subPlots, plotStyle, plotType=plotType, plotTypeData=plotTypeData) self.text_ctrl.SetValue(s) def onRun(self, event): diff --git a/pydatview/main.py b/pydatview/main.py index 7e169f7..f9521b3 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -973,7 +973,7 @@ def showApp(firstArg=None, dataframes=None, filenames=[], names=None): elif len(filenames)>0: frame.load_files(filenames, fileformats=None, bPlot=True) - #frame.onShowTool(toolName='Radial Average') + #frame.onShowTool(toolName='') #frame.onDataPlugin(toolName='Radial Average') #frame.onDataPlugin(toolName='Resample') #frame.onScript() diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index ec9e9ce..a48eb10 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -301,7 +301,7 @@ def apply(self, tabList, force=False, applyToAll=False): # self.collectErrors() - def script(self, tabList, scripterOptions=None, ID=None, subPlots=None, plotStyle=None): + def script(self, tabList, scripterOptions=None, ID=None, subPlots=None, plotStyle=None, plotType=None, plotTypeData=None): from pydatview.scripter import PythonScripter if scripterOptions is None: scripterOptions={} @@ -341,6 +341,10 @@ def script(self, tabList, scripterOptions=None, ID=None, subPlots=None, plotStyl #print('[INFO] Scripting routine for action {}'.format(action.name)) scripter.addAction(action.name, action_code, imports, code_init) + # --- Fake preplot actions that change the plot type + scripter.setPlotType(plotType, plotTypeData) + + # --- Preplot actions for action in self.actionsPlotFilters: action_code, imports, code_init = action.getScript() if action_code is None: diff --git a/pydatview/scripter.py b/pydatview/scripter.py index f17b45c..d3f1291 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -88,6 +88,71 @@ def addPreplotAction(self, action_name, code, imports=None, code_init=None): self.addImport(imp) self.preplot_actions[action_name] = (code_init, code.strip()) + def setPlotType(self, plotType, plotTypeOptions=None): + """ Setup a prePlot action depending on plot Type""" + if len(self.preplot_actions)>0: + raise Exception('PlotType should be the first preplot_action!') + opts = plotTypeOptions + action_code = None + imports = None + code_init = '' + if plotType is not None and plotType!='Regular': + if plotType=='PDF': + imports=['from pydatview.tools.stats import pdf_gaussian_kde, pdf_histogram'] + if opts is None: + action_code="""x, y = pdf_gaussian_kde(y, nOut=30)""" + else: + if opts['smooth']: + action_code="x, y = pdf_gaussian_kde(y, nOut={})".format(opts['nBins']) + else: + action_code="x, y = pdf_histogram(y, nBins={}, norm=True, count=False)".format(opts['nBins']) + + elif plotType=='FFT': + imports=['from pydatview.tools.spectral import fft_wrap'] + if opts is None: + opts={'yType':'PSD', 'avgMethod':'Welch', 'avgWindow':'Hamming', 'nExp':8, 'nPerDecade':8, 'bDetrend':False} + action_code = "x, y, Info = fft_wrap(x, y, output_type='{}', averaging='{}', averaging_window='{}', detrend={}, nExp={}, nPerDecade={})".format( + opts['yType'], opts['avgMethod'], opts['avgWindow'], opts['bDetrend'], opts['nExp'], opts['nPerDecade']) + + # TODO xType.. + # if xType=='1/x': + # if unit(PD.sx)=='s': + # PD.sx= 'Frequency [Hz]' + # else: + # PD.sx= '' + # elif xType=='x': + # PD.x=1/PD.x + # if unit(PD.sx)=='s': + # PD.sx= 'Period [s]' + # else: + # PD.sx= '' + # elif xType=='2pi/x': + # PD.x=2*np.pi*PD.x + # if unit(PD.sx)=='s': + # PD.sx= 'Cyclic frequency [rad/s]' + # else: + # PD.sx= '' + + elif plotType=='MinMax': + if opts is None: + action_code ="x = (x-np.min(x))/(np.max(x)-np.min(x))\n" + action_code+="y = (y-np.min(y))/(np.max(y)-np.min(y))" + else: + action_code = [] + if opts['xScale']: + action_code+=["x = (x-np.min(x))/(np.max(x)-np.min(x))"] + if opts['yScale']: + action_code+=["y = (y-np.min(y))/(np.max(y)-np.min(y))"] + action_code = '\n'.join(action_code) + + elif plotType=='Compare': + print('[WARN] Scripter - compare not implemented') + + if action_code is not None: + self.addPreplotAction('plotType:'+plotType, action_code, imports, code_init) + + + def addFormula(self, df_index, name, formula): self.df_formulae.append((df_index, name, formula)) @@ -117,7 +182,7 @@ def generate(self, pltshow=True): def forLoopOnDFs(): if self.opts['dfsFlavor'] == 'dict': - script.append("for k, df in dfs.items():") + script.append("for key, df in dfs.items():") elif self.opts['dfsFlavor'] == 'list': script.append("for df in dfs:") @@ -149,9 +214,13 @@ def forLoopOnDFs(): # --- List of files script.append("\n# --- Script parameters") - script.append("filenames = []") - for filename in self.filenames: - script.append(f"filenames += ['{filename}']") + nFiles = len(self.filenames) + if nFiles==1 and self.opts['oneTabPerFile']: + script.append("filename = '{}'".format(self.filenames[0])) + else: + script.append("filenames = []") + for filename in self.filenames: + script.append(f"filenames += ['{filename}']") # --- Init data/preplot/adder actions if len(self.actions)>0: @@ -163,12 +232,15 @@ def forLoopOnDFs(): script.append(actioncode[0].strip()) if len(self.preplot_actions)>0: - script.append("\n# --- Data for preplot actions") + script_pre = [] for actionname, actioncode in self.preplot_actions.items(): if actioncode[0] is not None and len(actioncode[0].strip())>0: if verboseCommentLevel>=2: - script.append("# Data for preplot action {}".format(actionname)) - script.append(actioncode[0].strip()) + script_pre.append("# Data for preplot action {}".format(actionname)) + script_pre.append(actioncode[0].strip()) + if len(script_pre)>0: + script.append("\n# --- Data for preplot actions") + script+=script_pre if len(self.adder_actions)>0: script.append("\n# --- Data for actions that add new dataframes") @@ -182,47 +254,58 @@ def forLoopOnDFs(): # --- List of Dataframes script.append("\n# --- Open and convert files to DataFrames") if self.opts['dfsFlavor'] == 'dict': - script.append("dfs = {}") - script.append("for iFile, filename in enumerate(filenames):") - if self.opts['oneTabPerFile']: - script.append(indent1 + "dfs[iFile] = weio.read(filename).toDataFrame()") + if nFiles==1 and self.opts['oneTabPerFile']: + script.append("dfs = {}") + script.append("dfs[0] = weio.read(filename).toDataFrame()") else: - script.append(indent1 + "dfs_or_df = weio.read(filename).toDataFrame()") - script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") - script.append(indent1 + "if isinstance(dfs_or_df, dict):") - script.append(indent2 + "for k,df in dfs_or_df.items():") - script.append(indent3 + "dfs[k+f'{iFile}'] = df") - script.append(indent1 + "else:") - script.append(indent2 + "dfs[f'tab{iFile}'] = dfs_or_df") + script.append("dfs = {}") + script.append("for iFile, filename in enumerate(filenames):") + if self.opts['oneTabPerFile']: + script.append(indent1 + "dfs[iFile] = weio.read(filename).toDataFrame()") + else: + script.append(indent1 + "dfs_or_df = weio.read(filename).toDataFrame()") + script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") + script.append(indent1 + "if isinstance(dfs_or_df, dict):") + script.append(indent2 + "for k,df in dfs_or_df.items():") + script.append(indent3 + "dfs[k+f'{iFile}'] = df") + script.append(indent1 + "else:") + script.append(indent2 + "dfs[f'tab{iFile}'] = dfs_or_df") elif self.opts['dfsFlavor'] == 'list': script.append("dfs = []") - script.append("for iFile, filename in enumerate(filenames):") - if self.opts['oneTabPerFile']: - script.append(indent1 + "df = weio.read(filenames[iFile]).toDataFrame()") - script.append(indent1 + "dfs.append(df)") + if nFiles==1 and self.opts['oneTabPerFile']: + script.append("dfs.append( weio.read(filename).toDataFrame() )") else: - script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") - script.append(indent1 + "dfs = weio.read(filenames[iFile]).toDataFrame()") - script.append(indent1 + "if isinstance(dfs_or_df, dict):") - script.append(indent2 + "dfs+= list(dfs_or_df.values() # NOTE: user will need to adapt this.") - script.append(indent1 + "else:") - script.append(indent2 + "dfs.append(dfs_or_df)") - - elif self.opts['dfsFlavor'] == 'enumeration': - for iFile, filename in enumerate(self.filenames): - iFile1 = iFile+1 + script.append("for iFile, filename in enumerate(filenames):") if self.opts['oneTabPerFile']: - script.append(f"df{iFile1} = weio.read(filenames[{iFile}]).toDataFrame()") + script.append(indent1 + "df = weio.read(filenames[iFile]).toDataFrame()") + script.append(indent1 + "dfs.append(df)") else: - if verboseCommentLevel>=1: - script.append("# NOTE: we need a different action if the file contains multiple dataframes") - script.append(f"dfs_or_df = weio.read('{filename}').toDataFrame()") - script.append("if isinstance(dfs_or_df, dict):") - script.append(indent1 + f"df{iFile1} = dfs_or_df.items()[0][1] # NOTE: user will need to adapt this.") - script.append("else:") - script.append(indent1 + f"df{iFile1} = dfs_or_df") + script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") + script.append(indent1 + "dfs_or_df = weio.read(filenames[iFile]).toDataFrame()") + script.append(indent1 + "if isinstance(dfs_or_df, dict):") + script.append(indent2 + "dfs+= list(dfs_or_df.values()) # NOTE: user will need to adapt this.") + script.append(indent1 + "else:") + script.append(indent2 + "dfs.append(dfs_or_df)") + + elif self.opts['dfsFlavor'] == 'enumeration': + if nFiles==1 and self.opts['oneTabPerFile']: + script.append(f"df1 = weio.read(filename).toDataFrame()") + else: + for iFile, filename in enumerate(self.filenames): + iFile1 = iFile+1 + if self.opts['oneTabPerFile']: + script.append(f"df{iFile1} = weio.read(filenames[{iFile}]).toDataFrame()") + else: + if verboseCommentLevel>=1: + script.append("# NOTE: we need a different action if the file contains multiple dataframes") + script.append(f"dfs_or_df = weio.read('{filename}').toDataFrame()") + script.append("if isinstance(dfs_or_df, dict):") + script.append(indent1 + f"df{iFile1} = dfs_or_df.items()[0][1] # NOTE: user will need to adapt this.") + script.append("else:") + script.append(indent1 + f"df{iFile1} = dfs_or_df") # --- Adder actions + nTabs = len(self.filenames) # Approximate if len(self.adder_actions)>0: def addActionCode(actioname, actioncode, ind): script.append(ind+ "# Apply action {}".format(actioname)) @@ -233,7 +316,7 @@ def addActionCode(actioname, actioncode, ind): script.append("\n# --- Apply adder actions to dataframes") script.append("dfs_add = [] ; names_add =[]") if self.opts['dfsFlavor'] == 'dict': - script.append("for k, df in dfs.items():") + script.append("for k, (key, df) in enumerate(dfs.items()):") script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") for actionname, actioncode in self.adder_actions.items(): addActionCode(actionname, actioncode[1], indent1) @@ -243,17 +326,16 @@ def addActionCode(actioname, actioncode, ind): script.append(indent2+"dfs[name_new] = df_new") elif self.opts['dfsFlavor'] == 'list': - script.append("for df in dfs):") + script.append("for k, df in enumerate(dfs):") script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") for actionname, actioncode in self.adder_actions.items(): addActionCode(actionname, actioncode[1], indent1) script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") script.append("for name_new, df_new in zip(names_add, dfs_new):") script.append(indent1+"if df_new is not None:") - script.append(indent2+"dfs+ = [df_new]") + script.append(indent2+"dfs += [df_new]") elif self.opts['dfsFlavor'] == 'enumeration': - nTabs = len(self.filenames) # Approximate for iTab in range(nTabs): script.append("filename = filenames[{}] # NOTE: this is approximate..".format(iTab)) script.append('df = df{}'.format(iTab+1)) @@ -279,7 +361,7 @@ def addActionCode(actioname, actioncode, ind): script.append(indent2 + "df['{}'] = {}".format(name, formula)) #df.insert(int(i+1), name, formula) script.append(indent1 + "except:") - script.append(indent2 + "print('[WARN] Cannot add column {} to dataframe)".format(name)) + script.append(indent2 + "print('[WARN] Cannot add column {} to dataframe')".format(name)) elif self.opts['dfsFlavor'] == 'enumeration': for iTab in range(nTabs): @@ -303,7 +385,8 @@ def addActionCode(actioname, actioncode, ind): if len(self.actions)>0: def addActionCode(actioname, actioncode, ind): - script.append(ind+ "# Apply action {}".format(actioname)) + if verboseCommentLevel>2: + script.append(ind+ "# Apply action {}".format(actioname)) lines = actioncode.split("\n") indented_lines = [ind + line for line in lines] script.append("\n".join(indented_lines)) @@ -316,7 +399,7 @@ def addActionCode(actioname, actioncode, ind): script.append(indent1 + "dfs[k] = df") elif self.opts['dfsFlavor'] == 'list': - script.append("for df in dfs):") + script.append("for k, df in enumerate(dfs):") for actionname, actioncode in self.actions.items(): addActionCode(actionname, actioncode[1], indent1) script.append(indent1 + "dfs[k] = df") @@ -396,14 +479,14 @@ def getAxesString(i,j): sDF = sDF_new script.append("df = "+sDF) if len(self.preplot_actions)>0: - script.append("x = df['{}']".format(column_x)) - script.append("y = df['{}']".format(column_y)) + script.append("x = df['{}'].values".format(column_x)) + script.append("y = df['{}'].values".format(column_y)) else: sPlotXY ="df['{}'], df['{}'], ".format(column_x, column_y) elif self.opts['dfsFlavor'] == 'enumeration': if len(self.preplot_actions)>0: - script.append("x = df{}['{}']".format(df_index+1, column_x)) - script.append("y = df{}['{}']".format(df_index+1, column_y)) + script.append("x = df{}['{}'].values".format(df_index+1, column_x)) + script.append("y = df{}['{}'].values".format(df_index+1, column_y)) else: sPlotXY ="df{}['{}'], df{}['{}'], ".format(df_index+1, column_x, df_index+1, column_y) diff --git a/pydatview/tools/spectral.py b/pydatview/tools/spectral.py index f999562..80e54e5 100644 --- a/pydatview/tools/spectral.py +++ b/pydatview/tools/spectral.py @@ -48,6 +48,7 @@ def fft_wrap(t,y,dt=None, output_type='amplitude',averaging='None',averaging_win output_type = output_type.lower() averaging = averaging.lower() averaging_window = averaging_window.lower() + t = np.asarray(t) y = np.asarray(y) n0 = len(y) nt = len(t) From 1506664b8232c3f9370269155e9dc709aac7b1c9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 26 Sep 2023 11:48:57 -0600 Subject: [PATCH 127/178] Plot: remember subplot spacings on redraw/reload (#147) --- pydatview/GUIPlotPanel.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index f5b2706..36933cb 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -611,7 +611,7 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): self.SetSizer(plotsizer) self.plotsizer=plotsizer; - self.set_subplot_spacing(init=True) + self.setSubplotSpacing(init=True) # --- Bindings/callback def setAddTablesCallback(self, callback): @@ -656,7 +656,7 @@ def onEsthToggle(self,event): self.plotsizer.Layout() event.Skip() - def set_subplot_spacing(self, init=False): + def setSubplotSpacing(self, init=False): """ Handle default subplot spacing @@ -669,11 +669,12 @@ def set_subplot_spacing(self, init=False): #self.fig.set_tight_layout(True) # NOTE: works almost fine, but problem with FFT multiple ## TODO this is definitely not generic, but tight fails.. #return - #if hasattr(self.fig, '_subplotsPar'): - # # See GUIToolBox.py configure_toolbar - # self.fig.subplots_adjust(**self.fig._subplotsPar) - # return - + if hasattr(self, 'subplotsPar'): + if self.subplotsPar is not None: + # See GUIToolBox.py configure_toolbar + #print('>>> subplotPar', self.subplotsPar) + self.fig.subplots_adjust(**self.subplotsPar) + return if init: # NOTE: at init size is (20,20) because sizer is not initialized yet @@ -706,8 +707,15 @@ def set_subplot_spacing(self, init=False): else: self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98) - def get_subplot_spacing(self, init=False): - pass + def getSubplotSpacing(self): + try: + params = self.fig.subplotpars + paramsD= {} + for key in ['left', 'bottom', 'right', 'top', 'wspace', 'hspace']: + paramsD[key]=getattr(params, key) + return paramsD + except: + return None # At Init we don't have a figure def plot_matrix_select(self, event): @@ -761,7 +769,7 @@ def sharex(self): return self.cbSync.IsChecked() and (not self.pltTypePanel.cbPDF.GetValue()) def set_subplots(self,nPlots): - self.set_subplot_spacing() + self.setSubplotSpacing() # Creating subplots for ax in self.fig.axes: self.fig.delaxes(ax) @@ -1516,6 +1524,7 @@ def load_and_draw(self): - Trigger changes to infoPanel """ + self.subplotsPar = self.getSubplotSpacing() self.clean_memory() self.getPlotData(self.pltTypePanel.plotType()) if len(self.plotData)==0: From 339e3b4cdddd5ba4c1a350d9bf8cd49242557320 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 26 Sep 2023 13:10:25 -0600 Subject: [PATCH 128/178] Scripter: using temp dir --- pydatview/GUIScripter.py | 2 +- pydatview/scripter.py | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pydatview/GUIScripter.py b/pydatview/GUIScripter.py index 6282941..4ef7e9b 100644 --- a/pydatview/GUIScripter.py +++ b/pydatview/GUIScripter.py @@ -17,7 +17,7 @@ def __init__(self, parent, mainframe, pipeLike, title): self.btGen = wx.Button(self.panel, label="Update") - self.btRun = wx.Button(self.panel, label="Run Script") + self.btRun = wx.Button(self.panel, label="Run Script (beta)") self.btSave = wx.Button(self.panel, label="Save to File") txtLib = wx.StaticText(self.panel, -1, 'Library:') diff --git a/pydatview/scripter.py b/pydatview/scripter.py index d3f1291..9876c07 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -544,7 +544,7 @@ def getAxesString(i,j): return "\n".join(script) - def run(self, script=None, method='subprocess', pltshow=True, scriptName='_pydatview_temp_script.py'): + def run(self, script=None, method='subprocess', pltshow=True, scriptName='./_pydatview_temp_script.py', tempDir=True): if script is None: script = self.generate(pltshow=pltshow) import tempfile @@ -555,15 +555,18 @@ def run(self, script=None, method='subprocess', pltshow=True, scriptName='_pydat if method=='subprocess': try: # --- Create a temporary file - #temp_dir = tempfile.TemporaryDirectory() - #script_file_path = os.path.join(temp_dir.name, "temp_script.py") - script_file_path = scriptName + if tempDir: + temp_dir = tempfile.TemporaryDirectory() + script_file_path = os.path.join(temp_dir.name, scriptName) + else: + script_file_path = scriptName with open(script_file_path, "w") as script_file: script_file.write(script) # Run the script as a system call result = subprocess.run(["python", script_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + #result = subprocess.run(["python", '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) # Print the output and errors (if any) #errors.append("Script Output:") @@ -571,12 +574,16 @@ def run(self, script=None, method='subprocess', pltshow=True, scriptName='_pydat if len(result.stderr)>0: errors.append(result.stderr) except Exception as e: - error.append("An error occurred: {e}") + errors.append(f"An error occurred: {e}") finally: # Clean up by deleting the temporary directory and its contents if len(errors)==0: - #temp_dir.cleanup() - os.remove(script_file_path) + try: + os.remove(script_file_path) + if tempDir: + temp_dir.cleanup() + except: + print('[FAIL] Failed to remove {}'.format(script_file_path)) else: raise NotImplementedError() if len(errors)>0: From f22ffb722b489f7d077a00dc7d4300f5f2abcd39 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 26 Sep 2023 13:17:24 -0600 Subject: [PATCH 129/178] GH: Installing pydatview before unittest --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b7b98c5..1056733 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,6 +85,7 @@ jobs: #pip install -U wxpython || true #pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-22.04 wxPython || true pip install -r _tools/travis_requirements.txt + pip install -e . - name: System info run: | From 2b32c4b44347e34c396d54f3b2d5b7cc7f3922b5 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 26 Sep 2023 15:22:14 -0600 Subject: [PATCH 130/178] Measure: avoid replot and rezoom (Closes #144) --- pydatview/GUIMeasure.py | 18 +++++--- pydatview/GUIPlotPanel.py | 91 +++++++++++++++++++++++---------------- pydatview/scripter.py | 3 +- 3 files changed, 69 insertions(+), 43 deletions(-) diff --git a/pydatview/GUIMeasure.py b/pydatview/GUIMeasure.py index 63c4259..b94fa7c 100644 --- a/pydatview/GUIMeasure.py +++ b/pydatview/GUIMeasure.py @@ -44,8 +44,20 @@ def clearPointLines(self): self.points= [] self.lines= [] - def set(self, axes, ax, x, y, PD): + def setAndPlot(self, axes, ax, x, y, PD): """ + Set a measure and plot it + - x,y : point where the user clicked (will likely be slightly off plotdata) + """ + self.set(ax, x, y) + + # Plot measure where the user clicked (only for the axis that the user chose) + # - plot intersection point, vertical line, and annotation + self.plot(axes, PD) + + def set(self, ax, x, y): + """ + Set a measure - x,y : point where the user clicked (will likely be slightly off plotdata) """ self.clearPlot() @@ -56,10 +68,6 @@ def set(self, axes, ax, x, y, PD): self.P_target_num = P_num self.pd_closest = pd_closest - # Plot measure where the user clicked (only for the axis that the user chose) - # - plot intersection point, vertical line, and annotation - self.plot(axes, PD) - def compute(self, PD): for ipd, pd in enumerate(PD): if pd !=self.pd_closest: diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 36933cb..15f5b34 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -725,18 +725,58 @@ def plot_matrix_select(self, event): def measure_select(self, event): if self.cbMeasure.IsChecked(): + # We do nothing, onMouseRelease will trigger the plot and setting pass - #self.cbAutoScale.SetValue(False) else: - # We clear - for measure in [self.leftMeasure, self.rightMeasure]: - measure.clear() + self.cleanMeasures() + # We redraw + self.redraw_same_data() + + def setAndPlotMeasures(self, ax, x, y, which=None): + if which is None: + which=[1,2] + if 1 in which: + # Left click, measure 1 - set values, compute all intersections and plot + self.leftMeasure.set(ax, x, y) if self.infoPanel is not None: - self.infoPanel.clearMeasurements() - self.lbDeltaX.SetLabel('') - self.lbDeltaY.SetLabel('') + self.infoPanel.showMeasure1() + if 2 in which: + # Right click, measure 2 - set values, compute all intersections and plot + self.rightMeasure.set(ax, x, y) + if self.infoPanel is not None: + self.infoPanel.showMeasure2() + self.plotMeasures(which=which) + + def plotMeasures(self, which=None): + if which is None: + which=[1,2] + axes=self.fig.axes + ## plot them + if 1 in which: + self.leftMeasure.plot (axes, self.plotData) + if 2 in which: + self.rightMeasure.plot(axes, self.plotData) + ## Update dx,dy label + self.lbDeltaX.SetLabel(self.rightMeasure.sDeltaX(self.leftMeasure)) + self.lbDeltaY.SetLabel(self.rightMeasure.sDeltaY(self.leftMeasure)) + + #if not self.cbAutoScale.IsChecked(): + # print('>>> On Mouse Release Restore LIMITS') + # self._restore_limits() + #else: + # print('>>> On Mouse Release Not Restore LIMITS') + # Update label + + def cleanMeasures(self): + # We clear + for measure in [self.leftMeasure, self.rightMeasure]: + measure.clear() + if self.infoPanel is not None: + self.infoPanel.clearMeasurements() + # Update dx,dy label + self.lbDeltaX.SetLabel('') + self.lbDeltaY.SetLabel('') - self.redraw_same_data() def redraw_event(self, event): self.redraw_same_data() @@ -806,23 +846,14 @@ def onMouseRelease(self, event): #self.cbAutoScale.SetValue(False) return if event.button == 1: - # Left click, measure 1 - set values, compute all intersections and plot - self.leftMeasure.set(self.fig.axes, ax, x, y, self.plotData) # Set and plot - if self.infoPanel is not None: - self.infoPanel.showMeasure1() + which =[1] # Left click, measure 1 elif event.button == 3: - # Left click, measure 2 - set values, compute all intersections and plot - self.rightMeasure.set(self.fig.axes, ax, x, y, self.plotData) # Set and plot - if self.infoPanel is not None: - self.infoPanel.showMeasure2() + which =[2] # Right click, measure 2 else: return - if not self.cbAutoScale.IsChecked(): - self._restore_limits() - # Update label - self.lbDeltaX.SetLabel(self.rightMeasure.sDeltaX(self.leftMeasure)) - self.lbDeltaY.SetLabel(self.rightMeasure.sDeltaY(self.leftMeasure)) - return + self.setAndPlotMeasures(ax, x, y, which) + return # We return as soon as one ax match the click location + def onDraw(self, event): self._store_limits() @@ -1259,21 +1290,7 @@ def plot_all(self, autoscale=True): ax.legend(fancybox=False, loc=lgdLoc, **font_options_legd) # --- End loop on axes - # --- Measure - if self.cbMeasure.IsChecked(): - # Compute and plot them - self.leftMeasure.plot (axes, self.plotData) - self.rightMeasure.plot(axes, self.plotData) - # Update dx,dy label - self.lbDeltaX.SetLabel(self.rightMeasure.sDeltaX(self.leftMeasure)) - self.lbDeltaY.SetLabel(self.rightMeasure.sDeltaY(self.leftMeasure)) - ## Update info panel - #if self.infoPanel is not None: - # self.infoPanel.setMeasurements(self.leftMeasure.get_xydata(), self.rightMeasure.get_xydata()) - else: - # Update dx,dy label - self.lbDeltaX.SetLabel('') - self.lbDeltaY.SetLabel('') + # --- Measure: Done as an overlay in plotMeasures # --- xlabel #axes[-1].set_xlabel(PD[axes[-1].iPD[0]].sx, **font_options) diff --git a/pydatview/scripter.py b/pydatview/scripter.py index 9876c07..706174f 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -43,6 +43,7 @@ def __init__(self, **opts): # Imports that we know we'll need self.addImport('import numpy as np') + #self.addImport("import warnings; warnings.filterwarnings('ignore')") self.addImport('import pydatview.io as weio') self.addImport('import matplotlib.pyplot as plt') @@ -56,7 +57,7 @@ def reset(self): self.df_selections = [] # List of tuples (df_index, column_x, column_y) self.df_formulae = [] # List of tuples (df_index, name, formula) self.dfs = [] - self.subPlots = {'i':1, 'j':1, 'x_labels':['x'], 'y_labels':['y'], 'IPD':None, 'hasLegend':[True]} + self.subPlots = {'i':1, 'j':1, 'x_labels':['x'], 'y_labels':['y'], 'IPD':None, 'hasLegend':[False]} self.plotStyle = _defaultPlotStyle def setOptions(self, **opts): From 1611a319d701c76c1ce19973ce28cf401017f1e0 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 26 Sep 2023 15:38:07 -0600 Subject: [PATCH 131/178] Live Plot: allowing user to apply actions and click around --- pydatview/GUIPlotPanel.py | 3 +++ pydatview/main.py | 29 +++++++++++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 15f5b34..6a9348d 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -735,6 +735,9 @@ def measure_select(self, event): def setAndPlotMeasures(self, ax, x, y, which=None): if which is None: which=[1,2] + if not hasattr(ax, 'PD'): + print('[WARN] Cannot measure on an empty plot') + return if 1 in which: # Left click, measure 1 - set values, compute all intersections and plot self.leftMeasure.set(ax, x, y) diff --git a/pydatview/main.py b/pydatview/main.py index f9521b3..ff294c7 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -609,8 +609,8 @@ def onLivePlotChange(self, event=None): if self.cbLivePlot.IsChecked(): if hasattr(self,'plotPanel'): #print('[INFO] Reenabling live plot') - self.plotPanel.Enable(True) - self.infoPanel.Enable(True) + #self.plotPanel.Enable(True) + #self.infoPanel.Enable(True) self.redrawCallback() else: if hasattr(self,'plotPanel'): @@ -618,20 +618,21 @@ def onLivePlotChange(self, event=None): for ax in self.plotPanel.fig.axes: ax.annotate('Live Plot Disabled', xy=(0.5, 0.5), size=20, xycoords='axes fraction', ha='center', va='center',) self.plotPanel.canvas.draw() - self.plotPanel.Enable(False) - self.infoPanel.Enable(False) + #self.plotPanel.Enable(False) + #self.infoPanel.Enable(False) def livePlotFreezeUnfreeze(self): - if self.cbLivePlot.IsChecked(): - if hasattr(self,'plotPanel'): - #print('[INFO] Enabling live plot') - self.plotPanel.Enable(True) - self.infoPanel.Enable(True) - else: - if hasattr(self,'plotPanel'): - #print('[INFO] Disabling live plot') - self.plotPanel.Enable(False) - self.infoPanel.Enable(False) + pass + #if self.cbLivePlot.IsChecked(): + # if hasattr(self,'plotPanel'): + # #print('[INFO] Enabling live plot') + # #self.plotPanel.Enable(True) + # self.infoPanel.Enable(True) + #else: + # if hasattr(self,'plotPanel'): + # #print('[INFO] Disabling live plot') + # #self.plotPanel.Enable(False) + # self.infoPanel.Enable(False) def redrawCallback(self): if hasattr(self,'plotPanel'): From 0b0e8cc047ae24f0d98059a093142ab0544320a3 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 26 Sep 2023 16:16:16 -0600 Subject: [PATCH 132/178] Markers: feature similar to measure but for multiple points (Closes #141)" --- pydatview/GUIMeasure.py | 3 +- pydatview/GUIPlotPanel.py | 65 +++++++++++++++++++++++++++++++++------ pydatview/main.py | 2 ++ 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/pydatview/GUIMeasure.py b/pydatview/GUIMeasure.py index b94fa7c..a5c46b3 100644 --- a/pydatview/GUIMeasure.py +++ b/pydatview/GUIMeasure.py @@ -83,7 +83,8 @@ def compute(self, PD): else: # Already computed xc, yc = self.P_target_raw - pd.xyMeas[self.index-1] = (xc, yc) + if self.index>0: + pd.xyMeas[self.index-1] = (xc, yc) def plotAnnotation(self, ax, xc, yc): #self.clearAnnotation() diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 6a9348d..57d27cf 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -44,7 +44,7 @@ from pydatview.plotdata import PDL_xlabel from pydatview.GUICommon import * from pydatview.GUIToolBox import MyMultiCursor, MyNavigationToolbar2Wx, TBAddTool, TBAddCheckTool -from pydatview.GUIMeasure import GUIMeasure +from pydatview.GUIMeasure import GUIMeasure, find_closest_i import pydatview.icons as icons font = {'size' : 8} @@ -462,6 +462,7 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): #self.SetBackgroundColour('red') self.leftMeasure = GUIMeasure(1, 'firebrick') self.rightMeasure = GUIMeasure(2, 'darkgreen') + self.markers = [] # List of GUIMeasures self.xlim_prev = [[0, 1]] self.ylim_prev = [[0, 1]] self.addTablesCallback = None @@ -523,6 +524,7 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): self.cbGrid = wx.CheckBox(self.ctrlPanel, -1, 'Grid',(10,10)) self.cbStepPlot = wx.CheckBox(self.ctrlPanel, -1, 'StepPlot',(10,10)) self.cbMeasure = wx.CheckBox(self.ctrlPanel, -1, 'Measure',(10,10)) + self.cbMarkPt = wx.CheckBox(self.ctrlPanel, -1, 'Mark Points',(10,10)) #self.cbSub.SetValue(True) # DEFAULT TO SUB? self.cbSync.SetValue(True) self.cbXHair.SetValue(self.data['CrossHair']) # Have cross hair by default @@ -535,11 +537,12 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): self.Bind(wx.EVT_CHECKBOX, self.log_select , self.cbLogY ) self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbSync ) self.Bind(wx.EVT_CHECKBOX, self.crosshair_event , self.cbXHair ) - self.Bind(wx.EVT_CHECKBOX, self.plot_matrix_select, self.cbPlotMatrix ) + self.Bind(wx.EVT_CHECKBOX, self.plot_matrix_event, self.cbPlotMatrix ) self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbAutoScale ) self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbGrid ) self.Bind(wx.EVT_CHECKBOX, self.redraw_event , self.cbStepPlot ) - self.Bind(wx.EVT_CHECKBOX, self.measure_select , self.cbMeasure ) + self.Bind(wx.EVT_CHECKBOX, self.measure_event , self.cbMeasure ) + self.Bind(wx.EVT_CHECKBOX, self.markpt_event , self.cbMarkPt ) # LAYOUT cb_sizer = wx.FlexGridSizer(rows=4, cols=3, hgap=0, vgap=0) cb_sizer.Add(self.cbCurveType , 0, flag=wx.ALL, border=1) @@ -553,6 +556,7 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): cb_sizer.Add(self.cbSync , 0, flag=wx.ALL, border=1) cb_sizer.Add(self.cbPlotMatrix, 0, flag=wx.ALL, border=1) cb_sizer.Add(self.cbMeasure , 0, flag=wx.ALL, border=1) + cb_sizer.Add(self.cbMarkPt , 0, flag=wx.ALL, border=1) self.ctrlPanel.SetSizer(cb_sizer) @@ -718,12 +722,12 @@ def getSubplotSpacing(self): return None # At Init we don't have a figure - def plot_matrix_select(self, event): + def plot_matrix_event(self, event): if self.infoPanel is not None: self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) self.redraw_same_data() - def measure_select(self, event): + def measure_event(self, event): if self.cbMeasure.IsChecked(): # We do nothing, onMouseRelease will trigger the plot and setting pass @@ -753,12 +757,11 @@ def setAndPlotMeasures(self, ax, x, y, which=None): def plotMeasures(self, which=None): if which is None: which=[1,2] - axes=self.fig.axes ## plot them if 1 in which: - self.leftMeasure.plot (axes, self.plotData) + self.leftMeasure.plot (self.fig.axes, self.plotData) if 2 in which: - self.rightMeasure.plot(axes, self.plotData) + self.rightMeasure.plot(self.fig.axes, self.plotData) ## Update dx,dy label self.lbDeltaX.SetLabel(self.rightMeasure.sDeltaX(self.leftMeasure)) self.lbDeltaY.SetLabel(self.rightMeasure.sDeltaY(self.leftMeasure)) @@ -780,6 +783,24 @@ def cleanMeasures(self): self.lbDeltaX.SetLabel('') self.lbDeltaY.SetLabel('') + def markpt_event(self, event): + if self.cbMarkPt.IsChecked(): + self.markers = [] + # We do nothing, onMouseRelease will trigger the plot and setting + else: + self.cleanMarkers() + # We redraw + self.redraw_same_data() + + def plotMarkers(self): + for marker in self.markers: + marker.plot (self.fig.axes, self.plotData) + + def cleanMarkers(self): + # We clear + for marker in self.markers: + marker.clear() + self.markers=[] def redraw_event(self, event): self.redraw_same_data() @@ -839,6 +860,7 @@ def onMouseClick(self, event): def onMouseRelease(self, event): if self.cbMeasure.GetValue(): + # --- Measures # Loop on axes for iax, ax in enumerate(self.fig.axes): if event.inaxes == ax: @@ -856,7 +878,32 @@ def onMouseRelease(self, event): return self.setAndPlotMeasures(ax, x, y, which) return # We return as soon as one ax match the click location - + elif self.cbMarkPt.GetValue(): + # --- Markers + for iax, ax in enumerate(self.fig.axes): + if event.inaxes == ax: + x, y = event.xdata, event.ydata + if self.clickLocation != (ax, x, y): + return + if event.button == 1: + # We add a marker + # from pydatview.tools.colors import fColrs, python_colors + n = len(self.markers) + #marker = GUIMeasure(1, python_colors(n+1)) + marker = GUIMeasure(1, 'firebrick') + #GUIMeasure(2, 'darkgreen') + self.markers.append(marker) + marker.setAndPlot(self.fig.axes, ax, x, y, self.plotData) + elif event.button == 3: + # find the closest marker + XY = np.array([m.P_target_raw for m in self.markers]) + i = find_closest_i(XY, (x,y)) + # We clear it fomr the plot + self.markers[i].clear() + # We delete it + del self.markers[i] + else: + return def onDraw(self, event): self._store_limits() diff --git a/pydatview/main.py b/pydatview/main.py index ff294c7..f4084af 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -282,6 +282,8 @@ def clean_memory(self,bReload=False): #print('Clean memory') # force Memory cleanup self.tabList.clean() + if hasattr(self,'plotPanel'): + self.plotPanel.markers = [] if not bReload: if hasattr(self,'selPanel'): self.selPanel.clean_memory() From d3919b0df9fc25e8c26a520e414da2f0be90bfe5 Mon Sep 17 00:00:00 2001 From: "E. Branlard" <1318316+ebranlard@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:09:02 -0600 Subject: [PATCH 133/178] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index c53ffcd..ddbe48b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ [![Build status](https://github.com/ebranlard/pyDatView/workflows/Tests/badge.svg)](https://github.com/ebranlard/pyDatView/actions?query=workflow%3A%22Tests%22) -Donate just a small amount, buy me a coffee! - +Donate just a small amount, buy me a coffee # pyDatView From 1ac3739e937b3db34c2bda24793161fbb626acf4 Mon Sep 17 00:00:00 2001 From: "E. Branlard" <1318316+ebranlard@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:12:09 -0600 Subject: [PATCH 134/178] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ddbe48b..6a6eaf6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Build status](https://github.com/ebranlard/pyDatView/workflows/Tests/badge.svg)](https://github.com/ebranlard/pyDatView/actions?query=workflow%3A%22Tests%22) -Donate just a small amount, buy me a coffee +Donate just a small amount, buy me a coffee # pyDatView From e635db089982caa4a2c12868685d31618332507e Mon Sep 17 00:00:00 2001 From: "E. Branlard" <1318316+ebranlard@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:12:57 -0600 Subject: [PATCH 135/178] Update README.md --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6a6eaf6..22c5151 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Build status](https://github.com/ebranlard/pyDatView/workflows/Tests/badge.svg)](https://github.com/ebranlard/pyDatView/actions?query=workflow%3A%22Tests%22) -Donate just a small amount, buy me a coffee +Donate just a small amount, buy me a coffee # pyDatView @@ -275,7 +275,5 @@ Follow the procedure mentioned in the README of the weio repository (in particua Any contributions to this project are welcome! If you find this project useful, you can also buy me a coffee (donate a small amount) with the link below: -Donate just a small amount, buy me a coffee! - - +Donate just a small amount, buy me a coffee From dc1fc32288a798075732c09fbb70831f23f98d18 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 2 Oct 2023 22:12:57 -0600 Subject: [PATCH 136/178] Plot: tight layout when clicking home and better handling (see #147 #65) --- pydatview/GUIPlotPanel.py | 105 +++++++++++++++++++++----------------- pydatview/GUIToolBox.py | 6 ++- pydatview/main.py | 40 +++++++++------ 3 files changed, 87 insertions(+), 64 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 57d27cf..e872fb3 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -451,6 +451,8 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): self.parent = parent self.plotData = [] self.toolPanel=None + self.subplotsPar=None + self.plotDone=False if data is not None: self.data = data else: @@ -477,8 +479,8 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): self.canvas.mpl_connect('draw_event', self.onDraw) self.clickLocation = (None, 0, 0) - self.navTBTop = MyNavigationToolbar2Wx(self.canvas, ['Home', 'Pan']) - self.navTBBottom = MyNavigationToolbar2Wx(self.canvas, ['Subplots', 'Save']) + self.navTBTop = MyNavigationToolbar2Wx(self.canvas, ['Home', 'Pan'], plotPanel=self) + self.navTBBottom = MyNavigationToolbar2Wx(self.canvas, ['Subplots', 'Save'], plotPanel=self) TBAddCheckTool(self.navTBBottom,'', icons.chart.GetBitmap(), self.onEsthToggle) self.esthToggle=False @@ -615,7 +617,7 @@ def __init__(self, parent, selPanel, pipeLike=None, infoPanel=None, data=None): self.SetSizer(plotsizer) self.plotsizer=plotsizer; - self.setSubplotSpacing(init=True) +# self.setSubplotSpacing(init=True) # --- Bindings/callback def setAddTablesCallback(self, callback): @@ -660,7 +662,7 @@ def onEsthToggle(self,event): self.plotsizer.Layout() event.Skip() - def setSubplotSpacing(self, init=False): + def setSubplotSpacing(self, init=False, tight=False): """ Handle default subplot spacing @@ -670,46 +672,57 @@ def setSubplotSpacing(self, init=False): - need to change if right axis needed - this will override the user settings """ - #self.fig.set_tight_layout(True) # NOTE: works almost fine, but problem with FFT multiple - ## TODO this is definitely not generic, but tight fails.. - #return - if hasattr(self, 'subplotsPar'): - if self.subplotsPar is not None: - # See GUIToolBox.py configure_toolbar - #print('>>> subplotPar', self.subplotsPar) - self.fig.subplots_adjust(**self.subplotsPar) - return + if tight: + self.setSubplotTight(draw=False) + return if init: - # NOTE: at init size is (20,20) because sizer is not initialized yet - bottom = 0.12 - left = 0.12 + subplotsParLoc={'bottom':0.12, 'top':0.97, 'left':0.12, 'right':0.98} + self.fig.subplots_adjust(**subplotsParLoc) + return + + if self.subplotsPar is not None: + if self.cbPlotMatrix.GetValue(): # TODO detect it + self.subplotsPar['right'] = 0.98 - subplotsPar['left'] + # See GUIToolBox.py configure_toolbar + self.fig.subplots_adjust(**self.subplotsPar) + return else: - if self.Size[1]<300: - bottom=0.20 - elif self.Size[1]<350: - bottom=0.18 - elif self.Size[1]<430: - bottom=0.16 - elif self.Size[1]<600: - bottom=0.13 - elif self.Size[1]<800: - bottom=0.09 - else: - bottom=0.07 - if self.Size[0]<300: - left=0.22 - elif self.Size[0]<450: - left=0.20 - elif self.Size[0]<950: - left=0.12 - else: - left=0.06 - #print(self.Size,'bottom', bottom, 'left',left) - if self.cbPlotMatrix.GetValue(): # TODO detect it - self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98-left) + self.setSubplotTight(draw=False) + + def setSubplotTight(self, draw=True): + self.fig.tight_layout() + self.subplotsPar = self.getSubplotSpacing() + + # --- Ensure some minimum spacing based on panel size + if self.Size[1]<300: + bottom=0.20 + elif self.Size[1]<350: + bottom=0.18 + elif self.Size[1]<430: + bottom=0.16 + elif self.Size[1]<600: + bottom=0.13 + elif self.Size[1]<800: + bottom=0.09 else: - self.fig.subplots_adjust(top=0.97,bottom=bottom,left=left,right=0.98) + bottom=0.07 + if self.Size[0]<300: + left=0.22 + elif self.Size[0]<450: + left=0.20 + elif self.Size[0]<950: + left=0.12 + else: + left=0.06 + + self.subplotsPar['left'] = max(self.subplotsPar['left'] , left) + self.subplotsPar['bottom'] = max(self.subplotsPar['bottom'], bottom) + self.subplotsPar['top'] = min(self.subplotsPar['top'] , 0.97) + self.subplotsPar['right'] = min(self.subplotsPar['right'] , 0.995) + self.fig.subplots_adjust(**self.subplotsPar) + if draw: + self.canvas.draw() def getSubplotSpacing(self): try: @@ -721,7 +734,6 @@ def getSubplotSpacing(self): except: return None # At Init we don't have a figure - def plot_matrix_event(self, event): if self.infoPanel is not None: self.infoPanel.togglePlotMatrix(self.cbPlotMatrix.GetValue()) @@ -833,7 +845,6 @@ def sharex(self): return self.cbSync.IsChecked() and (not self.pltTypePanel.cbPDF.GetValue()) def set_subplots(self,nPlots): - self.setSubplotSpacing() # Creating subplots for ax in self.fig.axes: self.fig.delaxes(ax) @@ -1591,7 +1602,10 @@ def load_and_draw(self): - Trigger changes to infoPanel """ - self.subplotsPar = self.getSubplotSpacing() + if self.plotDone: + self.subplotsPar = self.getSubplotSpacing() + else: + self.subplotsPar = None self.clean_memory() self.getPlotData(self.pltTypePanel.plotType()) if len(self.plotData)==0: @@ -1606,6 +1620,7 @@ def load_and_draw(self): self.redraw_same_data() if self.infoPanel is not None: self.infoPanel.showStats(self.plotData,self.pltTypePanel.plotType()) + self.plotDone=True def redraw_same_data(self, force_autoscale=False): """ @@ -1636,13 +1651,12 @@ def redraw_same_data(self, force_autoscale=False): self.clean_memory_plot() self.set_subplots(nPlots) self.distributePlots(mode,nPlots,spreadBy) + self.setSubplotSpacing() if not self.pltTypePanel.cbCompare.GetValue(): self.setLegendLabels(mode) - autoscale = (self.cbAutoScale.IsChecked()) or (force_autoscale) - self.plot_all(autoscale=autoscale) self.canvas.draw() @@ -1659,7 +1673,6 @@ def _restore_limits(self): ax.set_xlim(xlim) ax.set_ylim(ylim) - if __name__ == '__main__': import pandas as pd; from Tables import Table,TableList diff --git a/pydatview/GUIToolBox.py b/pydatview/GUIToolBox.py index c591816..57d9575 100644 --- a/pydatview/GUIToolBox.py +++ b/pydatview/GUIToolBox.py @@ -243,9 +243,10 @@ class MyNavigationToolbar2Wx(NavigationToolbar2Wx): - Tools can be removed, if not in `keep_tools` - Zoom is set by default, and the toggling between zoom and pan is handled internally """ - def __init__(self, canvas, keep_tools): + def __init__(self, canvas, keep_tools, plotPanel): # Taken from matplotlib/backend_wx.py but added style: self.VERSION = matplotlib.__version__ + self.plotPanel = plotPanel #print('MPL VERSION:',self.VERSION) if self.VERSION[0]=='2' or self.VERSION[0]=='1': wx.ToolBar.__init__(self, canvas.GetParent(), -1, style=wx.TB_HORIZONTAL | wx.NO_BORDER | wx.TB_FLAT | wx.TB_NODIVIDER) @@ -305,6 +306,9 @@ def pan(self, *args): def home(self, *args): """Restore the original view.""" + # Feature: if user click on home, we trigger a tight layout + self.plotPanel.setSubplotTight(draw=False) + # Feature: We force autoscale self.canvas.GetParent().redraw_same_data(force_autoscale=True) def set_message(self, s): diff --git a/pydatview/main.py b/pydatview/main.py index f4084af..c9fba11 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -113,6 +113,7 @@ def __init__(self, data=None): self.data = loadAppData(self) self.tabList=TableList(options=self.data['loaderOptions']) self.datareset = False + self.resized = False # used to trigger a tight layout after resize event # Global variables... setFontSize(self.data['fontSize']) setMonoFontSize(self.data['monoFontSize']) @@ -262,6 +263,7 @@ def __init__(self, data=None): self.Center() self.Show() self.Bind(wx.EVT_SIZE, self.OnResizeWindow) + self.Bind(wx.EVT_IDLE, self.OnIdle) self.Bind(wx.EVT_CLOSE, self.onClose) # Shortcuts @@ -801,25 +803,29 @@ def onShowLoaderMenu(self, event=None): def mainFrameUpdateLayout(self, event=None): - if hasattr(self,'selPanel'): - nWind=self.selPanel.splitter.nWindows - if self.Size[0]<=800: - sash=SIDE_COL[nWind] - else: - sash=SIDE_COL_LARGE[nWind] - self.resizeSideColumn(sash) - - def OnResizeWindow(self, event): try: - self.mainFrameUpdateLayout() - self.Layout() + if hasattr(self,'selPanel'): + nWind=self.selPanel.splitter.nWindows + if self.Size[0]<=800: + sash=SIDE_COL[nWind] + else: + sash=SIDE_COL_LARGE[nWind] + self.resizeSideColumn(sash) except: - pass - # NOTE: doesn't work... - #if hasattr(self,'plotPanel'): - # Subplot spacing changes based on figure size - #print('>>> RESIZE WINDOW') - #self.redraw() + print('[Fail] An error occured in mainFrameUpdateLayout') + + def OnIdle(self, event): + if self.resized: + self.resized = False + self.mainFrameUpdateLayout() + if hasattr(self,'plotPanel'): + self.plotPanel.setSubplotTight() + self.Thaw() + + def OnResizeWindow(self, event): + self.resized = True + self.Freeze() + self.Layout() # --- Side column def resizeSideColumn(self,width): From f4b6426f1cd06de26aee47c840367753b58b4df9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 2 Oct 2023 23:10:17 -0600 Subject: [PATCH 137/178] Tables: reloading a single table/file (#165) --- pydatview/GUISelectionPanel.py | 13 +++++++ pydatview/Tables.py | 62 +++++++++++++++++++++++++++++----- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 4f875cf..bc1dd53 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -293,6 +293,11 @@ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=Fal self.Append(item) self.Bind(wx.EVT_MENU, self.mainframe.onAdd, item) + if len(self.ISel)==1: + item = wx.MenuItem(self, -1, "Reload") + self.Append(item) + self.Bind(wx.EVT_MENU, self.OnReload, item) + if len(self.ISel)==1 and self.ISel[0]!=0: item = wx.MenuItem(self, -1, "Move Up") self.Append(item) @@ -346,6 +351,14 @@ def OnMoveTabUp(self, event=None): # Trigger a replot self.mainframe.onTabSelectionChange() + def OnReload(self, event=None): + # Reload table + # NOTE: position is forgotten.. + self.tabList.reloadOneTab(iTab=self.ISel[0]) + # Updating tables + self.selPanel.update_tabs(self.tabList) + # Trigger a replot + self.mainframe.onTabSelectionChange() def OnMoveTabDown(self, event=None): iOld = self.ISel[0] diff --git a/pydatview/Tables.py b/pydatview/Tables.py index b0f6238..5edb2c3 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -15,6 +15,7 @@ def __init__(self, tabs=None, options=None): if tabs is None: tabs =[] self._tabs = tabs + self.hasswap=False self.options = self.defaultOptions() if options is None else options @@ -78,6 +79,8 @@ def load_tables_from_files(self, filenames=[], fileformats=None, bAdd=False, bRe """ load multiple files into table list""" if not bAdd: self.clean() # TODO figure it out + if bReload: + self.hasswap=False if fileformats is None: fileformats=[None]*len(filenames) @@ -172,6 +175,37 @@ def _load_file_tabs(self, filename, fileformat=None, bReload=False): warn='Warn: No dataframe found in file: '+filename+'\n' return tabs, warn + def reloadOneTab(self, iTab, desired_fileformat=None): + filename = self._tabs[iTab].filename + if desired_fileformat is None: + fileformat = self._tabs[iTab].fileformat + else: + raise Exception('TODO figure how to prescirbe the file format on reload') + if filename is None or len(filename)==0: + raise Exception('Cannot reload Table as it was not set from a file') + + # Find all the tables that have the same filename. NOTE: some may have been deleted.. + ITab = [iTab for iTab, t in enumerate(self._tabs) if t.filename==filename] + + # Store list of names + OldNames = [t.name for t in self._tabs if t.filename==filename] + + # Load the file + tabs, warn = self._load_file_tabs(filename, fileformat=fileformat, bReload=False) + # Replace in tab list: + nTabs = len(tabs) + for i in range(nTabs): + if i>=len(ITab): + # we append + self._tabs.append(tabs[i]) + else: + # NOTE we assume that these tables are added succesively, + iTab = ITab[i] + self._tabs[iTab] = tabs[i] + if not self.hasswap: + # If swapped were used, we can't really reuse their old names + self._tabs[iTab].name = OldNames[i] + def haveSameColumns(self,I=None): if I is None: I=list(range(len(self._tabs))) @@ -192,6 +226,7 @@ def renameTable(self, iTab, newName): def swap(self, i1, i2): """ Swap two elements of the list""" + self.hasswap=True self._tabs[i1], self._tabs[i2] = self._tabs[i2], self._tabs[i1] def sort(self, method='byName'): @@ -499,7 +534,7 @@ class Table(object): # active_name : # raw_name : # filename : - def __init__(self,data=None, name='',filename='', fileformat=None, dayfirst=False): + def __init__(self, data=None, name='', filename='', fileformat=None, dayfirst=False): # Default init self.maskString='' self.mask=None @@ -512,9 +547,19 @@ def __init__(self,data=None, name='',filename='', fileformat=None, dayfirst=Fals self.fileformat_name = '' self.formulas = [] - if not isinstance(data,pd.DataFrame): + if not isinstance(data, pd.DataFrame): raise NotImplementedError('Tables that are not dataframe not implemented.') - # --- Modify input DataFrame + + # --- Modify and store input DataFrame + self.setData(data, dayfirst=dayfirst) + + # --- Trying to figure out how to name this table + if name is None or len(str(name))==0: + if data.columns.name is not None: + name=data.columns.name + self.setupName(name=str(name)) + + def setData(self, data, dayfirst=False): # Adding index if data.columns[0].lower().find('index')>=0: pass @@ -528,16 +573,17 @@ def __init__(self,data=None, name='',filename='', fileformat=None, dayfirst=Fals data=data.iloc[:,:-1] else: break - # --- Trying to figure out how to name this table - if name is None or len(str(name))==0: - if data.columns.name is not None: - name=data.columns.name + # --- Store in object self.data = data - self.setupName(name=str(name)) self.convertTimeColumns(dayfirst=dayfirst) + #def reload(self): + # Not Obvious how to do that for files thatreturn several tables + # if self.filename is None or len(self.filename)==0: + # raise Exception('Cannot reload Table, as it was not set from a file') + # print('>>> Table reload') def setupName(self,name=''): # Creates a "codename": path | basename | name | ext From 7bddd846787f50d6f898e7876355e80cce1e42c2 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 3 Oct 2023 21:13:26 -0600 Subject: [PATCH 138/178] StandardizeUnits: small fix for deg/s --- pydatview/plugins/data_standardizeUnits.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py index 7a69b84..0b664ee 100644 --- a/pydatview/plugins/data_standardizeUnits.py +++ b/pydatview/plugins/data_standardizeUnits.py @@ -79,6 +79,7 @@ def change_units_to_WE(s, c): svar, u = splitunit(s) u=u.lower() scalings = {} + # OLD = NEW scalings['rad/s'] = (30/np.pi,'rpm') # TODO decide scalings['rad' ] = (180/np.pi,'deg') scalings['n'] = (1e-3, 'kN') @@ -102,8 +103,10 @@ def change_units_to_SI(s, c): svar, u = splitunit(s) u=u.lower() scalings = {} + # OLD = NEW scalings['rpm'] = (np.pi/30,'rad/s') scalings['rad' ] = (180/np.pi,'deg') + scalings['deg/s' ] = (np.pi/180,'rad/s') scalings['kn'] = (1e3, 'N') scalings['knm'] = (1e3, 'Nm') scalings['kn-m'] = (1e3, 'Nm') From 70f42f286ec19de9993f36388b98d29406297b90 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 3 Oct 2023 21:25:10 -0600 Subject: [PATCH 139/178] Tables: transpose table (#140) --- pydatview/GUISelectionPanel.py | 15 +++++++++++++++ pydatview/Tables.py | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index bc1dd53..2e99db5 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -308,6 +308,11 @@ def __init__(self, parent, tabPanel, selPanel=None, mainframe=None, fullmenu=Fal self.Append(item) self.Bind(wx.EVT_MENU, self.OnMoveTabDown, item) + if len(self.ISel)>=1: + item = wx.MenuItem(self, -1, "Transpose") + self.Append(item) + self.Bind(wx.EVT_MENU, self.OnTransposeTabs, item) + if len(self.ISel)>1: item = wx.MenuItem(self, -1, "Merge (horizontally)") self.Append(item) @@ -369,6 +374,16 @@ def OnMoveTabDown(self, event=None): # Trigger a replot self.mainframe.onTabSelectionChange() + def OnTransposeTabs(self, event): + tabs = [self.tabList[i] for i in self.ISel] + for t in tabs: + t.transpose() + # Updating tables + self.selPanel.update_tabs(self.tabList) + # Trigger a replot + self.mainframe.onTabSelectionChange() + + def OnMergeTabs(self, event): # --- Figure out the common columns tabs = [self.tabList[i] for i in self.ISel] diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 5edb2c3..ee26279 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -579,6 +579,17 @@ def setData(self, data, dayfirst=False): self.data = data self.convertTimeColumns(dayfirst=dayfirst) + def transpose(self): + # Not done smartly.. + try: + df = self.data.drop(['Index'], axis=1) + except: + df = self.data + M = df.values.T + cols = ['C{}'.format(i) for i in range(M.shape[1])] + df = pd.DataFrame(data=M, columns=cols) + self.setData(df) + #def reload(self): # Not Obvious how to do that for files thatreturn several tables # if self.filename is None or len(self.filename)==0: From aa2c9d13428f7008400eb36b9ef4e16205d816de Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 5 Oct 2023 14:58:39 -0600 Subject: [PATCH 140/178] Window Resize, removing freeze (see #166) --- pydatview/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydatview/main.py b/pydatview/main.py index c9fba11..43df9dd 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -820,11 +820,11 @@ def OnIdle(self, event): self.mainFrameUpdateLayout() if hasattr(self,'plotPanel'): self.plotPanel.setSubplotTight() - self.Thaw() + #self.Thaw() # Commented see #166 def OnResizeWindow(self, event): self.resized = True - self.Freeze() + #self.Freeze() # Commented see #166 self.Layout() # --- Side column From 7852af98ac4269c862d5425408f708e34d4ace85 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Thu, 5 Oct 2023 15:34:52 -0600 Subject: [PATCH 141/178] Bug Fix: formulae didn't have access to numpy anymore --- pydatview/formulae.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pydatview/formulae.py b/pydatview/formulae.py index bf591b7..33ab061 100644 --- a/pydatview/formulae.py +++ b/pydatview/formulae.py @@ -1,4 +1,9 @@ from pydatview.common import no_unit + +# --- Give access to numpy and other useful functions for "eval" +import numpy as np +from numpy import cos, sin, exp, log, pi + # --------------------------------------------------------------------------------} # --- Formula # --------------------------------------------------------------------------------{ From 33e25c2f32cc0281cd14007017a557f8f04b754c Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Mon, 16 Oct 2023 20:31:37 -0600 Subject: [PATCH 142/178] Table sort: Bug fix: sorting by name was broken --- pydatview/GUISelectionPanel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 2e99db5..505b5b5 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -440,11 +440,11 @@ def OnExportTab(self, event): self.mainframe.exportTab(self.ISel[0]); def OnSort(self, event): - self.tabList.sort(method=method) + self.tabList.sort(method='byName') # Updating tables - self.update_tabs(self.tabList) + self.selPanel.update_tabs(self.tabList) # Trigger a replot - self.onTabSelectionChange() + self.selPanel.onTabSelectionChange() class ColumnPopup(wx.Menu): """ Popup Menu when right clicking on the column list """ From 920075616799e0439a4b41e10e75e194b4ef7b21 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 17 Oct 2023 10:07:30 -0600 Subject: [PATCH 143/178] OpenFAST: adding tool for renaming old DBG OUTS (OF2.3) --- pydatview/Tables.py | 18 +++++-- pydatview/plugins/__init__.py | 7 ++- pydatview/plugins/data_renameFldAero.py | 12 ++++- pydatview/plugins/data_renameOF23.py | 66 +++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 pydatview/plugins/data_renameOF23.py diff --git a/pydatview/Tables.py b/pydatview/Tables.py index ee26279..2466fd8 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -792,18 +792,28 @@ def convertTimeColumns(self, dayfirst=False): def renameColumn(self,iCol,newName): self.data.columns.values[iCol]=newName - def renameColumns(self, strReplDict=None): + def renameColumns(self, strReplDict=None, regReplDict=None): """ Rename all the columns of given table - strReplDict: a string replacement dictionary of the form: {'new':'old'} + - regReplDict: a regexp string replacement dictionary of the form: {'new':'old'} """ + cols = self.data.columns + newcols = [] if strReplDict is not None: - cols = self.data.columns - newcols = [] for c in cols: for new,old in strReplDict.items(): c = c.replace(old,new) newcols.append(c) - self.data.columns = newcols + elif regReplDict is not None: + import re + for c in cols: + for new,old in regReplDict.items(): + c = re.sub(old, new, c) + newcols.append(c) + else: + raise NotImplementedError('Provide a replace dictionary') + self.data.columns = newcols + def deleteColumns(self, ICol): """ Delete columns by index, not column names which can have duplicates""" diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index 87ab7ba..ff65431 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -60,6 +60,10 @@ def _data_renameFldAero(label, mainframe=None): from .data_renameFldAero import renameFldAeroAction return renameFldAeroAction(label, mainframe) +def _data_renameOF23(label, mainframe=None): + from .data_renameOF23 import renameOFChannelsAction + return renameOFChannelsAction(label, mainframe) + # --- Adding actions def _data_radialConcat(label, mainframe=None): from .data_radialConcat import radialConcatAction @@ -113,5 +117,6 @@ def _tool_curvefitting(*args, **kwargs): # DATA_PLUGINS_SIMPLE: simple data plugins constructors should return an Action OF_DATA_PLUGINS_SIMPLE=OrderedDict([ ('Radial Time Concatenation' , _data_radialConcat), - ('Rename "Fld" > "Aero' , _data_renameFldAero), + ('v3.4 - Rename "Fld" > "Aero' , _data_renameFldAero), + ('v2.3 - Rename "B*N* " > "AB*N* ' , _data_renameOF23), ]) diff --git a/pydatview/plugins/data_renameFldAero.py b/pydatview/plugins/data_renameFldAero.py index ab9b514..4f31376 100644 --- a/pydatview/plugins/data_renameFldAero.py +++ b/pydatview/plugins/data_renameFldAero.py @@ -30,11 +30,21 @@ def guiCallback(): tableFunctionCancel= renameAeroFld, guiCallback=guiCallback, mainframe=mainframe, # shouldnt be needed - data = data + data = data, + imports = _imports, + data_var = _data_var, + code = _code ) return action +# --------------------------------------------------------------------------------} +# --- Main methods +# --------------------------------------------------------------------------------{ +_imports=[] +_data_var='' +_code="""# TODO rename channels not implemented""" + def renameFldAero(tab, data=None): tab.renameColumns(strReplDict={'Aero':'Fld'}) # New:Old diff --git a/pydatview/plugins/data_renameOF23.py b/pydatview/plugins/data_renameOF23.py new file mode 100644 index 0000000..064b7f4 --- /dev/null +++ b/pydatview/plugins/data_renameOF23.py @@ -0,0 +1,66 @@ +import unittest +import numpy as np +from pydatview.common import splitunit +from pydatview.pipeline import ReversibleTableAction + +# --------------------------------------------------------------------------------} +# --- Action +# --------------------------------------------------------------------------------{ +def renameOFChannelsAction(label, mainframe=None): + """ + Return an "action" for the current plugin, to be used in the pipeline. + """ + guiCallback=None + data={} + if mainframe is not None: + # TODO TODO TODO Clean this up + def guiCallback(): + if hasattr(mainframe,'selPanel'): + mainframe.selPanel.colPanel1.setColumns() + mainframe.selPanel.colPanel2.setColumns() + mainframe.selPanel.colPanel3.setColumns() + mainframe.onTabSelectionChange() # trigger replot + if hasattr(mainframe,'pipePanel'): + pass + # Function that will be applied to all tables + + action = ReversibleTableAction( + name=label, + tableFunctionApply = renameToNew, + tableFunctionCancel= renameToOld, + guiCallback=guiCallback, + mainframe=mainframe, # shouldnt be needed + data = data, + imports = _imports, + data_var = _data_var, + code = _code + ) + + return action + +# --------------------------------------------------------------------------------} +# --- Main methods +# --------------------------------------------------------------------------------{ +_imports=[] +_data_var='' +_code="""# TODO rename channels not implemented""" + + +def renameToOld(tab, data=None): + tab.renameColumns( regReplDict={'B':'^AB(?=\dN)', 'AOA_':'Alpha_', 'AIn_':'AxInd_', 'ApI_':'TnInd_'}) # New:Old + +def renameToNew(tab, data=None): + tab.renameColumns( regReplDict={'AB':'^B(?=\dN)', 'Alpha_':'AOA_', 'AxInd_':'AIn_', 'TnInd_':'ApI_'}) # New:Old + + +class TestRenameOFChannels(unittest.TestCase): + + def test_change_units(self): + from pydatview.Tables import Table + tab = Table.createDummy(n=10, columns=['B1N001_[-]']) + renameToNew(tab) + self.assertEqual(tab.columns, ['Index','AB1N001_[-]']) + + +if __name__ == '__main__': + unittest.main() From 3bedca6cd67292889630c6c00fe8b59c9129d291 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 17 Oct 2023 10:26:43 -0600 Subject: [PATCH 144/178] Plot: Clean markers and measures (either or) --- pydatview/GUIPlotPanel.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index e872fb3..2e47c90 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -683,7 +683,7 @@ def setSubplotSpacing(self, init=False, tight=False): if self.subplotsPar is not None: if self.cbPlotMatrix.GetValue(): # TODO detect it - self.subplotsPar['right'] = 0.98 - subplotsPar['left'] + self.subplotsPar['right'] = 0.98 - self.subplotsPar['left'] # See GUIToolBox.py configure_toolbar self.fig.subplots_adjust(**self.subplotsPar) return @@ -741,12 +741,14 @@ def plot_matrix_event(self, event): def measure_event(self, event): if self.cbMeasure.IsChecked(): + # Can't measure and Mark points at the same time + self.cbMarkPt.SetValue(False) + self.cleanMarkers() # We do nothing, onMouseRelease will trigger the plot and setting - pass else: self.cleanMeasures() - # We redraw - self.redraw_same_data() + # We redraw after cleaning (measures or markers) + self.redraw_same_data() def setAndPlotMeasures(self, ax, x, y, which=None): if which is None: @@ -796,13 +798,17 @@ def cleanMeasures(self): self.lbDeltaY.SetLabel('') def markpt_event(self, event): + if self.cbMarkPt.IsChecked(): - self.markers = [] + # Can't measure and Mark points at the same time + self.cbMeasure.SetValue(False) + self.cleanMeasures() # We do nothing, onMouseRelease will trigger the plot and setting + self.markers = [] else: self.cleanMarkers() - # We redraw - self.redraw_same_data() + # We redraw after cleaning markesr or measures + self.redraw_same_data() def plotMarkers(self): for marker in self.markers: From 86deaa4a25612c6477653e7d28d1fc27248fd477 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 17 Oct 2023 11:47:02 -0600 Subject: [PATCH 145/178] Pluggins: removing tables for now --- pydatview/GUIPipelinePanel.py | 6 ++-- pydatview/fast/postpro.py | 2 +- pydatview/main.py | 2 +- pydatview/pipeline.py | 14 ++++---- pydatview/plugins/base_plugin.py | 6 ++-- pydatview/plugins/data_mask.py | 29 +++++++-------- pydatview/plugins/data_radialavg.py | 28 +++++++-------- pydatview/plugins/plotdata_binning.py | 22 ++++++------ pydatview/plugins/plotdata_filter.py | 22 ++++++------ pydatview/plugins/plotdata_sampler.py | 51 +++++++++++++++++++-------- 10 files changed, 104 insertions(+), 78 deletions(-) diff --git a/pydatview/GUIPipelinePanel.py b/pydatview/GUIPipelinePanel.py index f49b7a4..28d391b 100644 --- a/pydatview/GUIPipelinePanel.py +++ b/pydatview/GUIPipelinePanel.py @@ -190,10 +190,10 @@ def append(self, action, overwrite=True, apply=True, updateGUI=True, tabList=Non ac = self.find(action.name) if ac is not None: if ac.unique: - print('>>> Deleting unique action before inserting it again', ac.name) + #print('>>> Deleting unique action before inserting it again', ac.name) self.remove(ac, silent=True, updateGUI=False) # Add to pipeline - print('>>> GUIPipeline: Adding action',action.name) + #print('>>> GUIPipeline: Adding action',action.name) # Call parent class (data) Pipeline.append(self, action, overwrite=overwrite, apply=apply, updateGUI=updateGUI, tabList=tabList) # Add to GUI @@ -207,7 +207,7 @@ def append(self, action, overwrite=True, apply=True, updateGUI=True, tabList=Non def remove(self, action, silent=False, cancel=True, updateGUI=True, tabList=None): """ NOTE: the action is only removed from the pipeline, not deleted. """ - print('>>> Deleting action', action.name) + #print('>>> Deleting action', action.name) # Call parent class (data) Pipeline.remove(self, action, cancel=cancel, updateGUI=updateGUI, tabList=tabList) # Remove From GUI diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 02d8054..9ab4268 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -1147,7 +1147,7 @@ def spanwiseConcat(df): IdxAvailableForThisChannel = ColsInfoAD[ic]['Idx'] chanName = ColsInfoAD[ic]['name'] colName = ColsInfoAD[ic]['cols'][ir] - print('Channel {}: colName {}'.format(chanName, colName)) + #print('Channel {}: colName {}'.format(chanName, colName)) try: if ir+1 in IdxAvailableForThisChannel: data[ir*nt:(ir+1)*nt, ic+2] = df[colName].values diff --git a/pydatview/main.py b/pydatview/main.py index 43df9dd..78085e4 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -946,7 +946,7 @@ def InitLocale(self): # See Bug #128 - Issue with wxPython 4.1 on Windows import locale locale.setlocale(locale.LC_ALL, "C") - print('[INFO] Setting locale to C') + #print('[INFO] Setting locale to C') #self.SetAssertMode(wx.APP_ASSERT_SUPPRESS) # Try this # --------------------------------------------------------------------------------} diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index a48eb10..b579dbb 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -85,7 +85,7 @@ def apply(self, tabList, force=False, applyToAll=False): raise Exception('{}: cannot apply on None tabList'.format(self)) for t in tabList: - print('>>> Applying action', self.name, 'to', t.nickname) + #print('>>> Applying action', self.name, 'to', t.nickname) try: # TODO TODO TODO Collect errors here self.tableFunctionApply(t, data=self.data) @@ -201,12 +201,12 @@ def apply(self, tabList, force=False, applyToAll=False): if force: self.applied = False if self.applied: - print('>>> Action: Skipping irreversible action', self.name) + #print('>>> Action: Skipping irreversible action', self.name) return Action.apply(self, tabList) def cancel(self, *args, **kwargs): - print('>>> Action: Cancel: skipping irreversible action', self.name) + #print('>>> Action: Cancel: skipping irreversible action', self.name) pass def __repr__(self): @@ -222,7 +222,7 @@ def apply(self, tabList, force=False, applyToAll=False): if force: self.applied = False if self.applied: - print('>>> Action: Apply: Skipping irreversible action', self.name) + #print('>>> Action: Apply: Skipping irreversible action', self.name) return Action.apply(self, tabList) @@ -232,7 +232,7 @@ def cancel(self, tabList): print('[WARN] Cannot cancel action {} on None tablist'.format(self)) return for t in tabList: - print('>>> Action: Cancel: ', self, 'to', t.nickname) + #print('>>> Action: Cancel: ', self, 'to', t.nickname) try: self.tableFunctionCancel(t, data=self.data) except Exception as e: @@ -252,7 +252,7 @@ def apply(self, tabList, force=False, applyToAll=False): if force: self.applied = False if self.applied: - print('>>> Action: Skipping Adder action', self.name) + #print('>>> Action: Skipping Adder action', self.name) return # Call parent function applyAndAdd dfs_new, names_new, errors = Action.applyAndAdd(self, tabList) @@ -426,7 +426,7 @@ def remove(self, a, cancel=True, updateGUI=True, tabList=None): # Cancel the action in Editor if a.guiEditorObj is not None: try: - print('[Pipe] Canceling action in guiEditor because the action is removed') + #print('[Pipe] Canceling action in guiEditor because the action is removed') a.guiEditorObj.cancelAction() # NOTE: should not trigger a plot except: print('[FAIL] Pipeline: Failed to call cancelAction() in GUI.') diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py index ac9ca70..9f5cb93 100644 --- a/pydatview/plugins/base_plugin.py +++ b/pydatview/plugins/base_plugin.py @@ -12,7 +12,7 @@ from pydatview.plotdata import PlotData from pydatview.pipeline import AdderAction -TOOL_BORDER=15 +TOOL_BORDER=5 # --------------------------------------------------------------------------------} # --- Default class for tools @@ -67,7 +67,7 @@ class ActionEditor(GUIToolPanel): - the action data - a set of function handles to process some triggers and callbacks """ - def __init__(self, parent, action, buttons=None, tables=True): + def __init__(self, parent, action, buttons=None, tables=False): GUIToolPanel.__init__(self, parent) # --- Data @@ -172,7 +172,7 @@ def onHelp(self,event=None): # --------------------------------------------------------------------------------{ class PlotDataActionEditor(ActionEditor): - def __init__(self, parent, action, buttons=None, tables=True): + def __init__(self, parent, action, buttons=None, tables=False): """ """ ActionEditor.__init__(self, parent, action=action) diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index 545bff5..da10d6b 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -103,8 +103,8 @@ def __init__(self, parent, action=None): self.btAdd = self.getBtBitmap(self, u'Mask (add)','add' , self.onAdd) self.btApply = self.getToggleBtBitmap(self, 'Apply','cloud', self.onToggleApply) - self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) - self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better + #self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) + #self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better self.lb = wx.StaticText( self, -1, """(Example of mask: "({Time}>100) && ({Time}<50) && ({WS}==5)" or "{Date} > '2018-10-01'" or "['substring' in str(x) for x in {string_variable}]")""") self.textMask = wx.TextCtrl(self, wx.ID_ANY, 'Dummy', style = wx.TE_PROCESS_ENTER) @@ -119,14 +119,14 @@ def __init__(self, parent, action=None): btSizer.Add(self.btApply ,0,flag = wx.ALL|wx.EXPAND, border = 1) row_sizer = wx.BoxSizer(wx.HORIZONTAL) - row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) - row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) - row_sizer.Add(wx.StaticText(self, -1, 'Mask:'), 0, wx.CENTER|wx.LEFT, 5) - row_sizer.Add(self.textMask, 1, wx.CENTER|wx.LEFT|wx.EXPAND, 5) + #row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) + #row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Mask:'), 0, wx.CENTER|wx.LEFT|wx.ALIGN_CENTER_VERTICAL, 1) + row_sizer.Add(self.textMask, 1, wx.CENTER|wx.LEFT|wx.RIGHT, 1) vert_sizer = wx.BoxSizer(wx.VERTICAL) - vert_sizer.Add(self.lb ,0, flag = wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) - vert_sizer.Add(row_sizer ,1, flag = wx.EXPAND|wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 5) + vert_sizer.Add(self.lb ,0, flag = wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 4) + vert_sizer.Add(row_sizer ,1, flag = wx.EXPAND|wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM, border = 3) self.sizer = wx.BoxSizer(wx.HORIZONTAL) self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) @@ -135,7 +135,7 @@ def __init__(self, parent, action=None): # --- Events # NOTE: getBtBitmap and getToggleBtBitmap already specify the binding - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + #self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) self.textMask.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeAndPressEnter) # --- Init triggers @@ -160,6 +160,7 @@ def guessMask(self): def onParamChangeAndPressEnter(self, event=None): # We apply if self.data['active']: + self._GUI2Data() self.action.apply(self.tabList, force=True) self.action.updateGUI() # We call the guiCallback else: @@ -168,7 +169,7 @@ def onParamChangeAndPressEnter(self, event=None): # --- Table related def onTabChange(self,event=None): - iSel=self.cbTabs.GetSelection() + #iSel=self.cbTabs.GetSelection() # TODO need a way to retrieve "data" from action, perTab if iSel==0: maskString = self.tabList.commonMaskString # for "all" @@ -180,10 +181,10 @@ def onTabChange(self,event=None): def updateTabList(self,event=None): tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() #try: - iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) + #iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) + #self.cbTabs.Clear() + #[self.cbTabs.Append(tn) for tn in tabListNames] + #self.cbTabs.SetSelection(iSel) #except RuntimeError: # pass diff --git a/pydatview/plugins/data_radialavg.py b/pydatview/plugins/data_radialavg.py index 2326bd3..0e5ce53 100644 --- a/pydatview/plugins/data_radialavg.py +++ b/pydatview/plugins/data_radialavg.py @@ -103,9 +103,9 @@ def __init__(self, parent, action=None): self.btClose = self.getBtBitmap(self,'Close' ,'close' , self.destroy) self.btAdd = self.getBtBitmap(self,'Average','compute', self.onAdd) # ART_PLUS - self.lb = wx.StaticText( self, -1, """Select tables, averaging method and average parameter (`Period` methods uses the `azimuth` signal) """) - self.cbTabs = wx.ComboBox(self, choices=[], style=wx.CB_READONLY) - self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better + self.lb = wx.StaticText( self, -1, """Select averaging method and average parameter (`Period` methods uses the `azimuth` signal) """) + #self.cbTabs = wx.ComboBox(self, choices=[], style=wx.CB_READONLY) + #self.cbTabs.Enable(False) # <<< Cancelling until we find a way to select tables and action better self.cbMethod = wx.ComboBox(self, choices=sAVG_METHODS, style=wx.CB_READONLY) @@ -118,16 +118,16 @@ def __init__(self, parent, action=None): btSizer.Add(self.btAdd ,0, flag = wx.ALL|wx.EXPAND, border = 1) row_sizer = wx.BoxSizer(wx.HORIZONTAL) - row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) - row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) - row_sizer.Add(wx.StaticText(self, -1, 'Method:'), 0, wx.CENTER|wx.LEFT, 5) + #row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) + #row_sizer.Add(self.cbTabs , 0, wx.CENTER|wx.LEFT, 2) + row_sizer.Add(wx.StaticText(self, -1, 'Method:'), 0, wx.ALIGN_LEFT|wx.CENTER|wx.LEFT, 0) row_sizer.Add(self.cbMethod , 0, wx.CENTER|wx.LEFT, 2) row_sizer.Add(wx.StaticText(self, -1, 'Param:') , 0, wx.CENTER|wx.LEFT, 5) - row_sizer.Add(self.textAverageParam , 0, wx.CENTER|wx.LEFT|wx.RIGHT| wx.EXPAND, 2) + row_sizer.Add(self.textAverageParam , 0, wx.CENTER|wx.LEFT|wx.RIGHT, 2) vert_sizer = wx.BoxSizer(wx.VERTICAL) - vert_sizer.Add(self.lb ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) - vert_sizer.Add(row_sizer ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 5) + vert_sizer.Add(self.lb ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 4) + vert_sizer.Add(row_sizer ,0, flag =wx.ALIGN_LEFT|wx.TOP|wx.BOTTOM,border = 4) self.sizer = wx.BoxSizer(wx.HORIZONTAL) self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 5) @@ -136,7 +136,7 @@ def __init__(self, parent, action=None): # --- Events # NOTE: getBtBitmap and getToggleBtBitmap already specify the binding - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + #self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) # --- Init triggers self._Data2GUI() @@ -153,10 +153,10 @@ def onTabChange(self,event=None): def updateTabList(self,event=None): tabListNames = ['All opened tables']+self.tabList.getDisplayTabNames() #try: - iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) - self.cbTabs.Clear() - [self.cbTabs.Append(tn) for tn in tabListNames] - self.cbTabs.SetSelection(iSel) + #iSel=np.min([self.cbTabs.GetSelection(),len(tabListNames)]) + #self.cbTabs.Clear() + #[self.cbTabs.Append(tn) for tn in tabListNames] + #self.cbTabs.SetSelection(iSel) #except RuntimeError: # pass diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index beb76d0..8a1d796 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -95,14 +95,14 @@ class BinningToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): import wx - PlotDataActionEditor.__init__(self, parent, action, tables=True, **kwargs) + PlotDataActionEditor.__init__(self, parent, action, tables=False, **kwargs) # --- GUI elements self.scBins = wx.SpinCtrl(self, value='50', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(60,-1) ) self.textXMin = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) self.textXMax = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) - self.btXRange = self.getBtBitmap(self, 'Default','compute', self.reset) + self.btXRange = self.getBtBitmap(self, 'Update x','compute', self.reset) self.lbDX = wx.StaticText(self, -1, '') self.scBins.SetRange(3, 10000) @@ -111,12 +111,14 @@ def __init__(self, parent, action, **kwargs): lbInputs.SetFont(boldFont) # --- Layout - msizer = wx.FlexGridSizer(rows=1, cols=3, hgap=2, vgap=0) - msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + #msizer = wx.FlexGridSizer(rows=1, cols=3, hgap=2, vgap=0) + #msizer.Add(wx.StaticText(self, -1, ' ') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + #msizer.Add(wx.StaticText(self, -1, ' ') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + #msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + #msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) # msizer.Add(self.btXRange , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.LEFT, 1) - msizer2 = wx.FlexGridSizer(rows=2, cols=5, hgap=2, vgap=1) + msizer2 = wx.FlexGridSizer(rows=2, cols=5, hgap=4, vgap=1) msizer2.Add(lbInputs , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 0) msizer2.Add(wx.StaticText(self, -1, '#bins: ') , 0, wx.LEFT|wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL , 1) @@ -131,8 +133,8 @@ def __init__(self, parent, action, **kwargs): #msizer2.AddGrowableCol(4,1) vsizer = wx.BoxSizer(wx.VERTICAL) - vsizer.Add(msizer,0, flag = wx.TOP ,border = 1) - vsizer.Add(msizer2,0, flag = wx.TOP|wx.EXPAND ,border = 1) + #vsizer.Add(msizer,0, flag = wx.TOP ,border = 1) + vsizer.Add(msizer2,0, flag = wx.TOP,border = 1) #self.sizer = wx.BoxSizer(wx.HORIZONTAL) #self.sizer.Add(btSizer ,0, flag = wx.LEFT , border = 5) @@ -143,7 +145,7 @@ def __init__(self, parent, action, **kwargs): self.scBins.Bind (wx.EVT_SPINCTRL , self.onParamChangeArrow) self.scBins.Bind (wx.EVT_SPINCTRLDOUBLE, self.onParamChangeArrow) self.scBins.Bind (wx.EVT_TEXT_ENTER, self.onParamChangeEnter) - self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) + #self.cbTabs.Bind (wx.EVT_COMBOBOX, self.onTabChange) self.textXMin.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter) self.textXMax.Bind(wx.EVT_TEXT_ENTER, self.onParamChangeEnter) @@ -205,7 +207,7 @@ def onAdd(self,event=None): return # TODO put this in GUI2Data??? - iSel = self.cbTabs.GetSelection() + #iSel = self.cbTabs.GetSelection() icol, colname = self.plotPanel.selPanel.xCol self.data['icol'] = icol self.data['colname'] = colname diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index 468f0ef..a59a1e9 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -72,7 +72,7 @@ class FilterToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): import wx # avoided at module level for unittests - PlotDataActionEditor.__init__(self, parent, action, tables=True, **kwargs) + PlotDataActionEditor.__init__(self, parent, action, tables=False, **kwargs) # --- Data from pydatview.tools.signal_analysis import FILTERS @@ -86,25 +86,25 @@ def __init__(self, parent, action, **kwargs): self.lbInfo = wx.StaticText( self, -1, '') # --- Layout - horzSizerT = wx.BoxSizer(wx.HORIZONTAL) - horzSizerT.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) - horzSizerT.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) + #horzSizerT = wx.BoxSizer(wx.HORIZONTAL) + #horzSizerT.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) + #horzSizerT.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) horzSizer = wx.BoxSizer(wx.HORIZONTAL) - horzSizer.Add(wx.StaticText(self, -1, 'Filter:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) + horzSizer.Add(wx.StaticText(self, -1, 'Filter:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 0) horzSizer.Add(self.cbFilters , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) horzSizer.Add(self.lbParamName , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 5) horzSizer.Add(self.tParam , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.LEFT, 1) vertSizer = wx.BoxSizer(wx.VERTICAL) - vertSizer.Add(self.lbInfo ,0, flag = wx.LEFT ,border = 5) - vertSizer.Add(horzSizerT ,1, flag = wx.LEFT|wx.EXPAND,border = 1) - vertSizer.Add(horzSizer ,1, flag = wx.LEFT|wx.EXPAND,border = 1) + vertSizer.Add(self.lbInfo ,0, flag = wx.TOP ,border = 4) + #vertSizer.Add(horzSizerT ,1, flag = wx.LEFT|wx.EXPAND,border = 1) + vertSizer.Add(horzSizer ,1, flag = wx.TOP|wx.EXPAND,border = 6) # self.sizer = wx.BoxSizer(wx.HORIZONTAL) # self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) - self.sizer.Add(vertSizer ,1, flag = wx.EXPAND|wx.LEFT ,border = 1) + self.sizer.Add(vertSizer ,1, flag = wx.LEFT,border = 6) self.sizer.Layout() @@ -187,7 +187,7 @@ def _Data2GUI(self, data=None): def onAdd(self, event=None): # TODO put this in GUI2Data??? - iSel = self.cbTabs.GetSelection() + #iSel = self.cbTabs.GetSelection() icol, colname = self.plotPanel.selPanel.xCol self.data['icol'] = icol @@ -202,7 +202,7 @@ def onHelp(self,event=None): To filter perform the following step: -- Chose a filtering method: +- Choose a filtering method: - Moving average: apply a moving average filter, with a length specified by the window size (in indices) - High pass 1st order: apply a first oder high-pass filter, diff --git a/pydatview/plugins/plotdata_sampler.py b/pydatview/plugins/plotdata_sampler.py index 4c5cc26..40c6d81 100644 --- a/pydatview/plugins/plotdata_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -67,11 +67,13 @@ class SamplerToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): import wx - PlotDataActionEditor.__init__(self, parent, action, tables=True) + PlotDataActionEditor.__init__(self, parent, action, tables=False) # --- Data from pydatview.tools.signal_analysis import SAMPLERS self._SAMPLERS_USER = SAMPLERS.copy() + self.xmin=0 + self.xmax=1 # --- GUI elements self.cbMethods = wx.ComboBox(self, -1, choices=[s['name'] for s in self._SAMPLERS_USER], style=wx.CB_READONLY) @@ -80,17 +82,26 @@ def __init__(self, parent, action, **kwargs): self.textNewX = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER) self.textOldX = wx.TextCtrl(self, wx.ID_ANY|wx.TE_READONLY) self.textOldX.Enable(False) + self.textOldX2 = wx.TextCtrl(self, wx.ID_ANY|wx.TE_READONLY) + self.textOldX2.Enable(False) + + self.btXRange = self.getBtBitmap(self, 'Update x','compute', self.setCurrentX) # --- Layout - msizer = wx.FlexGridSizer(rows=2, cols=4, hgap=2, vgap=0) - msizer.Add(wx.StaticText(self, -1, 'Table:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(wx.StaticText(self, -1, 'Current x: '), 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) - msizer.Add(self.textOldX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) + msizer = wx.FlexGridSizer(rows=3, cols=4, hgap=2, vgap=0) msizer.Add(wx.StaticText(self, -1, 'Method:') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) msizer.Add(self.cbMethods , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) msizer.Add(self.lbNewX , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) msizer.Add(self.textNewX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) + msizer.Add(wx.StaticText(self, -1, ' ') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.btXRange , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + #msizer.Add(self.cbTabs , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(wx.StaticText(self, -1, 'Current x: '), 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.textOldX , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) + msizer.Add(wx.StaticText(self, -1, ' ') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(wx.StaticText(self, -1, ' ') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(wx.StaticText(self, -1, ' ') , 0, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM, 1) + msizer.Add(self.textOldX2 , 1, wx.ALIGN_LEFT|wx.ALIGN_CENTER_VERTICAL|wx.TOP|wx.BOTTOM|wx.EXPAND, 1) msizer.AddGrowableCol(3,1) #self.sizer = wx.BoxSizer(wx.HORIZONTAL) @@ -115,17 +126,25 @@ def getSamplerIndex(self, name): return i return -1 - def setCurrentX(self): + def setCurrentX(self, event=None): if len(self.plotPanel.plotData)==0: return - x= np.array(self.plotPanel.plotData[0].x).astype(str) + x= np.array(self.plotPanel.plotData[0].x) if len(x)<50: - s=', '.join(x) + s=', '.join(x.astype(str)) else: - s=', '.join(x[[0,1,2,3]]) + s=', '.join(x[[0,1,2,3]].astype(str)) s+=', ..., ' - s+=', '.join(x[[-3,-2,-1]]) + s+=', '.join(x[[-3,-2,-1]].astype(str)) self.textOldX.SetValue(s) + self.xmin = np.min(x) + self.xmax = np.max(x) + self.textOldX2.SetValue('xmin={}, xmax={}, length={}'.format(self.xmin, self.xmax, len(x))) + # Update linspace... + self._GUI2Data() + if self.data['name'] == 'Linspace': + self.data['param']= [self.xmin, self.xmax, int(self.data['param'][2])] + self.textNewX.SetValue('{} , {}, {:d}'.format(*self.data['param'])) def onMethodChange(self, event=None, init=True): """ Select the method, but does not applied it to the plotData @@ -134,6 +153,9 @@ def onMethodChange(self, event=None, init=True): """ iOpt = self.cbMethods.GetSelection() opts = self._SAMPLERS_USER[iOpt] + # Update linspace... + if opts['name'] == 'Linspace': + opts['param']=[self.xmin, self.xmax, int(opts['param'][2])] # check if selection is the same as the one currently used if self.data['name'] == opts['name']: opts['param'] = self.data['param'] @@ -181,10 +203,9 @@ def onToggleApply(self, event=None, init=False): # self.setCurrentX() - def onAdd(self, event=None): # TODO put this in GUI2Data??? - iSel = self.cbTabs.GetSelection() + #iSel = self.cbTabs.GetSelection() icol, colname = self.plotPanel.selPanel.xCol self.data['icol'] = icol @@ -199,11 +220,13 @@ def onHelp(self,event=None): To resample perform the following step: -- Chose a resampling method: +- Choose a resampling method: - replace: specify all the new x-values - insert : insert a list of x values to the existing ones - delete : delete a list of x values from the existing ones - every-n : use every n values + - linspace : a linear spacing, insert a xmin, xmax and number of values + (works only when x is increasing monotically) - time-based: downsample using sample averaging or upsample using linear interpolation, x-axis must already be in seconds - delta x : specify a delta for uniform spacing of x values From e56619fc04b640adc3bdfe5b6c7dc84293476a2b Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 17 Oct 2023 11:54:37 -0600 Subject: [PATCH 146/178] Export figure: fixed menu --- pydatview/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatview/main.py b/pydatview/main.py index 78085e4..f81bef3 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -673,7 +673,7 @@ def cleanGUI(self, event=None): def onSave(self, event=None): # using the navigation toolbar save functionality - self.plotPanel.navTB.save_figure() + self.plotPanel.navTBBottom.save_figure() def onAbout(self, event=None): io_userpath = os.path.join(weio.defaultUserDataDir(), 'pydatview_io') From 02a50ebf5fa624aa7e6abbed8a92eb91228f06ed Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 17 Oct 2023 16:23:16 -0600 Subject: [PATCH 147/178] Plot Markers: different IDs and colors --- pydatview/GUIMeasure.py | 11 +- pydatview/GUIPlotPanel.py | 16 +- pydatview/fast/postpro.py | 6 +- pydatview/tools/colors.py | 463 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 486 insertions(+), 10 deletions(-) create mode 100644 pydatview/tools/colors.py diff --git a/pydatview/GUIMeasure.py b/pydatview/GUIMeasure.py index a5c46b3..c47bdd0 100644 --- a/pydatview/GUIMeasure.py +++ b/pydatview/GUIMeasure.py @@ -1,11 +1,14 @@ import numpy as np -from pydatview.common import isString, isDate, getDt, pretty_time, pretty_num, pretty_date, isDateScalar +from pydatview.common import isString, isDate, getDt, pretty_time, pretty_num_short, pretty_date, isDateScalar class GUIMeasure: - def __init__(self, index, color): + def __init__(self, index, color, ID=None): # Main data self.index = index + if ID is None: + ID = self.index + self.ID = ID self.color = color self.P_target_raw = None # closest x-y point stored in "raw" form (including datetime) self.P_target_num = None # closest x-y point stored in "num" form (internal matplotlib xy) @@ -88,7 +91,7 @@ def compute(self, PD): def plotAnnotation(self, ax, xc, yc): #self.clearAnnotation() - sAnnotation = '{}: ({}, {})'.format(self.index, formatValue(xc), formatValue(yc)) + sAnnotation = '{}: ({}, {})'.format(self.ID, formatValue(xc), formatValue(yc)) bbox_args = dict(boxstyle='round', fc='0.9', alpha=0.75) annotation = ax.annotate(sAnnotation, (xc, yc), xytext=(5, -2), textcoords='offset points', color=self.color, bbox=bbox_args) self.annotations.append(annotation) @@ -211,7 +214,7 @@ def formatValue(value): elif isString(value): return value else: - return pretty_num(value) + return pretty_num_short(value).strip() except TypeError: return '' diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 2e47c90..122ed35 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -267,6 +267,7 @@ def plotType(self): def regular_select(self, event=None): self.clear_measures() + self.parent.cleanMarkers() self.parent.cbLogY.SetValue(False) # self.parent.spcPanel.Hide(); @@ -280,6 +281,7 @@ def regular_select(self, event=None): def compare_select(self, event=None): self.clear_measures() + self.parent.cleanMarkers() self.parent.cbLogY.SetValue(False) self.parent.show_hide(self.parent.cmpPanel, self.cbCompare.GetValue()) self.parent.spcPanel.Hide(); @@ -290,6 +292,7 @@ def compare_select(self, event=None): def fft_select(self, event=None): self.clear_measures() + self.parent.cleanMarkers() self.parent.show_hide(self.parent.spcPanel, self.cbFFT.GetValue()) self.parent.cbLogY.SetValue(self.cbFFT.GetValue()) self.parent.pdfPanel.Hide(); @@ -299,6 +302,7 @@ def fft_select(self, event=None): def pdf_select(self, event=None): self.clear_measures() + self.parent.cleanMarkers() self.parent.cbLogX.SetValue(False) self.parent.cbLogY.SetValue(False) self.parent.show_hide(self.parent.pdfPanel, self.cbPDF.GetValue()) @@ -310,6 +314,7 @@ def pdf_select(self, event=None): def minmax_select(self, event): self.clear_measures() + self.parent.cleanMarkers() self.parent.cbLogY.SetValue(False) self.parent.show_hide(self.parent.mmxPanel, self.cbMinMax.GetValue()) self.parent.spcPanel.Hide(); @@ -904,10 +909,14 @@ def onMouseRelease(self, event): return if event.button == 1: # We add a marker - # from pydatview.tools.colors import fColrs, python_colors + from pydatview.tools.colors import fColrs, python_colors n = len(self.markers) - #marker = GUIMeasure(1, python_colors(n+1)) - marker = GUIMeasure(1, 'firebrick') + IDs = set([m.ID for m in self.markers]) + All = set(np.arange(1,n+2)) + ID = list(All.difference(IDs))[0] # Should be only of size 1 + #marker = GUIMeasure(1, python_colors(n+1), ID=ID) + marker = GUIMeasure(1, fColrs(ID, cmap='darker'), ID=ID) + #marker = GUIMeasure(1, 'firebrick', ID=ID) #GUIMeasure(2, 'darkgreen') self.markers.append(marker) marker.setAndPlot(self.fig.axes, ax, x, y, self.plotData) @@ -1664,6 +1673,7 @@ def redraw_same_data(self, force_autoscale=False): autoscale = (self.cbAutoScale.IsChecked()) or (force_autoscale) self.plot_all(autoscale=autoscale) + self.plotMarkers() self.canvas.draw() diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 9ab4268..24855e5 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -962,7 +962,7 @@ def spanwisePostProRows(df, FST_In=None): Cols=df.columns.values if r_AD is not None: ColsInfoAD, nrMaxAD = spanwiseColAD(Cols) - if r_ED is not None: + if r_ED_bld is not None: ColsInfoED, nrMaxED = spanwiseColED(Cols) if r_BD is not None: ColsInfoBD, nrMaxBD = spanwiseColBD(Cols) @@ -975,9 +975,9 @@ def spanwisePostProRows(df, FST_In=None): M_AD = np.zeros((len(v), len(dfRad_AD), len(dfRad_AD.columns))) Col_AD=dfRad_AD.columns.values M_AD[i, :, : ] = dfRad_AD.values - if r_ED is not None and len(r_ED)>0: + if r_ED_bld is not None and len(r_ED_bld)>0: dfRad_ED = extract_spanwise_data(ColsInfoED, nrMaxED, df=None, ts=df.iloc[i]) - dfRad_ED = insert_spanwise_columns(dfRad_ED, r_ED, R=R, IR=IR_ED) + dfRad_ED = insert_spanwise_columns(dfRad_ED, r_ED_bld, R=R, IR=IR_ED) if i==0: M_ED = np.zeros((len(v), len(dfRad_ED), len(dfRad_ED.columns))) Col_ED=dfRad_ED.columns.values diff --git a/pydatview/tools/colors.py b/pydatview/tools/colors.py new file mode 100644 index 0000000..135f556 --- /dev/null +++ b/pydatview/tools/colors.py @@ -0,0 +1,463 @@ +import numpy as np +import matplotlib as mpl +import matplotlib.pyplot as plt +import matplotlib.colors as mcolors +import itertools +from colorsys import rgb_to_hls as rgb_to_hls_scalar +from colorsys import hls_to_rgb as hls_to_rgb_scalar +import unittest + +# --------------------------------------------------------------------------------} +# --- COLOR TOOLS +# --------------------------------------------------------------------------------{ +def rgb2hex(C,g=None,b=None): + if len(C)==3 : + r=C[0] + g=C[1] + b=C[2] + if r<1.1 and g<1.1 and b<1.1: + r=r*255 + g=g*255 + b=b*255 + + return '#%02X%02X%02X' % (r,g,b) + +def adjust_color_lightness_scalar(r, g, b, factor): + """ + r,g,b between 0 and 1 + factor between 0 and +infty, but lightness bounded between 0 and 1 + """ + h, l, s = rgb_to_hls_scalar(r, g, b) + l = max(min(l * factor, 1.0), 0.0) + r, g, b = hls_to_rgb_scalar(h, l, s) + return r,g,b + +def adjust_color_lightness(rgb, factor): + """ + r,g,b between 0 and 1 + factor between 0 and +infty, but lightness bounded between 0 and 1 + """ + hls = rgb_to_hls(rgb) + hls[...,1] = np.multiply(hls[...,1] , factor) + hls[ hls[...,1]<0, 1] = 0 + hls[ hls[...,1]>1, 1] = 1 + rgb = hls_to_rgb(hls) + return rgb + + +def lighten_color(rgb, factor=0.1): + if factor ==0: + return rgb + return adjust_color_lightness(rgb, 1 + factor) + +def darken_color(rgb, factor=0.1): + return adjust_color_lightness(rgb, 1 - factor) + +# --------------------------------------------------------------------------------} +# --- COLOR MAPS +# --------------------------------------------------------------------------------{ +def python_colors(i=None): + if i is None: + return plt.rcParams['axes.prop_cycle'].by_key()['color'] + else: + Colrs=plt.rcParams['axes.prop_cycle'].by_key()['color'] + return Colrs[ np.mod(i,len(Colrs)) ] + +# ---- ColorMap +def make_colormap(seq,values=None,name='CustomMap'): + """Return a LinearSegmentedColormap + seq: RGB-tuples. + values: corresponding values (location betwen 0 and 1) + + cmap=make_colormap([MW_Blue, [1.0,1.0,1.0],MW_Red]) + """ + hasAlpha=len(seq[0])==4 + if hasAlpha: + nComp=4 + else: + nComp=3 + + n=len(seq) + if values is None: + values=np.linspace(0,1,n) + + doubled = list(itertools.chain.from_iterable(itertools.repeat(s, 2) for s in seq)) + doubled[0] = (None,)* nComp + doubled[-1] = (None,)* nComp + cdict = {'red': [], 'green': [], 'blue': []} + if hasAlpha: + cdict['alpha']=[] + for i,v in enumerate(values): + if hasAlpha: + r1, g1, b1, a1 = doubled[2*i] + r2, g2, b2, a2 = doubled[2*i + 1] + else: + r1, g1, b1 = doubled[2*i] + r2, g2, b2 = doubled[2*i + 1] + cdict['red'].append((v, r1, r2)) + cdict['green'].append((v, g1, g2)) + cdict['blue'].append((v, b1, b2)) + if hasAlpha: + cdict['alpha'].append((v, a1, a2)) + print(cdict) + return mcolors.LinearSegmentedColormap(name, cdict) + + +def color_scales(n, color='blue'): + maps={ + 'blue':mpl.cm.Blues, + 'purple':mpl.cm.Purples, + 'orange':mpl.cm.Oranges, + 'red':mpl.cm.Reds, + 'green':mpl.cm.Greens, + } + norm = mpl.colors.Normalize(vmin=0, vmax=n) + cmap = mpl.cm.ScalarMappable(norm=norm, cmap=maps[color]) + cmap.set_array([]) + return [cmap.to_rgba(i) for i in np.arange(n)] + + + +# --- Diverging Brown Green +DV_BG=[np.array([140,81,10])/255., np.array([191,129,4])/255., np.array([223,194,1])/255., np.array([246,232,1])/255., np.array([245,245,2])/255., np.array([199,234,2])/255., np.array([128,205,1])/255., np.array([53,151,14])/255., np.array([1,102,94 ])/255.] + +# --- Diverging Red Blue +DV_RB=[ np.array([215,48,39])/255., np.array([244,109,67])/255., np.array([253,174,97])/255., np.array([254,224,144])/255., np.array([255,255,191])/255., np.array([224,243,248])/255., np.array([171,217,233])/255., np.array([116,173,209])/255., np.array([69,117,180])/255.] + +# --- Diverging Purple Green +DV_PG=[np.array([118,42,131])/255., np.array([153,112,171])/255., np.array([194,165,207])/255., np.array([231,212,232])/255., np.array([247,247,247])/255., np.array([217,240,211])/255., np.array([166,219,160])/255., np.array([90,174,97])/255., np.array([27,120,55])/255.] + +# Maureen Stone, for line plots +MW_Light_Blue = np.array([114,147,203])/255. +MW_Light_Orange = np.array([225,151,76])/255. +MW_Light_Green = np.array([132,186,91])/255. +MW_Light_Red = np.array([211,94,96])/255. +MW_Light_Gray = np.array([128,133,133])/255. +MW_Light_Purple = np.array([144,103,167])/255. +MW_Light_DarkRed = np.array([171,104,87])/255. +MW_Light_Kaki = np.array([204,194,16])/255. + +MW_Blue = np.array([57,106,177])/255. +MW_Orange = np.array([218,124,48])/255. +MW_Green = np.array([62,150,81])/255. +MW_Red = np.array([204,37,41])/255. +MW_Gray = np.array([83,81,84])/255. +MW_Purple = np.array([107,76,154])/255. +MW_DarkRed = np.array([146,36,40])/255. +MW_Kaki = np.array([148,139,61])/255. + +MathematicaBlue = np.array([63 ,63 ,153 ])/255.; +MathematicaRed = np.array([153,61 ,113 ])/255.; +MathematicaGreen = np.array([61 ,153,86 ])/255.; +MathematicaYellow = np.array([152,140,61 ])/255.; +MathematicaLightBlue = np.array([159,159,204 ])/255.; +MathematicaLightRed = np.array([204,158,184 ])/255.; +MathematicaLightGreen = np.array([158,204,170 ])/255.; +# +ManuDarkBlue = np.array([0 ,0 ,0.7 ]) ; +ManuDarkRed = np.array([138 ,42 ,93 ])/255.; +# ManuDarkOrange = np.array([245 ,131,1 ])/255.; +ManuDarkOrange = np.array([198 ,106,1 ])/255.; +ManuLightOrange = np.array([255.,212,96 ])/255.; +# +Red = np.array([1 ,0 ,0]); +Blue = np.array([0 ,0 ,1]); +Green = np.array([0 ,0.6,0]); +Yellow = np.array([0.8,0.8,0]); + +MatlabGreen = np.array([0 ,0.5 ,1 ]); +MatlabCyan = np.array([0.0e+0 ,750.0e-03,750.0e-03]); +MatlabMagenta = np.array([ 750.0e-03,0.0e+0 ,750.0e-03]); +MatlabYellow = np.array([750.0e-03 ,750.0e-03,0.0e+0 ]); +MatlabGrey = np.array([250.0e-03 ,250.0e-03,250.0e-03]); + +# cRed=plt.cm.Reds(np.linspace(0.9,1,2)) +# cGreen=plt.cm.Greens(np.linspace(0.9,1,2)) +# cPur=plt.cm.Purples(np.linspace(0.9,1,2)) +# cGray=plt.cm.Greys(np.linspace(0.9,1,2)) + +# --- Mathematica darkrainbow colormap: +darkrainbow=mcolors.LinearSegmentedColormap('CustomMap', + {'red': [[0.0, None, 0.23529411764705882], [0.1111111111111111, 0.25098039215686274, 0.25098039215686274], [0.2222222222222222, 0.2627450980392157, 0.2627450980392157], [0.3333333333333333, 0.2901960784313726, 0.2901960784313726], [0.4444444444444444, 0.41568627450980394, 0.41568627450980394], [0.5555555555555556, 0.6235294117647059, 0.6235294117647059], [0.6666666666666666, 0.8117647058823529, 0.8117647058823529], [0.7777777777777777, 0.8745098039215686, 0.8745098039215686], [0.8888888888888888, 0.807843137254902, 0.807843137254902], [1.0, 0.7294117647058823, None]], + 'green': [[0.0, None, 0.33725490196078434], [0.1111111111111111, 0.3411764705882353, 0.3411764705882353], [0.2222222222222222, 0.4196078431372549, 0.4196078431372549], [0.3333333333333333, 0.4745098039215686, 0.4745098039215686], [0.4444444444444444, 0.5529411764705883, 0.5529411764705883], [0.5555555555555556, 0.6705882352941176, 0.6705882352941176], [0.6666666666666666, 0.7647058823529411, 0.7647058823529411], [0.7777777777777777, 0.7294117647058823, 0.7294117647058823], [0.8888888888888888, 0.5019607843137255, 0.5019607843137255], [1.0, 0.23921568627450981, None]] , + 'blue': [[0.0, None, 0.5725490196078431], [0.1111111111111111, 0.5568627450980392, 0.5568627450980392], [0.2222222222222222, 0.3843137254901961, 0.3843137254901961], [0.3333333333333333, 0.27058823529411763, 0.27058823529411763], [0.4444444444444444, 0.23921568627450981, 0.23921568627450981], [0.5555555555555556, 0.2627450980392157, 0.2627450980392157], [0.6666666666666666, 0.30196078431372547, 0.30196078431372547], [0.7777777777777777, 0.3254901960784314, 0.3254901960784314], [0.8888888888888888, 0.2980392156862745, 0.2980392156862745], [1.0, 0.22745098039215686, None]]}) +# NOTE: generated with: +MathematicaDarkRainbow=[(60 /255,86 /255,146/255), (64 /255,87 /255,142/255), (67 /255,107/255,98 /255), (74 /255,121/255,69 /255), (106/255,141/255,61 /255), (159/255,171/255,67 /255), (207/255,195/255,77 /255), (223/255,186/255,83 /255), (206/255,128/255,76 /255), (186/255,61 /255,58 /255)] +# darkrainbow2= make_colormap(MathematicaDarkRainbow) +# --- Another rainbow: +# Colrs=[(152/255,0,0),(152/255,69 /255,0 /255), (167/255,127/255,3 /255), (12 /255,137/255,0 /255), (0 /255,75 /255,131/255)] +# Colrs.reverse() + +def fColrs_hex(*args): + return rgb2hex(fColrs(*args)) + +def fGray(x): + return [x,x,x] + +def fColrs(i=-1, n=-1, bBW=True, cmap=None): + # Possible calls + # M=fColrs() : returns a nx3 matrix of RBG colors + # C=fColrs(i) : cycle through colors, modulo the number of color + # G=fColrs(i,n) : return a grey color (out of n), where i=1 is black + # % Thrid argument add a switch possibility between black and white or colors: + # % G=fColrs(i,n,1) : return a grey color (out of n), where i=1 is black + # % G=fColrs(i,n,0) : cycle through colors + + # Table of Color used + if cmap is None: + mcolrs=np.array([ + MathematicaBlue, + MathematicaGreen, + ManuDarkRed, + ManuDarkOrange, + MathematicaLightBlue, + MathematicaLightGreen, + MathematicaLightRed, + ManuLightOrange, + Blue, + Green, + Red, + Yellow, + MatlabCyan, + MatlabMagenta ]); + elif cmap=='darker': + mcolrs=np.array( + ['firebrick','darkgreen', MathematicaBlue, ManuDarkOrange], dtype=object ) + else: + raise NotImplementedError() + + # + if i==-1: + return mcolrs + elif (i!=-1 and n==-1): + return mcolrs[np.mod(i-1,len(mcolrs))]; + elif (i!=-1 and n!=-1): + if bBW: + if n==1: + return [0,0,0] + else: + return [0.55,0.55,0.55]*(v-1)/(n-1); #grayscale + else: + return mcolrs[mod(i-1,len(mcolrs,1))] + else: + return mcolrs + +# --------------------------------------------------------------------------------} +# --- Colorbar +# --------------------------------------------------------------------------------{ +def manual_colorbar(fig, cmap, norm=None, **kwargs): + """ Adds a colorbar to a plot without linking it to a specific axis """ + if norm is None: + norm=mcolors.Normalize(vmin=0, vmax=1) + sm = plt.cm.ScalarMappable(norm=norm, cmap=cmap) + sm.set_array([]) + return fig.colorbar(sm, **kwargs) + +# +def test_colrs(): + from matplotlib import pyplot as plt + + x=np.linspace(0,2*np.pi,100); + plt.figure() + plt.title('fig_python') + plt.grid() + for i in range(30): + plt.plot(x,np.sin(x)+i,'-',color=fColrs(i)) + plt.xlabel('x coordinate [m]') + plt.ylabel('Velocity U_i [m/s]') + plt.xlim([0,2*pi]) + + plt.show() + + +def rgb_to_hls(rgb): + """ + convert float rgb values (in the range [0, 1]), in a numpy array to hls + values. + + Parameters + ---------- + rgb : (..., 3) array-like or tuple + All values must be in the range [0, 1] + + Returns + ------- + hls : (..., 3) ndarray + Colors converted to hls values in range [0, 1] + """ + # --- Handling of arguments + rgb = np.asarray(rgb) + # check length of the last dimension, should be _some_ sort of rgb + if rgb.shape[-1] != 3: + raise ValueError("Last dimension of input array must be 3; " + "shape {shp} was found.".format(shp=rgb.shape)) + # if we got passed a 1D array, try to treat as a single color and reshape as needed + in_ndim = rgb.ndim + if in_ndim == 1: + rgb = np.array(rgb, ndmin=2) + + hls = np.zeros_like(rgb) # We rely on zeros values for hue and saturation + + maxc = rgb.max(-1) # maxc = max(r, g, b) + minc = rgb.min(-1) # minc = min(r, g, b) + delta = rgb.ptp(-1) # max-min + + # --- Lightness + hls[...,1] = (minc+maxc)/2.0 + # --- Saturation (HLS) + idx = (maxc > 0) & (delta > 0) + hls[idx,2] = delta[idx]/(1-np.abs(2*hls[idx,1]-1)) # s=(max-min)/(1-|2l-1|) + # --- Hue + ipos = delta > 0 + # red is max + idx = (rgb[..., 0] == maxc) & ipos + hls[idx, 0] = (rgb[idx, 1] - rgb[idx, 2]) / delta[idx] + # green is max + idx = (rgb[..., 1] == maxc) & ipos + hls[idx, 0] = 2. + (rgb[idx, 2] - rgb[idx, 0]) / delta[idx] + # blue is max + idx = (rgb[..., 2] == maxc) & ipos + hls[idx, 0] = 4. + (rgb[idx, 0] - rgb[idx, 1]) / delta[idx] + hls[..., 0] = (hls[..., 0] / 6.0) % 1.0 + + if in_ndim == 1: + hls.shape = (3,) + return hls + + +def hls_to_rgb(hls): + """ + convert hls values in a numpy array to rgb values + all values assumed to be in range [0, 1] + + Parameters + ---------- + hls : (..., 3) array-like or tuple + All values assumed to be in range [0, 1] + + Returns + ------- + rgb : (..., 3) ndarray + Colors converted to RGB values in range [0, 1] + """ + hls = np.asarray(hls) + + # check length of the last dimension, should be _some_ sort of rgb + if hls.shape[-1] != 3: + raise ValueError("Last dimension of input array must be 3; " + "shape {shp} was found.".format(shp=hls.shape)) + + # if we got passed a 1D array, try to treat as + # a single color and reshape as needed + in_ndim = hls.ndim + if in_ndim == 1: + hls = np.array(hls, ndmin=2) + + # make sure we don't have an int image + hls = hls.astype(np.promote_types(hls.dtype, np.float32)) + + h = hls[..., 0] + l = hls[..., 1] + s = hls[..., 2] + + r = np.zeros_like(h) + g = np.zeros_like(h) + b = np.zeros_like(h) + + c = (1-np.abs(2*l-1))*s + hp = h*6.0 + x = c * (1-np.abs( (hp % 2) -1 )) + i = hp.astype(int) + m = l-c/2 + + idx = i % 6 == 0 + r[idx] = c[idx] + g[idx] = x[idx] + + idx = i == 1 + r[idx] = x[idx] + g[idx] = c[idx] + + idx = i == 2 + g[idx] = c[idx] + b[idx] = x[idx] + + idx = i == 3 + g[idx] = x[idx] + b[idx] = c[idx] + + idx = i == 4 + r[idx] = x[idx] + b[idx] = c[idx] + + idx = i == 5 + r[idx] = c[idx] + b[idx] = x[idx] + + r=r+m + g=g+m + b=b+m + + rgb = np.stack([r, g, b], axis=-1) + + if in_ndim == 1: + rgb.shape = (3,) + + return rgb + + +class TestColors(unittest.TestCase): + def test_rgb_hsv(self): + # --- Test Data + RGB=np.zeros((11,3)) + RGB[0,:]=np.array([0.0,0.0,0.0]) + RGB[1,:]=np.array([1.0,1.0,1.0]) + RGB[2,:]=np.array([0.3,0.5,0.4]) + RGB[3,:]=np.array([0.9,0.9,0.9]) + RGB[4,:]=np.array([0.1,0.9,0.9]) + RGB[5,:]=np.array([0.9,0.1,0.9]) + RGB[6,:]=np.array([0.9,0.9,0.1]) + RGB[7,:]=np.array([0.9,0.1,0.1]) + RGB[8,:]=np.array([0.1,0.9,0.1]) + RGB[9,:]=np.array([0.1,0.1,0.9]) + RGB[10,:]=np.array([0.1,0.1,0.1]) + # --- Converting back and forth, + HLS = rgb_to_hls(RGB) + RGB2 = hls_to_rgb(HLS) + np.testing.assert_almost_equal(RGB,RGB2) + + # --- Comparing results with scalar version + for i in np.arange(RGB.shape[0]): + h,l,s=rgb_to_hls_scalar(RGB[i,0],RGB[i,1],RGB[i,2]) + np.testing.assert_array_equal(HLS[i,:],[h,l,s]) + # --- Calling with tuple + hls_arr = rgb_to_hls((0.3,0.5,0.4)) + hls_tuple_ref = rgb_to_hls_scalar(0.3,0.5,0.4) + np.testing.assert_equal(hls_arr,np.asarray(hls_tuple_ref)) + + def test_adjust_lightness(self): + RGB=np.zeros((10,3)) + RGB[0,:]=np.array([0.0,0.0,0.0]) + RGB[1,:]=np.array([1.0,1.0,1.0]) + RGB[2,:]=np.array([0.3,0.5,0.4]) + RGB[3,:]=np.array([0.9,0.9,0.9]) + RGB[4,:]=np.array([0.1,0.9,0.9]) + RGB[5,:]=np.array([0.9,0.1,0.9]) + RGB[6,:]=np.array([0.9,0.9,0.1]) + RGB[7,:]=np.array([0.9,0.1,0.1]) + RGB[8,:]=np.array([0.1,0.9,0.1]) + RGB[9,:]=np.array([0.1,0.1,0.9]) + factor=1.5 + RGB_out=adjust_color_lightness(RGB,factor) + for i in np.arange(RGB.shape[0]): + r,g,b=adjust_color_lightness_scalar(RGB[i,0],RGB[i,1],RGB[i,2],factor) + np.testing.assert_almost_equal(RGB_out[i,:],[r,g,b]) + # --- Calling with tuple + rgb_in = (0.1,0.3,0.5) + rgb_arr = adjust_color_lightness(rgb_in,factor) + rgb_tuple_ref = adjust_color_lightness_scalar(rgb_in[0],rgb_in[1],rgb_in[2],factor) + np.testing.assert_almost_equal(rgb_arr,np.asarray(rgb_tuple_ref)) + +if __name__ == "__main__": +# test_colrs() + unittest.main() From d63727d245406efd2d97dfde05d0da9b2ed11983 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 17 Oct 2023 17:14:13 -0600 Subject: [PATCH 148/178] Equation: Destroy equation editor dialog --- pydatview/GUISelectionPanel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 505b5b5..05dbd7c 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -562,7 +562,6 @@ def showFormulaDialog(self, title, name='', formula='', edit=False): if dlg.ShowModal()==wx.ID_OK: sName = dlg.name.GetValue() sFormula = dlg.formula.GetValue() - dlg.Destroy() if len(self.ISel)>0: iFilt=self.ISel[-1] iFull=self.parent.Filt2Full[iFilt] @@ -592,6 +591,7 @@ def showFormulaDialog(self, title, name='', formula='', edit=False): bValid=True else: bCancelled=True + dlg.Destroy() if bCancelled: return if bValid: From ac12adbab5ebf263092bbf4cf086d2a8c7043d52 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 17 Oct 2023 17:28:30 -0600 Subject: [PATCH 149/178] Equation: use end modal to quit equation editor --- pydatview/GUISelectionPanel.py | 80 ++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 05dbd7c..6d3d44a 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -87,12 +87,12 @@ def __init__(self, title='', name='', formula='',columns=[],unit='',xcol='',xuni - self.btOK = wx.Button(self, wx.ID_OK)#, label = "OK" ) + btOK = wx.Button(self, wx.ID_OK)#, label = "OK" ) btCL = wx.Button(self,label = "Cancel") bt_sizer = wx.BoxSizer(wx.HORIZONTAL) - bt_sizer.Add(self.btOK, 0 ,wx.ALL,5) + bt_sizer.Add(btOK, 0 ,wx.ALL,5) bt_sizer.Add(btCL, 0 ,wx.ALL,5) - #btOK.Bind(wx.EVT_BUTTON,self.onOK ) + btOK.Bind(wx.EVT_BUTTON,self.onOK ) btCL.Bind(wx.EVT_BUTTON,self.onCancel) @@ -261,8 +261,11 @@ def onQuickFormula(self, event): else: raise Exception('Unknown quick formula {}'.s) + def onOK(self, event): + self.EndModal(wx.ID_OK) + def onCancel(self, event): - self.Destroy() + self.EndModal(wx.ID_CANCEL) # --------------------------------------------------------------------------------} # --- Popup menus # --------------------------------------------------------------------------------{ @@ -435,6 +438,7 @@ def OnRenameTab(self, event): if dlg.ShowModal() == wx.ID_OK: newName=dlg.GetValue() self.mainframe.renameTable(self.ISel[0],newName) + dlg.Destroy() def OnExportTab(self, event): self.mainframe.exportTab(self.ISel[0]); @@ -504,6 +508,7 @@ def OnRenameColumn(self, event=None): self.parent.updateColumn(iFilt,newName) #faster self.parent.selPanel.updateLayout() # a trigger for the plot is required but skipped for now + dlg.Destroy() def OnEditColumn(self, event): if len(self.ISel) != 1: @@ -557,41 +562,40 @@ def showFormulaDialog(self, title, name='', formula='', edit=False): xcol = no_unit(xcol) while (not bValid) and (not bCancelled): - dlg = FormulaDialog(title=title,columns=columns,xcol=xcol,xunit=xunit,unit=main_unit,name=sName,formula=sFormula) - dlg.CentreOnParent() - if dlg.ShowModal()==wx.ID_OK: - sName = dlg.name.GetValue() - sFormula = dlg.formula.GetValue() - if len(self.ISel)>0: - iFilt=self.ISel[-1] - iFull=self.parent.Filt2Full[iFilt] + with FormulaDialog(title=title,columns=columns,xcol=xcol,xunit=xunit,unit=main_unit,name=sName,formula=sFormula) as dlg: + dlg.CentreOnParent() + if dlg.ShowModal()==wx.ID_OK: + sName = dlg.name.GetValue() + sFormula = dlg.formula.GetValue() + if len(self.ISel)>0: + iFilt=self.ISel[-1] + iFull=self.parent.Filt2Full[iFilt] + else: + iFull = -1 + + ITab,STab = self.selPanel.getSelectedTables() + #if main.tabList.haveSameColumns(ITab): + sError='' + nError=0 + haveSameColumns= self.selPanel.tabList.haveSameColumns(ITab) + for iTab,sTab in zip(ITab,STab): + if haveSameColumns or self.parent.tab.active_name == sTab: + # apply formula to all tables with same columns, otherwise only to active table + if edit: + bValid=self.selPanel.tabList[iTab].setColumnByFormula(sName,sFormula,iFull) + iOffset = 0 # we'll stay on this column that we are editing + else: + bValid=self.selPanel.tabList[iTab].addColumnByFormula(sName,sFormula,iFull) + iOffset = 1 # we'll select this newly created column + if not bValid: + sError+='The formula didn''t eval for table {}\n'.format(sTab) + nError+=1 + if len(sError)>0: + Error(self.parent,sError) + if nError0: - Error(self.parent,sError) - if nError Date: Tue, 17 Oct 2023 17:42:42 -0600 Subject: [PATCH 150/178] Table: remember old column names before transpose --- pydatview/Tables.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 2466fd8..18d6bba 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -538,6 +538,7 @@ def __init__(self, data=None, name='', filename='', fileformat=None, dayfirst=Fa # Default init self.maskString='' self.mask=None + self.columns_pre_transpose = None self.filename = filename self.fileformat = fileformat @@ -580,13 +581,22 @@ def setData(self, data, dayfirst=False): self.convertTimeColumns(dayfirst=dayfirst) def transpose(self): - # Not done smartly.. + # Not done smartly, likely to duplicate memory.. try: df = self.data.drop(['Index'], axis=1) except: df = self.data M = df.values.T cols = ['C{}'.format(i) for i in range(M.shape[1])] + if self.columns_pre_transpose is None: + # It's the first time we transpose, we backup the columns + self.columns_pre_transpose = df.columns.copy() + else: + # We have transposed before, we restore the columns + if len(self.columns_pre_transpose) == len(cols): + cols = self.columns_pre_transpose + self.columns_pre_transpose = None + df = pd.DataFrame(data=M, columns=cols) self.setData(df) From 76f6677b6e0407dfba12431f194fa38436d944e4 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 10:24:07 -0600 Subject: [PATCH 151/178] Tables: Fix issue when columns are not string --- pydatview/Tables.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 18d6bba..4c87046 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -561,6 +561,9 @@ def __init__(self, data=None, name='', filename='', fileformat=None, dayfirst=Fa self.setupName(name=str(name)) def setData(self, data, dayfirst=False): + # sanitize columns, we only accept strings + data.columns = data.columns.astype(str) + # Adding index if data.columns[0].lower().find('index')>=0: pass From 61e363ecdba30ba0b4f1a06a0c59e4822ea81a0b Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 10:25:47 -0600 Subject: [PATCH 152/178] PlotData: partially handling weird data types (mix date / nan) --- pydatview/common.py | 13 ++- pydatview/io/matlabmat_file.py | 1 - pydatview/plotdata.py | 162 ++++++++++++++++++++++----------- 3 files changed, 118 insertions(+), 58 deletions(-) diff --git a/pydatview/common.py b/pydatview/common.py index da5ff35..2cf0761 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -355,12 +355,15 @@ def pretty_date(d, timespan=None): return s def pretty_num(x): - if np.isnan(x): + try: + if np.isnan(x): + return 'NA' + if abs(x)<1000 and abs(x)>1e-4: + return "{:9.4f}".format(x) + else: + return '{:.3e}'.format(x) + except: return 'NA' - if abs(x)<1000 and abs(x)>1e-4: - return "{:9.4f}".format(x) - else: - return '{:.3e}'.format(x) def pretty_num_short(x,digits=3): if digits==4: diff --git a/pydatview/io/matlabmat_file.py b/pydatview/io/matlabmat_file.py index e1aff75..2ed922d 100644 --- a/pydatview/io/matlabmat_file.py +++ b/pydatview/io/matlabmat_file.py @@ -63,7 +63,6 @@ def read(self, filename=None, **kwargs): raise EmptyFileError('File is empty:',self.filename) mfile = scipy.io.loadmat(self.filename) - import pdb; pdb.set_trace() def write(self, filename=None): """ Rewrite object to file, or write object to `filename` if provided """ diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 7dce907..4efd16d 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -136,7 +136,7 @@ def _post_init(PD, pipeline=None): PD.y0 =PD.y def __repr__(s): - s1='id:{}, it:{}, ix:{}, iy:{}, sx:"{}", sy:"{}", st:{}, syl:{}\n'.format(s.id,s.it,s.ix,s.iy,s.sx,s.sy,s.st,s.syl) + s1='id:{}, it:{}, ix:{}, iy:{}, sx:"{}", sy:"{}", st:{}, syl:{}'.format(s.id,s.it,s.ix,s.iy,s.sx,s.sy,s.st,s.syl) #s1='id:{}, it:{}, sx:"{}", xyMeas:{}\n'.format(s.id,s.it,s.sx,s.xyMeas) return s1 @@ -305,7 +305,6 @@ def computeRange(PD): PD._xAtYMin = PD._xAtYMinCalc(PD._yMin[0]) PD._xAtYMax = PD._xAtYMaxCalc(PD._yMax[0]) - # --------------------------------------------------------------------------------} # --- Stats functions that should only becalled once, could maybe use @attributes.. # --------------------------------------------------------------------------------{ @@ -315,8 +314,11 @@ def _yMinCalc(PD): elif PD.yIsDate: return PD.y[0],'{}'.format(PD.y[0]) else: - v=np.nanmin(PD.y) - s=pretty_num(v) + try: + v=np.nanmin(PD.y) + s=pretty_num(v) + except: + return np.nan, 'NA' return (v,s) def _yMaxCalc(PD): @@ -325,8 +327,11 @@ def _yMaxCalc(PD): elif PD.yIsDate: return PD.y[-1],'{}'.format(PD.y[-1]) else: - v=np.nanmax(PD.y) - s=pretty_num(v) + try: + v=np.nanmax(PD.y) + s=pretty_num(v) + except: + return np.nan, 'NA' return (v,s) def _xAtYMinCalc(PD, yMin): @@ -361,8 +366,11 @@ def _xMinCalc(PD): elif PD.xIsDate: return PD.x[0],'{}'.format(PD.x[0]) else: - v=np.nanmin(PD.x) - s=pretty_num(v) + try: + v=np.nanmin(PD.x) + s=pretty_num(v) + except: + return np.nan, 'NA' return (v,s) def _xMaxCalc(PD): @@ -371,8 +379,11 @@ def _xMaxCalc(PD): elif PD.xIsDate: return PD.x[-1],'{}'.format(PD.x[-1]) else: - v=np.nanmax(PD.x) - s=pretty_num(v) + try: + v=np.nanmax(PD.x) + s=pretty_num(v) + except: + return np.nan, 'NA' return (v,s) def xMin(PD): @@ -415,24 +426,33 @@ def yMean(PD): if PD.yIsString or PD.yIsDate: return None,'NA' else: - v=np.nanmean(PD.y) - s=pretty_num(v) + try: + v=np.nanmean(PD.y) + s=pretty_num(v) + except: + return np.nan, 'NA' return (v,s) def yMedian(PD): if PD.yIsString or PD.yIsDate: return None,'NA' else: - v=np.nanmedian(PD.y) - s=pretty_num(v) + try: + v=np.nanmedian(PD.y) + s=pretty_num(v) + except: + return np.nan, 'NA' return (v,s) def yStd(PD): if PD.yIsString or PD.yIsDate: return None,'NA' else: - v=np.nanstd(PD.y) - s=pretty_num(v) + try: + v=np.nanstd(PD.y) + s=pretty_num(v) + except: + return np.nan, 'NA' return (v,s) def yName(PD): @@ -480,18 +500,23 @@ def yRange(PD): dtAll=getDt([PD.x[0],PD.x[-1]]) return np.nan,pretty_time(dtAll) else: - v=np.nanmax(PD.y)-np.nanmin(PD.y) - s=pretty_num(v) - return v,s + try: + v=np.nanmax(PD.y)-np.nanmin(PD.y) + s=pretty_num(v) + return v,s + except: + return np.nan,'NA' def yAbsMax(PD): if PD.yIsString or PD.yIsDate: return 'NA','NA' else: - v=max(np.abs(PD._y0Min[0]),np.abs(PD._y0Max[0])) - s=pretty_num(v) - return v,s - + try: + v=max(np.abs(PD._y0Min[0]),np.abs(PD._y0Max[0])) + s=pretty_num(v) + return v,s + except: + return np.nan,'NA' def xRange(PD): if PD.xIsString: @@ -500,51 +525,71 @@ def xRange(PD): dtAll=getDt([PD.x[0],PD.x[-1]]) return np.nan,pretty_time(dtAll) else: - v=np.nanmax(PD.x)-np.nanmin(PD.x) - s=pretty_num(v) - return v,s + try: + v=np.nanmax(PD.x)-np.nanmin(PD.x) + s=pretty_num(v) + return v,s + except: + return np.nan, 'NA' def inty(PD): if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: return None,'NA' else: - v=np.trapz(y=PD.y,x=PD.x) - s=pretty_num(v) - return v,s + try: + v=np.trapz(y=PD.y,x=PD.x) + s=pretty_num(v) + return v,s + except: + return np.nan, 'NA' def intyintdx(PD): if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: return None,'NA' else: - v=np.trapz(y=PD.y,x=PD.x)/np.trapz(y=PD.x*0+1,x=PD.x) - s=pretty_num(v) - return v,s + try: + v=np.trapz(y=PD.y,x=PD.x)/np.trapz(y=PD.x*0+1,x=PD.x) + s=pretty_num(v) + return v,s + except: + return np.nan, 'NA' def intyx1(PD): if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: return None,'NA' else: - v=np.trapz(y=PD.y*PD.x,x=PD.x) - s=pretty_num(v) - return v,s + try: + v=np.trapz(y=PD.y*PD.x,x=PD.x) + s=pretty_num(v) + return v,s + except: + return np.nan, 'NA' + def intyx1_scaled(PD): if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: return None,'NA' else: - v=np.trapz(y=PD.y*PD.x,x=PD.x) - v=v/np.trapz(y=PD.y,x=PD.x) - s=pretty_num(v) - return v,s + try: + v=np.trapz(y=PD.y*PD.x,x=PD.x) + v=v/np.trapz(y=PD.y,x=PD.x) + s=pretty_num(v) + return v,s + except: + return np.nan, 'NA' def intyx2(PD): if PD.yIsString or PD.yIsDate or PD.xIsString or PD.xIsDate: return None,'NA' else: - v=np.trapz(y=PD.y*PD.x**2,x=PD.x) - s=pretty_num(v) - return v,s + try: + v=np.trapz(y=PD.y*PD.x**2,x=PD.x) + s=pretty_num(v) + return v,s + except: + return np.nan, 'NA' + # --------------------------------------------------------------------------------} # --- Measure - TODO: cleanup @@ -649,9 +694,13 @@ def dx(PD): dt=getDt(PD.x) return dt,pretty_time(dt) else: - v=PD.x[1]-PD.x[0] - s=pretty_num(v) - return v,s + try: + v=PD.x[1]-PD.x[0] + s=pretty_num(v) + return v,s + except: + print('plotdata: computing dx failed for {}'.format(PD)) + return np.nan, 'NA' def xMax(PD): if PD.xIsString: @@ -659,18 +708,24 @@ def xMax(PD): elif PD.xIsDate: return PD.x[-1],'{}'.format(PD.x[-1]) else: - v=np.nanmax(PD.x) - s=pretty_num(v) - return v,s + try: + v=np.nanmax(PD.x) + s=pretty_num(v) + return v,s + except: + return np.nan, 'NA' def xMin(PD): if PD.xIsString: return PD.x[0],PD.x[0] elif PD.xIsDate: return PD.x[0],'{}'.format(PD.x[0]) else: - v=np.nanmin(PD.x) - s=pretty_num(v) - return v,s + try: + v=np.nanmin(PD.x) + s=pretty_num(v) + return v,s + except: + return np.nan, 'NA' def leq(PD, m, method=None): from pydatview.tools.fatigue import equivalent_load @@ -684,7 +739,10 @@ def leq(PD, m, method=None): except ModuleNotFoundError: print('[INFO] module fatpack not installed, default to windap method for equivalent load') method='rainflow_windap' - v=equivalent_load(PD.x, PD.y, m=m, Teq=1, bins=100, method=method) + try: + v = equivalent_load(PD.x, PD.y, m=m, Teq=1, bins=100, method=method) + except: + v = np.nan return v,pretty_num(v) def Info(PD,var): From bd36efd0dbe659d26054048c49562bb25a7ff3a5 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 11:38:04 -0600 Subject: [PATCH 153/178] Exception: handling before main loop, and pretty print them --- pydatview/GUIInfoPanel.py | 78 +++++++++++++++++++++------------------ pydatview/common.py | 7 +++- pydatview/main.py | 71 +++++++++++++++++++++++------------ 3 files changed, 96 insertions(+), 60 deletions(-) diff --git a/pydatview/GUIInfoPanel.py b/pydatview/GUIInfoPanel.py index e67f151..e55c2ba 100644 --- a/pydatview/GUIInfoPanel.py +++ b/pydatview/GUIInfoPanel.py @@ -343,41 +343,49 @@ def showStats(self,PD,plotType='Regular',erase=True): def _showStats(self,erase=True): self.Freeze() - selCols=[c for c in self.Cols if c['s']] - # Adding columns - if erase: - self.clean() - AL={'L':wx.LIST_FORMAT_LEFT,'R':wx.LIST_FORMAT_RIGHT,'C':wx.LIST_FORMAT_CENTER} - for i,c in enumerate(selCols): - if c['s']: - self.tbStats.InsertColumn(i,c['name'], AL[c['al']]) - # Inserting items - index = self.tbStats.GetItemCount() - for pd in self.PD: - for j,c in enumerate(selCols): - # Calling dedicated function: either a function handle, f(PD), or a method, pd.m. - if 'm' in c.keys(): - v,sv=getattr(pd,c['m'])() - else: - v,sv=c['f'](pd) - - # Insert items - try: - if j==0: - self.tbStats.InsertItem(index, sv) - else: - self.tbStats.SetItem(index, j,sv) - except: - if j==0: - self.tbStats.InsertStringItem(index, sv) - else: - self.tbStats.SetStringItem(index, j,sv) - index +=1 - - for i in range(self.tbStats.GetColumnCount()): - self.tbStats.SetColumnWidth(i, wx.LIST_AUTOSIZE_USEHEADER) - self.tbStats.RefreshRows() - self.Thaw() + try: + selCols=[c for c in self.Cols if c['s']] + # Adding columns + if erase: + self.clean() + AL={'L':wx.LIST_FORMAT_LEFT,'R':wx.LIST_FORMAT_RIGHT,'C':wx.LIST_FORMAT_CENTER} + for i,c in enumerate(selCols): + if c['s']: + self.tbStats.InsertColumn(i,c['name'], AL[c['al']]) + # Inserting items + index = self.tbStats.GetItemCount() + for pd in self.PD: + for j,c in enumerate(selCols): + # Calling dedicated function: either a function handle, f(PD), or a method, pd.m. + try: + if 'm' in c.keys(): + v,sv=getattr(pd,c['m'])() + else: + v,sv=c['f'](pd) + except: + # print statement because developper should fix this.. + print('GUIInfoPanel: Stat {} failed'.format(c['name'])) + v= np.nan + sv='NA' + + # Insert items + try: + if j==0: + self.tbStats.InsertItem(index, sv) + else: + self.tbStats.SetItem(index, j,sv) + except: + if j==0: + self.tbStats.InsertStringItem(index, sv) + else: + self.tbStats.SetStringItem(index, j,sv) + index +=1 + + for i in range(self.tbStats.GetColumnCount()): + self.tbStats.SetColumnWidth(i, wx.LIST_AUTOSIZE_USEHEADER) + self.tbStats.RefreshRows() + finally: + self.Thaw() def setPlotMatrixCallbacks(self, callback_left, callback_right): diff --git a/pydatview/common.py b/pydatview/common.py index 2cf0761..721bec7 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -445,11 +445,14 @@ def Error(parent, message, caption = 'Error!'): dlg.ShowModal() dlg.Destroy() -def exception2string(excp, iMax=40, prefix=' | '): +def exception2string(excp, iMax=40, prefix=' | ', prevStack=True): if isinstance(excp, PyDatViewException): return prefix + excp.args[0] else: - stack = traceback.extract_stack()[:-3] + traceback.extract_tb(excp.__traceback__) + stack=[] + if prevStack: + stack += traceback.extract_stack()[:-3] + stack += traceback.extract_tb(excp.__traceback__) stacklist = traceback.format_list(stack) # --- Parse stacktrace for file/ line / content traceback_dicts=[] diff --git a/pydatview/main.py b/pydatview/main.py index f81bef3..eced2d4 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -20,21 +20,22 @@ # GUI import wx -from .GUIPlotPanel import PlotPanel -from .GUISelectionPanel import SelectionPanel,SEL_MODES,SEL_MODES_ID -from .GUISelectionPanel import ColumnPopup,TablePopup -from .GUIInfoPanel import InfoPanel -from .GUIPipelinePanel import PipelinePanel -from .GUIToolBox import GetKeyString, TBAddTool -from .Tables import TableList, Table +from pydatview.GUIPlotPanel import PlotPanel +from pydatview.GUISelectionPanel import SelectionPanel,SEL_MODES,SEL_MODES_ID +from pydatview.GUISelectionPanel import ColumnPopup,TablePopup +from pydatview.GUIInfoPanel import InfoPanel +from pydatview.GUIPipelinePanel import PipelinePanel +from pydatview.GUIToolBox import GetKeyString, TBAddTool +from pydatview.Tables import TableList, Table # Helper -from .common import * -from .GUICommon import * +from pydatview.common import exception2string, PyDatViewException +from pydatview.common import * +from pydatview.GUICommon import * import pydatview.io as weio # File Formats and File Readers # Pluggins -from .plugins import DATA_PLUGINS_WITH_EDITOR, DATA_PLUGINS_SIMPLE, TOOLS -from .plugins import OF_DATA_PLUGINS_WITH_EDITOR, OF_DATA_PLUGINS_SIMPLE -from .appdata import loadAppData, saveAppData, configFilePath, defaultAppData +from pydatview.plugins import DATA_PLUGINS_WITH_EDITOR, DATA_PLUGINS_SIMPLE, TOOLS +from pydatview.plugins import OF_DATA_PLUGINS_WITH_EDITOR, OF_DATA_PLUGINS_SIMPLE +from pydatview.appdata import loadAppData, saveAppData, configFilePath, defaultAppData # --------------------------------------------------------------------------------} # --- GLOBAL @@ -861,7 +862,7 @@ def MyExceptionHook(etype, value, trace): standard Python header: ``Traceback (most recent call last)``. """ from wx._core import wxAssertionError - # Printing exception + # Printing exception to screen traceback.print_exception(etype, value, trace) if etype==wxAssertionError: if wx.Platform == '__WXMAC__': @@ -870,17 +871,26 @@ def MyExceptionHook(etype, value, trace): # Then showing to user the last error frame = wx.GetApp().GetTopWindow() tmp = traceback.format_exception(etype, value, trace) + sException='' if tmp[-1].find('Exception: Error:')==0: - Error(frame,tmp[-1][18:]) + sException = tmp[-1][18:] elif tmp[-1].find('Exception: Warn:')==0: Warn(frame,tmp[-1][17:]) else: - exception = 'The following exception occured:\n\n'+ tmp[-1] + '\n'+tmp[-2].strip() - Error(frame,exception) + sException = 'The following exception occured:\n\n' + #sException += tmp[-1] + '\n' + #sException += tmp[-2].strip() + #if len(tmp)>2: + # sException += '\n'+tmp[-3].strip() + ## TODO make this a custom dialog where the user can copy paste the error more easily... + sException += exception2string(value, prevStack=False) + try: frame.Thaw() # Make sure any freeze event is stopped except: pass + if len(sException)>0: + Error(frame, sException) # --------------------------------------------------------------------------------} # --- Tests @@ -975,19 +985,34 @@ def showApp(firstArg=None, dataframes=None, filenames=[], names=None): elif isinstance(firstArg, pd.DataFrame): dataframes=[firstArg] # Load files or dataframe depending on interface - if (dataframes is not None) and (len(dataframes)>0): - if names is None: - names=['df{}'.format(i+1) for i in range(len(dataframes))] - frame.load_dfs(dataframes, names) - elif len(filenames)>0: - frame.load_files(filenames, fileformats=None, bPlot=True) + err = None + # NOTE: any exceptions occurring before MainLoop will result in the window to close + try: + if (dataframes is not None) and (len(dataframes)>0): + if names is None: + names=['df{}'.format(i+1) for i in range(len(dataframes))] + frame.load_dfs(dataframes, names) + elif len(filenames)>0: + frame.load_files(filenames, fileformats=None, bPlot=True) + except Exception as e: + try: + frame.Thaw() + except: + pass + # Print to screen: + traceback.print_exc() + # Store it to display it later in App + err = 'Errors occured while loading files:\n\n' + err += exception2string(e) #frame.onShowTool(toolName='') #frame.onDataPlugin(toolName='Radial Average') #frame.onDataPlugin(toolName='Resample') #frame.onScript() - + if err is not None: + wx.FutureCall(100, Error, frame, err) app.MainLoop() + # Nothing will be reached here until the window is opened def cmdline(): From c240bc24ce8e7c88125b81a087281a15e541b246 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 11:56:01 -0600 Subject: [PATCH 154/178] IO: Excel File: return a single DataFrame for one sheet (see #170) --- pydatview/io/excel_file.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pydatview/io/excel_file.py b/pydatview/io/excel_file.py index 0b9c9ab..8ea2728 100644 --- a/pydatview/io/excel_file.py +++ b/pydatview/io/excel_file.py @@ -75,8 +75,10 @@ def __repr__(self): def _toDataFrame(self): - #cols=['Alpha_[deg]','Cl_[-]','Cd_[-]','Cm_[-]'] - #dfs[name] = pd.DataFrame(data=..., columns=cols) - #df=pd.DataFrame(data=,columns=) - return self.data + if len(self.data)==1: + # Return a single dataframe + return self.data[list(self.data.keys())[0]] + else: + # Return dictionary + return self.data From 36343a12055d831285f460c41ea166499095c1c9 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 16:21:59 -0600 Subject: [PATCH 155/178] IO: small updates --- pydatview/io/fast_linearization_file.py | 783 +++++++++++++++--------- pydatview/io/flex_out_file.py | 2 +- pydatview/io/pickle_file.py | 7 +- 3 files changed, 510 insertions(+), 282 deletions(-) diff --git a/pydatview/io/fast_linearization_file.py b/pydatview/io/fast_linearization_file.py index 28e1ace..8477d62 100644 --- a/pydatview/io/fast_linearization_file.py +++ b/pydatview/io/fast_linearization_file.py @@ -7,13 +7,21 @@ File = dict class BrokenFormatError(Exception): pass +class SlowReaderNeededError(Exception): + pass + + +_lin_vec = ['x','xd','xdot','u','y','z','header'] +_lin_mat = ['A','B','C','D','dUdu','dUdy', 'StateRotation', 'M'] +_lin_dict = ['x_info','xdot_info','u_info','y_info'] + class FASTLinearizationFile(File): """ Read/write an OpenFAST linearization file. The object behaves like a dictionary. Main keys --------- - - 'x', 'xdot' 'u', 'y', 'A', 'B', 'C', 'D' + - 'x', 'xdot', 'xd', 'u', 'y', 'z', 'A', 'B', 'C', 'D' Main methods ------------ @@ -47,8 +55,16 @@ def __init__(self, filename=None, **kwargs): if filename: self.read(**kwargs) - def read(self, filename=None, **kwargs): - """ Reads the file self.filename, or `filename` if provided """ + def read(self, filename=None, starSub=None, removeStatesPattern=None): + """ Reads the file self.filename, or `filename` if provided + + - starSub: if None, raise an error if `****` are present + otherwise replace *** with `starSub` (e.g. 0) + - removeStatesPattern: if None, do nothing + otherwise search for states matching a pattern and remove them + e.g: 'tower|Drivetrain' or '^AD' + see removeStates in this file. + """ # --- Standard tests and exceptions (generic code) if filename: @@ -59,131 +75,71 @@ def read(self, filename=None, **kwargs): raise OSError(2,'File not found:',self.filename) if os.stat(self.filename).st_size == 0: raise EmptyFileError('File is empty:',self.filename) - # --- Calling (children) function to read - self._read(**kwargs) - def _read(self, *args, **kwargs): + # --- Main Data self['header']=[] - def extractVal(lines, key): - for l in lines: - if l.find(key)>=0: - return l.split(key)[1].split()[0] - return None - - def readToMarker(fid, marker, nMax): - lines=[] - for i, line in enumerate(fid): - if i>nMax: - raise BrokenFormatError('`{}` not found in file'.format(marker)) - if line.find(marker)>=0: - break - lines.append(line.strip()) - return lines, line - - def readOP(fid, n, name=''): - OP=[] - Var = {'RotatingFrame': [], 'DerivativeOrder': [], 'Description': []} - colNames=fid.readline().strip() - dummy= fid.readline().strip() - bHasDeriv= colNames.find('Derivative Order')>=0 - for i, line in enumerate(fid): - sp=line.strip().split() - if sp[1].find(',')>=0: - # Most likely this OP has three values (e.g. orientation angles) - # For now we discard the two other values - OP.append(float(sp[1][:-1])) - iRot=4 - else: - OP.append(float(sp[1])) - iRot=2 - Var['RotatingFrame'].append(sp[iRot]) - if bHasDeriv: - Var['DerivativeOrder'].append(int(sp[iRot+1])) - Var['Description'].append(' '.join(sp[iRot+2:]).strip()) - else: - Var['DerivativeOrder'].append(-1) - Var['Description'].append(' '.join(sp[iRot+1:]).strip()) - if i>=n-1: - break - OP=np.asarray(OP) - return OP, Var - - def readMat(fid, n, m, name=''): - pattern = re.compile(r"[\*]+") - vals=[pattern.sub(' inf ', fid.readline().strip() ).split() for i in np.arange(n)] - vals = np.array(vals) - try: - vals = np.array(vals).astype(float) # This could potentially fail - except: - raise Exception('Failed to convert into an array of float the matrix `{}`\n\tin linfile: {}'.format(name, self.filename)) - if vals.shape[0]!=n or vals.shape[1]!=m: - shape1 = vals.shape - shape2 = (n,m) - raise Exception('Shape of matrix `{}` has wrong dimension ({} instead of {})\n\tin linfile: {}'.format(name, shape1, shape2, name, self.filename)) - - nNaN = sum(np.isnan(vals.ravel())) - nInf = sum(np.isinf(vals.ravel())) - if nInf>0: - raise Exception('Some ill-formated/infinite values (e.g. `*******`) were found in the matrix `{}`\n\tin linflile: {}'.format(name, self.filename)) - if nNaN>0: - raise Exception('Some NaN values were found in the matrix `{}`\n\tin linfile: `{}`.'.format(name, self.filename)) - return vals - - - # Reading - with open(self.filename, 'r', errors="surrogateescape") as f: - # --- Reader header - self['header'], lastLine=readToMarker(f, 'Jacobians included', 30) - self['header'].append(lastLine) - nx = int(extractVal(self['header'],'Number of continuous states:')) - nxd = int(extractVal(self['header'],'Number of discrete states:' )) - nz = int(extractVal(self['header'],'Number of constraint states:')) - nu = int(extractVal(self['header'],'Number of inputs:' )) - ny = int(extractVal(self['header'],'Number of outputs:' )) - bJac = extractVal(self['header'],'Jacobians included in this file?') - try: - self['Azimuth'] = float(extractVal(self['header'],'Azimuth:')) - except: - self['Azimuth'] = None - try: - self['RotSpeed'] = float(extractVal(self['header'],'Rotor Speed:')) # rad/s - except: - self['RotSpeed'] = None - try: - self['WindSpeed'] = float(extractVal(self['header'],'Wind Speed:')) - except: - self['WindSpeed'] = None - - for i, line in enumerate(f): - line = line.strip() - if line.find('Order of continuous states:')>=0: - self['x'], self['x_info'] = readOP(f, nx, 'x') - elif line.find('Order of continuous state derivatives:')>=0: - self['xdot'], self['xdot_info'] = readOP(f, nx, 'xdot') - elif line.find('Order of inputs')>=0: - self['u'], self['u_info'] = readOP(f, nu, 'u') - elif line.find('Order of outputs')>=0: - self['y'], self['y_info'] = readOP(f, ny, 'y') - elif line.find('A:')>=0: - self['A'] = readMat(f, nx, nx, 'A') - elif line.find('B:')>=0: - self['B'] = readMat(f, nx, nu, 'B') - elif line.find('C:')>=0: - self['C'] = readMat(f, ny, nx, 'C') - elif line.find('D:')>=0: - self['D'] = readMat(f, ny, nu, 'D') - elif line.find('dUdu:')>=0: - self['dUdu'] = readMat(f, nu, nu,'dUdu') - elif line.find('dUdy:')>=0: - self['dUdy'] = readMat(f, nu, ny,'dUdy') - elif line.find('StateRotation:')>=0: - pass - # TODO - #StateRotation: - elif line.find('ED M:')>=0: - self['EDDOF'] = line[5:].split() - self['M'] = readMat(f, 24, 24,'M') + # --- StarValues replacement `*****` -> inf + starPattern = re.compile(r"[\*]+") + starSubStr = ' inf ' + starSubFn = lambda si: starPattern.sub(starSubStr, si) + + # Reading function, with slow or fast reader. See sub functions at end of this file + def doRead(slowReader=False): + with open(self.filename, 'r', errors="surrogateescape") as f: + # --- Reader header + self['header'], lastLine=readToMarker(f, 'Jacobians included', 30) + self['header'].append(lastLine) + nx = extractVal(self['header'],'Number of continuous states:' , dtype=int, NA=np.nan, missing=None) + nxd = extractVal(self['header'],'Number of discrete states:' , dtype=int, NA=np.nan, missing=None) + nz = extractVal(self['header'],'Number of constraint states:' , dtype=int, NA=np.nan, missing=None) + nu = extractVal(self['header'],'Number of inputs:' , dtype=int, NA=np.nan, missing=None) + ny = extractVal(self['header'],'Number of outputs:' , dtype=int, NA=np.nan, missing=None) + bJac = extractVal(self['header'],'Jacobians included in this file?', dtype=bool, NA=False, missing=None) + self['Azimuth'] = extractVal(self['header'], 'Azimuth:' , dtype=float, NA=np.nan, missing=None) + self['RotSpeed'] = extractVal(self['header'], 'Rotor Speed:', dtype=float, NA=np.nan, missing=None) # rad/s + self['WindSpeed'] = extractVal(self['header'], 'Wind Speed:' , dtype=float, NA=np.nan, missing=None) + self['t'] = extractVal(self['header'],'Simulation time:' , dtype=float, NA=np.nan, missing=None) + for i, line in enumerate(f): + line = line.strip() + if line.find('Order of continuous states:')>=0: + self['x'], self['x_info'] = readOP(f, nx, 'x', defaultDerivOrder=1, starSubFn=starSubFn, starSub=starSub) + elif line.find('Order of continuous state derivatives:')>=0: + self['xdot'], self['xdot_info'] = readOP(f, nx, 'xdot', defaultDerivOrder=2, starSubFn=starSubFn, starSub=starSub) + elif line.find('Order of discrete states:')>=0: + self['xd'], self['xd_info'] = readOP(f, nxd, 'xd', defaultDerivOrder=2, starSubFn=starSubFn, starSub=starSub) + elif line.find('Order of inputs')>=0: + self['u'], self['u_info'] = readOP(f, nu, 'u', defaultDerivOrder=0, starSubFn=starSubFn, starSub=starSub) + elif line.find('Order of outputs')>=0: + self['y'], self['y_info'] = readOP(f, ny, 'y', defaultDerivOrder=0, starSubFn=starSubFn, starSub=starSub) + elif line.find('Order of constraint states:')>=0: + self['z'], self['z_info'] = readOP(f, nz, 'z', defaultDerivOrder=0, starSubFn=starSubFn, starSub=starSub) + elif line.find('A:')>=0: + self['A'] = readMat(f, nx, nx, 'A', slowReader=slowReader, filename=self.filename, starSubFn=starSubFn, starSub=starSub) + elif line.find('B:')>=0: + self['B'] = readMat(f, nx, nu, 'B', slowReader=slowReader, filename=self.filename, starSubFn=starSubFn, starSub=starSub) + elif line.find('C:')>=0: + self['C'] = readMat(f, ny, nx, 'C', slowReader=slowReader, filename=self.filename, starSubFn=starSubFn, starSub=starSub) + elif line.find('D:')>=0: + self['D'] = readMat(f, ny, nu, 'D', slowReader=slowReader, filename=self.filename, starSubFn=starSubFn, starSub=starSub) + elif line.find('dUdu:')>=0: + self['dUdu'] = readMat(f, nu, nu,'dUdu', slowReader=slowReader, filename=self.filename, starSubFn=starSubFn, starSub=starSub) + elif line.find('dUdy:')>=0: + self['dUdy'] = readMat(f, nu, ny,'dUdy', slowReader=slowReader, filename=self.filename, starSubFn=starSubFn, starSub=starSub) + elif line.find('StateRotation:')>=0: + pass + # TODO + #StateRotation: + elif line.find('ED M:')>=0: + self['EDDOF'] = line[5:].split() + self['M'] = readMat(f, 24, 24,'M', slowReader=slowReader, filename=self.filename, starSubFn=starSubFn, starSub=starSub) + try: + doRead(slowReader=False) + except SlowReaderNeededError: + doRead(slowReader=True) + + if removeStatesPattern is not None: + self.removeStates(pattern=removeStatesPattern) def toString(self): s='' @@ -193,176 +149,113 @@ def _write(self): with open(self.filename,'w') as f: f.write(self.toString()) - def short_descr(self,slist): - def shortname(s): - s=s.strip() - s = s.replace('(m/s)' , '_[m/s]' ); - s = s.replace('(kW)' , '_[kW]' ); - s = s.replace('(deg)' , '_[deg]' ); - s = s.replace('(N)' , '_[N]' ); - s = s.replace('(kN-m)' , '_[kNm]' ); - s = s.replace('(N-m)' , '_[Nm]' ); - s = s.replace('(kN)' , '_[kN]' ); - s = s.replace('(rpm)' , '_[rpm]' ); - s = s.replace('(rad)' , '_[rad]' ); - s = s.replace('(rad/s)' , '_[rad/s]' ); - s = s.replace('(rad/s^2)', '_[rad/s^2]' ); - s = s.replace('(m/s^2)' , '_[m/s^2]'); - s = s.replace('(deg/s^2)','_[deg/s^2]'); - s = s.replace('(m)' , '_[m]' ); - s = s.replace(', m/s/s','_[m/s^2]'); - s = s.replace(', m/s^2','_[m/s^2]'); - s = s.replace(', m/s','_[m/s]'); - s = s.replace(', m','_[m]'); - s = s.replace(', rad/s/s','_[rad/s^2]'); - s = s.replace(', rad/s^2','_[rad/s^2]'); - s = s.replace(', rad/s','_[rad/s]'); - s = s.replace(', rad','_[rad]'); - s = s.replace(', -','_[-]'); - s = s.replace(', Nm/m','_[Nm/m]'); - s = s.replace(', Nm','_[Nm]'); - s = s.replace(', N/m','_[N/m]'); - s = s.replace(', N','_[N]'); - s = s.replace('(1)','1') - s = s.replace('(2)','2') - s = s.replace('(3)','3') - s= re.sub(r'\([^)]*\)','', s) # remove parenthesis - s = s.replace('ED ',''); - s = s.replace('BD_','BD_B'); - s = s.replace('IfW ',''); - s = s.replace('Extended input: ','') - s = s.replace('1st tower ','qt1'); - s = s.replace('2nd tower ','qt2'); - nd = s.count('First time derivative of ') - if nd>=0: - s = s.replace('First time derivative of ' ,''); - if nd==1: - s = 'd_'+s.strip() - elif nd==2: - s = 'dd_'+s.strip() - s = s.replace('Variable speed generator DOF ','psi_rot'); # NOTE: internally in FAST this is the azimuth of the rotor - s = s.replace('fore-aft bending mode DOF ' ,'FA' ); - s = s.replace('side-to-side bending mode DOF','SS' ); - s = s.replace('bending-mode DOF of blade ' ,'' ); - s = s.replace(' rotational-flexibility DOF, rad','-ROT' ); - s = s.replace('rotational displacement in ','rot' ); - s = s.replace('Drivetrain','DT' ); - s = s.replace('translational displacement in ','trans' ); - s = s.replace('finite element node ','N' ); - s = s.replace('-component position of node ','posN') - s = s.replace('-component inflow on tower node','TwrN') - s = s.replace('-component inflow on blade 1, node','Bld1N') - s = s.replace('-component inflow on blade 2, node','Bld2N') - s = s.replace('-component inflow on blade 3, node','Bld3N') - s = s.replace('-component inflow velocity at node','N') - s = s.replace('X translation displacement, node','TxN') - s = s.replace('Y translation displacement, node','TyN') - s = s.replace('Z translation displacement, node','TzN') - s = s.replace('X translation velocity, node','TVxN') - s = s.replace('Y translation velocity, node','TVyN') - s = s.replace('Z translation velocity, node','TVzN') - s = s.replace('X translation acceleration, node','TAxN') - s = s.replace('Y translation acceleration, node','TAyN') - s = s.replace('Z translation acceleration, node','TAzN') - s = s.replace('X orientation angle, node' ,'RxN') - s = s.replace('Y orientation angle, node' ,'RyN') - s = s.replace('Z orientation angle, node' ,'RzN') - s = s.replace('X rotation velocity, node' ,'RVxN') - s = s.replace('Y rotation velocity, node' ,'RVyN') - s = s.replace('Z rotation velocity, node' ,'RVzN') - s = s.replace('X rotation acceleration, node' ,'RAxN') - s = s.replace('Y rotation acceleration, node' ,'RAyN') - s = s.replace('Z rotation acceleration, node' ,'RAzN') - s = s.replace('X force, node','FxN') - s = s.replace('Y force, node','FyN') - s = s.replace('Z force, node','FzN') - s = s.replace('X moment, node','MxN') - s = s.replace('Y moment, node','MyN') - s = s.replace('Z moment, node','MzN') - s = s.replace('FX', 'Fx') - s = s.replace('FY', 'Fy') - s = s.replace('FZ', 'Fz') - s = s.replace('MX', 'Mx') - s = s.replace('MY', 'My') - s = s.replace('MZ', 'Mz') - s = s.replace('FKX', 'FKx') - s = s.replace('FKY', 'FKy') - s = s.replace('FKZ', 'FKz') - s = s.replace('MKX', 'MKx') - s = s.replace('MKY', 'MKy') - s = s.replace('MKZ', 'MKz') - s = s.replace('Nodes motion','') - s = s.replace('cosine','cos' ); - s = s.replace('sine','sin' ); - s = s.replace('collective','coll.'); - s = s.replace('Blade','Bld'); - s = s.replace('rotZ','TORS-R'); - s = s.replace('transX','FLAP-D'); - s = s.replace('transY','EDGE-D'); - s = s.replace('rotX','EDGE-R'); - s = s.replace('rotY','FLAP-R'); - s = s.replace('flapwise','FLAP'); - s = s.replace('edgewise','EDGE'); - s = s.replace('horizontal surge translation DOF','Surge'); - s = s.replace('horizontal sway translation DOF','Sway'); - s = s.replace('vertical heave translation DOF','Heave'); - s = s.replace('roll tilt rotation DOF','Roll'); - s = s.replace('pitch tilt rotation DOF','Pitch'); - s = s.replace('yaw rotation DOF','Yaw'); - s = s.replace('vertical power-law shear exponent','alpha') - s = s.replace('horizontal wind speed ','WS') - s = s.replace('propagation direction','WD') - s = s.replace(' pitch command','pitch') - s = s.replace('HSS_','HSS') - s = s.replace('Bld','B') - s = s.replace('tower','Twr') - s = s.replace('Tower','Twr') - s = s.replace('Nacelle','Nac') - s = s.replace('Platform','Ptfm') - s = s.replace('SrvD','SvD') - s = s.replace('Generator torque','Qgen') - s = s.replace('coll. blade-pitch command','PitchColl') - s = s.replace('wave elevation at platform ref point','WaveElevRefPoint') - s = s.replace('1)','1'); - s = s.replace('2)','2'); - s = s.replace('3)','3'); - s = s.replace(',',''); - s = s.replace(' ',''); - s=s.strip() - return s - return [shortname(s) for s in slist] - - def xdescr(self): - if 'x_info' in self.keys(): - return self.short_descr(self['x_info']['Description']) + @property + def nx(self): + if 'x' in self.keys(): + return len(self['x']) + return 0 + + @property + def nxd(self): + if 'xd' in self.keys(): + return len(self['xd']) + return 0 + + @property + def nu(self): + if 'u' in self.keys(): + return len(self['u']) + return 0 + + @property + def ny(self): + if 'y' in self.keys(): + return len(self['y']) + return 0 + + @property + def nz(self): + if 'z' in self.keys(): + return len(self['z']) + return 0 + + @property + def u_descr(self): + if self.nu>0: + return self['u_info']['Description'] else: return [] - def xdotdescr(self): - if 'xdot_info' in self.keys(): - return self.short_descr(self['xdot_info']['Description']) + @property + def x_descr(self): + if self.nx>0: + return self['x_info']['Description'] else: return [] - def ydescr(self): - if 'y_info' in self.keys(): - return self.short_descr(self['y_info']['Description']) + @property + def xd_descr(self): # Discrete states not derivative! + if self.nxd>0: + return self['xd_info']['Description'] else: return [] - def udescr(self): - if 'u_info' in self.keys(): - return self.short_descr(self['u_info']['Description']) + + @property + def xdot_descr(self): + if self.nx>0: + return self['xdot_info']['Description'] + else: + return [] + + @property + def y_descr(self): + if self.ny>0: + return self['y_info']['Description'] + else: + return [] + + @property + def z_descr(self): + if self.nz>0: + return self['z_info']['Description'] else: return [] + def __repr__(self): + s='<{} object> with attributes:\n'.format(type(self).__name__) + s+=' - filename: {}\n'.format(self.filename) + s+=' * nx : {}\n'.format(self.nx) + s+=' * nxd : {}\n'.format(self.nxd) + s+=' * nu : {}\n'.format(self.nu) + s+=' * ny : {}\n'.format(self.ny) + s+=' * nz : {}\n'.format(self.nz) + s+='keys:\n' + for k,v in self.items(): + if k in _lin_vec: + s+=' - {:15s}: shape: ({}) \n'.format(k,len(v)) + elif k in _lin_mat: + s+=' - {:15s}: shape: ({} x {})\n'.format(k,v.shape[0], v.shape[1]) + elif k in _lin_dict: + s+=' - {:15s}: dict with keys: {} \n'.format(k,list(v.keys())) + else: + s+=' - {:15s}: {}\n'.format(k,v) + s+='methods:\n' + s+=' - toDataFrame: convert A,B,C,D to dataframes\n' + s+=' - removeStates: remove states\n' + s+=' - eva: eigenvalue analysis\n' + + return s + def toDataFrame(self): import pandas as pd dfs={} - xdescr_short = self.xdescr() - xdotdescr_short = self.xdotdescr() - ydescr_short = self.ydescr() - udescr_short = self.udescr() + xdescr_short = short_descr(self.x_descr) + xddescr_short = short_descr(self.xd_descr) + xdotdescr_short = short_descr(self.xdot_descr) + udescr_short = short_descr(self.u_descr) + ydescr_short = short_descr(self.y_descr) + zdescr_short = short_descr(self.z_descr) if 'A' in self.keys(): dfs['A'] = pd.DataFrame(data = self['A'], index=xdescr_short, columns=xdescr_short) @@ -374,12 +267,16 @@ def toDataFrame(self): dfs['D'] = pd.DataFrame(data = self['D'], index=ydescr_short, columns=udescr_short) if 'x' in self.keys(): dfs['x'] = pd.DataFrame(data = np.asarray(self['x']).reshape((1,-1)), columns=xdescr_short) + if 'xd' in self.keys(): + dfs['xd'] = pd.DataFrame(data = np.asarray(self['xd']).reshape((1,-1))) if 'xdot' in self.keys(): dfs['xdot'] = pd.DataFrame(data = np.asarray(self['xdot']).reshape((1,-1)), columns=xdotdescr_short) if 'u' in self.keys(): dfs['u'] = pd.DataFrame(data = np.asarray(self['u']).reshape((1,-1)), columns=udescr_short) if 'y' in self.keys(): dfs['y'] = pd.DataFrame(data = np.asarray(self['y']).reshape((1,-1)), columns=ydescr_short) + if 'z' in self.keys(): + dfs['z'] = pd.DataFrame(data = np.asarray(self['z']).reshape((1,-1)), columns=zdescr_short) if 'M' in self.keys(): dfs['M'] = pd.DataFrame(data = self['M'], index=self['EDDOF'], columns=self['EDDOF']) if 'dUdu' in self.keys(): @@ -389,4 +286,330 @@ def toDataFrame(self): return dfs + def removeStates(self, pattern=None, Irm=None, verbose=True): + """ + remove states based on pattern or index + + - pattern: e.g: 'tower|Drivetrain' or '^AD' + """ + if self.nx==0: + return + desc = self['x_info']['Description'] + Iall = set(range(len(desc))) + sInfo='' + if pattern is not None: + Irm = [i for i, s in enumerate(desc) if re.search(pattern, s)] + sInfo=' with pattern `{}`'.format(pattern) + if verbose: + print('[INFO] removing {}/{} states{}'.format(len(Irm), len(Iall), sInfo)) + Ikeep = list(Iall.difference(Irm)) + Ikeep.sort() # safety + if len(Ikeep)==0: + raise Exception('All states have been removed{}!'.format(sInfo)) + # Remove states and info in vectors + self['x'] = self['x'][Ikeep] + self['xdot'] = self['xdot'][Ikeep] + for k in self['x_info'].keys(): + self['x_info'][k] = self['x_info'][k][Ikeep] + self['xdot_info'][k] = self['xdot_info'][k][Ikeep] + # Remove states in matrices + if 'A' in self.keys(): + self['A'] = self['A'][np.ix_(Ikeep,Ikeep)] + if 'B' in self.keys(): + self['B'] = self['B'][Ikeep,:] + if 'C' in self.keys(): + self['C'] = self['C'][:, Ikeep] + + + def eva(self, normQ=None, sort=True, discardIm=True): + """ Perform eigenvalue analysis of A matrix and return frequencies and damping """ + # --- Excerpt from welib.tools.eva.eigA + A = self['A'] + n,m = A.shape + if m!=n: + raise Exception('Matrix needs to be square') + # Basic EVA + D,Q = np.linalg.eig(A) + Lambda = np.diag(D) + v = np.diag(Lambda) + + # Selecting eigenvalues with positive imaginary part (frequency) + if discardIm: + Ipos = np.imag(v)>0 + Q = Q[:,Ipos] + v = v[Ipos] + + # Frequencies and damping based on compled eigenvalues + omega_0 = np.abs(v) # natural cylic frequency [rad/s] + freq_d = np.imag(v)/(2*np.pi) # damped frequency [Hz] + zeta = - np.real(v)/omega_0 # damping ratio + freq_0 = omega_0/(2*np.pi) # natural frequency [Hz] + # Sorting + if sort: + I = np.argsort(freq_0) + freq_d = freq_d[I] + freq_0 = freq_0[I] + zeta = zeta[I] + Q = Q[:,I] + + # Normalize Q + if normQ=='byMax': + for j in range(Q.shape[1]): + q_j = Q[:,j] + scale = np.max(np.abs(q_j)) + Q[:,j]= Q[:,j]/scale + return freq_d, zeta, Q, freq_0 + + +def short_descr(slist): + """ Shorten and "unify" the description from lin file """ + def shortname(s): + s=s.strip() + s = s.replace('(m/s)' , '_[m/s]' ); + s = s.replace('(kW)' , '_[kW]' ); + s = s.replace('(deg)' , '_[deg]' ); + s = s.replace('(N)' , '_[N]' ); + s = s.replace('(kN-m)' , '_[kNm]' ); + s = s.replace('(N-m)' , '_[Nm]' ); + s = s.replace('(kN)' , '_[kN]' ); + s = s.replace('(rpm)' , '_[rpm]' ); + s = s.replace('(rad)' , '_[rad]' ); + s = s.replace('(rad/s)' , '_[rad/s]' ); + s = s.replace('(rad/s^2)', '_[rad/s^2]' ); + s = s.replace('(m/s^2)' , '_[m/s^2]'); + s = s.replace('(deg/s^2)','_[deg/s^2]'); + s = s.replace('(m)' , '_[m]' ); + s = s.replace(', m/s/s','_[m/s^2]'); + s = s.replace(', m/s^2','_[m/s^2]'); + s = s.replace(', m/s','_[m/s]'); + s = s.replace(', m','_[m]'); + s = s.replace(', rad/s/s','_[rad/s^2]'); + s = s.replace(', rad/s^2','_[rad/s^2]'); + s = s.replace(', rad/s','_[rad/s]'); + s = s.replace(', rad','_[rad]'); + s = s.replace(', -','_[-]'); + s = s.replace(', Nm/m','_[Nm/m]'); + s = s.replace(', Nm','_[Nm]'); + s = s.replace(', N/m','_[N/m]'); + s = s.replace(', N','_[N]'); + s = s.replace('(1)','1') + s = s.replace('(2)','2') + s = s.replace('(3)','3') + s= re.sub(r'\([^)]*\)','', s) # remove parenthesis + s = s.replace('ED ',''); + s = s.replace('BD_','BD_B'); + s = s.replace('IfW ',''); + s = s.replace('Extended input: ','') + s = s.replace('1st tower ','qt1'); + s = s.replace('2nd tower ','qt2'); + nd = s.count('First time derivative of ') + if nd>=0: + s = s.replace('First time derivative of ' ,''); + if nd==1: + s = 'd_'+s.strip() + elif nd==2: + s = 'dd_'+s.strip() + s = s.replace('Variable speed generator DOF ','psi_rot'); # NOTE: internally in FAST this is the azimuth of the rotor + s = s.replace('fore-aft bending mode DOF ' ,'FA' ); + s = s.replace('side-to-side bending mode DOF','SS' ); + s = s.replace('bending-mode DOF of blade ' ,'' ); + s = s.replace(' rotational-flexibility DOF, rad','-ROT' ); + s = s.replace('rotational displacement in ','rot' ); + s = s.replace('Drivetrain','DT' ); + s = s.replace('translational displacement in ','trans' ); + s = s.replace('finite element node ','N' ); + s = s.replace('-component position of node ','posN') + s = s.replace('-component inflow on tower node','TwrN') + s = s.replace('-component inflow on blade 1, node','Bld1N') + s = s.replace('-component inflow on blade 2, node','Bld2N') + s = s.replace('-component inflow on blade 3, node','Bld3N') + s = s.replace('-component inflow velocity at node','N') + s = s.replace('X translation displacement, node','TxN') + s = s.replace('Y translation displacement, node','TyN') + s = s.replace('Z translation displacement, node','TzN') + s = s.replace('X translation velocity, node','TVxN') + s = s.replace('Y translation velocity, node','TVyN') + s = s.replace('Z translation velocity, node','TVzN') + s = s.replace('X translation acceleration, node','TAxN') + s = s.replace('Y translation acceleration, node','TAyN') + s = s.replace('Z translation acceleration, node','TAzN') + s = s.replace('X orientation angle, node' ,'RxN') + s = s.replace('Y orientation angle, node' ,'RyN') + s = s.replace('Z orientation angle, node' ,'RzN') + s = s.replace('X rotation velocity, node' ,'RVxN') + s = s.replace('Y rotation velocity, node' ,'RVyN') + s = s.replace('Z rotation velocity, node' ,'RVzN') + s = s.replace('X rotation acceleration, node' ,'RAxN') + s = s.replace('Y rotation acceleration, node' ,'RAyN') + s = s.replace('Z rotation acceleration, node' ,'RAzN') + s = s.replace('X force, node','FxN') + s = s.replace('Y force, node','FyN') + s = s.replace('Z force, node','FzN') + s = s.replace('X moment, node','MxN') + s = s.replace('Y moment, node','MyN') + s = s.replace('Z moment, node','MzN') + s = s.replace('FX', 'Fx') + s = s.replace('FY', 'Fy') + s = s.replace('FZ', 'Fz') + s = s.replace('MX', 'Mx') + s = s.replace('MY', 'My') + s = s.replace('MZ', 'Mz') + s = s.replace('FKX', 'FKx') + s = s.replace('FKY', 'FKy') + s = s.replace('FKZ', 'FKz') + s = s.replace('MKX', 'MKx') + s = s.replace('MKY', 'MKy') + s = s.replace('MKZ', 'MKz') + s = s.replace('Nodes motion','') + s = s.replace('cosine','cos' ); + s = s.replace('sine','sin' ); + s = s.replace('collective','coll.'); + s = s.replace('Blade','Bld'); + s = s.replace('rotZ','TORS-R'); + s = s.replace('transX','FLAP-D'); + s = s.replace('transY','EDGE-D'); + s = s.replace('rotX','EDGE-R'); + s = s.replace('rotY','FLAP-R'); + s = s.replace('flapwise','FLAP'); + s = s.replace('edgewise','EDGE'); + s = s.replace('horizontal surge translation DOF','Surge'); + s = s.replace('horizontal sway translation DOF','Sway'); + s = s.replace('vertical heave translation DOF','Heave'); + s = s.replace('roll tilt rotation DOF','Roll'); + s = s.replace('pitch tilt rotation DOF','Pitch'); + s = s.replace('yaw rotation DOF','Yaw'); + s = s.replace('vertical power-law shear exponent','alpha') + s = s.replace('horizontal wind speed ','WS') + s = s.replace('propagation direction','WD') + s = s.replace(' pitch command','pitch') + s = s.replace('HSS_','HSS') + s = s.replace('Bld','B') + s = s.replace('tower','Twr') + s = s.replace('Tower','Twr') + s = s.replace('Nacelle','Nac') + s = s.replace('Platform','Ptfm') + s = s.replace('SrvD','SvD') + s = s.replace('Generator torque','Qgen') + s = s.replace('coll. blade-pitch command','PitchColl') + s = s.replace('wave elevation at platform ref point','WaveElevRefPoint') + s = s.replace('1)','1'); + s = s.replace('2)','2'); + s = s.replace('3)','3'); + s = s.replace(',',''); + s = s.replace(' ',''); + s=s.strip() + return s + return [shortname(s) for s in slist] + + + +def extractVal(lines, key, NA=None, missing=None, dtype=float): + for l in lines: + if l.find(key)>=0: + #l = starPattern.sub(starSubStr, l) + try: + return dtype(l.split(key)[1].split()[0]) + except: + return NA + return missing + +def readToMarker(fid, marker, nMax): + lines=[] + for i, line in enumerate(fid): + if i>nMax: + raise BrokenFormatError('`{}` not found in file'.format(marker)) + if line.find(marker)>=0: + break + lines.append(line.strip()) + return lines, line + +def readOP(fid, n, name='', defaultDerivOrder=1, filename='', starSubFn=None, starSub=None): + OP=[] + Var = {'RotatingFrame': [], 'DerivativeOrder': [], 'Description': []} + colNames=fid.readline().strip() + dummy= fid.readline().strip() + bHasDeriv= colNames.find('Derivative Order')>=0 + for i, line in enumerate(fid): + line = line.strip() + line = starSubFn(line) + sp = line.split() + if sp[1].find(',')>=0: + # Most likely this OP has three values (e.g. orientation angles) + # For now we discard the two other values + OP.append(float(sp[1][:-1])) + iRot=4 + else: + OP.append(float(sp[1])) + iRot=2 + Var['RotatingFrame'].append(sp[iRot]) + if bHasDeriv: + Var['DerivativeOrder'].append(int(sp[iRot+1])) + Var['Description'].append(' '.join(sp[iRot+2:]).strip()) + else: + Var['DerivativeOrder'].append(defaultDerivOrder) + Var['Description'].append(' '.join(sp[iRot+1:]).strip()) + if i>=n-1: + break + OP = np.asarray(OP) + nInf = np.sum(np.isinf(OP)) + if nInf>0: + sErr = 'Some ill-formated/infinite values (e.g. `*******`) were found in the vector `{}`\n\tin linflile: {}'.format(name, filename) + if starSub is None: + raise Exception(sErr) + else: + print('[WARN] '+sErr) + OP[np.isinf(OP)] = starSub + + Var['RotatingFrame'] = np.asarray(Var['RotatingFrame']) + Var['DerivativeOrder'] = np.asarray(Var['DerivativeOrder']) + Var['Description'] = np.asarray(Var['Description']) + return OP, Var + + + +def readMat(fid, n, m, name='', slowReader=False, filename='', starSubFn=None, starSub=None): + if not slowReader: + try: + return np.array([fid.readline().strip().split() for i in np.arange(n)],dtype=float) + except: + print('[INFO] Failed to read some value in matrix {}, trying slower reader'.format(name)) + raise SlowReaderNeededError() + else: + #vals = vals.ravel() + #vals = np.array(list(map(starSubFn, vals))).reshape(n,m) + vals=np.array([starSubFn( fid.readline().strip() ).split() for i in np.arange(n)], dtype=str) + try: + vals = vals.astype(float) # This could potentially fail + except: + raise Exception('Failed to convert into an array of float the matrix `{}`\n\tin linfile: {}'.format(name, filename)) + if vals.shape[0]!=n or vals.shape[1]!=m: + shape1 = vals.shape + shape2 = (n,m) + raise Exception('Shape of matrix `{}` has wrong dimension ({} instead of {})\n\tin linfile: {}'.format(name, shape1, shape2, name, filename)) + + nNaN = np.sum(np.isnan(vals.ravel())) + nInf = np.sum(np.isinf(vals.ravel())) + if nInf>0: + sErr = 'Some ill-formated/infinite values (e.g. `*******`) were found in the matrix `{}`\n\tin linflile: {}'.format(name, filename) + if starSub is None: + raise Exception(sErr) + else: + print('[WARN] '+sErr) + vals[np.isinf(vals)] = starSub + if nNaN>0: + raise Exception('Some NaN values were found in the matrix `{}`\n\tin linfile: `{}`.'.format(name, filename)) + return vals + +if __name__ == '__main__': + f = FASTLinearizationFile('../../data/example_files/StandstillSemi_ForID_EDHD.1.lin') + print(f) + _, zeta1, _, freq1 = f.eva() + f.removeStates(pattern=r'^AD') + print(f) + dfs = f.toDataFrame() + _, zeta2, _, freq2 = f.eva() + print('f',freq1) + print('f',freq2) + print('d',zeta1) + print('d',zeta2) diff --git a/pydatview/io/flex_out_file.py b/pydatview/io/flex_out_file.py index ea8eb38..86fc617 100644 --- a/pydatview/io/flex_out_file.py +++ b/pydatview/io/flex_out_file.py @@ -134,7 +134,7 @@ def read_flex_res(filename, dtype=np.float32): def read_flex_sensor(sensor_file): - with open(sensor_file, encoding="utf-8") as fid: + with open(sensor_file, 'r') as fid: sensor_info_lines = fid.readlines()[2:] sensor_info = [] d=dict({ 'ID':[],'Gain':[],'Offset':[],'Unit':[],'Name':[],'Description':[]}); diff --git a/pydatview/io/pickle_file.py b/pydatview/io/pickle_file.py index 3626093..97bd289 100644 --- a/pydatview/io/pickle_file.py +++ b/pydatview/io/pickle_file.py @@ -65,9 +65,14 @@ def _setData(self, data): else: self['data'] = data + def addDict(self, data): + self._setData(data) + + def additem(self, key, data): + self[key]=data + def read(self, filename=None, **kwargs): """ Reads the file self.filename, or `filename` if provided """ - # --- Standard tests and exceptions (generic code) if filename: self.filename = filename From 9e3347caacda2191e0dc5cf9679109e48d993b08 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 16:24:57 -0600 Subject: [PATCH 156/178] Scripter: more tools for lib-harmony and oneRawTab option --- pydatview/Tables.py | 23 +- pydatview/common.py | 2 - pydatview/fast/fastfarm.py | 2 +- pydatview/fast/postpro.py | 56 ++++- pydatview/pipeline.py | 9 +- pydatview/plugins/data_radialavg.py | 18 +- pydatview/plugins/data_standardizeUnits.py | 81 +------ .../plugins/tests/test_standardizeUnits.py | 3 +- pydatview/scripter.py | 74 +++++-- pydatview/tools/pandalib.py | 209 ++++++++++++++++++ 10 files changed, 340 insertions(+), 137 deletions(-) create mode 100644 pydatview/tools/pandalib.py diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 4c87046..5e301bb 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -473,25 +473,6 @@ def applyFormulas(self, formulas): # names_new.append(name_new) # except Exception as e: # errors.append('Filtering failed for table: '+t.nickname+'\n'+exception2string(e)) -# return dfs_new, names_new, errors -# -# # --- Radial average related -# def radialAvg(self,avgMethod,avgParam): -# """ Apply radial average on table list -# TODO Make this part of the action -# """ -# dfs_new = [] -# names_new = [] -# errors=[] -# for i,t in enumerate(self._tabs): -# try: -# dfs, names = t.radialAvg(avgMethod,avgParam) -# for df,n in zip(dfs,names): -# if df is not None: -# dfs_new.append(df) -# names_new.append(n) -# except Exception as e: -# errors.append('Radial averaging failed for table: '+t.nickname+'\n'+exception2string(e)) # return dfs_new, names_new, errors @@ -755,8 +736,8 @@ def changeUnits(self, data=None): if data is None: data={'flavor':'WE'} # NOTE: moved to a plugin, but interface kept - from pydatview.plugins.data_standardizeUnits import changeUnits - changeUnits(self, data=data) + from pydatview.plugins.data_standardizeUnits import changeUnitsTab + changeUnitsTab(self, data=data) def convertTimeColumns(self, dayfirst=False): if len(self.data)>0: diff --git a/pydatview/common.py b/pydatview/common.py index 721bec7..1d0a23d 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -519,8 +519,6 @@ def redraw (self, *args, **kwargs): Info(self.parent, 'This is dum if __name__ == '__main__': -# from welib.tools.clean_exceptions import * - try: raise Exception('Hello') except Exception as excp: diff --git a/pydatview/fast/fastfarm.py b/pydatview/fast/fastfarm.py index 0eb3ac4..f377382 100644 --- a/pydatview/fast/fastfarm.py +++ b/pydatview/fast/fastfarm.py @@ -7,7 +7,7 @@ from pydatview.io.fast_output_file import FASTOutputFile from pydatview.io.turbsim_file import TurbSimFile -from . import postpro as fastlib +from pydatview.fast import postpro as fastlib # --------------------------------------------------------------------------------} # --- Small helper functions diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 24855e5..7c24bfe 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -4,13 +4,16 @@ import numpy as np import re -# --- fast libraries + import pydatview.io as weio from pydatview.common import PyDatViewException as WELIBException from pydatview.io.fast_input_file import FASTInputFile from pydatview.io.fast_output_file import FASTOutputFile from pydatview.io.fast_input_deck import FASTInputDeck + +# --- fast libraries from pydatview.fast.subdyn import SubDyn +import pydatview.fast.fastfarm as fastfarm # --------------------------------------------------------------------------------} # --- Tools for IO @@ -907,6 +910,57 @@ def spanwisePostPro(FST_In=None,avgMethod='constantwindow',avgParam=5,out_ext='. # Combine all into a dictionary return out +def radialAvg(filename, avgMethod, avgParam, raw_name='', df=None, raiseException=True): + """ + Wrapper function, for instance used by pyDatView apply either: + spanwisePostPro or spanwisePostProFF (FAST.Farm) + """ + + base,out_ext = os.path.splitext(filename) + if df is None: + df = FASTOutputFile(filename).toDataFrame() + + # --- Detect if it's a FAST Farm file + sCols = ''.join(df.columns) + if sCols.find('WkDf')>1 or sCols.find('CtT')>0: + # --- FAST FARM files + Files=[base+ext for ext in ['.fstf','.FSTF','.Fstf','.fmas','.FMAS','.Fmas'] if os.path.exists(base+ext)] + if len(Files)==0: + fst_in=None + #raise Exception('Error: No .fstf file found with name: '+base+'.fstf') + else: + fst_in=Files[0] + + dfRad,_,dfDiam = fastfarm.spanwisePostProFF(fst_in,avgMethod=avgMethod,avgParam=avgParam,D=1,df=df) + dfs_new = [dfRad,dfDiam] + names_new=[raw_name+'_rad', raw_name+'_diam'] + else: + # --- FAST files + # HACK for AD file to find the right .fst file + iDotAD=base.lower().find('.ad') + if iDotAD>1: + base=base[:iDotAD] + # + Files=[base+ext for ext in ['.fst','.FST','.Fst','.dvr','.Dvr','.DVR'] if os.path.exists(base+ext)] + if len(Files)==0: + fst_in=None + #raise Exception('Error: No .fst file found with name: '+base+'.fst') + else: + fst_in=Files[0] + + try: + out = spanwisePostPro(fst_in, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df = df) + dfRadED=out['ED_bld']; dfRadAD = out['AD']; dfRadBD = out['BD'] + dfs_new = [dfRadAD, dfRadED, dfRadBD] + names_new=[raw_name+'_AD', raw_name+'_ED', raw_name+'_BD'] + except: + if raiseException: + raise + else: + print('[WARN] radialAvg failed for filename {}'.format(filename)) + dfs_new =[None] + names_new=[''] + return dfs_new, names_new def spanwisePostProRows(df, FST_In=None): """ diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index b579dbb..8c6f6e2 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -307,15 +307,18 @@ def script(self, tabList, scripterOptions=None, ID=None, subPlots=None, plotStyl scripterOptions={} # --- Files and number of tables + fileNamesNonEmpty = [f for f in tabList.filenames if len(f)>0] fileNames = np.unique(tabList.filenames) - fileNamesTrue = [t for t in fileNames if len(t)>0] + fileNamesTrue = [f for f in fileNames if len(f)>0] + scripterOptions['oneTabPerFile'] = False + scripterOptions['oneRawTab'] = False if len(fileNames)==len(fileNamesTrue): # No tables were added if len(fileNamesTrue) == len(tabList): # Each file returned one table only scripterOptions['oneTabPerFile'] = True - else: - scripterOptions['oneTabPerFile'] = False + if len(fileNamesNonEmpty) == 1: + scripterOptions['oneRawTab'] = True scripter = PythonScripter() scripter.setFiles(fileNamesTrue) diff --git a/pydatview/plugins/data_radialavg.py b/pydatview/plugins/data_radialavg.py index 0e5ce53..f55e3ff 100644 --- a/pydatview/plugins/data_radialavg.py +++ b/pydatview/plugins/data_radialavg.py @@ -1,4 +1,5 @@ import numpy as np +from pydatview.common import PyDatViewException from pydatview.plugins.base_plugin import GUIToolPanel, TOOL_BORDER from pydatview.plugins.base_plugin import ActionEditor @@ -68,22 +69,19 @@ def guiCallback(): # --------------------------------------------------------------------------------} # --- Main methods # --------------------------------------------------------------------------------{ -_imports = ["from pydatview.plugins.data_radialavg import radialAvg"] -_imports += ["from pydatview.Tables import Table"] +_imports = ["from pydatview.fast.postpro import radialAvg"] _data_var='dataRadialAvg' -_code = """# NOTE: this code relies on pydatview for now. It will be adapted for welib/pyFAST later. -# For the most part, the underlying functions are: -# dfRad,_,dfDiam = fastfarm.spanwisePostProFF(filename, avgMethod=avgMethod,avgParam=avgParam, D=1, df=df) -# out = fastlib.spanwisePostPro(filename, avgMethod=avgMethod, avgParam=avgParam, out_ext=out_ext, df=df) -tab=Table(data=df, filename=filename) -dfs_new, names_new = radialAvg(tab, dataRadialAvg) -""" +_code = """dfs_new, names_new = radialAvg(filename, avgMethod=dataRadialAvg['avgMethod'], avgParam=dataRadialAvg['avgParam'], df=df, raiseException=False)""" # add method def radialAvg(tab, data=None): """ NOTE: radial average may return several dataframe""" + from pydatview.fast.postpro import radialAvg as radialAvgPostPro #print('>>> RadialAvg',data) - dfs_new, names_new = tab.radialAvg(data['avgMethod'],data['avgParam']) + dfs_new, names_new = radialAvgPostPro(filename=tab.filename, df=tab.data, avgMethod=data['avgMethod'],avgParam=data['avgParam']) + if all(df is None for df in dfs_new): + raise PyDatViewException('No OpenFAST radial data found for table: '+tab.nickname) + return dfs_new, names_new # --------------------------------------------------------------------------------} diff --git a/pydatview/plugins/data_standardizeUnits.py b/pydatview/plugins/data_standardizeUnits.py index 0b664ee..f81ad60 100644 --- a/pydatview/plugins/data_standardizeUnits.py +++ b/pydatview/plugins/data_standardizeUnits.py @@ -27,7 +27,7 @@ def guiCallback(): action = IrreversibleTableAction( name=label, - tableFunctionApply=changeUnits, + tableFunctionApply=changeUnitsTab, guiCallback=guiCallback, mainframe=mainframe, # shouldnt be needed data = data , @@ -41,82 +41,13 @@ def guiCallback(): # --------------------------------------------------------------------------------} # --- Main method # --------------------------------------------------------------------------------{ -_imports=['from pydatview.plugins.data_standardizeUnits import changeUnits'] -# _imports+=['from pydatview.Tables import Table'] +_imports=['from pydatview.tools.pandalib import changeUnits'] _data_var='changeUnitsData' -_code="""changeUnits(df, changeUnitsData)""" +_code="""changeUnits(df, flavor=changeUnitsData['flavor'])""" -def changeUnits(tab, data): - """ Change units of a table - NOTE: it relies on the Table class, which may change interface in the future.. - """ - if not isinstance(tab, pd.DataFrame): - df = tab.data - else: - df = tab - - if data['flavor']=='WE': - cols = [] - for i, colname in enumerate(df.columns): - colname_new, df.iloc[:,i] = change_units_to_WE(colname, df.iloc[:,i]) - cols.append(colname_new) - df.columns = cols - elif data['flavor']=='SI': - cols = [] - for i, colname in enumerate(df.columns): - colname_new, df.iloc[:,i] = change_units_to_SI(colname, df.iloc[:,i]) - cols.append(colname_new) - df.columns = cols - else: - raise NotImplementedError(data['flavor']) - -def change_units_to_WE(s, c): - """ - Change units to wind energy units - s: channel name (string) containing units, typically 'speed_[rad/s]' - c: channel (array) - """ - svar, u = splitunit(s) - u=u.lower() - scalings = {} - # OLD = NEW - scalings['rad/s'] = (30/np.pi,'rpm') # TODO decide - scalings['rad' ] = (180/np.pi,'deg') - scalings['n'] = (1e-3, 'kN') - scalings['nm'] = (1e-3, 'kNm') - scalings['n-m'] = (1e-3, 'kNm') - scalings['n*m'] = (1e-3, 'kNm') - scalings['w'] = (1e-3, 'kW') - if u in scalings.keys(): - scale, new_unit = scalings[u] - s = svar+'['+new_unit+']' - c *= scale - return s, c - -def change_units_to_SI(s, c): - """ - Change units to SI units - TODO, a lot more units conversion needed...will add them as we go - s: channel name (string) containing units, typically 'speed_[rad/s]' - c: channel (array) - """ - svar, u = splitunit(s) - u=u.lower() - scalings = {} - # OLD = NEW - scalings['rpm'] = (np.pi/30,'rad/s') - scalings['rad' ] = (180/np.pi,'deg') - scalings['deg/s' ] = (np.pi/180,'rad/s') - scalings['kn'] = (1e3, 'N') - scalings['knm'] = (1e3, 'Nm') - scalings['kn-m'] = (1e3, 'Nm') - scalings['kn*m'] = (1e3, 'Nm') - scalings['kw'] = (1e3, 'W') - if u in scalings.keys(): - scale, new_unit = scalings[u] - s = svar+'['+new_unit+']' - c *= scale - return s, c +def changeUnitsTab(tab, data): + from pydatview.tools.pandalib import changeUnits + changeUnits(tab.data, flavor=data['flavor']) if __name__ == '__main__': diff --git a/pydatview/plugins/tests/test_standardizeUnits.py b/pydatview/plugins/tests/test_standardizeUnits.py index 841ee3b..50a36ad 100644 --- a/pydatview/plugins/tests/test_standardizeUnits.py +++ b/pydatview/plugins/tests/test_standardizeUnits.py @@ -6,6 +6,7 @@ class TestChangeUnits(unittest.TestCase): def test_change_units(self): + # TODO move this as a pandalib test from pydatview.Tables import Table data = np.ones((1,3)) data[:,0] *= 2*np.pi/60 # rad/s @@ -13,7 +14,7 @@ def test_change_units(self): data[:,2] *= 10*np.pi/180 # rad df = pd.DataFrame(data=data, columns=['om [rad/s]','F [N]', 'angle_[rad]']) tab=Table(data=df) - changeUnits(tab, data={'flavor':'WE'}) + changeUnitsTab(tab, data={'flavor':'WE'}) np.testing.assert_almost_equal(tab.data.values[:,1],[1]) np.testing.assert_almost_equal(tab.data.values[:,2],[2]) np.testing.assert_almost_equal(tab.data.values[:,3],[10]) diff --git a/pydatview/scripter.py b/pydatview/scripter.py index 706174f..caf521d 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -8,8 +8,8 @@ 'pydatview.fast.postpro':'welib.fast.postpro', } _PYFAST={ - 'pydatview.io':'pyFAST.input_output', - 'pydatview.tools.tictoc':'pyFAST.tools.tictoc', + 'pydatview.io' :'pyFAST.input_output', + 'pydatview.tools' :'pyFAST.tools', 'pydatview.fast.postpro':'pyFAST.postpro', # I think... } @@ -23,6 +23,7 @@ 'libFlavor':'pydatview', 'dfsFlavor':'dict', 'oneTabPerFile':False, + 'oneRawTab':False, 'indent':' ', 'verboseCommentLevel':1, } @@ -187,6 +188,15 @@ def forLoopOnDFs(): elif self.opts['dfsFlavor'] == 'list': script.append("for df in dfs:") + def onlyOneFile(): + return len(self.filenames)==1 + def oneTabPerFile(): + return self.opts['oneTabPerFile'] + def oneRawTab(): + return self.opts['oneRawTab'] + def dontListFiles(): + return oneRawTab() or (onlyOneFile() and oneTabPerFile()) + script = [] @@ -215,8 +225,7 @@ def forLoopOnDFs(): # --- List of files script.append("\n# --- Script parameters") - nFiles = len(self.filenames) - if nFiles==1 and self.opts['oneTabPerFile']: + if dontListFiles(): script.append("filename = '{}'".format(self.filenames[0])) else: script.append("filenames = []") @@ -255,7 +264,7 @@ def forLoopOnDFs(): # --- List of Dataframes script.append("\n# --- Open and convert files to DataFrames") if self.opts['dfsFlavor'] == 'dict': - if nFiles==1 and self.opts['oneTabPerFile']: + if dontListFiles(): script.append("dfs = {}") script.append("dfs[0] = weio.read(filename).toDataFrame()") else: @@ -273,7 +282,7 @@ def forLoopOnDFs(): script.append(indent2 + "dfs[f'tab{iFile}'] = dfs_or_df") elif self.opts['dfsFlavor'] == 'list': script.append("dfs = []") - if nFiles==1 and self.opts['oneTabPerFile']: + if dontListFiles(): script.append("dfs.append( weio.read(filename).toDataFrame() )") else: script.append("for iFile, filename in enumerate(filenames):") @@ -289,7 +298,7 @@ def forLoopOnDFs(): script.append(indent2 + "dfs.append(dfs_or_df)") elif self.opts['dfsFlavor'] == 'enumeration': - if nFiles==1 and self.opts['oneTabPerFile']: + if dontListFiles(): script.append(f"df1 = weio.read(filename).toDataFrame()") else: for iFile, filename in enumerate(self.filenames): @@ -317,32 +326,51 @@ def addActionCode(actioname, actioncode, ind): script.append("\n# --- Apply adder actions to dataframes") script.append("dfs_add = [] ; names_add =[]") if self.opts['dfsFlavor'] == 'dict': - script.append("for k, (key, df) in enumerate(dfs.items()):") - script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") - for actionname, actioncode in self.adder_actions.items(): - addActionCode(actionname, actioncode[1], indent1) - script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") + if dontListFiles(): + script.append('df = dfs[0] # NOTE: we assume that only one dataframe is present' ) + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], indent0) + script.append(indent0+"dfs_add += dfs_new ; names_add += names_new") + else: + script.append("for k, (key, df) in enumerate(dfs.items()):") + script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], indent1) + script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") script.append("for name_new, df_new in zip(names_add, dfs_new):") script.append(indent1+"if df_new is not None:") script.append(indent2+"dfs[name_new] = df_new") elif self.opts['dfsFlavor'] == 'list': - script.append("for k, df in enumerate(dfs):") - script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") - for actionname, actioncode in self.adder_actions.items(): - addActionCode(actionname, actioncode[1], indent1) - script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") + if dontListFiles(): + script.append('df = dfs[0]') + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], indent0) + script.append(indent0+"dfs_add += dfs_new ; names_add += names_new") + else: + script.append("for k, df in enumerate(dfs):") + script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], indent1) + script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") script.append("for name_new, df_new in zip(names_add, dfs_new):") script.append(indent1+"if df_new is not None:") script.append(indent2+"dfs += [df_new]") elif self.opts['dfsFlavor'] == 'enumeration': - for iTab in range(nTabs): - script.append("filename = filenames[{}] # NOTE: this is approximate..".format(iTab)) - script.append('df = df{}'.format(iTab+1)) - for actionname, actioncode in self.adder_actions.items(): - addActionCode(actionname, actioncode[1], '') - script.append("df{} = dfs_new[0] # NOTE: we only keep the first table here..".format(nTabs+iTab+1)) + if dontListFiles(): + for iTab in range(nTabs): + script.append('df = df{}'.format(iTab+1)) + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], '') + script.append("df{} = dfs_new[0] # NOTE: we only keep the first table here..".format(nTabs+iTab+1)) + else: + for iTab in range(nTabs): + script.append("filename = filenames[{}] # NOTE: this is approximate..".format(iTab)) + script.append('df = df{}'.format(iTab+1)) + for actionname, actioncode in self.adder_actions.items(): + addActionCode(actionname, actioncode[1], '') + script.append("df{} = dfs_new[0] # NOTE: we only keep the first table here..".format(nTabs+iTab+1)) nTabs += nTabs diff --git a/pydatview/tools/pandalib.py b/pydatview/tools/pandalib.py new file mode 100644 index 0000000..5da1833 --- /dev/null +++ b/pydatview/tools/pandalib.py @@ -0,0 +1,209 @@ +import pandas as pd +import numpy as np +import re + + +def pd_interp1(x_new, xLabel, df): + """ Interpolate a panda dataframe based on a set of new value + This function assumes that the dataframe is a simple 2d-table + """ + from pyFAST.tools.signal_analysis import multiInterp + x_old = df[xLabel].values + data_new=multiInterp(x_new, x_old, df.values.T) + return pd.DataFrame(data=data_new.T, columns=df.columns.values) + #nRow,nCol = df.shape + #nRow = len(xnew) + #data = np.zeros((nRow,nCol)) + #xref =df[xLabel].values.astype(float) + #for col,i in zip(df.columns.values,range(nCol)): + # yref = df[col].values + # if yref.dtype!=float: + # raise Exception('Wrong type for yref, consider using astype(float)') + # data[:,i] = np.interp(xnew, xref, yref) + #return pd.DataFrame(data=data, columns = df.columns) + +def create_dummy_dataframe(size): + return pd.DataFrame(data={'col1': np.linspace(0,1,size), 'col2': np.random.normal(0,1,size)}) + + + +def remap_df(df, ColMap, bColKeepNewOnly=False, inPlace=False, dataDict=None, verbose=False): + """ + NOTE: see welib.fast.postpro + + Add/rename columns of a dataframe, potentially perform operations between columns + + dataDict: dictionary of data to be made available as "variable" in the column mapping + 'key' (new) : value (old) + + Example: + + ColumnMap={ + 'WS_[m/s]' : '{Wind1VelX_[m/s]}' , # create a new column from existing one + 'RtTSR_[-]' : '{RtTSR_[-]} * 2 + {RtAeroCt_[-]}' , # change value of column + 'RotSpeed_[rad/s]' : '{RotSpeed_[rpm]} * 2*np.pi/60 ', # new column [rpm] -> [rad/s] + 'q_p' : ['Q_P_[rad]', '{PtfmSurge_[deg]}*np.pi/180'] # List of possible matches + } + # Read + df = weio.read('FASTOutBin.outb').toDataFrame() + # Change columns based on formulae, potentially adding new columns + df = fastlib.remap_df(df, ColumnMap, inplace=True) + + """ + # Insert dataDict into namespace + if dataDict is not None: + for k,v in dataDict.items(): + exec('{:s} = dataDict["{:s}"]'.format(k,k)) + + + if not inPlace: + df=df.copy() + ColMapMiss=[] + ColNew=[] + RenameMap=dict() + # Loop for expressions + for k0,v in ColMap.items(): + k=k0.strip() + if type(v) is not list: + values = [v] + else: + values = v + Found = False + for v in values: + v=v.strip() + if Found: + break # We avoid replacing twice + if v.find('{')>=0: + # --- This is an advanced substitution using formulae + search_results = re.finditer(r'\{.*?\}', v) + expr=v + if verbose: + print('Attempt to insert column {:15s} with expr {}'.format(k,v)) + # For more advanced operations, we use an eval + bFail=False + for item in search_results: + col=item.group(0)[1:-1] + if col not in df.columns: + ColMapMiss.append(col) + bFail=True + expr=expr.replace(item.group(0),'df[\''+col+'\']') + #print(k0, '=', expr) + if not bFail: + df[k]=eval(expr) + ColNew.append(k) + else: + print('[WARN] Column not present in dataframe, cannot evaluate: ',expr) + else: + #print(k0,'=',v) + if v not in df.columns: + ColMapMiss.append(v) + if verbose: + print('[WARN] Column not present in dataframe: ',v) + else: + if k in RenameMap.keys(): + print('[WARN] Not renaming {} with {} as the key is already present'.format(k,v)) + else: + RenameMap[k]=v + Found=True + + # Applying renaming only now so that expressions may be applied in any order + for k,v in RenameMap.items(): + if verbose: + print('Renaming column {:15s} > {}'.format(v,k)) + k=k.strip() + iCol = list(df.columns).index(v) + df.columns.values[iCol]=k + ColNew.append(k) + df.columns = df.columns.values # Hack to ensure columns are updated + + if len(ColMapMiss)>0: + print('[FAIL] The following columns were not found in the dataframe:',ColMapMiss) + #print('Available columns are:',df.columns.values) + + if bColKeepNewOnly: + ColNew = [c for c,_ in ColMap.items() if c in ColNew]# Making sure we respec order from user + ColKeepSafe = [c for c in ColNew if c in df.columns.values] + ColKeepMiss = [c for c in ColNew if c not in df.columns.values] + if len(ColKeepMiss)>0: + print('[WARN] Signals missing and omitted for ColKeep:\n '+'\n '.join(ColKeepMiss)) + df=df[ColKeepSafe] + return df + +def changeUnits(df, flavor='SI', inPlace=True): + """ Change units of a dataframe + + # TODO harmonize with dfToSIunits in welib.fast.tools.lin.py ! + """ + def splitunit(s): + iu=s.rfind('[') + if iu>0: + return s[:iu], s[iu+1:].replace(']','') + else: + return s, '' + def change_units_to_WE(s, c): + """ + Change units to wind energy units + s: channel name (string) containing units, typically 'speed_[rad/s]' + c: channel (array) + """ + svar, u = splitunit(s) + u=u.lower() + scalings = {} + # OLD = NEW + scalings['rad/s'] = (30/np.pi,'rpm') # TODO decide + scalings['rad' ] = (180/np.pi,'deg') + scalings['n'] = (1e-3, 'kN') + scalings['nm'] = (1e-3, 'kNm') + scalings['n-m'] = (1e-3, 'kNm') + scalings['n*m'] = (1e-3, 'kNm') + scalings['w'] = (1e-3, 'kW') + if u in scalings.keys(): + scale, new_unit = scalings[u] + s = svar+'['+new_unit+']' + c *= scale + return s, c + + def change_units_to_SI(s, c): + """ + Change units to SI units + TODO, a lot more units conversion needed...will add them as we go + s: channel name (string) containing units, typically 'speed_[rad/s]' + c: channel (array) + """ + svar, u = splitunit(s) + u=u.lower() + scalings = {} + # OLD = NEW + scalings['rpm'] = (np.pi/30,'rad/s') + scalings['rad' ] = (180/np.pi,'deg') + scalings['deg/s' ] = (np.pi/180,'rad/s') + scalings['kn'] = (1e3, 'N') + scalings['knm'] = (1e3, 'Nm') + scalings['kn-m'] = (1e3, 'Nm') + scalings['kn*m'] = (1e3, 'Nm') + scalings['kw'] = (1e3, 'W') + if u in scalings.keys(): + scale, new_unit = scalings[u] + s = svar+'['+new_unit+']' + c *= scale + return s, c + + if not inPlace: + raise NotImplementedError() + + if flavor == 'WE': + cols = [] + for i, colname in enumerate(df.columns): + colname_new, df.iloc[:,i] = change_units_to_WE(colname, df.iloc[:,i]) + cols.append(colname_new) + df.columns = cols + elif flavor == 'SI': + cols = [] + for i, colname in enumerate(df.columns): + colname_new, df.iloc[:,i] = change_units_to_SI(colname, df.iloc[:,i]) + cols.append(colname_new) + df.columns = cols + else: + raise NotImplementedError(flavor) + return df + From e8f483938beb581fb56d456d28ea233ef6ecad2d Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 16:54:05 -0600 Subject: [PATCH 157/178] Scripter: adding help (see #170) --- pydatview/GUIScripter.py | 52 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/pydatview/GUIScripter.py b/pydatview/GUIScripter.py index 4ef7e9b..b1fcca4 100644 --- a/pydatview/GUIScripter.py +++ b/pydatview/GUIScripter.py @@ -1,5 +1,49 @@ import wx import wx.stc as stc +from pydatview.common import CHAR, Info + +_HELP = """Script generation + +Generates a python code that perform the same actions and plot as pyDatView. + +Disclaimer: + Not all of pyDatView features are supported and the user will likely have to adjust the code + manually in order to get a fully working script. + +This feature is intended to: + - Generate a concise code (therefore with limited error handling) + - Include most of the actions from pyDatView pipelines. + - Ease the transition from pyDatView to python scripting. + +Feature: + - Save the python script to a file (button "Save to file") + - Attempt to execute/test the script using a system call (button "Run Script") + - Update the code if you change things in pyDatView (button "Update") + - Modify it directly in the text editor + - Modify it using some of the menu provided to change how the code is generated. + (see Options below) + +Requirements: + You need to be familiar with git and python. + You need to install ONE of the following python library: + - library= welib, repository= https://github.com/ebranlard/welib + - library= pydatview, repository= https://github.com/ebranlard/pyDatView + - library= pyFAST, repository= https://github.com/openfast/python-toolbox + You can install a given library as follows: + git clone repository # Replace repository with the address above + cd library # Replace library with the folder generated after cloning + pip install -e . # Note the "." + Make sure to chose the library you installed using the Options (see below) + +Options: + - Library: chose the library you want to use (will affect the import statements) + See "Requirements" for the different library and how to install them. + - DF storage: chose how you want to store/name the pandas dataframes. + - "enumeration" is the simplest: df1, df2, etc. + - "dict" will store the dataframes in dictionaries + - "list" will store the dataframes in a list dfs[0], dfs[1], etc. + - Comment level: the verbosity of the comments in the code. +""" class GUIScripterFrame(wx.Frame): def __init__(self, parent, mainframe, pipeLike, title): @@ -15,7 +59,7 @@ def __init__(self, parent, mainframe, pipeLike, title): self.text_ctrl = stc.StyledTextCtrl(self.panel, style=wx.TE_MULTILINE) self.setup_syntax_highlighting() - + self.btHelp= wx.Button(self.panel, label=CHAR['help']+' '+"Help", style=wx.BU_EXACTFIT) self.btGen = wx.Button(self.panel, label="Update") self.btRun = wx.Button(self.panel, label="Run Script (beta)") self.btSave = wx.Button(self.panel, label="Save to File") @@ -42,6 +86,7 @@ def __init__(self, parent, mainframe, pipeLike, title): vbox.Add(self.text_ctrl, proportion=1, flag=wx.EXPAND | wx.ALL, border=2) + hbox.Add(self.btHelp, flag=wx.LEFT|wx.ALIGN_CENTER_VERTICAL, border=5) hbox.Add( txtLib, flag=wx.LEFT|wx.ALIGN_CENTER_VERTICAL, border=5) hbox.Add(self.cbLib, proportion=1, flag=wx.EXPAND|wx.ALL, border=5) hbox.Add( txtDFS, flag=wx.LEFT|wx.ALIGN_CENTER_VERTICAL, border=5) @@ -56,6 +101,7 @@ def __init__(self, parent, mainframe, pipeLike, title): self.panel.SetSizerAndFit(vbox) # -- Binding + self.btHelp.Bind(wx.EVT_BUTTON, self.onHelp) self.btSave.Bind(wx.EVT_BUTTON, self.onSave) self.btRun.Bind(wx.EVT_BUTTON, self.onRun) self.btGen.Bind(wx.EVT_BUTTON, self.generateScript) @@ -65,6 +111,7 @@ def __init__(self, parent, mainframe, pipeLike, title): self.generateScript() + def _GUI2Data(self, *args, **kwargs): # GUI2Data data={} @@ -146,6 +193,9 @@ def onSave(self, event): dialog.Destroy() + def onHelp(self,event=None): + Info(self, _HELP) + def setup_syntax_highlighting(self): # --- Basic mono_font = wx.Font(10, wx.FONTFAMILY_TELETYPE, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL) From d1303a472a5db7e3b60dde705eb507a005550da0 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 16:57:01 -0600 Subject: [PATCH 158/178] Plugins: Fix changeUnit test after renaming of the function --- pydatview/plugins/tests/test_standardizeUnits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatview/plugins/tests/test_standardizeUnits.py b/pydatview/plugins/tests/test_standardizeUnits.py index 50a36ad..bfa9540 100644 --- a/pydatview/plugins/tests/test_standardizeUnits.py +++ b/pydatview/plugins/tests/test_standardizeUnits.py @@ -1,7 +1,7 @@ import unittest import numpy as np import pandas as pd -from pydatview.plugins.data_standardizeUnits import changeUnits +from pydatview.plugins.data_standardizeUnits import changeUnitsTab class TestChangeUnits(unittest.TestCase): From 978131696b6b8246118a4ec0720b56e2e955e45a Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 17:02:50 -0600 Subject: [PATCH 159/178] Scripter: unique filenames preserving order (see #170) --- pydatview/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index 8c6f6e2..3f3d20f 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -308,8 +308,8 @@ def script(self, tabList, scripterOptions=None, ID=None, subPlots=None, plotStyl # --- Files and number of tables fileNamesNonEmpty = [f for f in tabList.filenames if len(f)>0] - fileNames = np.unique(tabList.filenames) - fileNamesTrue = [f for f in fileNames if len(f)>0] + fileNames = np.unique(tabList.filenames) + fileNamesTrue = list(dict.fromkeys(fileNamesNonEmpty)) # Unique and non Empty, preserving order scripterOptions['oneTabPerFile'] = False scripterOptions['oneRawTab'] = False if len(fileNames)==len(fileNamesTrue): From d898055396c865424b3f54bfcbd3b8c820359c5c Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 17:30:58 -0600 Subject: [PATCH 160/178] Pipeline: not coloring links for users with dark background --- pydatview/GUIPipelinePanel.py | 6 +++--- pydatview/fast/postpro.py | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pydatview/GUIPipelinePanel.py b/pydatview/GUIPipelinePanel.py index 28d391b..a081376 100644 --- a/pydatview/GUIPipelinePanel.py +++ b/pydatview/GUIPipelinePanel.py @@ -21,7 +21,7 @@ def __init__(self, parent, action, style=wx.TAB_TRAVERSAL): lko = hl.HyperLinkCtrl(self, -1, name) lko.AutoBrowse(False) lko.SetUnderlines(False, False, False) - lko.SetColours(wx.BLACK, wx.BLACK, wx.BLACK) + #lko.SetColours(wx.BLACK, wx.BLACK, wx.BLACK) lko.DoPopup(False) #lko.SetBold(True) lko.SetToolTip(wx.ToolTip('Change "'+name+'"')) @@ -30,7 +30,7 @@ def __init__(self, parent, action, style=wx.TAB_TRAVERSAL): lkc = hl.HyperLinkCtrl(self, -1, 'x') lkc.AutoBrowse(False) lkc.EnableRollover(True) - lkc.SetColours(wx.BLACK, wx.BLACK, (200,0,0)) + #lkc.SetColours(wx.BLACK, wx.BLACK, (200,0,0)) lkc.DoPopup(False) #lkc.SetBold(True) lkc.SetToolTip(wx.ToolTip('Remove "'+name+'"')) @@ -67,7 +67,7 @@ def __init__(self, parent, pipeline, style=wx.TAB_TRAVERSAL): lke = hl.HyperLinkCtrl(self, -1, 'Errors (0)') lke.AutoBrowse(False) lke.EnableRollover(True) - lke.SetColours(wx.BLACK, wx.BLACK, (200,0,0)) + #lke.SetColours(wx.BLACK, wx.BLACK, (200,0,0)) lke.DoPopup(False) #lkc.SetBold(True) lke.SetToolTip(wx.ToolTip('View errors.')) diff --git a/pydatview/fast/postpro.py b/pydatview/fast/postpro.py index 7c24bfe..840fd11 100644 --- a/pydatview/fast/postpro.py +++ b/pydatview/fast/postpro.py @@ -4,14 +4,13 @@ import numpy as np import re - import pydatview.io as weio from pydatview.common import PyDatViewException as WELIBException + +# --- fast libraries from pydatview.io.fast_input_file import FASTInputFile from pydatview.io.fast_output_file import FASTOutputFile from pydatview.io.fast_input_deck import FASTInputDeck - -# --- fast libraries from pydatview.fast.subdyn import SubDyn import pydatview.fast.fastfarm as fastfarm From d247d1e8ec04c2c385f473d2ac5634a5cd47378d Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 17:33:59 -0600 Subject: [PATCH 161/178] Plugin: LogDec: bug fix seValue instead of setValue --- pydatview/plugins/tool_logdec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatview/plugins/tool_logdec.py b/pydatview/plugins/tool_logdec.py index 5834349..29a742b 100644 --- a/pydatview/plugins/tool_logdec.py +++ b/pydatview/plugins/tool_logdec.py @@ -151,7 +151,7 @@ def onCompute(self,event=None): # if True: fn, zeta, info = freqDampEstimator(x, t, opts=self.data) except: - self.results.SeValue('Failed. The signal needs to look like the decay of a first order system.') + self.results.SetValue('Failed. The signal needs to look like the decay of a first order system.') event.Skip() return From 4cdf935e12468341685d9827a76a174fe4a39faf Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 20 Oct 2023 17:40:14 -0600 Subject: [PATCH 162/178] Scripter: update of help for windows exe --- pydatview/GUIScripter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydatview/GUIScripter.py b/pydatview/GUIScripter.py index b1fcca4..37a6b0d 100644 --- a/pydatview/GUIScripter.py +++ b/pydatview/GUIScripter.py @@ -24,7 +24,8 @@ (see Options below) Requirements: - You need to be familiar with git and python. + If you used an windows installer (".exe"), use the "pyDatView" library (see "Options"). + Otherwise, you need to be familiar with git and python. You need to install ONE of the following python library: - library= welib, repository= https://github.com/ebranlard/welib - library= pydatview, repository= https://github.com/ebranlard/pyDatView From b0b1e9d6b21b1518af340ef2d2df63b63a61d92a Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 24 Oct 2023 19:56:55 -0600 Subject: [PATCH 163/178] Curve Fitting: bug fix when user change keys in constants (see #172) --- pydatview/main.py | 2 +- pydatview/plugins/tool_curvefitting.py | 11 +++++++++-- pydatview/tools/curve_fitting.py | 5 +++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pydatview/main.py b/pydatview/main.py index eced2d4..95e37a9 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -1005,7 +1005,7 @@ def showApp(firstArg=None, dataframes=None, filenames=[], names=None): err = 'Errors occured while loading files:\n\n' err += exception2string(e) - #frame.onShowTool(toolName='') + #frame.onShowTool(toolName='Curve fitting') #frame.onDataPlugin(toolName='Radial Average') #frame.onDataPlugin(toolName='Resample') #frame.onScript() diff --git a/pydatview/plugins/tool_curvefitting.py b/pydatview/plugins/tool_curvefitting.py index 4d923d5..3130998 100644 --- a/pydatview/plugins/tool_curvefitting.py +++ b/pydatview/plugins/tool_curvefitting.py @@ -1,8 +1,9 @@ import wx import numpy as np +import pandas as pd import copy from pydatview.plugins.base_plugin import GUIToolPanel -from pydatview.common import Error, Info, pretty_num_short +from pydatview.common import Error, Info, pretty_num_short, PyDatViewException from pydatview.tools.curve_fitting import model_fit, extract_key_miscnum, extract_key_num, MODELS, FITTERS, set_common_keys @@ -193,7 +194,13 @@ def onCurveFit(self,event=None): bounds=self.textBounds.GetLineText(0).replace('np.inf','inf') # Guess p0=self.textGuess.GetLineText(0).replace('np.inf','inf') - fun_kwargs=extract_key_num(self.textConstants.GetLineText(0).replace('np.inf','inf')) + fun_kwargs = extract_key_num(self.textConstants.GetLineText(0).replace('np.inf','inf')) + fun_kwargs_ref = extract_key_num(d['consts']) + if set(fun_kwargs_ref.keys()) != set(fun_kwargs.keys()): + Error(self, 'The Field `Constants` should contain the keys: {}'.format(list(fun_kwargs_ref.keys()))) + return + + #print('>>> Model fit sFunc :',sFunc ) #print('>>> Model fit p0 :',p0 ) #print('>>> Model fit bounds:',bounds ) diff --git a/pydatview/tools/curve_fitting.py b/pydatview/tools/curve_fitting.py index 64c71ee..b92cf6e 100644 --- a/pydatview/tools/curve_fitting.py +++ b/pydatview/tools/curve_fitting.py @@ -364,6 +364,7 @@ def gentorque(x, p): return GenTrq +# TODO TODO TODO WHY DID I GO FOR STRINGS AND NOT DICTIONARIES FOR CONSTS AND BOUNDS???? MODELS =[ # {'label':'User defined model', # 'name':'eval:', @@ -483,7 +484,7 @@ def model_fit(func, x, y, p0=None, bounds=None, **fun_kwargs): consts = FITTERS[i]['consts'] args, missing = set_common_keys(consts, fun_kwargs) if len(missing)>0: - raise Exception('Curve fitting with `{}` requires the following arguments {}. Missing: {}'.format(func,consts.keys(),missing)) + raise Exception('Curve fitting with `{}` requires the following arguments {}. Missing: {}'.format(func, list(consts.keys()), missing)) # Calling the class fitter = FitterDict['handle'](x=x, y=y, p0=p0, bounds=bounds, **fun_kwargs) else: @@ -586,7 +587,7 @@ def func(x, p): self.model['consts'], missing = set_common_keys(self.model['consts'], fun_kwargs ) if len(missing)>0: - raise Exception('Curve fitting with function `{}` requires the following arguments {}. Missing: {}'.format(func.__name__,consts.keys(),missing)) + raise Exception('Curve fitting with function `{}` requires the following arguments {}. Missing: {}'.format(self.model['name'],list(self.model['consts'].keys()),missing)) def setup_bounds(self, bounds, nParams): if bounds is not None: From cd11ea5f731cbcf55437f2b490ddf1ff4d202fb4 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 24 Oct 2023 22:25:52 -0600 Subject: [PATCH 164/178] Tables: improved parsing of strings and datetimes --- pydatview/Tables.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 5e301bb..7f7a1dc 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -1,6 +1,7 @@ import numpy as np import os.path from dateutil import parser +import datetime import pandas as pd from pydatview.common import no_unit, ellude_common, getDt, exception2string, PyDatViewException import pydatview.io as weio # File Formats and File Readers @@ -740,11 +741,32 @@ def changeUnits(self, data=None): changeUnitsTab(self, data=data) def convertTimeColumns(self, dayfirst=False): + + def convertTimeColumn(c): + print('[INFO] Converting column {} to datetime, dayfirst: {}. May take a while...'.format(c, dayfirst)) + try: + # TODO THIS CAN BE VERY SLOW... + self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst, infer_datetime_format=True).to_pydatetime() + print(' Done.') + except: + try: + print('[FAIL] Attempting without infer datetime. May take a while...') + self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst, infer_datetime_format=False).to_pydatetime() + print(' Done.') + except: + # Happens if values are e.g. "Monday, Tuesday" + print('[FAIL] Inferring column as string instead') + + if len(self.data)>0: for i,c in enumerate(self.data.columns.values): y = self.data.iloc[:,i] if y.dtype == object: - if isinstance(y.values[0], str): + if isinstance(y.values[0], datetime.datetime): + isDate=True + convertTimeColumn(c) + + elif isinstance(y.values[0], str): # tring to convert to date try: vals = parser.parse(y.values[0]) @@ -755,21 +777,10 @@ def convertTimeColumns(self, dayfirst=False): else: isDate=False if isDate: - print('[INFO] Converting column {} to datetime, dayfirst: {}. May take a while...'.format(c, dayfirst)) - try: - # TODO THIS CAN BE VERY SLOW... - self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst, infer_datetime_format=True).to_pydatetime() - print(' Done.') - except: - try: - print('[FAIL] Attempting without infer datetime. May take a while...') - self.data[c]=pd.to_datetime(self.data[c].values, dayfirst=dayfirst, infer_datetime_format=False).to_pydatetime() - print(' Done.') - except: - # Happens if values are e.g. "Monday, Tuesday" - print('[FAIL] Inferring column as string instead') + convertTimeColumn(c) else: print('Column {} inferred as string'.format(c)) + self.data[c] = self.data[c].str.strip() elif isinstance(y.values[0], (float, int)): try: self.data[c]=self.data[c].astype(float) @@ -777,7 +788,7 @@ def convertTimeColumns(self, dayfirst=False): except: self.data[c]=self.data[c].astype(str) print('Column {} inferred and converted to string'.format(c)) - else : + else: print('>> Unknown type:',type(y.values[0])) #print(self.data.dtypes) From 0bd0b9ac2366eeac14a7c373e00b7b796fd2fa61 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 24 Oct 2023 22:26:37 -0600 Subject: [PATCH 165/178] Measure: avoid issues with dates and strings --- pydatview/GUIMeasure.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pydatview/GUIMeasure.py b/pydatview/GUIMeasure.py index c47bdd0..7fc95d2 100644 --- a/pydatview/GUIMeasure.py +++ b/pydatview/GUIMeasure.py @@ -106,8 +106,11 @@ def plotPoint(self, ax, xc, yc, ms=3): def plotLine(self, ax): """ plot vertical line across axis""" - line = ax.axvline(x=self.P_target_raw[0], color=self.color, linewidth=0.5) - self.lines.append(line) + try: + line = ax.axvline(x=self.P_target_raw[0], color=self.color, linewidth=0.5) + self.lines.append(line) + except: + print('[FAIL] GUIMeasure: cannot plot line', self.P_target_raw[0]) def plot(self, axes, PD): """ From 3a5c6f8a7c8f8e94cd3da3a774f5feee1e393128 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 24 Oct 2023 22:28:40 -0600 Subject: [PATCH 166/178] GUIPlotPanel: limit confusion between plotdata and pandas --- pydatview/GUIPlotPanel.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index 122ed35..b36c5b0 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -999,12 +999,12 @@ def setPD_MinMax(self,PD): self.mmxPanel.cbxMinMax.SetValue(False) raise e # Used to be Warn - def setPD_FFT(self,pd): + def setPD_FFT(self, PD): """ Convert plot data to FFT data based on GUI options""" data = self.spcPanel._GUI2Data() # Convert plotdata to FFT data try: - Info = pd.toFFT(**data) + Info = PD.toFFT(**data) # Trigger if hasattr(Info,'nExp') and Info.nExp!=data['nExp']: self.spcPanel.scP2.SetValue(Info.nExp) @@ -1015,7 +1015,7 @@ def setPD_FFT(self,pd): raise e - def transformPlotData(self,PD): + def transformPlotData(self, PD): """" Apply MinMax, PDF or FFT transform to plot based on GUI data """ @@ -1023,7 +1023,7 @@ def transformPlotData(self,PD): if plotType=='MinMax': self.setPD_MinMax(PD) elif plotType=='PDF': - self.setPD_PDF(PD,PD.c) + self.setPD_PDF(PD, PD.c) elif plotType=='FFT': self.setPD_FFT(PD) @@ -1039,16 +1039,16 @@ def getPlotData(self,plotType): try: for i,idx in enumerate(ID): # Initialize each plotdata based on selected table and selected id channels - pd=PlotData(); - pd.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeLike) + PD = PlotData(); + PD.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeLike) # Possible change of data if plotType=='MinMax': - self.setPD_MinMax(pd) + self.setPD_MinMax(PD) elif plotType=='PDF': - self.setPD_PDF(pd,pd.c) + self.setPD_PDF(PD, PD.c) elif plotType=='FFT': - self.setPD_FFT(pd) - self.plotData.append(pd) + self.setPD_FFT(PD) + self.plotData.append(PD) except Exception as e: self.plotData=[] raise e From efd11736bc56d5119c4dac7dd7afda16a0d2eb15 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Tue, 24 Oct 2023 22:31:21 -0600 Subject: [PATCH 167/178] MinMax: adding y-centering options (see #175) --- pydatview/GUIPlotPanel.py | 50 +++++++++++++++++++++++++++++++++++---- pydatview/plotdata.py | 50 +++++++++++++++++++++++++++++---------- tests/test_plotdata.py | 45 ++++++++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 18 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index b36c5b0..d19a11e 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -81,24 +81,54 @@ def _GUI2Data(self): class MinMaxPanel(wx.Panel): def __init__(self, parent): super(MinMaxPanel,self).__init__(parent) - self.parent = parent + # Data + self.parent = parent + self.yRef = None self.cbxMinMax = wx.CheckBox(self, -1, 'xMinMax',(10,10)) self.cbyMinMax = wx.CheckBox(self, -1, 'yMinMax',(10,10)) self.cbxMinMax.SetValue(False) self.cbyMinMax.SetValue(True) + lbCentering = wx.StaticText( self, -1, 'Y-centering:') + self.lbyRef = wx.StaticText( self, -1, ' ') + self.cbyMean = wx.ComboBox(self, choices=['None', 'Mid=0', 'Mid=ref', 'Mean=0', 'Mean=ref'], style=wx.CB_READONLY) + #self.cbyMean = wx.ComboBox(self, choices=['None', '0', 'Mean of means'] , style=wx.CB_READONLY) + self.cbyMean.SetSelection(0) dummy_sizer = wx.BoxSizer(wx.HORIZONTAL) dummy_sizer.Add(self.cbxMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) dummy_sizer.Add(self.cbyMinMax ,0, flag=wx.CENTER|wx.LEFT, border = 1) + dummy_sizer.Add(lbCentering ,0, flag=wx.CENTER|wx.LEFT, border = 2) + dummy_sizer.Add(self.cbyMean ,0, flag=wx.CENTER|wx.LEFT, border = 1) + dummy_sizer.Add(self.lbyRef ,0, flag=wx.CENTER|wx.LEFT, border = 2) self.SetSizer(dummy_sizer) self.Bind(wx.EVT_CHECKBOX, self.onMinMaxChange) + self.cbyMean.Bind(wx.EVT_COMBOBOX, self.onMeanChange) self.Hide() - def onMinMaxChange(self,event=None): + def setYRef(self, yRef=None): + self.yRef = yRef + if yRef is None: + self.lbyRef.SetLabel('') + else: + self.lbyRef.SetLabel('Y-ref: '+pretty_num(yRef)) + + + def onMinMaxChange(self, event=None): + self.parent.load_and_draw(); # DATA HAS CHANGED + + def onMeanChange(self, event=None): + self.setYRef(None) + if self.cbyMean.GetValue()=='None': + self.cbyMinMax.Enable(True) + else: + self.cbyMinMax.Enable(False) self.parent.load_and_draw(); # DATA HAS CHANGED def _GUI2Data(self): data={'yScale':self.cbyMinMax.IsChecked(), - 'xScale':self.cbxMinMax.IsChecked()} + 'xScale':self.cbxMinMax.IsChecked(), + 'yCenter':self.cbyMean.GetValue(), + 'yRef':self.yRef, + } return data class CompCtrlPanel(wx.Panel): @@ -990,9 +1020,19 @@ def setPD_PDF(self,PD,c): if nBins_out != data['nBins']: self.pdfPanel.scBins.SetValue(data['nBins']) - def setPD_MinMax(self,PD): + def setPD_MinMax(self, PD, firstCall=False): """ Convert plot data to MinMax data based on GUI options""" data = self.mmxPanel._GUI2Data() + if data['yCenter'] in ['Mean=ref', 'Mid=ref']: + if firstCall: + try: + data['yRef'] = PD._y0Mean[0] # Will fail for strings + if np.isnan(data['yRef']): # Will fail for datetimes + data['yRef'] = 0 + except: + data['yRef'] = 0 + print('[WARN] Fail to get yRef, setting it to 0') + self.mmxPanel.setYRef(data['yRef']) # Update GUI try: PD.toMinMax(**data) except Exception as e: @@ -1043,7 +1083,7 @@ def getPlotData(self,plotType): PD.fromIDs(tabs, i, idx, SameCol, pipeline=self.pipeLike) # Possible change of data if plotType=='MinMax': - self.setPD_MinMax(PD) + self.setPD_MinMax(PD, firstCall=i==0) elif plotType=='PDF': self.setPD_PDF(PD, PD.c) elif plotType=='FFT': diff --git a/pydatview/plotdata.py b/pydatview/plotdata.py index 4efd16d..efa9c4d 100644 --- a/pydatview/plotdata.py +++ b/pydatview/plotdata.py @@ -197,21 +197,47 @@ def toPDF(PD, nBins=30, smooth=False): return nBins - def toMinMax(PD, xScale=False, yScale=True): + def toMinMax(PD, xScale=False, yScale=True, yCenter='None', yRef=0): """ Convert plot data to MinMax data based on GUI options NOTE: inPlace """ - if yScale: - if PD.yIsString: - raise Exception('Warn: Cannot compute min-max for strings') - mi = PD._y0Min[0] #mi= np.nanmin(PD.y) - mx = PD._y0Max[0] #mx= np.nanmax(PD.y) - if mi == mx: - PD.y=PD.y*0 - else: - PD.y = (PD.y-mi)/(mx-mi) - PD._yMin=0,'0' - PD._yMax=1,'1' + # --- Scaling and offset for the y axis + ymi = PD._y0Min[0] + ymx = PD._y0Max[0] + yOff = 0 + if yCenter in ['Mean=0', 'Mid=0']: + yRef = 0 + if yCenter=='None': + if yScale: + if PD.yIsString: + raise Exception('Warn: Cannot compute min-max for strings') + if ymi == ymx: + PD.y=PD.y*0 + else: + PD.y = (PD.y-ymi)/(ymx-ymi) + PD._yMin=0,'0' + PD._yMax=1,'1' + elif yCenter in ['Mean=0','Mean=ref']: + if PD.yIsString or PD.yIsDate: + raise Exception('Warn: Cannot offset y-center for strings or dates') + yOff = yRef - PD._y0Mean[0] + PD.y = PD.y + yOff # NOTE: we don't use "+=" to avoid casting issue between int and float + ymi = ymi + yOff + ymx = ymx + yOff + PD._yMin=ymi,str(ymi) + PD._yMax=ymx,str(ymx) + elif yCenter in ['Mid=0','Mid=ref']: + if PD.yIsString or PD.yIsDate: + raise Exception('Warn: Cannot offset y-center for strings or dates') + yOff = yRef - (ymx+ymi)/2 + PD.y = PD.y + yOff # NOTE: we don't use "+=" to avoid casting issue between int and float + ymi = ymi + yOff + ymx = ymx + yOff + PD._yMin=ymi,str(ymi) + PD._yMax=ymx,str(ymx) + else: + raise NotImplementedError('yCenter {}'.format(yCenter)) + # --- Scaling for the x axis if xScale: if PD.xIsString: raise Exception('Warn: Cannot compute min-max for strings') diff --git a/tests/test_plotdata.py b/tests/test_plotdata.py index bb0bc9c..6575145 100644 --- a/tests/test_plotdata.py +++ b/tests/test_plotdata.py @@ -31,11 +31,54 @@ def test_MinMax(self): x = np.linspace(-2,2,100) y = x**3 PD = PlotData(x,y) - PD.toMinMax(xScale=True,yScale=True) + # --- Scale both + PD.toMinMax(xScale=True, yScale=True, yCenter='None') self.assertAlmostEqual(np.min(PD.x),0.0) self.assertAlmostEqual(np.min(PD.y),0.0) + self.assertAlmostEqual(PD._xMin[0],0.0) + self.assertAlmostEqual(PD._yMin[0],0.0) self.assertAlmostEqual(np.max(PD.x),1.0) self.assertAlmostEqual(np.max(PD.y),1.0) + self.assertAlmostEqual(PD._xMax[0] ,1.0) + self.assertAlmostEqual(PD._yMax[0] ,1.0) + + # --- Y Center 0 + x = np.linspace(-2,2,100) + y = x**3 + 10 + PD = PlotData(x,y) + PD.toMinMax(xScale=False, yScale=False, yCenter='Mean=0') + self.assertAlmostEqual(np.mean(PD.y),0.0) + self.assertAlmostEqual(np.min(PD.y),-8.0) + self.assertAlmostEqual(PD._yMin[0] ,-8.0) + self.assertAlmostEqual(np.max(PD.y),8.0) + self.assertAlmostEqual(PD._yMax[0] ,8.0) + + PD = PlotData(x,y) + PD.toMinMax(xScale=False, yScale=False, yCenter='Mid=0') + self.assertAlmostEqual(np.min(PD.y),-8.0) + self.assertAlmostEqual(PD._yMin[0] ,-8.0) + self.assertAlmostEqual(np.max(PD.y),8.0) + self.assertAlmostEqual(PD._yMax[0] ,8.0) + + # --- Y Center ref + x = np.linspace(-2,2,100) + y = x**3 + 10 + PD = PlotData(x,y) + PD.toMinMax(xScale=False, yScale=False, yCenter='Mean=ref', yRef=20) + self.assertAlmostEqual(np.mean(PD.y),20+0.0) + self.assertAlmostEqual(np.min(PD.y) ,20+-8.0) + self.assertAlmostEqual(PD._yMin[0] ,20+-8.0) + self.assertAlmostEqual(np.max(PD.y) ,20+8.0) + self.assertAlmostEqual(PD._yMax[0] ,20+8.0) + + PD = PlotData(x,y) + PD.toMinMax(xScale=False, yScale=False, yCenter='Mid=ref', yRef=20) + self.assertAlmostEqual(np.min(PD.y),20+-8.0) + self.assertAlmostEqual(PD._yMin[0] ,20+-8.0) + self.assertAlmostEqual(np.max(PD.y),20+8.0) + self.assertAlmostEqual(PD._yMax[0] ,20+8.0) + + def test_PDF(self): # --- Test the PDF conversion of plotdata From 1eb36a3224e73cf968e77dbb0cb401f05e37f1b8 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 00:14:44 -0600 Subject: [PATCH 168/178] Mask: clear previous mask before applying a new one --- pydatview/Tables.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydatview/Tables.py b/pydatview/Tables.py index 7f7a1dc..f2f6a40 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -635,6 +635,8 @@ def clearMask(self): self.mask=None def applyMaskString(self, sMask, bAdd=True): + # Remove any existing filter + self.clearMask() # Apply mask on Table df = self.data df_new = None From 013c9d5a3ff7c4e88ce3ae1cc59fa022a550ac6d Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 00:15:37 -0600 Subject: [PATCH 169/178] Mask: only storing successfully applied masks --- pydatview/plugins/data_mask.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index da10d6b..5201648 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -53,8 +53,9 @@ def maskAction(label='mask', mainframe=None, data=None): def applyMask(tab, data): # dfs, names, errors = tabList.applyCommonMaskString(maskString, bAdd=False) - data['formattedMaskString'] = formatMaskString(tab.data, data['maskString']) - dfs, name = tab.applyMaskString(data['formattedMaskString'], bAdd=False) + formattedMaskString = formatMaskString(tab.data, data['maskString']) + dfs, name = tab.applyMaskString(formattedMaskString, bAdd=False) # Might raise an Exception + data['formattedMaskString'] = formattedMaskString # We only store the "succesful" masks def removeMask(tab, data): tab.clearMask() From b4770c5d623ce1908ee747cc12a579005f254dbf Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 00:16:08 -0600 Subject: [PATCH 170/178] IO: BTS file, less spectral warnings --- pydatview/io/turbsim_file.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/pydatview/io/turbsim_file.py b/pydatview/io/turbsim_file.py index 9d3cd7b..e6a1390 100644 --- a/pydatview/io/turbsim_file.py +++ b/pydatview/io/turbsim_file.py @@ -647,22 +647,25 @@ def toDataFrame(self): # Mid csd try: - fc, chi_uu, chi_vv, chi_ww = self.csd_longi() - cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] - data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) - dfs['Mid_csd_longi'] = pd.DataFrame(data = data ,columns = cols) - - # Mid csd - fc, chi_uu, chi_vv, chi_ww = self.csd_lat() - cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] - data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) - dfs['Mid_csd_lat'] = pd.DataFrame(data = data ,columns = cols) - - # Mid csd - fc, chi_uu, chi_vv, chi_ww = self.csd_vert() - cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] - data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) - dfs['Mid_csd_vert'] = pd.DataFrame(data = data ,columns = cols) + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings('ignore') #, category=DeprecationWarning) + fc, chi_uu, chi_vv, chi_ww = self.csd_longi() + cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] + data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) + dfs['Mid_csd_longi'] = pd.DataFrame(data = data ,columns = cols) + + # Mid csd + fc, chi_uu, chi_vv, chi_ww = self.csd_lat() + cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] + data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) + dfs['Mid_csd_lat'] = pd.DataFrame(data = data ,columns = cols) + + # Mid csd + fc, chi_uu, chi_vv, chi_ww = self.csd_vert() + cols = ['f_[Hz]','chi_uu_[-]', 'chi_vv_[-]','chi_ww_[-]'] + data = np.column_stack((fc, chi_uu, chi_vv, chi_ww)) + dfs['Mid_csd_vert'] = pd.DataFrame(data = data ,columns = cols) except ModuleNotFoundError: print('Module scipy.signal not available') except ImportError: From b6cb46adba1c0b1db2c4e68a0a62a5b344ba530b Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 00:17:26 -0600 Subject: [PATCH 171/178] Scripter: plottype should be last --- pydatview/pipeline.py | 6 +++--- pydatview/scripter.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pydatview/pipeline.py b/pydatview/pipeline.py index 3f3d20f..f9f1373 100644 --- a/pydatview/pipeline.py +++ b/pydatview/pipeline.py @@ -344,9 +344,6 @@ def script(self, tabList, scripterOptions=None, ID=None, subPlots=None, plotStyl #print('[INFO] Scripting routine for action {}'.format(action.name)) scripter.addAction(action.name, action_code, imports, code_init) - # --- Fake preplot actions that change the plot type - scripter.setPlotType(plotType, plotTypeData) - # --- Preplot actions for action in self.actionsPlotFilters: action_code, imports, code_init = action.getScript() @@ -356,6 +353,9 @@ def script(self, tabList, scripterOptions=None, ID=None, subPlots=None, plotStyl #print('[INFO] Scripting routine for plot action {}'.format(action.name)) scripter.addPreplotAction(action.name, action_code, imports, code_init) + # --- Fake preplot actions that change the plot type + scripter.setPlotType(plotType, plotTypeData) + # --- Formulae from pydatview.formulae import formatFormula for it, tab in enumerate(tabList): diff --git a/pydatview/scripter.py b/pydatview/scripter.py index caf521d..fa50d30 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -92,8 +92,6 @@ def addPreplotAction(self, action_name, code, imports=None, code_init=None): def setPlotType(self, plotType, plotTypeOptions=None): """ Setup a prePlot action depending on plot Type""" - if len(self.preplot_actions)>0: - raise Exception('PlotType should be the first preplot_action!') opts = plotTypeOptions action_code = None imports = None From 207f55563b312e9a2a2d7e8eaca70958324a2123 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 00:18:26 -0600 Subject: [PATCH 172/178] Scripter: using preproXY to reduce the number of lines --- pydatview/GUIScripter.py | 14 +-- pydatview/common.py | 5 + pydatview/plugins/plotdata_binning.py | 2 +- pydatview/plugins/plotdata_sampler.py | 2 +- pydatview/scripter.py | 128 ++++++++++++++------------ 5 files changed, 81 insertions(+), 70 deletions(-) diff --git a/pydatview/GUIScripter.py b/pydatview/GUIScripter.py index 37a6b0d..ebfdb14 100644 --- a/pydatview/GUIScripter.py +++ b/pydatview/GUIScripter.py @@ -43,7 +43,7 @@ - "enumeration" is the simplest: df1, df2, etc. - "dict" will store the dataframes in dictionaries - "list" will store the dataframes in a list dfs[0], dfs[1], etc. - - Comment level: the verbosity of the comments in the code. + - Verbosity: the verbosity of the code and comments. """ class GUIScripterFrame(wx.Frame): @@ -61,9 +61,9 @@ def __init__(self, parent, mainframe, pipeLike, title): self.setup_syntax_highlighting() self.btHelp= wx.Button(self.panel, label=CHAR['help']+' '+"Help", style=wx.BU_EXACTFIT) - self.btGen = wx.Button(self.panel, label="Update") - self.btRun = wx.Button(self.panel, label="Run Script (beta)") - self.btSave = wx.Button(self.panel, label="Save to File") + self.btGen = wx.Button(self.panel, label=CHAR['update']+' '+"Update", style=wx.BU_EXACTFIT) + self.btRun = wx.Button(self.panel, label=CHAR['compute']+' '+"Run", style=wx.BU_EXACTFIT) + self.btSave = wx.Button(self.panel, label=CHAR['save']+' '+"Save", style=wx.BU_EXACTFIT) txtLib = wx.StaticText(self.panel, -1, 'Library:') libflavors = ["welib", "pydatview", "pyFAST"] @@ -75,10 +75,10 @@ def __init__(self, parent, mainframe, pipeLike, title): self.cbDFS = wx.Choice(self.panel, choices=DFSflavors) self.cbDFS.SetSelection(0) - txtCom= wx.StaticText(self.panel, -1, 'Comment level:') - ComLevels = ["1", "2"] + txtCom= wx.StaticText(self.panel, -1, 'Verbosity:') + ComLevels = ["1", "2", "3"] self.cbCom = wx.Choice(self.panel, choices=ComLevels) - self.cbCom.SetSelection(0) + self.cbCom.SetSelection(1) # --- Layout diff --git a/pydatview/common.py b/pydatview/common.py index 1d0a23d..0a6e287 100644 --- a/pydatview/common.py +++ b/pydatview/common.py @@ -28,6 +28,11 @@ class PyDatViewException(Exception): 'help' : u'\u2753', 'pencil' : u'\u270f', # draw 'pick' : u'\u26cf', +'downarrow': u'\u2193', +'update' : u'\u27F3', +# 'update' : u'\u21BB', +# 'update' : u'\U0001F5D8', +'save' : u'\U0001F5AB', 'hammer' : u'\U0001f528', 'wrench' : u'\U0001f527', 'ruler' : u'\U0001F4CF', # measure diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index 8a1d796..313156c 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -102,7 +102,7 @@ def __init__(self, parent, action, **kwargs): self.textXMin = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) self.textXMax = wx.TextCtrl(self, wx.ID_ANY, '', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(70,-1)) - self.btXRange = self.getBtBitmap(self, 'Update x','compute', self.reset) + self.btXRange = self.getBtBitmap(self, 'Update x', 'update', self.reset) self.lbDX = wx.StaticText(self, -1, '') self.scBins.SetRange(3, 10000) diff --git a/pydatview/plugins/plotdata_sampler.py b/pydatview/plugins/plotdata_sampler.py index 40c6d81..9a5ff0f 100644 --- a/pydatview/plugins/plotdata_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -85,7 +85,7 @@ def __init__(self, parent, action, **kwargs): self.textOldX2 = wx.TextCtrl(self, wx.ID_ANY|wx.TE_READONLY) self.textOldX2.Enable(False) - self.btXRange = self.getBtBitmap(self, 'Update x','compute', self.setCurrentX) + self.btXRange = self.getBtBitmap(self, 'Update x', 'update', self.setCurrentX) # --- Layout msizer = wx.FlexGridSizer(rows=3, cols=4, hgap=2, vgap=0) diff --git a/pydatview/scripter.py b/pydatview/scripter.py index fa50d30..2aaae23 100644 --- a/pydatview/scripter.py +++ b/pydatview/scripter.py @@ -25,7 +25,7 @@ 'oneTabPerFile':False, 'oneRawTab':False, 'indent':' ', - 'verboseCommentLevel':1, + 'verboseCommentLevel':2, } _defaultPlotStyle={ 'grid':False, 'logX':False, 'logY':False, @@ -141,8 +141,16 @@ def setPlotType(self, plotType, plotTypeOptions=None): action_code = [] if opts['xScale']: action_code+=["x = (x-np.min(x))/(np.max(x)-np.min(x))"] - if opts['yScale']: - action_code+=["y = (y-np.min(y))/(np.max(y)-np.min(y))"] + comment='' + if opts['yCenter'].find('ref')>0: + comment=' # TODO: add yRef' + if opts['yCenter']=='None': + if opts['yScale']: + action_code+=["y = (y-np.min(y))/(np.max(y)-np.min(y))"] + elif opts['yCenter'].startswith('Mean'): + action_code+=["y -= np.mean(y)"+comment] + elif opts['yCenter'].startswith('Mid'): + action_code+=["y -= (np.min(y)+np.max(y))/2"+comment] action_code = '\n'.join(action_code) elif plotType=='Compare': @@ -180,6 +188,15 @@ def needFormulae(self): def generate(self, pltshow=True): + script = [] + verboseCommentLevel = self.opts['verboseCommentLevel'] + indent0= '' + indent1= self.opts['indent'] + indent2= indent1 + indent1 + indent3= indent2 + indent1 + plotStyle = self.plotStyle + + # --- Helper functions def forLoopOnDFs(): if self.opts['dfsFlavor'] == 'dict': script.append("for key, df in dfs.items():") @@ -195,17 +212,18 @@ def oneRawTab(): def dontListFiles(): return oneRawTab() or (onlyOneFile() and oneTabPerFile()) + def addActionCode(actioname, actioncode, ind, codeFail='pass'): + if verboseCommentLevel>=3: + script.append(ind+ "# Apply action {}".format(actioname)) + if verboseCommentLevel>=2: + script.append(ind+'try:') + script.append('\n'.join(ind+indent1+l for l in actioncode.splitlines())) + script.append(ind+'except:') + script.append(ind+indent1+codeFail) + else: + script.append('\n'.join(ind +l for l in actioncode.splitlines())) - script = [] - - verboseCommentLevel = self.opts['verboseCommentLevel'] - indent0= '' - indent1= self.opts['indent'] - indent2= indent1 + indent1 - indent3= indent2 + indent1 - plotStyle = self.plotStyle - # --- Disclaimer script.append('""" Script generated by pyDatView - The script will likely need to be adapted."""') @@ -231,11 +249,15 @@ def dontListFiles(): script.append(f"filenames += ['{filename}']") # --- Init data/preplot/adder actions + if len(self.actions)>0 or len(self.preplot_actions)>0 or len(self.adder_actions)>0: + script.append("\n# --- Data for different actions") + if len(self.actions)>0: - script.append("\n# --- Data for actions") + if verboseCommentLevel>=3: + script.append("# --- Data for actions") for actionname, actioncode in self.actions.items(): if actioncode[0] is not None and len(actioncode[0].strip())>0: - if verboseCommentLevel>=2: + if verboseCommentLevel>=3: script.append("# Data for action {}".format(actionname)) script.append(actioncode[0].strip()) @@ -243,18 +265,20 @@ def dontListFiles(): script_pre = [] for actionname, actioncode in self.preplot_actions.items(): if actioncode[0] is not None and len(actioncode[0].strip())>0: - if verboseCommentLevel>=2: + if verboseCommentLevel>=3: script_pre.append("# Data for preplot action {}".format(actionname)) script_pre.append(actioncode[0].strip()) if len(script_pre)>0: - script.append("\n# --- Data for preplot actions") + if verboseCommentLevel>=3: + script.append("# --- Data for preplot actions") script+=script_pre if len(self.adder_actions)>0: - script.append("\n# --- Data for actions that add new dataframes") + if verboseCommentLevel>=3: + script.append("# --- Data for actions that add new dataframes") for actionname, actioncode in self.adder_actions.items(): if actioncode[0] is not None and len(actioncode[0].strip())>0: - if verboseCommentLevel>=2: + if verboseCommentLevel>=3: script.append("# Data for adder action {}".format(actionname)) script.append(actioncode[0].strip()) @@ -272,7 +296,7 @@ def dontListFiles(): script.append(indent1 + "dfs[iFile] = weio.read(filename).toDataFrame()") else: script.append(indent1 + "dfs_or_df = weio.read(filename).toDataFrame()") - script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") + #script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") script.append(indent1 + "if isinstance(dfs_or_df, dict):") script.append(indent2 + "for k,df in dfs_or_df.items():") script.append(indent3 + "dfs[k+f'{iFile}'] = df") @@ -288,7 +312,7 @@ def dontListFiles(): script.append(indent1 + "df = weio.read(filenames[iFile]).toDataFrame()") script.append(indent1 + "dfs.append(df)") else: - script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") + #script.append(indent1 + "# NOTE: we need a different action if the file contains multiple dataframes") script.append(indent1 + "dfs_or_df = weio.read(filenames[iFile]).toDataFrame()") script.append(indent1 + "if isinstance(dfs_or_df, dict):") script.append(indent2 + "dfs+= list(dfs_or_df.values()) # NOTE: user will need to adapt this.") @@ -308,32 +332,26 @@ def dontListFiles(): script.append("# NOTE: we need a different action if the file contains multiple dataframes") script.append(f"dfs_or_df = weio.read('{filename}').toDataFrame()") script.append("if isinstance(dfs_or_df, dict):") - script.append(indent1 + f"df{iFile1} = dfs_or_df.items()[0][1] # NOTE: user will need to adapt this.") + script.append(indent1 + f"df{iFile1} = next(iter(dfs_or_df.values())) # NOTE: user will need to adapt this.") script.append("else:") script.append(indent1 + f"df{iFile1} = dfs_or_df") # --- Adder actions nTabs = len(self.filenames) # Approximate if len(self.adder_actions)>0: - def addActionCode(actioname, actioncode, ind): - script.append(ind+ "# Apply action {}".format(actioname)) - lines = actioncode.split("\n") - indented_lines = [ind + line for line in lines] - script.append("\n".join(indented_lines)) - script.append("\n# --- Apply adder actions to dataframes") script.append("dfs_add = [] ; names_add =[]") if self.opts['dfsFlavor'] == 'dict': if dontListFiles(): script.append('df = dfs[0] # NOTE: we assume that only one dataframe is present' ) for actionname, actioncode in self.adder_actions.items(): - addActionCode(actionname, actioncode[1], indent0) + addActionCode(actionname, actioncode[1], indent0, codeFail='dfs_new=[]; names_new=[]') script.append(indent0+"dfs_add += dfs_new ; names_add += names_new") else: script.append("for k, (key, df) in enumerate(dfs.items()):") script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") for actionname, actioncode in self.adder_actions.items(): - addActionCode(actionname, actioncode[1], indent1) + addActionCode(actionname, actioncode[1], indent1, codeFail='dfs_new=[]; names_new=[]') script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") script.append("for name_new, df_new in zip(names_add, dfs_new):") script.append(indent1+"if df_new is not None:") @@ -343,13 +361,13 @@ def addActionCode(actioname, actioncode, ind): if dontListFiles(): script.append('df = dfs[0]') for actionname, actioncode in self.adder_actions.items(): - addActionCode(actionname, actioncode[1], indent0) + addActionCode(actionname, actioncode[1], indent0, codeFail='dfs_new=[]; names_new=[]') script.append(indent0+"dfs_add += dfs_new ; names_add += names_new") else: script.append("for k, df in enumerate(dfs):") script.append(indent1 + "filename = filenames[k] # NOTE: this is approximate..") for actionname, actioncode in self.adder_actions.items(): - addActionCode(actionname, actioncode[1], indent1) + addActionCode(actionname, actioncode[1], indent1, codeFail='dfs_new=[]; names_new=[]') script.append(indent1+"dfs_add += dfs_new ; names_add += names_new") script.append("for name_new, df_new in zip(names_add, dfs_new):") script.append(indent1+"if df_new is not None:") @@ -410,14 +428,6 @@ def addActionCode(actioname, actioncode, ind): # --- Data Actions if len(self.actions)>0: - - def addActionCode(actioname, actioncode, ind): - if verboseCommentLevel>2: - script.append(ind+ "# Apply action {}".format(actioname)) - lines = actioncode.split("\n") - indented_lines = [ind + line for line in lines] - script.append("\n".join(indented_lines)) - script.append("\n# --- Apply actions to dataframes") if self.opts['dfsFlavor'] == 'dict': script.append("for k, df in dfs.items():") @@ -438,10 +448,17 @@ def addActionCode(actioname, actioncode, ind): addActionCode(actionname, actioncode[1], '') script.append('df{} = df'.format(iTab+1)) + if len(self.preplot_actions)>0: + script.append("\n# --- Plot preprocessing") + script.append("def preproXY(x, y):") + for actionname, actioncode in self.preplot_actions.items(): + script.append('\n'.join(indent1+l for l in actioncode[1].splitlines())) + script.append(indent1+"return x, y") + # --- Plot Styling script.append("\n# --- Plot") # Plot Styling - if verboseCommentLevel>=2: + if verboseCommentLevel>=3: script.append("# Plot styling") # NOTE: dfs not known for enumerate script.append("lw = {} ".format(plotStyle['LineWidth'])) @@ -452,7 +469,7 @@ def addActionCode(actioname, actioncode, ind): #script.append("cols=['r', 'g', 'b'] * 100") if self.opts['dfsFlavor'] == 'dict': script.append("tabNames = list(dfs.keys())") - if verboseCommentLevel>=2: + if verboseCommentLevel>=3: script.append("# Subplots") if self.subPlots['i']==1 and self.subPlots['j']==1: @@ -493,10 +510,8 @@ def getAxesString(i,j): if sAxes_new != sAxes: sAxes=sAxes_new script.append(sAxes_new) - if verboseCommentLevel>=2: - script.append("\n# Selecting data for df{}".format(df_index+1)) - if len(self.preplot_actions)>0: - sPlotXY='x, y, ' + if verboseCommentLevel>=3: + script.append("\n# Selecting data and plotting for df{}".format(df_index+1)) if self.opts['dfsFlavor'] in ['dict', 'list']: if self.opts['dfsFlavor'] == 'dict': sDF_new = "dfs[tabNames[{}]]".format(df_index) @@ -505,27 +520,18 @@ def getAxesString(i,j): if sDF_new != sDF: sDF = sDF_new script.append("df = "+sDF) - if len(self.preplot_actions)>0: - script.append("x = df['{}'].values".format(column_x)) - script.append("y = df['{}'].values".format(column_y)) - else: - sPlotXY ="df['{}'], df['{}'], ".format(column_x, column_y) + sPlotXY ="df['{}'], df['{}']".format(column_x, column_y) elif self.opts['dfsFlavor'] == 'enumeration': - if len(self.preplot_actions)>0: - script.append("x = df{}['{}'].values".format(df_index+1, column_x)) - script.append("y = df{}['{}'].values".format(df_index+1, column_y)) - else: - sPlotXY ="df{}['{}'], df{}['{}'], ".format(df_index+1, column_x, df_index+1, column_y) + sPlotXY ="df{}['{}'], df{}['{}']".format(df_index+1, column_x, df_index+1, column_y) if len(self.preplot_actions)>0: - if verboseCommentLevel>=2: - script.append("# Applying preplot action for df{}".format(df_index+1)) - for actionname, actioncode in self.preplot_actions.items(): - script.append(actioncode[1]) + #script.append("x, y = preproXY({})".format(sPlotXY)) + sPlotXY="*preproXY({}), ".format(sPlotXY) + #sPlotXY='x, y, ' + else: + sPlotXY=sPlotXY+', ' # --- Plot - if verboseCommentLevel>=2: - script.append("# Plotting for df{}".format(df_index+1)) label =column_y.replace('_',' ') # TODO for table comparison plotLine = "ax.plot(" plotLine += sPlotXY From 44b236cd4351d72c05c08ab1eee16e970ac75527 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 11:07:05 -0600 Subject: [PATCH 173/178] Plot: setting ground for more esthetics options --- pydatview/GUIPlotPanel.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index d19a11e..c881c8a 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -394,7 +394,7 @@ def __init__(self, parent, data): self.cbLgdFont.SetSelection(i) # Line Width Font lbLW = wx.StaticText( self, -1, 'Line width:') - LWChoices = ['0.5','1.0','1.25','1.5','2.0','2.5','3.0'] + LWChoices = ['0.5','1.0','1.25','1.5','1.75','2.0','2.5','3.0'] self.cbLW = wx.ComboBox(self, choices=LWChoices , style=wx.CB_READONLY) try: i = LWChoices.index(str(data['LineWidth'])) @@ -1229,7 +1229,12 @@ def getPlotOptions(self, PD=None): plot_options['step'] = self.cbStepPlot.IsChecked() plot_options['logX'] = self.cbLogX.IsChecked() plot_options['logY'] = self.cbLogY.IsChecked() - plot_options['grid'] = self.cbGrid.IsChecked() + if self.cbGrid.IsChecked(): + plot_options['grid'] = {'visible': self.cbGrid.IsChecked(), 'linestyle':'-', 'linewidth':0.5, 'color':'#b0b0b0'} + else: + plot_options['grid'] = {'visible': False} + #plot_options['tick_params'] = {'direction':'in', 'top':True, 'right':True, 'labelright':False, 'labeltop':False, 'which':'both'} + plot_options['tick_params'] = {} plot_options['lw']=plotStyle['LineWidth'] plot_options['ms']=plotStyle['MarkerSize'] @@ -1279,7 +1284,6 @@ def plot_all(self, autoscale=True): # --- PlotStyles plotStyle, plot_options, font_options, font_options_legd = self.getPlotOptions() - # --- Loop on axes. Either use ax.iPD to chose the plot data, or rely on plotmatrix for axis_idx, ax_left in enumerate(axes): ax_right = None @@ -1337,7 +1341,7 @@ def plot_all(self, autoscale=True): except: pass - ax_left.grid(plot_options['grid']) + ax_left.grid(**plot_options['grid']) if ax_right is not None: l = ax_left.get_ylim() l2 = ax_right.get_ylim() @@ -1347,13 +1351,17 @@ def plot_all(self, autoscale=True): if len(ax_left.lines) == 0: ax_left.set_yticks(ax_right.get_yticks()) ax_left.yaxis.set_visible(False) - ax_right.grid(plot_options['grid']) + ax_right.grid(**plot_options['grid']) # Special Grids if self.pltTypePanel.cbCompare.GetValue(): if self.cmpPanel.rbType.GetStringSelection()=='Y-Y': xmin,xmax=ax_left.get_xlim() - ax_left.plot([xmin,xmax],[xmin,xmax],'k--',linewidth=0.5) + + # Ticks + ax_left.tick_params(**plot_options['tick_params']) + if ax_right is not None: + ax_right.tick_params(**plot_options['tick_params']) # Labels yleft_labels = [] From 37167ba77b1bfff65a087c6466baef997e5af371 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 15:57:57 -0600 Subject: [PATCH 174/178] Doc: adding help documentation for tools --- pydatview/plugins/__init__.py | 4 +- pydatview/plugins/base_plugin.py | 76 +++++++++++--------- pydatview/plugins/data_mask.py | 37 +++++++++- pydatview/plugins/data_radialavg.py | 53 ++++++++++++-- pydatview/plugins/plotdata_binning.py | 45 ++++++------ pydatview/plugins/plotdata_filter.py | 51 +++++++------ pydatview/plugins/plotdata_removeOutliers.py | 21 +++++- pydatview/plugins/plotdata_sampler.py | 58 ++++++++------- pydatview/plugins/tool_curvefitting.py | 36 +++++----- pydatview/plugins/tool_logdec.py | 7 +- 10 files changed, 239 insertions(+), 149 deletions(-) diff --git a/pydatview/plugins/__init__.py b/pydatview/plugins/__init__.py index ff65431..658d584 100644 --- a/pydatview/plugins/__init__.py +++ b/pydatview/plugins/__init__.py @@ -112,11 +112,11 @@ def _tool_curvefitting(*args, **kwargs): # TOOLS: tool plugins constructor should return a Panel class # OF_DATA_TOOLS={} OF_DATA_PLUGINS_WITH_EDITOR=OrderedDict([ # TODO - ('Radial Average', _data_radialavg), + ('Nodal Average', _data_radialavg), ]) # DATA_PLUGINS_SIMPLE: simple data plugins constructors should return an Action OF_DATA_PLUGINS_SIMPLE=OrderedDict([ - ('Radial Time Concatenation' , _data_radialConcat), + ('Nodal Time Concatenation' , _data_radialConcat), ('v3.4 - Rename "Fld" > "Aero' , _data_renameFldAero), ('v2.3 - Rename "B*N* " > "AB*N* ' , _data_renameOF23), ]) diff --git a/pydatview/plugins/base_plugin.py b/pydatview/plugins/base_plugin.py index 9f5cb93..b4ba0c4 100644 --- a/pydatview/plugins/base_plugin.py +++ b/pydatview/plugins/base_plugin.py @@ -18,9 +18,10 @@ # --- Default class for tools # --------------------------------------------------------------------------------{ class GUIToolPanel(wx.Panel): - def __init__(self, parent): + def __init__(self, parent, help_string =''): super(GUIToolPanel,self).__init__(parent) self.parent = parent + self.help_string = help_string def destroyData(self): if hasattr(self, 'action'): @@ -56,6 +57,10 @@ def getToggleBtBitmap(self,par,label,Type=None,callback=None,bitmap=False): par.Bind(wx.EVT_TOGGLEBUTTON, callback, bt) return bt + def onHelp(self, event=None): + Info(self, self.help_string) + + # --------------------------------------------------------------------------------} # --- Default class for GUI to edit plugin and control an action # --------------------------------------------------------------------------------{ @@ -67,8 +72,8 @@ class ActionEditor(GUIToolPanel): - the action data - a set of function handles to process some triggers and callbacks """ - def __init__(self, parent, action, buttons=None, tables=False): - GUIToolPanel.__init__(self, parent) + def __init__(self, parent, action, buttons=None, tables=False, help_string=''): + GUIToolPanel.__init__(self, parent, help_string=help_string) # --- Data self.data = action.data @@ -163,19 +168,15 @@ def onAdd(self, event=None): def onClear(self, event=None): self.redrawHandle() - def onHelp(self,event=None): - Info(self, """Dummy help""") - - # --------------------------------------------------------------------------------} # --- Default class for GUI to edit plugin and control a plot data action # --------------------------------------------------------------------------------{ class PlotDataActionEditor(ActionEditor): - def __init__(self, parent, action, buttons=None, tables=False): + def __init__(self, parent, action, sButtons=None, tables=False, help_string='', nBtRows=None, nBtCols=None): """ """ - ActionEditor.__init__(self, parent, action=action) + ActionEditor.__init__(self, parent, action=action, help_string=help_string) # --- Data self.data = action.data @@ -189,24 +190,35 @@ def __init__(self, parent, action, buttons=None, tables=False): self.action.guiEditorObj = self # --- GUI elements - if buttons is None: - buttons=['Close', 'Add', 'Help', 'Clear', 'Plot', 'Apply'] + if sButtons is None: + sButtons=['Close', 'Add', 'Help', 'Clear', 'Plot', 'Apply'] self.btAdd = None self.btHelp = None self.btClear = None self.btPlot = None - nButtons = 0 - self.btClose = self.getBtBitmap(self,'Close','close',self.destroy); nButtons+=1 - if 'Add' in buttons: - self.btAdd = self.getBtBitmap(self, 'Add','add' , self.onAdd); nButtons+=1 - if 'Help' in buttons: - self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp); nButtons+=1 - if 'Clear' in buttons: - self.btClear = self.getBtBitmap(self, 'Clear Plot','sun' , self.onClear); nButtons+=1 - if 'Plot' in buttons: - self.btPlot = self.getBtBitmap(self, 'Plot ' ,'chart' , self.onPlot); nButtons+=1 - self.btApply = self.getToggleBtBitmap(self,'Apply','cloud',self.onToggleApply); nButtons+=1 - + nButtons = len(sButtons) + buttons=[] + for lbl in sButtons: + if lbl=='Close': + self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) + buttons.append(self.btClose) + elif lbl=='Add': + self.btAdd = self.getBtBitmap(self, 'Add', 'add' , self.onAdd) + buttons.append(self.btAdd) + elif lbl=='Help': + self.btHelp = self.getBtBitmap(self, 'Help', 'help', self.onHelp) + buttons.append(self.btHelp) + elif lbl=='Clear': + self.btClear = self.getBtBitmap(self, 'Clear Plot', 'sun', self.onClear) + buttons.append(self.btClear) + elif lbl=='Plot': + self.btPlot = self.getBtBitmap(self, 'Plot ', 'chart' , self.onPlot) + buttons.append(self.btPlot) + elif lbl=='Apply': + self.btApply = self.getToggleBtBitmap(self, 'Apply', 'cloud', self.onToggleApply) + buttons.append(self.btApply) + else: + raise NotImplementedError('Button: {}'.format(lbl)) if tables: self.cbTabs= wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) @@ -215,17 +227,13 @@ def __init__(self, parent, action, buttons=None, tables=False): self.cbTabs = None # --- Layout - btSizer = wx.FlexGridSizer(rows=int(nButtons/2), cols=2, hgap=2, vgap=0) - btSizer.Add(self.btClose , 0, flag = wx.ALL|wx.EXPAND, border = 1) - if self.btClear is not None: - btSizer.Add(self.btClear , 0, flag = wx.ALL|wx.EXPAND, border = 1) - if self.btAdd is not None: - btSizer.Add(self.btAdd , 0, flag = wx.ALL|wx.EXPAND, border = 1) - if self.btPlot is not None: - btSizer.Add(self.btPlot , 0, flag = wx.ALL|wx.EXPAND, border = 1) - if self.btHelp is not None: - btSizer.Add(self.btHelp , 0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btApply , 0, flag = wx.ALL|wx.EXPAND, border = 1) + if nBtCols is None: + nBtCols=2 + if nBtRows is None: + nBtRows = int(np.ceil(nButtons/nBtCols)) + btSizer = wx.FlexGridSizer(rows=nBtRows, cols=nBtCols, hgap=2, vgap=0) + for bt in buttons: + btSizer.Add(bt, 0, flag = wx.ALL|wx.EXPAND, border = 1) self.sizer = wx.BoxSizer(wx.HORIZONTAL) self.sizer.Add(btSizer ,0, flag = wx.LEFT ,border = 1) diff --git a/pydatview/plugins/data_mask.py b/pydatview/plugins/data_mask.py index 5201648..9bd4a00 100644 --- a/pydatview/plugins/data_mask.py +++ b/pydatview/plugins/data_mask.py @@ -4,6 +4,38 @@ from pydatview.common import CHAR, Error, Info, pretty_num_short from pydatview.common import DummyMainFrame from pydatview.pipeline import ReversibleTableAction +_HELP="""Mask + +The masking operation is used to select a subset of the data based on some criteria +defined by a "mask". Currently the mask is applied to all the tables opened in pyDatView but +if it fails on some tables, the errors are not fatal (they will be reported in the "Errors" +link in the taskbar). + +Usage: + - Fill a mask string in the field "Mask" + - Click on the Button Mask to apply the mask to all tables. + - Click on the Button"Mask (add)" to create copy of the tables with the masked applied + (this is useful if you want to further export these tables, or perform comparisons between + unmask and masked data sets). + +Mask specifications: + - The mask should be in "Python" syntax, except that the columns are referenced using + a special notation. + - Columns are referenced with curly brackets. The units are removed for convenience. + For instance: "{Time}" corresponds to the columns 'Time_[s]' or 'Time (s)' or 'Time'. + - Basic selection can be done with the operators '==', '<', '<=', etc. + - The Python syntax for logical operations is: + - and: (A) && (B) + - or : (A) || (B) + - Numpy operations may be performed using "np." + +Example of masks: + - "({Time}>100) && ({Time}<50) && ({WindSpeed}==5)": Select data where time is between 50 + and 100 and where the WindSpeed is 5. + - "{Date} > '2018-10-01'": Select dates after a given date + - "['John' in str(x) for x in {Names}]": Select Names that contain 'John' + +""" # --------------------------------------------------------------------------------} # --- Data # --------------------------------------------------------------------------------{ @@ -92,7 +124,7 @@ def formatMaskString(df, sMask): class MaskToolPanel(ActionEditor): def __init__(self, parent, action=None): import wx # avoided at module level for unittests - ActionEditor.__init__(self, parent, action=action) + ActionEditor.__init__(self, parent, action=action, help_string=_HELP) # --- Creating "Fake data" for testing only! if action is None: @@ -102,6 +134,7 @@ def __init__(self, parent, action=None): # --- GUI elements self.btClose = self.getBtBitmap(self, 'Close','close', self.destroy) self.btAdd = self.getBtBitmap(self, u'Mask (add)','add' , self.onAdd) + self.btHelp = self.getBtBitmap(self, u'Help' ,'help' , self.onHelp) self.btApply = self.getToggleBtBitmap(self, 'Apply','cloud', self.onToggleApply) #self.cbTabs = wx.ComboBox(self, -1, choices=[], style=wx.CB_READONLY) @@ -115,7 +148,7 @@ def __init__(self, parent, action=None): # --- Layout btSizer = wx.FlexGridSizer(rows=2, cols=2, hgap=2, vgap=0) btSizer.Add(self.btClose ,0,flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(wx.StaticText(self, -1, '') ,0,flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btHelp ,0,flag = wx.ALL|wx.EXPAND, border = 1) btSizer.Add(self.btAdd ,0,flag = wx.ALL|wx.EXPAND, border = 1) btSizer.Add(self.btApply ,0,flag = wx.ALL|wx.EXPAND, border = 1) diff --git a/pydatview/plugins/data_radialavg.py b/pydatview/plugins/data_radialavg.py index f55e3ff..53bfe6a 100644 --- a/pydatview/plugins/data_radialavg.py +++ b/pydatview/plugins/data_radialavg.py @@ -7,6 +7,50 @@ # from pydatview.common import DummyMainFrame from pydatview.pipeline import AdderAction +_HELP = """Nodal/Radial averaging + +Nodal averaging is useful to extract OpenFAST/FAST.Farm outputs as function +of a given nodal variable(typically: the radial position along the blade span, or +the tower height, or downstream wake locations). + +OpenFAST outputs these nodal variables in different channels such as: + B1N001Alpha(t), B1N002Alpha(t), etc. + +The nodal averaging tool perform a time average of these channels, so that they can +be plotted as function of their spatial variable (e.g. the radial position). +For the example above, the nodal averaging would return Alpha(r), with r the radial +position. + +Two methods are available to perform the time averaging: + - Last `n` seconds: average over the `n` last seconds of the time series, where `n` is, + the user input parameter value. Setting `n` > tmax will correspond + to the entire time series. + - Last `n` periods: average the time series over the last `n` rotor revolutions. + This is the recommend approach for quasi-periodic signals + (such as wind turbines outputs), with a rotor. + The script will need a column named "Azimuth" to perform correctly. + +Behind the scene, the script: + - Determines whether it is an OpenFAST or FAST.Farm output. + - Attempts to open the OpenFAST input files to find out the nodal locations ( + (e.g., by opening the AeroDyn blade file, the ElastoDyn file, etc.) + If the files can't be read the variables will be plotted as function of "index" + instead of the spatial coordinates (e.g., r). + Better results are therefore obtained if the input files are accessible by pyDatView. + +Requirements: + - A column named "time" (case and unit incensitive) need to be present. + - For the period averaging, a column named "azimuth" need to be present. + - For better results, OpenFAST inputs files should be accessible (i.e. the output file + should not have been moved or renamed) + + +NOTE: + - The action "nodal time concatenation" takes all the time series of a given variable + and concatenate them into one channel. + B1Alpha = [B1N001Alpha(t), B1N002Alpha(t), etc. ] + +""" # --------------------------------------------------------------------------------} # --- Radial @@ -90,7 +134,7 @@ def radialAvg(tab, data=None): class RadialToolPanel(ActionEditor): def __init__(self, parent, action=None): import wx # avoided at module level for unittests - super(RadialToolPanel,self).__init__(parent, action=action) + super(RadialToolPanel,self).__init__(parent, action=action, help_string=_HELP) # --- Creating "Fake data" for testing only! if action is None: @@ -100,6 +144,7 @@ def __init__(self, parent, action=None): # --- GUI elements self.btClose = self.getBtBitmap(self,'Close' ,'close' , self.destroy) self.btAdd = self.getBtBitmap(self,'Average','compute', self.onAdd) # ART_PLUS + self.btHelp = self.getBtBitmap(self, 'Help','help', self.onHelp) self.lb = wx.StaticText( self, -1, """Select averaging method and average parameter (`Period` methods uses the `azimuth` signal) """) #self.cbTabs = wx.ComboBox(self, choices=[], style=wx.CB_READONLY) @@ -110,10 +155,11 @@ def __init__(self, parent, action=None): self.textAverageParam = wx.TextCtrl(self, wx.ID_ANY, '', size = (36,-1), style=wx.TE_PROCESS_ENTER) # --- Layout - btSizer = wx.FlexGridSizer(rows=2, cols=1, hgap=0, vgap=0) + btSizer = wx.FlexGridSizer(rows=2, cols=2, hgap=0, vgap=0) #btSizer = wx.BoxSizer(wx.VERTICAL) btSizer.Add(self.btClose ,0, flag = wx.ALL|wx.EXPAND, border = 1) - btSizer.Add(self.btAdd ,0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btAdd ,0, flag = wx.ALL|wx.EXPAND, border = 1) + btSizer.Add(self.btHelp ,0, flag = wx.ALL|wx.EXPAND, border = 1) row_sizer = wx.BoxSizer(wx.HORIZONTAL) #row_sizer.Add(wx.StaticText(self, -1, 'Tab:') , 0, wx.CENTER|wx.LEFT, 0) @@ -175,7 +221,6 @@ def _Data2GUI(self): self.textAverageParam.SetValue(str(self.data['avgParam'])) - if __name__ == '__main__': from pydatview.plugins.base_plugin import demoGUIPlugin demoGUIPlugin(RadialToolPanel, actionCreator=radialAvgAction, mainLoop=False, title='Radial Avg') diff --git a/pydatview/plugins/plotdata_binning.py b/pydatview/plugins/plotdata_binning.py index 313156c..9e62b1b 100644 --- a/pydatview/plugins/plotdata_binning.py +++ b/pydatview/plugins/plotdata_binning.py @@ -2,6 +2,27 @@ from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info, pretty_num_short from pydatview.pipeline import PlotDataAction +_HELP = """Binning. + +The binning operation computes average y values for a set of x ranges. + +To bin perform the following step: + +- Specify the number of bins (#bins) +- Specify the min and max of the x values (or click on "Update x") +- Click on one of the following buttons: + - Plot: will display the binned data on the figure + - Apply: will perform the binning on the fly for all new plots + (click on Clear to stop applying) + - Add: will create new table(s) with binned values for all + signals. This process might take some time. + Select a table or choose all (default) + + - Update x: retrieve the minimum and maximum x values of the current plot and update the + corresponding fields in the GUI. The values can then used to make sure the bins + cover the full range of the data. + +""" # --------------------------------------------------------------------------------} # --- Data # --------------------------------------------------------------------------------{ @@ -95,7 +116,7 @@ class BinningToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): import wx - PlotDataActionEditor.__init__(self, parent, action, tables=False, **kwargs) + PlotDataActionEditor.__init__(self, parent, action, tables=False, help_string=_HELP, **kwargs) # --- GUI elements self.scBins = wx.SpinCtrl(self, value='50', style = wx.TE_PROCESS_ENTER|wx.TE_RIGHT, size=wx.Size(60,-1) ) @@ -215,28 +236,6 @@ def onAdd(self,event=None): # Call parent class PlotDataActionEditor.onAdd(self) - def onHelp(self,event=None): - Info(self,"""Binning. - -The binning operation computes average y values for a set of x ranges. - -To bin perform the following step: - -- Specify the number of bins (#bins) -- Specify the min and max of the x values (or click on "Default") - -- Click on one of the following buttons: - - Plot: will display the binned data on the figure - - Apply: will perform the binning on the fly for all new plots - (click on Clear to stop applying) - - Add: will create new table(s) with biined values for all - signals. This process might take some time. - Select a table or choose all (default) -""") - - - - if __name__ == '__main__': from pydatview.plugins.base_plugin import demoPlotDataActionPanel diff --git a/pydatview/plugins/plotdata_filter.py b/pydatview/plugins/plotdata_filter.py index a59a1e9..384b123 100644 --- a/pydatview/plugins/plotdata_filter.py +++ b/pydatview/plugins/plotdata_filter.py @@ -3,6 +3,30 @@ from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction import platform + +_HELP="""Filtering. + +The filtering operation changes the "y" values of a table/plot, +applying a given filter (typically cutting off some frequencies). + +To filter perform the following step: + +- Choose a filtering method: + - Moving average: apply a moving average filter, with + a length specified by the window size (in indices) + - High pass 1st order: apply a first order high-pass filter, + passing the frequencies above the cutoff frequency parameter. + - Low pass 1st order: apply a first order low-pass filter, + passing the frequencies below the cutoff frequency parameter. + +- Click on one of the following buttons: + - Plot: will display the filtered data on the figure + - Apply: will perform the filtering on the fly for all new plots + - Add: will create new table(s) with filtered values for all + signals. This process might take some time. + Currently done for all tables. +""" + # --------------------------------------------------------------------------------} # --- Data # --------------------------------------------------------------------------------{ @@ -72,7 +96,7 @@ class FilterToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): import wx # avoided at module level for unittests - PlotDataActionEditor.__init__(self, parent, action, tables=False, **kwargs) + PlotDataActionEditor.__init__(self, parent, action, tables=False, help_string=_HELP, **kwargs) # --- Data from pydatview.tools.signal_analysis import FILTERS @@ -194,31 +218,6 @@ def onAdd(self, event=None): # Call parent class PlotDataActionEditor.onAdd(self) - def onHelp(self,event=None): - Info(self,"""Filtering. - -The filtering operation changes the "y" values of a table/plot, -applying a given filter (typically cutting off some frequencies). - -To filter perform the following step: - -- Choose a filtering method: - - Moving average: apply a moving average filter, with - a length specified by the window size (in indices) - - High pass 1st order: apply a first oder high-pass filter, - passing the frequencies above the cutoff frequency parameter. - - Low pass 1st order: apply a first oder low-pass filter, - passing the frequencies below the cutoff frequency parameter. - -- Click on one of the following buttons: - - Plot: will display the filtered data on the figure - - Apply: will perform the filtering on the fly for all new plots - - Add: will create new table(s) with filtered values for all - signals. This process might take some time. - Currently done for all tables. -""") - - if __name__ == '__main__': from pydatview.plugins.base_plugin import demoPlotDataActionPanel diff --git a/pydatview/plugins/plotdata_removeOutliers.py b/pydatview/plugins/plotdata_removeOutliers.py index 1cae4f2..ed14e37 100644 --- a/pydatview/plugins/plotdata_removeOutliers.py +++ b/pydatview/plugins/plotdata_removeOutliers.py @@ -3,6 +3,23 @@ from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction import platform +_HELP="""Outlier removal + +Removes outliers from the plotted data. + +Currently, the only method implemented is the "Median-std". + +Usage: + - Adjust the parameter of the method. + - Click on "Apply" to apply the filtering on the fly to all the data being plotted. + +Methods: + - Median-std: rejects data points that are away from the median by a distance + corresponding to a fraction of the standard deviation of the signal. + This fraction is referred to as "Median deviations". + The smaller the fraction the more data will be removed. + +""" # --------------------------------------------------------------------------------} # --- Data # --------------------------------------------------------------------------------{ @@ -61,7 +78,7 @@ class RemoveOutliersToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): import wx - PlotDataActionEditor.__init__(self, parent, action, tables=False, buttons=[''], **kwargs) + PlotDataActionEditor.__init__(self, parent, action, tables=False, sButtons=['Close','Help','Apply'], nBtCols=3, help_string=_HELP, **kwargs) # --- GUI elements #self.btClose = self.getBtBitmap(self,'Close','close',self.destroy) @@ -77,8 +94,6 @@ def __init__(self, parent, action, **kwargs): # --- Layout hsizer = wx.BoxSizer(wx.HORIZONTAL) -# self.sizer.Add(self.btClose,0,flag = wx.LEFT|wx.CENTER,border = 1) -# self.sizer.Add(self.btApply,0,flag = wx.LEFT|wx.CENTER,border = 5) hsizer.Add(lb1 ,0,flag = wx.LEFT|wx.CENTER,border = 5) hsizer.Add(self.tMD ,0,flag = wx.LEFT|wx.CENTER,border = 5) hsizer.Add(self.lb ,0,flag = wx.LEFT|wx.CENTER,border = 5) diff --git a/pydatview/plugins/plotdata_sampler.py b/pydatview/plugins/plotdata_sampler.py index 9a5ff0f..780a874 100644 --- a/pydatview/plugins/plotdata_sampler.py +++ b/pydatview/plugins/plotdata_sampler.py @@ -2,6 +2,33 @@ from pydatview.plugins.base_plugin import PlotDataActionEditor, TOOL_BORDER from pydatview.common import Error, Info from pydatview.pipeline import PlotDataAction +_HELP = """Resampling. + +The resampling operation changes the "x" values of a table/plot and +adapt the "y" values accordingly. + +To resample perform the following step: + +- Choose a resampling method: + - replace: specify all the new x-values + - insert : insert a list of x values to the existing ones + - delete : delete a list of x values from the existing ones + - every-n : use every n values + - linspace : a linear spacing, insert a xmin, xmax and number of values + (works only when x is increasing monotically) + - time-based: downsample using sample averaging or upsample using + linear interpolation, x-axis must already be in seconds + - delta x : specify a delta for uniform spacing of x values + +- Specify the x values as a space or comma separated list + +- Click on one of the following buttons: + - Plot: will display the resampled data on the figure + - Apply: will perform the resampling on the fly for all new plots + - Add: will create new table(s) with resampled values for all + signals. This process might take some time. + Select a table or choose all (default) +""" # --------------------------------------------------------------------------------} # --- Data # --------------------------------------------------------------------------------{ @@ -67,7 +94,7 @@ class SamplerToolPanel(PlotDataActionEditor): def __init__(self, parent, action, **kwargs): import wx - PlotDataActionEditor.__init__(self, parent, action, tables=False) + PlotDataActionEditor.__init__(self, parent, action, help_string=_HELP, tables=False) # --- Data from pydatview.tools.signal_analysis import SAMPLERS @@ -212,35 +239,6 @@ def onAdd(self, event=None): # Call parent class PlotDataActionEditor.onAdd(self) - def onHelp(self,event=None): - Info(self,"""Resampling. - -The resampling operation changes the "x" values of a table/plot and -adapt the "y" values accordingly. - -To resample perform the following step: - -- Choose a resampling method: - - replace: specify all the new x-values - - insert : insert a list of x values to the existing ones - - delete : delete a list of x values from the existing ones - - every-n : use every n values - - linspace : a linear spacing, insert a xmin, xmax and number of values - (works only when x is increasing monotically) - - time-based: downsample using sample averaging or upsample using - linear interpolation, x-axis must already be in seconds - - delta x : specify a delta for uniform spacing of x values - -- Specify the x values as a space or comma separated list - -- Click on one of the following buttons: - - Plot: will display the resampled data on the figure - - Apply: will perform the resampling on the fly for all new plots - - Add: will create new table(s) with resampled values for all - signals. This process might take some time. - Select a table or choose all (default) -""") - if __name__ == '__main__': from pydatview.plugins.base_plugin import demoPlotDataActionPanel diff --git a/pydatview/plugins/tool_curvefitting.py b/pydatview/plugins/tool_curvefitting.py index 3130998..8656924 100644 --- a/pydatview/plugins/tool_curvefitting.py +++ b/pydatview/plugins/tool_curvefitting.py @@ -5,8 +5,23 @@ from pydatview.plugins.base_plugin import GUIToolPanel from pydatview.common import Error, Info, pretty_num_short, PyDatViewException from pydatview.tools.curve_fitting import model_fit, extract_key_miscnum, extract_key_num, MODELS, FITTERS, set_common_keys +_HELP = """Curve fitting +To perform a curve fit, adjusts the "Inputs section on the left": +- Select a predefined equation to fit, using the scrolldown menu. +- Adjust the initial guess for the parameters (if wanted) +- (Only for few models: set constants values) +- Click on "Fit" + +If you select a user-defined model: +- Equation parameters are specified using curly brackets +- Numpy functions are available using "np." +Buttons: +- Clear: remove the fit from the plot +- Add: add the fit data to the list of tables (can then be exported) + +""" # --------------------------------------------------------------------------------} # --- Curve Fitting # --------------------------------------------------------------------------------{ @@ -27,7 +42,7 @@ class CurveFitToolPanel(GUIToolPanel): def __init__(self, parent): - super(CurveFitToolPanel,self).__init__(parent) + super(CurveFitToolPanel,self).__init__(parent, help_string=_HELP) # Data self.x = None @@ -243,23 +258,4 @@ def onAdd(self,event=None): name='model_fit' if self.x is not None and self.y_fit is not None: df=pd.DataFrame({self.sx:self.x, self.sy:self.y_fit}) - self.parent.addTables([df], [name], bAdd=True) - - def onHelp(self,event=None): - Info(self,"""Curve fitting is still in beta. -To perform a curve fit, adjusts the "Inputs section on the left": -- Select a predefined equation to fit, using the scrolldown menu. -- Adjust the initial gues for the parameters (if wanted) -- (Only for few models: set constants values) -- Click on "Fit" - -If you select a user-defined model: -- Equation parameters are specified using curly brackets -- Numpy functions are available using "np" - -Buttons: -- Clear: remove the fit from the plot -- Add: add the fit data to the list of tables (can then be exported) - -""") diff --git a/pydatview/plugins/tool_logdec.py b/pydatview/plugins/tool_logdec.py index 29a742b..92e1d02 100644 --- a/pydatview/plugins/tool_logdec.py +++ b/pydatview/plugins/tool_logdec.py @@ -6,7 +6,7 @@ _HELP = """Damping and frequency estimation. -This tool attemps to estimate the natural frequency and damping ratio of the signal, +This tool attempts to estimate the natural frequency and damping ratio of the signal, assuming it has a dominant frequency and an exponential decay or growth. The damping ratio is referred to as "zeta". @@ -40,7 +40,7 @@ # --------------------------------------------------------------------------------{ class LogDecToolPanel(GUIToolPanel): def __init__(self, parent): - super(LogDecToolPanel,self).__init__(parent) + super(LogDecToolPanel,self).__init__(parent, help_string=_HELP) self.data = {} # --- GUI btClose = self.getBtBitmap(self,'Close' ,'close' ,self.destroy ) @@ -106,9 +106,6 @@ def onClear(self): #.mainframe.plotPanel.load_and_draw # or action.guiCallback return self.parent.load_and_draw() - def onHelp(self,event=None): - Info(self, _HELP) - # --- Fairly generic def _GUI2Data(self): self.data['method'] = 'fromPeaks' From 16d4c202348684dd54472e263580170dd2e99c3e Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 16:40:47 -0600 Subject: [PATCH 175/178] Doc: mentioning new features in README --- README.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 22c5151..7b9ca19 100644 --- a/README.md +++ b/README.md @@ -106,9 +106,12 @@ Documentation is scarce for now, but here are some tips for using the program: - You can open several files at once, with same or different filetypes. Upon opening multiple files, a new table appears with the list of open files. - To add multiple channels or data from multiple files to a plot, use `ctrl + click` or shift-click to make selections. - Look for the menus indicated by the "sandwich" symbol (3 horizontal bars ੆). These menus are also accessible with right clicks. - - The menus will allow you to edit tables (rename, delete them), add or remove columns (for instance to convert a signal from one unit to another unit), or change the values displayed in the information table at the bottom. - - Few options are also available in the menus `data` and `tools` located at the top of the program. - - The modes and file format drop down menus at the top can usually be kept on `auto`. If a file cannot be read, pay attention to the file extension used, and possibly select a specific file format in the dropdown menu instead of `auto`. + - The menus will allow you to edit tables (rename, delete, reload, merge), add or remove columns (for instance to convert a signal from one unit to another unit), or change the values displayed in the information table at the bottom. + - Different "actions" (e.g. filtering, binning, masking) are available in the menus `data` and `tools` located at the top of the program. + - The modes and fileformat drop down menus at the top can usually be kept on `auto`. If a file cannot be read, pay attention to the file extension used, and possibly select a specific file format in the dropdown menu instead of `auto`. + - Above the taskbar is the "Pipeline" which lists the different actions (e.g. binning, filtering, mask) that are applied to the different tables before being plotted. The pipeline actions will be reapplied on reload, and python code for them will be generated when exporting a script. + - Different plot styling options can be found below the plot area. The button next to the "Save" icon can be used to customize the esthetics of the plot (e.g. fontsize, linewidth, legend location). + - Live plotting can be disabled using the check box "Live plot". This is useful when manipulating large datasets, and potentially wanting to delete some columns without plotting them. @@ -119,6 +122,8 @@ Main features: - Reload of data (e.g. on file change) - Display of statistics - Export figure as pdf, png, eps, svg +- Export python script +- Export data as csv, or other file formats Different kind of plots: - Scatter plots or line plots @@ -128,13 +133,19 @@ Different kind of plots: Plot options: - Logarithmic scales on x and y axis -- Scaling of data between 0 and 1 using min and max +- Scaling data ("min/max") when ranges and means are different - Synchronization of the x-axis of the sub-figures while zooming +- Markers annotations and Measurements +- Plot styling options Data manipulation options: - Remove columns in a table, add columns using a given formula, and export the table to csv - Mask part of the data (for instance selecting times above a certain value to remove the transient). Apply the mask temporarily, or create a new table from it - - Estimate logarithmic decrement from a signal tthat is decaying + - Filter data (e.g. moving averaging, low-pass filters) + - Bin data + - Change units + - Estimate frequency and damping from a signal + - Curve fitting - Extract radial data from OpenFAST input files From 81b03b3f8137f0e65d3cb47d580eb8a78ae8c740 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 16:53:07 -0600 Subject: [PATCH 176/178] Version 0.4: update of version number --- .github/workflows/tests.yml | 2 +- _tools/NewRelease.md | 1 + installer.cfg | 2 +- pydatview/main.py | 2 +- setup.py | 4 ++-- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1056733..7f079c2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: run: | git fetch --unshallow > /dev/null git fetch --tags > /dev/null - export CURRENT_TAG="v0.3" + export CURRENT_TAG="v0.4" export CURRENT_DEV_TAG="$CURRENT_TAG-dev" # BRANCH FAILS export BRANCH1=`git rev-parse --abbrev-ref HEAD` diff --git a/_tools/NewRelease.md b/_tools/NewRelease.md index a8256d0..0078a7e 100644 --- a/_tools/NewRelease.md +++ b/_tools/NewRelease.md @@ -2,6 +2,7 @@ # Creating a new release Steps: +- Change version in setup.py - Change PROG\_VERSION in pydateview/main.py - Change version in installler.cfg - Change CURRENT\_DEV\_TAG in .github/workflows/tests.yml (not pretty...) diff --git a/installer.cfg b/installer.cfg index c6d5941..53fec1e 100644 --- a/installer.cfg +++ b/installer.cfg @@ -1,6 +1,6 @@ [Application] name=pyDatView -version=0.3 +version=0.4 entry_point=pydatview:show_sys_args icon=ressources/pyDatView.ico diff --git a/pydatview/main.py b/pydatview/main.py index 95e37a9..cbbefc6 100644 --- a/pydatview/main.py +++ b/pydatview/main.py @@ -41,7 +41,7 @@ # --- GLOBAL # --------------------------------------------------------------------------------{ PROG_NAME='pyDatView' -PROG_VERSION='v0.3-local' +PROG_VERSION='v0.4-local' SIDE_COL = [160,160,300,420,530] SIDE_COL_LARGE = [200,200,360,480,600] BOT_PANL =85 diff --git a/setup.py b/setup.py index f8dd4f5..2ac5332 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ setup( name='pydatview', - version='1.0', + version='0.4', description='GUI to display tabulated data from files or pandas dataframes', - url='http://github.com/elmanuelito/pyDatView/', + url='http://github.com/ebranlard/pyDatView/', author='Emmanuel Branlard', author_email='lastname@gmail.com', license='MIT', From 82d8ac0e0bf3e4e002ef9f15ac8bac778eda1ce0 Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Wed, 25 Oct 2023 18:13:16 -0600 Subject: [PATCH 177/178] Version rc0.4: Misc updates --- pydatview/GUIPlotPanel.py | 3 ++- pydatview/GUISelectionPanel.py | 2 +- pydatview/Tables.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pydatview/GUIPlotPanel.py b/pydatview/GUIPlotPanel.py index c881c8a..b26cedf 100644 --- a/pydatview/GUIPlotPanel.py +++ b/pydatview/GUIPlotPanel.py @@ -1695,7 +1695,8 @@ def redraw_same_data(self, force_autoscale=False): self.cleanPlot(); return elif len(self.plotData) == 1: - if self.plotData[0].xIsString or self.plotData[0].yIsString or self.plotData[0].xIsDate or self.plotData[0].yIsDate: + if self.plotData[0].xIsString or self.plotData[0].yIsString: + # or self.plotData[0].xIsDate or self.plotData[0].yIsDate: self.cbAutoScale.SetValue(True) else: if len(self.xlim_prev)==0: # Might occur if some date didn't plot before (e.g. strings) diff --git a/pydatview/GUISelectionPanel.py b/pydatview/GUISelectionPanel.py index 6d3d44a..1e5b6b2 100644 --- a/pydatview/GUISelectionPanel.py +++ b/pydatview/GUISelectionPanel.py @@ -1172,7 +1172,7 @@ def autoMode(self): self.threeColumnsMode() else: #self.simColumnsMode(self) - raise Exception('Too many panels selected with significant columns differences.') + raise PyDatViewException('Too many panels selected with significant columns differences.') def sameColumnsMode(self): self.currentMode = 'sameColumnsMode' diff --git a/pydatview/Tables.py b/pydatview/Tables.py index f2f6a40..b8df2b4 100644 --- a/pydatview/Tables.py +++ b/pydatview/Tables.py @@ -233,7 +233,8 @@ def swap(self, i1, i2): def sort(self, method='byName'): if method=='byName': tabnames_display=self.getDisplayTabNames() - self._tabs = [t for _,t in sorted(zip(tabnames_display,self._tabs))] + # Cannot use sorted(zip()) below + self._tabs = [self._tabs[i] for i in np.argsort(tabnames_display)] else: raise PyDatViewException('Sorting method unknown: `{}`'.format(method)) From dbcd123acab09cbd067f19acd13ca6ab119c8efb Mon Sep 17 00:00:00 2001 From: Emmanuel Branlard Date: Fri, 27 Oct 2023 14:23:01 -0600 Subject: [PATCH 178/178] IO: FASTOut: small fix for data with one timestamp --- pydatview/io/fast_output_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydatview/io/fast_output_file.py b/pydatview/io/fast_output_file.py index b5e7a53..c6dc0a7 100644 --- a/pydatview/io/fast_output_file.py +++ b/pydatview/io/fast_output_file.py @@ -173,8 +173,9 @@ def toDataFrame(self): df= self.data df.columns=cols else: + self.data = np.atleast_2d(self.data) if len(cols)!=self.data.shape[1]: - raise BrokenFormatError('Inconstistent number of columns between headers ({}) and data ({}) for file {}'.format(len(cols), self.data.shape[1], self.filename)) + raise BrokenFormatError('Inconsistent number of columns between headers ({}) and data ({}) for file {}'.format(len(cols), self.data.shape[1], self.filename)) df = pd.DataFrame(data=self.data,columns=cols) return df