From 3dc69654e07069c46145403c3859a829c28ad686 Mon Sep 17 00:00:00 2001 From: Vecko Date: Wed, 31 Jul 2024 17:09:22 +0200 Subject: [PATCH] Update ast, os, sys standard libraries to module imports --- parcels/fieldset.py | 6 ++-- parcels/interaction/interactionkernel.py | 4 +-- parcels/kernel.py | 40 ++++++++++++------------ parcels/particleset.py | 4 +-- parcels/rng.py | 16 +++++----- tests/test_data/create_testfields.py | 6 ++-- tests/test_data/fieldset_nemo.py | 16 +++++----- tests/test_data/fieldset_nemo_error.py | 16 +++++----- tests/test_fieldset.py | 31 +++++++++--------- tests/test_grids.py | 20 ++++++------ tests/test_kernel_execution.py | 6 ++-- tests/test_kernel_language.py | 4 +-- tests/test_mpirun.py | 12 +++---- 13 files changed, 90 insertions(+), 91 deletions(-) diff --git a/parcels/fieldset.py b/parcels/fieldset.py index e10276c1d..29262c085 100644 --- a/parcels/fieldset.py +++ b/parcels/fieldset.py @@ -1,8 +1,8 @@ import importlib.util +import os import sys from copy import deepcopy from glob import glob -from os import path import dask.array as da import numpy as np @@ -304,7 +304,7 @@ def parse_wildcards(cls, paths, filenames, var): notfound_paths = filenames[var] if isinstance(filenames, dict) and var in filenames else filenames raise OSError(f"FieldSet files not found for variable {var}: {str(notfound_paths)}") for fp in paths: - if not path.exists(fp): + if not os.path.exists(fp): raise OSError(f"FieldSet file not found: {fp}") return paths @@ -1074,7 +1074,7 @@ def from_modulefile(cls, filename, modulename="create_fieldset", **kwargs): modulename: name of the function in the python file that returns a FieldSet object. Default is "create_fieldset". """ # check if filename exists - if not path.exists(filename): + if not os.path.exists(filename): raise IOError(f"FieldSet module file {filename} does not exist") # Importing the source file directly (following https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly) diff --git a/parcels/interaction/interactionkernel.py b/parcels/interaction/interactionkernel.py index 2a0bb6061..836ce1055 100644 --- a/parcels/interaction/interactionkernel.py +++ b/parcels/interaction/interactionkernel.py @@ -1,6 +1,6 @@ import inspect +import sys from collections import defaultdict -from sys import version_info import numpy as np @@ -93,7 +93,7 @@ def check_kernel_signature_on_version(self): numkernelargs = [] if self._pyfunc is not None and isinstance(self._pyfunc, list): for func in self._pyfunc: - if version_info[0] < 3: + if sys.version_info[0] < 3: numkernelargs.append( len(inspect.getargspec(func).args) ) diff --git a/parcels/kernel.py b/parcels/kernel.py index 06d8193f9..1825b0efe 100644 --- a/parcels/kernel.py +++ b/parcels/kernel.py @@ -1,15 +1,15 @@ +import ast import functools +import hashlib import inspect import math # noqa +import os import random # noqa import re +import sys import types -from ast import FunctionDef, parse from copy import deepcopy from ctypes import byref, c_double, c_int -from hashlib import md5 -from os import path, remove -from sys import platform, version_info from time import time as ostime import _ctypes @@ -117,7 +117,7 @@ def _cache_key(self): field_keys = "-".join( [f"{name}:{field.units.__class__.__name__}" for name, field in self.field_args.items()]) key = self.name + self.ptype._cache_key + field_keys + ('TIME:%f' % ostime()) - return md5(key.encode('utf-8')).hexdigest() + return hashlib.md5(key.encode('utf-8')).hexdigest() @staticmethod def fix_indentation(string): @@ -177,7 +177,7 @@ def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, p self.funcvars = None self.funccode = funccode or inspect.getsource(pyfunc.__code__) # Parse AST if it is not provided explicitly - self.py_ast = py_ast or parse(BaseKernel.fix_indentation(self.funccode)).body[0] + self.py_ast = py_ast or ast.parse(BaseKernel.fix_indentation(self.funccode)).body[0] if pyfunc is None: # Extract user context by inspecting the call stack stack = inspect.stack() @@ -193,7 +193,7 @@ def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, p finally: del stack # Remove cyclic references # Compile and generate Python function from AST - py_mod = parse("") + py_mod = ast.parse("") py_mod.body = [self.py_ast] exec(compile(py_mod, "", "exec"), user_ctx) self._pyfunc = user_ctx[self.funcname] @@ -223,7 +223,7 @@ def __init__(self, fieldset, ptype, pyfunc=None, funcname=None, funccode=None, p self.field_args[sF_name] = getattr(f, sF_component) self.const_args = kernelgen.const_args loopgen = LoopGenerator(fieldset, ptype) - if path.isfile(self._c_include): + if os.path.isfile(self._c_include): with open(self._c_include) as f: c_include_str = f.read() else: @@ -274,7 +274,7 @@ def _cache_key(self): field_keys = "-".join( [f"{name}:{field.units.__class__.__name__}" for name, field in self.field_args.items()]) key = self.name + self.ptype._cache_key + field_keys + ('TIME:%f' % ostime()) - return md5(key.encode('utf-8')).hexdigest() + return hashlib.md5(key.encode('utf-8')).hexdigest() def add_scipy_positionupdate_kernels(self): # Adding kernels that set and update the coordinate changes @@ -339,7 +339,7 @@ def check_fieldsets_in_kernels(self, pyfunc): def check_kernel_signature_on_version(self): numkernelargs = 0 if self._pyfunc is not None: - if version_info[0] < 3: + if sys.version_info[0] < 3: numkernelargs = len(inspect.getargspec(self._pyfunc).args) else: numkernelargs = len(inspect.getfullargspec(self._pyfunc).args) @@ -392,11 +392,11 @@ def get_kernel_compile_files(self): if type(basename) in (list, dict, tuple, ndarray): src_file_or_files = ["", ] * len(basename) for i, src_file in enumerate(basename): - src_file_or_files[i] = f"{path.join(dyn_dir, src_file)}.c" + src_file_or_files[i] = f"{os.path.join(dyn_dir, src_file)}.c" else: - src_file_or_files = f"{path.join(dyn_dir, basename)}.c" - lib_file = f"{path.join(dyn_dir, lib_path)}.{'dll' if platform == 'win32' else 'so'}" - log_file = f"{path.join(dyn_dir, basename)}.log" + src_file_or_files = f"{os.path.join(dyn_dir, basename)}.c" + lib_file = f"{os.path.join(dyn_dir, lib_path)}.{'dll' if sys.platform == 'win32' else 'so'}" + log_file = f"{os.path.join(dyn_dir, basename)}.log" return src_file_or_files, lib_file, log_file def compile(self, compiler): @@ -430,8 +430,8 @@ def merge(self, kernel, kclass): funcname = self.funcname + kernel.funcname func_ast = None if self.py_ast is not None: - func_ast = FunctionDef(name=funcname, args=self.py_ast.args, body=self.py_ast.body + kernel.py_ast.body, - decorator_list=[], lineno=1, col_offset=0) + func_ast = ast.FunctionDef(name=funcname, args=self.py_ast.args, body=self.py_ast.body + kernel.py_ast.body, + decorator_list=[], lineno=1, col_offset=0) delete_cfiles = self.delete_cfiles and kernel.delete_cfiles return kclass(self.fieldset, self.ptype, pyfunc=None, funcname=funcname, funccode=self.funccode + kernel.funccode, @@ -483,10 +483,10 @@ def from_list(cls, fieldset, ptype, pyfunc_list, *args, **kwargs): @staticmethod def cleanup_remove_files(lib_file, all_files_array, delete_cfiles): if lib_file is not None: - if path.isfile(lib_file): # and delete_cfiles - [remove(s) for s in [lib_file, ] if path is not None and path.exists(s)] + if os.path.isfile(lib_file): # and delete_cfiles + [os.remove(s) for s in [lib_file, ] if os.path is not None and os.path.exists(s)] if delete_cfiles and len(all_files_array) > 0: - [remove(s) for s in all_files_array if path is not None and path.exists(s)] + [os.remove(s) for s in all_files_array if os.path is not None and os.path.exists(s)] @staticmethod def cleanup_unload_lib(lib): @@ -495,7 +495,7 @@ def cleanup_unload_lib(lib): # naming scheme which is required on Windows OS'es to deal with updates to a Parcels' kernel. if lib is not None: try: - _ctypes.FreeLibrary(lib._handle) if platform == 'win32' else _ctypes.dlclose(lib._handle) + _ctypes.FreeLibrary(lib._handle) if sys.platform == 'win32' else _ctypes.dlclose(lib._handle) except: pass diff --git a/parcels/particleset.py b/parcels/particleset.py index 5ae9b4025..a62004b6f 100644 --- a/parcels/particleset.py +++ b/parcels/particleset.py @@ -1,9 +1,9 @@ +import os import sys from abc import ABC from copy import copy from datetime import date, datetime from datetime import timedelta as delta -from os import path import cftime import numpy as np @@ -855,7 +855,7 @@ def execute(self, pyfunc=AdvectionRK4, pyfunc_inter=None, endtime=None, runtime= if self.particledata.ptype.uses_jit: self.kernel.remove_lib() cppargs = ['-DDOUBLE_COORD_VARIABLES'] if self.particledata.lonlatdepth_dtype else None - self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs, incdirs=[path.join(get_package_dir(), 'include'), "."])) + self.kernel.compile(compiler=GNUCompiler(cppargs=cppargs, incdirs=[os.path.join(get_package_dir(), 'include'), "."])) self.kernel.load_lib() if output_file: output_file.add_metadata('parcels_kernels', self.kernel.name) diff --git a/parcels/rng.py b/parcels/rng.py index 23c52814b..54421b826 100644 --- a/parcels/rng.py +++ b/parcels/rng.py @@ -1,7 +1,7 @@ +import os +import sys import uuid from ctypes import c_float, c_int -from os import path, remove -from sys import platform import _ctypes import numpy.ctypeslib as npct @@ -78,7 +78,7 @@ def __del__(self): def unload_lib(self): # Unload the currently loaded dynamic linked library to be secure if self._lib is not None and self._loaded and _ctypes is not None: - _ctypes.FreeLibrary(self._lib._handle) if platform == 'win32' else _ctypes.dlclose(self._lib._handle) + _ctypes.FreeLibrary(self._lib._handle) if sys.platform == 'win32' else _ctypes.dlclose(self._lib._handle) del self._lib self._lib = None self._loaded = False @@ -90,22 +90,22 @@ def load_lib(self): def remove_lib(self): # If file already exists, pull new names. This is necessary on a Windows machine, because # Python's ctype does not deal in any sort of manner well with dynamic linked libraries on this OS. - if self._lib is not None and self._loaded and _ctypes is not None and path.isfile(self.lib_file): - [remove(s) for s in [self.src_file, self.lib_file, self.log_file]] + if self._lib is not None and self._loaded and _ctypes is not None and os.path.isfile(self.lib_file): + [os.remove(s) for s in [self.src_file, self.lib_file, self.log_file]] def compile(self, compiler=None): if self.src_file is None or self.lib_file is None or self.log_file is None: basename = 'parcels_random_%s' % uuid.uuid4() lib_filename = "lib" + basename - basepath = path.join(get_cache_dir(), f"{basename}") - libpath = path.join(get_cache_dir(), f"{lib_filename}") + basepath = os.path.join(get_cache_dir(), f"{basename}") + libpath = os.path.join(get_cache_dir(), f"{lib_filename}") self.src_file = f"{basepath}.c" self.lib_file = f"{libpath}.so" self.log_file = f"{basepath}.log" ccompiler = compiler if ccompiler is None: cppargs = [] - incdirs = [path.join(get_package_dir(), 'include'), ] + incdirs = [os.path.join(get_package_dir(), 'include'), ] ccompiler = GNUCompiler(cppargs=cppargs, incdirs=incdirs) if self._lib is None: with open(self.src_file, 'w+') as f: diff --git a/tests/test_data/create_testfields.py b/tests/test_data/create_testfields.py index bbc259d77..ce759432d 100644 --- a/tests/test_data/create_testfields.py +++ b/tests/test_data/create_testfields.py @@ -9,7 +9,7 @@ except: asizeof = None -from os import path +import os import xarray as xr @@ -66,8 +66,8 @@ def generate_perlin_testfield(): print(f"Perlin V-field requires {V.size * V.itemsize} bytes of memory.") fieldset = FieldSet.from_data(data, dimensions, mesh='spherical', transpose=False) # fieldset.write("perlinfields") # can also be used, but then has a ghost depth dimension - write_simple_2Dt(fieldset.U, path.join(path.dirname(__file__), 'perlinfields'), varname='vozocrtx') - write_simple_2Dt(fieldset.V, path.join(path.dirname(__file__), 'perlinfields'), varname='vomecrty') + write_simple_2Dt(fieldset.U, os.path.join(os.path.dirname(__file__), 'perlinfields'), varname='vozocrtx') + write_simple_2Dt(fieldset.V, os.path.join(os.path.dirname(__file__), 'perlinfields'), varname='vomecrty') def write_simple_2Dt(field, filename, varname=None): diff --git a/tests/test_data/fieldset_nemo.py b/tests/test_data/fieldset_nemo.py index 207a5a126..aa8347af2 100644 --- a/tests/test_data/fieldset_nemo.py +++ b/tests/test_data/fieldset_nemo.py @@ -1,17 +1,17 @@ -from os import path +import os import parcels def create_fieldset(indices=None): - data_path = path.join(path.dirname(__file__)) + data_path = os.path.join(os.path.dirname(__file__)) - filenames = {'U': {'lon': path.join(data_path, 'mask_nemo_cross_180lon.nc'), - 'lat': path.join(data_path, 'mask_nemo_cross_180lon.nc'), - 'data': path.join(data_path, 'Uu_eastward_nemo_cross_180lon.nc')}, - 'V': {'lon': path.join(data_path, 'mask_nemo_cross_180lon.nc'), - 'lat': path.join(data_path, 'mask_nemo_cross_180lon.nc'), - 'data': path.join(data_path, 'Vv_eastward_nemo_cross_180lon.nc')}} + filenames = {'U': {'lon': os.path.join(data_path, 'mask_nemo_cross_180lon.nc'), + 'lat': os.path.join(data_path, 'mask_nemo_cross_180lon.nc'), + 'data': os.path.join(data_path, 'Uu_eastward_nemo_cross_180lon.nc')}, + 'V': {'lon': os.path.join(data_path, 'mask_nemo_cross_180lon.nc'), + 'lat': os.path.join(data_path, 'mask_nemo_cross_180lon.nc'), + 'data': os.path.join(data_path, 'Vv_eastward_nemo_cross_180lon.nc')}} variables = {'U': 'U', 'V': 'V'} dimensions = {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'} indices = indices or {} diff --git a/tests/test_data/fieldset_nemo_error.py b/tests/test_data/fieldset_nemo_error.py index a2c14f802..918fd6ccd 100644 --- a/tests/test_data/fieldset_nemo_error.py +++ b/tests/test_data/fieldset_nemo_error.py @@ -1,17 +1,17 @@ -from os import path +import os import parcels def random_function_name(): - data_path = path.join(path.dirname(__file__)) + data_path = os.path.join(os.path.dirname(__file__)) - filenames = {'U': {'lon': path.join(data_path, 'mask_nemo_cross_180lon.nc'), - 'lat': path.join(data_path, 'mask_nemo_cross_180lon.nc'), - 'data': path.join(data_path, 'Uu_eastward_nemo_cross_180lon.nc')}, - 'V': {'lon': path.join(data_path, 'mask_nemo_cross_180lon.nc'), - 'lat': path.join(data_path, 'mask_nemo_cross_180lon.nc'), - 'data': path.join(data_path, 'Vv_eastward_nemo_cross_180lon.nc')}} + filenames = {'U': {'lon': os.path.join(data_path, 'mask_nemo_cross_180lon.nc'), + 'lat': os.path.join(data_path, 'mask_nemo_cross_180lon.nc'), + 'data': os.path.join(data_path, 'Uu_eastward_nemo_cross_180lon.nc')}, + 'V': {'lon': os.path.join(data_path, 'mask_nemo_cross_180lon.nc'), + 'lat': os.path.join(data_path, 'mask_nemo_cross_180lon.nc'), + 'data': os.path.join(data_path, 'Vv_eastward_nemo_cross_180lon.nc')}} variables = {'U': 'U', 'V': 'V'} dimensions = {'lon': 'glamf', 'lat': 'gphif', 'time': 'time_counter'} return parcels.FieldSet.from_nemo(filenames, variables, dimensions) diff --git a/tests/test_fieldset.py b/tests/test_fieldset.py index ecb23fd05..4b8b383e7 100644 --- a/tests/test_fieldset.py +++ b/tests/test_fieldset.py @@ -3,7 +3,6 @@ import os import sys from datetime import timedelta as delta -from os import path import cftime import dask @@ -139,7 +138,7 @@ def test_fieldset_from_parcels(xdim, ydim, tmpdir, filename='test_parcels'): def test_field_from_netcdf_variables(): - data_path = path.join(path.dirname(__file__), 'test_data/') + data_path = os.path.join(os.path.dirname(__file__), 'test_data/') filename = data_path + 'perlinfieldsU.nc' dims = {'lon': 'x', 'lat': 'y'} @@ -185,7 +184,7 @@ def test_fieldset_nonstandardtime(calendar, cftime_datetime, tmpdir, filename='t @pytest.mark.parametrize('with_timestamps', [True, False]) def test_field_from_netcdf(with_timestamps): - data_path = path.join(path.dirname(__file__), 'test_data/') + data_path = os.path.join(os.path.dirname(__file__), 'test_data/') filenames = {'lon': data_path + 'mask_nemo_cross_180lon.nc', 'lat': data_path + 'mask_nemo_cross_180lon.nc', @@ -201,7 +200,7 @@ def test_field_from_netcdf(with_timestamps): def test_fieldset_from_modulefile(): - data_path = path.join(path.dirname(__file__), 'test_data/') + data_path = os.path.join(os.path.dirname(__file__), 'test_data/') fieldset = FieldSet.from_modulefile(data_path + 'fieldset_nemo.py') assert fieldset.U.creation_log == 'from_nemo' @@ -219,7 +218,7 @@ def test_fieldset_from_modulefile(): def test_field_from_netcdf_fieldtypes(): - data_path = path.join(path.dirname(__file__), 'test_data/') + data_path = os.path.join(os.path.dirname(__file__), 'test_data/') filenames = {'varU': {'lon': data_path + 'mask_nemo_cross_180lon.nc', 'lat': data_path + 'mask_nemo_cross_180lon.nc', @@ -240,7 +239,7 @@ def test_field_from_netcdf_fieldtypes(): def test_fieldset_from_cgrid_interpmethod(): - data_path = path.join(path.dirname(__file__), 'test_data/') + data_path = os.path.join(os.path.dirname(__file__), 'test_data/') filenames = {'lon': data_path + 'mask_nemo_cross_180lon.nc', 'lat': data_path + 'mask_nemo_cross_180lon.nc', @@ -329,7 +328,7 @@ def test_illegal_dimensionsdict(calltype): dimensions['test'] = None FieldSet.from_data(data, dimensions) elif calltype == 'from_nemo': - fname = path.join(path.dirname(__file__), 'test_data', 'mask_nemo_cross_180lon.nc') + fname = os.path.join(os.path.dirname(__file__), 'test_data', 'mask_nemo_cross_180lon.nc') filenames = {'dx': fname, 'mesh_mask': fname} variables = {'dx': 'e1u'} dimensions = {'lon': 'glamu', 'lat': 'gphiu', 'test': 'test'} @@ -500,7 +499,7 @@ def test_fieldset_celledgesizes(mesh): @pytest.mark.parametrize('dx, dy', [('e1u', 'e2u'), ('e1v', 'e2v')]) def test_fieldset_celledgesizes_curvilinear(dx, dy): - fname = path.join(path.dirname(__file__), 'test_data', 'mask_nemo_cross_180lon.nc') + fname = os.path.join(os.path.dirname(__file__), 'test_data', 'mask_nemo_cross_180lon.nc') filenames = {'dx': fname, 'dy': fname, 'mesh_mask': fname} variables = {'dx': dx, 'dy': dy} dimensions = {'dx': {'lon': 'glamu', 'lat': 'gphiu'}, @@ -516,7 +515,7 @@ def test_fieldset_celledgesizes_curvilinear(dx, dy): def test_fieldset_write_curvilinear(tmpdir): - fname = path.join(path.dirname(__file__), 'test_data', 'mask_nemo_cross_180lon.nc') + fname = os.path.join(os.path.dirname(__file__), 'test_data', 'mask_nemo_cross_180lon.nc') filenames = {'dx': fname, 'mesh_mask': fname} variables = {'dx': 'e1u'} dimensions = {'lon': 'glamu', 'lat': 'gphiu'} @@ -536,7 +535,7 @@ def test_fieldset_write_curvilinear(tmpdir): def test_curv_fieldset_add_periodic_halo(): - fname = path.join(path.dirname(__file__), 'test_data', 'mask_nemo_cross_180lon.nc') + fname = os.path.join(os.path.dirname(__file__), 'test_data', 'mask_nemo_cross_180lon.nc') filenames = {'dx': fname, 'dy': fname, 'mesh_mask': fname} variables = {'dx': 'e1u', 'dy': 'e1v'} dimensions = {'dx': {'lon': 'glamu', 'lat': 'gphiu'}, @@ -702,8 +701,8 @@ def periodicBoundaryConditions(particle, fieldset, time): process = psutil.Process(os.getpid()) mem_0 = process.memory_info().rss - fnameU = path.join(path.dirname(__file__), 'test_data', 'perlinfieldsU.nc') - fnameV = path.join(path.dirname(__file__), 'test_data', 'perlinfieldsV.nc') + fnameU = os.path.join(os.path.dirname(__file__), 'test_data', 'perlinfieldsU.nc') + fnameV = os.path.join(os.path.dirname(__file__), 'test_data', 'perlinfieldsV.nc') ufiles = [fnameU, ] * 4 vfiles = [fnameV, ] * 4 timestamps = np.arange(0, 4, 1) * 86400.0 @@ -737,8 +736,8 @@ def periodicBoundaryConditions(particle, fieldset, time): @pytest.mark.parametrize('chunksize', [False, 'auto', {'lat': ('y', 32), 'lon': ('x', 32)}, {'time': ('time_counter', 1), 'lat': ('y', 32), 'lon': ('x', 32)}]) @pytest.mark.parametrize('deferLoad', [True, False]) def test_from_netcdf_chunking(mode, time_periodic, chunksize, deferLoad): - fnameU = path.join(path.dirname(__file__), 'test_data', 'perlinfieldsU.nc') - fnameV = path.join(path.dirname(__file__), 'test_data', 'perlinfieldsV.nc') + fnameU = os.path.join(os.path.dirname(__file__), 'test_data', 'perlinfieldsU.nc') + fnameV = os.path.join(os.path.dirname(__file__), 'test_data', 'perlinfieldsV.nc') ufiles = [fnameU, ] * 4 vfiles = [fnameV, ] * 4 timestamps = np.arange(0, 4, 1) * 86400.0 @@ -1008,7 +1007,7 @@ def generate_dataset(xdim, ydim, zdim=1, tdim=1): @pytest.mark.parametrize('mode', ['scipy', 'jit']) def test_fieldset_frompop(mode): - filenames = path.join(path.join(path.dirname(__file__), 'test_data'), 'POPtestdata_time.nc') + filenames = os.path.join(os.path.join(os.path.dirname(__file__), 'test_data'), 'POPtestdata_time.nc') variables = {'U': 'U', 'V': 'V', 'W': 'W', 'T': 'T'} dimensions = {'lon': 'lon', 'lat': 'lat', 'time': 'time'} @@ -1097,7 +1096,7 @@ def SampleU(particle, fieldset, time): def test_daskfieldfilebuffer_dimnames(): DaskFileBuffer.add_to_dimension_name_map_global({'lat': 'nydim', 'lon': 'nxdim'}) - fnameU = path.join(path.dirname(__file__), 'test_data', 'perlinfieldsU.nc') + fnameU = os.path.join(os.path.dirname(__file__), 'test_data', 'perlinfieldsU.nc') dimensions = {'lon': 'nav_lon', 'lat': 'nav_lat'} fb = DaskFileBuffer(fnameU, dimensions, indices={}) assert ('nxdim' in fb._static_name_maps['lon']) and ('ntdim' not in fb._static_name_maps['time']) diff --git a/tests/test_grids.py b/tests/test_grids.py index 76a11e0ea..9553386a3 100644 --- a/tests/test_grids.py +++ b/tests/test_grids.py @@ -1,6 +1,6 @@ import math +import os from datetime import timedelta as delta -from os import path import numpy as np import pytest @@ -359,7 +359,7 @@ def sampleSpeed(particle, fieldset, time): @pytest.mark.parametrize('mode', ['scipy', 'jit']) def test_nemo_grid(mode): - data_path = path.join(path.dirname(__file__), 'test_data/') + data_path = os.path.join(os.path.dirname(__file__), 'test_data/') filenames = {'U': {'lon': data_path + 'mask_nemo_cross_180lon.nc', 'lat': data_path + 'mask_nemo_cross_180lon.nc', @@ -393,7 +393,7 @@ def sampleVel(particle, fieldset, time): @pytest.mark.parametrize('mode', ['scipy', 'jit']) def test_advect_nemo(mode): - data_path = path.join(path.dirname(__file__), 'test_data/') + data_path = os.path.join(os.path.dirname(__file__), 'test_data/') filenames = {'U': {'lon': data_path + 'mask_nemo_cross_180lon.nc', 'lat': data_path + 'mask_nemo_cross_180lon.nc', @@ -510,7 +510,7 @@ def sampleVel(particle, fieldset, time): @pytest.mark.parametrize('vert_mode', ['zlev', 'slev1']) @pytest.mark.parametrize('time', [True, False]) def test_cgrid_uniform_3dvel_spherical(mode, vert_mode, time): - data_path = path.join(path.dirname(__file__), 'test_data/') + data_path = os.path.join(os.path.dirname(__file__), 'test_data/') dim_file = xr.open_dataset(data_path + 'mask_nemo_cross_180lon.nc') u_file = xr.open_dataset(data_path + 'Uu_eastward_nemo_cross_180lon.nc') v_file = xr.open_dataset(data_path + 'Vv_eastward_nemo_cross_180lon.nc') @@ -571,7 +571,7 @@ def sampleVel(particle, fieldset, time): @pytest.mark.parametrize('vert_discretisation', ['zlevel', 'slevel', 'slevel2']) @pytest.mark.parametrize('deferred_load', [True, False]) def test_popgrid(mode, vert_discretisation, deferred_load): - mesh = path.join(path.join(path.dirname(__file__), 'test_data'), 'POPtestdata_time.nc') + mesh = os.path.join(os.path.join(os.path.dirname(__file__), 'test_data'), 'POPtestdata_time.nc') if vert_discretisation == 'zlevel': w_dep = 'w_dep' elif vert_discretisation == 'slevel': @@ -874,9 +874,9 @@ def test_bgrid_interpolation(gridindexingtype, mode, extrapolation): else: zi = 2 if gridindexingtype == 'mom5': - ufile = path.join(path.join(path.dirname(__file__), 'test_data'), 'access-om2-01_u.nc') - vfile = path.join(path.join(path.dirname(__file__), 'test_data'), 'access-om2-01_v.nc') - wfile = path.join(path.join(path.dirname(__file__), 'test_data'), 'access-om2-01_wt.nc') + ufile = os.path.join(os.path.join(os.path.dirname(__file__), 'test_data'), 'access-om2-01_u.nc') + vfile = os.path.join(os.path.join(os.path.dirname(__file__), 'test_data'), 'access-om2-01_v.nc') + wfile = os.path.join(os.path.join(os.path.dirname(__file__), 'test_data'), 'access-om2-01_wt.nc') filenames = {"U": {"lon": ufile, "lat": ufile, "depth": wfile, "data": ufile}, "V": {"lon": ufile, "lat": ufile, "depth": wfile, "data": vfile}, @@ -897,8 +897,8 @@ def test_bgrid_interpolation(gridindexingtype, mode, extrapolation): w = ds_w.wt.isel(time=0, sw_ocean=zi, yt_ocean=yi, xt_ocean=xi) elif gridindexingtype == 'pop': - datafname = path.join(path.join(path.dirname(__file__), 'test_data'), 'popdata.nc') - coordfname = path.join(path.join(path.dirname(__file__), 'test_data'), 'popcoordinates.nc') + datafname = os.path.join(os.path.join(os.path.dirname(__file__), 'test_data'), 'popdata.nc') + coordfname = os.path.join(os.path.join(os.path.dirname(__file__), 'test_data'), 'popcoordinates.nc') filenames = {"U": {"lon": coordfname, "lat": coordfname, "depth": coordfname, "data": datafname}, "V": {"lon": coordfname, "lat": coordfname, "depth": coordfname, "data": datafname}, "W": {"lon": coordfname, "lat": coordfname, "depth": coordfname, "data": datafname}} diff --git a/tests/test_kernel_execution.py b/tests/test_kernel_execution.py index b19914131..b51c3263b 100644 --- a/tests/test_kernel_execution.py +++ b/tests/test_kernel_execution.py @@ -1,5 +1,5 @@ +import os import sys -from os import path import numpy as np import pytest @@ -335,9 +335,9 @@ def test_execution_keep_cfiles_and_nocompilation_warnings(fieldset, delete_cfile logfile = pset.kernel.log_file del pset.kernel if delete_cfiles: - assert not path.exists(cfile) + assert not os.path.exists(cfile) else: - assert path.exists(cfile) + assert os.path.exists(cfile) with open(logfile) as f: assert 'warning' not in f.read(), 'Compilation WARNING in log file' diff --git a/tests/test_kernel_language.py b/tests/test_kernel_language.py index 83fbb3fb7..931620040 100644 --- a/tests/test_kernel_language.py +++ b/tests/test_kernel_language.py @@ -1,6 +1,6 @@ +import os import random as py_random from contextlib import nullcontext as does_not_raise -from os import path import numpy as np import pytest @@ -356,7 +356,7 @@ def func(U, lon, dt): } """ else: - c_include = path.join(path.dirname(__file__), 'customed_header.h') + c_include = os.path.join(os.path.dirname(__file__), 'customed_header.h') def ckernel(particle, fieldset, time): func('parcels_customed_Cfunc_pointer_args', fieldset.U, particle_dlon, particle.dt) # noqa diff --git a/tests/test_mpirun.py b/tests/test_mpirun.py index edce3946a..eb3862c6a 100644 --- a/tests/test_mpirun.py +++ b/tests/test_mpirun.py @@ -1,6 +1,6 @@ +import os import sys from glob import glob -from os import path, system import numpy as np import pytest @@ -11,19 +11,19 @@ @pytest.mark.parametrize('repeatdt, maxage', [(200*86400, 600*86400), (100*86400, 100*86400)]) @pytest.mark.parametrize('nump', [8]) def test_mpi_run(tmpdir, repeatdt, maxage, nump): - stommel_file = path.join(path.dirname(__file__), '..', 'docs', 'examples', 'example_stommel.py') + stommel_file = os.path.join(os.path.dirname(__file__), '..', 'docs', 'examples', 'example_stommel.py') outputMPI = tmpdir.join('StommelMPI') outputMPI_partition_function = tmpdir.join('StommelMPI_partition_function') outputNoMPI = tmpdir.join('StommelNoMPI.zarr') - system('mpirun -np 2 python %s -p %d -o %s -r %d -a %d -wf False -cpf True' % (stommel_file, nump, outputMPI_partition_function, repeatdt, maxage)) - system('mpirun -np 2 python %s -p %d -o %s -r %d -a %d -wf False' % (stommel_file, nump, outputMPI, repeatdt, maxage)) - system('python %s -p %d -o %s -r %d -a %d -wf False' % (stommel_file, nump, outputNoMPI, repeatdt, maxage)) + os.system('mpirun -np 2 python %s -p %d -o %s -r %d -a %d -wf False -cpf True' % (stommel_file, nump, outputMPI_partition_function, repeatdt, maxage)) + os.system('mpirun -np 2 python %s -p %d -o %s -r %d -a %d -wf False' % (stommel_file, nump, outputMPI, repeatdt, maxage)) + os.system('python %s -p %d -o %s -r %d -a %d -wf False' % (stommel_file, nump, outputNoMPI, repeatdt, maxage)) ds2 = xr.open_zarr(outputNoMPI) for mpi_run in [outputMPI, outputMPI_partition_function]: - files = glob(path.join(mpi_run, "proc*")) + files = glob(os.path.join(mpi_run, "proc*")) ds1 = xr.concat([xr.open_zarr(f) for f in files], dim='trajectory', compat='no_conflicts', coords='minimal').sortby(['trajectory'])