From 3975b29b75e1c7d5286dc654cbb4cdf72f438115 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 13 Sep 2023 16:02:00 +0900 Subject: [PATCH 01/15] Profiling with PPE --- pfio/v2/local.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index 9611d232..4052a0ff 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -4,6 +4,21 @@ import shutil from typing import Optional +try: + from pytorch_pfn_extras.profiler import record_function, record_iterable + +except ImportError: + print("no ppe") + # IF PPE is not available, wrap with noop + def record_function(*args): + def wrapper(f): + return f + return wrapper + + def record_iterable(tag, iter, *args): + yield from iter + + from .fs import FS, FileStat, format_repr @@ -83,6 +98,7 @@ def __repr__(self): }, ) + @record_function("local-open") def open(self, file_path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): @@ -94,6 +110,12 @@ def open(self, file_path, mode='r', def list(self, path: Optional[str] = '', recursive=False, detail=False): + for e in record_iterable("local-list", + self._list(path, recursive, detail)): + yield e + + def _list(self, path: Optional[str] = '', recursive=False, + detail=False): path_or_prefix = os.path.join(self.cwd, "" if path is None else path) @@ -128,6 +150,7 @@ def _recursive_list(self, prefix_end_index: int, path: str, yield from self._recursive_list(prefix_end_index, e.path, detail) + @record_function("local-stat") def stat(self, path): path = os.path.join(self.cwd, path) return LocalFileStat(os.stat(path), path) From b35a47af85a00852eebb1a396d0c90363a3836e9 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 13 Sep 2023 16:10:19 +0900 Subject: [PATCH 02/15] Profiling with PPE --- pfio/v2/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index 4052a0ff..e6c9e50f 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -8,7 +8,7 @@ from pytorch_pfn_extras.profiler import record_function, record_iterable except ImportError: - print("no ppe") + # IF PPE is not available, wrap with noop def record_function(*args): def wrapper(f): From b3ba64d6fb209e4e4e5a795306e44423c4e10b2a Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 13 Sep 2023 16:19:15 +0900 Subject: [PATCH 03/15] Make mypy happy --- pfio/v2/local.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index e6c9e50f..5cab765d 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -10,12 +10,12 @@ except ImportError: # IF PPE is not available, wrap with noop - def record_function(*args): + def record_function(*args): # type: ignore def wrapper(f): return f return wrapper - def record_iterable(tag, iter, *args): + def record_iterable(tag, iter, *args): # type: ignore yield from iter From a723f1bab646909e525406fea39bbcb2ad31b6d1 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 13 Sep 2023 16:22:13 +0900 Subject: [PATCH 04/15] Make flake8 happy --- pfio/v2/local.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index 5cab765d..4023235e 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -10,12 +10,12 @@ except ImportError: # IF PPE is not available, wrap with noop - def record_function(*args): # type: ignore + def record_function(*args): # type: ignore # NOQA def wrapper(f): return f return wrapper - def record_iterable(tag, iter, *args): # type: ignore + def record_iterable(tag, iter, *args): # type: ignore # NOQA yield from iter From dd0dd1c14b864800aa91c74aff0de62eb748e90b Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 13 Sep 2023 16:28:42 +0900 Subject: [PATCH 05/15] Make isort and mypy even happier --- pfio/v2/local.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index 4023235e..a126c311 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -5,17 +5,18 @@ from typing import Optional try: - from pytorch_pfn_extras.profiler import record_function, record_iterable + from pytorch_pfn_extras.profiler import ( # type: ignore # NOQA + record_function, record_iterable) except ImportError: # IF PPE is not available, wrap with noop - def record_function(*args): # type: ignore # NOQA + def record_function(*args): # type: ignore # NOQA def wrapper(f): return f return wrapper - def record_iterable(tag, iter, *args): # type: ignore # NOQA + def record_iterable(tag, iter, *args): # type: ignore # NOQA yield from iter From 338afd9c7c2857a6386ae305265ef794e11267e5 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 13 Sep 2023 16:38:15 +0900 Subject: [PATCH 06/15] Support all methods --- pfio/v2/local.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index a126c311..37f13fa2 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -160,23 +160,28 @@ def isdir(self, path: str): path = os.path.join(self.cwd, path) return os.path.isdir(path) + @record_function("local-mkdir") def mkdir(self, file_path: str, mode=0o777, *args, dir_fd=None): path = os.path.join(self.cwd, file_path) return os.mkdir(path, mode, *args, dir_fd=None) + @record_function("local-makedirs") def makedirs(self, file_path: str, mode=0o777, exist_ok=False): path = os.path.join(self.cwd, file_path) return os.makedirs(path, mode, exist_ok) + @record_function("local-exists") def exists(self, file_path: str): path = os.path.join(self.cwd, file_path) return os.path.exists(path) + @record_function("local-rename") def rename(self, src, dst): s = os.path.join(self.cwd, src) d = os.path.join(self.cwd, dst) return os.rename(s, d) + @record_function("local-remove") def remove(self, file_path: str, recursive=False): file_path = os.path.join(self.cwd, file_path) if recursive: @@ -186,6 +191,7 @@ def remove(self, file_path: str, recursive=False): return os.remove(file_path) + @record_function("local-glob") def glob(self, pattern: str): return [ str(item.relative_to(self.cwd)) From 41e6a3b8c602dffb6ce5386cae90c788297098c6 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Wed, 13 Sep 2023 16:53:45 +0900 Subject: [PATCH 07/15] Align tag naming rule with PPE (module.Klass:method) --- pfio/v2/local.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index 37f13fa2..8ba73e0f 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -99,7 +99,7 @@ def __repr__(self): }, ) - @record_function("local-open") + @record_function("pfio.v2.Local:open") def open(self, file_path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): @@ -111,7 +111,7 @@ def open(self, file_path, mode='r', def list(self, path: Optional[str] = '', recursive=False, detail=False): - for e in record_iterable("local-list", + for e in record_iterable("pfio.v2.Local:list", self._list(path, recursive, detail)): yield e @@ -151,7 +151,7 @@ def _recursive_list(self, prefix_end_index: int, path: str, yield from self._recursive_list(prefix_end_index, e.path, detail) - @record_function("local-stat") + @record_function("pfio.v2.Local:stat") def stat(self, path): path = os.path.join(self.cwd, path) return LocalFileStat(os.stat(path), path) @@ -160,28 +160,28 @@ def isdir(self, path: str): path = os.path.join(self.cwd, path) return os.path.isdir(path) - @record_function("local-mkdir") + @record_function("pfio.v2.Local:mkdir") def mkdir(self, file_path: str, mode=0o777, *args, dir_fd=None): path = os.path.join(self.cwd, file_path) return os.mkdir(path, mode, *args, dir_fd=None) - @record_function("local-makedirs") + @record_function("pfio.v2.Local:makedirs") def makedirs(self, file_path: str, mode=0o777, exist_ok=False): path = os.path.join(self.cwd, file_path) return os.makedirs(path, mode, exist_ok) - @record_function("local-exists") + @record_function("pfio.v2.Local:exists") def exists(self, file_path: str): path = os.path.join(self.cwd, file_path) return os.path.exists(path) - @record_function("local-rename") + @record_function("pfio.v2.Local:rename") def rename(self, src, dst): s = os.path.join(self.cwd, src) d = os.path.join(self.cwd, dst) return os.rename(s, d) - @record_function("local-remove") + @record_function("pfio.v2.Local:remove") def remove(self, file_path: str, recursive=False): file_path = os.path.join(self.cwd, file_path) if recursive: @@ -191,7 +191,7 @@ def remove(self, file_path: str, recursive=False): return os.remove(file_path) - @record_function("local-glob") + @record_function("pfio.v2.Local:glob") def glob(self, pattern: str): return [ str(item.relative_to(self.cwd)) From 36a6fcae1ff5f1961eabc77607b96a2a8986d658 Mon Sep 17 00:00:00 2001 From: KantaTamura Date: Wed, 21 Aug 2024 11:37:32 +0900 Subject: [PATCH 08/15] Support PPE tracing --- pfio/v2/local.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index 8ba73e0f..7fd8f3b0 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -5,18 +5,17 @@ from typing import Optional try: - from pytorch_pfn_extras.profiler import ( # type: ignore # NOQA - record_function, record_iterable) + from pytorch_pfn_extras.profiler import record_function, record_iterable except ImportError: # IF PPE is not available, wrap with noop - def record_function(*args): # type: ignore # NOQA + def record_function(tag, trace, *args): # type: ignore # NOQA def wrapper(f): return f return wrapper - def record_iterable(tag, iter, *args): # type: ignore # NOQA + def record_iterable(tag, iter, trace, *args): # type: ignore # NOQA yield from iter @@ -99,7 +98,7 @@ def __repr__(self): }, ) - @record_function("pfio.v2.Local:open") + @record_function("pfio.v2.Local:open", trace=True) def open(self, file_path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): @@ -112,7 +111,8 @@ def open(self, file_path, mode='r', def list(self, path: Optional[str] = '', recursive=False, detail=False): for e in record_iterable("pfio.v2.Local:list", - self._list(path, recursive, detail)): + self._list(path, recursive, detail), + trace=True): yield e def _list(self, path: Optional[str] = '', recursive=False, @@ -151,7 +151,7 @@ def _recursive_list(self, prefix_end_index: int, path: str, yield from self._recursive_list(prefix_end_index, e.path, detail) - @record_function("pfio.v2.Local:stat") + @record_function("pfio.v2.Local:stat", trace=True) def stat(self, path): path = os.path.join(self.cwd, path) return LocalFileStat(os.stat(path), path) @@ -160,28 +160,28 @@ def isdir(self, path: str): path = os.path.join(self.cwd, path) return os.path.isdir(path) - @record_function("pfio.v2.Local:mkdir") + @record_function("pfio.v2.Local:mkdir", trace=True) def mkdir(self, file_path: str, mode=0o777, *args, dir_fd=None): path = os.path.join(self.cwd, file_path) return os.mkdir(path, mode, *args, dir_fd=None) - @record_function("pfio.v2.Local:makedirs") + @record_function("pfio.v2.Local:makedirs", trace=True) def makedirs(self, file_path: str, mode=0o777, exist_ok=False): path = os.path.join(self.cwd, file_path) return os.makedirs(path, mode, exist_ok) - @record_function("pfio.v2.Local:exists") + @record_function("pfio.v2.Local:exists", trace=True) def exists(self, file_path: str): path = os.path.join(self.cwd, file_path) return os.path.exists(path) - @record_function("pfio.v2.Local:rename") + @record_function("pfio.v2.Local:rename", trace=True) def rename(self, src, dst): s = os.path.join(self.cwd, src) d = os.path.join(self.cwd, dst) return os.rename(s, d) - @record_function("pfio.v2.Local:remove") + @record_function("pfio.v2.Local:remove", trace=True) def remove(self, file_path: str, recursive=False): file_path = os.path.join(self.cwd, file_path) if recursive: @@ -191,7 +191,7 @@ def remove(self, file_path: str, recursive=False): return os.remove(file_path) - @record_function("pfio.v2.Local:glob") + @record_function("pfio.v2.Local:glob", trace=True) def glob(self, pattern: str): return [ str(item.relative_to(self.cwd)) From af772bf49371ce7874242c4a5a81d583c55ca2d7 Mon Sep 17 00:00:00 2001 From: KantaTamura Date: Wed, 21 Aug 2024 12:52:19 +0900 Subject: [PATCH 09/15] Local: Support io class method --- pfio/v2/local.py | 47 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index 7fd8f3b0..b5366947 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -5,11 +5,24 @@ from typing import Optional try: - from pytorch_pfn_extras.profiler import record_function, record_iterable + from pytorch_pfn_extras.profiler import record, record_function, record_iterable except ImportError: + class _DummyRecord: + def __init__(self): + pass + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + pass + # IF PPE is not available, wrap with noop + def record(tag, trace, *args): # type: ignore # NOQA + return _DummyRecord() + def record_function(tag, trace, *args): # type: ignore # NOQA def wrapper(f): return f @@ -22,6 +35,28 @@ def record_iterable(tag, iter, trace, *args): # type: ignore # NOQA from .fs import FS, FileStat, format_repr +class LocalProfileWrapper: + def __init__(self, fp): + self.fp = fp + + def __enter__(self): + self.fp.__enter__() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.fp.__exit__(exc_type, exc_value, traceback) + + def __getattr__(self, name): + attr = getattr(self.fp, name) + if callable(attr): + def wrapper(*args, **kwargs): + with record(f"pfio.v2.Local:{attr.__name__}", trace=True): + return attr(*args, **kwargs) + return wrapper + else: + return attr + + class LocalFileStat(FileStat): """Detailed information of a POSIX file @@ -104,9 +139,13 @@ def open(self, file_path, mode='r', newline=None, closefd=True, opener=None): path = os.path.join(self.cwd, file_path) - return io.open(path, mode, - buffering, encoding, errors, - newline, closefd, opener) + + fp = io.open(path, mode, + buffering, encoding, errors, + newline, closefd, opener) + + # Add ppe recorder to io class methods (e.g. read, write) + return LocalProfileWrapper(fp) def list(self, path: Optional[str] = '', recursive=False, detail=False): From 7cc4b58e65b00481d9691de5737079d52fd51648 Mon Sep 17 00:00:00 2001 From: KantaTamura Date: Wed, 21 Aug 2024 18:11:50 +0900 Subject: [PATCH 10/15] Add trace switching Can be specified in the `trace` argument when creating a Local() instance. --- pfio/v2/local.py | 84 ++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/pfio/v2/local.py b/pfio/v2/local.py index b5366947..81aca891 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -5,7 +5,7 @@ from typing import Optional try: - from pytorch_pfn_extras.profiler import record, record_function, record_iterable + from pytorch_pfn_extras.profiler import record, record_iterable except ImportError: @@ -23,11 +23,6 @@ def __exit__(self, exc_type, exc_value, traceback): def record(tag, trace, *args): # type: ignore # NOQA return _DummyRecord() - def record_function(tag, trace, *args): # type: ignore # NOQA - def wrapper(f): - return f - return wrapper - def record_iterable(tag, iter, trace, *args): # type: ignore # NOQA yield from iter @@ -95,9 +90,11 @@ def __init__(self, _stat, filename): class Local(FS): - def __init__(self, cwd=None, create=False, **_): + def __init__(self, cwd=None, trace=False, create=False, **_): super().__init__() + self.trace = trace + if cwd is None: self._cwd = '' else: @@ -133,25 +130,28 @@ def __repr__(self): }, ) - @record_function("pfio.v2.Local:open", trace=True) def open(self, file_path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): - path = os.path.join(self.cwd, file_path) + with record("pfio.v2.Local:open", trace=self.trace): + path = os.path.join(self.cwd, file_path) - fp = io.open(path, mode, - buffering, encoding, errors, - newline, closefd, opener) + fp = io.open(path, mode, + buffering, encoding, errors, + newline, closefd, opener) - # Add ppe recorder to io class methods (e.g. read, write) - return LocalProfileWrapper(fp) + # Add ppe recorder to io class methods (e.g. read, write) + if self.trace: + return LocalProfileWrapper(fp) + else: + return fp def list(self, path: Optional[str] = '', recursive=False, detail=False): for e in record_iterable("pfio.v2.Local:list", self._list(path, recursive, detail), - trace=True): + trace=self.trace): yield e def _list(self, path: Optional[str] = '', recursive=False, @@ -190,51 +190,51 @@ def _recursive_list(self, prefix_end_index: int, path: str, yield from self._recursive_list(prefix_end_index, e.path, detail) - @record_function("pfio.v2.Local:stat", trace=True) def stat(self, path): - path = os.path.join(self.cwd, path) - return LocalFileStat(os.stat(path), path) + with record("pfio.v2.Local:stat", trace=self.trace): + path = os.path.join(self.cwd, path) + return LocalFileStat(os.stat(path), path) def isdir(self, path: str): path = os.path.join(self.cwd, path) return os.path.isdir(path) - @record_function("pfio.v2.Local:mkdir", trace=True) def mkdir(self, file_path: str, mode=0o777, *args, dir_fd=None): - path = os.path.join(self.cwd, file_path) - return os.mkdir(path, mode, *args, dir_fd=None) + with record("pfio.v2.Local:mkdir", trace=self.trace): + path = os.path.join(self.cwd, file_path) + return os.mkdir(path, mode, *args, dir_fd=None) - @record_function("pfio.v2.Local:makedirs", trace=True) def makedirs(self, file_path: str, mode=0o777, exist_ok=False): - path = os.path.join(self.cwd, file_path) - return os.makedirs(path, mode, exist_ok) + with record("pfio.v2.Local:makedirs", trace=self.trace): + path = os.path.join(self.cwd, file_path) + return os.makedirs(path, mode, exist_ok) - @record_function("pfio.v2.Local:exists", trace=True) def exists(self, file_path: str): - path = os.path.join(self.cwd, file_path) - return os.path.exists(path) + with record("pfio.v2.Local:exists", trace=self.trace): + path = os.path.join(self.cwd, file_path) + return os.path.exists(path) - @record_function("pfio.v2.Local:rename", trace=True) def rename(self, src, dst): - s = os.path.join(self.cwd, src) - d = os.path.join(self.cwd, dst) - return os.rename(s, d) + with record("pfio.v2.Local:rename", trace=self.trace): + s = os.path.join(self.cwd, src) + d = os.path.join(self.cwd, dst) + return os.rename(s, d) - @record_function("pfio.v2.Local:remove", trace=True) def remove(self, file_path: str, recursive=False): - file_path = os.path.join(self.cwd, file_path) - if recursive: - return shutil.rmtree(file_path) - if os.path.isdir(file_path): - return os.rmdir(file_path) + with record("pfio.v2.Local:remove", trace=self.trace): + file_path = os.path.join(self.cwd, file_path) + if recursive: + return shutil.rmtree(file_path) + if os.path.isdir(file_path): + return os.rmdir(file_path) - return os.remove(file_path) + return os.remove(file_path) - @record_function("pfio.v2.Local:glob", trace=True) def glob(self, pattern: str): - return [ - str(item.relative_to(self.cwd)) - for item in pathlib.Path(self.cwd).glob(pattern)] + with record("pfio.v2.Local:glob", trace=self.trace): + return [ + str(item.relative_to(self.cwd)) + for item in pathlib.Path(self.cwd).glob(pattern)] def _canonical_name(self, file_path: str) -> str: return "file:/" + os.path.normpath(os.path.join(self.cwd, file_path)) From 0ffb9ae699e1a6d7c40714ff0d576103a524e829 Mon Sep 17 00:00:00 2001 From: KantaTamura Date: Thu, 22 Aug 2024 17:29:29 +0900 Subject: [PATCH 11/15] Split PPE methods --- pfio/v2/_profiler.py | 21 +++++++++++++++++++++ pfio/v2/local.py | 24 +----------------------- 2 files changed, 22 insertions(+), 23 deletions(-) create mode 100644 pfio/v2/_profiler.py diff --git a/pfio/v2/_profiler.py b/pfio/v2/_profiler.py new file mode 100644 index 00000000..6023e307 --- /dev/null +++ b/pfio/v2/_profiler.py @@ -0,0 +1,21 @@ +try: + from pytorch_pfn_extras.profiler import record, record_iterable + +except ImportError: + + class _DummyRecord: + def __init__(self): + pass + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + pass + + # IF PPE is not available, wrap with noop + def record(tag, trace, *args): # type: ignore # NOQA + return _DummyRecord() + + def record_iterable(tag, iter, trace, *args): # type: ignore # NOQA + yield from iter diff --git a/pfio/v2/local.py b/pfio/v2/local.py index 81aca891..09945b0b 100644 --- a/pfio/v2/local.py +++ b/pfio/v2/local.py @@ -4,29 +4,7 @@ import shutil from typing import Optional -try: - from pytorch_pfn_extras.profiler import record, record_iterable - -except ImportError: - - class _DummyRecord: - def __init__(self): - pass - - def __enter__(self): - pass - - def __exit__(self, exc_type, exc_value, traceback): - pass - - # IF PPE is not available, wrap with noop - def record(tag, trace, *args): # type: ignore # NOQA - return _DummyRecord() - - def record_iterable(tag, iter, trace, *args): # type: ignore # NOQA - yield from iter - - +from ._profiler import record, record_iterable from .fs import FS, FileStat, format_repr From 19561797a51413686081c64cc96920cb960a311d Mon Sep 17 00:00:00 2001 From: KantaTamura Date: Mon, 26 Aug 2024 09:08:54 +0900 Subject: [PATCH 12/15] mypy: ignore ppe --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 9771dec0..f9de3660 100644 --- a/mypy.ini +++ b/mypy.ini @@ -20,3 +20,6 @@ ignore_missing_imports = True [mypy-pyarrow.*] ignore_missing_imports = True + +[mypy-pytorch_pfn_extras.*] +ignore_missing_imports = True \ No newline at end of file From 62d4b0b03ad135a6331b922e16967398425b2c09 Mon Sep 17 00:00:00 2001 From: KantaTamura Date: Wed, 21 Aug 2024 18:12:18 +0900 Subject: [PATCH 13/15] add trace test --- tests/v2_tests/test_local.py | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/v2_tests/test_local.py b/tests/v2_tests/test_local.py index be83663b..957cf6d0 100644 --- a/tests/v2_tests/test_local.py +++ b/tests/v2_tests/test_local.py @@ -1,4 +1,5 @@ import io +import json import os import pickle import tempfile @@ -292,5 +293,48 @@ def test_from_url_create_option(self): assert os.path.exists(path) and os.path.isdir(path) +def test_local_rw_profiling(): + ppe = pytest.importorskip("pytorch_pfn_extras") + + with Local(trace=True) as fs: + ppe.profiler.clear_tracer() + + with fs.open('foo.txt', 'w') as fp: + fp.write('bar') + + dict = ppe.profiler.get_tracer().state_dict() + keys = [event["name"] for event in json.loads(dict['_event_list'])] + + assert "pfio.v2.Local:open" in keys + assert "pfio.v2.Local:write" in keys + + with Local(trace=True) as fs: + ppe.profiler.clear_tracer() + + with fs.open('foo.txt', 'r') as fp: + tmp = fp.read() + assert tmp == 'bar' + + dict = ppe.profiler.get_tracer().state_dict() + keys = [event["name"] for event in json.loads(dict['_event_list'])] + + assert "pfio.v2.Local:open" in keys + assert "pfio.v2.Local:read" in keys + + # no profile case + with Local(trace=False) as fs: + ppe.profiler.clear_tracer() + + with fs.open('foo.txt', 'r') as fp: + tmp = fp.read() + assert tmp == 'bar' + + dict = ppe.profiler.get_tracer().state_dict() + keys = [event["name"] for event in json.loads(dict['_event_list'])] + + assert "pfio.v2.Local:open" not in keys + assert "pfio.v2.Local:read" not in keys + + if __name__ == '__main__': unittest.main() From 033d4202267f6ab21b3a60a680db6206183627d2 Mon Sep 17 00:00:00 2001 From: KantaTamura Date: Fri, 23 Aug 2024 14:36:09 +0900 Subject: [PATCH 14/15] setup: add trace context --- setup.py | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9395211b..62bbe81a 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ 'doc': ['sphinx', 'sphinx_rtd_theme'], 'bench': ['numpy>=1.19.5', 'torch>=1.9.0', 'Pillow<=8.2.0'], 'hdfs': ['pyarrow>=6.0.0'], + 'trace': ['pytorch-pfn-extras'], }, # When updating install requires, docs/requirements.txt should be updated too install_requires=['boto3', 'deprecation', 'urllib3'], diff --git a/tox.ini b/tox.ini index 08c4b21d..0ec44d6c 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = py38,py39,py310,py311,py312,doc [testenv] -deps = .[test] +deps = .[test, trace] skipsdist = True commands = flake8 pfio tests From 19f5baec594de49866732f6b90d5dbb544f235d6 Mon Sep 17 00:00:00 2001 From: KantaTamura Date: Mon, 26 Aug 2024 15:19:12 +0900 Subject: [PATCH 15/15] doc: add output tracer results tips --- docs/source/tips.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/source/tips.rst b/docs/source/tips.rst index 14f9b079..4bf2efda 100644 --- a/docs/source/tips.rst +++ b/docs/source/tips.rst @@ -17,3 +17,35 @@ A. Default buffer size of file objects from ``pfio.v2.S3`` is 16MB, with open_url('s3://bucket/very-large-file', 'wb') as dst: with open('very-large-local-file', 'rb') as src: shutil.copyfileobj(src, dst, length=16*1024*1024) + +Q. How to output tracer results (readable by Chrome Tracing). +============================================================= + +A. As shown in the sample program below, + by executing `initialize_writer()` and `flush()` after tracing, + JSON that can be read by Chrome tracing can be output. + +.. code-block:: python + + import json + import pytorch_pfn_extras as ppe + + from pfio.v2 import Local, Path, from_url + + tracer = ppe.profiler.get_tracer() + + with Local(trace=True) as fs: + for f in fs.list(): + if fs.isdir(f.strip()): + dir += 1 + continue + + fil += 1 + len(fs.open(f).read()) + + w = ppe.writing.SimpleWriter(out_dir="") + + # output '[' + tracer.initialize_writer("trace.json", w) + # output json dump + tracer.flush("trace.json", w) \ No newline at end of file