-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
use custom file-system pickle based cache
- Loading branch information
mhewedy
committed
May 24, 2024
1 parent
a2625d9
commit e714bbe
Showing
3 changed files
with
21 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,28 @@ | ||
import logging | ||
|
||
import diskcache | ||
import os | ||
import pickle | ||
|
||
import util | ||
|
||
cache = diskcache.Cache(util.get_root_fs() + '/cache') | ||
root_fs_cache = util.get_root_fs() + '/cache' | ||
|
||
|
||
def _cache_path(namespace, key): | ||
cache_namespace = root_fs_cache + '/' + namespace | ||
os.makedirs(cache_namespace, exist_ok=True) | ||
return os.path.join(cache_namespace, f'{key}.pkl') | ||
|
||
|
||
def put(namespace, key: str, value): | ||
cache.set(f'{namespace}:{key.strip().lower()}', value) | ||
def get(namespace, key): | ||
try: | ||
with open(_cache_path(namespace, f'{key.strip().lower()}'), 'rb') as f: | ||
value = pickle.load(f) | ||
logging.info(f'cache hit for {key} in namespace: {namespace}, with value: {value}') | ||
return value | ||
except FileNotFoundError: | ||
return None | ||
|
||
|
||
def get(namespace, key: str): | ||
result = cache.get(f'{namespace}:{key.strip().lower()}') | ||
if result is not None: | ||
logging.info(f'cache hit for {namespace}:{key} -> {result}') | ||
return result | ||
def put(namespace, key, value): | ||
with open(_cache_path(namespace, f'{key.strip().lower()}'), 'wb') as f: | ||
pickle.dump(value, f) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -38,4 +38,3 @@ typing_extensions==4.11.0 | |
tzlocal==5.2 | ||
urllib3==2.2.1 | ||
yarl==1.9.4 | ||
diskcache==5.6.3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters