fixed lahman.py; added test_lahman.py

docs/lahman.md

@@ -1,10 +1,10 @@
 # Lahman Data Acquisition Functions
 
-Pull data from [Sean Lahman's database](http://www.seanlahman.com/baseball-archive/statistics/), also hosted by [Chadwick Bureau on GitHub](https://github.com/chadwickbureau/baseballdatabank) -- our new source -- using the following functions:
+Pulls data linked from [Sean Lahman's database](http://seanlahman.com/) now hosted on dropbox -- using the following functions:
 
 ```python
 from pybaseball.lahman import *
-download_lahman() #download the entire lahman database to your current working directory
+download_lahman()
 
 # a table of all player biographical info and ids
 people = people()
@@ -81,7 +81,7 @@ schools = schools()
 series_post = series_post()
 
 # data on teams by year: record, division, stadium, attendance, etc
-teams = teams()
+teams = teams_core()
 
 # current and historical franchises, whether they're still active, and their ids
 teams_franchises = teams_franchises()

pybaseball/__init__.py

@@ -79,7 +79,6 @@
 from .lahman import schools
 from .lahman import series_post
 from .lahman import teams_core
-from .lahman import teams_upstream
 from .lahman import teams_franchises
 from .lahman import teams_half
 from .lahman import download_lahman

pybaseball/lahman.py

@@ -1,136 +1,161 @@
+from datetime import timedelta
 from io import BytesIO
+from os import makedirs
 from os import path
-from typing import Optional
-from zipfile import ZipFile
 
+from bs4 import BeautifulSoup
 import pandas as pd
+from pathlib import Path
+from py7zr import SevenZipFile
 import requests
+from requests_cache import CachedSession
 
 from . import cache
 
-url = "https://github.com/chadwickbureau/baseballdatabank/archive/master.zip"
-base_string = "baseballdatabank-master"
-
-_handle = None
-
-def get_lahman_zip() -> Optional[ZipFile]:
- # Retrieve the Lahman database zip file, returns None if file already exists in cwd.
- # If we already have the zip file, keep re-using that.
- # Making this a function since everything else will be re-using these lines
- global _handle
- if path.exists(path.join(cache.config.cache_directory, base_string)):
- _handle = None
- elif not _handle:
- s = requests.get(url, stream=True)
- _handle = ZipFile(BytesIO(s.content))
- return _handle
-
-def download_lahman():
- # download entire lahman db to present working directory
- z = get_lahman_zip()
- if z is not None:
- z.extractall(cache.config.cache_directory)
- z = get_lahman_zip()
- # this way we'll now start using the extracted zip directory
- # instead of the session ZipFile object
-
-def _get_file(tablename: str, quotechar: str = "'") -> pd.DataFrame:
- z = get_lahman_zip()
- f = f'{base_string}/{tablename}'
+# NB: response will be cached for 30 days unless force is True
+def _get_response(force:bool=False) -> requests.Response:
+ session = _get_session()
+ response = session.get("http://seanlahman.com", refresh=force)
+ return response
+
+# For example, "https://www.dropbox.com/scl/fi/hy0sxw6gaai7ghemrshi8/lahman_1871-2023_csv.7z?rlkey=edw1u63zzxg48gvpcmr3qpnhz&dl=1"
+def _get_download_url(force:bool=False) -> str:
+ response = _get_response(force)
+ soup = BeautifulSoup(response.content, "html.parser")
+
+ anchor = soup.find("a", string="Comma-delimited version")
+ url = anchor["href"].replace("dl=0", "dl=1")
+
+ return url
+
+def _get_cache_dir() -> str:
+ return f"{cache.config.cache_directory}/lahman"
+
+def _get_session() -> CachedSession:
+ return CachedSession(_get_cache_dir(), expire_after=timedelta(days=30))
+
+def _get_base_string() -> str:
+ url = _get_download_url()
+ path = Path(url)
+
+ return path.stem
+
+def _get_file_path(filename: str = "") -> str:
+ base_string = _get_base_string()
+ return path.join(_get_cache_dir(), base_string, filename)
+
+def _get_table(filename: str,
+ quotechar: str = "'",
+ encoding=None,
+ on_bad_lines="error") -> pd.DataFrame:
+ filepath = _get_file_path(filename)
  data = pd.read_csv(
- f"{path.join(cache.config.cache_directory, f)}" if z is None else z.open(f),
+ filepath,
  header=0,
- sep=',',
- quotechar=quotechar
+ sep=",",
+ quotechar=quotechar,
+ encoding=encoding,
+ on_bad_lines=on_bad_lines,
  )
  return data
 
+# Return whether download happened (True) or if cache used (False)
+def download_lahman(force: bool = False) -> bool:
+ if force or not path.exists(_get_file_path()):
+ cache_dir = _get_cache_dir()
+ base_string = _get_base_string()
+ makedirs(f"{cache_dir}/{base_string}", exist_ok=True)
 
-# do this for every table in the lahman db so they can exist as separate functions
-def parks() -> pd.DataFrame:
- return _get_file('core/Parks.csv')
+ url = _get_download_url(force)
+ stream = requests.get(url, stream=True)
+ with SevenZipFile(BytesIO(stream.content)) as zip:
+ zip.extractall(cache_dir)
+ return True
+ return False
 
+# do this for every table in the lahman db so they can exist as separate functions
 def all_star_full() -> pd.DataFrame:
- return _get_file("core/AllstarFull.csv")
+ return _get_table("AllstarFull.csv")
 
 def appearances() -> pd.DataFrame:
- return _get_file("core/Appearances.csv")
+ return _get_table("Appearances.csv")
 
 def awards_managers() -> pd.DataFrame:
- return _get_file("contrib/AwardsManagers.csv")
+ return _get_table("AwardsManagers.csv")
 
 def awards_players() -> pd.DataFrame:
- return _get_file("contrib/AwardsPlayers.csv")
+ return _get_table("AwardsPlayers.csv")
 
 def awards_share_managers() -> pd.DataFrame:
- return _get_file("contrib/AwardsShareManagers.csv")
+ return _get_table("AwardsShareManagers.csv")
 
 def awards_share_players() -> pd.DataFrame:
- return _get_file("contrib/AwardsSharePlayers.csv")
+ return _get_table("AwardsSharePlayers.csv")
 
 def batting() -> pd.DataFrame:
- return _get_file("core/Batting.csv")
+ return _get_table("Batting.csv")
 
 def batting_post() -> pd.DataFrame:
- return _get_file("core/BattingPost.csv")
+ return _get_table("BattingPost.csv")
 
 def college_playing() -> pd.DataFrame:
- return _get_file("contrib/CollegePlaying.csv")
+ return _get_table("CollegePlaying.csv")
 
 def fielding() -> pd.DataFrame:
- return _get_file("core/Fielding.csv")
+ return _get_table("Fielding.csv")
 
 def fielding_of() -> pd.DataFrame:
- return _get_file("core/FieldingOF.csv")
+ return _get_table("FieldingOF.csv")
 
 def fielding_of_split() -> pd.DataFrame:
- return _get_file("core/FieldingOFsplit.csv")
+ return _get_table("FieldingOFsplit.csv")
 
 def fielding_post() -> pd.DataFrame:
- return _get_file("core/FieldingPost.csv")
+ return _get_table("FieldingPost.csv")
 
 def hall_of_fame() -> pd.DataFrame:
- return _get_file("contrib/HallOfFame.csv")
+ return _get_table("HallOfFame.csv")
 
 def home_games() -> pd.DataFrame:
- return _get_file("core/HomeGames.csv")
+ return _get_table("HomeGames.csv")
 
 def managers() -> pd.DataFrame:
- return _get_file("core/Managers.csv")
+ return _get_table("Managers.csv")
 
 def managers_half() -> pd.DataFrame:
- return _get_file("core/ManagersHalf.csv")
+ return _get_table("ManagersHalf.csv")
 
 def master() -> pd.DataFrame:
  # Alias for people -- the new name for master
  return people()
 
+def parks() -> pd.DataFrame:
+ return _get_table("Parks.csv", encoding="unicode_escape")
+
 def people() -> pd.DataFrame:
- return _get_file("core/People.csv")
+ return _get_table("People.csv", encoding="unicode_escape")
 
 def pitching() -> pd.DataFrame:
- return _get_file("core/Pitching.csv")
+ return _get_table("Pitching.csv")
 
 def pitching_post() -> pd.DataFrame:
- return _get_file("core/PitchingPost.csv")
+ return _get_table("PitchingPost.csv")
 
 def salaries() -> pd.DataFrame:
- return _get_file("contrib/Salaries.csv")
+ return _get_table("Salaries.csv")
 
 def schools() -> pd.DataFrame:
- return _get_file("contrib/Schools.csv", quotechar='"') # different here bc of doublequotes used in some school names
+ # NB: one line is bad; "brklyncuny" should use double quotes, but doesn't
+ return _get_table("Schools.csv", quotechar='"', on_bad_lines="skip")
 
 def series_post() -> pd.DataFrame:
- return _get_file("core/SeriesPost.csv")
+ return _get_table("SeriesPost.csv")
 
 def teams_core() -> pd.DataFrame:
- return _get_file("core/Teams.csv")
-
-def teams_upstream() -> pd.DataFrame:
- return _get_file("upstream/Teams.csv") # manually maintained file
+ return _get_table("Teams.csv")
 
 def teams_franchises() -> pd.DataFrame:
- return _get_file("core/TeamsFranchises.csv")
+ return _get_table("TeamsFranchises.csv")
 
 def teams_half() -> pd.DataFrame:
- return _get_file("core/TeamsHalf.csv")
+ return _get_table("TeamsHalf.csv")

setup.py

@@ -92,6 +92,8 @@
  'matplotlib>=2.0.0',
  'tqdm>=4.50.0',
  'attrs>=20.3.0',
+ 'py7zr>=0.22.0',
+ 'requests_cache>=1.2.1',
  ],
 
  # List additional groups of dependencies here (e.g. development

tests/pybaseball/conftest.py

@@ -131,83 +131,70 @@ def get_contents(filename: str) -> str:
 
  return get_contents
 
-
 @pytest.fixture()
-def get_data_file_dataframe(data_dir: str) -> GetDataFrameCallable:
+def get_data_file_bytes(data_dir: str) -> Callable[[str], bytes]:
  """
- Returns a function that will allow getting a dataframe from a csv file in the tests data directory easily
+ Returns a function that will allow getting the contents of a file in the tests data directory easily
  """
- def get_dataframe(filename: str, parse_dates: _ParseDates = False) -> pd.DataFrame:
+ def get_bytes(filename: str) -> bytes:
  """
- Get the DatFrame representation of the contents of a csv file in the tests data directory
+ Get the byte contents of a file in the tests data directory
 
 
  ARGUMENTS:
- filename : str : the name of the file within the tests data directory to load into a DataFrame
+ filename : str : the name of the file within the tests data directory to get the contents of
  """
- return pd.read_csv(os.path.join(data_dir, filename), index_col=0, parse_dates=parse_dates).reset_index(drop=True).convert_dtypes(convert_string=False)
-
- return get_dataframe
+ with open(os.path.join(data_dir, filename), 'rb') as _file:
+ data = _file.read()
+  return data
 
+ return get_bytes
 
 @pytest.fixture()
-def response_get_monkeypatch(monkeypatch: MonkeyPatch) -> Callable:
+def get_data_file_dataframe(data_dir: str) -> GetDataFrameCallable:
  """
- Returns a function that will monkeypatch the requests.get function call to return expected data 
+ Returns a function that will allow getting a dataframe from a csv file in the tests data directory easily
  """
- def setup(result: Union[str, bytes], expected_url: Optional[str] = None) -> None:
+ def get_dataframe(filename: str, parse_dates: _ParseDates = False) -> pd.DataFrame:
  """
- Get the DatFrame representation of the contents of a csv file in the tests data directory
+  Get the DatFrame representation of the contents of a csv file in the tests data directory
 
 
  ARGUMENTS:
- result : str : the payload to return in the contents of the request.get call
- expected_url : str (optional) : an expected_url to test the get call against
- to ensure the correct endpoint is hit
+ filename : str : the name of the file within the tests data directory to load into a DataFrame
  """
- def _monkeypatch(url: str, params: Optional[Dict] = None, timeout: Optional[int] = None) -> object:
- final_url = url
-
- if params:
- query_params = urllib.parse.urlencode(params, safe=',')
- final_url = f"{final_url}?{query_params}"
-
- if expected_url is not None:
- # These prints are desired as these are long and get cut off in the test outpute.
- # These will only render on failed tests, so only when you would want to see them anyway.
- print("expected", expected_url)
- print("received", final_url)
- assert final_url.endswith(expected_url)
-
- class DummyResponse:
- def __init__(self, content: Union[str, bytes]):
- self.content = content
- self.text = content
- self.status_code = 200
- self.url = final_url
+ return pd.read_csv(os.path.join(data_dir, filename), index_col=0, parse_dates=parse_dates).reset_index(drop=True).convert_dtypes(convert_string=False)
 
-  return DummyResponse(result)
+ return get_dataframe
 
- monkeypatch.setattr(requests, 'get', _monkeypatch)
 
- return setup
+@pytest.fixture()
+def response_get_monkeypatch(monkeypatch: MonkeyPatch) -> Callable:
+ return _get_monkeypatch(monkeypatch, requests)
 
 @pytest.fixture()
 def bref_get_monkeypatch(monkeypatch: MonkeyPatch) -> Callable:
+ return _get_monkeypatch(monkeypatch, BRefSession())
+
+@pytest.fixture()
+def target_get_monkeypatch(monkeypatch: MonkeyPatch, target: str | object) -> Callable:
+ return _get_monkeypatch(monkeypatch, target)
+
+def _get_monkeypatch(monkeypatch: MonkeyPatch, target: str | object) -> Callable:
  """
- Returns a function that will monkeypatch the BRefSession.get function call to return expected data 
+ Returns a function that will monkeypatch the input target's get() function call to return supplied result.
  """
  def setup(result: Union[str, bytes], expected_url: Optional[str] = None) -> None:
  """
- Get the DatFrame representation of the contents of a csv file in the tests data directory
+  Get the result when calling the get() function
 
 
  ARGUMENTS:
- result : str   : the payload to return in the contents of the request.get call
+ result : str | bytes : the payload to return in the contents of the request.get call
  expected_url : str (optional) : an expected_url to test the get call against
  to ensure the correct endpoint is hit
  """
- def _monkeypatch(url: str, params: Optional[Dict] = None, timeout: Optional[int] = None) -> object:
+ def _monkeypatch(url: str, params: Optional[Dict] = None, stream = False, timeout: Optional[int] = None) -> object:
  final_url = url
 
  if params:
@@ -230,6 +217,6 @@ def __init__(self, content: Union[str, bytes]):
 
  return DummyResponse(result)
 
- monkeypatch.setattr(BRefSession(), 'get', _monkeypatch)
+ monkeypatch.setattr(target, 'get', _monkeypatch)
 
  return setup