00_main.R

source("0_clean_lm.R")

WP = if(!exists("WP")) TRUE else WP ### default to expected points models

########################
####### LOAD DATA ######
########################

### full dataset
data_full_0 <- read_csv("data8a.csv") 

### for some reason, the pointspread from nflFastR is flipped, so flip it back!
data_full_0 <- data_full_0 %>% mutate(posteam_spread = -posteam_spread)

### standardizing the columns as-is 
data_full_0A = 
  data_full_0 %>% 
  mutate(
    posteam_coach = ifelse(home == 1, home_coach, away_coach),
    ### for WP models
    scoreTimeRatio = compute_scoreTimeRatio(score_differential, game_seconds_remaining),
    utm = as.numeric(half_seconds_remaining <= 120),
    not_utm = 1-utm,
    gtg = (yardline_100 <= 10),
    half_sec_rem_std = 1-(half_seconds_remaining/1800), ### in [0,1] where 0 is start and 1 is end.
    ### features for Lock and Nettleton WP
    AdjustedScore_LN = score_differential / sqrt(game_seconds_remaining + 1),
    total_score = posteam_score + defteam_score,
    ### market-derived offensive and defensive qualities
    market_OQOT_minus_DQDT = (total_line + -posteam_spread) / 2,
    market_OQDT_minus_DQOT = (total_line - -posteam_spread) / 2,
    market_OQOT_minus_DQDT_std = std(market_OQOT_minus_DQDT),
    market_OQDT_minus_DQOT_std = std(market_OQDT_minus_DQOT),
  ) %>%
  ### fix era numeric column
  mutate(era_A = case_when(era0==1~0, era1==1~1, era2==1~2, era3==1~3, era4==1~4, TRUE~NA_real_)) %>%
  mutate(era_B = case_when(era0==1~0, era1==1~1, era2==1~2, era3==1|era4==1~3, TRUE~NA_real_)) %>%
  ### drive_id, the unique index of all drives in the dataset
  left_join(
    data_full_0 %>% distinct(game_id, drive) %>% mutate(drive_id = 1:n())
  )

### look at the standardizations
mean(data_full_0A$market_OQOT_minus_DQDT)
sd(data_full_0A$market_OQOT_minus_DQDT)
mean(data_full_0A$market_OQDT_minus_DQOT)
sd(data_full_0A$market_OQDT_minus_DQOT)

# ### check
# temp = data_full_0 %>% distinct(
#   game_id, total_line, posteam, defteam,
#   posteam_spread, market_OQOT_minus_DQDT, market_OQDT_minus_DQOT,
#   market_OQOT_minus_DQDT_std, market_OQDT_minus_DQOT_std,
# ) 

###
data_full_AA = data_full_0A %>%
  ### remember, we used era0 and era1 to construct WP0 and EPA0, so remove them
  filter(era0 != 1 & era1 != 1) %>%
  ### this analysis was originally completed using data <= 2021
  filter(season <= 2021) 

### datasets for WP
data_full_WP = data_full_AA
data_full_110_WP = data_full_WP %>% filter(down == 1 & (ydstogo == 10 | yardline_100 <= 10) )

data_full = data_full_WP
data_full_110 = data_full_110_WP

# ### datasets for EP
# data_full_A = data_full_AA %>% drop_na(pass_or_rush)
# omega = 0.15
# data_full_EP = data_full_A %>% filter(omega <= wp0 & wp0 <= 1-omega)
# data_full_110_EP = data_full_EP %>% filter(down == 1 & (ydstogo == 10 | yardline_100 <= 10) )
# 
# if (WP) {
#   data_full = data_full_WP
#   data_full_110 = data_full_110_WP
# } else { # EP
#   data_full = data_full_EP
#   data_full_110 = data_full_110_EP
# }

print(paste(
  "our primary dataset of all non-tied games from", min(data_full_WP$season), "to", max(data_full_WP$season),
  "consists of", length(unique(data_full_WP$game_id)), "games,",
   nrow(data_full_WP), "plays, and", nrow(data_full_110_WP), "first-down plays.",
  "There are", round(nrow(data_full_110_WP)/length(unique(data_full_WP$game_id))), 
  "first-down plays per game on average."
))

##########################################
### Get FG, PUNT, and CONVERT Datasets ###
##########################################

fg_df = 
  data_full_WP %>%
  filter(field_goal_attempt == 1) %>%
  select(row_idx, field_goal_attempt, fg_made, yardline_100, posteam, season, week, kicker_name, kq) %>%
  filter(yardline_100 <= 50) 
tail(fg_df)

punt_df = 
  data_full_WP %>%
  filter(punt_attempt == 1) %>%
  select(row_idx, yardline_100, next_ydl, posteam, season, week, punter_name, pq) %>%
  filter(yardline_100 >= 30)
tail(punt_df)

go_df = 
  data_full_WP %>%
  filter(down == 3 | down == 4) %>%
  drop_na(pass_or_rush) %>%
  select(row_idx, season, posteam_spread, market_OQDT_minus_DQOT, market_OQOT_minus_DQDT_std,
         yardline_100, down, pass_or_rush, ydstogo, yards_gained) %>%
  mutate(convert = as.numeric(yards_gained >= ydstogo))
go_df

go_df_ = 
  data_full_WP %>%
  filter(down == 3 | down == 4) %>%
  drop_na(pass_or_rush) %>%
  mutate(convert = as.numeric(yards_gained >= ydstogo))
go_df_

#### dataframe of all fourth down decisions
all_fourth_downs = 
  data_full_WP %>%
  filter(down==4) %>%
  mutate(
  decision_actual = case_when(
    !is.na(kicker_player_name) ~ "FG",
    !is.na(punter_player_name) ~ "Punt",
    TRUE ~ "Go"
  )) %>%
  mutate(
    label_decision_actual = case_when(
      decision_actual == "Go" ~ 0,
      decision_actual == "FG" ~ 1,
      decision_actual == "Punt" ~ 2
    )
  )
all_fourth_downs$fgp = as.numeric(all_fourth_downs$decision_actual == "FG")
all_fourth_downs$puntp = as.numeric(all_fourth_downs$decision_actual == "Punt")
tail(
  all_fourth_downs %>% select(
    row_idx, posteam, season, posteam_coach, decision_actual,label_decision_actual,
    kicker_player_name, punter_player_name,
    yardline_100, ydstogo, down, score_differential, posteam_spread, game_seconds_remaining,
    # fgp, puntp, kq, pq,
  )
)

################################################################################

#### dataframe of ALL fourth down decisions
ALL_fourth_downs = 
  data_full_0A %>%
  filter(down==4) %>%
  mutate(
    decision_actual = case_when(
      !is.na(kicker_player_name) ~ "FG",
      !is.na(punter_player_name) ~ "Punt",
      TRUE ~ "Go"
    )) %>%
  mutate(
    label_decision_actual = case_when(
      decision_actual == "Go" ~ 0,
      decision_actual == "FG" ~ 1,
      decision_actual == "Punt" ~ 2
    )
  )
ALL_fourth_downs$fgp = as.numeric(ALL_fourth_downs$decision_actual == "FG")
ALL_fourth_downs$puntp = as.numeric(ALL_fourth_downs$decision_actual == "Punt")
tail(
  ALL_fourth_downs %>% select(
    row_idx, posteam, season, posteam_coach, decision_actual,label_decision_actual,
    kicker_player_name, punter_player_name,
    yardline_100, ydstogo, down, score_differential, posteam_spread, game_seconds_remaining,
    fgp, puntp, 
    # kq, pq,
  )
)


# ### data for Adi's opportunity week class
# data_full_WP
# adi1 = 
#   data_full_WP %>%
#   filter(season %in% c(2020, 2021)) %>%
#   select(
#     game_id, season, game_seconds_remaining,posteam,defteam, home_team,
#     yardline_100, ydstogo, down, score_differential, posteam_score, defteam_score,
#     posteam_timeouts_remaining, defteam_timeouts_remaining,
#     qbq_ot_0_sum, oq_rot_0_total_sum, dq_dt_0_againstRun_sum, dq_dt_0_againstPass_sum,
#     qbq_dt_0_sum, oq_rdt_0_sum, dq_ot_0_againstRun_sum, dq_ot_0_againstPass_sum,
#     label_win, 
#   )
# dim(adi1)
# adi1
# write_csv(adi1, "data_NFL_wp_demo.csv")

# ### data for summer lab 2024
# write_csv(
#   fg_df %>% select(-c(row_idx, field_goal_attempt)) %>%
#     filter(season >= 2010) %>%
#     rename(ydl = yardline_100, kicker = kicker_name, kq = kq_0_sum_std)
#   , "data_field_goals_kq.csv"
# )
# write_csv(
#   fg_df %>% select(-c(row_idx, field_goal_attempt, kq_0_sum_std)) %>%
#     filter(season >= 2010) %>%
#     rename(ydl = yardline_100, kicker = kicker_name)
#   , "data_field_goals.csv"
# )
# write_csv(
#   punt_df %>% 
#     select(-c(row_idx, week, posteam)) %>% 
#     filter(season >= 2015) %>%
#     rename(ydl = yardline_100, punter = punter_name, pq = pq_0_sum_std)
#   , "data_punts.csv"
# )
# 
# df_lab_EP = 
#   data_full %>%
#   select(
#     game_id,season,play_id,pts_next_score,label,
#     yardline_100,down,ydstogo,half,half_seconds_remaining,
#     posteam_timeouts_remaining,defteam_timeouts_remaining,
#     posteam_spread
#   ) %>%
#   filter(season %in% 2019:2024)
# df_lab_EP
# write_csv(df_lab_EP, "data_EP.csv")
# df_lab_WP = 
#   # data_full_110_WP
#   data_full_WP %>%
#   select(
#     game_id, season, label_win,
#     score_differential, game_seconds_remaining, posteam_spread,
#     posteam_timeouts_remaining,defteam_timeouts_remaining,
#     yardline_100,
#     down,ydstogo,
#   ) %>%
#   filter(season %in% 2016:2024)
# df_lab_WP
# write_csv(df_lab_WP, "data_WP.csv")