From edcb718c9e7075576777ea4d66e5088c88057cf1 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Sun, 21 Jun 2026 19:46:32 +1000 Subject: [PATCH 1/3] [hansen_singleton] replace pandas-datareader with direct FRED/Fama-French fetch pandas-datareader 0.10.0 (last released 2021, unmaintained) breaks at import under pandas 3.0 -- it relies on pandas' private deprecate_kwarg, whose signature changed -- so hansen_singleton_1982 and hansen_singleton_1983 fail to execute under anaconda 2026.06 (see #923). There is no pandas-3.0-compatible pandas-datareader release to pin to. Replace the two web.DataReader calls with small direct downloads that use only the standard library + pandas: - FRED: pd.read_csv from the fredgraph.csv endpoint - Fama-French: parse the F-F_Research_Data_Factors zip from the Ken French data library Since no extra package is needed, the in-notebook `!pip install pandas-datareader` cell and the now-dead date_parser warnings filter are removed too. Verified the new fetch returns byte-identical FRED and Fama-French data to the old pandas-datareader path on pandas 2.3.3, and that the full data construction runs clean with FutureWarning/DeprecationWarning promoted to errors (i.e. pandas-3.0 safe). Closes #924 Co-Authored-By: Claude Opus 4.8 --- lectures/hansen_singleton_1982.md | 75 ++++++++++++++++++++++--------- lectures/hansen_singleton_1983.md | 75 ++++++++++++++++++++++--------- 2 files changed, 110 insertions(+), 40 deletions(-) diff --git a/lectures/hansen_singleton_1982.md b/lectures/hansen_singleton_1982.md index 560cd5c4b..34fd39d4b 100644 --- a/lectures/hansen_singleton_1982.md +++ b/lectures/hansen_singleton_1982.md @@ -51,31 +51,20 @@ Though maximum likelihood estimators (such as the MLE in {doc}`hansen_singleton_ Relative to the full paper, we only estimate one return at a time (value-weighted stock returns), using only monthly nondurable consumption (`ND`), and omitting their maximum-likelihood comparison (Table II) and multi-return systems (Table III). -In addition to what comes with Anaconda, this lecture requires `pandas-datareader` - ```{code-cell} ipython3 -:tags: [hide-output] - -!pip install pandas-datareader -``` - -```{code-cell} ipython3 -import warnings +import io +import urllib.request +import zipfile import matplotlib.pyplot as plt import numpy as np import pandas as pd from IPython.display import Latex from numba import njit -from pandas_datareader import data as web from scipy import stats from scipy.optimize import minimize from statsmodels.sandbox.regression import gmm from statsmodels.tsa.stattools import acf - -warnings.filterwarnings( - "ignore", message=".*date_parser.*", category=FutureWarning -) ``` We also define a helper to display DataFrames as LaTeX arrays in the hidden cell below @@ -1000,7 +989,7 @@ Because the Ken French return is not identical to the original CRSP NYSE value-w Both this lecture and the companion lecture {doc}`hansen_singleton_1983` use the same data construction. -The hidden cell below pulls the relevant FRED series, constructs per capita real consumption, and joins with Ken French market returns via `pandas-datareader`. +The hidden cell below pulls the relevant FRED series, constructs per capita real consumption, and joins with Ken French market returns. The data are downloaded directly from the [FRED](https://fred.stlouisfed.org/) and [Ken French](https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/data_library.html) data libraries. ```{code-cell} ipython3 :tags: [hide-cell] @@ -1018,6 +1007,55 @@ def to_month_end(index): return pd.PeriodIndex(pd.DatetimeIndex(index), freq="M").to_timestamp("M") +def read_fred(codes, start, end): + """ + Download FRED series as a date-indexed DataFrame whose columns are the + requested FRED codes (replaces pandas-datareader's "fred" reader). + """ + base = "https://fred.stlouisfed.org/graph/fredgraph.csv" + columns = [] + for code in codes: + url = f"{base}?id={code}&cosd={start:%Y-%m-%d}&coed={end:%Y-%m-%d}" + columns.append( + pd.read_csv(url, index_col=0, parse_dates=True, na_values=".")) + fred = pd.concat(columns, axis=1).astype("float64") + fred.index.name = "DATE" + return fred + + +def read_famafrench_factors(start, end): + """ + Download the monthly Fama-French research factors directly from the Ken + French data library (replaces pandas-datareader's "famafrench" reader). + Returns a month-``Period``-indexed DataFrame with values in percent. + """ + url = ("https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/" + "F-F_Research_Data_Factors_CSV.zip") + with urllib.request.urlopen(url) as response: + archive = zipfile.ZipFile(io.BytesIO(response.read())) + text = archive.read(archive.namelist()[0]).decode("utf-8") + + # The file is a text preamble, then the monthly table (rows keyed by + # YYYYMM), then an annual table (rows keyed by YYYY). Keep the monthly + # rows, stopping once that contiguous block ends. + records = [] + for line in text.splitlines(): + cells = [cell.strip() for cell in line.split(",")] + key = cells[0] + if len(key) == 6 and key.isdigit(): + records.append([key] + [float(x) for x in cells[1:5]]) + elif records: + break + factors = pd.DataFrame( + records, columns=["date", "Mkt-RF", "SMB", "HML", "RF"]) + factors.index = pd.PeriodIndex( + pd.to_datetime(factors["date"], format="%Y%m"), freq="M") + factors = factors.drop(columns="date") + window = ((factors.index >= pd.Period(start, "M")) + & (factors.index <= pd.Period(end, "M"))) + return factors.loc[window] + + def load_hs_monthly_data( start="1959-02-01", end="1978-12-01", @@ -1034,8 +1072,7 @@ def load_hs_monthly_data( sample_start = start_period.to_timestamp("M") sample_end = end_period.to_timestamp("M") - fred = web.DataReader( - list(fred_codes.values()), "fred", fetch_start, fetch_end) + fred = read_fred(list(fred_codes.values()), fetch_start, fetch_end) fred = fred.rename(columns={v: k for k, v in fred_codes.items()}) fred.index = to_month_end(fred.index) fred["cons_real_level"] = fred["cons_nd_real_index"] @@ -1050,9 +1087,7 @@ def load_hs_monthly_data( fred["cons_price_index"] / fred["cons_price_index"].shift(1) ) - ff = web.DataReader( - "F-F_Research_Data_Factors", "famafrench", - fetch_start, fetch_end)[0].copy() + ff = read_famafrench_factors(fetch_start, fetch_end).copy() ff.columns = [str(col).strip() for col in ff.columns] if ("Mkt-RF" not in ff.columns) or ("RF" not in ff.columns): raise KeyError( diff --git a/lectures/hansen_singleton_1983.md b/lectures/hansen_singleton_1983.md index ee70ff25b..c6c6fcbc4 100644 --- a/lectures/hansen_singleton_1983.md +++ b/lectures/hansen_singleton_1983.md @@ -64,31 +64,20 @@ To keep lecture this lecture narrowly focused, we estimate one return at a time * we use only monthly nondurable consumption (`ND`). -In addition to what comes with Anaconda, this lecture requires `pandas-datareader` - -```{code-cell} ipython3 -:tags: [hide-output] - -!pip install pandas-datareader -``` - ```{code-cell} ipython3 -import warnings +import io +import urllib.request +import zipfile from itertools import combinations import matplotlib.pyplot as plt import numpy as np import pandas as pd from IPython.display import Latex -from pandas_datareader import data as web from scipy import stats from scipy.linalg import LinAlgError, cholesky, solve_triangular from scipy.optimize import minimize from statsmodels.stats.stattools import durbin_watson - -warnings.filterwarnings( - "ignore", message=".*date_parser.*", category=FutureWarning -) ``` We also define a helper to display DataFrames as LaTeX arrays in the hidden cell below @@ -1439,7 +1428,7 @@ While Hansen-Singleton use CRSP value-weighted NYSE returns, we use the Ken Fren The consumption series is constructed from consumption of nondurables (`ND`) with the nondurables deflator. -The hidden cell below pulls the relevant FRED series, constructs per capita real consumption, and joins with the Ken French returns +The hidden cell below pulls the relevant FRED series, constructs per capita real consumption, and joins with the Ken French returns. The data are downloaded directly from the [FRED](https://fred.stlouisfed.org/) and [Ken French](https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/data_library.html) data libraries. ```{code-cell} ipython3 :tags: [hide-cell] @@ -1457,6 +1446,55 @@ def to_month_end(index): return pd.PeriodIndex(pd.DatetimeIndex(index), freq="M").to_timestamp("M") +def read_fred(codes, start, end): + """ + Download FRED series as a date-indexed DataFrame whose columns are the + requested FRED codes (replaces pandas-datareader's "fred" reader). + """ + base = "https://fred.stlouisfed.org/graph/fredgraph.csv" + columns = [] + for code in codes: + url = f"{base}?id={code}&cosd={start:%Y-%m-%d}&coed={end:%Y-%m-%d}" + columns.append( + pd.read_csv(url, index_col=0, parse_dates=True, na_values=".")) + fred = pd.concat(columns, axis=1).astype("float64") + fred.index.name = "DATE" + return fred + + +def read_famafrench_factors(start, end): + """ + Download the monthly Fama-French research factors directly from the Ken + French data library (replaces pandas-datareader's "famafrench" reader). + Returns a month-``Period``-indexed DataFrame with values in percent. + """ + url = ("https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/" + "F-F_Research_Data_Factors_CSV.zip") + with urllib.request.urlopen(url) as response: + archive = zipfile.ZipFile(io.BytesIO(response.read())) + text = archive.read(archive.namelist()[0]).decode("utf-8") + + # The file is a text preamble, then the monthly table (rows keyed by + # YYYYMM), then an annual table (rows keyed by YYYY). Keep the monthly + # rows, stopping once that contiguous block ends. + records = [] + for line in text.splitlines(): + cells = [cell.strip() for cell in line.split(",")] + key = cells[0] + if len(key) == 6 and key.isdigit(): + records.append([key] + [float(x) for x in cells[1:5]]) + elif records: + break + factors = pd.DataFrame( + records, columns=["date", "Mkt-RF", "SMB", "HML", "RF"]) + factors.index = pd.PeriodIndex( + pd.to_datetime(factors["date"], format="%Y%m"), freq="M") + factors = factors.drop(columns="date") + window = ((factors.index >= pd.Period(start, "M")) + & (factors.index <= pd.Period(end, "M"))) + return factors.loc[window] + + def load_hs_monthly_data( start="1959-02-01", end="1978-12-01", @@ -1473,8 +1511,7 @@ def load_hs_monthly_data( sample_start = start_period.to_timestamp("M") sample_end = end_period.to_timestamp("M") - fred = web.DataReader( - list(fred_codes.values()), "fred", fetch_start, fetch_end) + fred = read_fred(list(fred_codes.values()), fetch_start, fetch_end) fred = fred.rename(columns={v: k for k, v in fred_codes.items()}) fred.index = to_month_end(fred.index) fred["cons_real_level"] = fred["cons_nd_real_index"] @@ -1488,9 +1525,7 @@ def load_hs_monthly_data( fred["cons_price_index"] / fred["cons_price_index"].shift(1) ) - ff = web.DataReader( - "F-F_Research_Data_Factors", "famafrench", - fetch_start, fetch_end)[0].copy() + ff = read_famafrench_factors(fetch_start, fetch_end).copy() ff.columns = [str(col).strip() for col in ff.columns] if ("Mkt-RF" not in ff.columns) or ("RF" not in ff.columns): raise KeyError( From 6128fd88be411de14f2c081978e3dd56611eeae6 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Sun, 21 Jun 2026 20:08:58 +1000 Subject: [PATCH 2/3] [hansen_singleton] read vendored data snapshot instead of fetching inline Switch both lectures to read the pre-built monthly CSV from _static/lecture_specific/hansen_singleton_198{2,3}/ (added in PR #926) via its raw GitHub URL, replacing the inline FRED / Fama-French download helpers from the previous commit. The data construction now lives in the per-lecture make_data.py maintenance scripts; the lectures just read the frozen snapshot. This keeps the build reproducible and off the live data endpoints, and still removes the pandas-datareader dependency that breaks under pandas 3.0. Depends on PR #926 (must land on main first so the raw URL resolves). Co-Authored-By: Claude Opus 4.8 --- lectures/hansen_singleton_1982.md | 132 ++++------------------------ lectures/hansen_singleton_1983.md | 138 ++++-------------------------- 2 files changed, 34 insertions(+), 236 deletions(-) diff --git a/lectures/hansen_singleton_1982.md b/lectures/hansen_singleton_1982.md index 34fd39d4b..2c348adb2 100644 --- a/lectures/hansen_singleton_1982.md +++ b/lectures/hansen_singleton_1982.md @@ -52,10 +52,6 @@ Though maximum likelihood estimators (such as the MLE in {doc}`hansen_singleton_ Relative to the full paper, we only estimate one return at a time (value-weighted stock returns), using only monthly nondurable consumption (`ND`), and omitting their maximum-likelihood comparison (Table II) and multi-return systems (Table III). ```{code-cell} ipython3 -import io -import urllib.request -import zipfile - import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -989,126 +985,30 @@ Because the Ken French return is not identical to the original CRSP NYSE value-w Both this lecture and the companion lecture {doc}`hansen_singleton_1983` use the same data construction. -The hidden cell below pulls the relevant FRED series, constructs per capita real consumption, and joins with Ken French market returns. The data are downloaded directly from the [FRED](https://fred.stlouisfed.org/) and [Ken French](https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/data_library.html) data libraries. +The hidden cell below loads a vendored monthly dataset of gross real returns and gross consumption growth. The data are built from the [FRED](https://fred.stlouisfed.org/) and [Ken French](https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/data_library.html) data libraries by the maintenance script at [`_static/lecture_specific/hansen_singleton_1982/make_data.py`](https://github.com/QuantEcon/lecture-python.myst/blob/main/lectures/_static/lecture_specific/hansen_singleton_1982/make_data.py) and read here directly from GitHub. ```{code-cell} ipython3 :tags: [hide-cell] -fred_codes = { - "population_16plus": "CNP16OV", - "cons_nd_real_index": "DNDGRA3M086SBEA", - "cons_nd_price_index": "DNDGRG3M086SBEA", -} - -def to_month_end(index): - """ - Convert a date index to month-end timestamps. - """ - return pd.PeriodIndex(pd.DatetimeIndex(index), freq="M").to_timestamp("M") +DATA_URL = ( + "https://github.com/QuantEcon/lecture-python.myst/raw/refs/heads/main/" + "lectures/_static/lecture_specific/hansen_singleton_1982/" + "hansen_singleton_1982_data.csv" +) -def read_fred(codes, start, end): - """ - Download FRED series as a date-indexed DataFrame whose columns are the - requested FRED codes (replaces pandas-datareader's "fred" reader). - """ - base = "https://fred.stlouisfed.org/graph/fredgraph.csv" - columns = [] - for code in codes: - url = f"{base}?id={code}&cosd={start:%Y-%m-%d}&coed={end:%Y-%m-%d}" - columns.append( - pd.read_csv(url, index_col=0, parse_dates=True, na_values=".")) - fred = pd.concat(columns, axis=1).astype("float64") - fred.index.name = "DATE" - return fred - - -def read_famafrench_factors(start, end): - """ - Download the monthly Fama-French research factors directly from the Ken - French data library (replaces pandas-datareader's "famafrench" reader). - Returns a month-``Period``-indexed DataFrame with values in percent. +def load_hs_monthly_data(start="1959-02-01", end="1978-12-01"): """ - url = ("https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/" - "F-F_Research_Data_Factors_CSV.zip") - with urllib.request.urlopen(url) as response: - archive = zipfile.ZipFile(io.BytesIO(response.read())) - text = archive.read(archive.namelist()[0]).decode("utf-8") - - # The file is a text preamble, then the monthly table (rows keyed by - # YYYYMM), then an annual table (rows keyed by YYYY). Keep the monthly - # rows, stopping once that contiguous block ends. - records = [] - for line in text.splitlines(): - cells = [cell.strip() for cell in line.split(",")] - key = cells[0] - if len(key) == 6 and key.isdigit(): - records.append([key] + [float(x) for x in cells[1:5]]) - elif records: - break - factors = pd.DataFrame( - records, columns=["date", "Mkt-RF", "SMB", "HML", "RF"]) - factors.index = pd.PeriodIndex( - pd.to_datetime(factors["date"], format="%Y%m"), freq="M") - factors = factors.drop(columns="date") - window = ((factors.index >= pd.Period(start, "M")) - & (factors.index <= pd.Period(end, "M"))) - return factors.loc[window] - - -def load_hs_monthly_data( - start="1959-02-01", - end="1978-12-01", -): - """ - Build monthly gross real return and gross consumption-growth series. - """ - start_period = pd.Timestamp(start).to_period("M") - end_period = pd.Timestamp(end).to_period("M") - - # Pull one extra month to build the first in-sample growth rate. - fetch_start = (start_period - 1).to_timestamp(how="start") - fetch_end = end_period.to_timestamp("M") - sample_start = start_period.to_timestamp("M") - sample_end = end_period.to_timestamp("M") - - fred = read_fred(list(fred_codes.values()), fetch_start, fetch_end) - fred = fred.rename(columns={v: k for k, v in fred_codes.items()}) - fred.index = to_month_end(fred.index) - fred["cons_real_level"] = fred["cons_nd_real_index"] - fred["cons_price_index"] = fred["cons_nd_price_index"] - fred["consumption_per_capita"] = fred["cons_real_level"] \ - / fred["population_16plus"] - fred["gross_cons_growth"] = ( - fred["consumption_per_capita"] - / fred["consumption_per_capita"].shift(1) - ) - fred["gross_inflation_cons"] = ( - fred["cons_price_index"] / fred["cons_price_index"].shift(1) - ) + Load the monthly gross real return and gross consumption-growth series. - ff = read_famafrench_factors(fetch_start, fetch_end).copy() - ff.columns = [str(col).strip() for col in ff.columns] - if ("Mkt-RF" not in ff.columns) or ("RF" not in ff.columns): - raise KeyError( - "Fama-French data missing required columns: 'Mkt-RF' and 'RF'.") - - # Mkt-RF and RF are reported in percent per month. - ff["gross_nom_return"] = 1.0 + (ff["Mkt-RF"] + ff["RF"]) / 100.0 - ff.index = ff.index.to_timestamp(how="end") - ff.index = to_month_end(ff.index) - market = ff[["gross_nom_return"]] - - out = fred.join(market, how="inner") - out["gross_real_return"] = out["gross_nom_return"] \ - / out["gross_inflation_cons"] - out = out.loc[sample_start:sample_end].dropna() - - required_cols = [ - "gross_real_return", - "gross_cons_growth", - ] - return out[required_cols].copy() + The data are a vendored snapshot built by the maintenance script at + ``_static/lecture_specific/hansen_singleton_1982/make_data.py``, which + constructs them from FRED and the Ken French data library. + """ + frame = pd.read_csv(DATA_URL, index_col=0, parse_dates=True) + start = pd.Timestamp(start).to_period("M").to_timestamp("M") + end = pd.Timestamp(end).to_period("M").to_timestamp("M") + return frame.loc[start:end] def get_estimation_data( diff --git a/lectures/hansen_singleton_1983.md b/lectures/hansen_singleton_1983.md index c6c6fcbc4..3ccc3631b 100644 --- a/lectures/hansen_singleton_1983.md +++ b/lectures/hansen_singleton_1983.md @@ -65,9 +65,6 @@ To keep lecture this lecture narrowly focused, we estimate one return at a time * we use only monthly nondurable consumption (`ND`). ```{code-cell} ipython3 -import io -import urllib.request -import zipfile from itertools import combinations import matplotlib.pyplot as plt @@ -1428,131 +1425,32 @@ While Hansen-Singleton use CRSP value-weighted NYSE returns, we use the Ken Fren The consumption series is constructed from consumption of nondurables (`ND`) with the nondurables deflator. -The hidden cell below pulls the relevant FRED series, constructs per capita real consumption, and joins with the Ken French returns. The data are downloaded directly from the [FRED](https://fred.stlouisfed.org/) and [Ken French](https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/data_library.html) data libraries. +The hidden cell below loads a vendored monthly dataset of returns and consumption series. The data are built from the [FRED](https://fred.stlouisfed.org/) and [Ken French](https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/data_library.html) data libraries by the maintenance script at [`_static/lecture_specific/hansen_singleton_1983/make_data.py`](https://github.com/QuantEcon/lecture-python.myst/blob/main/lectures/_static/lecture_specific/hansen_singleton_1983/make_data.py) and read here directly from GitHub. ```{code-cell} ipython3 :tags: [hide-cell] -fred_codes = { - "population_16plus": "CNP16OV", - "cons_nd_real_index": "DNDGRA3M086SBEA", - "cons_nd_price_index": "DNDGRG3M086SBEA", -} - -def to_month_end(index): - """ - Convert a date index to month-end timestamps. - """ - return pd.PeriodIndex(pd.DatetimeIndex(index), freq="M").to_timestamp("M") +DATA_URL = ( + "https://github.com/QuantEcon/lecture-python.myst/raw/refs/heads/main/" + "lectures/_static/lecture_specific/hansen_singleton_1983/" + "hansen_singleton_1983_data.csv" +) -def read_fred(codes, start, end): - """ - Download FRED series as a date-indexed DataFrame whose columns are the - requested FRED codes (replaces pandas-datareader's "fred" reader). - """ - base = "https://fred.stlouisfed.org/graph/fredgraph.csv" - columns = [] - for code in codes: - url = f"{base}?id={code}&cosd={start:%Y-%m-%d}&coed={end:%Y-%m-%d}" - columns.append( - pd.read_csv(url, index_col=0, parse_dates=True, na_values=".")) - fred = pd.concat(columns, axis=1).astype("float64") - fred.index.name = "DATE" - return fred - - -def read_famafrench_factors(start, end): - """ - Download the monthly Fama-French research factors directly from the Ken - French data library (replaces pandas-datareader's "famafrench" reader). - Returns a month-``Period``-indexed DataFrame with values in percent. - """ - url = ("https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/" - "F-F_Research_Data_Factors_CSV.zip") - with urllib.request.urlopen(url) as response: - archive = zipfile.ZipFile(io.BytesIO(response.read())) - text = archive.read(archive.namelist()[0]).decode("utf-8") - - # The file is a text preamble, then the monthly table (rows keyed by - # YYYYMM), then an annual table (rows keyed by YYYY). Keep the monthly - # rows, stopping once that contiguous block ends. - records = [] - for line in text.splitlines(): - cells = [cell.strip() for cell in line.split(",")] - key = cells[0] - if len(key) == 6 and key.isdigit(): - records.append([key] + [float(x) for x in cells[1:5]]) - elif records: - break - factors = pd.DataFrame( - records, columns=["date", "Mkt-RF", "SMB", "HML", "RF"]) - factors.index = pd.PeriodIndex( - pd.to_datetime(factors["date"], format="%Y%m"), freq="M") - factors = factors.drop(columns="date") - window = ((factors.index >= pd.Period(start, "M")) - & (factors.index <= pd.Period(end, "M"))) - return factors.loc[window] - - -def load_hs_monthly_data( - start="1959-02-01", - end="1978-12-01", -): - """ - Build monthly gross real return and gross consumption-growth series. +def load_hs_monthly_data(start="1959-02-01", end="1978-12-01"): """ - start_period = pd.Timestamp(start).to_period("M") - end_period = pd.Timestamp(end).to_period("M") - - # Pull one extra month to build the first in-sample growth rate - fetch_start = (start_period - 1).to_timestamp(how="start") - fetch_end = end_period.to_timestamp("M") - sample_start = start_period.to_timestamp("M") - sample_end = end_period.to_timestamp("M") - - fred = read_fred(list(fred_codes.values()), fetch_start, fetch_end) - fred = fred.rename(columns={v: k for k, v in fred_codes.items()}) - fred.index = to_month_end(fred.index) - fred["cons_real_level"] = fred["cons_nd_real_index"] - fred["cons_price_index"] = fred["cons_nd_price_index"] - fred["consumption_per_capita"] = fred["cons_real_level"] \ - / fred["population_16plus"] - fred["gross_cons_growth"] = ( - fred["consumption_per_capita"] / fred["consumption_per_capita"].shift(1) - ) - fred["gross_inflation_cons"] = ( - fred["cons_price_index"] / fred["cons_price_index"].shift(1) - ) + Load the monthly series used in the lecture: gross real market return, + gross consumption growth, gross consumption inflation, per capita real + consumption, and gross real T-bill return. - ff = read_famafrench_factors(fetch_start, fetch_end).copy() - ff.columns = [str(col).strip() for col in ff.columns] - if ("Mkt-RF" not in ff.columns) or ("RF" not in ff.columns): - raise KeyError( - "Fama-French data missing required columns: 'Mkt-RF' and 'RF'.") - - # Mkt-RF and RF are reported in percent per month - ff["gross_nom_return"] = 1.0 + (ff["Mkt-RF"] + ff["RF"]) / 100.0 - ff["gross_nom_tbill"] = 1.0 + ff["RF"] / 100.0 - ff.index = ff.index.to_timestamp(how="end") - ff.index = to_month_end(ff.index) - market = ff[["gross_nom_return", "gross_nom_tbill"]] - - out = fred.join(market, how="inner") - out["gross_real_return"] = out["gross_nom_return"] \ - / out["gross_inflation_cons"] - out["gross_real_tbill"] = out["gross_nom_tbill"] \ - / out["gross_inflation_cons"] - out = out.loc[sample_start:sample_end].dropna() - - required_cols = [ - "gross_real_return", - "gross_cons_growth", - "gross_inflation_cons", - "consumption_per_capita", - "gross_real_tbill", - ] - return out[required_cols].copy() + The data are a vendored snapshot built by the maintenance script at + ``_static/lecture_specific/hansen_singleton_1983/make_data.py``, which + constructs them from FRED and the Ken French data library. + """ + frame = pd.read_csv(DATA_URL, index_col=0, parse_dates=True) + start = pd.Timestamp(start).to_period("M").to_timestamp("M") + end = pd.Timestamp(end).to_period("M").to_timestamp("M") + return frame.loc[start:end] def get_estimation_data( From 135e0ddb51567535f35bf3d2d36ca5d432f50716 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Sun, 21 Jun 2026 20:36:17 +1000 Subject: [PATCH 3/3] [hansen_singleton] cache the vendored CSV at cell scope Read the snapshot once into a module-level _data and have load_hs_monthly_data slice a copy of it, instead of re-downloading/parsing on every call. This removes the redundant fetch in hansen_singleton_1983 (which loads via both get_estimation_data and get_tbill_estimation_data). The .copy() keeps callers from mutating the cached frame. Addresses Copilot review on PR #925. Co-Authored-By: Claude Opus 4.8 --- lectures/hansen_singleton_1982.md | 6 ++++-- lectures/hansen_singleton_1983.md | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lectures/hansen_singleton_1982.md b/lectures/hansen_singleton_1982.md index 2c348adb2..9959ea8c4 100644 --- a/lectures/hansen_singleton_1982.md +++ b/lectures/hansen_singleton_1982.md @@ -996,6 +996,9 @@ DATA_URL = ( "hansen_singleton_1982_data.csv" ) +# Read the vendored snapshot once; load_hs_monthly_data just slices it. +_data = pd.read_csv(DATA_URL, index_col=0, parse_dates=True) + def load_hs_monthly_data(start="1959-02-01", end="1978-12-01"): """ @@ -1005,10 +1008,9 @@ def load_hs_monthly_data(start="1959-02-01", end="1978-12-01"): ``_static/lecture_specific/hansen_singleton_1982/make_data.py``, which constructs them from FRED and the Ken French data library. """ - frame = pd.read_csv(DATA_URL, index_col=0, parse_dates=True) start = pd.Timestamp(start).to_period("M").to_timestamp("M") end = pd.Timestamp(end).to_period("M").to_timestamp("M") - return frame.loc[start:end] + return _data.loc[start:end].copy() def get_estimation_data( diff --git a/lectures/hansen_singleton_1983.md b/lectures/hansen_singleton_1983.md index 3ccc3631b..0be605a08 100644 --- a/lectures/hansen_singleton_1983.md +++ b/lectures/hansen_singleton_1983.md @@ -1436,6 +1436,9 @@ DATA_URL = ( "hansen_singleton_1983_data.csv" ) +# Read the vendored snapshot once; load_hs_monthly_data just slices it. +_data = pd.read_csv(DATA_URL, index_col=0, parse_dates=True) + def load_hs_monthly_data(start="1959-02-01", end="1978-12-01"): """ @@ -1447,10 +1450,9 @@ def load_hs_monthly_data(start="1959-02-01", end="1978-12-01"): ``_static/lecture_specific/hansen_singleton_1983/make_data.py``, which constructs them from FRED and the Ken French data library. """ - frame = pd.read_csv(DATA_URL, index_col=0, parse_dates=True) start = pd.Timestamp(start).to_period("M").to_timestamp("M") end = pd.Timestamp(end).to_period("M").to_timestamp("M") - return frame.loc[start:end] + return _data.loc[start:end].copy() def get_estimation_data(