# --- Setup: load our helpers and the data --------------------------------
import sys, pathlib
for _p in [pathlib.Path.cwd(), pathlib.Path.cwd().parent]:
    if (_p / 'lib' / 'statslab.py').exists():
        sys.path.insert(0, str(_p / 'lib')); break
import statslab as sl

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
sl.use_course_style()

GAPMINDER_URL = 'https://raw.githubusercontent.com/kirenz/datasets/master/gapminder.csv'
gap = sl.load_csv('gapminder.csv', url=GAPMINDER_URL)

# Use a single year so each country appears exactly once (a clean cross-section).
g07 = gap[gap['year'] == 2007].copy()

print(f'{len(g07)} countries in 2007')
print(f'GDP per capita ranges from {g07["gdpPercap"].min():,.0f} to {g07["gdpPercap"].max():,.0f} (USD)')
print(f'life expectancy ranges from {g07["lifeExp"].min():.1f} to {g07["lifeExp"].max():.1f} years')
g07[['country', 'continent', 'gdpPercap', 'lifeExp']].head()

142 countries in 2007
GDP per capita ranges from 278 to 49,357 (USD)
life expectancy ranges from 39.6 to 82.6 years

x = g07['gdpPercap'].to_numpy()      # how rich (USD per person)
y = g07['lifeExp'].to_numpy()        # how long (years)

fig, ax = plt.subplots()
ax.scatter(x, y, s=28, alpha=0.6, color='#4c72b0', edgecolor='white', lw=0.4)
ax.set_xlabel('GDP per capita (USD per person)')
ax.set_ylabel('life expectancy (years)')
ax.set_title('Gapminder 2007: richer countries live longer — but not in a straight line')
plt.show()

r_raw = g07['gdpPercap'].corr(g07['lifeExp'])
print(f'correlation r (GDP per capita, life expectancy) = {r_raw:.3f}')
print('\nThat is a "moderate-to-strong positive" linear trend on the textbook scale.')
print('But your eyes already saw a relationship TIGHTER than 0.68 looks — the curve is the reason.')

correlation r (GDP per capita, life expectancy) = 0.679

That is a "moderate-to-strong positive" linear trend on the textbook scale.
But your eyes already saw a relationship TIGHTER than 0.68 looks — the curve is the reason.

fit_raw = stats.linregress(x, y)
b1, b0 = fit_raw.slope, fit_raw.intercept
print(f'slope b1     = {b1:.5f} years per extra USD of GDP/person')
print(f'intercept b0 = {b0:.2f} years')
print(f'R^2 = r^2    = {fit_raw.rvalue**2:.3f}   (the line explains ~{fit_raw.rvalue**2*100:.0f}% of the variation)')

line_x = np.linspace(x.min(), x.max(), 200)
line_y = b0 + b1 * line_x

fig, ax = plt.subplots()
ax.scatter(x, y, s=28, alpha=0.6, color='#4c72b0', edgecolor='white', lw=0.4, label='countries')
ax.plot(line_x, line_y, color='#D55E00', lw=2.5, label='least-squares straight line')
ax.set_xlabel('GDP per capita (USD per person)')
ax.set_ylabel('life expectancy (years)')
ax.set_title('The best STRAIGHT line cuts straight through a curved truth')
ax.legend(); plt.show()

slope b1     = 0.00064 years per extra USD of GDP/person
intercept b0 = 59.57 years
R^2 = r^2    = 0.461   (the line explains ~46% of the variation)

predicted_raw = b0 + b1 * x
residuals_raw = y - predicted_raw

fig, ax = plt.subplots()
ax.scatter(predicted_raw, residuals_raw, s=28, alpha=0.6, color='#4c72b0', edgecolor='white', lw=0.4)
ax.axhline(0, color='#D55E00', lw=2, ls='--', label='residual = 0 (a perfect prediction)')
ax.set_xlabel('predicted life expectancy (years)')
ax.set_ylabel('residual = actual − predicted (years)')
ax.set_title('Residual plot for the straight line: a clear arch (∩-shape), not random scatter')
ax.legend(); plt.show()

print(f'Residuals still average to ~0: mean = {residuals_raw.mean():.3f} years')
print('...but they are NOT shapeless. They rise into positive then fall back — an arch (∩).')
print('That leftover curve is the straight line confessing it is the wrong shape.')

Residuals still average to ~0: mean = 0.000 years
...but they are NOT shapeless. They rise into positive then fall back — an arch (∩).
That leftover curve is the straight line confessing it is the wrong shape.

skew_raw = stats.skew(g07['gdpPercap'])
log_gdp  = np.log10(g07['gdpPercap'])
skew_log = stats.skew(log_gdp)

print(f'skew of gdpPercap        = {skew_raw:+.2f}   <- strongly RIGHT-skewed (long rich tail)')
print(f'skew of log10(gdpPercap) = {skew_log:+.2f}   <- nearly symmetric: the log tamed it')

# Same data, same y, but wealth now on a log10 ruler.
gl = np.log10(x)
fig, ax = plt.subplots()
ax.scatter(gl, y, s=28, alpha=0.6, color='#009E73', edgecolor='white', lw=0.4)
ax.set_xlabel('log10(GDP per capita)   [3 = 1,000 USD,  4 = 10,000 USD]')
ax.set_ylabel('life expectancy (years)')
ax.set_title('Switch wealth to a log ruler and the curve STRAIGHTENS into a line')
plt.show()

skew of gdpPercap        = +1.21   <- strongly RIGHT-skewed (long rich tail)
skew of log10(gdpPercap) = -0.15   <- nearly symmetric: the log tamed it

r_log = log_gdp.corr(g07['lifeExp'])
fit_log = stats.linregress(gl, y)
b1L, b0L = fit_log.slope, fit_log.intercept

print(f'correlation r:      raw = {r_raw:.3f}   ->   log = {r_log:.3f}    (jumped up!)')
print(f'R^2 = r^2:          raw = {r_raw**2:.3f}   ->   log = {r_log**2:.3f}')
print(f'\nlog-line slope b1 = {b1L:.2f} years per 10x increase in GDP per capita')
print(f'log-line intercept b0 = {b0L:.2f} years')

logline_x = np.linspace(gl.min(), gl.max(), 200)
logline_y = b0L + b1L * logline_x

fig, ax = plt.subplots()
ax.scatter(gl, y, s=28, alpha=0.6, color='#009E73', edgecolor='white', lw=0.4, label='countries')
ax.plot(logline_x, logline_y, color='#E69F00', lw=2.5, label='least-squares line (log wealth)')
ax.set_xlabel('log10(GDP per capita)')
ax.set_ylabel('life expectancy (years)')
ax.set_title('After the log transform: the straight line fits well')
ax.legend(); plt.show()

correlation r:      raw = 0.679   ->   log = 0.809    (jumped up!)
R^2 = r^2:          raw = 0.461   ->   log = 0.654

log-line slope b1 = 16.59 years per 10x increase in GDP per capita
log-line intercept b0 = 4.95 years

predicted_log = b0L + b1L * gl
residuals_log = y - predicted_log

fig, (axL, axR) = plt.subplots(1, 2, figsize=(13, 5.0), sharey=True)

axL.scatter(predicted_raw, residuals_raw, s=24, alpha=0.6, color='#4c72b0', edgecolor='white', lw=0.4)
axL.axhline(0, color='#D55E00', lw=2, ls='--')
axL.set_title('BEFORE: straight line on raw GDP\n(residuals trace an arch ∩ — misfit)')
axL.set_xlabel('predicted life expectancy (years)')
axL.set_ylabel('residual (years)')

axR.scatter(predicted_log, residuals_log, s=24, alpha=0.6, color='#009E73', edgecolor='white', lw=0.4)
axR.axhline(0, color='#D55E00', lw=2, ls='--')
axR.set_title('AFTER: line on log10(GDP)\n(residuals look like a shapeless band — good)')
axR.set_xlabel('predicted life expectancy (years)')

fig.suptitle('Reading the residuals: the log transform erased the curve', fontweight='bold')
plt.tight_layout(); plt.show()

print('Left: a patterned arch (∩) — the model is the wrong shape.')
print('Right: scatter with no obvious trend — the straight line now fits the data.')

Left: a patterned arch (∩) — the model is the wrong shape.
Right: scatter with no obvious trend — the straight line now fits the data.

	country	continent	gdpPercap	lifeExp
11	Afghanistan	Asia	974.580338	43.828
23	Albania	Europe	5937.029526	76.423
35	Algeria	Africa	6223.367465	72.301
47	Angola	Africa	4797.231267	42.731
59	Argentina	Americas	12779.379640	75.320

	Ames (core lesson)	Gapminder (this lesson)
Honest form of the relationship	roughly straight	clearly curved (concave)
Did the raw straight line fit?	yes, fairly directly	no — systematic, patterned miss
Residual plot of the raw fit	roughly flat (slight fanning)	a clear arch (∩ curve)
What $r$ reported	a fair summary ($\approx 0.71$)	an understatement ($\approx 0.68$)
Fix needed	none	log-transform wealth ($r \to 0.81$)

Relationships on a different dataset: wealth & life expectancy¶

1) The first move (always): a scatterplot¶

2) The correlation r — and why it understates a curve¶

3) Fit the least-squares line anyway — and let it betray itself¶

4) Residuals — the diagnostic that names the problem¶

5) The fix: a log transform straightens the curve¶

6) The contrast with Ames — and why r and the line have a blind spot¶

7) And still: correlation is not causation¶

Now you try¶

What you learned¶