# --- Setup: load our helpers and the data --------------------------------
import sys, pathlib
for _p in [pathlib.Path.cwd(), pathlib.Path.cwd().parent]:
    if (_p / 'lib' / 'statslab.py').exists():
        sys.path.insert(0, str(_p / 'lib')); break
import statslab as sl

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
sl.use_course_style()
rng = np.random.default_rng(12)  # fixed seed -> reproducible; learner can change it

TITANIC_URL = 'https://web.stanford.edu/class/archive/cs/cs109/cs109.1166/stuff/titanic.csv'
titanic = sl.load_csv('titanic.csv', url=TITANIC_URL)

print(f'{len(titanic)} passengers, {titanic.shape[1]} columns')
titanic.head()

887 passengers, 8 columns

n_total    = len(titanic)
n_survived = int(titanic['Survived'].sum())      # the 1s
p_survived = titanic['Survived'].mean()          # mean of 0/1 == the fraction

print(f"Survivors: {n_survived} out of {n_total}")
print(f"P(survived) = {n_survived}/{n_total} = {p_survived:.4f}")
print(f"So about {p_survived*100:.0f}% of passengers in this dataset survived.")

Survivors: 342 out of 887
P(survived) = 342/887 = 0.3856
So about 39% of passengers in this dataset survived.

# Way 1 — restrict to the subgroup, then take the survival rate.
women = titanic[titanic['Sex'] == 'female']
men   = titanic[titanic['Sex'] == 'male']
p_surv_given_female = women['Survived'].mean()
p_surv_given_male   = men['Survived'].mean()

# Way 2 — P(survived AND female) / P(female), using the whole dataset.
p_female              = (titanic['Sex'] == 'female').mean()
p_survived_and_female = ((titanic['Sex'] == 'female') & (titanic['Survived'] == 1)).mean()
p_via_division        = p_survived_and_female / p_female

print(f"P(survived | female), restrict & count : {p_surv_given_female:.4f}")
print(f"P(survived | female), joint / marginal : {p_via_division:.4f}   <- same number")
print()
print(f"P(survived | female) = {p_surv_given_female:.4f}")
print(f"P(survived | male)   = {p_surv_given_male:.4f}")
print(f"(compare to the overall P(survived) = {p_survived:.4f})")

P(survived | female), restrict & count : 0.7420
P(survived | female), joint / marginal : 0.7420   <- same number

P(survived | female) = 0.7420
P(survived | male)   = 0.1902
(compare to the overall P(survived) = 0.3856)

counts = pd.crosstab(titanic['Sex'], titanic['Survived'],
                     margins=True, margins_name='Total')
counts.columns = ['Died (0)', 'Survived (1)', 'Total']
print("COUNTS — how many passengers fall in each box:")
counts

COUNTS — how many passengers fall in each box:

# Same table as PROPORTIONS of all 887 passengers (every box / grand total).
props = pd.crosstab(titanic['Sex'], titanic['Survived'],
                    normalize=True, margins=True, margins_name='Total')
props.columns = ['Died (0)', 'Survived (1)', 'Total']
print("PROPORTIONS — each box as a fraction of everyone:")
props.round(4)

PROPORTIONS — each box as a fraction of everyone:

lhs = p_survived_and_female                 # P(survived and female), counted directly
rhs = p_surv_given_female * p_female        # P(survived | female) * P(female)

print(f"P(survived and female)            = {lhs:.4f}")
print(f"P(survived | female) * P(female)  = {rhs:.4f}")
print(f"Match? {np.isclose(lhs, rhs)}   <- the multiplication rule, confirmed by counting.")

P(survived and female)            = 0.2627
P(survived | female) * P(female)  = 0.2627
Match? True   <- the multiplication rule, confirmed by counting.

# A coin flip per passenger — pure noise, unrelated to the Titanic.
coin = rng.choice(['heads', 'tails'], size=len(titanic))
titanic_coin = titanic.assign(coin=coin)

p_surv_given_heads = titanic_coin.loc[titanic_coin['coin'] == 'heads', 'Survived'].mean()
p_surv_given_tails = titanic_coin.loc[titanic_coin['coin'] == 'tails', 'Survived'].mean()

print(f"P(survived)               = {p_survived:.4f}")
print(f"P(survived | coin=heads)  = {p_surv_given_heads:.4f}   <- ~the same (independent)")
print(f"P(survived | coin=tails)  = {p_surv_given_tails:.4f}   <- ~the same (independent)")
print()
print("Sex MOVES the survival rate a lot; a coin flip barely nudges it.")
print(f"  conditioning on sex  : {p_surv_given_female:.3f} vs {p_surv_given_male:.3f}  (huge gap)")
print(f"  conditioning on coin : {p_surv_given_heads:.3f} vs {p_surv_given_tails:.3f}  (tiny gap)")

P(survived)               = 0.3856
P(survived | coin=heads)  = 0.4130   <- ~the same (independent)
P(survived | coin=tails)  = 0.3596   <- ~the same (independent)

Sex MOVES the survival rate a lot; a coin flip barely nudges it.
  conditioning on sex  : 0.742 vs 0.190  (huge gap)
  conditioning on coin : 0.413 vs 0.360  (tiny gap)

class_rates = titanic.groupby('Pclass')['Survived'].mean()
for c, r in class_rates.items():
    print(f"P(survived | Pclass={c}) = {r:.4f}")

fig, ax = plt.subplots()
bars = ax.bar(['1st class', '2nd class', '3rd class'], class_rates.values,
              color=['#4c72b0', '#E69F00', '#D55E00'], edgecolor='white')
ax.axhline(p_survived, color='black', ls='--', lw=1.5,
           label=f'overall P(survived) = {p_survived:.3f}')
for b, r in zip(bars, class_rates.values):
    ax.text(b.get_x() + b.get_width()/2, r + 0.01, f'{r:.0%}', ha='center', fontweight='bold')
ax.set_ylabel('P(survived | class)'); ax.set_ylim(0, 0.75)
ax.set_title('Survival rate by passenger class')
ax.legend(); plt.show()

P(survived | Pclass=1) = 0.6296
P(survived | Pclass=2) = 0.4728
P(survived | Pclass=3) = 0.2444

# Bayes: flip P(survived|female) into P(female|survived).
p_female_given_survived_bayes = (p_surv_given_female * p_female) / p_survived

# Direct check: among survivors only, what fraction were female?
survivors = titanic[titanic['Survived'] == 1]
p_female_given_survived_direct = (survivors['Sex'] == 'female').mean()

print(f"P(survived | female)              = {p_surv_given_female:.4f}")
print(f"P(female | survived) via Bayes    = {p_female_given_survived_bayes:.4f}")
print(f"P(female | survived) from counting= {p_female_given_survived_direct:.4f}   <- matches Bayes")
print()
print("Punchline:  P(survived | female) = 0.742  but  P(female | survived) = 0.681.")
print("Same two events, opposite condition, DIFFERENT numbers. Order matters!")

P(survived | female)              = 0.7420
P(female | survived) via Bayes    = 0.6813
P(female | survived) from counting= 0.6813   <- matches Bayes

Punchline:  P(survived | female) = 0.742  but  P(female | survived) = 0.681.
Same two events, opposite condition, DIFFERENT numbers. Order matters!

n_draws = 4000                       # tinker: try 200, or 20000
picks = rng.choice(titanic['Survived'].to_numpy(), size=n_draws, replace=True)
running_freq = np.cumsum(picks) / np.arange(1, n_draws + 1)   # survivors-so-far / draws-so-far

print(f"After    10 draws: running P(survived) = {running_freq[9]:.4f}")
print(f"After   100 draws: running P(survived) = {running_freq[99]:.4f}")
print(f"After  {n_draws} draws: running P(survived) = {running_freq[-1]:.4f}")
print(f"True (whole-group) P(survived)           = {p_survived:.4f}")

fig, ax = plt.subplots()
ax.plot(np.arange(1, n_draws + 1), running_freq, color='#4c72b0', lw=1)
ax.axhline(p_survived, color='#D55E00', ls='--', lw=2,
           label=f'true P(survived) = {p_survived:.3f}')
ax.set_xscale('log')   # log x-axis so the early lurching and late settling both show
ax.set_xlabel('number of random passengers drawn (log scale)')
ax.set_ylabel('survivors so far / draws so far')
ax.set_title('A probability emerging: running survival frequency converges')
ax.legend(); plt.show()

After    10 draws: running P(survived) = 0.6000
After   100 draws: running P(survived) = 0.5100
After  4000 draws: running P(survived) = 0.3880
True (whole-group) P(survived)           = 0.3856

	Survived	Pclass	Name	Sex	Age	Siblings/Spouses Aboard	Fare
0	0	3	Mr. Owen Harris Braund	male	22.0	1	7.2500
1	1	1	Mrs. John Bradley (Florence Briggs Thayer) Cum...	female	38.0	1	71.2833
2	1	3	Miss. Laina Heikkinen	female	26.0	0	7.9250
3	1	1	Mrs. Jacques Heath (Lily May Peel) Futrelle	female	35.0	1	53.1000
4	0	3	Mr. William Henry Allen	male	35.0	0	8.0500

Lesson 5 — Probability & conditional probability¶

1) Probability = favorable ÷ total¶

2) Conditional probability — restrict to a subgroup¶

3) The contingency table: marginal, joint & conditional in one grid¶

4) Independence — when knowing $B$ tells you nothing¶

5) A conditional gradient: survival by passenger class¶

6) A first taste of Bayes — flipping the conditional¶

7) Watching a probability happen: the long-run frequency¶

Now you try¶

What you learned¶

	Died (0)	Survived (1)	Total
Sex
female	0.0913	0.2627	0.354
male	0.5231	0.1229	0.646
Total	0.6144	0.3856	1.000