Exploratory Data Analysis (EDA)

Exploratory Data Analysis, introduced by John Tukey in 1977, is the approach of analysing datasets to summarise their main characteristics — mainly through visual methods — before formal modelling or hypothesis testing.

Tukey's principle: "The greatest value of a picture is when it forces us to notice what we never expected to see."

Objectives of EDA

1. Understand the structure and quality of the data
2. Discover patterns, trends and relationships
3. Detect outliers and anomalies
4. Check the assumptions required by planned models (normality, linearity, independence)
5. Identify the most important variables
6. Generate hypotheses worth testing formally
7. Guide feature engineering and model selection

EDA vs Confirmatory Data Analysis (CDA)

Types of EDA

Step 1 — Understanding the Structure

import pandas as pd
import numpy as np
import seaborn as sns

df = sns.load_dataset("tips")     # a standard practice dataset

print(df.shape)          # (244, 7)  -> 244 rows, 7 columns
print(df.head())
print(df.info())         # dtypes, non-null counts, memory usage
print(df.columns.tolist())
print(df.dtypes)
print(df.isnull().sum())         # missing values per column
print(df.duplicated().sum())     # duplicate rows
print(df.nunique())              # distinct values per column — spots ID/constant columns

Step 2 — Univariate Analysis

# NUMERIC variables
print(df.describe().round(2))
#        total_bill     tip    size
# count      244.00  244.00  244.00
# mean        19.79    3.00    2.57
# std          8.90    1.38    0.95
# min          3.07    1.00    1.00
# 25%         13.35    2.00    2.00
# 50%         17.80    2.90    2.00
# 75%         24.13    3.56    3.00
# max         50.81   10.00    6.00

print("Skewness:\n", df[["total_bill", "tip"]].skew().round(3))
# total_bill  1.133   -> right-skewed: a few very large bills
# tip         1.465

# CATEGORICAL variables
print(df.describe(include="object"))
for col in ["sex", "smoker", "day", "time"]:
    print(f"\n{col}:")
    print(df[col].value_counts())
    print((df[col].value_counts(normalize=True) * 100).round(1))

import matplotlib.pyplot as plt

fig, axes = plt.subplots(2, 2, figsize=(12, 8))

df["total_bill"].hist(bins=25, ax=axes[0, 0], edgecolor="black")
axes[0, 0].set_title("Distribution of Total Bill")

df.boxplot(column="total_bill", ax=axes[0, 1])
axes[0, 1].set_title("Total Bill — Outliers")

df["day"].value_counts().plot(kind="bar", ax=axes[1, 0], color="teal")
axes[1, 0].set_title("Count by Day")

sns.kdeplot(data=df, x="tip", ax=axes[1, 1], fill=True)
axes[1, 1].set_title("Tip — Density")

plt.tight_layout()
plt.show()

Step 3 — Bivariate Analysis

# Numeric vs Numeric
print("Correlation:\n", df[["total_bill", "tip", "size"]].corr().round(3))
#             total_bill    tip   size
# total_bill       1.000  0.676  0.598
# tip              0.676  1.000  0.489
# size             0.598  0.489  1.000

# Numeric vs Categorical — group-wise summary
print(df.groupby("day")["total_bill"].agg(["count", "mean", "median", "std"]).round(2))
print(df.groupby(["sex", "smoker"])["tip"].mean().round(2))

# Categorical vs Categorical — cross-tabulation
print(pd.crosstab(df["day"], df["time"]))
print(pd.crosstab(df["sex"], df["smoker"], normalize="index").round(3))

fig, axes = plt.subplots(1, 3, figsize=(16, 4))

sns.scatterplot(data=df, x="total_bill", y="tip", hue="time", ax=axes[0])
axes[0].set_title("Tip vs Total Bill")

sns.boxplot(data=df, x="day", y="total_bill", ax=axes[1])
axes[1].set_title("Bill Distribution by Day")

sns.barplot(data=df, x="sex", y="tip", hue="smoker", ax=axes[2])
axes[2].set_title("Average Tip by Sex and Smoker")

plt.tight_layout()
plt.show()

Step 4 — Multivariate Analysis

# Pair plot — every numeric pair at once, with a categorical colour dimension
sns.pairplot(df, vars=["total_bill", "tip", "size"], hue="time", diag_kind="kde")
plt.show()

# Correlation heatmap
plt.figure(figsize=(6, 5))
sns.heatmap(df[["total_bill", "tip", "size"]].corr(), annot=True,
            cmap="coolwarm", center=0, fmt=".2f", square=True)
plt.title("Correlation Heatmap")
plt.show()

# Faceting — the same relationship split across categories
g = sns.FacetGrid(df, col="time", row="smoker", height=3.5)
g.map_dataframe(sns.scatterplot, x="total_bill", y="tip")
plt.show()

Step 5 — Anomaly and Missing-Value Patterns

def find_outliers(series):
    Q1, Q3 = series.quantile(0.25), series.quantile(0.75)
    IQR = Q3 - Q1
    low, high = Q1 - 1.5 * IQR, Q3 + 1.5 * IQR
    out = series[(series < low) | (series > high)]
    return len(out), round(len(out) / len(series) * 100, 2), out.tolist()[:5]

for col in ["total_bill", "tip", "size"]:
    n, pct, sample = find_outliers(df[col])
    print(f"{col:12} {n:3d} outliers ({pct}%)  e.g. {sample}")
# total_bill    9 outliers (3.69%)  e.g. [48.27, 48.17, 50.81, 45.35, 40.55]
# tip           9 outliers (3.69%)
# size          9 outliers (3.69%)

Step 6 — Feature Engineering Discovered Through EDA

# EDA showed tip rises with bill — is the RATE constant?
df["tip_pct"] = (df["tip"] / df["total_bill"] * 100).round(2)

print(df["tip_pct"].describe().round(2))
print(df.groupby("day")["tip_pct"].mean().round(2))
print(df.groupby("size")["tip_pct"].mean().round(2))
# size
# 1    21.73     <- solo diners tip the highest percentage
# 2    16.57
# 3    15.22
# 4    14.59
# 5    14.15
# 6    15.62
# INSIGHT: tip PERCENTAGE falls as party size grows — invisible in raw tip amounts

That last finding is the essence of EDA: the raw correlation said "bigger bills get bigger tips" (obvious), but a derived feature revealed the genuinely actionable pattern.

An EDA Checklist

Automated EDA Tools

# Generate a full EDA report in one line
# from ydata_profiling import ProfileReport
# ProfileReport(df, title="Tips EDA").to_file("eda_report.html")

# Other options: sweetviz, dtale, autoviz

These are useful for a first pass, but they never replace domain-driven exploration — a tool cannot know that "tip percentage" is the metric that actually matters to a restaurant.

The next three lessons dissect the three workhorse EDA charts individually: histograms, box plots and scatter plots.