Python Reference

Core language, data structures, OOP, and the libraries that carry the data-engineering and ML workload.

1. Core Syntax

x = 10
name = "Ada"
is_ready = True

if x > 5 and is_ready:
    print(f"{name} is ready, x={x}")
elif x == 5:
    print("exactly five")
else:
    print("not ready")

for i in range(5):
    if i == 3:
        break
    print(i)

i = 0
while i < 3:
    i += 1

2. Data Structures

nums = [3, 1, 4, 1, 5]
nums.append(9); nums.sort()

person = {"name": "Ada", "role": "engineer"}
person["team"] = "data"
for k, v in person.items():
    print(k, v)

unique = set(nums)
coords = (10.5, 20.1)

3. Functions & Comprehensions

def greet(name, greeting="Hello"):
    return f"{greeting}, {name}!"

square = lambda n: n * n

squares = [n * n for n in range(10) if n % 2 == 0]
lookup = {n: n * n for n in range(5)}

def total(*args, **kwargs):
    return sum(args) + sum(kwargs.values())

4. OOP: Classes & Objects

Object-oriented programming (OOP) organises code around objects that bundle data (attributes) with the behaviour that works on that data (methods). A class is the blueprint; an object (instance) is one thing built from it. self is the instance the method was called on.

class BankAccount:
    bank_name = "MiTech Bank"  # class attribute: shared by every instance

    def __init__(self, owner: str, balance: float = 0.0):
        self.owner = owner  # instance attributes: one copy per object
        self.balance = balance

    def deposit(self, amount: float) -> None:
        self.balance += amount

acc = BankAccount("Asha", 100)
acc.deposit(50)
print(acc.balance)                          # 150
print(acc.bank_name, BankAccount.bank_name) # MiTech Bank MiTech Bank

Instance, class and static methods

class Employee:
    raise_pct = 5

    def __init__(self, name: str, salary: float):
        self.name = name
        self.salary = salary

    def apply_raise(self) -> None:  # instance method
        self.salary = round(self.salary * (1 + self.raise_pct / 100), 2)

    @classmethod
    def from_csv(cls, row: str) -> "Employee":  # alternative constructor
        name, salary = row.split(",")
        return cls(name.strip(), float(salary))

    @staticmethod
    def is_valid_salary(value: float) -> bool:  # plain helper
        return value > 0

e = Employee.from_csv("Ben, 1000")
e.apply_raise()
print(e.name, e.salary, Employee.is_valid_salary(-5))  # Ben 1050.0 False

Trap: mutable class attributes are shared

class Team:
    members = []  # ONE list shared by every Team object

    def add(self, name):
        self.members.append(name)

a, b = Team(), Team()
a.add("Asha")
print(b.members)  # ['Asha']: b sees a's member!

class Team:
    def __init__(self):
        self.members = []  # fix: create mutable state per instance in __init__

5. OOP: Encapsulation

Encapsulation means keeping an object's data and the code that changes it together, and only letting outsiders change that data through methods that enforce the rules (invariants). Callers depend on the interface, not the internals, so the internals can change safely.

class Account:
    def __init__(self, owner: str, opening: float = 0):
        self.owner = owner
        self.__balance = opening  # private: change it only through methods

    def deposit(self, amount: float) -> None:
        if amount <= 0:
            raise ValueError("deposit must be positive")
        self.__balance += amount

    def withdraw(self, amount: float) -> None:
        if amount > self.__balance:
            raise ValueError("insufficient funds")
        self.__balance -= amount

    @property
    def balance(self) -> float:  # read-only view of the private field
        return self.__balance

acc = Account("Asha", 100)
acc.withdraw(30)
print(acc.balance)                  # 70

try:
    acc.balance = 1_000_000         # no setter defined
except AttributeError as e:
    print("blocked:", type(e).__name__)  # blocked: AttributeError

print(acc._Account__balance)  # 70: name mangling hides, it does not protect

Validating with a property setter

A @property looks like a plain attribute to callers but runs code. Start with a public attribute, and switch to a property later if you need validation; callers do not change.

class Employee:
    def __init__(self, name: str, salary: float):
        self.name = name
        self.salary = salary  # goes through the setter below, so it is validated too

    @property
    def salary(self) -> float:
        return self._salary

    @salary.setter
    def salary(self, value: float) -> None:
        if value < 0:
            raise ValueError("salary cannot be negative")
        self._salary = value

e = Employee("Chen", 900)
e.salary = 950                          # fine
try:
    Employee("Dia", -1)
except ValueError as err:
    print(err)  # salary cannot be negative

6. OOP: Abstraction

Abstraction means exposing what an object does and hiding how. In Python, an abstract base class (ABC) defines a contract: subclasses must implement its abstract methods, and the ABC itself cannot be instantiated.

from abc import ABC, abstractmethod
import csv, io

class DataSource(ABC):
    @abstractmethod
    def read(self) -> list[dict]:
        """Return all records as dictionaries."""

    def count(self) -> int:  # concrete method every source inherits
        return len(self.read())

class CsvSource(DataSource):
    def __init__(self, text: str):
        self.text = text

    def read(self) -> list[dict]:
        return list(csv.DictReader(io.StringIO(self.text)))

class ApiSource(DataSource):
    def read(self) -> list[dict]:
        return [{"id": "1", "amount": "250"}]  # pretend this called an HTTP API

print(CsvSource("id,amount\n1,100\n2,200\n").count())   # 2
print(ApiSource().count())                               # 1

try:
    DataSource()
except TypeError:
    print("abstract classes cannot be instantiated")

Protocols (structural typing) describe an interface without inheritance: any class with the right methods is accepted by type checkers. This is the typed form of duck typing.

from typing import Protocol

class Writer(Protocol):
    def write(self, rows: list[dict]) -> None: ...

class ConsoleWriter:  # does NOT inherit from Writer
    def write(self, rows: list[dict]) -> None:
        print(f"wrote {len(rows)} rows")

def save(rows: list[dict], writer: Writer) -> None:
    writer.write(rows)

save([{"id": 1}], ConsoleWriter())         # wrote 1 rows

7. OOP: Inheritance

Inheritance lets a child class reuse and extend a parent class: an "is-a" relationship (a Manager is an Employee). The child can add attributes and methods, override parent methods, and call the parent version with super().

class Employee:
    def __init__(self, name: str, salary: float):
        self.name = name
        self.salary = salary

    def annual_cost(self) -> float:
        return self.salary * 12

class Manager(Employee):  # Manager is-an Employee
    def __init__(self, name: str, salary: float, reports: list[str]):
        super().__init__(name, salary)  # reuse the parent's setup
        self.reports = reports

    def annual_cost(self) -> float:  # override, extending the parent's result
        return super().annual_cost() + 5000  # plus a management allowance

m = Manager("Asha", 1000, ["Ben", "Chen"])
print(m.annual_cost())                                # 17000
print(isinstance(m, Employee), issubclass(Manager, Employee))  # True True

Multiple inheritance and the MRO

With several parents, Python decides which method to use with the Method Resolution Order (MRO), computed by C3 linearisation. super() follows the MRO, not just "the parent", which is how the classic diamond calls each class exactly once:

class A:
    def hello(self): return "A"

class B(A):
    def hello(self): return "B -> " + super().hello()

class C(A):
    def hello(self): return "C -> " + super().hello()

class D(B, C):
    def hello(self): return "D -> " + super().hello()

print(D().hello())                       # D -> B -> C -> A
print([k.__name__ for k in D.__mro__])  # ['D', 'B', 'C', 'A', 'object']

Mixins

A mixin is a small class that adds one capability and is meant to be combined with others rather than used alone:

import json

class JsonMixin:
    def to_json(self) -> str:
        return json.dumps(self.__dict__, sort_keys=True)

class Customer(JsonMixin):
    def __init__(self, cid: str, city: str):
        self.cid, self.city = cid, city

print(Customer("C1", "Pune").to_json())  # {"cid": "C1", "city": "Pune"}

Prefer shallow hierarchies. Deep or wide inheritance trees couple classes tightly; if the relationship is "has-a" rather than "is-a", use composition instead (see the next sections).

8. OOP: Polymorphism

Polymorphism means "many forms": the same call works on different types, each responding in its own way. Code written against the shared interface does not need to know which concrete class it has.

Method overriding

class Shape:
    def area(self) -> float:
        raise NotImplementedError

class Rectangle(Shape):
    def __init__(self, w, h): self.w, self.h = w, h
    def area(self): return self.w * self.h

class Circle(Shape):
    def __init__(self, r): self.r = r
    def area(self): return round(3.14159 * self.r ** 2, 2)

for shape in [Rectangle(2, 3), Circle(1)]:
    print(type(shape).__name__, shape.area())  # Rectangle 6 / Circle 3.14

Duck typing

"If it walks like a duck and quacks like a duck, it is a duck." Python checks for the method at call time, not the class, so any object with the right method works, with no shared base class needed:

class S3File:
    def read(self): return "from s3"

class LocalFile:
    def read(self): return "from disk"

def load(f):  # works with anything that has .read()
    return f.read().upper()

print(load(S3File()), load(LocalFile()))   # FROM S3 FROM DISK

Operator overloading with dunder methods

from functools import total_ordering

@total_ordering  # builds <=, >, >= from __eq__ and __lt__
class Money:
    def __init__(self, amount: float, currency: str = "INR"):
        self.amount, self.currency = amount, currency

    def __add__(self, other: "Money") -> "Money":
        if self.currency != other.currency:
            raise ValueError("currency mismatch")
        return Money(self.amount + other.amount, self.currency)

    def __radd__(self, other):        # lets sum() start from 0
        return self if other == 0 else self.__add__(other)

    def __eq__(self, other):
        return (self.amount, self.currency) == (other.amount, other.currency)

    def __lt__(self, other):
        return self.amount < other.amount

    def __repr__(self):
        return f"Money({self.amount}, {self.currency!r})"

total = sum([Money(100), Money(250), Money(50)])
print(total)                          # Money(400, 'INR')
print(Money(100) < Money(250), max(Money(5), Money(9)))  # True Money(9, 'INR')

No overloading by signature: use defaults or singledispatch

Python does not pick a method by argument types like Java (a second def with the same name simply replaces the first). Use default arguments, *args, or functools.singledispatch to dispatch on the type of the first argument:

from functools import singledispatch

@singledispatch
def to_sql(value) -> str:
    raise TypeError(f"unsupported type: {type(value).__name__}")

@to_sql.register
def _(value: str) -> str:
    return "'" + value.replace("'", "''") + "'"  # escape quotes

@to_sql.register
def _(value: float) -> str:
    return repr(value)

@to_sql.register(type(None))
def _(value) -> str:
    return "NULL"

print(to_sql("O'Brien"), to_sql(2.5), to_sql(None))  # 'O''Brien' 2.5 NULL

9. Composition, dataclasses & Dunder Methods

Composition over inheritance

Composition builds objects from other objects ("has-a"): a Pipeline has a source and a sink, rather than being a kind of source. Parts can be swapped at runtime and tested in isolation, and changes do not ripple through a class hierarchy.

dataclasses

@dataclass generates __init__, __repr__ and __eq__ from type-annotated fields, which is ideal for records and configuration objects:

from dataclasses import dataclass, field

@dataclass(frozen=True)  # frozen: immutable and hashable
class Transaction:
    txn_id: str
    amount: float
    channel: str = "UPI"
    tags: tuple[str, ...] = ()

    def __post_init__(self):  # validation after the generated __init__
        if self.amount <= 0:
            raise ValueError("amount must be positive")

@dataclass
class Batch:
    name: str
    txns: list[Transaction] = field(default_factory=list)  # never use [] as a default

t1 = Transaction("T1", 250.0)
print(t1)  # Transaction(txn_id='T1', amount=250.0, channel='UPI', tags=())
print(t1 == Transaction("T1", 250.0))    # True: value equality
b = Batch("2026-10-26"); b.txns.append(t1)
print(len(b.txns), len({t1, t1}))  # 1 1: frozen dataclasses can go in sets

Common dunder (magic) methods

__slots__ = ("a", "b") removes the per-instance __dict__, which saves memory when you create millions of small objects.

10. OOP in a Data Pipeline

All four pillars working together: abstraction (abstract Source, Transform and Sink contracts), inheritance (concrete classes implement them), polymorphism (the pipeline calls read / apply / write without knowing the concrete types), encapsulation (the sink protects its rows) and composition (a Pipeline is assembled from parts).

from abc import ABC, abstractmethod
from dataclasses import dataclass, field

class Source(ABC):
    @abstractmethod
    def read(self) -> list[dict]: ...

class Transform(ABC):
    @abstractmethod
    def apply(self, rows: list[dict]) -> list[dict]: ...

class Sink(ABC):
    @abstractmethod
    def write(self, rows: list[dict]) -> None: ...

class ListSource(Source):
    def __init__(self, rows: list[dict]):
        self._rows = rows
    def read(self) -> list[dict]:
        return [dict(r) for r in self._rows]  # copies: callers can't mutate our data

class DropNullAmounts(Transform):
    def apply(self, rows):
        return [r for r in rows if r.get("amount") is not None]

class AddTax(Transform):
    def __init__(self, rate: float):
        self.rate = rate
    def apply(self, rows):
        return [{**r, "amount_with_tax": round(r["amount"] * (1 + self.rate), 2)}
                for r in rows]

class MemorySink(Sink):
    def __init__(self):
        self.__rows: list[dict] = []  # encapsulated
    def write(self, rows):
        self.__rows.extend(rows)
    @property
    def rows(self) -> tuple[dict, ...]:
        return tuple(self.__rows)  # read-only view

@dataclass
class Pipeline:  # composition: has-a source, transforms, sink
    source: Source
    sink: Sink
    transforms: list[Transform] = field(default_factory=list)

    def run(self) -> int:
        rows = self.source.read()
        for t in self.transforms:  # polymorphism: any Transform works
            rows = t.apply(rows)
        self.sink.write(rows)
        return len(rows)

sink = MemorySink()
pipe = Pipeline(
    source=ListSource([
        {"id": 1, "amount": 100.0},
        {"id": 2, "amount": None},
        {"id": 3, "amount": 50.0},
    ]),
    sink=sink,
    transforms=[DropNullAmounts(), AddTax(0.18)],
)
print(pipe.run())                     # 2
print(sink.rows[0])  # {'id': 1, 'amount': 100.0, 'amount_with_tax': 118.0}

Swapping ListSource for an S3 or JDBC source, or MemorySink for a Delta writer, needs no change to Pipeline. That is the practical payoff of OOP in data engineering, and it makes each part easy to unit-test.

11. OOP Interview Questions

12. Files & Exceptions

with open("data.csv") as f:
    for line in f:
        process(line)

try:
    value = int(input("Enter a number: "))
except ValueError as e:
    print(f"Invalid input: {e}")
finally:
    print("Done")

13. pandas Essentials

import pandas as pd

df = pd.read_csv("orders.csv")
df.head()
df.info()

df[df["amount"] > 100]
df.groupby("region")["amount"].agg(["sum", "mean"])
df["amount_usd"] = df["amount"] * df["fx_rate"]
df.merge(other_df, on="order_id", how="left")
df.to_parquet("orders.parquet")

14. boto3 (AWS SDK)

import boto3

s3 = boto3.client("s3")
s3.upload_file("local.csv", "my-bucket", "data/local.csv")
s3.download_file("my-bucket", "data/local.csv", "local.csv")

for obj in s3.list_objects_v2(Bucket="my-bucket", Prefix="data/")["Contents"]:
    print(obj["Key"], obj["Size"])

15. Decorators, Generators & Context Managers

import time
from contextlib import contextmanager

# Decorator: wraps a function to add behavior
def timed(fn):
    def wrapper(*args, **kwargs):
        start = time.time()
        result = fn(*args, **kwargs)
        print(f"{fn.__name__} took {time.time() - start:.3f}s")
        return result
    return wrapper

@timed
def slow_query():
    time.sleep(1)

# Generator: lazily yields values instead of building a full list
def batches(items, size):
    for i in range(0, len(items), size):
        yield items[i:i + size]

for batch in batches(range(100), 10):
    process(batch)

# Context manager: guarantees setup/teardown even on error
@contextmanager
def timer(label):
    start = time.time()
    yield
    print(f"{label}: {time.time() - start:.3f}s")

with timer("load"):
    load_data()

16. Type Hints

from typing import Optional

def greet(name: str, times: int = 1) -> list[str]:
    return [f"Hello, {name}!"] * times

def find_user(user_id: int) -> Optional[dict]:
    return users.get(user_id)  # None if missing

class Order:
    id: int
    amount: float
    status: str = "pending"

Type hints aren't enforced at runtime — they document intent and let tools like mypy catch mismatches before code ships.

17. Environments & Testing

python -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
pip freeze > requirements.txt

# test_orders.py
import pytest
from orders import total

def test_total_sums_amounts():
    assert total([10, 20, 30]) == 60

def test_total_empty_list():
    assert total([]) == 0

@pytest.fixture
def sample_order():
    return {"id": 1, "amount": 49.99}

def test_order_amount(sample_order):
    assert sample_order["amount"] > 0

A virtual environment (venv) isolates a project's dependencies from the system Python and from other projects; pytest discovers any function named test_* and runs it, with fixtures supplying reusable setup.