Pandas Processing Large Files (chunksize)

When processing large data files, memory may not be sufficient to load all data at once. Pandas provides chunked reading functionality to process data in batches.

Read CSV in chunks

chunksize parameter

Example

import pandas as pd
import io

# Simulate large file content (replace with file path in actual use)
data = """id,value
1,100
2,200
3,300
4,400
5,500
6,600
7,700
8,800
9,900
10,1000
"""

# Use chunksize to read in chunks
chunks = pd.read_csv(io.StringIO(data), chunksize=3)

print("Chunk processing:")
for i, chunk in enumerate(chunks):
    print(f"\n"Chunk {i+1}:")
    print(chunk)

Aggregation processing

Example

import pandas as pd
import io
import numpy as np

# Simulate large file
data = "value\n" + "\n".join([str(i) for i in range(1, 1001)])

# Calculate sum by chunk
total = 0
count = 0

for chunk in pd.read_csv(io.StringIO(data), chunksize=100):
    total += chunk["value"].sum()
    count += len(chunk)

print(f"Total rows: {count}")
print(f"Sum: {total}")
print(f"Average: {total / count}")

Incremental processing

Filtering

Example

import pandas as pd
import io

# Simulate data
data = """id,value,category
1,100,A
2,200,B
3,150,A
4,300,C
5,250,B
6,180,A
"""

# Filter data with specific conditions
filtered_chunks = []

for chunk in pd.read_csv(io.StringIO(data), chunksize=2):
    filtered = chunk[chunk["category"] == "A"]
    if len(filtered) > 0:
        filtered_chunks.append(filtered)

result = pd.concat(filtered_chunks)
print("Filtering category='A' data:")
print(result)

Multiprocessing parallel processing

Example

import pandas as pd
import io
from concurrent.futures import ProcessPoolExecutor

# Parallel processing (suitable for CPU-intensive tasks)
def process_chunk(chunk):
    """Process a single chunk"""
    return chunk["value"].sum()

# Simulate data
data = "value\n" + "\n".join([str(i) for i in range(1, 101)])

# Prepare data chunks
chunks = list(pd.read_csv(io.StringIO(data), chunksize=10))

# Serial processing
total = sum(process_chunk(chunk) for chunk in chunks)
print(f"Serial processing sum: {total}")

Processing large JSON files

Example

import pandas as pd
import json
import io

# Simulate JSON Lines format data
jsonl_data = '\n'.join([
    json.dumps({"id": i, "value": i * 10})
    for i in range(1, 101)
])

# Read JSON Lines in chunks
chunks = []
chunk_size = 20

for chunk in pd.read_json(io.StringIO(jsonl_data), lines=True, chunksize=chunk_size):
    chunks.append(chunk)
    if len(chunks) >= 3:  # Only process the first 3 chunks as an example
        break

result = pd.concat(chunks)
print(f"Read {len(result)} rows of data")
print(result.head())

Chunked processing can significantly reduce memory usage, but it will increase processing time. Choose a reasonable chunk size to balance memory and time.

Other extensions