Need to find all email addresses in a text? Extract IP addresses from log files? Validate that a string is a valid phone number? Regular expressions are the answer. The `re` module gives you the power of regex in Python. In this lesson, you'll learn to write patterns that can search, extract, and validate any text format you encounter.

1. Learning Objectives

By the end of this lesson, you will be able to:

  • Write regex patterns for common text formats
  • Use re.search(), re.match(), re.findall(), re.sub()
  • Extract data with capture groups
  • Parse log files with regex
  • Validate user input formats
  • Optimize regex performance

2. Why This Matters

Real-world scenario: Your log files contain thousands of lines. You need to extract all IP addresses, timestamps, and error codes. Doing this manually is impossible. With regex, one line of code can extract everything you need.

3. Core Concepts

Regex Pattern Basics

import re

# Literal characters
pattern = r"error"
text = "This is an error message"
match = re.search(pattern, text)
print(match.group() if match else None)  # 'error'

# Character classes
# \d - any digit (0-9)
# \w - any word character (a-z, A-Z, 0-9, _)
# \s - any whitespace (space, tab, newline)
# . - any character except newline

# Quantifiers
# * - zero or more
# + - one or more
# ? - zero or one
# {3} - exactly 3
# {2,5} - 2 to 5
# {3,} - 3 or more

# Anchors
# ^ - start of string
# $ - end of string
# \b - word boundary

# Examples
phone_pattern = r"\d{3}-\d{3}-\d{4}"
email_pattern = r"\w+@\w+\.\w+"
ip_pattern = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"
Regex pattern basics

Common Patterns

# Email address
email_regex = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"

# IP address
ip_regex = r"(?:\d{1,3}\.){3}\d{1,3}"

# URL
url_regex = r"https?://(?:[\w-]+\.)+[\w-]+(?:/[\w-./?%&=]*)?"

# Date (YYYY-MM-DD)
date_regex = r"\d{4}-\d{2}-\d{2}"

# Timestamp (2024-01-31 14:30:00)
timestamp_regex = r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}"

# UUID
uuid_regex = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"

# HTML tag
tag_regex = r"<[^>]+>"

# Username (alphanumeric, underscore, 3-20 chars)
username_regex = r"^[a-zA-Z0-9_]{3,20}$"

# Password (at least 8 chars, one uppercase, one lowercase, one digit)
password_regex = r"^(?=.*[a-z])(?=.*[A-Z])(?=.*\d).{8,}$"
Common regex patterns

re Module Functions

import re

text = "The IP 192.168.1.1 is accessing port 8080 at 2024-01-31 14:30:00"

# search() - find first match
match = re.search(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", text)
if match:
    print(f"Found IP: {match.group()}")

# findall() - find all matches
ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", text)
print(f"All IPs: {ips}")

# finditer() - iterator with match objects
for match in re.finditer(r"\d+", text):
    print(f"Number: {match.group()} at position {match.start()}")

# match() - match from start of string
result = re.match(r"The IP", text)
print(f"Matches start: {bool(result)}")

# fullmatch() - match entire string
result = re.fullmatch(r"The IP .*", text)
print(f"Matches full: {bool(result)}")

# sub() - replace
text_sub = re.sub(r"\d{4}-\d{2}-\d{2}", "DATE", text)
print(f"Replaced: {text_sub}")

# split() - split by pattern
parts = re.split(r"\s+", text)
print(f"Split by whitespace: {parts}")
re module functions

Capture Groups

import re

# Basic groups (parentheses)
log_line = "2024-01-31 14:30:00 ERROR Failed to connect to 192.168.1.1:8080"
pattern = r"(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) (\w+) (.+) (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)"

match = re.search(pattern, log_line)
if match:
    print(f"Date: {match.group(1)}")
    print(f"Time: {match.group(2)}")
    print(f"Level: {match.group(3)}")
    print(f"Message: {match.group(4)}")
    print(f"IP: {match.group(5)}")
    print(f"Port: {match.group(6)}")

# Named groups
pattern = r"(?P<date>\d{4}-\d{2}-\d{2}) (?P<time>\d{2}:\d{2}:\d{2}) (?P<level>\w+) (?P<message>.+) (?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d+)"
match = re.search(pattern, log_line)
if match:
    print(f"Date: {match.group('date')}")
    print(f"Level: {match.group('level')}")
    print(f"IP: {match.group('ip')}")

# Non-capturing groups (?:...)
pattern = r"(?:\d{1,3}\.){3}\d{1,3}"  # IP without capturing

# Using groups in substitution
text = "John Doe, 30 years old"
pattern = r"(\w+) (\w+), (\d+) years old"
result = re.sub(pattern, r"Name: \2, \1 | Age: \3", text)
print(result)  # Name: Doe, John | Age: 30
Using capture groups

Flags and Compilation

import re

# Common flags
# re.IGNORECASE (re.I) - case insensitive
# re.MULTILINE (re.M) - ^ and $ match line boundaries
# re.DOTALL (re.S) - . matches newline as well
# re.VERBOSE (re.X) - allow comments and whitespace in pattern

# Case insensitive
pattern = re.compile(r"error", re.IGNORECASE)
text = "This is an ERROR message"
print(pattern.search(text))  # Matches 'ERROR'

# Multiline
pattern = re.compile(r"^\d+", re.MULTILINE)
text = "1. First line\n2. Second line\n3. Third line"
matches = pattern.findall(text)
print(matches)  # ['1', '2', '3']

# Verbose (readable patterns)
pattern = re.compile(r"""
    (?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})  # IP address
    \s+                                          # whitespace
    (?P<timestamp>\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})  # timestamp
    \s+                                          # whitespace
    (?P<level>\w+)                               # log level
""", re.VERBOSE)

# Pre-compile for performance (use same pattern multiple times)
email_regex = re.compile(r"\w+@\w+\.\w+")

for line in large_text_lines:
    emails = email_regex.findall(line)
    process_emails(emails)
Pattern flags and compilation

Lookahead and Lookbehind

import re

# Positive lookahead (?=...) - matches if pattern follows
text = "price: $100, discount: $20, total: $80"
# Find dollar amounts followed by a comma
matches = re.findall(r"\$\d+(?=,)", text)
print(matches)  # ['$100']

# Negative lookahead (?!...) - matches if pattern does NOT follow
matches = re.findall(r"\$\d+(?!,)", text)
print(matches)  # ['$20', '$80']

# Positive lookbehind (?<=...) - matches if pattern precedes
text = "User: alice, User: bob, Guest: charlie"
matches = re.findall(r"(?<=User: )\w+", text)
print(matches)  # ['alice', 'bob']

# Negative lookbehind (?<!...) - matches if pattern does NOT precede
matches = re.findall(r"(?<!User: )\w+", text)
print(matches)  # ['alice,', 'bob,', 'charlie']  # Note the commas

# Combined example: find 'error' not preceded by 'no '
text = "error occurred, no error found, another error"
matches = re.findall(r"(?<!no )error", text)
print(matches)  # ['error', 'error'] (first and third, not the second)
Lookahead and lookbehind assertions

4. Complete Project: Log Parser with Regex

#!/usr/bin/env python3
"""
log_parser.py - Parse log files using regular expressions
"""

import re
import json
from pathlib import Path
from collections import Counter, defaultdict
from datetime import datetime

class LogParser:
    """Parse various log formats with regex"""
    
    # Common log format patterns
    PATTERNS = {
        'nginx': re.compile(
            r'(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(?P<timestamp>[^\]]+)\] "(?P<method>[A-Z]+) (?P<path>[^ ]+) HTTP/\d\.\d" (?P<status>\d{3}) (?P<size>\d+) "(?P<referer>[^"]*)" "(?P<user_agent>[^"]*)"'
        ),
        'apache': re.compile(
            r'(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - (?P<user>[^ ]+) \[(?P<timestamp>[^\]]+)\] "(?P<method>[A-Z]+) (?P<path>[^ ]+) HTTP/\d\.\d" (?P<status>\d{3}) (?P<size>\d+)'
        ),
        'syslog': re.compile(
            r'(?P<timestamp>\w+\s+\d+\s+\d{2}:\d{2}:\d{2}) (?P<hostname>[\w.-]+) (?P<service>[\w\[\.\]/]+): (?P<message>.+)'
        ),
        'error_log': re.compile(
            r'\[(?P<timestamp>\w+\s+\w+\s+\d+\s+\d{2}:\d{2}:\d{2}\.\d+)\] \[(?P<level>[^\]]+)\] \[(?P<pid>\d+)\] (?P<message>.+)'
        ),
        'custom': re.compile(
            r'(?P<date>\d{4}-\d{2}-\d{2}) (?P<time>\d{2}:\d{2}:\d{2}) - (?P<level>\w+) - (?P<message>.+)'
        )
    }
    
    def __init__(self, log_type='nginx'):
        self.pattern = self.PATTERNS.get(log_type, self.PATTERNS['nginx'])
        self.entries = []
        self.stats = defaultdict(Counter)
    
    def parse_line(self, line):
        """Parse a single log line"""
        match = self.pattern.match(line.strip())
        if match:
            entry = match.groupdict()
            self.entries.append(entry)
            
            # Update statistics
            if 'status' in entry:
                self.stats['status_codes'][entry['status']] += 1
            if 'method' in entry:
                self.stats['methods'][entry['method']] += 1
            if 'ip' in entry:
                self.stats['ips'][entry['ip']] += 1
            if 'path' in entry:
                self.stats['paths'][entry['path']] += 1
            if 'level' in entry:
                self.stats['levels'][entry['level']] += 1
            
            return entry
        return None
    
    def parse_file(self, filepath):
        """Parse an entire log file"""
        with open(filepath, 'r') as f:
            for line_num, line in enumerate(f, 1):
                entry = self.parse_line(line)
                if not entry:
                    print(f"Warning: Could not parse line {line_num}")
        
        return self.entries
    
    def get_top(self, key, n=10):
        """Get top N values for a field"""
        if key in self.stats:
            return self.stats[key].most_common(n)
        return []
    
    def filter_by(self, **kwargs):
        """Filter entries by field values"""
        results = self.entries
        for field, value in kwargs.items():
            results = [e for e in results if e.get(field) == value]
        return results
    
    def generate_report(self):
        """Generate analysis report"""
        report = {
            'total_entries': len(self.entries),
            'top_ips': self.get_top('ips', 10),
            'top_paths': self.get_top('paths', 10),
            'status_codes': dict(self.stats.get('status_codes', {})),
            'methods': dict(self.stats.get('methods', {})),
            'error_rate': 0,
            'summary': {}
        }
        
        if 'status_codes' in self.stats:
            total = sum(self.stats['status_codes'].values())
            errors = sum(v for k, v in self.stats['status_codes'].items() if k.startswith('4') or k.startswith('5'))
            report['error_rate'] = (errors / total * 100) if total > 0 else 0
        
        return report
    
    def print_report(self):
        """Print formatted report"""
        report = self.generate_report()
        
        print("\n" + "="*60)
        print("LOG ANALYSIS REPORT")
        print("="*60)
        print(f"Total entries: {report['total_entries']}")
        print(f"Error rate: {report['error_rate']:.2f}%")
        
        print(f"\n📊 STATUS CODES")
        for code, count in sorted(report['status_codes'].items()):
            bar = '█' * (count * 50 // report['total_entries'])
            print(f"   {code}: {count:6} {bar}")
        
        print(f"\n🌐 TOP IPs")
        for ip, count in report['top_ips'][:5]:
            print(f"   {ip:20} {count:6} requests")
        
        print(f"\n📁 TOP PATHS")
        for path, count in report['top_paths'][:5]:
            print(f"   {path[:40]:40} {count:6} requests")
        
        print(f"\n🔄 HTTP METHODS")
        for method, count in report['methods'].items():
            print(f"   {method}: {count}")
        
        print("="*60 + "\n")
    
    def save_json(self, filepath='log_analysis.json'):
        """Save entries to JSON"""
        with open(filepath, 'w') as f:
            json.dump(self.entries, f, indent=2)
        print(f"Saved {len(self.entries)} entries to {filepath}")


def main():
    import argparse
    
    parser = argparse.ArgumentParser(description='Log Parser')
    parser.add_argument('log_file', help='Path to log file')
    parser.add_argument('--type', '-t', choices=['nginx', 'apache', 'syslog', 'error_log', 'custom'], default='nginx', help='Log format type')
    parser.add_argument('--save', '-s', action='store_true', help='Save parsed data')
    parser.add_argument('--output', '-o', default='log_analysis.json', help='Output file')
    parser.add_argument('--status', help='Filter by status code')
    parser.add_argument('--ip', help='Filter by IP address')
    
    args = parser.parse_args()
    
    if not Path(args.log_file).exists():
        print(f"Error: File {args.log_file} not found")
        return
    
    parser_obj = LogParser(args.type)
    parser_obj.parse_file(args.log_file)
    
    if args.status or args.ip:
        filters = {}
        if args.status:
            filters['status'] = args.status
        if args.ip:
            filters['ip'] = args.ip
        filtered = parser_obj.filter_by(**filters)
        print(f"\nFiltered: {len(filtered)} entries")
        for entry in filtered[:10]:
            print(entry)
    else:
        parser_obj.print_report()
    
    if args.save:
        parser_obj.save_json(args.output)


if __name__ == '__main__':
    main()
Complete log parser with regex

5. Common Errors & Solutions

6. Summary Checklist

7. Next Steps

Next lesson: Python Lesson 4.7: Working with YAML & Config Files

Tags: python, regex, re-module, pattern-matching, text-processing

Gataya Med

DevOps Engineer & Backend Developer. Sharing insights on cloud, automation, and scalable systems.

gatayaMed View Profile @https://github.com/gatayaMed

Comments (0)

Sarah Chen January 31, 2025

This is exactly what I needed! The initContainer approach solved our migration issues completely. Thanks for the detailed guide!

Reply

Leave a Comment