SessionManager

Manages multiple session instances for Scrapling spiders

The SessionManager class manages pre-configured session instances for spiders. It allows you to register multiple sessions (FetcherSession, AsyncStealthySession, etc.) and route requests to specific sessions using session IDs.

Class Definition

from scrapling.spiders.session import SessionManager

class SessionManager:
    """Manages pre-configured session instances."""

Constructor

def __init__(self) -> None

Creates an empty session manager. Sessions must be added via the add() method.

Methods

add

def add(
    self,
    session_id: str,
    session: Session,
    *,
    default: bool = False,
    lazy: bool = False
) -> SessionManager

Register a session instance.

session_id · str · required

Unique name to reference this session in requests.

session · Session · required

Pre-configured session instance (FetcherSession, AsyncStealthySession, or AsyncDynamicSession).

default · bool · default: "False"

If True, this becomes the default session. The first session added is automatically the default.

lazy · bool · default: "False"

If True, the session will only be started when a request uses its ID (useful for expensive sessions like browser-based ones).

Returns: self (for chaining)

Raises: ValueError if session_id already exists

Example:

from scrapling.spiders.session import SessionManager
from scrapling.fetchers import FetcherSession, AsyncStealthySession

manager = SessionManager()

# Add default session
manager.add("default", FetcherSession())

# Add stealth session (lazy loading)
manager.add("stealth", AsyncStealthySession(), lazy=True)

# Chain multiple adds
manager.add("api", FetcherSession()).add("browser", AsyncDynamicSession(), lazy=True)

remove

def remove(self, session_id: str) -> None

Remove a session by ID.

session_id · str · required

ID of the session to remove.

Raises: KeyError if session_id not found

pop

def pop(self, session_id: str) -> Session

Remove and return a session.

session_id · str · required

ID of the session to remove and return.

Returns: The removed session instance

Raises: KeyError if session_id not found

Example:

# Remove and close a session
old_session = manager.pop("outdated")
await old_session.__aexit__(None, None, None)

get

def get(self, session_id: str) -> Session

Get a session by ID.

session_id · str · required

ID of the session to retrieve.

Returns: The session instance

Raises: KeyError if session_id not found (with helpful error message listing available sessions)

Example:

session = manager.get("stealth")
# Use session directly if needed

fetch

async def fetch(self, request: Request) -> Response

Fetch a request using the appropriate session. Automatically starts lazy sessions on first use.

request · Request · required

The request to fetch.

Returns: Response object with merged metadata from the request

Process:

1. Determine session ID from request (or use default)
2. Get the session instance
3. Start lazy session if needed (thread-safe)
4. Fetch using the session
5. Merge request.meta into response.meta
6. Attach request to response.request

start

async def start(self) -> None

Start all non-lazy sessions. Called automatically by the async context manager.

Example:

manager = SessionManager()
manager.add("default", FetcherSession())
manager.add("lazy", AsyncStealthySession(), lazy=True)

await manager.start()  # Only starts "default", not "lazy"

close

async def close(self) -> None

Close all registered sessions. Called automatically by the async context manager.

Properties

default_session_id

@property
def default_session_id(self) -> str

Get the ID of the default session.

Returns: Default session ID

Raises: RuntimeError if no sessions are registered

session_ids

@property
def session_ids(self) -> list[str]

Get list of all registered session IDs.

Returns: List of session ID strings

Special Methods

Async Context Manager

async def __aenter__(self) -> SessionManager
async def __aexit__(self, *exc) -> None

Supports async context manager protocol for automatic start/close.

Example:

manager = SessionManager()
manager.add("default", FetcherSession())

async with manager:
    # Sessions are started
    response = await manager.fetch(request)
    # Sessions are closed on exit

Contains

def __contains__(self, session_id: str) -> bool

Check if a session ID is registered.

Example:

if "stealth" in manager:
    print("Stealth session available")

Length

def __len__(self) -> int

Get the number of registered sessions.

Example:

print(f"Manager has {len(manager)} sessions")

Usage Examples

Basic Setup in Spider

from scrapling.spiders import Spider
from scrapling.fetchers import FetcherSession

class MySpider(Spider):
    name = "example"
    
    def configure_sessions(self, manager):
        # The manager parameter is a SessionManager instance
        manager.add("default", FetcherSession())

Multiple Sessions

from scrapling.spiders import Spider
from scrapling.fetchers import FetcherSession, AsyncStealthySession, AsyncDynamicSession

class MySpider(Spider):
    name = "multi_session"
    
    def configure_sessions(self, manager):
        # Fast session for APIs
        manager.add("api", FetcherSession())
        
        # Stealth session for protected pages (lazy load)
        manager.add("stealth", AsyncStealthySession(), lazy=True)
        
        # Browser session for JS-heavy pages (lazy load)
        manager.add("browser", AsyncDynamicSession(), lazy=True)
    
    async def parse(self, response):
        # Use different sessions for different requests
        yield Request(
            "https://api.example.com/data",
            sid="api",
            callback=self.parse_api
        )
        
        yield Request(
            "https://protected.example.com",
            sid="stealth",
            callback=self.parse_stealth
        )
        
        yield Request(
            "https://spa.example.com",
            sid="browser",
            callback=self.parse_spa
        )

Session-Specific Configuration

def configure_sessions(self, manager):
    # Configure HTTP session with custom settings
    http_session = FetcherSession(
        headers={"User-Agent": "MyBot/1.0"},
        follow_redirects=True,
        timeout=30
    )
    manager.add("http", http_session)
    
    # Configure stealth session with custom fingerprint
    stealth_session = AsyncStealthySession(
        fingerprint={
            "os": "windows",
            "browser": "chrome"
        }
    )
    manager.add("stealth", stealth_session, lazy=True)

Dynamic Session Switching

class MySpider(Spider):
    def configure_sessions(self, manager):
        manager.add("fast", FetcherSession())
        manager.add("slow", AsyncStealthySession(), lazy=True)
    
    async def parse(self, response):
        for url in response.css("a::attr(href)").getall():
            # Use fast session for most pages
            sid = "fast"
            
            # Switch to slow session for specific patterns
            if "protected" in url or "login" in url:
                sid = "slow"
            
            yield Request(response.urljoin(url), sid=sid)

Manual Session Management (Advanced)

import anyio
from scrapling.spiders.session import SessionManager
from scrapling.fetchers import FetcherSession
from scrapling.spiders import Request

async def main():
    manager = SessionManager()
    manager.add("default", FetcherSession())
    
    async with manager:
        # Create request
        request = Request("https://example.com")
        request.sid = "default"
        request.update_fingerprint()
        
        # Fetch manually
        response = await manager.fetch(request)
        print(response.text)

anyio.run(main)

Lazy Loading Benefits

class MySpider(Spider):
    def configure_sessions(self, manager):
        # Regular session - starts immediately
        manager.add("fast", FetcherSession())
        
        # Lazy session - only starts when first used
        # This saves resources if the session is never needed
        manager.add("heavy", AsyncDynamicSession(), lazy=True)
    
    async def parse(self, response):
        # If we never yield a request with sid="heavy",
        # that browser session never starts!
        
        if response.css(".needs-js"):
            # Browser session starts here on first use
            yield Request(
                response.url,
                sid="heavy",
                callback=self.parse_js
            )

Session Types

The manager supports these session types:

FetcherSession

from scrapling.fetchers import FetcherSession

# HTTP client for static content
session = FetcherSession(
    headers={"User-Agent": "MyBot"},
    timeout=30,
    follow_redirects=True
)
manager.add("http", session)

AsyncStealthySession

from scrapling.fetchers import AsyncStealthySession

# Stealth browser session
session = AsyncStealthySession(
    headless=True,
    fingerprint={"os": "windows", "browser": "chrome"}
)
manager.add("stealth", session, lazy=True)

AsyncDynamicSession

from scrapling.fetchers import AsyncDynamicSession

# Full browser session for JS rendering
session = AsyncDynamicSession(
    headless=True,
    auto_scroll=True
)
manager.add("browser", session, lazy=True)

Thread Safety

Lazy session initialization is thread-safe via async lock:

# Multiple concurrent requests can safely use the same lazy session
# Only one will initialize it
async with self._lazy_lock:
    if not session._is_alive:
        await session.__aenter__()

Error Handling

try:
    session = manager.get("nonexistent")
except KeyError as e:
    # Error message includes available sessions:
    # "Session 'nonexistent' not found. Available: default, stealth, browser"
    print(e)

try:
    manager.add("default", FetcherSession())
    manager.add("default", FetcherSession())  # Duplicate!
except ValueError as e:
    # "Session 'default' already registered"
    print(e)

See Also

• Spider - Using sessions in spiders
• Request - Specifying session IDs in requests
• FetcherSession - HTTP session documentation
• AsyncStealthySession - Stealth browser session