feat(deep_crawling): allow configuring batch_size and dispatcher

crawl4ai/deep_crawling/bff_strategy.py

@@ -10,12 +10,18 @@
 from .scorers import URLScorer
 from . import DeepCrawlStrategy
 
-from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+from ..types import (
+    AsyncWebCrawler,
+    BaseDispatcher,
+    CrawlerRunConfig,
+    CrawlResult,
+    RunManyReturn,
+)
 from ..utils import normalize_url_for_deep_crawl
 
 from math import inf as infinity
 
-# Configurable batch size for processing items from the priority queue
+# Default batch size for processing items from the priority queue
 BATCH_SIZE = 10
 
 
@@ -47,13 +53,20 @@ def __init__(
         on_state_change: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
         # Optional cancellation callback - checked before each URL is processed
         should_cancel: Optional[Callable[[], Union[bool, Awaitable[bool]]]] = None,
+        # Number of items pulled from the priority queue per round and handed
+        # to arun_many() at once. Defaults to 10.
+        batch_size: int = BATCH_SIZE,
+        # Optional dispatcher forwarded to arun_many() for each batch
+        dispatcher: Optional[BaseDispatcher] = None,
     ):
         self.max_depth = max_depth
         self.filter_chain = filter_chain
         self.url_scorer = url_scorer
         self.include_external = include_external
         self.score_threshold = score_threshold
         self.max_pages = max_pages
+        self.batch_size = batch_size
+        self.dispatcher = dispatcher
         # self.logger = logger or logging.getLogger(__name__)
         # Ensure logger is always a Logger instance, not a dict from serialization
         if isinstance(logger, logging.Logger):
@@ -253,15 +266,16 @@ async def _arun_best_first(
 
             # Calculate how many more URLs we can process in this batch
             remaining = self.max_pages - self._pages_crawled
-            batch_size = min(BATCH_SIZE, remaining)
-            if batch_size <= 0:
+            effective_batch_size = min(self.batch_size, remaining)
+            if effective_batch_size <= 0:
                 # No more pages to crawl
                 self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
                 break
-                
+
             batch: List[Tuple[float, int, str, Optional[str]]] = []
-            # Retrieve up to BATCH_SIZE items from the priority queue.
-            for _ in range(BATCH_SIZE):
+            # Retrieve up to effective_batch_size items from the priority queue,
+            # so the final round never fetches past the max_pages budget.
+            for _ in range(effective_batch_size):
                 if queue.empty():
                     break
                 item = await queue.get()
@@ -286,7 +300,12 @@ async def _arun_best_first(
             # make subsequent queue ordering depend on network timing.
             urls = [item[2] for item in batch]
             batch_config = config.clone(deep_crawl_strategy=None, stream=True)
-            stream_gen = await crawler.arun_many(urls=urls, config=batch_config)
+            arun_many_kwargs = (
+                {"dispatcher": self.dispatcher} if self.dispatcher is not None else {}
+            )
+            stream_gen = await crawler.arun_many(
+                urls=urls, config=batch_config, **arun_many_kwargs
+            )
             results_by_url: Dict[str, CrawlResult] = {}
             async for result in stream_gen:
                 results_by_url[result.url] = result

crawl4ai/deep_crawling/bfs_strategy.py

@@ -8,8 +8,8 @@
 from ..models import TraversalStats
 from .filters import FilterChain
 from .scorers import URLScorer
-from . import DeepCrawlStrategy  
-from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
+from . import DeepCrawlStrategy
+from ..types import AsyncWebCrawler, BaseDispatcher, CrawlerRunConfig, CrawlResult
 from ..utils import normalize_url_for_deep_crawl, efficient_normalize_url_for_deep_crawl
 from math import inf as infinity
 
@@ -36,13 +36,16 @@ def __init__(
         on_state_change: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
         # Optional cancellation callback - checked before each URL is processed
         should_cancel: Optional[Callable[[], Union[bool, Awaitable[bool]]]] = None,
+        # Optional dispatcher forwarded to arun_many() for each level
+        dispatcher: Optional[BaseDispatcher] = None,
     ):
         self.max_depth = max_depth
         self.filter_chain = filter_chain
         self.url_scorer = url_scorer
         self.include_external = include_external
         self.score_threshold = score_threshold
         self.max_pages = max_pages
+        self.dispatcher = dispatcher
         # self.logger = logger or logging.getLogger(__name__)
         # Ensure logger is always a Logger instance, not a dict from serialization
         if isinstance(logger, logging.Logger):
@@ -260,7 +263,15 @@ async def _arun_batch(
 
             # Clone the config to disable deep crawling recursion and enforce batch mode.
             batch_config = config.clone(deep_crawl_strategy=None, stream=False)
-            batch_results = await crawler.arun_many(urls=urls, config=batch_config)
+            # Only pass `dispatcher` when explicitly set, so the call shape is
+            # unchanged (and test doubles built against the old signature keep
+            # working) for the common case of relying on arun_many()'s own default.
+            arun_many_kwargs = (
+                {"dispatcher": self.dispatcher} if self.dispatcher is not None else {}
+            )
+            batch_results = await crawler.arun_many(
+                urls=urls, config=batch_config, **arun_many_kwargs
+            )
 
             for result in batch_results:
                 url = result.url
@@ -360,8 +371,13 @@ async def _arun_stream(
             visited.update(urls)
 
             stream_config = config.clone(deep_crawl_strategy=None, stream=True)
-            stream_gen = await crawler.arun_many(urls=urls, config=stream_config)
-
+            arun_many_kwargs = (
+                {"dispatcher": self.dispatcher} if self.dispatcher is not None else {}
+            )
+            stream_gen = await crawler.arun_many(
+                urls=urls, config=stream_config, **arun_many_kwargs
+            )
+
             # Keep track of processed results for this batch
             results_count = 0
             async for result in stream_gen:

crawl4ai/deep_crawling/dfs_strategy.py

@@ -76,7 +76,12 @@ async def _arun_batch(
 
             # Clone config to disable recursive deep crawling.
             batch_config = config.clone(deep_crawl_strategy=None, stream=False)
-            url_results = await crawler.arun_many(urls=[url], config=batch_config)
+            arun_many_kwargs = (
+                {"dispatcher": self.dispatcher} if self.dispatcher is not None else {}
+            )
+            url_results = await crawler.arun_many(
+                urls=[url], config=batch_config, **arun_many_kwargs
+            )
 
             for result in url_results:
                 result.metadata = result.metadata or {}
@@ -185,7 +190,12 @@ async def _arun_stream(
             visited.add(url)
 
             stream_config = config.clone(deep_crawl_strategy=None, stream=True)
-            stream_gen = await crawler.arun_many(urls=[url], config=stream_config)
+            arun_many_kwargs = (
+                {"dispatcher": self.dispatcher} if self.dispatcher is not None else {}
+            )
+            stream_gen = await crawler.arun_many(
+                urls=[url], config=stream_config, **arun_many_kwargs
+            )
             async for result in stream_gen:
                 result.metadata = result.metadata or {}
                 result.metadata["depth"] = depth

docs/md_v2/core/deep-crawling.md

@@ -87,6 +87,7 @@ strategy = BFSDeepCrawlStrategy(
 - **`score_threshold`**: Minimum score for URLs to be crawled (default: -inf)
 - **`filter_chain`**: FilterChain instance for URL filtering
 - **`url_scorer`**: Scorer instance for evaluating URLs
+- **`dispatcher`**: Dispatcher forwarded to `arun_many()` for each batch (optional, see [8.3](#83-using-batch_size-and-dispatcher))
 
 ### 2.2 DFSDeepCrawlStrategy (Depth-First Search)
 
@@ -111,6 +112,7 @@ strategy = DFSDeepCrawlStrategy(
 - **`score_threshold`**: Minimum score for URLs to be crawled (default: -inf)
 - **`filter_chain`**: FilterChain instance for URL filtering
 - **`url_scorer`**: Scorer instance for evaluating URLs
+- **`dispatcher`**: Dispatcher forwarded to `arun_many()` for each batch (optional, see [8.3](#83-using-batch_size-and-dispatcher))
 
 ### 2.3 BestFirstCrawlingStrategy (⭐️ - Recommended Deep crawl strategy)
 
@@ -141,6 +143,7 @@ This crawling approach:
 - Helps focus crawl resources on the most relevant content
 - Can limit total pages crawled with `max_pages`
 - Does not need `score_threshold` as it naturally prioritizes by score
+- Crawls URLs in rounds of `batch_size` (default: 10), see [8.3](#83-using-batch_size-and-dispatcher)
 
 ---
 
@@ -462,6 +465,36 @@ strategy = DFSDeepCrawlStrategy(
 
 Note that for BestFirstCrawlingStrategy, score_threshold is not needed since pages are already processed in order of highest score first.
 
+### 8.3 Using batch_size and dispatcher
+
+Deep crawl strategies hand URLs to `arun_many()` in batches: BFS one level at a time, DFS one URL at a time, and BestFirst in rounds of `batch_size` URLs. You can pass a `dispatcher` to any strategy to control how each batch is crawled, for example to rate-limit requests to a site:
+
+```python
+from crawl4ai import RateLimiter
+from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher
+
+dispatcher = MemoryAdaptiveDispatcher(
+    max_session_permit=5,      # At most 5 concurrent pages
+    rate_limiter=RateLimiter(  # Back off on 429/503 responses
+        base_delay=(1.0, 2.0),
+        max_delay=30.0,
+        max_retries=2
+    ),
+)
+
+strategy = BestFirstCrawlingStrategy(
+    max_depth=2,
+    url_scorer=scorer,
+    max_pages=50,
+    batch_size=5,              # URLs pulled from the priority queue per round (default: 10)
+    dispatcher=dispatcher,     # Forwarded to arun_many() for every batch (optional)
+)
+```
+
+Note that `arun_many()` runs once per batch, so the dispatcher is started and stopped around every batch rather than once for the whole crawl. The rate limiter's per-domain delays carry over between batches, but the dispatcher's memory monitoring restarts each batch, and an attached `CrawlerMonitor` resets its timer and terminal display each batch.
+
+See [Advanced Multi-URL Crawling](../advanced/multi-url-crawling.md) for all dispatcher options.
+
 ## 9. Common Pitfalls & Tips
 
 1.**Set realistic limits.** Be cautious with `max_depth` values > 3, which can exponentially increase crawl size. Use `max_pages` to set hard limits.