distiller¶

def distill(
    self,
    prompt: str,
    paths: Optional[Union[str, Path, List[Path]]] = None,
    *,  # Force keyword-only arguments for clarity
    format: str = "markdown",
    model: Optional[str] = None,
    max_tokens: Optional[int] = None,
    mode: str = "balanced",
    include_git: bool = True,
    session_name: Optional[str] = None,
    include_patterns: Optional[List[str]] = None,
    exclude_patterns: Optional[List[str]] = None,
    full: bool = False,
    condense: bool = False,
    remove_comments: bool = False,
    pinned_files: Optional[List[Path]] = None,
    include_tests: Optional[bool] = None,
    docstring_weight: Optional[float] = None,
    summarize_imports: bool = True,
    timeout: Optional[float] = None,
) -> ContextResult:
    """Distill relevant context from codebase based on prompt.

    This is the main method that extracts, ranks, and aggregates
    the most relevant files and information for a given prompt.

    Args:
        prompt: The user's query or task description
        paths: Paths to analyze (default: current directory)
        format: Output format (markdown, xml, json)
        model: Target LLM model for token counting
        max_tokens: Maximum tokens for context
        mode: Analysis mode (fast, balanced, thorough)
        include_git: Whether to include git context
        session_name: Session name for stateful context
        include_patterns: File patterns to include
        exclude_patterns: File patterns to exclude

    Returns:
        ContextResult with the distilled context

    Example:
        >>> distiller = Distiller(config)
        >>> result = distiller.distill(
        ...     "implement OAuth2 authentication",
        ...     paths="./src",
        ...     mode="thorough",
        ...     max_tokens=50000
        ... )
        >>> print(result.context)
    """
    start_time = time.time()
    deadline = start_time + timeout if timeout and timeout > 0 else None
    timed_out = False

    def _check_timeout(stage: str) -> bool:
        nonlocal timed_out
        if deadline is not None and time.time() >= deadline:
            timed_out = True
            self.logger.warning(f"Distillation timed out during {stage}")
            return True
        return False

    self.logger.info(f"Distilling context for: {prompt[:100]}...")

    # 1. Parse and understand the prompt
    parse_start = time.time()
    prompt_context = self._parse_prompt(prompt)
    self.logger.debug(f"Prompt parsing took {time.time() - parse_start:.2f}s")
    _check_timeout("prompt parsing")

    # Override test inclusion if explicitly specified
    if include_tests is not None:
        prompt_context.include_tests = include_tests
        self.logger.debug(f"Override: test inclusion set to {include_tests}")

    # 2. Determine paths to analyze
    paths = self._normalize_paths(paths)

    # 3. Discover relevant files
    discover_start = time.time()
    files = self._discover_files(
        paths=paths,
        prompt_context=prompt_context,
        include_patterns=include_patterns,
        exclude_patterns=exclude_patterns,
    )
    self.logger.debug(f"File discovery took {time.time() - discover_start:.2f}s")
    _check_timeout("file discovery")

    # 4. Analyze files for structure and content
    # Prepend pinned files (avoid duplicates) while preserving original discovery order
    if pinned_files:
        # Preserve the explicit order given by the caller (tests rely on this)
        # Do NOT filter by existence – tests pass synthetic Paths.
        pinned_strs = [str(p) for p in pinned_files]
        pinned_set = set(pinned_strs)
        ordered: List[Path] = []
        # First, add pinned files (re-using the discovered Path object if present
        # so downstream identity / patch assertions still work).
        discovered_map = {str(f): f for f in files}
        for p_str, p_obj in zip(pinned_strs, pinned_files):
            if p_str in discovered_map:
                f = discovered_map[p_str]
            else:
                f = p_obj  # fallback to provided Path
            if f not in ordered:
                ordered.append(f)
        # Then append remaining discovered files preserving original discovery order.
        for f in files:
            if str(f) not in pinned_set and f not in ordered:
                ordered.append(f)
        files = ordered

    analyzed_files = self._analyze_files(
        files=files, mode=mode, prompt_context=prompt_context, deadline=deadline
    )
    _check_timeout("file analysis")

    # 5. Rank files by relevance
    rank_start = time.time()
    ranked_files = self._rank_files(
        files=analyzed_files,
        prompt_context=prompt_context,
        mode=mode,
        deadline=deadline,
    )
    self.logger.debug(f"File ranking took {time.time() - rank_start:.2f}s")
    _check_timeout("ranking")

    # If we hit timeout before aggregation, return partial context quickly
    if timed_out:
        end_time = time.time()
        duration = end_time - start_time
        partial_files = ranked_files[:10] if ranked_files else []
        context_lines = [
            "Distillation timed out before completion.",
            f"Elapsed: {format_duration(duration)}",
        ]
        if timeout and timeout > 0:
            context_lines[-1] += f" (limit: {int(timeout)}s)"
        if partial_files:
            context_lines.append("")
            context_lines.append("Top files considered:")
            for f in partial_files:
                context_lines.append(f"- {getattr(f, 'path', f)}")

        metadata = {
            "mode": mode,
            "files_analyzed": len(analyzed_files),
            "files_included": len(partial_files),
            "model": model,
            "format": format,
            "session": session_name,
            "prompt": prompt,
            "full_mode": full,
            "condense": condense,
            "remove_comments": remove_comments,
            "included_files": partial_files,
            "total_tokens": 0,
            "timed_out": True,
            "timeout_seconds": timeout,
            "timing": {
                "duration": duration,
                "formatted_duration": format_duration(duration),
                "start_datetime": datetime.fromtimestamp(start_time).isoformat(),
                "end_datetime": datetime.fromtimestamp(end_time).isoformat(),
            },
        }
        return self._build_result(
            formatted="\n".join(context_lines),
            metadata=metadata,
        )

    # 6. Add git context if requested
    git_context = None
    if include_git:
        git_context = self._get_git_context(
            paths=paths, prompt_context=prompt_context, files=ranked_files
        )

    # 7. Aggregate files within token budget
    aggregate_start = time.time()
    aggregated = self._aggregate_files(
        files=ranked_files,
        prompt_context=prompt_context,
        max_tokens=max_tokens or self.config.max_tokens,
        model=model,
        git_context=git_context,
        full=full,
        condense=condense,
        remove_comments=remove_comments,
        docstring_weight=docstring_weight,
        summarize_imports=summarize_imports,
    )
    self.logger.debug(f"File aggregation took {time.time() - aggregate_start:.2f}s")
    _check_timeout("aggregation")

    # 8. Format the output
    formatted = self._format_output(
        aggregated=aggregated,
        format=format,
        prompt_context=prompt_context,
        session_name=session_name,
    )

    # 9. Build final result with debug information
    metadata = {
        "mode": mode,
        "files_analyzed": len(files),
        "files_included": len(aggregated["included_files"]),
        "model": model,
        "format": format,
        "session": session_name,
        "prompt": prompt,
        "full_mode": full,
        "condense": condense,
        "remove_comments": remove_comments,
        # Include the aggregated data for _build_result to use
        "included_files": aggregated["included_files"],
        "total_tokens": aggregated.get("total_tokens", 0),
    }

    # Add debug information for verbose mode
    # Add prompt parsing details
    metadata["prompt_context"] = {
        "task_type": prompt_context.task_type,
        "intent": prompt_context.intent,
        "keywords": prompt_context.keywords,
        "synonyms": getattr(prompt_context, "synonyms", []),
        "entities": prompt_context.entities,
    }

    # Expose NLP normalization metrics if available from parser
    try:
        if (
            isinstance(prompt_context.metadata, dict)
            and "nlp_normalization" in prompt_context.metadata
        ):
            metadata["nlp_normalization"] = prompt_context.metadata["nlp_normalization"]
    except Exception:
        pass

    # Add ranking details
    metadata["ranking_details"] = {
        "algorithm": mode,
        "threshold": self.config.ranking.threshold,
        "files_ranked": len(analyzed_files),
        "files_above_threshold": len(ranked_files),
        "top_files": [
            {
                "path": str(f.path),
                "score": f.relevance_score,
                "match_details": {
                    "keywords_matched": getattr(f, "keywords_matched", []),
                    "semantic_score": getattr(f, "semantic_score", 0),
                },
            }
            for f in ranked_files[:10]  # Top 10 files
        ],
    }

    # Add aggregation details
    metadata["aggregation_details"] = {
        "strategy": aggregated.get("strategy", "unknown"),
        "min_relevance": aggregated.get("min_relevance", 0),
        "files_considered": len(ranked_files),
        "files_rejected": len(ranked_files) - len(aggregated["included_files"]),
        "rejection_reasons": aggregated.get("rejection_reasons", {}),
    }

    end_time = time.time()
    duration = end_time - start_time
    metadata["timed_out"] = timed_out
    metadata["timeout_seconds"] = timeout
    metadata["timing"] = {
        "duration": duration,
        "formatted_duration": format_duration(duration),
        "start_datetime": datetime.fromtimestamp(start_time).isoformat(),
        "end_datetime": datetime.fromtimestamp(end_time).isoformat(),
    }

    return self._build_result(
        formatted=formatted,
        metadata=metadata,
    )