diff --git a/scripts/issue_classifier/.gitignore b/scripts/issue_classifier/.gitignore new file mode 100644 index 00000000000..aed612bb0f5 --- /dev/null +++ b/scripts/issue_classifier/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +.venv/ +data/*.jsonl +data/*.json +!data/.gitkeep diff --git a/scripts/issue_classifier/README.md b/scripts/issue_classifier/README.md new file mode 100644 index 00000000000..3313f368bab --- /dev/null +++ b/scripts/issue_classifier/README.md @@ -0,0 +1,69 @@ +# GitHub Issue Classification System + +Classify and label GitHub issues in the vaadin/flow repository using LLM-assisted analysis. + +## Quick Start + +```bash +# 1. Set up the environment +cd scripts/issue_classifier +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt + +# 2. Set your Anthropic API key +export ANTHROPIC_API_KEY="your-key-here" + +# 3. Ensure gh CLI is authenticated +gh auth login + +# 4. Run the workflow +cd scripts # Must run from scripts/ directory +python -m issue_classifier fetch # Fetch all issues +python -m issue_classifier classify --all # Classify with LLM +python -m issue_classifier review # Review classifications +python -m issue_classifier apply --dry-run # Preview changes +python -m issue_classifier apply # Apply labels +``` + +## Commands + +| Command | Description | +|---------|-------------| +| `fetch` | Fetch all open issues from GitHub | +| `classify` | Classify issues using LLM | +| `resume` | Resume classification from checkpoint | +| `review` | Interactive review of classifications | +| `apply` | Apply approved label changes | +| `report` | Generate summary reports | +| `stats` | Show quick statistics | +| `show ` | Show details for a specific issue | + +## Workflow + +1. **Fetch** - Download all open issues to `data/issues/issues.jsonl` +2. **Classify** - Use Claude to analyze and classify each issue +3. **Review** - Manually approve/modify each classification (required) +4. **Apply** - Apply approved labels to GitHub + +## Data Files + +- `data/issues/issues.jsonl` - All issues in JSON Lines format +- `data/issues/issues_meta.json` - Progress metadata +- `data/issues/audit_log.jsonl` - Record of all label changes + +## Classification Schema + +Each issue is classified with: + +- **Type**: `bug`, `enhancement`, or `feature request` +- **Impact** (bugs only): `High` or `Low` +- **Severity** (bugs only): `Major` or `Minor` +- **Modules**: Which parts of the codebase are affected +- **Good First Issue**: Whether suitable for new contributors + +## Requirements + +- Python 3.9+ +- `gh` CLI authenticated (`gh auth login`) +- `ANTHROPIC_API_KEY` environment variable diff --git a/scripts/issue_classifier/__init__.py b/scripts/issue_classifier/__init__.py new file mode 100644 index 00000000000..1cc697f1686 --- /dev/null +++ b/scripts/issue_classifier/__init__.py @@ -0,0 +1,3 @@ +"""GitHub Issue Classification System for vaadin/flow.""" + +__version__ = "0.1.0" diff --git a/scripts/issue_classifier/__main__.py b/scripts/issue_classifier/__main__.py new file mode 100644 index 00000000000..c0db282810e --- /dev/null +++ b/scripts/issue_classifier/__main__.py @@ -0,0 +1,6 @@ +"""Allow running as `python -m issue_classifier`.""" + +from .cli import main + +if __name__ == "__main__": + main() diff --git a/scripts/issue_classifier/apply.py b/scripts/issue_classifier/apply.py new file mode 100644 index 00000000000..d128d8cf831 --- /dev/null +++ b/scripts/issue_classifier/apply.py @@ -0,0 +1,312 @@ +"""Batch label updates for GitHub issues.""" + +import json +import subprocess +import time +from datetime import datetime, timezone + +from rich.console import Console +from rich.panel import Panel +from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn +from rich.table import Table + +from .config import ( + APPLY_BATCH_DELAY_SECONDS, + APPLY_BATCH_SIZE, + AUDIT_LOG, + ISSUES_FILE, + REPO, +) +from .fetch import load_issues, load_metadata, update_metadata + +console = Console() + + +def run_gh_command(args: list[str], check: bool = True) -> tuple[bool, str]: + """Run a gh CLI command and return success status and output.""" + cmd = ["gh"] + args + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=check) + return True, result.stdout + except subprocess.CalledProcessError as e: + return False, e.stderr + + +def add_label_to_issue(issue_number: int, label: str) -> bool: + """Add a label to an issue.""" + success, output = run_gh_command([ + "issue", "edit", + str(issue_number), + "--repo", REPO, + "--add-label", label, + ], check=False) + + if not success: + console.print(f"[red]Failed to add '{label}' to #{issue_number}: {output}[/red]") + + return success + + +def remove_label_from_issue(issue_number: int, label: str) -> bool: + """Remove a label from an issue.""" + success, output = run_gh_command([ + "issue", "edit", + str(issue_number), + "--repo", REPO, + "--remove-label", label, + ], check=False) + + if not success: + console.print( + f"[red]Failed to remove '{label}' from #{issue_number}: {output}[/red]" + ) + + return success + + +def log_audit_entry(entry: dict) -> None: + """Append an entry to the audit log.""" + with open(AUDIT_LOG, "a") as f: + f.write(json.dumps(entry) + "\n") + + +def get_approved_issues() -> list[dict]: + """Get all approved issues that have label changes.""" + issues = load_issues() + + approved = [] + for issue in issues: + if issue.get("review_status") != "approved": + continue + + proposed = issue.get("proposed_labels", {}) + if proposed.get("add") or proposed.get("remove"): + approved.append(issue) + + return approved + + +def preview_changes() -> None: + """Show a preview of all changes that would be applied.""" + approved = get_approved_issues() + + if not approved: + console.print("[yellow]No approved issues with label changes[/yellow]") + return + + console.print(f"\n[bold]Preview of {len(approved)} issues to update:[/bold]\n") + + table = Table() + table.add_column("Issue", style="cyan") + table.add_column("Title", max_width=40) + table.add_column("Add Labels", style="green") + table.add_column("Remove Labels", style="red") + + for issue in approved[:50]: # Show first 50 + proposed = issue.get("proposed_labels", {}) + table.add_row( + f"#{issue['number']}", + issue["title"][:40], + ", ".join(proposed.get("add", [])) or "-", + ", ".join(proposed.get("remove", [])) or "-", + ) + + console.print(table) + + if len(approved) > 50: + console.print(f"[dim]... and {len(approved) - 50} more issues[/dim]") + + # Summary + total_adds = sum( + len(i.get("proposed_labels", {}).get("add", [])) + for i in approved + ) + total_removes = sum( + len(i.get("proposed_labels", {}).get("remove", [])) + for i in approved + ) + + console.print() + console.print(Panel( + f"Issues: {len(approved)}\n" + f"Labels to add: {total_adds}\n" + f"Labels to remove: {total_removes}", + title="Summary", + border_style="blue", + )) + + +def apply_changes( + dry_run: bool = False, + batch_size: int = APPLY_BATCH_SIZE, + batch_delay: int = APPLY_BATCH_DELAY_SECONDS, +) -> dict: + """Apply label changes to approved issues.""" + approved = get_approved_issues() + + if not approved: + console.print("[yellow]No approved issues with label changes[/yellow]") + return {"applied": 0, "failed": 0} + + if dry_run: + console.print("[yellow]DRY RUN - No changes will be made[/yellow]") + preview_changes() + return {"applied": 0, "failed": 0, "dry_run": True} + + console.print(f"[blue]Applying changes to {len(approved)} issues[/blue]") + + issues = load_issues() + issue_map = {i["number"]: idx for idx, i in enumerate(issues)} + + applied = 0 + failed = 0 + batch_count = 0 + + with Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TaskProgressColumn(), + console=console, + ) as progress: + task = progress.add_task("Applying labels...", total=len(approved)) + + for i, issue in enumerate(approved): + # Batch delay + if i > 0 and i % batch_size == 0: + batch_count += 1 + progress.update( + task, + description=f"Batch {batch_count} complete, waiting {batch_delay}s...", + ) + time.sleep(batch_delay) + + proposed = issue.get("proposed_labels", {}) + issue_applied = True + + # Add labels + for label in proposed.get("add", []): + if not add_label_to_issue(issue["number"], label): + issue_applied = False + + # Remove labels + for label in proposed.get("remove", []): + if not remove_label_from_issue(issue["number"], label): + issue_applied = False + + if issue_applied: + applied += 1 + + # Log to audit + log_audit_entry({ + "timestamp": datetime.now(timezone.utc).isoformat(), + "issue_number": issue["number"], + "labels_added": proposed.get("add", []), + "labels_removed": proposed.get("remove", []), + "status": "success", + }) + + # Update issue status + idx = issue_map[issue["number"]] + issues[idx]["review_status"] = "applied" + issues[idx]["applied_at"] = datetime.now(timezone.utc).isoformat() + + else: + failed += 1 + log_audit_entry({ + "timestamp": datetime.now(timezone.utc).isoformat(), + "issue_number": issue["number"], + "labels_added": proposed.get("add", []), + "labels_removed": proposed.get("remove", []), + "status": "failed", + }) + + progress.update( + task, + advance=1, + description=f"Updated #{issue['number']}", + ) + + # Save updated issues + with open(ISSUES_FILE, "w") as f: + for issue in issues: + f.write(json.dumps(issue) + "\n") + + # Update metadata + meta = load_metadata() or {} + update_metadata({"applied_count": meta.get("applied_count", 0) + applied}) + + # Summary + console.print() + console.print(Panel( + f"Applied: {applied}\nFailed: {failed}", + title="Apply Summary", + border_style="green" if failed == 0 else "yellow", + )) + + return {"applied": applied, "failed": failed} + + +def verify_applied(sample_size: int = 10) -> None: + """Verify labels were applied correctly by checking a sample.""" + issues = load_issues() + + # Get recently applied issues + applied = [i for i in issues if i.get("review_status") == "applied"] + + if not applied: + console.print("[yellow]No applied issues to verify[/yellow]") + return + + # Sample + import random + sample = random.sample(applied, min(sample_size, len(applied))) + + console.print(f"[blue]Verifying {len(sample)} issues...[/blue]\n") + + verified = 0 + mismatched = 0 + + for issue in sample: + # Fetch current labels from GitHub + success, output = run_gh_command([ + "issue", "view", + str(issue["number"]), + "--repo", REPO, + "--json", "labels", + ]) + + if not success: + console.print(f"[red]Failed to fetch #{issue['number']}[/red]") + continue + + current_labels = set( + label["name"] + for label in json.loads(output).get("labels", []) + ) + + # Check if expected labels are present + expected_adds = set(issue.get("proposed_labels", {}).get("add", [])) + expected_removes = set(issue.get("proposed_labels", {}).get("remove", [])) + + adds_present = expected_adds.issubset(current_labels) + removes_absent = expected_removes.isdisjoint(current_labels) + + if adds_present and removes_absent: + console.print(f"[green]✓ #{issue['number']}[/green]") + verified += 1 + else: + console.print(f"[red]✗ #{issue['number']}[/red]") + if not adds_present: + missing = expected_adds - current_labels + console.print(f" [dim]Missing: {missing}[/dim]") + if not removes_absent: + still_present = expected_removes & current_labels + console.print(f" [dim]Still present: {still_present}[/dim]") + mismatched += 1 + + console.print() + console.print(Panel( + f"Verified: {verified}\nMismatched: {mismatched}", + title="Verification Summary", + border_style="green" if mismatched == 0 else "red", + )) diff --git a/scripts/issue_classifier/classify.py b/scripts/issue_classifier/classify.py new file mode 100644 index 00000000000..d2a4e12945a --- /dev/null +++ b/scripts/issue_classifier/classify.py @@ -0,0 +1,398 @@ +"""LLM-assisted classification of GitHub issues.""" + +import json +import time +from typing import Any + +from anthropic import Anthropic +from rich.console import Console +from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn + +from .config import ( + ANTHROPIC_MODEL, + CLASSIFY_CHUNK_SIZE, + GOOD_FIRST_ISSUE_CRITERIA, + ISSUES_FILE, + MODULE_KEYWORDS, + TRIAGE_CRITERIA, + get_anthropic_api_key, +) +from .fetch import load_issues, load_metadata, update_metadata + +console = Console() + + +def build_classification_prompt(issue: dict) -> str: + """Build the classification prompt for an issue.""" + module_context = "\n".join( + f"- {module}: {', '.join(keywords)}" + for module, keywords in MODULE_KEYWORDS.items() + ) + + return f"""Analyze this GitHub issue from the vaadin/flow repository and classify it. + +## Issue Details + +**Number:** #{issue['number']} +**Title:** {issue['title']} +**Age:** {issue['age_days']} days +**Author:** {issue['author']} +**Comments:** {issue['comment_count']} +**Existing Labels:** {', '.join(issue['existing_labels']) if issue['existing_labels'] else 'None'} + +**Body:** +{issue['body_preview']} + +## Classification Task + +Classify this issue with the following: + +1. **Type** (required): One of: bug, enhancement, feature request + - bug: Something is broken or not working as expected + - enhancement: Improvement to existing functionality + - feature request: New functionality that doesn't exist yet + +2. **Impact** (for bugs only): High or Low + - High: Affects many users, blocks workflows, causes data loss, security issue + - Low: Edge case, minor inconvenience, workaround available + +3. **Severity** (for bugs only): Major or Minor + - Major: Core functionality broken, crashes, security vulnerabilities + - Minor: Cosmetic issues, non-critical bugs, minor inconveniences + +4. **Modules** (if detectable): Which parts of the codebase are affected + Module keywords for reference: +{module_context} + +5. **Good First Issue** (yes/no): Would this be suitable for a new contributor? +{GOOD_FIRST_ISSUE_CRITERIA} + +6. **Confidence** (0.0-1.0): How confident are you in this classification? + +7. **Reasoning**: Brief explanation of your classification (1-2 sentences) + +## Triage Analysis (for bugs only) + +For bugs, also assess: + +8. **Needs Test Case** (yes/no): Does this bug need a reproducible test case before it can be fixed? +{TRIAGE_CRITERIA['needs_test_case']} + +9. **AI Fixable** (yes/no): Could an AI assistant likely fix this bug without further information? +{TRIAGE_CRITERIA['ai_fixable']} + +10. **Potentially Fixed** (yes/no): Might this issue have already been fixed? +{TRIAGE_CRITERIA['potentially_fixed']} + +11. **Potentially Outdated** (yes/no): Is this issue possibly outdated or about deprecated features? +{TRIAGE_CRITERIA['potentially_outdated']} + +## Response Format + +Respond with ONLY a JSON object in this exact format: +{{ + "type": "bug" | "enhancement" | "feature request", + "impact": "High" | "Low" | null, + "severity": "Major" | "Minor" | null, + "modules": ["module1", "module2"], + "good_first_issue": true | false, + "confidence": 0.85, + "reasoning": "Brief explanation", + "triage": {{ + "needs_test_case": true | false, + "ai_fixable": true | false, + "potentially_fixed": true | false, + "potentially_outdated": true | false, + "triage_notes": "Brief notes on triage assessment" + }} +}} + +Important: +- impact and severity should only be set for bugs, otherwise null +- modules should be an empty array if no specific module is detectable +- Be conservative with good_first_issue - only mark true if clearly suitable +- triage should be null for non-bugs, only populated for bugs +- ai_fixable should be conservative - only true if clearly straightforward +""" + + +def parse_classification_response(response_text: str) -> dict | None: + """Parse the LLM response into a classification dict.""" + try: + # Try to extract JSON from the response + text = response_text.strip() + + # Handle markdown code blocks + if "```json" in text: + start = text.find("```json") + 7 + end = text.find("```", start) + text = text[start:end].strip() + elif "```" in text: + start = text.find("```") + 3 + end = text.find("```", start) + text = text[start:end].strip() + + return json.loads(text) + except json.JSONDecodeError as e: + console.print(f"[red]Failed to parse classification response: {e}[/red]") + console.print(f"[dim]Response was: {response_text[:500]}...[/dim]") + return None + + +def classify_issue(client: Anthropic, issue: dict) -> dict | None: + """Classify a single issue using the LLM.""" + prompt = build_classification_prompt(issue) + + try: + response = client.messages.create( + model=ANTHROPIC_MODEL, + max_tokens=800, + messages=[{"role": "user", "content": prompt}], + ) + + response_text = response.content[0].text + return parse_classification_response(response_text) + + except Exception as e: + console.print(f"[red]Error classifying issue #{issue['number']}: {e}[/red]") + return None + + +def determine_proposed_labels(issue: dict, classification: dict) -> dict[str, list[str]]: + """Determine which labels to add/remove based on classification.""" + existing = set(issue["existing_labels"]) + to_add = [] + to_remove = [] + + # Type label + type_label = classification.get("type") + if type_label and type_label not in existing: + to_add.append(type_label) + + # Impact label (for bugs) + if classification.get("type") == "bug" and classification.get("impact"): + impact_label = f"Impact: {classification['impact']}" + if impact_label not in existing: + to_add.append(impact_label) + + # Severity label (for bugs) + if classification.get("type") == "bug" and classification.get("severity"): + severity_label = f"Severity: {classification['severity']}" + if severity_label not in existing: + to_add.append(severity_label) + + # Good First Issue + if classification.get("good_first_issue") and "Good First Issue" not in existing: + to_add.append("Good First Issue") + + return {"add": to_add, "remove": to_remove} + + +def save_issue_update(issues: list[dict], index: int, issue: dict) -> None: + """Save an updated issue back to the JSONL file.""" + issues[index] = issue + + with open(ISSUES_FILE, "w") as f: + for iss in issues: + f.write(json.dumps(iss) + "\n") + + +def classify_chunk( + start_index: int, + chunk_size: int = CLASSIFY_CHUNK_SIZE, + rate_limit_delay: float = 0.5, +) -> int: + """Classify a chunk of issues starting from the given index.""" + issues = load_issues() + meta = load_metadata() + + if not issues: + console.print("[yellow]No issues to classify. Run fetch first.[/yellow]") + return 0 + + total = len(issues) + end_index = min(start_index + chunk_size, total) + + console.print( + f"[blue]Classifying issues {start_index + 1} to {end_index} of {total}[/blue]" + ) + + # Initialize Anthropic client + api_key = get_anthropic_api_key() + client = Anthropic(api_key=api_key) + + classified_count = 0 + + with Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TaskProgressColumn(), + console=console, + ) as progress: + task = progress.add_task( + "Classifying...", + total=end_index - start_index, + ) + + for i in range(start_index, end_index): + issue = issues[i] + + # Skip already classified issues + if issue.get("classification") is not None: + progress.update(task, advance=1) + continue + + # Classify the issue + classification = classify_issue(client, issue) + + if classification: + issue["classification"] = classification + issue["proposed_labels"] = determine_proposed_labels( + issue, classification + ) + classified_count += 1 + + # Save immediately after each classification + save_issue_update(issues, i, issue) + + progress.update( + task, + advance=1, + description=f"Classified #{issue['number']}", + ) + + # Rate limiting + if i < end_index - 1: + time.sleep(rate_limit_delay) + + # Update metadata + update_metadata({ + "last_processed_index": end_index, + "classified_count": meta.get("classified_count", 0) + classified_count, + }) + + console.print(f"[green]Classified {classified_count} issues in this chunk[/green]") + return classified_count + + +def classify_all( + chunk_size: int = CLASSIFY_CHUNK_SIZE, + rate_limit_delay: float = 0.5, +) -> int: + """Classify all unclassified issues.""" + issues = load_issues() + + if not issues: + console.print("[yellow]No issues to classify. Run fetch first.[/yellow]") + return 0 + + total = len(issues) + total_classified = 0 + + # Find first unclassified issue + start_index = 0 + for i, issue in enumerate(issues): + if issue.get("classification") is None: + start_index = i + break + else: + console.print("[green]All issues are already classified![/green]") + return 0 + + console.print(f"[blue]Starting classification from issue {start_index + 1}[/blue]") + + while start_index < total: + classified = classify_chunk(start_index, chunk_size, rate_limit_delay) + total_classified += classified + start_index += chunk_size + + if start_index < total: + console.print( + f"[dim]Progress: {start_index}/{total} " + f"({start_index * 100 // total}%)[/dim]" + ) + + console.print(f"\n[bold green]Classification complete![/bold green]") + console.print(f" Total classified: {total_classified}") + + return total_classified + + +def resume_classification( + chunk_size: int = CLASSIFY_CHUNK_SIZE, + rate_limit_delay: float = 0.5, +) -> int: + """Resume classification from where we left off.""" + meta = load_metadata() + + if not meta: + console.print("[yellow]No metadata found. Run fetch first.[/yellow]") + return 0 + + start_index = meta.get("last_processed_index", 0) + console.print(f"[blue]Resuming from index {start_index}[/blue]") + + return classify_all(chunk_size, rate_limit_delay) + + +def get_classification_stats() -> dict[str, Any]: + """Get statistics about classification progress.""" + issues = load_issues() + + if not issues: + return {"total": 0, "classified": 0, "pending": 0} + + classified = sum(1 for i in issues if i.get("classification") is not None) + pending = len(issues) - classified + + # Count by type + type_counts: dict[str, int] = {} + impact_counts: dict[str, int] = {} + module_counts: dict[str, int] = {} + good_first_count = 0 + + # Triage counts + triage_counts = { + "needs_test_case": 0, + "ai_fixable": 0, + "potentially_fixed": 0, + "potentially_outdated": 0, + } + + for issue in issues: + if issue.get("classification"): + cls = issue["classification"] + + # Type + t = cls.get("type", "unknown") + type_counts[t] = type_counts.get(t, 0) + 1 + + # Impact + if cls.get("impact"): + impact_counts[cls["impact"]] = impact_counts.get(cls["impact"], 0) + 1 + + # Modules + for mod in cls.get("modules", []): + module_counts[mod] = module_counts.get(mod, 0) + 1 + + # Good First Issue + if cls.get("good_first_issue"): + good_first_count += 1 + + # Triage stats (for bugs) + triage = cls.get("triage") + if triage: + for key in triage_counts: + if triage.get(key): + triage_counts[key] += 1 + + return { + "total": len(issues), + "classified": classified, + "pending": pending, + "by_type": type_counts, + "by_impact": impact_counts, + "by_module": module_counts, + "good_first_issues": good_first_count, + "triage": triage_counts, + } diff --git a/scripts/issue_classifier/cli.py b/scripts/issue_classifier/cli.py new file mode 100644 index 00000000000..125b36be8ba --- /dev/null +++ b/scripts/issue_classifier/cli.py @@ -0,0 +1,260 @@ +"""CLI entry point for the issue classifier.""" + +import click +from rich.console import Console + +console = Console() + + +@click.group() +@click.version_option(version="0.1.0") +def cli() -> None: + """GitHub Issue Classification System for vaadin/flow. + + This tool helps classify and label GitHub issues using LLM assistance. + All classifications require manual review before labels are applied. + """ + pass + + +@cli.command() +@click.option("--verify", is_flag=True, help="Verify fetch against GitHub count") +def fetch(verify: bool) -> None: + """Fetch all open issues from GitHub. + + Uses the gh CLI to fetch issues and stores them in data/issues/issues.jsonl. + Requires gh CLI to be authenticated (gh auth login). + """ + from .fetch import fetch_and_save, verify_fetch + + count = fetch_and_save() + + if verify and count > 0: + verify_fetch() + + +@cli.command() +@click.option( + "--chunk-size", + default=20, + help="Number of issues to classify per chunk", +) +@click.option( + "--rate-limit", + default=0.5, + help="Delay between API calls in seconds", +) +@click.option( + "--all", + "classify_all_flag", + is_flag=True, + help="Classify all unclassified issues", +) +def classify(chunk_size: int, rate_limit: float, classify_all_flag: bool) -> None: + """Classify issues using LLM. + + Requires ANTHROPIC_API_KEY environment variable to be set. + Classifications are saved incrementally and can be resumed. + """ + from .classify import classify_all, classify_chunk, get_classification_stats + from .fetch import load_metadata + + meta = load_metadata() + if not meta: + console.print( + "[red]No issues found. Run 'fetch' first.[/red]" + ) + return + + if classify_all_flag: + classify_all(chunk_size, rate_limit) + else: + start_index = meta.get("last_processed_index", 0) + classify_chunk(start_index, chunk_size, rate_limit) + + # Show stats + stats = get_classification_stats() + console.print( + f"\n[dim]Progress: {stats['classified']}/{stats['total']} classified[/dim]" + ) + + +@cli.command() +@click.option( + "--chunk-size", + default=20, + help="Number of issues to classify per chunk", +) +@click.option( + "--rate-limit", + default=0.5, + help="Delay between API calls in seconds", +) +def resume(chunk_size: int, rate_limit: float) -> None: + """Resume classification from last checkpoint.""" + from .classify import resume_classification + + resume_classification(chunk_size, rate_limit) + + +@cli.command() +@click.option( + "--start-from", + default=0, + help="Start review from this index", +) +@click.option( + "--issue", + "issue_number", + type=int, + help="Review a single issue by number", +) +def review(start_from: int, issue_number: int | None) -> None: + """Interactive review of classified issues. + + All classifications must be approved before labels can be applied. + Use arrow keys or type the action letter to select an option. + """ + from .review import get_review_stats, review_issues, review_single + + if issue_number: + review_single(issue_number) + else: + # Show stats first + stats = get_review_stats() + console.print( + f"[dim]Pending review: {stats['pending_review']} | " + f"Approved: {stats['approved']}[/dim]\n" + ) + review_issues(start_from) + + +@cli.command() +@click.option("--dry-run", is_flag=True, help="Preview changes without applying") +@click.option( + "--batch-size", + default=30, + help="Number of issues per batch", +) +@click.option( + "--batch-delay", + default=60, + help="Delay between batches in seconds", +) +@click.option("--verify", is_flag=True, help="Verify applied labels") +def apply(dry_run: bool, batch_size: int, batch_delay: int, verify: bool) -> None: + """Apply approved label changes to GitHub. + + Only issues with review_status='approved' will be updated. + Uses the gh CLI to apply labels. + """ + from .apply import apply_changes, preview_changes, verify_applied + + if verify: + verify_applied() + return + + if dry_run: + preview_changes() + else: + result = apply_changes( + dry_run=False, + batch_size=batch_size, + batch_delay=batch_delay, + ) + + if result.get("applied", 0) > 0: + console.print( + "\n[dim]Run 'apply --verify' to verify the changes[/dim]" + ) + + +@cli.command() +@click.option( + "--type", + "report_type", + type=click.Choice(["progress", "labels", "unlabeled", "good-first", "triage"]), + default="progress", + help="Type of report to generate", +) +def report(report_type: str) -> None: + """Generate summary reports. + + Available reports: + - progress: Overall classification progress + - labels: Summary of proposed label changes + - unlabeled: Report on unlabeled issues + - good-first: Good First Issue candidates + - triage: Triage analysis (AI fixable, outdated, etc.) + """ + from .report import ( + print_good_first_issues, + print_label_summary, + print_progress_report, + print_triage_report, + print_unlabeled_report, + ) + + if report_type == "progress": + print_progress_report() + elif report_type == "labels": + print_label_summary() + elif report_type == "unlabeled": + print_unlabeled_report() + elif report_type == "good-first": + print_good_first_issues() + elif report_type == "triage": + print_triage_report() + + +@cli.command() +def stats() -> None: + """Show quick statistics.""" + from .classify import get_classification_stats + from .fetch import load_issues, load_metadata + from .review import get_review_stats + + meta = load_metadata() + issues = load_issues() + + if not issues: + console.print("[yellow]No data available. Run fetch first.[/yellow]") + return + + review_stats = get_review_stats() + class_stats = get_classification_stats() + + console.print("\n[bold]Quick Stats[/bold]\n") + console.print(f" Total issues: {len(issues)}") + console.print(f" Classified: {class_stats['classified']}") + console.print(f" Pending review: {review_stats['pending_review']}") + console.print(f" Approved: {review_stats['approved']}") + + if meta: + console.print(f"\n [dim]Last fetch: {meta.get('fetched_at', 'unknown')}[/dim]") + + +@cli.command() +@click.argument("issue_number", type=int) +def show(issue_number: int) -> None: + """Show details for a specific issue.""" + from .fetch import load_issues + from .review import display_issue + + issues = load_issues() + + for issue in issues: + if issue["number"] == issue_number: + display_issue(issue) + return + + console.print(f"[red]Issue #{issue_number} not found[/red]") + + +def main() -> None: + """Main entry point.""" + cli() + + +if __name__ == "__main__": + main() diff --git a/scripts/issue_classifier/config.py b/scripts/issue_classifier/config.py new file mode 100644 index 00000000000..088d666a7af --- /dev/null +++ b/scripts/issue_classifier/config.py @@ -0,0 +1,152 @@ +"""Configuration constants for the issue classifier.""" + +import os +from pathlib import Path + +# Repository settings +REPO = "vaadin/flow" + +# File paths +PROJECT_ROOT = Path(__file__).parent.parent.parent +DATA_DIR = Path(__file__).parent / "data" +ISSUES_FILE = DATA_DIR / "issues.jsonl" +META_FILE = DATA_DIR / "issues_meta.json" +AUDIT_LOG = DATA_DIR / "audit_log.jsonl" + +# API settings +FETCH_PAGE_SIZE = 100 +CLASSIFY_CHUNK_SIZE = 20 +APPLY_BATCH_SIZE = 30 +APPLY_BATCH_DELAY_SECONDS = 60 + +# Classification labels +TYPE_LABELS = ["bug", "enhancement", "feature request"] +IMPACT_LABELS = ["Impact: High", "Impact: Low"] +SEVERITY_LABELS = ["Severity: Major", "Severity: Minor"] + +MODULE_LABELS = { + "signals": "signals", + "binder": "flow-data", + "navigation": "navigation", + "push": "push", + "dnd": "dnd", + "spring": "vaadin-spring", + "build-tools": "build-tools", + "flow-server": "flow-server", + "flow-client": "flow-client", + "flow-data": "flow-data", + "security": "security", + "i18n": "i18n", + "a11y": "accessibility", + "pwa": "pwa", + "testing": "testing", +} + +COMMUNITY_LABELS = ["Good First Issue", "Help wanted"] + +# Module detection keywords for LLM context +MODULE_KEYWORDS = { + "signals": [ + "Signal", "reactive", "state management", "NumberSignal", + "ListSignal", "computed", "effect" + ], + "navigation": [ + "Router", "route", "redirect", "BeforeEnter", "BeforeLeave", + "navigation", "RouteConfiguration", "RouteParameters", "QueryParameters" + ], + "push": [ + "WebSocket", "Atmosphere", "push", "real-time", "PushConnection", + "server push", "@Push" + ], + "binder": [ + "Binder", "binding", "validation", "converter", "Validator", + "field binding", "bean validation" + ], + "build-tools": [ + "Maven", "Gradle", "Vite", "frontend build", "npm", "pnpm", + "webpack", "bundle", "plugin" + ], + "flow-server": [ + "Element", "StateNode", "StateTree", "Component", "UI", + "VaadinSession", "VaadinService" + ], + "flow-client": [ + "TypeScript", "JavaScript", "client-side", "Flow.ts", + "frontend", "lit", "web component" + ], + "spring": [ + "Spring", "autowire", "Bean", "SpringUI", "VaadinServlet", + "Spring Boot", "Spring Security" + ], + "security": [ + "security", "authentication", "authorization", "CSRF", + "XSS", "injection", "vulnerability" + ], + "dnd": [ + "drag", "drop", "DnD", "drag and drop", "draggable" + ], +} + +# Good First Issue criteria +GOOD_FIRST_ISSUE_CRITERIA = """ +A "Good First Issue" should meet these criteria: +- Clear, well-defined scope with a single specific task +- Documentation or test improvements +- Single-file changes likely +- No deep framework knowledge required +- Has a clear solution path that can be described +- Doesn't require understanding of complex internals +""" + +# Triage analysis criteria +TRIAGE_CRITERIA = { + "needs_test_case": """ +A bug "needs a test case" if: +- The reporter hasn't provided steps to reproduce +- The issue description is vague about when/how it occurs +- There's no code sample or minimal reproduction +- The behavior is intermittent or environment-specific +- It's unclear what the expected vs actual behavior is +""", + "ai_fixable": """ +An issue is "likely AI-fixable without further info" if: +- The problem is clearly described with specific error messages +- The fix is localized to a small area of code +- It's a straightforward bug (typo, off-by-one, null check, etc.) +- The expected behavior is unambiguous +- No external dependencies or complex state involved +- Similar patterns exist elsewhere in the codebase +- It doesn't require deep domain knowledge or architectural decisions +""", + "potentially_fixed": """ +An issue is "potentially already fixed" if: +- It's older than 6 months and mentions a specific version +- It describes behavior that seems basic/critical (likely caught) +- The reporter hasn't responded to follow-ups +- Similar issues have been closed as fixed +- It mentions APIs or features that have been reworked +""", + "potentially_outdated": """ +An issue is "potentially outdated/deprecated" if: +- It references old Vaadin versions (< 23) +- It mentions deprecated APIs (Polymer, bower, webjars, etc.) +- It's about features that have been replaced (e.g., old build system) +- The technology stack mentioned is no longer supported +- It references removed or significantly changed APIs +""", +} + +# LLM settings +ANTHROPIC_MODEL = "claude-sonnet-4-20250514" +MAX_BODY_PREVIEW_LENGTH = 2000 + + +def get_anthropic_api_key() -> str: + """Get the Anthropic API key from environment.""" + key = os.environ.get("ANTHROPIC_API_KEY") + if not key: + raise ValueError( + "ANTHROPIC_API_KEY environment variable is not set. " + "Please set it to use LLM classification." + ) + return key diff --git a/scripts/issue_classifier/data/.gitkeep b/scripts/issue_classifier/data/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/issue_classifier/fetch.py b/scripts/issue_classifier/fetch.py new file mode 100644 index 00000000000..90ed36f4550 --- /dev/null +++ b/scripts/issue_classifier/fetch.py @@ -0,0 +1,227 @@ +"""Fetch issues from GitHub using gh CLI.""" + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +from rich.console import Console +from rich.progress import Progress, SpinnerColumn, TextColumn + +from .config import ( + DATA_DIR, + FETCH_PAGE_SIZE, + ISSUES_FILE, + MAX_BODY_PREVIEW_LENGTH, + META_FILE, + REPO, +) + +console = Console() + + +def run_gh_command(args: list[str]) -> str: + """Run a gh CLI command and return the output.""" + cmd = ["gh"] + args + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return result.stdout + + +def fetch_issues_page(page: int, per_page: int = FETCH_PAGE_SIZE) -> list[dict]: + """Fetch a single page of issues from GitHub.""" + # Calculate the range for this page + # gh doesn't support pagination directly, so we use --limit with offset simulation + # We'll fetch all at once with a high limit since gh handles it efficiently + args = [ + "issue", "list", + "--repo", REPO, + "--state", "open", + "--json", "number,title,body,labels,createdAt,updatedAt,author,comments", + "--limit", str(per_page), + ] + + output = run_gh_command(args) + return json.loads(output) if output.strip() else [] + + +def fetch_all_issues() -> list[dict]: + """Fetch all open issues from the repository.""" + console.print(f"[blue]Fetching all open issues from {REPO}...[/blue]") + + # gh CLI handles pagination internally when we use a high limit + args = [ + "issue", "list", + "--repo", REPO, + "--state", "open", + "--json", "number,title,body,labels,createdAt,updatedAt,author,comments", + "--limit", "5000", # High limit to get all issues + ] + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Fetching issues from GitHub...", total=None) + output = run_gh_command(args) + issues = json.loads(output) if output.strip() else [] + progress.update(task, description=f"Fetched {len(issues)} issues") + + return issues + + +def transform_issue(issue: dict) -> dict: + """Transform a raw GitHub issue into our tracking format.""" + now = datetime.now(timezone.utc) + created_at = datetime.fromisoformat(issue["createdAt"].replace("Z", "+00:00")) + age_days = (now - created_at).days + + # Extract label names + existing_labels = [label["name"] for label in issue.get("labels", [])] + + # Truncate body for preview + body = issue.get("body") or "" + body_preview = body[:MAX_BODY_PREVIEW_LENGTH] + if len(body) > MAX_BODY_PREVIEW_LENGTH: + body_preview += "..." + + return { + "number": issue["number"], + "title": issue["title"], + "body_preview": body_preview, + "created_at": issue["createdAt"], + "age_days": age_days, + "existing_labels": existing_labels, + "author": issue.get("author", {}).get("login", "unknown"), + "comment_count": issue.get("comments", 0), + "classification": None, + "proposed_labels": None, + "review_status": "pending", + } + + +def save_issues(issues: list[dict], filepath: Path = ISSUES_FILE) -> None: + """Save issues to JSONL file.""" + filepath.parent.mkdir(parents=True, exist_ok=True) + + with open(filepath, "w") as f: + for issue in issues: + f.write(json.dumps(issue) + "\n") + + console.print(f"[green]Saved {len(issues)} issues to {filepath}[/green]") + + +def save_metadata(total: int, fetched_at: str) -> None: + """Save metadata about the fetch operation.""" + meta = { + "total_issues": total, + "fetched_at": fetched_at, + "last_processed_index": 0, + "classified_count": 0, + "reviewed_count": 0, + "applied_count": 0, + } + + with open(META_FILE, "w") as f: + json.dump(meta, f, indent=2) + + console.print(f"[green]Saved metadata to {META_FILE}[/green]") + + +def load_issues(filepath: Path = ISSUES_FILE) -> list[dict]: + """Load issues from JSONL file.""" + if not filepath.exists(): + return [] + + issues = [] + with open(filepath) as f: + for line in f: + if line.strip(): + issues.append(json.loads(line)) + + return issues + + +def load_metadata() -> dict | None: + """Load metadata from file.""" + if not META_FILE.exists(): + return None + + with open(META_FILE) as f: + return json.load(f) + + +def update_metadata(updates: dict) -> None: + """Update metadata file with new values.""" + meta = load_metadata() or {} + meta.update(updates) + + with open(META_FILE, "w") as f: + json.dump(meta, f, indent=2) + + +def fetch_and_save() -> int: + """Main fetch operation: fetch all issues and save to file.""" + # Ensure data directory exists + DATA_DIR.mkdir(parents=True, exist_ok=True) + + # Fetch all issues + raw_issues = fetch_all_issues() + + if not raw_issues: + console.print("[yellow]No issues found![/yellow]") + return 0 + + # Transform to our format + console.print("[blue]Transforming issues...[/blue]") + transformed = [transform_issue(issue) for issue in raw_issues] + + # Sort by issue number (oldest first) + transformed.sort(key=lambda x: x["number"]) + + # Save issues + save_issues(transformed) + + # Save metadata + fetched_at = datetime.now(timezone.utc).isoformat() + save_metadata(len(transformed), fetched_at) + + # Print summary + console.print("\n[bold green]Fetch complete![/bold green]") + console.print(f" Total issues: {len(transformed)}") + + # Count issues by label status + unlabeled = sum(1 for i in transformed if not i["existing_labels"]) + console.print(f" Unlabeled issues: {unlabeled} ({unlabeled*100//len(transformed)}%)") + + return len(transformed) + + +def verify_fetch() -> bool: + """Verify the fetch by comparing with gh issue count.""" + console.print("[blue]Verifying fetch...[/blue]") + + # Get count from gh CLI + args = [ + "issue", "list", + "--repo", REPO, + "--state", "open", + "--json", "number", + "--limit", "5000", + ] + output = run_gh_command(args) + gh_count = len(json.loads(output)) if output.strip() else 0 + + # Get count from our file + issues = load_issues() + file_count = len(issues) + + console.print(f" GitHub count: {gh_count}") + console.print(f" File count: {file_count}") + + if gh_count == file_count: + console.print("[green]✓ Counts match![/green]") + return True + else: + console.print(f"[yellow]⚠ Counts differ by {abs(gh_count - file_count)}[/yellow]") + return False diff --git a/scripts/issue_classifier/report.py b/scripts/issue_classifier/report.py new file mode 100644 index 00000000000..de922ec44b4 --- /dev/null +++ b/scripts/issue_classifier/report.py @@ -0,0 +1,389 @@ +"""Summary reports for issue classification.""" + +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from .classify import get_classification_stats +from .fetch import load_issues, load_metadata +from .review import get_review_stats + +console = Console() + + +def print_progress_report() -> None: + """Print a comprehensive progress report.""" + meta = load_metadata() + issues = load_issues() + + if not issues: + console.print("[yellow]No data available. Run fetch first.[/yellow]") + return + + # Overall progress + total = len(issues) + classified = sum(1 for i in issues if i.get("classification") is not None) + approved = sum(1 for i in issues if i.get("review_status") == "approved") + applied = sum(1 for i in issues if i.get("review_status") == "applied") + + console.print("\n[bold]Issue Classification Progress Report[/bold]\n") + + # Progress table + progress_table = Table(title="Pipeline Progress") + progress_table.add_column("Stage", style="cyan") + progress_table.add_column("Count", justify="right") + progress_table.add_column("Percentage", justify="right") + + progress_table.add_row("Total Issues", str(total), "100%") + progress_table.add_row( + "Classified", + str(classified), + f"{classified * 100 // total}%" if total else "0%", + ) + progress_table.add_row( + "Approved", + str(approved), + f"{approved * 100 // total}%" if total else "0%", + ) + progress_table.add_row( + "Applied", + str(applied), + f"{applied * 100 // total}%" if total else "0%", + ) + + console.print(progress_table) + + # Classification stats + stats = get_classification_stats() + + if stats.get("by_type"): + console.print("\n[bold]Classification by Type[/bold]") + type_table = Table() + type_table.add_column("Type", style="cyan") + type_table.add_column("Count", justify="right") + + for t, count in sorted(stats["by_type"].items(), key=lambda x: -x[1]): + type_table.add_row(t, str(count)) + + console.print(type_table) + + if stats.get("by_impact"): + console.print("\n[bold]Impact Distribution (Bugs)[/bold]") + impact_table = Table() + impact_table.add_column("Impact", style="cyan") + impact_table.add_column("Count", justify="right") + + for impact, count in sorted(stats["by_impact"].items()): + impact_table.add_row(impact, str(count)) + + console.print(impact_table) + + if stats.get("by_module"): + console.print("\n[bold]Top Modules[/bold]") + module_table = Table() + module_table.add_column("Module", style="cyan") + module_table.add_column("Count", justify="right") + + for mod, count in sorted( + stats["by_module"].items(), + key=lambda x: -x[1], + )[:10]: + module_table.add_row(mod, str(count)) + + console.print(module_table) + + if stats.get("good_first_issues"): + console.print(f"\n[green]Good First Issues: {stats['good_first_issues']}[/green]") + + # Triage stats + if stats.get("triage"): + triage = stats["triage"] + console.print("\n[bold]Triage Analysis (Bugs)[/bold]") + triage_table = Table() + triage_table.add_column("Category", style="yellow") + triage_table.add_column("Count", justify="right") + + triage_table.add_row("Needs Test Case", str(triage.get("needs_test_case", 0))) + triage_table.add_row("AI Fixable", str(triage.get("ai_fixable", 0))) + triage_table.add_row("Potentially Fixed", str(triage.get("potentially_fixed", 0))) + triage_table.add_row("Potentially Outdated", str(triage.get("potentially_outdated", 0))) + + console.print(triage_table) + + # Metadata + if meta: + console.print(f"\n[dim]Last fetch: {meta.get('fetched_at', 'unknown')}[/dim]") + + +def print_label_summary() -> None: + """Print a summary of proposed label changes.""" + issues = load_issues() + + if not issues: + console.print("[yellow]No data available. Run fetch first.[/yellow]") + return + + # Count proposed labels + add_counts: dict[str, int] = {} + remove_counts: dict[str, int] = {} + + for issue in issues: + if issue.get("review_status") not in ("approved", "applied"): + continue + + proposed = issue.get("proposed_labels", {}) + + for label in proposed.get("add", []): + add_counts[label] = add_counts.get(label, 0) + 1 + + for label in proposed.get("remove", []): + remove_counts[label] = remove_counts.get(label, 0) + 1 + + console.print("\n[bold]Label Changes Summary[/bold]\n") + + if add_counts: + console.print("[green]Labels to Add:[/green]") + add_table = Table() + add_table.add_column("Label", style="green") + add_table.add_column("Count", justify="right") + + for label, count in sorted(add_counts.items(), key=lambda x: -x[1]): + add_table.add_row(label, str(count)) + + console.print(add_table) + + if remove_counts: + console.print("\n[red]Labels to Remove:[/red]") + remove_table = Table() + remove_table.add_column("Label", style="red") + remove_table.add_column("Count", justify="right") + + for label, count in sorted(remove_counts.items(), key=lambda x: -x[1]): + remove_table.add_row(label, str(count)) + + console.print(remove_table) + + total_add = sum(add_counts.values()) + total_remove = sum(remove_counts.values()) + + console.print(f"\n[bold]Total: +{total_add} / -{total_remove} label operations[/bold]") + + +def print_unlabeled_report() -> None: + """Print a report on unlabeled issues.""" + issues = load_issues() + + if not issues: + console.print("[yellow]No data available. Run fetch first.[/yellow]") + return + + unlabeled = [i for i in issues if not i.get("existing_labels")] + + console.print(f"\n[bold]Unlabeled Issues: {len(unlabeled)}[/bold]\n") + + if not unlabeled: + console.print("[green]All issues have labels![/green]") + return + + # Group by classification status + classified = [i for i in unlabeled if i.get("classification")] + unclassified = [i for i in unlabeled if not i.get("classification")] + + console.print(f" Classified: {len(classified)}") + console.print(f" Unclassified: {len(unclassified)}") + + # Show oldest unlabeled + console.print("\n[bold]Oldest Unlabeled Issues:[/bold]") + + table = Table() + table.add_column("Issue", style="cyan") + table.add_column("Age (days)", justify="right") + table.add_column("Title", max_width=50) + + oldest = sorted(unlabeled, key=lambda x: -x.get("age_days", 0))[:10] + + for issue in oldest: + table.add_row( + f"#{issue['number']}", + str(issue.get("age_days", 0)), + issue["title"][:50], + ) + + console.print(table) + + +def export_good_first_issues() -> list[dict]: + """Export list of Good First Issue candidates.""" + issues = load_issues() + + candidates = [] + for issue in issues: + cls = issue.get("classification", {}) + if cls.get("good_first_issue"): + candidates.append({ + "number": issue["number"], + "title": issue["title"], + "url": f"https://github.com/vaadin/flow/issues/{issue['number']}", + "type": cls.get("type"), + "confidence": cls.get("confidence", 0), + "reasoning": cls.get("reasoning", ""), + }) + + return sorted(candidates, key=lambda x: -x.get("confidence", 0)) + + +def print_good_first_issues() -> None: + """Print Good First Issue candidates.""" + candidates = export_good_first_issues() + + if not candidates: + console.print("[yellow]No Good First Issue candidates found.[/yellow]") + return + + console.print(f"\n[bold]Good First Issue Candidates: {len(candidates)}[/bold]\n") + + table = Table() + table.add_column("Issue", style="cyan") + table.add_column("Type", style="dim") + table.add_column("Confidence", justify="right") + table.add_column("Title", max_width=50) + + for c in candidates[:20]: + table.add_row( + f"#{c['number']}", + c.get("type", ""), + f"{c.get('confidence', 0):.0%}", + c["title"][:50], + ) + + console.print(table) + + if len(candidates) > 20: + console.print(f"[dim]... and {len(candidates) - 20} more[/dim]") + + +def print_triage_report() -> None: + """Print detailed triage analysis for bugs.""" + issues = load_issues() + + if not issues: + console.print("[yellow]No data available. Run fetch first.[/yellow]") + return + + # Collect bugs with triage info + ai_fixable = [] + needs_test_case = [] + potentially_fixed = [] + potentially_outdated = [] + + for issue in issues: + cls = issue.get("classification", {}) + if cls.get("type") != "bug": + continue + + triage = cls.get("triage", {}) + if not triage: + continue + + issue_info = { + "number": issue["number"], + "title": issue["title"], + "age_days": issue.get("age_days", 0), + "confidence": cls.get("confidence", 0), + "notes": triage.get("triage_notes", ""), + } + + if triage.get("ai_fixable"): + ai_fixable.append(issue_info) + if triage.get("needs_test_case"): + needs_test_case.append(issue_info) + if triage.get("potentially_fixed"): + potentially_fixed.append(issue_info) + if triage.get("potentially_outdated"): + potentially_outdated.append(issue_info) + + console.print("\n[bold]Triage Report[/bold]\n") + + # AI Fixable - most actionable + console.print(f"[bold green]AI Fixable Issues: {len(ai_fixable)}[/bold green]") + console.print("[dim]These bugs may be fixable by AI without additional information[/dim]\n") + + if ai_fixable: + table = Table() + table.add_column("Issue", style="cyan") + table.add_column("Age", justify="right") + table.add_column("Title", max_width=50) + + for i in sorted(ai_fixable, key=lambda x: -x["confidence"])[:15]: + table.add_row( + f"#{i['number']}", + f"{i['age_days']}d", + i["title"][:50], + ) + + console.print(table) + if len(ai_fixable) > 15: + console.print(f"[dim]... and {len(ai_fixable) - 15} more[/dim]") + + # Potentially outdated - cleanup candidates + console.print(f"\n[bold yellow]Potentially Outdated: {len(potentially_outdated)}[/bold yellow]") + console.print("[dim]May be related to deprecated features or old versions[/dim]\n") + + if potentially_outdated: + table = Table() + table.add_column("Issue", style="cyan") + table.add_column("Age", justify="right") + table.add_column("Title", max_width=50) + + for i in sorted(potentially_outdated, key=lambda x: -x["age_days"])[:15]: + table.add_row( + f"#{i['number']}", + f"{i['age_days']}d", + i["title"][:50], + ) + + console.print(table) + if len(potentially_outdated) > 15: + console.print(f"[dim]... and {len(potentially_outdated) - 15} more[/dim]") + + # Potentially fixed - verification candidates + console.print(f"\n[bold blue]Potentially Fixed: {len(potentially_fixed)}[/bold blue]") + console.print("[dim]May have been fixed already - needs verification[/dim]\n") + + if potentially_fixed: + table = Table() + table.add_column("Issue", style="cyan") + table.add_column("Age", justify="right") + table.add_column("Title", max_width=50) + + for i in sorted(potentially_fixed, key=lambda x: -x["age_days"])[:15]: + table.add_row( + f"#{i['number']}", + f"{i['age_days']}d", + i["title"][:50], + ) + + console.print(table) + if len(potentially_fixed) > 15: + console.print(f"[dim]... and {len(potentially_fixed) - 15} more[/dim]") + + # Needs test case - waiting for info + console.print(f"\n[bold red]Needs Test Case: {len(needs_test_case)}[/bold red]") + console.print("[dim]Need reproducible test case before they can be addressed[/dim]\n") + + if needs_test_case: + table = Table() + table.add_column("Issue", style="cyan") + table.add_column("Age", justify="right") + table.add_column("Title", max_width=50) + + for i in sorted(needs_test_case, key=lambda x: -x["age_days"])[:15]: + table.add_row( + f"#{i['number']}", + f"{i['age_days']}d", + i["title"][:50], + ) + + console.print(table) + if len(needs_test_case) > 15: + console.print(f"[dim]... and {len(needs_test_case) - 15} more[/dim]") diff --git a/scripts/issue_classifier/requirements.txt b/scripts/issue_classifier/requirements.txt new file mode 100644 index 00000000000..3d8f3cf031d --- /dev/null +++ b/scripts/issue_classifier/requirements.txt @@ -0,0 +1,3 @@ +click>=8.0 +rich>=13.0 +anthropic>=0.40.0 diff --git a/scripts/issue_classifier/review.py b/scripts/issue_classifier/review.py new file mode 100644 index 00000000000..060570e1fe6 --- /dev/null +++ b/scripts/issue_classifier/review.py @@ -0,0 +1,375 @@ +"""Interactive review interface for classified issues.""" + +import json +from typing import Literal + +from rich.console import Console +from rich.panel import Panel +from rich.prompt import Confirm, Prompt +from rich.table import Table +from rich.text import Text + +from .config import ( + COMMUNITY_LABELS, + IMPACT_LABELS, + ISSUES_FILE, + SEVERITY_LABELS, + TYPE_LABELS, +) +from .fetch import load_issues, load_metadata, update_metadata + +console = Console() + + +def display_issue(issue: dict) -> None: + """Display an issue for review.""" + # Header + console.print() + console.print( + Panel( + f"[bold]#{issue['number']}[/bold] {issue['title']}", + title="Issue", + border_style="blue", + ) + ) + + # Metadata table + meta_table = Table(show_header=False, box=None, padding=(0, 2)) + meta_table.add_column("Key", style="dim") + meta_table.add_column("Value") + + meta_table.add_row("Age", f"{issue['age_days']} days") + meta_table.add_row("Author", issue.get("author", "unknown")) + meta_table.add_row("Comments", str(issue.get("comment_count", 0))) + meta_table.add_row( + "Existing Labels", + ", ".join(issue["existing_labels"]) if issue["existing_labels"] else "None", + ) + + console.print(meta_table) + + # Body preview + console.print() + console.print(Panel(issue["body_preview"] or "[dim]No body[/dim]", title="Body")) + + # Classification + if issue.get("classification"): + cls = issue["classification"] + + cls_table = Table(show_header=False, box=None, padding=(0, 2)) + cls_table.add_column("Key", style="cyan") + cls_table.add_column("Value") + + cls_table.add_row("Type", cls.get("type", "unknown")) + if cls.get("impact"): + cls_table.add_row("Impact", cls["impact"]) + if cls.get("severity"): + cls_table.add_row("Severity", cls["severity"]) + cls_table.add_row("Modules", ", ".join(cls.get("modules", [])) or "None") + cls_table.add_row( + "Good First Issue", "Yes" if cls.get("good_first_issue") else "No" + ) + cls_table.add_row("Confidence", f"{cls.get('confidence', 0):.0%}") + cls_table.add_row("Reasoning", cls.get("reasoning", "")) + + console.print() + console.print(Panel(cls_table, title="LLM Classification", border_style="green")) + + # Triage analysis (for bugs) + triage = cls.get("triage") + if triage: + triage_table = Table(show_header=False, box=None, padding=(0, 2)) + triage_table.add_column("Key", style="yellow") + triage_table.add_column("Value") + + def yes_no_style(val: bool) -> str: + return "[green]Yes[/green]" if val else "[dim]No[/dim]" + + triage_table.add_row( + "Needs Test Case", + yes_no_style(triage.get("needs_test_case", False)), + ) + triage_table.add_row( + "AI Fixable", + yes_no_style(triage.get("ai_fixable", False)), + ) + triage_table.add_row( + "Potentially Fixed", + yes_no_style(triage.get("potentially_fixed", False)), + ) + triage_table.add_row( + "Potentially Outdated", + yes_no_style(triage.get("potentially_outdated", False)), + ) + if triage.get("triage_notes"): + triage_table.add_row("Notes", triage["triage_notes"]) + + console.print() + console.print(Panel(triage_table, title="Triage Analysis", border_style="yellow")) + + # Proposed labels + if issue.get("proposed_labels"): + props = issue["proposed_labels"] + + if props.get("add"): + add_text = Text() + for label in props["add"]: + add_text.append(f"+ {label}", style="green") + add_text.append(" ") + console.print() + console.print(Panel(add_text, title="Labels to Add", border_style="green")) + + if props.get("remove"): + remove_text = Text() + for label in props["remove"]: + remove_text.append(f"- {label}", style="red") + remove_text.append(" ") + console.print() + console.print( + Panel(remove_text, title="Labels to Remove", border_style="red") + ) + + +def get_review_action() -> Literal["approve", "modify", "skip", "quit"]: + """Get the user's review action.""" + console.print() + console.print("[dim]Actions: [a]pprove, [m]odify, [s]kip, [q]uit[/dim]") + action = Prompt.ask( + "Action", + choices=["a", "m", "s", "q", "approve", "modify", "skip", "quit"], + default="a", + ) + + action_map = { + "a": "approve", + "m": "modify", + "s": "skip", + "q": "quit", + } + return action_map.get(action, action) + + +def modify_classification(issue: dict) -> dict: + """Allow user to modify the classification.""" + cls = issue.get("classification", {}).copy() + + console.print("\n[yellow]Modify classification (press Enter to keep current):[/yellow]") + + # Type + current_type = cls.get("type", "") + type_choice = Prompt.ask( + f"Type [{'/'.join(TYPE_LABELS)}]", + default=current_type, + ) + if type_choice in TYPE_LABELS: + cls["type"] = type_choice + + # Impact (for bugs) + if cls.get("type") == "bug": + current_impact = cls.get("impact", "") + impact_choice = Prompt.ask( + "Impact [High/Low/none]", + default=current_impact or "none", + ) + if impact_choice in ["High", "Low"]: + cls["impact"] = impact_choice + elif impact_choice == "none": + cls["impact"] = None + + # Severity + current_severity = cls.get("severity", "") + severity_choice = Prompt.ask( + "Severity [Major/Minor/none]", + default=current_severity or "none", + ) + if severity_choice in ["Major", "Minor"]: + cls["severity"] = severity_choice + elif severity_choice == "none": + cls["severity"] = None + else: + cls["impact"] = None + cls["severity"] = None + + # Good First Issue + gfi = Confirm.ask( + "Good First Issue?", + default=cls.get("good_first_issue", False), + ) + cls["good_first_issue"] = gfi + + # Recalculate proposed labels + proposed = determine_proposed_labels_from_cls(issue, cls) + + return {"classification": cls, "proposed_labels": proposed} + + +def determine_proposed_labels_from_cls(issue: dict, cls: dict) -> dict[str, list[str]]: + """Determine proposed labels from a classification.""" + existing = set(issue["existing_labels"]) + to_add = [] + + # Type label + type_label = cls.get("type") + if type_label and type_label not in existing: + to_add.append(type_label) + + # Impact label (for bugs) + if cls.get("type") == "bug" and cls.get("impact"): + impact_label = f"Impact: {cls['impact']}" + if impact_label not in existing: + to_add.append(impact_label) + + # Severity label (for bugs) + if cls.get("type") == "bug" and cls.get("severity"): + severity_label = f"Severity: {cls['severity']}" + if severity_label not in existing: + to_add.append(severity_label) + + # Good First Issue + if cls.get("good_first_issue") and "Good First Issue" not in existing: + to_add.append("Good First Issue") + + return {"add": to_add, "remove": []} + + +def save_issues(issues: list[dict]) -> None: + """Save all issues back to the JSONL file.""" + with open(ISSUES_FILE, "w") as f: + for issue in issues: + f.write(json.dumps(issue) + "\n") + + +def review_issues(start_from: int = 0, filter_status: str | None = None) -> None: + """Interactive review of classified issues.""" + issues = load_issues() + meta = load_metadata() + + if not issues: + console.print("[yellow]No issues to review. Run fetch first.[/yellow]") + return + + # Filter to classified, unreviewed issues + review_queue = [] + for i, issue in enumerate(issues): + if issue.get("classification") is None: + continue + if filter_status and issue.get("review_status") != filter_status: + continue + if issue.get("review_status") == "approved": + continue + review_queue.append((i, issue)) + + if not review_queue: + console.print("[green]No issues to review![/green]") + return + + # Skip to start position + review_queue = [(i, iss) for i, iss in review_queue if i >= start_from] + + console.print( + f"[blue]Starting review of {len(review_queue)} issues[/blue]" + ) + console.print("[dim]Tip: Issues must be approved before labels can be applied[/dim]") + + reviewed_count = 0 + approved_count = 0 + + for idx, (i, issue) in enumerate(review_queue): + console.print(f"\n[dim]─── Issue {idx + 1} of {len(review_queue)} ───[/dim]") + + display_issue(issue) + + action = get_review_action() + + if action == "quit": + console.print("[yellow]Exiting review...[/yellow]") + break + + elif action == "skip": + console.print("[dim]Skipped[/dim]") + continue + + elif action == "modify": + modifications = modify_classification(issue) + issue["classification"] = modifications["classification"] + issue["proposed_labels"] = modifications["proposed_labels"] + issue["review_status"] = "approved" + issues[i] = issue + save_issues(issues) + console.print("[green]✓ Modified and approved[/green]") + approved_count += 1 + + elif action == "approve": + issue["review_status"] = "approved" + issues[i] = issue + save_issues(issues) + console.print("[green]✓ Approved[/green]") + approved_count += 1 + + reviewed_count += 1 + + # Update metadata + current_reviewed = meta.get("reviewed_count", 0) if meta else 0 + update_metadata({"reviewed_count": current_reviewed + approved_count}) + + # Summary + console.print() + console.print(Panel( + f"Reviewed: {reviewed_count}\nApproved: {approved_count}", + title="Review Summary", + border_style="green", + )) + + +def get_review_stats() -> dict: + """Get statistics about review progress.""" + issues = load_issues() + + if not issues: + return {"total": 0, "classified": 0, "pending_review": 0, "approved": 0} + + classified = sum(1 for i in issues if i.get("classification") is not None) + approved = sum(1 for i in issues if i.get("review_status") == "approved") + pending_review = classified - approved + + return { + "total": len(issues), + "classified": classified, + "pending_review": pending_review, + "approved": approved, + } + + +def review_single(issue_number: int) -> None: + """Review a single issue by number.""" + issues = load_issues() + + for i, issue in enumerate(issues): + if issue["number"] == issue_number: + if not issue.get("classification"): + console.print( + f"[yellow]Issue #{issue_number} has not been classified yet[/yellow]" + ) + return + + display_issue(issue) + action = get_review_action() + + if action == "approve": + issue["review_status"] = "approved" + issues[i] = issue + save_issues(issues) + console.print("[green]✓ Approved[/green]") + + elif action == "modify": + modifications = modify_classification(issue) + issue["classification"] = modifications["classification"] + issue["proposed_labels"] = modifications["proposed_labels"] + issue["review_status"] = "approved" + issues[i] = issue + save_issues(issues) + console.print("[green]✓ Modified and approved[/green]") + + return + + console.print(f"[red]Issue #{issue_number} not found[/red]")