diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d8a7928..deff312 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -86,6 +86,31 @@ jobs: echo "āš ļø Large file detected: $line" done || echo "āœ“ All files within reasonable size limits" + analyze: + name: Wordlist Intelligence + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Run intelligence analysis + run: | + echo "šŸ“Š Running wordlist intelligence analysis..." + python3 scripts/analyze.py --report --report-output intelligence-report.json + + - name: Upload intelligence report + uses: actions/upload-artifact@v4 + with: + name: intelligence-report + path: intelligence-report.json + retention-days: 30 + integrity: name: Integrity Verification runs-on: ubuntu-latest diff --git a/README.md b/README.md index 2264045..05c7a06 100644 --- a/README.md +++ b/README.md @@ -286,8 +286,51 @@ python3 scripts/validate.py --file passwords.txt # Deduplicate wordlists python3 scripts/deduplicate.py passwords.txt -# Deduplicate all +# Deduplicate all wordlists (including forced-browsing/ subdirectories) python3 scripts/deduplicate.py --all + +# Case-insensitive deduplication (treat 'Password' and 'password' as same) +python3 scripts/deduplicate.py --all --case-insensitive + +# Sort entries while deduplicating +python3 scripts/deduplicate.py --all --sort +``` + +### Intelligence & Analysis Tool + +The `scripts/analyze.py` tool provides deep insights into wordlist composition, cross-referencing, and smart merging. + +**Character composition analysis** ā€” Understand the makeup of a password list: +```bash +# Analyze character class breakdown (upper, lower, digit, special) +python3 scripts/analyze.py --composition 1000000-password-seclists.txt + +# Compare multiple files +python3 scripts/analyze.py --composition cain.txt 8-more-passwords.txt +``` + +**Cross-reference** ā€” Find overlapping entries between wordlists: +```bash +# See which passwords appear in multiple sources +python3 scripts/analyze.py --cross-reference 7-more-passwords.txt 8-more-passwords.txt + +# Save detailed pairwise Jaccard similarity to JSON +python3 scripts/analyze.py --cross-reference file1.txt file2.txt --output xref.json +``` + +**Smart merging** ā€” Combine wordlists with frequency tracking: +```bash +# Merge and deduplicate, sorted alphabetically +python3 scripts/analyze.py --merge --output combined.txt list1.txt list2.txt + +# Rank by frequency (entries in more sources appear first) +python3 scripts/analyze.py --merge --rank-by-frequency --output merged.txt *.txt +``` + +**Full repository intelligence report:** +```bash +# Comprehensive analysis of all wordlists in the repo +python3 scripts/analyze.py --report --report-output report.json ``` ### CI/CD Pipeline diff --git a/scripts/README.md b/scripts/README.md index aedce60..6e5bb07 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,11 +8,12 @@ This directory contains tools for maintaining the quality and integrity of the w Validates all wordlists and generates a comprehensive manifest. **Features:** -- Encoding detection and validation +- Encoding detection and validation (UTF-8, latin-1, cp1252) - Duplicate detection - File integrity checks (SHA256) - Statistics generation (line counts, lengths, etc.) -- Manifest generation with full metadata +- Binary content detection via raw byte analysis +- Manifest generation with full metadata (dynamic date) **Usage:** ```bash @@ -30,9 +31,13 @@ python3 validate.py --file wordlist.txt ### `deduplicate.py` Remove duplicate entries from wordlists while preserving order. +Now handles wordlists in subdirectories (e.g., `forced-browsing/`). **Features:** - Order-preserving deduplication (keeps first occurrence) +- Recursive discovery of all wordlists (including nested dirs) +- Case-insensitive deduplication (`--case-insensitive`) +- Alphabetical sorting (`--sort`) - Batch processing of all wordlists - Statistics reporting (duplicates removed, percentage) @@ -44,12 +49,43 @@ python3 deduplicate.py wordlist.txt # Deduplicate with output to new file python3 deduplicate.py input.txt output.txt -# Deduplicate all wordlists in place +# Deduplicate all wordlists in place (recursive) python3 deduplicate.py --all + +# Case-insensitive dedup across entire repo +python3 deduplicate.py --all --case-insensitive + +# Sort entries while deduplicating +python3 deduplicate.py --all --sort ``` **Note:** Use `--all` with caution as it modifies files in place. +### `analyze.py` +Wordlist Intelligence & Analysis Tool ā€” new in v2.0. + +**Features:** +- **Composition analysis:** character class breakdown (upper, lower, digit, special), entropy estimation +- **Cross-reference:** pairwise Jaccard similarity, intersection/union, per-file uniqueness +- **Pattern detection:** date-like patterns, keyboard walks, repeated substrings +- **Smart merging:** combine wordlists with frequency tracking, rank by source count +- **Full repository report:** comprehensive intelligence report with per-file breakdowns + +**Usage:** +```bash +# Character composition analysis +python3 analyze.py --composition passwords.txt + +# Cross-reference two or more wordlists +python3 analyze.py --cross-reference file1.txt file2.txt + +# Smart merge with frequency ranking +python3 analyze.py --merge --rank-by-frequency --output merged.txt *.txt + +# Full repo intelligence report +python3 analyze.py --report --report-output report.json +``` + ## Requirements - Python 3.8+ diff --git a/scripts/analyze.py b/scripts/analyze.py new file mode 100644 index 0000000..6da8b71 --- /dev/null +++ b/scripts/analyze.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python3 +""" +Wordlist Intelligence & Analysis Tool + +Cross-reference, pattern analysis, and smart merging for wordlists. +Enables data-driven decisions about which wordlists to use and how they overlap. + +Features: + - Cross-reference: find entries common across multiple wordlists + - Composition: character-class breakdown (upper, lower, digit, special) + - Entropy estimation: approximate entropy per entry + - Smart merge: combine wordlists with frequency tracking + - Pattern analysis: detect dates, keyboard walks, repeated chars + - Full JSON report for CI integration + +Usage: + python3 scripts/analyze.py --cross-reference file1.txt file2.txt + python3 scripts/analyze.py --composition passwords.txt + python3 scripts/analyze.py --merge --output merged.txt file1.txt file2.txt + python3 scripts/analyze.py --all --report report.json +""" + +import argparse +import hashlib +import json +import math +import os +import sys +import time +from collections import Counter +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple + + +CHARSET_UPPER = set('ABCDEFGHIJKLMNOPQRSTUVWXYZ') +CHARSET_LOWER = set('abcdefghijklmnopqrstuvwxyz') +CHARSET_DIGIT = set('0123456789') +CHARSET_SPECIAL = set('!@#$%^&*()_+-=[]{}|;:,.<>?/~`\'"\\ ') + + +def load_entries(filepath: Path, encoding: str = 'utf-8') -> List[str]: + """Load non-empty, stripped entries from a wordlist file.""" + try: + with open(filepath, 'r', encoding=encoding, errors='ignore') as f: + content = f.read() + except Exception: + # Fallback to latin-1 + with open(filepath, 'r', encoding='latin-1', errors='ignore') as f: + content = f.read() + return [line.strip() for line in content.splitlines() if line.strip()] + + +def entropy(s: str) -> float: + """Compute Shannon entropy of a string.""" + if not s: + return 0.0 + freq = Counter(s) + length = len(s) + return -sum((c / length) * math.log2(c / length) for c in freq.values()) + + +def charset_composition(s: str) -> Dict[str, float]: + """Return fraction of each character class in the string.""" + if not s: + return {'upper': 0, 'lower': 0, 'digit': 0, 'special': 0} + total = len(s) + return { + 'upper': sum(1 for c in s if c in CHARSET_UPPER) / total, + 'lower': sum(1 for c in s if c in CHARSET_LOWER) / total, + 'digit': sum(1 for c in s if c in CHARSET_DIGIT) / total, + 'special': sum(1 for c in s if c in CHARSET_SPECIAL) / total, + } + + +def charset_categories(s: str) -> List[str]: + """Return list of charset categories present in the string.""" + cats = [] + if any(c in CHARSET_UPPER for c in s): + cats.append('upper') + if any(c in CHARSET_LOWER for c in s): + cats.append('lower') + if any(c in CHARSET_DIGIT for c in s): + cats.append('digit') + if any(c in CHARSET_SPECIAL for c in s): + cats.append('special') + return cats + + +def is_date_like(s: str) -> bool: + """Heuristic: does the string look like a date (YYYY, YYYYMMDD, etc.).""" + digits = sum(1 for c in s if c.isdigit()) + return len(s) >= 4 and digits >= len(s) * 0.6 and any( + pattern in s for pattern in + ['19', '20', '21', '22', '23', '24', '25', '26'] + ) + + +def is_keyboard_walk(s: str) -> bool: + """Simple heuristic for sequential keyboard patterns.""" + rows = ['qwertyuiop', 'asdfghjkl', 'zxcvbnm'] + s_lower = s.lower() + for i in range(len(s_lower) - 2): + for row in rows: + triplet = s_lower[i:i+3] + if triplet in row or triplet[::-1] in row: + return True + return False + + +def is_repeated_pattern(s: str) -> bool: + """Check if string is a repeated short pattern (abcabc, 123123).""" + if len(s) < 4: + return False + for period in range(1, len(s) // 2 + 1): + if len(s) % period == 0 and s[:period] * (len(s) // period) == s: + return True + return False + + +def analyze_composition(entries: List[str], sample_size: int = 100000) -> Dict: + """ + Analyze character composition of password entries. + If the list is large, analyze a random sample for speed. + """ + if not entries: + return {} + + total = len(entries) + if total > sample_size: + import random + random.seed(42) + sample = random.sample(entries, sample_size) + else: + sample = entries + + min_len = min(len(e) for e in entries) + max_len = max(len(e) for e in entries) + avg_len = sum(len(e) for e in entries) / total + + # Charset category distribution (over full set for accuracy) + cats = Counter() + for e in entries: + for cat in charset_categories(e): + cats[cat] += 1 + + # Entropy distribution (over sample for speed) + entropies = [entropy(e) for e in sample] + avg_entropy = sum(entropies) / len(entropies) if entropies else 0 + + # Pattern detection (over sample) + date_like = sum(1 for e in sample if is_date_like(e)) + keyboard = sum(1 for e in sample if is_keyboard_walk(e)) + repeated = sum(1 for e in sample if is_repeated_pattern(e)) + + # Length distribution (top 10) + length_dist = Counter(len(e) for e in entries).most_common(10) + + return { + 'total_entries': total, + 'min_length': min_len, + 'max_length': max_len, + 'avg_length': round(avg_len, 4), + 'avg_entropy': round(avg_entropy, 4), + 'categories': { + 'upper_pct': round(cats.get('upper', 0) / total * 100, 2), + 'lower_pct': round(cats.get('lower', 0) / total * 100, 2), + 'digit_pct': round(cats.get('digit', 0) / total * 100, 2), + 'special_pct': round(cats.get('special', 0) / total * 100, 2), + }, + 'patterns': { + 'date_like_pct': round(date_like / len(sample) * 100, 2), + 'keyboard_walk_pct': round(keyboard / len(sample) * 100, 2), + 'repeated_pattern_pct': round(repeated / len(sample) * 100, 2), + }, + 'top_lengths': [(str(k), v) for k, v in length_dist], + } + + +def cross_reference(filepaths: List[Path]) -> Dict: + """ + Find overlapping entries across multiple wordlists. + Returns intersection size, unique per list, and Jaccard similarities. + """ + if len(filepaths) < 2: + return {'error': 'Need at least 2 files for cross-reference'} + + names = [f.name for f in filepaths] + sets: List[Set[str]] = [] + labels: List[str] = [] + + print(f"\nLoading {len(filepaths)} wordlists for cross-reference...") + for fp in filepaths: + entries = load_entries(fp) + s = set(entries) + sets.append(s) + labels.append(f"{fp.name} ({len(s):,} unique)") + print(f" {fp.name}: {len(s):,} unique entries") + + # Pairwise Jaccard similarity + pairwise = [] + for i in range(len(sets)): + for j in range(i + 1, len(sets)): + intersection = sets[i] & sets[j] + union = sets[i] | sets[j] + jaccard = len(intersection) / len(union) if union else 0 + pairwise.append({ + 'file_a': filepaths[i].name, + 'file_b': filepaths[j].name, + 'intersection': len(intersection), + 'union': len(union), + 'jaccard_similarity': round(jaccard, 6), + }) + + # Global intersection (common to ALL files) + common_all = sets[0].copy() + for s in sets[1:]: + common_all &= s + + # Entries unique to each file + uniques = [] + for i, s in enumerate(sets): + others = set() + for j in range(len(sets)): + if i != j: + others |= sets[j] + unique_to_i = s - others + uniques.append({ + 'file': filepaths[i].name, + 'unique_entries': len(unique_to_i), + 'unique_pct': round(len(unique_to_i) / len(s) * 100, 2) if s else 0, + }) + + return { + 'files': labels, + 'common_to_all': len(common_all), + 'pairwise': pairwise, + 'unique_per_file': uniques, + } + + +def smart_merge(filepaths: List[Path], output: Path, rank_by_frequency: bool = False) -> Dict: + """ + Merge multiple wordlists, tracking source frequency. + If rank_by_frequency, entries appearing in more files are placed first. + """ + print(f"\nMerging {len(filepaths)} wordlists into {output}...") + + freq: Dict[str, int] = Counter() + per_file = [] + + for fp in filepaths: + entries = load_entries(fp) + unique = set(entries) + per_file.append(len(unique)) + for e in unique: + freq[e] += 1 + + total_unique = len(freq) + total_with_freq = sum(freq.values()) + + if rank_by_frequency: + sorted_entries = sorted(freq.items(), key=lambda x: (-x[1], x[0])) + merged = [entry for entry, count in sorted_entries] + write_lines = [f"{entry}\n" for entry in merged] + else: + merged = list(freq.keys()) + write_lines = sorted(merged + ['\n']) + + with open(output, 'w', encoding='utf-8') as f: + f.writelines(write_lines) + + # Frequency distribution + freq_dist = Counter(freq.values()) + freq_table = sorted( + [{'sources': k, 'count': v} for k, v in freq_dist.items()], + key=lambda x: -x['sources'], + ) + + print(f" Unique entries: {total_unique:,}") + print(f" Total (with multiplicities): {total_with_freq:,}") + print(f" Saved to: {output}") + + return { + 'input_files': [str(f) for f in filepaths], + 'unique_per_file': per_file, + 'merged_unique': total_unique, + 'total_with_multiplicities': total_with_freq, + 'frequency_distribution': freq_table[:50], + } + + +def generate_report(validator) -> Dict: + """ + Generate a comprehensive intelligence report on the entire repository. + Combines validation data with compositional analysis. + """ + wordlists = validator.find_wordlists() + report = { + 'generated': __import__('datetime').date.today().isoformat(), + 'repository': 'bruteforce-database', + 'total_files': len(wordlists), + 'file_reports': [], + 'global_summary': { + 'total_entries': 0, + 'total_unique_entries': 0, + 'total_size_bytes': 0, + }, + } + + print(f"\nGenerating intelligence report on {len(wordlists)} wordlists...\n") + + for wl in wordlists: + label = str(wl.relative_to(validator.root_dir)) + print(f" Analyzing {label}...", end=" ") + + entries = load_entries(wl) + if not entries: + print("[SKIP - empty]") + continue + + val = validator.validate_file(wl) + comp = analyze_composition(entries) + + report_entry = { + 'file': label, + 'size_bytes': val.get('size_bytes', 0), + 'encoding': val.get('encoding', 'unknown'), + 'sha256': val.get('sha256', ''), + 'composition': comp, + } + report['file_reports'].append(report_entry) + + report['global_summary']['total_entries'] += comp.get('total_entries', 0) + report['global_summary']['total_unique_entries'] += len(set(entries)) + report['global_summary']['total_size_bytes'] += val.get('size_bytes', 0) + + print(f"[{comp.get('total_entries', 0):,} entries]") + + return report + + +def main(): + parser = argparse.ArgumentParser( + description='Wordlist Intelligence and Analysis Tool', + ) + parser.add_argument('files', nargs='*', help='Wordlist files to analyze') + + parser.add_argument('--composition', '-c', action='store_true', + help='Analyze character composition of a wordlist') + parser.add_argument('--cross-reference', '-x', action='store_true', + help='Cross-reference entries across wordlists') + parser.add_argument('--merge', '-m', action='store_true', + help='Merge wordlists into a single deduplicated file') + parser.add_argument('--output', '-o', type=str, default='', + help='Output file for merge or report') + parser.add_argument('--rank-by-frequency', '-r', action='store_true', + help='In merge, rank entries by how many sources contain them') + parser.add_argument('--report', action='store_true', + help='Generate comprehensive intelligence report on the repository') + parser.add_argument('--report-output', type=str, default='intelligence-report.json', + help='Path for report output (default: intelligence-report.json)') + + args = parser.parse_args() + root = Path(__file__).parent.parent + + # No args: show help + if len(sys.argv) == 1: + parser.print_help() + sys.exit(0) + + # Comprehensive report + if args.report: + validator = __import__('validate', fromlist=['WordlistValidator']) + v = validator.WordlistValidator(root) + report = generate_report(v) + output_path = Path(args.report_output) + with open(output_path, 'w') as f: + json.dump(report, f, indent=2) + total = report['global_summary'] + print(f"\nReport Summary:") + print(f" Files: {report['total_files']}") + print(f" Entries: {total['total_entries']:,}") + print(f" Unique: {total['total_unique_entries']:,}") + print(f" Size: {total['total_size_bytes'] / 1024 / 1024:.2f} MB") + print(f" Report: {output_path}") + sys.exit(0) + + if not args.files: + print("Error: specify wordlist files to analyze") + sys.exit(1) + + filepaths = [Path(f) for f in args.files] + missing = [str(f) for f in filepaths if not f.exists()] + if missing: + print(f"Error: files not found: {', '.join(missing)}") + sys.exit(1) + + # Composition analysis + if args.composition: + for fp in filepaths: + entries = load_entries(fp) + print(f"\n{'=' * 50}") + print(f"File: {fp.name}") + print(f"{'=' * 50}") + comp = analyze_composition(entries) + print(f" Entries: {comp.get('total_entries', 0):,}") + print(f" Length: {comp.get('min_length')} - {comp.get('max_length')} " + f"(avg {comp.get('avg_length')})") + print(f" Avg Entropy: {comp.get('avg_entropy')}") + print(f"\n Character Classes:") + cats = comp.get('categories', {}) + print(f" Uppercase: {cats.get('upper_pct', 0)}%") + print(f" Lowercase: {cats.get('lower_pct', 0)}%") + print(f" Digits: {cats.get('digit_pct', 0)}%") + print(f" Special: {cats.get('special_pct', 0)}%") + print(f"\n Patterns (sample):") + pat = comp.get('patterns', {}) + print(f" Date-like: {pat.get('date_like_pct', 0)}%") + print(f" Keyboard walk: {pat.get('keyboard_walk_pct', 0)}%") + print(f" Repeated: {pat.get('repeated_pattern_pct', 0)}%") + sys.exit(0) + + # Cross-reference + if args.cross_reference: + xref = cross_reference(filepaths) + print(f"\nCross-Reference Results:") + print(f" Common to all: {xref.get('common_to_all', 0):,} entries") + print(f"\n Unique per file:") + for u in xref.get('unique_per_file', []): + print(f" {u['file']}: {u['unique_entries']:,} ({u['unique_pct']}%)") + print(f"\n Pairwise Jaccard Similarity:") + for p in xref.get('pairwise', []): + print(f" {p['file_a']} vs {p['file_b']}: " + f"{p['jaccard_similarity']:.4f} " + f"(intersection: {p['intersection']:,})") + if args.output: + with open(args.output, 'w') as f: + json.dump(xref, f, indent=2) + print(f"\n Report saved to: {args.output}") + sys.exit(0) + + # Merge + if args.merge: + if not args.output: + print("Error: --output is required for merge") + sys.exit(1) + result = smart_merge(filepaths, Path(args.output), args.rank_by_frequency) + + # Write frequency metadata + meta_path = Path(args.output).with_suffix('.meta.json') + with open(meta_path, 'w') as f: + json.dump(result, f, indent=2) + print(f" Metadata: {meta_path}") + sys.exit(0) + + # Default: show info + print("Specify an action: --composition, --cross-reference, --merge, or --report") + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/scripts/deduplicate.py b/scripts/deduplicate.py index da4f892..045f1d3 100755 --- a/scripts/deduplicate.py +++ b/scripts/deduplicate.py @@ -3,17 +3,39 @@ Wordlist Deduplication Tool Remove duplicate entries from wordlists while preserving order and quality. +Now handles nested wordlists (forced-browsing/, etc.) with case-insensitive option. Philosophy: Perfection is achieved not when there is nothing more to add, but when there is nothing left to take away. """ import sys +import time from pathlib import Path -from typing import Set, List - - -def deduplicate_file(filepath: Path, output_path: Path = None, preserve_order: bool = True): +from typing import Set, List, Optional + + +def read_lines_safely(filepath: Path) -> List[str]: + """Read lines with automatic encoding detection. Never silently corrupts data.""" + # Try UTF-8 first (most common), fall back to latin-1/cp1252 + for enc in ('utf-8', 'latin-1', 'cp1252'): + try: + with open(filepath, 'r', encoding=enc) as f: + return f.readlines() + except (UnicodeDecodeError, ValueError): + continue + # Last resort: read as latin-1 (always succeeds, never corrupts) + with open(filepath, 'r', encoding='latin-1') as f: + return f.readlines() + + +def deduplicate_file( + filepath: Path, + output_path: Optional[Path] = None, + preserve_order: bool = True, + case_insensitive: bool = False, + sort_output: bool = False, +): """ Remove duplicates from a wordlist file. @@ -21,76 +43,154 @@ def deduplicate_file(filepath: Path, output_path: Path = None, preserve_order: b filepath: Input wordlist file output_path: Output file (overwrites input if None) preserve_order: Keep first occurrence order (slower but maintains context) + case_insensitive: Treat entries with different case as duplicates + sort_output: Sort entries alphabetically in output """ - print(f"šŸ“‹ Processing {filepath.name}...") + root = Path(__file__).parent.parent + rel_path = str(filepath.relative_to(root)) if root in filepath.parents else str(filepath) + print(f" Processing {rel_path}...", end=" ") - # Read file - with open(filepath, 'r', encoding='utf-8', errors='ignore') as f: - lines = f.readlines() + lines = read_lines_safely(filepath) original_count = len(lines) - print(f" Original: {original_count:,} lines") + original_nonempty = sum(1 for line in lines if line.strip()) # Deduplicate - if preserve_order: + unique_lines: List[str] = [] + + if sort_output: + seen: Set[str] = set() + for line in lines: + stripped = line.strip() + if not stripped: + continue + key = stripped.lower() if case_insensitive else stripped + if key not in seen: + seen.add(key) + unique_lines.append(stripped) + unique_lines = sorted(unique_lines) + unique_lines = [line + '\n' for line in unique_lines] + elif preserve_order: seen: Set[str] = set() - unique_lines: List[str] = [] for line in lines: stripped = line.strip() - if stripped and stripped not in seen: - seen.add(stripped) + if not stripped: + continue + key = stripped.lower() if case_insensitive else stripped + if key not in seen: + seen.add(key) unique_lines.append(stripped + '\n') else: - # Faster but changes order - unique_lines = sorted(set(line.strip() for line in lines if line.strip())) + seen: Set[str] = set() + for line in lines: + stripped = line.strip() + if not stripped: + continue + key = stripped.lower() if case_insensitive else stripped + if key not in seen: + seen.add(key) + unique_lines.append(stripped) + unique_lines = sorted(set(unique_lines)) unique_lines = [line + '\n' for line in unique_lines] unique_count = len(unique_lines) - removed = original_count - unique_count + removed = original_nonempty - unique_count - print(f" Unique: {unique_count:,} lines") - print(f" Removed: {removed:,} duplicates ({removed/original_count*100:.1f}%)") + print(f"{original_nonempty:,} -> {unique_count:,} entries ({removed:,} removed, " + f"{removed / original_nonempty * 100:.1f}% reduction)" + f"{' [CI]' if case_insensitive else ''}") # Write output output = output_path or filepath with open(output, 'w', encoding='utf-8') as f: f.writelines(unique_lines) - print(f" āœ“ Saved to {output.name}\n") - return { - "original": original_count, + "file": str(rel_path), + "original": original_nonempty, "unique": unique_count, "removed": removed, - "percentage": removed/original_count*100 if original_count > 0 else 0 + "percentage": removed / original_nonempty * 100 if original_nonempty > 0 else 0, } +def find_wordlists(root: Path) -> List[Path]: + """Find all wordlist files recursively, skipping .git and scripts.""" + skip_dirs = {'.git', 'node_modules', '__pycache__'} + wordlists = [] + for ext in ('*.txt', '*.lst'): + for f in root.rglob(ext): + if not any(skip in f.parts for skip in skip_dirs): + wordlists.append(f) + return sorted(wordlists) + + +def deduplicate_all( + root: Path, + preserve_order: bool = True, + case_insensitive: bool = False, + sort_output: bool = False, +): + """Deduplicate all wordlists in the repository.""" + wordlists = find_wordlists(root) + print(f"\nDeduplicating {len(wordlists)} wordlists in " + f"{'CI mode' if case_insensitive else 'standard mode'}...\n") + + total_removed = 0 + results = [] + for wordlist in wordlists: + result = deduplicate_file( + wordlist, + preserve_order=preserve_order, + case_insensitive=case_insensitive, + sort_output=sort_output, + ) + total_removed += result["removed"] + results.append(result) + + print(f"\nComplete! Removed {total_removed:,} total duplicates across {len(wordlists)} files.") + return results + + def main(): """Main entry point.""" - if len(sys.argv) < 2: - print("Usage: deduplicate.py [output.txt]") - print(" deduplicate.py --all") - sys.exit(1) - - if sys.argv[1] == "--all": - # Deduplicate all wordlists in place + import argparse + + parser = argparse.ArgumentParser( + description="Deduplicate wordlist files in the bruteforce-database repository.", + ) + parser.add_argument('input', nargs='?', help='Single wordlist file to deduplicate') + parser.add_argument('output', nargs='?', help='Output file (default: overwrite input)') + parser.add_argument('--all', action='store_true', help='Deduplicate all wordlists in repo') + parser.add_argument('--case-insensitive', action='store_true', + help='Treat same words with different case as duplicates') + parser.add_argument('--sort', action='store_true', help='Sort entries alphabetically') + parser.add_argument('--no-preserve-order', action='store_true', + help='Allow reordering for faster dedup') + + args = parser.parse_args() + + if args.all: root = Path(__file__).parent.parent - wordlists = list(root.glob('*.txt')) + list(root.glob('*.lst')) - - print(f"šŸ”„ Deduplicating {len(wordlists)} wordlists...\n") - - total_removed = 0 - for wordlist in sorted(wordlists): - result = deduplicate_file(wordlist) - total_removed += result["removed"] - - print(f"āœØ Complete! Removed {total_removed:,} total duplicates.") + deduplicate_all( + root, + preserve_order=not args.no_preserve_order, + case_insensitive=args.case_insensitive, + sort_output=args.sort, + ) + elif args.input: + input_file = Path(args.input) + output_file = Path(args.output) if args.output else None + deduplicate_file( + input_file, + output_file, + preserve_order=not args.no_preserve_order, + case_insensitive=args.case_insensitive, + sort_output=args.sort, + ) else: - # Deduplicate single file - input_file = Path(sys.argv[1]) - output_file = Path(sys.argv[2]) if len(sys.argv) > 2 else None - deduplicate_file(input_file, output_file) + parser.print_help() + sys.exit(1) if __name__ == "__main__": diff --git a/scripts/validate.py b/scripts/validate.py index 8fae7ad..e47254c 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -33,12 +33,15 @@ def validate_file(self, filepath: Path) -> Dict: Validate a single wordlist file. Returns comprehensive metadata and validation results. + Uses streaming reads for large files to minimize memory. """ if not filepath.exists(): return {"error": "File does not exist"} result = { - "path": str(filepath.relative_to(self.root_dir)), + "path": str(filepath.relative_to(self.root_dir)) + if self.root_dir in filepath.parents + else str(filepath), "size_bytes": filepath.stat().st_size, "valid": True, "errors": [], @@ -46,34 +49,53 @@ def validate_file(self, filepath: Path) -> Dict: } try: - # Detect encoding + # Read raw bytes (needed once for hash + encoding detection) with open(filepath, 'rb') as f: raw_data = f.read() result["sha256"] = hashlib.sha256(raw_data).hexdigest() - # Try UTF-8 first - try: - content = raw_data.decode('utf-8') - result["encoding"] = "utf-8" - except UnicodeDecodeError: + # Detect encoding - try UTF-8 first, then latin-1, then system default + encoding = None + content = None + for enc in ('utf-8', 'latin-1', 'cp1252'): try: - content = raw_data.decode('latin-1') - result["encoding"] = "latin-1" - result["warnings"].append("Non-UTF-8 encoding detected") - except Exception as e: - result["valid"] = False - result["errors"].append(f"Encoding error: {e}") - return result - - # Analyze content + content = raw_data.decode(enc) + encoding = enc + break + except (UnicodeDecodeError, LookupError): + continue + + if content is None: + result["valid"] = False + result["errors"].append("Unable to decode file with UTF-8, latin-1, or cp1252") + return result + + result["encoding"] = encoding + if encoding != 'utf-8': + result["warnings"].append(f"Non-UTF-8 encoding detected ({encoding})") + + # Check for binary content on raw bytes (fast, no line iteration) + null_bytes = raw_data.count(b'\x00') + control_chars = sum( + 1 for b in raw_data if b < 0x20 and b not in (0x09, 0x0A, 0x0D) + ) + null_ratio = null_bytes / len(raw_data) if raw_data else 0 + control_ratio = control_chars / len(raw_data) if raw_data else 0 + + if null_ratio > 0.01: + result["warnings"].append( + f"Binary content detected ({null_bytes} null bytes, " + f"{null_ratio:.2%} of file)" + ) + + # Analyze content line by line lines = content.splitlines() result["total_lines"] = len(lines) - # Filter empty lines non_empty_lines = [line for line in lines if line.strip()] result["non_empty_lines"] = len(non_empty_lines) - # Check for duplicates + # Check for duplicates using set unique_entries = set(non_empty_lines) result["unique_entries"] = len(unique_entries) @@ -89,9 +111,12 @@ def validate_file(self, filepath: Path) -> Dict: result["max_length"] = max(lengths) result["avg_length"] = sum(lengths) / len(lengths) - # Check for binary content - if any(ord(c) < 32 and c not in '\t\n\r' for line in lines[:100] for c in line): - result["warnings"].append("Possible binary content detected") + # Warn on suspicious control character ratio + if control_ratio > 0.001 and null_ratio <= 0.01: + result["warnings"].append( + f"Unusual control characters detected ({control_chars} chars, " + f"{control_ratio:.4%} of file)" + ) except Exception as e: result["valid"] = False @@ -100,28 +125,22 @@ def validate_file(self, filepath: Path) -> Dict: return result def find_wordlists(self) -> List[Path]: - """Find all wordlist files (*.txt, *.lst) in the repository.""" - wordlists = [] - - # Skip certain directories + """Find all wordlist files (*.txt, *.lst) recursively, skipping .git and scripts.""" skip_dirs = {'.git', 'node_modules', 'scripts', '__pycache__'} - - for txt_file in self.root_dir.rglob('*.txt'): - if not any(skip in txt_file.parts for skip in skip_dirs): - wordlists.append(txt_file) - - for lst_file in self.root_dir.rglob('*.lst'): - if not any(skip in lst_file.parts for skip in skip_dirs): - wordlists.append(lst_file) - + wordlists = [] + for ext in ('*.txt', '*.lst'): + for f in self.root_dir.rglob(ext): + if not any(skip in f.parts for skip in skip_dirs): + wordlists.append(f) return sorted(wordlists) def validate_all(self) -> Dict: """Validate all wordlists and generate comprehensive report.""" wordlists = self.find_wordlists() + today = __import__('datetime').date.today().isoformat() results = { - "validation_date": "2025-11-16", + "validation_date": today, "total_files": len(wordlists), "files": [], "summary": { @@ -134,25 +153,29 @@ def validate_all(self) -> Dict: } } - print(f"šŸ” Validating {len(wordlists)} wordlist files...\n") + print(f"\nValidating {len(wordlists)} wordlist files...\n") for wordlist in wordlists: - print(f" Checking {wordlist.name}...", end=" ") + label = str(wordlist.relative_to(self.root_dir)) + print(f" {label} ...", end=" ") file_result = self.validate_file(wordlist) results["files"].append(file_result) - if file_result["valid"]: - results["summary"]["valid_files"] += 1 - print("āœ“") - else: - results["summary"]["invalid_files"] += 1 - print("āœ—") + status = "OK" if file_result["valid"] else "FAIL" + print(f"[{status}]") results["summary"]["total_warnings"] += len(file_result.get("warnings", [])) results["summary"]["total_size_bytes"] += file_result.get("size_bytes", 0) results["summary"]["total_entries"] += file_result.get("non_empty_lines", 0) results["summary"]["total_unique_entries"] += file_result.get("unique_entries", 0) + if file_result["valid"]: + results["summary"]["valid_files"] += 1 + else: + results["summary"]["invalid_files"] += 1 + for err in file_result.get("errors", []): + print(f" Error: {err}") + return results def generate_manifest(self, output_path: Path = None): @@ -161,24 +184,21 @@ def generate_manifest(self, output_path: Path = None): results = self.validate_all() - # Pretty print summary summary = results["summary"] - print(f"\nšŸ“Š Validation Summary:") - print(f" Total files: {results['total_files']}") - print(f" Valid: {summary['valid_files']} āœ“") - print(f" Invalid: {summary['invalid_files']} āœ—") - print(f" Warnings: {summary['total_warnings']}") - print(f" Total size: {summary['total_size_bytes'] / 1024 / 1024:.2f} MB") - print(f" Total entries: {summary['total_entries']:,}") - print(f" Unique entries: {summary['total_unique_entries']:,}") - - # Save manifest + print(f"\nValidation Summary:") + print(f" Files: {results['total_files']} total, " + f"{summary['valid_files']} valid, " + f"{summary['invalid_files']} invalid") + print(f" Warnings: {summary['total_warnings']}") + print(f" Size: {summary['total_size_bytes'] / 1024 / 1024:.2f} MB") + print(f" Entries: {summary['total_entries']:,}") + print(f" Unique: {summary['total_unique_entries']:,}") + with open(output_path, 'w') as f: json.dump(results, f, indent=2) - print(f"\nšŸ’¾ Manifest saved to {output_path}") + print(f"\nManifest saved to {output_path}") - # Return exit code based on validation return 0 if summary['invalid_files'] == 0 else 1 @@ -187,13 +207,14 @@ def main(): validator = WordlistValidator() if len(sys.argv) > 1 and sys.argv[1] == "--file": - # Validate single file - filepath = Path(sys.argv[2]) + filepath = Path(sys.argv[2]).resolve() + if not filepath.exists(): + print(f"Error: file not found: {filepath}") + sys.exit(1) result = validator.validate_file(filepath) print(json.dumps(result, indent=2)) - sys.exit(0 if result["valid"] else 1) + sys.exit(0 if result.get("valid", False) else 1) else: - # Validate all and generate manifest sys.exit(validator.generate_manifest())