feat: add keywords CLI tool for text vectorization (#122)

.rubocop.yml

@@ -30,6 +30,7 @@ Metrics/BlockLength:
     - 'test/**/*'
     - 'lib/classifier/extensions/vector.rb'
     - 'lib/classifier/cli.rb'
+    - 'lib/classifier/keywords/cli.rb'
 
 # Allow longer methods in complex algorithms (SVD, etc.) and CLI
 Metrics/MethodLength:
@@ -39,6 +40,7 @@ Metrics/MethodLength:
     - 'lib/classifier/extensions/vector.rb'
     - 'lib/classifier/lsi/content_node.rb'
     - 'lib/classifier/cli.rb'
+    - 'lib/classifier/keywords/cli.rb'
 
 # Allow higher complexity for mathematical algorithms and CLI
 Metrics/AbcSize:
@@ -49,6 +51,7 @@ Metrics/AbcSize:
     - 'lib/classifier/lsi.rb'
     - 'lib/classifier/lsi/content_node.rb'
     - 'lib/classifier/cli.rb'
+    - 'lib/classifier/keywords/cli.rb'
 
 Metrics/CyclomaticComplexity:
   Max: 10
@@ -63,6 +66,7 @@ Metrics/PerceivedComplexity:
     - 'lib/classifier/extensions/vector.rb'
     - 'lib/classifier/lsi/content_node.rb'
     - 'lib/classifier/cli.rb'
+    - 'lib/classifier/keywords/cli.rb'
 
 # Class length limits - algorithms, tests and CLI can be longer
 Metrics/ClassLength:

README.md

@@ -63,6 +63,49 @@ classifier "Great product, highly recommend"
 # => positive
 ```
 
+Extract keywords and analyze term importance using TF-IDF instantly:
+
+```bash
+# Extract from a raw string
+keywords "Ruby is a programming language"
+# => ruby:0.52 programming:0.41 language:0.38
+
+# Extract from a file
+keywords extract article.txt
+# => machine:0.61 learning:0.58 neural:0.45 network:0.42
+
+# Pipeline with stdin and web data
+curl -s https://example.com/article | keywords extract
+
+# Get top 5 terms only
+keywords -n 5 "long document with many terms..."
+
+# Use a custom model file
+keywords -m custom_model.json "Ruby is a programming language"
+```
+
+Build your own vocabulary (fit data):
+```bash
+# Fit from multiple files
+keywords fit corpus/*.txt
+
+# Fit from stdin (each line is treated as a separate document)
+cat documents.txt | keywords fit
+
+# Tune vocabulary filters during fitting
+keywords fit --min-df 2 --max-df 0.85 --ngram 1,2 corpus/*.txt
+```
+
+Inspect your model:
+```bash
+# Check model statistics and parameters
+keywords info
+# => Documents: 1,234
+# => Vocabulary: 5,678
+# => Min DF: 1
+# => Max DF: 1.0
+```
+
 [CLI Guide →](https://rubyclassifier.com/docs/guides/cli/basics)
 
 ### Claude Code Plugin

classifier.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 3.1'
   s.files = Dir['{lib,sig,exe}/**/*.{rb,rbs}', 'ext/**/*.{c,h,rb}', 'exe/*', 'bin/*', 'LICENSE', '*.md', 'test/*']
   s.bindir = 'exe'
-  s.executables = ['classifier']
+  s.executables = %w[classifier keywords]
   s.extensions = ['ext/classifier/extconf.rb']
   s.license = 'LGPL'
 

exe/classifier

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Force UTF-8 encoding for proper handling of model data and user input
 Encoding.default_external = Encoding::UTF_8
 Encoding.default_internal = Encoding::UTF_8
 

exe/keywords

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Force UTF-8 encoding for proper handling of model data and user input
+Encoding.default_external = Encoding::UTF_8
+Encoding.default_internal = Encoding::UTF_8
+
+require 'classifier/keywords/cli'
+
+result = Classifier::Keywords::CLI.new(ARGV).run
+
+warn result[:error] unless result[:error].empty?
+puts result[:output] unless result[:output].empty?
+
+exit result[:exit_code]

lib/classifier/extensions/word_hash.rb

@@ -33,6 +33,12 @@ def clean_word_hash(min_word_length = 3)
     word_hash_for_words(gsub(/[^\w\s]/, '').split, min_word_length)
   end
 
+  # Builds a mapping between stemmed roots and their most frequent original words.
+  # @rbs (?Integer) -> Hash[Symbol, String]
+  def stem_to_word_hash(min_word_length = 3)
+    mapping_stem_to_word_for_words(gsub(/[^\w\s]/, '').split, min_word_length)
+  end
+
   private
 
   # @rbs (Array[String], Integer) -> Hash[Symbol, Integer]
@@ -54,6 +60,20 @@ def word_hash_for_symbols(words)
     d
   end
 
+  # @rbs (Array[String], Integer) -> Hash[Symbol, Integer]
+  def mapping_stem_to_word_for_words(words, min_word_length)
+    h = {}
+    words.map { _1.tap(&:downcase!) }.tally.each do |word, count|
+      next unless !CORPUS_SKIP_WORDS.include?(word) && word.length >= min_word_length
+
+      stem = word.stem.intern
+      h[stem] ||= [word, count]
+      h[stem] = [word, count] if h.dig(stem, 1) < count
+    end
+    h.each_key { |k| h[k] = h[k].first }
+    h
+  end
+
   CORPUS_SKIP_WORDS = ::Set.new(%w[
                                   a
                                   again