From ab7cf2b3ec953cd0c2409a2a967cfed4ec9dd136 Mon Sep 17 00:00:00 2001 From: Benct Philip Jonsson Date: Tue, 22 Nov 2016 19:03:59 +0100 Subject: [PATCH] Allow loading patterns and exceptions from files or an array reference *The* place to find suitable files is right from the people who maintain TeX hyphenation files. --- hyphenmin.csv | 78 +++++++++++ lib/Text/Hyphen.pm | 61 +++++++- t/04-files.t | 78 +++++++++++ t/data/hyph-pt.hyp.txt | 2 + t/data/hyph-pt.lic.txt | 56 ++++++++ t/data/hyph-pt.pat.txt | 307 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 580 insertions(+), 2 deletions(-) create mode 100644 hyphenmin.csv create mode 100644 t/04-files.t create mode 100644 t/data/hyph-pt.hyp.txt create mode 100644 t/data/hyph-pt.lic.txt create mode 100644 t/data/hyph-pt.pat.txt diff --git a/hyphenmin.csv b/hyphenmin.csv new file mode 100644 index 0000000..647e026 --- /dev/null +++ b/hyphenmin.csv @@ -0,0 +1,78 @@ +"See: http://mirror.ctan.org/language/hyph-utf8/source/generic/hyph-utf8/languages.rb",,, +grc-x-ibycus,ibycus,2,2 +el-polyton,greek,1,1 +el-monoton,monogreek,1,1 +grc,ancientgreek,1,1 +cop,coptic,1,1 +de-1901,german,2,2 +de-1996,ngerman,2,2 +de-ch-1901,swissgerman,2,2 +ru,russian,2,2 +uk,ukrainian,2,2 +af,afrikaans,1,2 +ca,catalan,2,2 +cs,czech,2,3 +sk,slovak,2,3 +cy,welsh,2,3 +da,danish,2,2 +eo,esperanto,2,2 +es,spanish,2,2 +eu,basque,2,2 +fr,french,2,2 +gl,galician,2,2 +et,estonian,2,3 +fi,finnish,2,2 +hr,croatian,2,2 +hu,hungarian,2,2 +hy,armenian,1,2 +ia,interlingua,2,2 +id,indonesian,2,2 +is,icelandic,2,2 +ga,irish,2,3 +it,italian,2,2 +rm,romansh,2,2 +fur,friulan,2,2 +pms,piedmontese,2,2 +kmr,kurmanji,2,2 +la,latin,2,2 +la-x-classic,classiclatin,2,2 +la-x-liturgic,liturgicallatin,2,2 +lt,lithuanian,2,2 +lv,latvian,2,2 +nl,dutch,2,2 +oc,occitan,2,2 +pl,polish,2,2 +pt,portuguese,2,3 +zh-latn-pinyin,pinyin,1,1 +ro,romanian,2,2 +sl,slovenian,2,2 +hsb,uppersorbian,2,2 +sv,swedish,2,2 +tk,turkmen,2,2 +tr,turkish,2,2 +en-gb,ukenglish,2,3 +en-us,usenglishmax,2,3 +sh-latn,serbian,2,2 +sh-cyrl,serbianc,2,2 +mn-cyrl,mongolian,2,2 +mn-cyrl-x-lmc,mongolianlmc,2,2 +bg,bulgarian,2,2 +sa,sanskrit,1,3 +no,norwegian,2,2 +nb,bokmal,2,2 +nn,nynorsk,2,2 +as,assamese,1,1 +bn,bengali,1,1 +gu,gujarati,1,1 +hi,hindi,1,1 +kn,kannada,1,1 +ml,malayalam,1,1 +mr,marathi,1,1 +or,oriya,1,1 +pa,panjabi,1,1 +ta,tamil,1,1 +te,telugu,1,1 +th,thai,2,3 +mul-ethi,ethiopic,1,1 +ka,georgian,1,2 +cu,churchslavonic,1,2 diff --git a/lib/Text/Hyphen.pm b/lib/Text/Hyphen.pm index d3441c1..586dc84 100644 --- a/lib/Text/Hyphen.pm +++ b/lib/Text/Hyphen.pm @@ -5,6 +5,8 @@ use strict; use 5.006; +use Carp qw(croak); + =head1 NAME Text::Hyphen - determine positions for hyphens inside words @@ -82,6 +84,34 @@ English. Minimal suffix to leave wothout any hyphens. Defaults to 2 for English. +=item load_patterns, load_exceptions + +The name of a file containing hyphenation patterns or exceptions +respectively, or an array reference with patterns or exceptions +respectively, or C to explicitly not load any exceptions +or to explicitly use the defaults. If these options were missing +in the call to the constructor the builtin defaults (Knuth's data +for US English) are loaded. + +A file should contain nothing but whitespace-separated patterns +or exceptions. Suitable files can be found at +L +Those files contain UTF-8 hyphenation data for many languages, +converted to UTF-8 by the people who maintain hyphenation files +for TeX. The patterns are in the files ending in F<.pat.txt> and +the exceptions are in the files ending in F<.hyp.txt>. This +distribution contains a file F which contains +mappings from the language codes in the file names to the long +TeX names of the languages, and recommended C and +C values derived from data found at CTAN. Note that +this module has I been tested with all those data. + +=item binmode + +A suitable second argument to L, +used when reading files specified with C +or C. Defaults to C<:encoding(UTF-8)>. + =back =cut @@ -107,9 +137,19 @@ sub _add_pattern { sub _load_patterns { my $self = shift; - $self->_add_pattern($_) foreach @{$self->_PATTERNS}; + my ( $patterns, $exceptions ) = qw(patterns exceptions); + for my $data ( $patterns, $exceptions ) { + my $key = "load_$data"; + my $builtin = uc "_$data"; + $data + = exists( $self->{$key} ) ? $self->_load_data( $self->{$key} ) + : $self->{load_patterns} ? [] + : $self->$builtin; + } + + $self->_add_pattern($_) foreach @{$patterns}; - foreach my $ex (@{$self->_EXCEPTIONS}) { + foreach my $ex (@{$exceptions}) { (my $word = $ex) =~ tr/-//d; # wo-rd-le => { wordle => [0, 0, 1, 0, 1, 0, 0] } # ||a||a|| @@ -118,6 +158,23 @@ sub _load_patterns { } } +sub _load_data { + my($self, $arg) = @_; + return $arg if 'ARRAY' eq ref $arg; + return [] unless defined $arg; + croak "No such file or not a file: $arg" + unless !-d $arg and -f $arg; + local *FH; + open FH, '<', $arg or croak "Couldn't open file ($!): $arg"; + my $binmode = exists($self->{binmode}) ? $self->{binmode} : ':encoding(UTF-8)'; + binmode FH, $binmode if $binmode; + my @items = map { grep { length $_ } split /\s+/, $_ } ; + close FH or croak "Couldn't close file ($!): $arg"; + # use DDP; + # pp @items; + return \@items; +} + sub _PATTERNS { return [qw( .ach4 .ad4der .af1t .al3t .am5at .an5c .ang4 .ani5m .ant4 .an3te .anti5s .ar5s diff --git a/t/04-files.t b/t/04-files.t new file mode 100644 index 0000000..b95d836 --- /dev/null +++ b/t/04-files.t @@ -0,0 +1,78 @@ +#!/usr/bin/perl +use strict; +use warnings; + +# These must be loaded before Test::More! +use utf8; +use open qw[ :std :utf8 ]; + +use Test::More; + +BEGIN { use_ok('Text::Hyphen') }; + +my @data = ( + # Portuguese was chosen because it is the smallest data set with both + # patterns and exceptions at *the* source for these files: + # http://mirror.ctan.org/language/hyph-utf8/tex/generic/hyph-utf8/patterns/txt/ + [ 'Portuguese', + [ load_patterns => 't/data/hyph-pt.pat.txt', + load_exceptions => 't/data/hyph-pt.hyp.txt'], + { + 'Todos' => 'To-dos', + 'consciência' => 'cons-ci-ên-cia', + 'devem' => 'de-vem', + 'dignidade' => 'dig-ni-da-de', + 'direitos' => 'di-rei-tos', + 'dotados' => 'do-ta-dos', + 'espírito' => 'es-pí-ri-to', + 'fraternidade' => 'fra-ter-ni-da-de', + 'hardware' => 'hard-ware', + 'humanos' => 'hu-ma-nos', + 'iguais' => 'iguais', + 'livres' => 'li-vres', + 'nascem' => 'nas-cem', + 'outros' => 'ou-tros', + 'razão' => 'ra-zão', + 'relação' => 're-la-ção', + 'seres' => 'se-res', + 'software' => 'soft-ware', + }, + ], + # Default (English) patterns are expected to perform crappy + # on Portuguese words! This test is here to make sure the + # Portuguese patterns *were* loaded before and are *not* loaded now. + [ 'default', + [], + { 'Todos' => 'To-dos', + 'consciência' => 'con-sciên-cia', + 'devem' => 'de-vem', + 'dignidade' => 'dig-nidade', + 'direitos' => 'di-re-itos', + 'dotados' => 'dota-dos', + 'espírito' => 'es-píri-to', + 'fraternidade' => 'frater-nidade', + 'hardware' => 'hard-ware', + 'humanos' => 'hu-manos', + 'iguais' => 'iguais', + 'livres' => 'livres', + 'nascem' => 'nascem', + 'outros' => 'out-ros', + 'razão' => 'razão', + 'relação' => 're-lação', + 'seres' => 'seres', + 'software' => 'soft-ware', + }, + ], +); + +for my $lang ( @data ) { + my($name, $opts, $words, $print) = @$lang; + + my $hyp = new_ok 'Text::Hyphen', $opts, "$name hyphenator"; + + for my $word ( sort keys %$words ) { + is $hyp->hyphenate($word), $words->{$word}, "$name: $word"; + } +} + +done_testing; diff --git a/t/data/hyph-pt.hyp.txt b/t/data/hyph-pt.hyp.txt new file mode 100644 index 0000000..678628b --- /dev/null +++ b/t/data/hyph-pt.hyp.txt @@ -0,0 +1,2 @@ +hard-ware +soft-ware diff --git a/t/data/hyph-pt.lic.txt b/t/data/hyph-pt.lic.txt new file mode 100644 index 0000000..351c10d --- /dev/null +++ b/t/data/hyph-pt.lic.txt @@ -0,0 +1,56 @@ +% This file has been converted for the hyph-utf8 project from pthyph.tex +% (Version 1.2, 1996-07-21), whose authors have been identified as Pedro J. de +% Rezende and J.Joao Dias Almeida . The licence terms are unchanged. +% +% See http://www.hyphenation.org for details on the project. +% --------------------------------------------------------------------- +% BSD 3-Clause License (https://opensource.org/licenses/BSD-3-Clause): +% +% Copyright (c) 1987, Pedro J. de Rezende (rezende@ic.unicamp.br) and J.Joao Dias Almeida (jj@di.uminho.pt) +% +% All rights reserved. +% +% Redistribution and use in source and binary forms, with or without +% modification, are permitted provided that the following conditions are met: +% * Redistributions of source code must retain the above copyright +% notice, this list of conditions and the following disclaimer. +% * Redistributions in binary form must reproduce the above copyright +% notice, this list of conditions and the following disclaimer in the +% documentation and/or other materials provided with the distribution. +% * Neither the name of the University of Campinas, of the University of +% Minho nor the names of its contributors may be used to endorse or +% promote products derived from this software without specific prior +% written permission. +% +% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +% ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +% WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +% DISCLAIMED. IN NO EVENT SHALL PEDRO J. DE REZENDE OR J.JOAO DIAS ALMEIDA BE +% LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +% CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +% GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +% HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +% LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +% OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% The Portuguese TeX hyphenation table. +% (C) 2015 by Pedro J. de Rezende (rezende@ic.unicamp.br) +% and J.Joao Dias Almeida (jj@di.uminho.pt) +% Version: 1.3 Release date: 12/08/2015 +% +% (C) 1996 by Pedro J. de Rezende (rezende@ic.unicamp.br) +% and J.Joao Dias Almeida (jj@di.uminho.pt) +% Version: 1.2 Release date: 07/21/1996 +% +% (C) 1994 by Pedro J. de Rezende (rezende@ic.unicamp.br) +% Version: 1.1 Release date: 04/12/1994 +% +% (C) 1987 by Pedro J. de Rezende +% Version: 1.0 Release date: 02/13/1987 +% +% ----------------------------------------------------------------- +% Remember! If you *must* change it, then call the resulting file +% something else and attach your name to your *documented* changes. +% ================================================================= +% diff --git a/t/data/hyph-pt.pat.txt b/t/data/hyph-pt.pat.txt new file mode 100644 index 0000000..654f8a1 --- /dev/null +++ b/t/data/hyph-pt.pat.txt @@ -0,0 +1,307 @@ +1b2l +1b2r +1ba +1be +1bi +1bo +1bu +1bá +1bâ +1bã +1bé +1bí +1bó +1bú +1bê +1bõ +1c2h +1c2l +1c2r +1ca +1ce +1ci +1co +1cu +1cá +1câ +1cã +1cé +1cí +1có +1cú +1cê +1cõ +1ça +1çe +1çi +1ço +1çu +1çá +1çâ +1çã +1çé +1çí +1çó +1çú +1çê +1çõ +1d2l +1d2r +1da +1de +1di +1do +1du +1dá +1dâ +1dã +1dé +1dí +1dó +1dú +1dê +1dõ +1f2l +1f2r +1fa +1fe +1fi +1fo +1fu +1fá +1fâ +1fã +1fé +1fí +1fó +1fú +1fê +1fõ +1g2l +1g2r +1ga +1ge +1gi +1go +1gu +1gu4a +1gu4e +1gu4i +1gu4o +1gá +1gâ +1gã +1gé +1gí +1gó +1gú +1gê +1gõ +1ja +1je +1ji +1jo +1ju +1já +1jâ +1jã +1jé +1jí +1jó +1jú +1jê +1jõ +1k2l +1k2r +1ka +1ke +1ki +1ko +1ku +1ká +1kâ +1kã +1ké +1kí +1kó +1kú +1kê +1kõ +1l2h +1la +1le +1li +1lo +1lu +1lá +1lâ +1lã +1lé +1lí +1ló +1lú +1lê +1lõ +1ma +1me +1mi +1mo +1mu +1má +1mâ +1mã +1mé +1mí +1mó +1mú +1mê +1mõ +1n2h +1na +1ne +1ni +1no +1nu +1ná +1nâ +1nã +1né +1ní +1nó +1nú +1nê +1nõ +1p2l +1p2r +1pa +1pe +1pi +1po +1pu +1pá +1pâ +1pã +1pé +1pí +1pó +1pú +1pê +1põ +1qu4a +1qu4e +1qu4i +1qu4o +1ra +1re +1ri +1ro +1ru +1rá +1râ +1rã +1ré +1rí +1ró +1rú +1rê +1rõ +1sa +1se +1si +1so +1su +1sá +1sâ +1sã +1sé +1sí +1só +1sú +1sê +1sõ +1t2l +1t2r +1ta +1te +1ti +1to +1tu +1tá +1tâ +1tã +1té +1tí +1tó +1tú +1tê +1tõ +1v2l +1v2r +1va +1ve +1vi +1vo +1vu +1vá +1vâ +1vã +1vé +1ví +1vó +1vú +1vê +1võ +1w2l +1w2r +1xa +1xe +1xi +1xo +1xu +1xá +1xâ +1xã +1xé +1xí +1xó +1xú +1xê +1xõ +1za +1ze +1zi +1zo +1zu +1zá +1zâ +1zã +1zé +1zí +1zó +1zú +1zê +1zõ +a3a +a3e +a3o +c3c +e3a +e3e +e3o +i3a +i3e +i3i +i3o +i3â +i3ê +i3ô +o3a +o3e +o3o +r3r +s3s +u3a +u3e +u3o +u3u +1-