diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7acabee --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,36 @@ +name: CI + +on: + push: + branches: + - master + - 'feature/**' + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go: [oldstable, stable] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + + - name: gofmt + run: test -z "$(gofmt -l .)" + + - name: vet + run: go vet ./... + + - name: test + run: go test -race ./... + + - name: fuzz + # A short differential-fuzzing smoke on top of the seed corpus, + # which `go test` above already runs; see FuzzMatchRegexp. + run: go test -fuzz=FuzzMatchRegexp -fuzztime=30s . diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 0000000..c48d30c --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,54 @@ +name: Fuzz + +on: + schedule: + # Nightly; the odd minute avoids the on-the-hour load spikes. + - cron: '17 3 * * *' + workflow_dispatch: + # Allows manual runs from the Actions tab. + +jobs: + fuzz: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: stable + + - name: locate fuzz cache + id: cache-dir + run: echo "dir=$(go env GOCACHE)/fuzz" >> "$GITHUB_OUTPUT" + + - name: restore corpus + uses: actions/cache/restore@v4 + with: + path: ${{ steps.cache-dir.outputs.dir }} + # A fresh key every run plus a prefix restore key: the job always + # starts from the most recent corpus and always saves the evolved + # one, so the coverage exploration accumulates across nights. + key: fuzz-corpus-${{ github.run_id }} + restore-keys: | + fuzz-corpus- + + - name: fuzz + run: go test -fuzz=FuzzMatchRegexp -fuzztime=30m . + + - name: save corpus + # Save even when the fuzzing found a failure -- especially then. + if: always() + uses: actions/cache/save@v4 + with: + path: ${{ steps.cache-dir.outputs.dir }} + key: fuzz-corpus-${{ github.run_id }} + + - name: upload crashers + # A found failure is minimized and written into testdata/fuzz/; + # keep it as an artifact so it can be committed as a regression. + if: failure() + uses: actions/upload-artifact@v4 + with: + name: crashers + path: testdata/fuzz/ diff --git a/.gitignore b/.gitignore index b4ae623..534b466 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ glob.iml *.dot *.png *.svg +patterns.txt +*.bench diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1ef96cc..0000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: go -go: - - "1.7.X" - - "1.8.X" - - "1.9.X" - - "1.10.X" - - master - -matrix: - allow_failures: - - go: master -fast_finish: true - -script: - - go test -v ./... diff --git a/bench.sh b/bench.sh index 804cf22..8570697 100755 --- a/bench.sh +++ b/bench.sh @@ -1,26 +1,41 @@ -#! /bin/bash +#!/bin/bash +# +# Compares the benchmarks of the current branch against a git revision: +# +# ./bench.sh v0.2.3 # all the benchmarks +# ./bench.sh master 'Match' # the ones matching a -bench regexp +# +# The results are written to *.bench files in the current directory and +# compared with benchstat (go install golang.org/x/perf/cmd/benchstat@latest). -bench() { - filename="/tmp/$1-$2.bench" - if test -e "${filename}"; - then - echo "Already exists ${filename}" - else - backup=`git rev-parse --abbrev-ref HEAD` - git checkout $1 - echo -n "Creating ${filename}... " - go test ./... -run=NONE -bench=$2 > "${filename}" -benchmem - echo "OK" - git checkout ${backup} - sleep 5 - fi -} +set -eu + +prev=$1 +what=${2:-.} +curr=$(git rev-parse --abbrev-ref HEAD) +rnd=$(head -c4 "$out" + echo "OK" + git checkout -q "$curr" + sleep 5 +} -bench ${to} $2 -bench ${current} $2 +bench "$prev" +bench "$curr" -benchcmp $3 "/tmp/${to}-$2.bench" "/tmp/${current}-$2.bench" +benchstat "$(file "$prev")" "$(file "$curr")" diff --git a/cmd/globdraw/main.go b/cmd/globdraw/main.go deleted file mode 100644 index 585880d..0000000 --- a/cmd/globdraw/main.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "github.com/gobwas/glob" - "github.com/gobwas/glob/match" - "github.com/gobwas/glob/match/debug" - "os" - "strings" - "unicode/utf8" -) - -func main() { - pattern := flag.String("p", "", "pattern to draw") - sep := flag.String("s", "", "comma separated list of separators characters") - flag.Parse() - - if *pattern == "" { - flag.Usage() - os.Exit(1) - } - - var separators []rune - if len(*sep) > 0 { - for _, c := range strings.Split(*sep, ",") { - if r, w := utf8.DecodeRuneInString(c); len(c) > w { - fmt.Println("only single charactered separators are allowed") - os.Exit(1) - } else { - separators = append(separators, r) - } - } - } - - glob, err := glob.Compile(*pattern, separators...) - if err != nil { - fmt.Println("could not compile pattern:", err) - os.Exit(1) - } - - matcher := glob.(match.Matcher) - fmt.Fprint(os.Stdout, debug.Graphviz(*pattern, matcher)) -} diff --git a/cmd/globtest/main.go b/cmd/globtest/main.go index 95c102f..34f26ae 100644 --- a/cmd/globtest/main.go +++ b/cmd/globtest/main.go @@ -1,13 +1,17 @@ package main import ( + "errors" "flag" "fmt" - "github.com/gobwas/glob" "os" "strings" "testing" + "text/tabwriter" "unicode/utf8" + + "github.com/gobwas/glob" + "github.com/gobwas/glob/internal/debug" ) func benchString(r testing.BenchmarkResult) string { @@ -31,52 +35,91 @@ func benchString(r testing.BenchmarkResult) string { return fmt.Sprintf("%8d\t%s\t%s allocs", r.N, ns, allocs) } +// pointAt renders the pattern with a pointer at the offset of the syntax +// error, followed by its reason: +// +// {a,b +// ----^ unclosed `{` +func pointAt(pattern string, err *glob.SyntaxError) string { + // The offset is in bytes, while the pointer is drawn in columns. + col := utf8.RuneCountInString(pattern[:min(err.Offset, len(pattern))]) + return fmt.Sprintf("%s\n%s^ %s\n\n", pattern, strings.Repeat("-", col), err.Reason) +} + func main() { - pattern := flag.String("p", "", "pattern to draw") - sep := flag.String("s", "", "comma separated list of separators") - fixture := flag.String("f", "", "fixture") - verbose := flag.Bool("v", false, "verbose") + var ( + pattern = flag.String("p", "", "pattern to draw") + sep = flag.String("s", "", "comma separated list of separators") + fixture = flag.String("f", "", "fixture") + benchCompile = flag.Bool("bench-compile", false, "benchmark compilation time") + benchMatch = flag.Bool("bench-match", false, "benchmark matching time") + verbose = flag.Bool("v", false, "print the pattern, fixture, compiled matcher tree and result; without it only the exit status tells: 0 on match, 1 otherwise") + ) + + // Expose testing package flags. + testing.Init() + flag.Parse() if *pattern == "" { flag.Usage() - os.Exit(1) + os.Exit(2) } var separators []rune for _, c := range strings.Split(*sep, ",") { - if r, w := utf8.DecodeRuneInString(c); len(c) > w { - fmt.Println("only single charactered separators are allowed") - os.Exit(1) - } else { - separators = append(separators, r) + if len(c) == 0 { + continue } + r, w := utf8.DecodeRuneInString(c) + if len(c) > w { + fmt.Fprintln(os.Stderr, "only single charactered separators are allowed") + os.Exit(2) + } + separators = append(separators, r) } g, err := glob.Compile(*pattern, separators...) if err != nil { - fmt.Println("could not compile pattern:", err) - os.Exit(1) + fmt.Fprintln(os.Stderr, "could not compile pattern:", err) + var syntaxErr *glob.SyntaxError + if errors.As(err, &syntaxErr) { + fmt.Fprint(os.Stderr, "\n"+pointAt(*pattern, syntaxErr)) + } + os.Exit(2) } + matched := g.Match(*fixture) - if !*verbose { - fmt.Println(g.Match(*fixture)) - return + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + if *verbose { + fmt.Fprintf(w, "pattern:\t%s\n", g) + fmt.Fprintf(w, "fixture:\t%s\n", *fixture) + fmt.Fprintf(w, "matchers:\t%s\n", debug.Tree(g)) + fmt.Fprintf(w, "result:\t%t\n", matched) + // Flush before the benchmarks: they take a while, and the result + // must not wait for them. + w.Flush() } - fmt.Printf("result: %t\n", g.Match(*fixture)) - - cb := testing.Benchmark(func(b *testing.B) { - for i := 0; i < b.N; i++ { - glob.Compile(*pattern, separators...) - } - }) - fmt.Println("compile:", benchString(cb)) + if *benchCompile { + b := testing.Benchmark(func(b *testing.B) { + for i := 0; i < b.N; i++ { + glob.Compile(*pattern, separators...) + } + }) + fmt.Fprintf(w, "compile:\t%s\n", benchString(b)) + } + if *benchMatch { + b := testing.Benchmark(func(b *testing.B) { + for i := 0; i < b.N; i++ { + g.Match(*fixture) + } + }) + fmt.Fprintf(w, "match:\t%s\n", benchString(b)) + } + w.Flush() - mb := testing.Benchmark(func(b *testing.B) { - for i := 0; i < b.N; i++ { - g.Match(*fixture) - } - }) - fmt.Println("match: ", benchString(mb)) + if !matched { + os.Exit(1) + } } diff --git a/compiler/compiler.go b/compiler/compiler.go deleted file mode 100644 index 02e7de8..0000000 --- a/compiler/compiler.go +++ /dev/null @@ -1,525 +0,0 @@ -package compiler - -// TODO use constructor with all matchers, and to their structs private -// TODO glue multiple Text nodes (like after QuoteMeta) - -import ( - "fmt" - "reflect" - - "github.com/gobwas/glob/match" - "github.com/gobwas/glob/syntax/ast" - "github.com/gobwas/glob/util/runes" -) - -func optimizeMatcher(matcher match.Matcher) match.Matcher { - switch m := matcher.(type) { - - case match.Any: - if len(m.Separators) == 0 { - return match.NewSuper() - } - - case match.AnyOf: - if len(m.Matchers) == 1 { - return m.Matchers[0] - } - - return m - - case match.List: - if m.Not == false && len(m.List) == 1 { - return match.NewText(string(m.List)) - } - - return m - - case match.BTree: - m.Left = optimizeMatcher(m.Left) - m.Right = optimizeMatcher(m.Right) - - r, ok := m.Value.(match.Text) - if !ok { - return m - } - - var ( - leftNil = m.Left == nil - rightNil = m.Right == nil - ) - if leftNil && rightNil { - return match.NewText(r.Str) - } - - _, leftSuper := m.Left.(match.Super) - lp, leftPrefix := m.Left.(match.Prefix) - la, leftAny := m.Left.(match.Any) - - _, rightSuper := m.Right.(match.Super) - rs, rightSuffix := m.Right.(match.Suffix) - ra, rightAny := m.Right.(match.Any) - - switch { - case leftSuper && rightSuper: - return match.NewContains(r.Str, false) - - case leftSuper && rightNil: - return match.NewSuffix(r.Str) - - case rightSuper && leftNil: - return match.NewPrefix(r.Str) - - case leftNil && rightSuffix: - return match.NewPrefixSuffix(r.Str, rs.Suffix) - - case rightNil && leftPrefix: - return match.NewPrefixSuffix(lp.Prefix, r.Str) - - case rightNil && leftAny: - return match.NewSuffixAny(r.Str, la.Separators) - - case leftNil && rightAny: - return match.NewPrefixAny(r.Str, ra.Separators) - } - - return m - } - - return matcher -} - -func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { - if len(matchers) == 0 { - return nil, fmt.Errorf("compile error: need at least one matcher") - } - if len(matchers) == 1 { - return matchers[0], nil - } - if m := glueMatchers(matchers); m != nil { - return m, nil - } - - idx := -1 - maxLen := -1 - var val match.Matcher - for i, matcher := range matchers { - if l := matcher.Len(); l != -1 && l >= maxLen { - maxLen = l - idx = i - val = matcher - } - } - - if val == nil { // not found matcher with static length - r, err := compileMatchers(matchers[1:]) - if err != nil { - return nil, err - } - return match.NewBTree(matchers[0], nil, r), nil - } - - left := matchers[:idx] - var right []match.Matcher - if len(matchers) > idx+1 { - right = matchers[idx+1:] - } - - var l, r match.Matcher - var err error - if len(left) > 0 { - l, err = compileMatchers(left) - if err != nil { - return nil, err - } - } - - if len(right) > 0 { - r, err = compileMatchers(right) - if err != nil { - return nil, err - } - } - - return match.NewBTree(val, l, r), nil -} - -func glueMatchers(matchers []match.Matcher) match.Matcher { - if m := glueMatchersAsEvery(matchers); m != nil { - return m - } - if m := glueMatchersAsRow(matchers); m != nil { - return m - } - return nil -} - -func glueMatchersAsRow(matchers []match.Matcher) match.Matcher { - if len(matchers) <= 1 { - return nil - } - - var ( - c []match.Matcher - l int - ) - for _, matcher := range matchers { - if ml := matcher.Len(); ml == -1 { - return nil - } else { - c = append(c, matcher) - l += ml - } - } - return match.NewRow(l, c...) -} - -func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher { - if len(matchers) <= 1 { - return nil - } - - var ( - hasAny bool - hasSuper bool - hasSingle bool - min int - separator []rune - ) - - for i, matcher := range matchers { - var sep []rune - - switch m := matcher.(type) { - case match.Super: - sep = []rune{} - hasSuper = true - - case match.Any: - sep = m.Separators - hasAny = true - - case match.Single: - sep = m.Separators - hasSingle = true - min++ - - case match.List: - if !m.Not { - return nil - } - sep = m.List - hasSingle = true - min++ - - default: - return nil - } - - // initialize - if i == 0 { - separator = sep - } - - if runes.Equal(sep, separator) { - continue - } - - return nil - } - - if hasSuper && !hasAny && !hasSingle { - return match.NewSuper() - } - - if hasAny && !hasSuper && !hasSingle { - return match.NewAny(separator) - } - - if (hasAny || hasSuper) && min > 0 && len(separator) == 0 { - return match.NewMin(min) - } - - every := match.NewEveryOf() - - if min > 0 { - every.Add(match.NewMin(min)) - - if !hasAny && !hasSuper { - every.Add(match.NewMax(min)) - } - } - - if len(separator) > 0 { - every.Add(match.NewContains(string(separator), true)) - } - - return every -} - -func minimizeMatchers(matchers []match.Matcher) []match.Matcher { - var done match.Matcher - var left, right, count int - - for l := 0; l < len(matchers); l++ { - for r := len(matchers); r > l; r-- { - if glued := glueMatchers(matchers[l:r]); glued != nil { - var swap bool - - if done == nil { - swap = true - } else { - cl, gl := done.Len(), glued.Len() - swap = cl > -1 && gl > -1 && gl > cl - swap = swap || count < r-l - } - - if swap { - done = glued - left = l - right = r - count = r - l - } - } - } - } - - if done == nil { - return matchers - } - - next := append(append([]match.Matcher{}, matchers[:left]...), done) - if right < len(matchers) { - next = append(next, matchers[right:]...) - } - - if len(next) == len(matchers) { - return next - } - - return minimizeMatchers(next) -} - -// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree -func minimizeTree(tree *ast.Node) *ast.Node { - switch tree.Kind { - case ast.KindAnyOf: - return minimizeTreeAnyOf(tree) - default: - return nil - } -} - -// minimizeAnyOf tries to find common children of given node of AnyOf pattern -// it searches for common children from left and from right -// if any common children are found – then it returns new optimized ast tree -// else it returns nil -func minimizeTreeAnyOf(tree *ast.Node) *ast.Node { - if !areOfSameKind(tree.Children, ast.KindPattern) { - return nil - } - - commonLeft, commonRight := commonChildren(tree.Children) - commonLeftCount, commonRightCount := len(commonLeft), len(commonRight) - if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts - return nil - } - - var result []*ast.Node - if commonLeftCount > 0 { - result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...)) - } - - var anyOf []*ast.Node - for _, child := range tree.Children { - reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount] - var node *ast.Node - if len(reuse) == 0 { - // this pattern is completely reduced by commonLeft and commonRight patterns - // so it become nothing - node = ast.NewNode(ast.KindNothing, nil) - } else { - node = ast.NewNode(ast.KindPattern, nil, reuse...) - } - anyOf = appendIfUnique(anyOf, node) - } - switch { - case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing: - result = append(result, anyOf[0]) - case len(anyOf) > 1: - result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...)) - } - - if commonRightCount > 0 { - result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...)) - } - - return ast.NewNode(ast.KindPattern, nil, result...) -} - -func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { - if len(nodes) <= 1 { - return - } - - // find node that has least number of children - idx := leastChildren(nodes) - if idx == -1 { - return - } - tree := nodes[idx] - treeLength := len(tree.Children) - - // allocate max able size for rightCommon slice - // to get ability insert elements in reverse order (from end to start) - // without sorting - commonRight = make([]*ast.Node, treeLength) - lastRight := treeLength // will use this to get results as commonRight[lastRight:] - - var ( - breakLeft bool - breakRight bool - commonTotal int - ) - for i, j := 0, treeLength-1; commonTotal < treeLength && j >= 0 && !(breakLeft && breakRight); i, j = i+1, j-1 { - treeLeft := tree.Children[i] - treeRight := tree.Children[j] - - for k := 0; k < len(nodes) && !(breakLeft && breakRight); k++ { - // skip least children node - if k == idx { - continue - } - - restLeft := nodes[k].Children[i] - restRight := nodes[k].Children[j+len(nodes[k].Children)-treeLength] - - breakLeft = breakLeft || !treeLeft.Equal(restLeft) - - // disable searching for right common parts, if left part is already overlapping - breakRight = breakRight || (!breakLeft && j <= i) - breakRight = breakRight || !treeRight.Equal(restRight) - } - - if !breakLeft { - commonTotal++ - commonLeft = append(commonLeft, treeLeft) - } - if !breakRight { - commonTotal++ - lastRight = j - commonRight[j] = treeRight - } - } - - commonRight = commonRight[lastRight:] - - return -} - -func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node { - for _, n := range target { - if reflect.DeepEqual(n, val) { - return target - } - } - return append(target, val) -} - -func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool { - for _, n := range nodes { - if n.Kind != kind { - return false - } - } - return true -} - -func leastChildren(nodes []*ast.Node) int { - min := -1 - idx := -1 - for i, n := range nodes { - if idx == -1 || (len(n.Children) < min) { - min = len(n.Children) - idx = i - } - } - return idx -} - -func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) { - var matchers []match.Matcher - for _, desc := range tree.Children { - m, err := compile(desc, sep) - if err != nil { - return nil, err - } - matchers = append(matchers, optimizeMatcher(m)) - } - return matchers, nil -} - -func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) { - switch tree.Kind { - case ast.KindAnyOf: - // todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go) - if n := minimizeTree(tree); n != nil { - return compile(n, sep) - } - matchers, err := compileTreeChildren(tree, sep) - if err != nil { - return nil, err - } - return match.NewAnyOf(matchers...), nil - - case ast.KindPattern: - if len(tree.Children) == 0 { - return match.NewNothing(), nil - } - matchers, err := compileTreeChildren(tree, sep) - if err != nil { - return nil, err - } - m, err = compileMatchers(minimizeMatchers(matchers)) - if err != nil { - return nil, err - } - - case ast.KindAny: - m = match.NewAny(sep) - - case ast.KindSuper: - m = match.NewSuper() - - case ast.KindSingle: - m = match.NewSingle(sep) - - case ast.KindNothing: - m = match.NewNothing() - - case ast.KindList: - l := tree.Value.(ast.List) - m = match.NewList([]rune(l.Chars), l.Not) - - case ast.KindRange: - r := tree.Value.(ast.Range) - m = match.NewRange(r.Lo, r.Hi, r.Not) - - case ast.KindText: - t := tree.Value.(ast.Text) - m = match.NewText(t.Text) - - default: - return nil, fmt.Errorf("could not compile tree: unknown node type") - } - - return optimizeMatcher(m), nil -} - -func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) { - m, err := compile(tree, sep) - if err != nil { - return nil, err - } - - return m, nil -} diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go deleted file mode 100644 index b58b1eb..0000000 --- a/compiler/compiler_test.go +++ /dev/null @@ -1,624 +0,0 @@ -package compiler - -import ( - "github.com/gobwas/glob/match" - "github.com/gobwas/glob/match/debug" - "github.com/gobwas/glob/syntax/ast" - "reflect" - "testing" -) - -var separators = []rune{'.'} - -func TestCommonChildren(t *testing.T) { - for i, test := range []struct { - nodes []*ast.Node - left []*ast.Node - right []*ast.Node - }{ - { - nodes: []*ast.Node{ - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"z"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), - ), - }, - }, - { - nodes: []*ast.Node{ - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"z"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), - ), - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"b"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), - ), - }, - left: []*ast.Node{ - ast.NewNode(ast.KindText, ast.Text{"a"}), - }, - right: []*ast.Node{ - ast.NewNode(ast.KindText, ast.Text{"c"}), - }, - }, - { - nodes: []*ast.Node{ - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"b"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), - ast.NewNode(ast.KindText, ast.Text{"d"}), - ), - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"b"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), - ast.NewNode(ast.KindText, ast.Text{"d"}), - ), - }, - left: []*ast.Node{ - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"b"}), - }, - right: []*ast.Node{ - ast.NewNode(ast.KindText, ast.Text{"c"}), - ast.NewNode(ast.KindText, ast.Text{"d"}), - }, - }, - { - nodes: []*ast.Node{ - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"b"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), - ), - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"b"}), - ast.NewNode(ast.KindText, ast.Text{"b"}), - ast.NewNode(ast.KindText, ast.Text{"c"}), - ), - }, - left: []*ast.Node{ - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"b"}), - }, - right: []*ast.Node{ - ast.NewNode(ast.KindText, ast.Text{"c"}), - }, - }, - { - nodes: []*ast.Node{ - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"d"}), - ), - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"d"}), - ), - ast.NewNode(ast.KindNothing, nil, - ast.NewNode(ast.KindText, ast.Text{"a"}), - ast.NewNode(ast.KindText, ast.Text{"e"}), - ), - }, - left: []*ast.Node{ - ast.NewNode(ast.KindText, ast.Text{"a"}), - }, - right: []*ast.Node{}, - }, - } { - left, right := commonChildren(test.nodes) - if !nodesEqual(left, test.left) { - t.Errorf("[%d] left, right := commonChildren(); left = %v; want %v", i, left, test.left) - } - if !nodesEqual(right, test.right) { - t.Errorf("[%d] left, right := commonChildren(); right = %v; want %v", i, right, test.right) - } - } -} - -func nodesEqual(a, b []*ast.Node) bool { - if len(a) != len(b) { - return false - } - for i, av := range a { - if !av.Equal(b[i]) { - return false - } - } - return true -} - -func TestGlueMatchers(t *testing.T) { - for id, test := range []struct { - in []match.Matcher - exp match.Matcher - }{ - { - []match.Matcher{ - match.NewSuper(), - match.NewSingle(nil), - }, - match.NewMin(1), - }, - { - []match.Matcher{ - match.NewAny(separators), - match.NewSingle(separators), - }, - match.EveryOf{match.Matchers{ - match.NewMin(1), - match.NewContains(string(separators), true), - }}, - }, - { - []match.Matcher{ - match.NewSingle(nil), - match.NewSingle(nil), - match.NewSingle(nil), - }, - match.EveryOf{match.Matchers{ - match.NewMin(3), - match.NewMax(3), - }}, - }, - { - []match.Matcher{ - match.NewList([]rune{'a'}, true), - match.NewAny([]rune{'a'}), - }, - match.EveryOf{match.Matchers{ - match.NewMin(1), - match.NewContains("a", true), - }}, - }, - } { - act, err := compileMatchers(test.in) - if err != nil { - t.Errorf("#%d convert matchers error: %s", id, err) - continue - } - - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func TestCompileMatchers(t *testing.T) { - for id, test := range []struct { - in []match.Matcher - exp match.Matcher - }{ - { - []match.Matcher{ - match.NewSuper(), - match.NewSingle(separators), - match.NewText("c"), - }, - match.NewBTree( - match.NewText("c"), - match.NewBTree( - match.NewSingle(separators), - match.NewSuper(), - nil, - ), - nil, - ), - }, - { - []match.Matcher{ - match.NewAny(nil), - match.NewText("c"), - match.NewAny(nil), - }, - match.NewBTree( - match.NewText("c"), - match.NewAny(nil), - match.NewAny(nil), - ), - }, - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }, - match.NewRow( - 4, - match.Matchers{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }..., - ), - }, - } { - act, err := compileMatchers(test.in) - if err != nil { - t.Errorf("#%d convert matchers error: %s", id, err) - continue - } - - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func TestConvertMatchers(t *testing.T) { - for id, test := range []struct { - in, exp []match.Matcher - }{ - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - match.NewAny(nil), - }, - []match.Matcher{ - match.NewRow( - 4, - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }..., - ), - match.NewAny(nil), - }, - }, - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - match.NewAny(nil), - match.NewSingle(nil), - match.NewSingle(nil), - match.NewAny(nil), - }, - []match.Matcher{ - match.NewRow( - 3, - match.Matchers{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - }..., - ), - match.NewMin(3), - }, - }, - } { - act := minimizeMatchers(test.in) - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func TestCompiler(t *testing.T) { - for id, test := range []struct { - ast *ast.Node - result match.Matcher - sep []rune - }{ - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ), - result: match.NewText("abc"), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAny, nil), - ), - sep: separators, - result: match.NewAny(separators), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAny, nil), - ), - result: match.NewSuper(), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindSuper, nil), - ), - result: match.NewSuper(), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindSingle, nil), - ), - sep: separators, - result: match.NewSingle(separators), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindRange, ast.Range{ - Lo: 'a', - Hi: 'z', - Not: true, - }), - ), - result: match.NewRange('a', 'z', true), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindList, ast.List{ - Chars: "abc", - Not: true, - }), - ), - result: match.NewList([]rune{'a', 'b', 'c'}, true), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindSingle, nil), - ast.NewNode(ast.KindSingle, nil), - ast.NewNode(ast.KindSingle, nil), - ), - sep: separators, - result: match.EveryOf{Matchers: match.Matchers{ - match.NewMin(3), - match.NewContains(string(separators), true), - }}, - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindSingle, nil), - ast.NewNode(ast.KindSingle, nil), - ast.NewNode(ast.KindSingle, nil), - ), - result: match.NewMin(3), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindSingle, nil), - ), - sep: separators, - result: match.NewBTree( - match.NewRow( - 4, - match.Matchers{ - match.NewText("abc"), - match.NewSingle(separators), - }..., - ), - match.NewAny(separators), - nil, - ), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"/"}), - ast.NewNode(ast.KindAnyOf, nil, - ast.NewNode(ast.KindText, ast.Text{"z"}), - ast.NewNode(ast.KindText, ast.Text{"ab"}), - ), - ast.NewNode(ast.KindSuper, nil), - ), - sep: separators, - result: match.NewBTree( - match.NewText("/"), - nil, - match.NewBTree( - match.NewAnyOf(match.NewText("z"), match.NewText("ab")), - nil, - match.NewSuper(), - ), - ), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindSuper, nil), - ast.NewNode(ast.KindSingle, nil), - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindSingle, nil), - ), - sep: separators, - result: match.NewBTree( - match.NewRow( - 5, - match.Matchers{ - match.NewSingle(separators), - match.NewText("abc"), - match.NewSingle(separators), - }..., - ), - match.NewSuper(), - nil, - ), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ), - result: match.NewSuffix("abc"), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindAny, nil), - ), - result: match.NewPrefix("abc"), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindText, ast.Text{"def"}), - ), - result: match.NewPrefixSuffix("abc", "def"), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindAny, nil), - ), - result: match.NewContains("abc", false), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindAny, nil), - ast.NewNode(ast.KindAny, nil), - ), - sep: separators, - result: match.NewBTree( - match.NewText("abc"), - match.NewAny(separators), - match.NewAny(separators), - ), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindSuper, nil), - ast.NewNode(ast.KindSingle, nil), - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindSuper, nil), - ast.NewNode(ast.KindSingle, nil), - ), - result: match.NewBTree( - match.NewText("abc"), - match.NewMin(1), - match.NewMin(1), - ), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ), - result: match.NewText("abc"), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAnyOf, nil, - ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAnyOf, nil, - ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ), - ), - ), - ), - ), - result: match.NewText("abc"), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAnyOf, nil, - ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindSingle, nil), - ), - ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindList, ast.List{Chars: "def"}), - ), - ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ), - ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ), - ), - ), - result: match.NewBTree( - match.NewText("abc"), - nil, - match.AnyOf{Matchers: match.Matchers{ - match.NewSingle(nil), - match.NewList([]rune{'d', 'e', 'f'}, false), - match.NewNothing(), - }}, - ), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindRange, ast.Range{Lo: 'a', Hi: 'z'}), - ast.NewNode(ast.KindRange, ast.Range{Lo: 'a', Hi: 'x', Not: true}), - ast.NewNode(ast.KindAny, nil), - ), - result: match.NewBTree( - match.NewRow( - 2, - match.Matchers{ - match.NewRange('a', 'z', false), - match.NewRange('a', 'x', true), - }..., - ), - nil, - match.NewSuper(), - ), - }, - { - ast: ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindAnyOf, nil, - ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindList, ast.List{Chars: "abc"}), - ast.NewNode(ast.KindText, ast.Text{"ghi"}), - ), - ast.NewNode(ast.KindPattern, nil, - ast.NewNode(ast.KindText, ast.Text{"abc"}), - ast.NewNode(ast.KindList, ast.List{Chars: "def"}), - ast.NewNode(ast.KindText, ast.Text{"ghi"}), - ), - ), - ), - result: match.NewRow( - 7, - match.Matchers{ - match.NewText("abc"), - match.AnyOf{Matchers: match.Matchers{ - match.NewList([]rune{'a', 'b', 'c'}, false), - match.NewList([]rune{'d', 'e', 'f'}, false), - }}, - match.NewText("ghi"), - }..., - ), - }, - } { - m, err := Compile(test.ast, test.sep) - if err != nil { - t.Errorf("compilation error: %s", err) - continue - } - - if !reflect.DeepEqual(m, test.result) { - t.Errorf("[%d] Compile():\nexp: %#v\nact: %#v\n\ngraphviz:\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher))) - continue - } - } -} diff --git a/fuzz_test.go b/fuzz_test.go new file mode 100644 index 0000000..54e179b --- /dev/null +++ b/fuzz_test.go @@ -0,0 +1,296 @@ +package glob + +import ( + "errors" + "fmt" + "regexp" + "strings" + "testing" + "unicode/utf8" +) + +// FuzzMatchRegexp verifies Pattern.Match() against the regexp package. +// +// The fuzzer generates arbitrary (pattern, string, separator) inputs; each +// compilable pattern is translated into an equivalent anchored regular +// expression by translateGlob() -- a naive, independent re-implementation +// of the glob syntax -- and the match results are compared. Since the +// translator shares no code with Compile(), this validates the lexer, the +// parser, the compile-time rewrites and the matching engine all at once. +// +// Run with: +// +// go test -fuzz=FuzzMatchRegexp +// +// Without -fuzz only the seed corpus is run, as a regression test. +func FuzzMatchRegexp(f *testing.F) { + for _, seed := range []struct { + pattern string + fixture string + dotSep bool + }{ + { + pattern: "", + fixture: "", + }, + { + pattern: "abc", + fixture: "abc", + }, + { + pattern: "a*c", + fixture: "abbbc", + }, + { + pattern: "a*a*a*a*b", + fixture: "aaaaaaab", + }, + { + pattern: "**a*b", + fixture: "axc.ab", + dotSep: true, + }, + { + pattern: "*a**b", + fixture: "xa.b", + dotSep: true, + }, + { + pattern: "{*,**,?}", + fixture: "a.b", + dotSep: true, + }, + { + pattern: "{a,ab}c", + fixture: "abc", + }, + { + pattern: "{*.google.*,yandex.*}", + fixture: "www.google.com", + dotSep: true, + }, + { + pattern: "{https://*gobwas.com,http://exclude.gobwas.com}", + fixture: "https://safe.gobwas.com", + }, + { + pattern: "[!a]*", + fixture: "this is a test3", + }, + { + pattern: "[a-z][!a-x]*cat*[h][!b]*eyes*", + fixture: "my cat has very bright eyes", + }, + { + pattern: `\*`, + fixture: "*", + }, + { + pattern: "*ä", + fixture: "åä", + }, + { + pattern: "{,a}{a,}a", + fixture: "a", + }, + { + pattern: "{{a,b},c}d", + fixture: "bd", + }, + { + // A star inside an alternative must not discard the restart + // point of a star outside of it; see the starsFloor handling + // in matchContext.storeStar(). + pattern: "*{*0,}", + fixture: "1", + }, + { + // U+FFFD is a valid character, not an invalid byte: the class + // must match it, and the pattern must compile. + pattern: "[!a]", + fixture: "�", + }, + { + pattern: "[�]*", + fixture: "�x", + }, + } { + f.Add(seed.pattern, seed.fixture, seed.dotSep) + } + f.Fuzz(func(t *testing.T, pattern, s string, dotSep bool) { + if len(pattern) > 32 || len(s) > 64 { + t.Skip("too large") + } + if !utf8.ValidString(pattern) || !utf8.ValidString(s) { + // The regexp package normalizes invalid UTF-8; the comparison + // makes no sense for it. + t.Skip("invalid utf8") + } + var sep []rune + if dotSep { + sep = []rune{'.'} + if strings.Count(pattern, "*") > 4 { + // Backtracking over many separator-limited stars is + // combinatorial in the worst case; keep the fuzzing fast. + t.Skip("too many stars") + } + } + p, err := Compile(pattern, sep...) + if err != nil { + t.Skip("does not compile") + } + reStr, err := translateGlob(pattern, sep) + if err != nil { + t.Fatalf("Compile() accepted %#q but the translator did not: %v", pattern, err) + } + re, err := regexp.Compile(reStr) + if err != nil { + t.Fatalf("bad regexp translation of %#q: %v", pattern, err) + } + var ( + got = p.Match(s) + want = re.MatchString(s) + ) + if got != want { + t.Errorf( + "sep=%q pattern=%#q (regexp %#q) string=%#q: Match()=%t, regexp=%t", + string(sep), pattern, re, s, got, want, + ) + } + }) +} + +// translateGlob translates the glob pattern into an equivalent anchored +// regular expression. It is intentionally independent from Compile(): a naive +// rune-by-rune re-implementation of the documented syntax, sharing no code +// with the lexer or the parser, to serve as the differential oracle. +func translateGlob(pattern string, sep []rune) (string, error) { + var sb strings.Builder + sb.WriteString("^(?:") + rs := []rune(pattern) + depth := 0 + for i := 0; i < len(rs); i++ { + switch r := rs[i]; r { + case '\\': + // A trailing backslash does not compile, so it never gets here. + if i+1 < len(rs) { + i++ + sb.WriteString(regexp.QuoteMeta(string(rs[i]))) + } + + case '*': + if i+1 < len(rs) && rs[i+1] == '*' { + i++ + sb.WriteString(`(?s:.*)`) + } else { + sb.WriteString(sepClassRegexp(sep) + `*`) + } + + case '?': + sb.WriteString(sepClassRegexp(sep)) + + case '{': + depth++ + sb.WriteString(`(?:`) + + case '}': + if depth == 0 { + // A literal outside of braces. + sb.WriteString(regexp.QuoteMeta(`}`)) + break + } + depth-- + sb.WriteString(`)`) + + case ',': + if depth == 0 { + // A literal outside of braces. + sb.WriteString(`,`) + break + } + sb.WriteString(`|`) + + case '[': + n, err := translateClass(&sb, rs[i:]) + if err != nil { + return "", err + } + i += n - 1 + + default: + sb.WriteString(regexp.QuoteMeta(string(r))) + } + } + if depth != 0 { + return "", errors.New("unclosed `{`") + } + sb.WriteString(`)$`) + return sb.String(), nil +} + +// translateClass translates the `[...]` character class rs begins with and +// returns the number of runes consumed. Mirroring the syntax, a class is +// either a range `[a-z]` or a set `[abc]`, both optionally negated with the +// leading `!`; the range boundary characters are taken verbatim, the set +// characters may be escaped. +func translateClass(sb *strings.Builder, rs []rune) (int, error) { + i := 1 // Skip the `[`. + not := "" + if i < len(rs) && rs[i] == '!' { + not = "^" + i++ + } + if i+1 < len(rs) && rs[i+1] == '-' { + // A range. + if i+3 >= len(rs) || rs[i+3] != ']' { + return 0, errors.New("unclosed range") + } + lo, hi := rs[i], rs[i+2] + if hi < lo { + return 0, errors.New("range hi character is less than lo") + } + fmt.Fprintf(sb, `[%s\x{%x}-\x{%x}]`, not, lo, hi) + return i + 4, nil + } + // A set. + var chars []rune + escaped := false + for ; i < len(rs); i++ { + r := rs[i] + switch { + case escaped: + chars = append(chars, r) + escaped = false + case r == '\\': + escaped = true + case r == ']': + if len(chars) == 0 { + return 0, errors.New("empty class") + } + sb.WriteString("[" + not) + for _, c := range chars { + fmt.Fprintf(sb, `\x{%x}`, c) + } + sb.WriteString("]") + return i + 1, nil + default: + chars = append(chars, r) + } + } + return 0, errors.New("unclosed `[`") +} + +// sepClassRegexp returns a class matching any single rune except the +// separators, e.g. `[^\x{2e}]` -- or the match-all when there are none. +func sepClassRegexp(sep []rune) string { + if len(sep) == 0 { + return `(?s:.)` + } + var sb strings.Builder + sb.WriteString("[^") + for _, r := range sep { + fmt.Fprintf(&sb, `\x{%x}`, r) + } + sb.WriteString("]") + return sb.String() +} diff --git a/glob.go b/glob.go index 2afde34..fb3e475 100644 --- a/glob.go +++ b/glob.go @@ -1,80 +1,158 @@ package glob import ( - "github.com/gobwas/glob/compiler" + "fmt" + + "github.com/gobwas/glob/internal/debug" "github.com/gobwas/glob/syntax" ) -// Glob represents compiled glob pattern. -type Glob interface { - Match(string) bool +// SyntaxError is returned by [Compile] when the given pattern can not be +// parsed. Offset points at the place in the pattern the error was detected +// at, so the tooling can do things like: +// +// {a,b +// ----^ unclosed `{` +type SyntaxError struct { + // Offset is a byte offset in the pattern. + Offset int + // Reason describes the error. + Reason string } -// Compile creates Glob for given pattern and strings (if any present after pattern) as separators. -// The pattern syntax is: +func (s *SyntaxError) Error() string { + return fmt.Sprintf("glob: syntax error at %d: %s", s.Offset, s.Reason) +} + +// Pattern represents a compiled glob pattern. // -// pattern: -// { term } +// A pattern is compiled into a tree of matchers: // -// term: -// `*` matches any sequence of non-separator characters -// `**` matches any sequence of characters -// `?` matches any single non-separator character -// `[` [ `!` ] { character-range } `]` -// character class (must be non-empty) -// `{` pattern-list `}` -// pattern alternatives -// c matches character c (c != `*`, `**`, `?`, `\`, `[`, `{`, `}`) -// `\` c matches character c +// `a` => "a" +// `a*` => ["a"·*] +// `{a*,b}` => {["a"·*]|"b"} // -// character-range: -// c matches character c (c != `\\`, `-`, `]`) -// `\` c matches character c -// lo `-` hi matches character c for lo <= c <= hi +// Matching is a backtracking walk over that tree; see [Pattern.Match]. +type Pattern struct { + // str is the pattern text the Pattern was compiled from; see + // [Pattern.String]. + str string + + // sep are the separators the Pattern was compiled with; see + // [Pattern.Separators]. + sep []rune + + // m is the root of the matcher tree; see [matcher]. + m matcher + + // state tells whether matching m needs the backtracking state, that + // is, whether it may save checkpoints; see [needsState]. A pattern + // without them is matched with a plain call chain. + state bool + + // The match preconditions: every matching string is at least minLen + // bytes and ends with suffix. They fail the obvious mismatches in O(1) + // instead of a backtracking walk -- e.g. `a*a*a*b` requires the + // trailing `b`, no matter how the stars go. + // + // A precondition pays off only when it catches a mismatch earlier than + // the walk would, which is why: + // + // - there is no required prefix: the walk is left-to-right, so a + // leading literal is the first thing checked anyway, while a bad + // suffix or length is discovered last, after the whole + // backtracking exploration; + // + // - they are computed for the stateful patterns only: a stateless + // pattern is a plain call chain whose matchers perform these very + // checks themselves (e.g. suffixMatcher is a HasSuffix), so the + // precondition would only duplicate them. + minLen int + suffix string +} + +// String returns the source text used to compile the pattern, the same way +// [regexp.Regexp.String] does. // -// pattern-list: -// pattern { `,` pattern } -// comma-separated (without spaces) patterns +// Note that separators are not part of String: they are given to Compile +// alongside the pattern text. +func (p *Pattern) String() string { + return p.str +} + +// Separators returns the separators the pattern was compiled with, in the +// order they were given to Compile; nil when there are none. // -func Compile(pattern string, separators ...rune) (Glob, error) { - ast, err := syntax.Parse(pattern) - if err != nil { - return nil, err - } +// The returned slice is the very one given to Compile, sharing its backing +// array: it is not copied on the way in or out. Matching does not use it. +func (p *Pattern) Separators() []rune { + return p.sep +} - matcher, err := compiler.Compile(ast, separators) - if err != nil { - return nil, err +func init() { + // The matcher tree is unexported; hand its rendering to the in-module + // tooling (cmd/globtest -v) without widening the public API. + debug.Tree = func(p any) string { + return p.(*Pattern).m.String() } +} - return matcher, nil +// Compile compiles the glob pattern. The separators, if given, are the +// characters `*` and `?` do not match (`**` does); they can not be changed +// after the compilation, see [Pattern.Separators]. A malformed pattern is +// reported with a [*SyntaxError]. +// +// The pattern syntax is: +// +// pattern: +// { term } +// +// term: +// `*` matches any sequence of non-separator characters +// `**` matches any sequence of characters +// `?` matches any single non-separator character +// `[` [ `!` ] class `]` +// character class; `!` negates it +// `{` pattern-list `}` +// pattern alternatives +// c matches character c (c != `*`, `**`, `?`, `\`, `[`, `{`, `}`) +// `\` c matches character c +// +// class: +// lo `-` hi matches character c for lo <= c <= hi +// { c } matches any of the listed characters (c != `\`, `]`; +// `\` c matches c, `-` is literal here); must be non-empty +// +// pattern-list: +// pattern { `,` pattern } +// comma-separated (without spaces) patterns +func Compile(pattern string, separators ...rune) (*Pattern, error) { + return compile(pattern, separators) } -// MustCompile is the same as Compile, except that if Compile returns error, this will panic -func MustCompile(pattern string, separators ...rune) Glob { +// MustCompile is the same as Compile, except that if Compile returns error, +// this will panic. +func MustCompile(pattern string, separators ...rune) *Pattern { g, err := Compile(pattern, separators...) if err != nil { panic(err) } - return g } -// QuoteMeta returns a string that quotes all glob pattern meta characters -// inside the argument text; For example, QuoteMeta(`{foo*}`) returns `\[foo\*\]`. +// QuoteMeta returns a copy of the s having all glob meta characters escaped. func QuoteMeta(s string) string { + // 2 is a pessimistic way of allocating an extra byte per each byte in s. b := make([]byte, 2*len(s)) - - // a byte loop is correct because all meta characters are ASCII j := 0 + // A byte loop is correct here because all meta characters are ASCII. for i := 0; i < len(s); i++ { - if syntax.Special(s[i]) { + if syntax.IsSpecial(s[i]) { b[j] = '\\' j++ } b[j] = s[i] j++ } - return string(b[0:j]) } diff --git a/glob_test.go b/glob_test.go index 810036f..9fb8521 100644 --- a/glob_test.go +++ b/glob_test.go @@ -1,184 +1,743 @@ package glob import ( + "errors" + "fmt" + "path/filepath" "regexp" + "slices" + "strings" "testing" + "time" ) -const ( - pattern_all = "[a-z][!a-x]*cat*[h][!b]*eyes*" - regexp_all = `^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` - fixture_all_match = "my cat has very bright eyes" - fixture_all_mismatch = "my dog has very bright eyes" - - pattern_plain = "google.com" - regexp_plain = `^google\.com$` - fixture_plain_match = "google.com" - fixture_plain_mismatch = "gobwas.com" - - pattern_multiple = "https://*.google.*" - regexp_multiple = `^https:\/\/.*\.google\..*$` - fixture_multiple_match = "https://account.google.com" - fixture_multiple_mismatch = "https://google.com" - - pattern_alternatives = "{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}" - regexp_alternatives = `^(https:\/\/.*\.google\..*|.*yandex\..*|.*yahoo\..*|.*mail\.ru)$` - fixture_alternatives_match = "http://yahoo.com" - fixture_alternatives_mismatch = "http://google.com" - - pattern_alternatives_suffix = "{https://*gobwas.com,http://exclude.gobwas.com}" - regexp_alternatives_suffix = `^(https:\/\/.*gobwas\.com|http://exclude.gobwas.com)$` - fixture_alternatives_suffix_first_match = "https://safe.gobwas.com" - fixture_alternatives_suffix_first_mismatch = "http://safe.gobwas.com" - fixture_alternatives_suffix_second = "http://exclude.gobwas.com" - - pattern_prefix = "abc*" - regexp_prefix = `^abc.*$` - pattern_suffix = "*def" - regexp_suffix = `^.*def$` - pattern_prefix_suffix = "ab*ef" - regexp_prefix_suffix = `^ab.*ef$` - fixture_prefix_suffix_match = "abcdef" - fixture_prefix_suffix_mismatch = "af" - - pattern_alternatives_combine_lite = "{abc*def,abc?def,abc[zte]def}" - regexp_alternatives_combine_lite = `^(abc.*def|abc.def|abc[zte]def)$` - fixture_alternatives_combine_lite = "abczdef" - - pattern_alternatives_combine_hard = "{abc*[a-c]def,abc?[d-g]def,abc[zte]?def}" - regexp_alternatives_combine_hard = `^(abc.*[a-c]def|abc.[d-g]def|abc[zte].def)$` - fixture_alternatives_combine_hard = "abczqdef" -) - -type test struct { - pattern, match string - should bool - delimiters []rune -} - -func glob(s bool, p, m string, d ...rune) test { - return test{p, m, s, d} +func TestCompileErrors(t *testing.T) { + for _, test := range []struct { + name string + pat string + offset int + reason string + }{ + { + pat: "{a,b", + offset: 4, + reason: "unclosed `{`", + }, + { + pat: "[abc", + offset: 4, + reason: "unexpected end of input", + }, + { + pat: "[a-cx]", + offset: 5, + reason: "expected close range character", + }, + { + // The class is lexed as a whole: the offset is past its `]`. + pat: "[c-a]", + offset: 5, + reason: "range hi character is less than lo", + }, + { + name: "trailing backslash", + pat: `a\`, + offset: 2, + reason: "trailing backslash", + }, + { + name: "invalid utf8", + pat: "a\xffb", + offset: 1, + reason: "invalid UTF-8 sequence", + }, + } { + t.Run(test.name, func(t *testing.T) { + _, err := Compile(test.pat) + var se *SyntaxError + if !errors.As(err, &se) { + t.Fatalf("Compile(%q) = %v; want *SyntaxError", test.pat, err) + } + if se.Offset != test.offset || se.Reason != test.reason { + t.Errorf( + "Compile(%q) = (%d, %q); want (%d, %q)", + test.pat, se.Offset, se.Reason, test.offset, test.reason, + ) + } + }) + } } -func TestGlob(t *testing.T) { - for _, test := range []test{ - glob(true, "* ?at * eyes", "my cat has very bright eyes"), - - glob(true, "", ""), - glob(false, "", "b"), - - glob(true, "*ä", "åä"), - glob(true, "abc", "abc"), - glob(true, "a*c", "abc"), - glob(true, "a*c", "a12345c"), - glob(true, "a?c", "a1c"), - glob(true, "a.b", "a.b", '.'), - glob(true, "a.*", "a.b", '.'), - glob(true, "a.**", "a.b.c", '.'), - glob(true, "a.?.c", "a.b.c", '.'), - glob(true, "a.?.?", "a.b.c", '.'), - glob(true, "?at", "cat"), - glob(true, "?at", "fat"), - glob(true, "*", "abc"), - glob(true, `\*`, "*"), - glob(true, "**", "a.b.c", '.'), - - glob(false, "?at", "at"), - glob(false, "?at", "fat", 'f'), - glob(false, "a.*", "a.b.c", '.'), - glob(false, "a.?.c", "a.bb.c", '.'), - glob(false, "*", "a.b.c", '.'), - - glob(true, "*test", "this is a test"), - glob(true, "this*", "this is a test"), - glob(true, "*is *", "this is a test"), - glob(true, "*is*a*", "this is a test"), - glob(true, "**test**", "this is a test"), - glob(true, "**is**a***test*", "this is a test"), - - glob(false, "*is", "this is a test"), - glob(false, "*no*", "this is a test"), - glob(true, "[!a]*", "this is a test3"), - - glob(true, "*abc", "abcabc"), - glob(true, "**abc", "abcabc"), - glob(true, "???", "abc"), - glob(true, "?*?", "abc"), - glob(true, "?*?", "ac"), - glob(false, "sta", "stagnation"), - glob(true, "sta*", "stagnation"), - glob(false, "sta?", "stagnation"), - glob(false, "sta?n", "stagnation"), - - glob(true, "{abc,def}ghi", "defghi"), - glob(true, "{abc,abcd}a", "abcda"), - glob(true, "{a,ab}{bc,f}", "abc"), - glob(true, "{*,**}{a,b}", "ab"), - glob(false, "{*,**}{a,b}", "ac"), - - glob(true, "/{rate,[a-z][a-z][a-z]}*", "/rate"), - glob(true, "/{rate,[0-9][0-9][0-9]}*", "/rate"), - glob(true, "/{rate,[a-z][a-z][a-z]}*", "/usd"), +func TestPatternMatch(t *testing.T) { + for i, test := range []struct { + sep []rune + pat string + str string + exp bool + }{ + { + pat: "a*a*a*a*b", + str: strings.Repeat("a", 100), + exp: false, + }, + { + pat: "a*a*a*a*b", + str: strings.Repeat("a", 100) + "b", + exp: true, + }, + { + pat: "{a,ab}c", + str: "abc", + exp: true, + }, + { + pat: "* ?at * eyes", + str: "my cat has very bright eyes", + exp: true, + }, + { + pat: "", + str: "", + exp: true, + }, + { + pat: "", + str: "b", + exp: false, + }, + { + pat: "*ä", + str: "åä", + exp: true, + }, + { + pat: "abc", + str: "abc", + exp: true, + }, + { + pat: "a*c", + str: "abc", + exp: true, + }, + { + pat: "a*c", + str: "a12345c", + exp: true, + }, + { + pat: "a?c", + str: "a1c", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "a.b", + str: "a.b", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "a.*", + str: "a.b", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "a.**", + str: "a.b.c", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "a.?.c", + str: "a.b.c", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "a.?.?", + str: "a.b.c", + exp: true, + }, + { + pat: "?at", + str: "cat", + exp: true, + }, + { + pat: "?at", + str: "fat", + exp: true, + }, + { + pat: "*", + str: "abc", + exp: true, + }, + { + pat: "*a*", + str: "a", + exp: true, + }, + { + pat: "\\*", + str: "*", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "**", + str: "a.b.c", + exp: true, + }, + { + pat: "?at", + str: "at", + exp: false, + }, + { + sep: []rune{'f'}, + pat: "?at", + str: "fat", + exp: false, + }, + { + sep: []rune{'.'}, + pat: "a.*", + str: "a.b.c", + exp: false, + }, + { + sep: []rune{'.'}, + pat: "a.?.c", + str: "a.bb.c", + exp: false, + }, + { + sep: []rune{'.'}, + pat: "*", + str: "a.b.c", + exp: false, + }, + { + pat: "*test", + str: "this is a test", + exp: true, + }, + { + pat: "this*", + str: "this is a test", + exp: true, + }, + { + pat: "*is *", + str: "this is a test", + exp: true, + }, + { + pat: "*is*a*", + str: "this is a test", + exp: true, + }, + { + pat: "**test**", + str: "this is a test", + exp: true, + }, + { + pat: "**is**a***test*", + str: "this is a test", + exp: true, + }, + { + pat: "*test*", + str: "test", + exp: true, + }, + { + pat: "*is", + str: "this is a test", + exp: false, + }, + { + pat: "*no*", + str: "this is a test", + exp: false, + }, + { + pat: "[!a]*", + str: "this is a test3", + exp: true, + }, + { + pat: "*abc", + str: "abcabc", + exp: true, + }, + { + pat: "**abc", + str: "abcabc", + exp: true, + }, + { + pat: "???", + str: "abc", + exp: true, + }, + { + pat: "?*?", + str: "abc", + exp: true, + }, + { + pat: "?*?", + str: "ac", + exp: true, + }, + { + pat: "sta", + str: "stagnation", + exp: false, + }, + { + pat: "sta*", + str: "stagnation", + exp: true, + }, + { + pat: "sta?", + str: "stagnation", + exp: false, + }, + { + pat: "sta?n", + str: "stagnation", + exp: false, + }, + { + pat: "{abc,def}ghi", + str: "defghi", + exp: true, + }, + { + pat: "{abc,abcd}a", + str: "abcda", + exp: true, + }, + { + pat: "{,a}", + str: "", + exp: true, + }, + { + pat: "{a,}", + str: "", + exp: true, + }, + { + pat: "{a,ab}{bc,f}", + str: "abc", + exp: true, + }, + { + pat: "{*,**}{a,b}", + str: "ab", + exp: true, + }, + { + pat: "{*,**}{a,b}", + str: "ac", + exp: false, + }, + { + pat: "/{rate,[a-z][a-z][a-z]}*", + str: "/rate", + exp: true, + }, + { + pat: "/{rate,[0-9][0-9][0-9]}*", + str: "/rate", + exp: true, + }, + { + pat: "/{rate,[a-z][a-z][a-z]}*", + str: "/usd", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "{*.google.*,*.yandex.*}", + str: "www.google.com", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "{*.google.*,*.yandex.*}", + str: "www.yandex.com", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "{*.google.*,*.yandex.*}", + str: "yandex.com", + exp: false, + }, + { + sep: []rune{'.'}, + pat: "{*.google.*,*.yandex.*}", + str: "google.com", + exp: false, + }, + { + sep: []rune{'.'}, + pat: "{*.google.*,yandex.*}", + str: "www.google.com", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "{*.google.*,yandex.*}", + str: "yandex.com", + exp: true, + }, + { + sep: []rune{'.'}, + pat: "{*.google.*,yandex.*}", + str: "www.yandex.com", + exp: false, + }, + { + sep: []rune{'.'}, + pat: "{*.google.*,yandex.*}", + str: "google.com", + exp: false, + }, + { + pat: "*//{,*.}example.com", + str: "https://www.example.com", + exp: true, + }, + { + pat: "*//{,*.}example.com", + str: "http://example.com", + exp: true, + }, + { + pat: "*//{,*.}example.com", + str: "http://example.com.net", + exp: false, + }, + { + sep: []rune{'.'}, + pat: "{a*,b}c", + str: "abc", + exp: true, + }, + { + pat: "[a-z][!a-x]*cat*[h][!b]*eyes*", + str: "my cat has very bright eyes", + exp: true, + }, + { + pat: "[a-z][!a-x]*cat*[h][!b]*eyes*", + str: "my dog has very bright eyes", + exp: false, + }, - glob(true, "{*.google.*,*.yandex.*}", "www.google.com", '.'), - glob(true, "{*.google.*,*.yandex.*}", "www.yandex.com", '.'), - glob(false, "{*.google.*,*.yandex.*}", "yandex.com", '.'), - glob(false, "{*.google.*,*.yandex.*}", "google.com", '.'), + { + pat: "google.com", + str: "google.com", + exp: true, + }, + { + pat: "google.com", + str: "gobwas.com", + exp: false, + }, - glob(true, "{*.google.*,yandex.*}", "www.google.com", '.'), - glob(true, "{*.google.*,yandex.*}", "yandex.com", '.'), - glob(false, "{*.google.*,yandex.*}", "www.yandex.com", '.'), - glob(false, "{*.google.*,yandex.*}", "google.com", '.'), + { + pat: "https://*.google.*", + str: "https://account.google.com", + exp: true, + }, + { + pat: "https://*.google.*", + str: "https://google.com", + exp: false, + }, - glob(true, "*//{,*.}example.com", "https://www.example.com"), - glob(true, "*//{,*.}example.com", "http://example.com"), - glob(false, "*//{,*.}example.com", "http://example.com.net"), + { + pat: "{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}", + str: "http://yahoo.com", + exp: true, + }, + { + pat: "{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}", + str: "http://google.com", + exp: false, + }, - glob(true, pattern_all, fixture_all_match), - glob(false, pattern_all, fixture_all_mismatch), + { + pat: "{https://*gobwas.com,http://exclude.gobwas.com}", + str: "https://safe.gobwas.com", + exp: true, + }, + { + pat: "{https://*gobwas.com,http://exclude.gobwas.com}", + str: "http://safe.gobwas.com", + exp: false, + }, + { + pat: "{https://*gobwas.com,http://exclude.gobwas.com}", + str: "http://exclude.gobwas.com", + exp: true, + }, - glob(true, pattern_plain, fixture_plain_match), - glob(false, pattern_plain, fixture_plain_mismatch), + { + pat: "{abc*[a-c]def,abc?[d-g]def,abc[zte]?def}", + str: "abczqdef", + exp: true, + }, - glob(true, pattern_multiple, fixture_multiple_match), - glob(false, pattern_multiple, fixture_multiple_mismatch), + { + pat: "{abc*def,abc?def,abc[zte]def}", + str: "abczdef", + exp: true, + }, - glob(true, pattern_alternatives, fixture_alternatives_match), - glob(false, pattern_alternatives, fixture_alternatives_mismatch), + { + pat: "abc*", + str: "abcdef", + exp: true, + }, + { + pat: "abc*", + str: "af", + exp: false, + }, - glob(true, pattern_alternatives_suffix, fixture_alternatives_suffix_first_match), - glob(false, pattern_alternatives_suffix, fixture_alternatives_suffix_first_mismatch), - glob(true, pattern_alternatives_suffix, fixture_alternatives_suffix_second), + { + pat: "*def", + str: "abcdef", + exp: true, + }, + { + pat: "*def", + str: "af", + exp: false, + }, - glob(true, pattern_alternatives_combine_hard, fixture_alternatives_combine_hard), + { + pat: "ab*ef", + str: "abcdef", + exp: true, + }, + { + pat: "ab*ef", + str: "af", + exp: false, + }, - glob(true, pattern_alternatives_combine_lite, fixture_alternatives_combine_lite), + { + pat: "{a,ab,abc}", + str: "ab", + exp: true, + }, + { + pat: "{a,ab,abc}", + str: "", + exp: false, + }, + { + pat: "{a,ab,abc,abcd}{b,bc,bcd,bcde}{c,cd,cde,cdef}", + str: "abcdbcdecdef", + exp: true, + }, + { + pat: "{{a,b},c}", + str: "a", + exp: true, + }, + { + pat: "{{a,b},c}", + str: "c", + exp: true, + }, - glob(true, pattern_prefix, fixture_prefix_suffix_match), - glob(false, pattern_prefix, fixture_prefix_suffix_mismatch), + // A separator-limited `*` following a `**` may get stuck at a + // separator; matching must then backtrack to the pending `**` + // restart point. See the star checkpoints stack in Pattern.Match(). + { + sep: []rune{'.'}, + pat: "**a*b", + str: ".a.ab", // ** = ".a.", * = "" + exp: true, + }, + { + sep: []rune{'.'}, + pat: "**a*b", + str: "axc.ab", // ** = "axc.", * = "" + exp: true, + }, + { + sep: []rune{'.'}, + pat: "**a*b", + str: "a.axb", // ** = "a.", * = "x" + exp: true, + }, + { + sep: []rune{'.'}, + pat: "*a**b", + str: "xa.b", // * = "x", ** = "." + exp: true, + }, + { + sep: []rune{'.'}, + pat: "**.x", + str: "a.b.x", // ** = "a.b" + exp: true, + }, + { + sep: []rune{'.'}, + pat: "*a**b", + str: "x.ab", // `*` can not extend over the first separator. + exp: false, + }, - glob(true, pattern_suffix, fixture_prefix_suffix_match), - glob(false, pattern_suffix, fixture_prefix_suffix_mismatch), + // A star inside an alternative must not discard the restart point + // of a star outside of it: here the outer `*` must consume "1" and + // the empty alternative must be taken. Found by FuzzMatchRegexp; + // see the starsFloor handling in matchContext.storeStar(). + { + pat: "*{*0,}", + str: "1", + exp: true, + }, - glob(true, pattern_prefix_suffix, fixture_prefix_suffix_match), - glob(false, pattern_prefix_suffix, fixture_prefix_suffix_mismatch), + // Braces without commas hold a single alternative: `{ab*}` is the + // pattern `ab*`, not `{ab,*}`. See the opList handling in compile(). + { + pat: "{ab*}x", + str: "abzx", + exp: true, + }, + { + pat: "{ab*}x", + str: "zzx", + exp: false, + }, + { + pat: "{ab*}", + str: "ab", + exp: true, + }, + { + pat: "{a}{b}{c}", + str: "abc", + exp: true, + }, + { + pat: "{**/daxing}/x", + str: "a/daxing/x", + exp: true, + }, + { + pat: "{**/daxing}/x", + str: "zz/x", + exp: false, + }, + // U+FFFD is a character like any other, both in the pattern and in + // the input: it must not be taken for an invalid byte. + { + pat: "[!a]", + str: "�", + exp: true, + }, + { + pat: "[!a-c]", + str: "�", + exp: true, + }, + { + pat: "[�]", + str: "�", + exp: true, + }, + { + pat: "a�b", + str: "a�b", + exp: true, + }, + { + pat: "?", + str: "�", + exp: true, + }, } { - t.Run("", func(t *testing.T) { - g := MustCompile(test.pattern, test.delimiters...) - result := g.Match(test.match) - if result != test.should { + + suffix := fmt.Sprintf("-%02d", i) + t.Run("glob"+suffix, func(t *testing.T) { + t.Logf("testing pattern=%#q", test.pat) + var ( + pat = MustCompile(test.pat, test.sep...) + start = time.Now() + result = pat.Match(test.str) + latency = time.Since(start) + ) + t.Logf( + "delim=%#q pattern=%#q fixture=%#q result=%t want=%t latency=%.4fms", + test.sep, test.pat, test.str, result, test.exp, + latency.Seconds()*1000, + ) + if result != test.exp { t.Errorf( - "pattern %q matching %q should be %v but got %v\n%s", - test.pattern, test.match, test.should, result, g, + "pattern %q matching %q should be %v but got %v", + test.pat, test.str, test.exp, result, ) } + if latency > 5*time.Millisecond { + t.Errorf("too slow: %.4fms", latency.Seconds()*1000) + } }) + if len(test.sep) == 0 && !strings.ContainsAny(test.pat, "{}") { + // NOTE: filepath.Match doesn't support `{}`. + t.Run("filepath"+suffix, func(t *testing.T) { + // NOTE: filepath.Match negates a character class with `^` + // instead of `!`. + pat := strings.ReplaceAll(test.pat, "[!", "[^") + start := time.Now() + result, err := filepath.Match(pat, test.str) + latency := time.Since(start) + t.Logf( + "[filepath] pattern=%#q fixture=%#q result=%t want=%t latency=%.4fms", + test.pat, test.str, result, test.exp, + latency.Seconds()*1000, + ) + if err != nil { + t.Errorf("filepath.Match(%q) failed: %v", test.pat, err) + } + if result != test.exp { + t.Errorf( + "filepath.Match(%q, %q) = %t; test expects %t", + test.pat, test.str, result, test.exp, + ) + } + }) + } } } +func ExampleQuoteMeta() { + s := "{foo*}" + // Output: \{foo\*\} + fmt.Println(QuoteMeta(s)) +} + func TestQuoteMeta(t *testing.T) { for id, test := range []struct { - in, out string + in string + out string }{ { in: `[foo*]`, @@ -207,325 +766,373 @@ func TestQuoteMeta(t *testing.T) { } } -func BenchmarkParseGlob(b *testing.B) { - for i := 0; i < b.N; i++ { - Compile(pattern_all) - } -} -func BenchmarkParseRegexp(b *testing.B) { - for i := 0; i < b.N; i++ { - regexp.MustCompile(regexp_all) - } -} - -func BenchmarkAllGlobMatch(b *testing.B) { - m, _ := Compile(pattern_all) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_all_match) - } -} -func BenchmarkAllGlobMatchParallel(b *testing.B) { - m, _ := Compile(pattern_all) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _ = m.Match(fixture_all_match) +func BenchmarkPattern(b *testing.B) { + for _, test := range []struct { + name string + pat string + input map[string]string + }{ + { + name: "segments", + pat: `{a,ab,abc}`, + input: map[string]string{ + "0": "a", + "1": "ab", + "2": "abc", + "empty": "", + }, + }, + { + name: "long_segments", + pat: `{a,ab,abc,abcd}{b,bc,bcd,bcde}{c,cd,cde,cdef}`, + input: map[string]string{ + "long": "abcdbcdecdef", + }, + }, + { + name: "cat", + pat: `[a-z][!a-x]*cat*[h][!b]*eyes*`, + input: map[string]string{ + "match": "my cat has very bright eyes", + "mismatch": "my dog has very bright eyes", + }, + }, + { + name: "wildcard", + pat: `https://*.google.*`, + input: map[string]string{ + "match": "https://account.google.com", + "mismatch": "https://google.com", + }, + }, + { + name: "alternatives", + pat: `{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}`, + input: map[string]string{ + "match": "http://yahoo.com", + "mismatch": "http://google.com", + }, + }, + { + name: "alternatives_suffix_first", + pat: `{https://*gobwas.com,http://exclude.gobwas.com}`, + input: map[string]string{ + "match": "https://safe.gobwas.com", + "mismatch": "http://safe.gobwas.com", + }, + }, + { + name: "alternatives_suffix_second", + pat: `{https://*gobwas.com,http://exclude.gobwas.com}`, + input: map[string]string{ + "match": "http://exclude.gobwas.com", + //"mismatch": "", + }, + }, + { + name: "alternatives_combine_lite", + pat: `{abc*def,abc?def,abc[zte]def}`, + input: map[string]string{ + "match": "abczdef", + //"mismatch": "", + }, + }, + { + name: "alternatives_combine_hard", + pat: `{abc*[a-c]def,abc?[d-g]def,abc[zte]?def}`, + input: map[string]string{ + "match": "abczqdef", + //"mismatch": "", + }, + }, + { + name: "plain", + pat: `google.com`, + input: map[string]string{ + "match": "google.com", + "mismatch": "gobwas.com", + }, + }, + { + name: "prefix", + pat: `abc*`, + input: map[string]string{ + "match": "abcdef", + "mismatch": "af", + }, + }, + { + name: "suffix", + pat: `*def`, + input: map[string]string{ + "match": "abcdef", + "mismatch": "af", + }, + }, + { + name: "prefix_and_suffix", + pat: `ab*ef`, + input: map[string]string{ + "match": "abcdef", + "mismatch": "af", + }, + }, + } { + b.Run(test.name+"-compile", func(b *testing.B) { + for i := 0; i < b.N; i++ { + MustCompile(test.pat) + } + }) + pat := MustCompile(test.pat) + for _, key := range keysInOrder(test.input) { + str := test.input[key] + b.Run(test.name+"-match-"+key, func(b *testing.B) { + for i := 0; i < b.N; i++ { + pat.Match(str) + } + }) } - }) -} - -func BenchmarkAllRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_all) - f := []byte(fixture_all_match) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkAllGlobMismatch(b *testing.B) { - m, _ := Compile(pattern_all) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_all_mismatch) } } -func BenchmarkAllGlobMismatchParallel(b *testing.B) { - m, _ := Compile(pattern_all) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _ = m.Match(fixture_all_mismatch) +// BenchmarkCompareGlobAndRegexp is what the regexp table in the readme is +// made of. The regexps are the exact equivalents of the globs: anchored, +// with the meta characters escaped and, since `*` matches a newline as any +// other character, with the `s` flag (see issue #21). +func BenchmarkCompareGlobAndRegexp(b *testing.B) { + for _, test := range []struct { + name string + glob string + regexp string + input map[string]string + }{ + { + name: "cat", + glob: `[a-z][!a-x]*cat*[h][!b]*eyes*`, + regexp: `(?s)^[a-z][^a-x].*cat.*[h][^b].*eyes.*$`, + input: map[string]string{ + "match": "my cat has very bright eyes", + "mismatch": "my dog has very bright eyes", + }, + }, + { + name: "wildcard", + glob: `https://*.google.*`, + regexp: `(?s)^https://.*\.google\..*$`, + input: map[string]string{ + "match": "https://account.google.com", + "mismatch": "https://google.com", + }, + }, + { + name: "alternatives", + glob: `{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}`, + regexp: `(?s)^(https://.*\.google\..*|.*yandex\..*|.*yahoo\..*|.*mail\.ru)$`, + input: map[string]string{ + "match": "http://yahoo.com", + "mismatch": "http://google.com", + }, + }, + { + name: "alternatives_suffix_first", + glob: `{https://*gobwas.com,http://exclude.gobwas.com}`, + regexp: `(?s)^(https://.*gobwas\.com|http://exclude\.gobwas\.com)$`, + input: map[string]string{ + "match": "https://safe.gobwas.com", + "mismatch": "http://safe.gobwas.com", + }, + }, + { + name: "alternatives_suffix_second", + glob: `{https://*gobwas.com,http://exclude.gobwas.com}`, + regexp: `(?s)^(https://.*gobwas\.com|http://exclude\.gobwas\.com)$`, + input: map[string]string{ + "match": "http://exclude.gobwas.com", + //"mismatch": "", + }, + }, + { + name: "alternatives_combine_lite", + glob: `{abc*def,abc?def,abc[zte]def}`, + regexp: `(?s)^(abc.*def|abc.def|abc[zte]def)$`, + input: map[string]string{ + "match": "abczdef", + //"mismatch": "", + }, + }, + { + name: "alternatives_combine_hard", + glob: `{abc*[a-c]def,abc?[d-g]def,abc[zte]?def}`, + regexp: `(?s)^(abc.*[a-c]def|abc.[d-g]def|abc[zte].def)$`, + input: map[string]string{ + "match": "abczqdef", + //"mismatch": "", + }, + }, + { + name: "plain", + glob: `google.com`, + regexp: `^google\.com$`, + input: map[string]string{ + "match": "google.com", + "mismatch": "gobwas.com", + }, + }, + { + name: "prefix", + glob: `abc*`, + regexp: `(?s)^abc.*$`, + input: map[string]string{ + "match": "abcdef", + "mismatch": "af", + }, + }, + { + name: "suffix", + glob: `*def`, + regexp: `(?s)^.*def$`, + input: map[string]string{ + "match": "abcdef", + "mismatch": "af", + }, + }, + { + name: "prefix_and_suffix", + glob: `ab*ef`, + regexp: `(?s)^ab.*ef$`, + input: map[string]string{ + "match": "abcdef", + "mismatch": "af", + }, + }, + } { + b.Run(test.name+"-glob-compile", func(b *testing.B) { + for i := 0; i < b.N; i++ { + MustCompile(test.glob) + } + }) + b.Run(test.name+"-regexp-compile", func(b *testing.B) { + for i := 0; i < b.N; i++ { + regexp.MustCompile(test.regexp) + } + }) + var ( + pat = MustCompile(test.glob) + exp = regexp.MustCompile(test.regexp) + ) + for _, key := range keysInOrder(test.input) { + str := test.input[key] + b.Run(test.name+"-glob-"+key, func(b *testing.B) { + for i := 0; i < b.N; i++ { + pat.Match(str) + } + }) + b.Run(test.name+"-regexp-"+key, func(b *testing.B) { + for i := 0; i < b.N; i++ { + exp.MatchString(str) + } + }) } - }) -} -func BenchmarkAllRegexpMismatch(b *testing.B) { - m := regexp.MustCompile(regexp_all) - f := []byte(fixture_all_mismatch) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} - -func BenchmarkMultipleGlobMatch(b *testing.B) { - m, _ := Compile(pattern_multiple) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_multiple_match) } } -func BenchmarkMultipleRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_multiple) - f := []byte(fixture_multiple_match) - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkMultipleGlobMismatch(b *testing.B) { - m, _ := Compile(pattern_multiple) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_multiple_mismatch) - } -} -func BenchmarkMultipleRegexpMismatch(b *testing.B) { - m := regexp.MustCompile(regexp_multiple) - f := []byte(fixture_multiple_mismatch) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} - -func BenchmarkAlternativesGlobMatch(b *testing.B) { - m, _ := Compile(pattern_alternatives) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_alternatives_match) - } -} -func BenchmarkAlternativesGlobMismatch(b *testing.B) { - m, _ := Compile(pattern_alternatives) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_alternatives_mismatch) - } -} -func BenchmarkAlternativesRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_alternatives) - f := []byte(fixture_alternatives_match) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkAlternativesRegexpMismatch(b *testing.B) { - m := regexp.MustCompile(regexp_alternatives) - f := []byte(fixture_alternatives_mismatch) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} - -func BenchmarkAlternativesSuffixFirstGlobMatch(b *testing.B) { - m, _ := Compile(pattern_alternatives_suffix) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_alternatives_suffix_first_match) - } -} -func BenchmarkAlternativesSuffixFirstGlobMismatch(b *testing.B) { - m, _ := Compile(pattern_alternatives_suffix) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_alternatives_suffix_first_mismatch) - } -} -func BenchmarkAlternativesSuffixSecondGlobMatch(b *testing.B) { - m, _ := Compile(pattern_alternatives_suffix) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_alternatives_suffix_second) - } -} -func BenchmarkAlternativesCombineLiteGlobMatch(b *testing.B) { - m, _ := Compile(pattern_alternatives_combine_lite) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_alternatives_combine_lite) - } -} -func BenchmarkAlternativesCombineHardGlobMatch(b *testing.B) { - m, _ := Compile(pattern_alternatives_combine_hard) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_alternatives_combine_hard) - } -} -func BenchmarkAlternativesSuffixFirstRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_alternatives_suffix) - f := []byte(fixture_alternatives_suffix_first_match) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkAlternativesSuffixFirstRegexpMismatch(b *testing.B) { - m := regexp.MustCompile(regexp_alternatives_suffix) - f := []byte(fixture_alternatives_suffix_first_mismatch) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkAlternativesSuffixSecondRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_alternatives_suffix) - f := []byte(fixture_alternatives_suffix_second) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkAlternativesCombineLiteRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_alternatives_combine_lite) - f := []byte(fixture_alternatives_combine_lite) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkAlternativesCombineHardRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_alternatives_combine_hard) - f := []byte(fixture_alternatives_combine_hard) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} - -func BenchmarkPlainGlobMatch(b *testing.B) { - m, _ := Compile(pattern_plain) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_plain_match) - } -} -func BenchmarkPlainRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_plain) - f := []byte(fixture_plain_match) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkPlainGlobMismatch(b *testing.B) { - m, _ := Compile(pattern_plain) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_plain_mismatch) - } -} -func BenchmarkPlainRegexpMismatch(b *testing.B) { - m := regexp.MustCompile(regexp_plain) - f := []byte(fixture_plain_mismatch) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} - -func BenchmarkPrefixGlobMatch(b *testing.B) { - m, _ := Compile(pattern_prefix) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_prefix_suffix_match) - } -} -func BenchmarkPrefixRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_prefix) - f := []byte(fixture_prefix_suffix_match) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} -func BenchmarkPrefixGlobMismatch(b *testing.B) { - m, _ := Compile(pattern_prefix) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_prefix_suffix_mismatch) - } -} -func BenchmarkPrefixRegexpMismatch(b *testing.B) { - m := regexp.MustCompile(regexp_prefix) - f := []byte(fixture_prefix_suffix_mismatch) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) - } -} - -func BenchmarkSuffixGlobMatch(b *testing.B) { - m, _ := Compile(pattern_suffix) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_prefix_suffix_match) - } -} -func BenchmarkSuffixRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_suffix) - f := []byte(fixture_prefix_suffix_match) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) +func TestPatternString(t *testing.T) { + for _, test := range []struct { + name string + str string + }{ + { + str: "", + }, + { + str: "foo", + }, + { + str: "*.github.com", + }, + { + str: "{cat,bat,[fr]at}", + }, + { + str: `\*escaped\?`, + }, + { + str: "ångstr[ö]m", + }, + } { + t.Run(test.name, func(t *testing.T) { + p, err := Compile(test.str, '.', '/') + if err != nil { + t.Fatalf("Compile(%q): %v", test.str, err) + } + if act := p.String(); act != test.str { + t.Errorf("String() = %q; want %q", act, test.str) + } + if act := fmt.Sprint(p); act != test.str { + t.Errorf("fmt.Sprint() = %q; want %q", act, test.str) + } + }) } } -func BenchmarkSuffixGlobMismatch(b *testing.B) { - m, _ := Compile(pattern_suffix) - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_prefix_suffix_mismatch) +func TestPatternSeparators(t *testing.T) { + for _, test := range []struct { + name string + sep []rune + }{ + { + sep: nil, + }, + { + sep: []rune{'.'}, + }, + { + sep: []rune{'.', '/'}, + }, + { + sep: []rune{'ö', '/', '.'}, + }, + } { + t.Run(test.name, func(t *testing.T) { + p := MustCompile("*", test.sep...) + act := p.Separators() + if !slices.Equal(act, test.sep) { + t.Errorf("Separators() = %v; want %v", act, test.sep) + } + if len(test.sep) == 0 && act != nil { + t.Errorf("Separators() = %v; want nil", act) + } + }) } } -func BenchmarkSuffixRegexpMismatch(b *testing.B) { - m := regexp.MustCompile(regexp_suffix) - f := []byte(fixture_prefix_suffix_mismatch) - for i := 0; i < b.N; i++ { - _ = m.Match(f) +// TestPatternSeparatorsAliasing checks that Separators() returns the slice +// given to Compile as is, and that matching does not depend on it. +func TestPatternSeparatorsAliasing(t *testing.T) { + sep := []rune{'.'} + p := MustCompile("*", sep...) + if got := p.Separators(); &got[0] != &sep[0] { + t.Errorf( + "Separators() does not share the backing array of the given slice", + ) } -} - -func BenchmarkPrefixSuffixGlobMatch(b *testing.B) { - m, _ := Compile(pattern_prefix_suffix) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_prefix_suffix_match) + sep[0] = 'x' + if got := p.Separators(); !slices.Equal(got, sep) { + t.Errorf("Separators() = %v; want %v", got, sep) } -} -func BenchmarkPrefixSuffixRegexpMatch(b *testing.B) { - m := regexp.MustCompile(regexp_prefix_suffix) - f := []byte(fixture_prefix_suffix_match) - - for i := 0; i < b.N; i++ { - _ = m.Match(f) + if p.Match("a.b") { + t.Errorf("Match() picked up the modification of the given slice") } -} -func BenchmarkPrefixSuffixGlobMismatch(b *testing.B) { - m, _ := Compile(pattern_prefix_suffix) - - for i := 0; i < b.N; i++ { - _ = m.Match(fixture_prefix_suffix_mismatch) + if !p.Match("axb") { + t.Errorf("Match() picked up the modification of the given slice") } } -func BenchmarkPrefixSuffixRegexpMismatch(b *testing.B) { - m := regexp.MustCompile(regexp_prefix_suffix) - f := []byte(fixture_prefix_suffix_mismatch) - for i := 0; i < b.N; i++ { - _ = m.Match(f) +func keysInOrder(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) } + slices.Sort(keys) + return keys } diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f3acb1a --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/gobwas/glob + +go 1.22.0 diff --git a/internal/debug/debug_disabled.go b/internal/debug/debug_disabled.go new file mode 100644 index 0000000..1dcc50d --- /dev/null +++ b/internal/debug/debug_disabled.go @@ -0,0 +1,8 @@ +//go:build !globdebug +// +build !globdebug + +package debug + +const Enabled = false + +func Printf(f string, args ...any) {} diff --git a/internal/debug/debug_enabled.go b/internal/debug/debug_enabled.go new file mode 100644 index 0000000..26aeebd --- /dev/null +++ b/internal/debug/debug_enabled.go @@ -0,0 +1,14 @@ +//go:build globdebug +// +build globdebug + +package debug + +import ( + "fmt" +) + +const Enabled = true + +func Printf(f string, args ...any) { + fmt.Printf(f, args...) +} diff --git a/internal/debug/tree.go b/internal/debug/tree.go new file mode 100644 index 0000000..c716a14 --- /dev/null +++ b/internal/debug/tree.go @@ -0,0 +1,8 @@ +package debug + +// Tree renders the matcher tree of a compiled *glob.Pattern. +// +// It is set by package glob at init time: the pattern internals are +// unexported, and this keeps them so while letting the in-module tooling +// (cmd/globtest -v) print them. The format is not stable. +var Tree func(pattern any) string diff --git a/issues_test.go b/issues_test.go new file mode 100644 index 0000000..1282aea --- /dev/null +++ b/issues_test.go @@ -0,0 +1,63 @@ +package glob + +import "testing" + +// TestOpenIssues verifies the reproduction cases from the GitHub issues +// reported against the pre-v1 versions. Each case is labeled with its issue +// number: https://github.com/gobwas/glob/issues/. +func TestOpenIssues(t *testing.T) { + for _, c := range []struct { + issue int + pat, str string + sep []rune + exp bool + }{ + {33, "test*201901*.12*", "test.20190101.120000", nil, true}, + {33, "test.*201901*.14*", "test.20190101.130000", nil, false}, + {33, "test.*201901*.14*", "test.20190101.140000", nil, true}, + {36, "test,pattern", "test,pattern", nil, true}, + {36, `test\,pattern`, "test,pattern", nil, true}, + {41, "hell?", "hello", nil, true}, + {41, "hell[o]", "hello", nil, true}, + {41, "ångstr?m", "ångström", nil, true}, + {41, "ångstr[ö]m", "ångström", nil, true}, + {43, "{*.,}google*", "a.google.com", nil, true}, + {43, "{*.,}google*", "agoogle.com", nil, false}, + {43, "{*.,}google*", "google.com", nil, true}, + {43, "{,*.}google*", "a.google.com", nil, true}, + {43, "{,*.}google*", "agoogle.com", nil, false}, + {43, "{,*.}google*", "google.com", nil, true}, + {50, "*yandex:*.exe:page.*", "yandex:service.exe:page.12345", nil, true}, + {50, "yandex:*.exe:page.*", "yandex:service.exe:page.12345", nil, true}, + {50, "{*yandex:*.exe:page.*,x}", "yandex:service.exe:page.12345", nil, true}, + {50, "{*yandex:*.exe:page.*}", "yandex:service.exe:page.12345", nil, true}, + {51, "a*ant", "an ant", nil, true}, + {51, "a*ant", "ant", nil, false}, + {51, "br*r", "br", nil, false}, + {51, "br*r", "brother", nil, true}, + {51, "so*so", "so so", nil, true}, + {51, "so*so", "so", nil, false}, + {54, "1?5helloö", "155helloö", nil, true}, + {54, "1?5ö", "155ö", nil, true}, + {54, "1?ö5hello", "15ö5hello", nil, true}, + {54, "1ö?5", "1ö55", nil, true}, + {61, "start*art", "start", nil, false}, + {62, "{a,}{a,}a", "a", nil, true}, + {66, `{**/daxing,daxing,x,y}/**/*dev*.yaml`, "playground/daxing/generated/dev.yaml", nil, true}, + {66, `{**/daxing,daxing,x}/**/*dev*.yaml`, "playground/daxing/generated/dev.yaml", nil, true}, + {66, `{**/daxing,daxing}/**/*dev*.yaml`, "playground/daxing/generated/dev.yaml", nil, true}, + {66, `{**/daxing}/**/*dev*.yaml`, "playground/daxing/generated/dev.yaml", nil, true}, + } { + got := MustCompile(c.pat, c.sep...).Match(c.str) + if got != c.exp { + t.Errorf("issue #%d: Match(%q, %q) = %t; want %t", c.issue, c.pat, c.str, got, c.exp) + } + } + // Crashers: old versions panicked, new must return a syntax error. + if _, err := Compile("/{a{*.json{", '/'); err == nil { + t.Errorf("issue #59: Compile succeeded; want error") + } + if _, err := Compile("0{", '.', '/'); err == nil { + t.Errorf("issue #56: Compile succeeded; want error") + } +} diff --git a/match.go b/match.go new file mode 100644 index 0000000..2b84257 --- /dev/null +++ b/match.go @@ -0,0 +1,788 @@ +package glob + +import ( + "fmt" + "slices" + "strconv" + "strings" + "sync" + "unicode/utf8" + "unsafe" + + "github.com/gobwas/glob/internal/debug" +) + +// Match reports whether s matches the pattern. +func (p *Pattern) Match(s string) bool { + var x matchContext + if p.state { + if len(s) < p.minLen || !strings.HasSuffix(s, p.suffix) { + return false + } + state := acquireState() + defer releaseState(state) + x.state = state + } + for { + n, match := p.m.Match(x, s[x.offset:]) + // Note: debug.Enabled is a build-tag constant; when it is false the + // whole block (including the argument evaluation) is compiled away. + if debug.Enabled && x.state != nil { + debug.Printf("stack: %s\n", formatStack(x.state.stack)) + debug.Printf("stars: %s\n", formatStack(x.state.stars)) + } + if match && n == len(s[x.offset:]) { + if debug.Enabled { + debug.Printf("match!\n") + } + return true + } + if x.state == nil { + // The pattern never saves checkpoints (see [needsState]): + // nothing to backtrack to. + return false + } + var ( + c checkpoint + k checkpointKind + ) + switch { + case len(x.state.stars) > 0: + if debug.Enabled { + debug.Printf("has star\n") + } + c = popLast(&x.state.stars) + k = checkpointStars + + case len(x.state.stack) > 0: + if debug.Enabled { + debug.Printf("has stack\n") + } + c = popLast(&x.state.stack) + k = checkpointStack + + default: + if debug.Enabled { + debug.Printf("no match\n") + } + return false + } + x.offset = c.offset + x.kind = k + x.frame = frame{ + path: c.path, + depth: 0, + } + } +} + +// matcher is a node of the tree a pattern compiles into. +// +// Match matches the beginning of s and reports how many bytes it consumed; +// the caller goes on with the rest. A matcher may consume nothing and still +// match: a void does, and so does a non-terminal star -- it stores its +// restart points instead and lets the walk continue, to be resumed from one +// of them on a mismatch later; see [Pattern.Match]. The context tells where +// in the input and in the tree the matcher is; see [matchContext]. +// +// String renders the node for the debug output and cmd/globtest -v; the +// notation is described at [Pattern]. +type matcher interface { + Match(matchContext, string) (n int, matched bool) + String() string +} + +// frame tells where in the matcher tree the walk currently is: the path from +// the root down to the current node, and the depth of the node. +// +// A checkpoint stores the frame's path; resuming it re-enters the tree from +// the root and follows the path back to the very node that saved it (an +// alternative to try, or a star to restart) -- see [Pattern.Match], +// [multiMatcher.Match] and [altMatcher.Match]. +// +// For `{a*b,c}y`, compiled into [{["a"·*·"b"]|"c"}·"y"], the frame at each +// node the walk visits is: +// +// node path depth +// [{["a"·*·"b"]|"c"}·"y"] [] 0 the root +// {["a"·*·"b"]|"c"} [0]* 1 +// ["a"·*·"b"] [0 0]* 2 +// "a" [0 0 0]* 3 +// * [0 0 1] 3 +// "b" [0 0 2] 3 +// "c" [0 1] 2 reached only by resuming the +// checkpoint the alt saved +// "y" [1] 1 +// +// * the zeros are virtual: turning to child #0 records nothing, so the +// path actually stored is [] -- see path below. +type frame struct { + // path addresses the current node: path[d] is the index of the child taken + // at depth d, that is, in the multiMatcher or altMatcher d levels below + // the root. + // + // For `{a*b,c}y`, compiled into [{["a"·*·"b"]|"c"}·"y"]: + // + // [0 1] the "c" alternative -- what the alt checkpoints when it + // enters ["a"·*·"b"], to be tried on a mismatch + // + // [0 0 1] the star: child #1 of ["a"·*·"b"], which is alternative #0 + // of the alt, which is child #0 of the root sequence -- what + // the star's restart points carry + // + // It is recorded lazily: an index is stored only when the walk turns to a + // child other than #0 (see [matchContext.branch]), so the path may be + // shorter than the current depth -- the missing trailing entries are + // implicitly zero, and [frame.index] reads them as such. In the example + // above, at "a" the path is still empty, not [0 0 0]: the alt, alternative + // #0 and "a" were all entered as child #0. + path []int + + // depth is the depth of the current node (the one [frame.path] leads to): + // 0 at the root, 1 at its children, and so on; [matchContext.next] + // increments it on every descent. It is also where the node's own entry in + // path lives: path[depth] is the child to take next -- [frame.index] reads + // it, [matchContext.branch] writes it. + // + // It is not len(path): the path is recorded lazily, so it may fall short + // of the depth, and, when resuming a checkpoint, it is the whole path of + // the checkpoint, reaching beyond the depth all the way down to the node + // to resume at. + // + // To say it the other way, depth is what len(path) would be were the path + // always recorded in full -- the virtual zeros included -- and cut at the + // current node: the length of the node's full address in the tree. + // + // For `{a*b,c}y`, compiled into [{["a"·*·"b"]|"c"}·"y"], the nodes at + // each depth are: + // + // 0 [{["a"·*·"b"]|"c"}·"y"] the root sequence + // 1 {["a"·*·"b"]|"c"}, "y" its children + // 2 ["a"·*·"b"], "c" the alternatives + // 3 "a", *, "b" the children of ["a"·*·"b"] + depth int +} + +// index returns the index of the child to take at the current node: the one +// the path leads to when resuming a checkpoint, #0 otherwise. +func (v frame) index() int { + if v.depth >= len(v.path) { + return 0 + } + return v.path[v.depth] +} + +// checkpoint is a place to resume the walk from on a mismatch. +type checkpoint struct { + // offset is the position in the input to resume at. + offset int + + // path leads to the node that saved the checkpoint; see [frame]. + // + // Unlike a live frame's, it is always at full length -- the virtual zeros + // written out -- so len(path) is the depth of that node, and a checkpoint + // needs no depth of its own: resuming starts at the root and tells it has + // arrived by comparing the walk's depth against len(path); see + // [matchContext.branch], [matchContext.storeStar] and [altMatcher.Match]. + path []int +} + +// checkpointKind tells which pile a checkpoint was taken from during the +// backtracking in [Pattern.Match]. +type checkpointKind int + +const ( + // checkpointStack is an alternative checkpoint saved by altMatcher. + checkpointStack checkpointKind = iota + // checkpointStars is a star restart point saved by starMatcher. + checkpointStars +) + +// matchContext is what a matcher is called with: where in the input and in +// the tree it is, plus the backtracking state shared by the whole walk. It +// is passed by value, so a matcher's changes to it are seen by its +// descendants only. +type matchContext struct { + // offset is the position in the whole input the current node matches from. + // The matchers see only the remainder of the input, so it is what a + // checkpoint records to resume at the same place; see [checkpoint]. + offset int + + // frame is where in the matcher tree the current node is; see [frame]. + frame frame + + // state holds the checkpoint piles and the path arena shared by the + // whole walk. + state *matchState + + // kind tells which pile the checkpoint being resumed was taken from. + // See [altMatcher.Match] for its use. + kind checkpointKind + + // starsFloor is the number of star restart points that existed when + // the walk entered the current alternative. The entries below it were + // born outside of the alternative and must not be discarded by the + // stars inside it; see [matchContext.storeStar]. + starsFloor int +} + +// push saves an alternative checkpoint at the current offset for the node f +// leads to; see [altMatcher.Match]. +func (x matchContext) push(f frame) { + x.state.stack = append(x.state.stack, checkpoint{ + offset: x.offset, + path: f.path, + }) + if debug.Enabled { + debug.Printf( + "checkpoint offset=%d path=%v\n", + x.offset, f.path, + ) + } +} + +// storeStar saves a restart point for the current star at offset bytes +// further in the input; reset tells whether the star may discard the pending +// restart points first, see below. +func (x matchContext) storeStar(offset int, reset bool) { + path := x.frame.path + if d := x.frame.depth; len(path) < d { + // The walk records an index in path only when it turns to a child + // other than the first one; levels entered at child #0 are implicit. + // Store the path at its full length (the missing entries are always + // zeros) so that the alts above can tell this checkpoint from their + // own. See [altMatcher.Match]. + p := x.state.allocPath(d) + copy(p, path) + path = p + } + if reset { + // This star can extend over anything the pending restart points could + // reach -- they are redundant, discard them. See + // research.swtch.com/glob. + // + // However, only the restart points born inside the current alternative + // may be discarded. An outer star, when resumed, re-enters the + // enclosing alt and may pick another alternative -- something this + // star, locked inside its own alternative, can not absorb. See the + // `*{*0,}` test: the outer star must survive the inner one to reach + // the empty alternative. + x.state.stars = x.state.stars[:x.starsFloor] + } + x.state.stars = append(x.state.stars, checkpoint{ + offset: x.offset + offset, + path: path, + }) + if debug.Enabled { + debug.Printf( + "star offset=%d path=%v reset=%t\n", + x.offset+offset, path, reset, + ) + } +} + +// next returns the context for a child of the current node, matching offset +// bytes further in the input: one level deeper in the tree. +func (x matchContext) next(offset int) matchContext { + x.offset = x.offset + offset + x.frame.depth += 1 + return x +} + +// branch returns a copy of the current frame with its path turned to child i +// at the current level, discarding the deeper levels. The new path is +// allocated from the state's arena. +func (x matchContext) branch(i int) frame { + f := x.frame + path := x.state.allocPath(f.depth + 1) + copy(path, f.path) + path[f.depth] = i + f.path = path + return f +} + +// matchState holds the backtracking state of a single [Pattern.Match] call. +// The states are pooled globally: the buffers keep their grown capacity +// between the matches, so a steady-state Match does not allocate them. +type matchState struct { + // stars are the star restart points and stack the alternative + // checkpoints, both LIFO. On a mismatch the walk resumes from the most + // recent restart point, if any, before the most recent alternative; see + // [Pattern.Match]. + stars []checkpoint + stack []checkpoint + + // arena is the buffer the checkpoint paths are allocated from; see + // [matchState.allocPath]. It is bulk-freed when the match ends, which + // spares the per-path lifetime reasoning: a path may be shared between + // the current frame and several checkpoints. + arena []int +} + +// allocPath returns a zeroed []int of length n allocated from the state's +// arena. When the arena runs out of capacity, a fresh chunk is started; the +// paths allocated from the previous chunks stay valid, since the chunks are +// kept alive by the paths referencing them. +func (st *matchState) allocPath(n int) []int { + if cap(st.arena)-len(st.arena) < n { + st.arena = make([]int, 0, max(2*cap(st.arena), n, 32)) + } + p := st.arena[len(st.arena) : len(st.arena)+n : len(st.arena)+n] + st.arena = st.arena[:len(st.arena)+n] + clear(p) + return p +} + +var statePool sync.Pool // Pool[*matchState] + +// acquireState takes a state from the pool, or makes a new one. +func acquireState() *matchState { + if st, _ := statePool.Get().(*matchState); st != nil { + return st + } + return &matchState{} +} + +// releaseState empties the state and puts it back to the pool. +func releaseState(st *matchState) { + resetCheckpoints(&st.stars) + resetCheckpoints(&st.stack) + // The arena holds no references; keep the (largest) chunk as is. + st.arena = st.arena[:0] + statePool.Put(st) +} + +// resetCheckpoints empties s keeping its capacity. The whole backing array +// is zeroed (not only the live part) to drop the references to the +// checkpoint paths popped during the match. +func resetCheckpoints(s *[]checkpoint) { + full := (*s)[:cap(*s)] + clear(full) + *s = full[:0] +} + +// multiMatcher is a sequence, ["a"·*·"b"]: it matches its children one +// after another, each on the input the previous ones left. +type multiMatcher []matcher + +func (ms multiMatcher) String() string { + var sb strings.Builder + sb.WriteByte('[') + for i, m := range ms { + if i > 0 { + sb.WriteString("·") + } + sb.WriteString(m.String()) + } + sb.WriteByte(']') + return sb.String() +} + +func (ms multiMatcher) Match(x matchContext, s string) (n int, ok bool) { + for i := x.frame.index(); i < len(ms); i++ { + if i != x.frame.index() && x.state != nil { + // The path is recorded for the checkpoints the descendants may + // save; in a stateless walk (see [needsState]) there are none + // and nobody would ever read it. + x.frame = x.branch(i) + } + child := x.next(n) + k, ok := ms[i].Match(child, s[n:]) + if debug.Enabled { + debug.Printf( + "[%T@%p] #%d match %#q against %[5]T(%[5]s) at path=%v depth=%d => %d %t\n", + ms, unsafe.SliceData(ms), i, s[n:], ms[i], + child.frame.path, child.frame.depth, k, ok, + ) + } + if !ok { + return 0, false + } + n += k + } + return n, true +} + +// altMatcher is a group of alternatives, {"a"|"b"}: it matches the one the +// walk is at (the first one when entered anew), having saved a checkpoint +// for the next one to be tried on a mismatch later. +type altMatcher []matcher + +func (ms altMatcher) String() string { + var sb strings.Builder + sb.WriteByte('{') + for i, m := range ms { + if i > 0 { + sb.WriteString("|") + } + sb.WriteString(m.String()) + } + sb.WriteByte('}') + return sb.String() +} + +func (ms altMatcher) Match(x matchContext, s string) (int, bool) { + i := x.frame.index() + // Save a checkpoint for the next alternative to consider it in case of + // a mismatch later (if any). This must be done only when: + // + // - the alt is entered for the first time (the resume path ends above + // this level, or there is none); + // + // - the resume path ends exactly at this level with an alternative + // checkpoint -- its job is "try alternative #i", so the one for the + // next alternative must be saved now. Note that a star restart point + // may end at this level too (a star being a direct child of the alt, + // as in `{*,b}`) -- it must not trigger a save. + // + // Otherwise the walk is merely passing through this alt on its way to + // resume a deeper checkpoint -- the one for the next alternative was + // already saved when the alt was entered for the first time, and saving + // it again on every star restart would blow the stack up exponentially. + // See the "alternatives" tests. + if next := i + 1; next < len(ms) { + d := len(x.frame.path) + if x.frame.depth >= d || (x.frame.depth == d-1 && x.kind == checkpointStack) { + x.push(x.branch(next)) + } + } + // The stars below this level may only discard the restart points born + // inside the same alternative; see [matchContext.storeStar]. + x.starsFloor = len(x.state.stars) + child := x.next(0) + n, match := ms[i].Match(child, s) + if debug.Enabled { + debug.Printf( + "[%T@%p] #%d match %#q against %[5]T(%[5]s) at path=%v depth=%d => %d %t\n", + ms, unsafe.SliceData(ms), i, s, ms[i], + child.frame.path, child.frame.depth, n, match, + ) + } + return n, match +} + +// textMatcher is a literal, "abc". +type textMatcher struct { + Text string +} + +func (m *textMatcher) String() string { + return strconv.Quote(m.Text) +} + +func (m *textMatcher) Match(_ matchContext, s string) (int, bool) { + if strings.HasPrefix(s, m.Text) { + return len(m.Text), true + } + return 0, false +} + +// charMatcher is a `?`: any single character but a separator. +type charMatcher struct { + Sep []rune +} + +func (m *charMatcher) String() string { + var sb strings.Builder + sb.WriteByte('?') + if len(m.Sep) > 0 { + sb.WriteByte('(') + formatRunes(&sb, m.Sep) + sb.WriteByte(')') + } + return sb.String() +} + +func (m *charMatcher) Match(_ matchContext, s string) (int, bool) { + if len(s) == 0 { + return 0, false + } + r, n := utf8.DecodeRuneInString(s) + if slices.Contains(m.Sep, r) { + return 0, false + } + return n, true +} + +// starMatcher is a `*` or a `**`: any sequence of characters, but for the +// separators in the former case. +type starMatcher struct { + // Sep are the separators the star may not extend over; empty for `**`. + Sep []rune + // SepStr is Sep as a string, for the byte-wise scans below. + SepStr string + + // Next is the literal the matcher right after this star begins with + // (when it is a textMatcher), set by [annotateStars]. A restart point + // at a position where the literal does not occur is a guaranteed + // mismatch, so the star jumps between its occurrences instead of + // retrying at every rune. + Next string + // Terminal is set by [annotateStars] when nothing follows this star + // anywhere in the pattern: the star then consumes everything in its + // reach at once, and no restart point can change the outcome. + Terminal bool +} + +// reach returns the length of the prefix of s the star may extend over: +// everything up to the nearest separator. +func (m *starMatcher) reach(s string) int { + if m.SepStr == "" { + return len(s) + } + if e := strings.IndexAny(s, m.SepStr); e >= 0 { + return e + } + return len(s) +} + +// storeSkip stores the restart point at the next occurrence of the m.Next +// literal instead of the next rune. +func (m *starMatcher) storeSkip(x matchContext, s string) { + reach := m.reach(s) + // Look for the occurrences starting within the star's reach; the + // literal itself may extend past it (it may contain the separators). + // Note that a valid UTF-8 literal can not match at a mid-rune + // position, so the one-byte skip below is rune-safe. + end := min(reach+len(m.Next), len(s)) + j := strings.Index(s[1:end], m.Next) + if j < 0 || 1+j > reach { + return + } + x.storeStar(1+j, len(m.Sep) == 0) +} + +func (m *starMatcher) String() string { + var sb strings.Builder + sb.WriteByte('*') + if len(m.Sep) > 0 { + sb.WriteByte('(') + formatRunes(&sb, m.Sep) + sb.WriteByte(')') + } + return sb.String() +} + +func (m *starMatcher) Match(x matchContext, s string) (int, bool) { + if m.Terminal { + // Nothing follows this star in the pattern: either it consumes + // the whole remainder within its reach, or the match fails. + return m.reach(s), true + } + if len(s) == 0 { + return 0, true + } + if m.Next != "" { + m.storeSkip(x, s) + return 0, true + } + r, n := utf8.DecodeRuneInString(s) + if !slices.Contains(m.Sep, r) { + // The star may extend over the rune: save the restart point past + // it. A separator-free star (`**`) can extend over anything the + // pending restart points could reach, so it may discard them; see + // [matchContext.storeStar]. + x.storeStar(n, len(m.Sep) == 0) + } + return 0, true +} + +// runeRangeMatcher is a character range class, `[a-z]` or `[!a-z]`. +type runeRangeMatcher struct { + Lo rune + Hi rune + Not bool +} + +func (m *runeRangeMatcher) String() string { + var sb strings.Builder + if m.Not { + sb.WriteByte('!') + } + sb.WriteByte('[') + sb.WriteRune(m.Lo) + sb.WriteByte('-') + sb.WriteRune(m.Hi) + sb.WriteByte(']') + return sb.String() +} + +func (m *runeRangeMatcher) Match(_ matchContext, s string) (int, bool) { + // Note that an invalid byte decodes as U+FFFD, and is matched as such, + // the same way regexp does; only the empty input is a mismatch. + r, n := utf8.DecodeRuneInString(s) + if n == 0 { + return 0, false + } + ok := m.Lo <= r && r <= m.Hi + if ok != m.Not { + return n, true + } + return 0, false +} + +// runeSetMatcher is a character set class, `[abc]` or `[!abc]`. +type runeSetMatcher struct { + Set map[rune]struct{} + Not bool +} + +// formatRunes writes rs, sorted and comma-separated, to sb. +func formatRunes(sb *strings.Builder, rs []rune) { + rs = slices.Clone(rs) // Not to reorder the caller's, e.g. a matcher's Sep. + slices.Sort(rs) + for i, r := range rs { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteRune(r) + } +} + +func (m *runeSetMatcher) String() string { + rs := make([]rune, 0, len(m.Set)) + for r := range m.Set { + rs = append(rs, r) + } + var sb strings.Builder + if m.Not { + sb.WriteByte('!') + } + sb.WriteByte('[') + formatRunes(&sb, rs) + sb.WriteByte(']') + return sb.String() +} + +func (m *runeSetMatcher) Match(_ matchContext, s string) (int, bool) { + // See the note in runeRangeMatcher.Match. + r, n := utf8.DecodeRuneInString(s) + if n == 0 { + return 0, false + } + if _, has := m.Set[r]; has != m.Not { + return n, true + } + return 0, false +} + +// voidMatcher matches the empty string: an empty alternative, `{a,}`. In a +// sequence it is dropped by [normalizeSequence]. +type voidMatcher struct{} + +func (*voidMatcher) String() string { + return "void" +} + +func (*voidMatcher) Match(matchContext, string) (int, bool) { + return 0, true +} + +// The shaped matchers below are the compile-time rewrites of the common +// terminal sub-sequences; see [specialize]. Nothing may follow them in the +// pattern, so each one either consumes the whole remainder of the input or +// fails -- deterministically, storing no checkpoints. + +// prefixMatcher is a terminal `abc*`. +type prefixMatcher struct { + Text string + Sep string +} + +func (m *prefixMatcher) String() string { + return "prefix(" + strconv.Quote(m.Text) + ")" +} + +func (m *prefixMatcher) Match(_ matchContext, s string) (int, bool) { + if strings.HasPrefix(s, m.Text) && noSep(s[len(m.Text):], m.Sep) { + return len(s), true + } + return 0, false +} + +// suffixMatcher is a terminal `*abc`. +type suffixMatcher struct { + Text string + Sep string +} + +func (m *suffixMatcher) String() string { + return "suffix(" + strconv.Quote(m.Text) + ")" +} + +func (m *suffixMatcher) Match(_ matchContext, s string) (int, bool) { + if strings.HasSuffix(s, m.Text) && noSep(s[:len(s)-len(m.Text)], m.Sep) { + return len(s), true + } + return 0, false +} + +// prefixSuffixMatcher is a terminal `abc*def`. +type prefixSuffixMatcher struct { + Prefix string + Suffix string + Sep string +} + +func (m *prefixSuffixMatcher) String() string { + return "prefix_suffix(" + strconv.Quote(m.Prefix) + "," + strconv.Quote(m.Suffix) + ")" +} + +func (m *prefixSuffixMatcher) Match(_ matchContext, s string) (int, bool) { + // The length check keeps the prefix and the suffix from overlapping: + // `a*ant` must not match `ant`. + if len(s) >= len(m.Prefix)+len(m.Suffix) && + strings.HasPrefix(s, m.Prefix) && + strings.HasSuffix(s, m.Suffix) && + noSep(s[len(m.Prefix):len(s)-len(m.Suffix)], m.Sep) { + return len(s), true + } + return 0, false +} + +// containsMatcher is a terminal `*abc*` with the separator-free stars. +type containsMatcher struct { + Text string +} + +func (m *containsMatcher) String() string { + return "contains(" + strconv.Quote(m.Text) + ")" +} + +func (m *containsMatcher) Match(_ matchContext, s string) (int, bool) { + if strings.Contains(s, m.Text) { + return len(s), true + } + return 0, false +} + +// noSep reports whether s contains none of the separators. +func noSep(s, sep string) bool { + return sep == "" || !strings.ContainsAny(s, sep) +} + +// formatCheckpoint renders c as path@offset for the debug output. +func formatCheckpoint(c checkpoint) string { + return fmt.Sprintf("%v@%d", c.path, c.offset) +} + +// formatStack renders the checkpoint pile s for the debug output, the most +// recent one last. +func formatStack(s []checkpoint) string { + var sb strings.Builder + for i, c := range s { + if i > 0 { + sb.WriteString(" -> ") + } + sb.WriteString(formatCheckpoint(c)) + } + return sb.String() +} + +// popLast panics if s is empty. +func popLast[T any, E ~[]T](s *E) T { + n := len(*s) + r := (*s)[n-1] + *s = (*s)[:n-1] + return r +} diff --git a/match/any.go b/match/any.go deleted file mode 100644 index 514a9a5..0000000 --- a/match/any.go +++ /dev/null @@ -1,45 +0,0 @@ -package match - -import ( - "fmt" - "github.com/gobwas/glob/util/strings" -) - -type Any struct { - Separators []rune -} - -func NewAny(s []rune) Any { - return Any{s} -} - -func (self Any) Match(s string) bool { - return strings.IndexAnyRunes(s, self.Separators) == -1 -} - -func (self Any) Index(s string) (int, []int) { - found := strings.IndexAnyRunes(s, self.Separators) - switch found { - case -1: - case 0: - return 0, segments0 - default: - s = s[:found] - } - - segments := acquireSegments(len(s)) - for i := range s { - segments = append(segments, i) - } - segments = append(segments, len(s)) - - return 0, segments -} - -func (self Any) Len() int { - return lenNo -} - -func (self Any) String() string { - return fmt.Sprintf("", string(self.Separators)) -} diff --git a/match/any_of.go b/match/any_of.go deleted file mode 100644 index 8e65356..0000000 --- a/match/any_of.go +++ /dev/null @@ -1,82 +0,0 @@ -package match - -import "fmt" - -type AnyOf struct { - Matchers Matchers -} - -func NewAnyOf(m ...Matcher) AnyOf { - return AnyOf{Matchers(m)} -} - -func (self *AnyOf) Add(m Matcher) error { - self.Matchers = append(self.Matchers, m) - return nil -} - -func (self AnyOf) Match(s string) bool { - for _, m := range self.Matchers { - if m.Match(s) { - return true - } - } - - return false -} - -func (self AnyOf) Index(s string) (int, []int) { - index := -1 - - segments := acquireSegments(len(s)) - for _, m := range self.Matchers { - idx, seg := m.Index(s) - if idx == -1 { - continue - } - - if index == -1 || idx < index { - index = idx - segments = append(segments[:0], seg...) - continue - } - - if idx > index { - continue - } - - // here idx == index - segments = appendMerge(segments, seg) - } - - if index == -1 { - releaseSegments(segments) - return -1, nil - } - - return index, segments -} - -func (self AnyOf) Len() (l int) { - l = -1 - for _, m := range self.Matchers { - ml := m.Len() - switch { - case l == -1: - l = ml - continue - - case ml == -1: - return -1 - - case l != ml: - return -1 - } - } - - return -} - -func (self AnyOf) String() string { - return fmt.Sprintf("", self.Matchers) -} diff --git a/match/any_of_test.go b/match/any_of_test.go deleted file mode 100644 index 3b478cf..0000000 --- a/match/any_of_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestAnyOfIndex(t *testing.T) { - for id, test := range []struct { - matchers Matchers - fixture string - index int - segments []int - }{ - { - Matchers{ - NewAny(nil), - NewText("b"), - NewText("c"), - }, - "abc", - 0, - []int{0, 1, 2, 3}, - }, - { - Matchers{ - NewPrefix("b"), - NewSuffix("c"), - }, - "abc", - 0, - []int{3}, - }, - { - Matchers{ - NewList([]rune("[def]"), false), - NewList([]rune("[abc]"), false), - }, - "abcdef", - 0, - []int{1}, - }, - } { - everyOf := NewAnyOf(test.matchers...) - index, segments := everyOf.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} diff --git a/match/any_test.go b/match/any_test.go deleted file mode 100644 index 358f553..0000000 --- a/match/any_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestAnyIndex(t *testing.T) { - for id, test := range []struct { - sep []rune - fixture string - index int - segments []int - }{ - { - []rune{'.'}, - "abc", - 0, - []int{0, 1, 2, 3}, - }, - { - []rune{'.'}, - "abc.def", - 0, - []int{0, 1, 2, 3}, - }, - } { - p := NewAny(test.sep) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexAny(b *testing.B) { - m := NewAny(bench_separators) - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexAnyParallel(b *testing.B) { - m := NewAny(bench_separators) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/btree.go b/match/btree.go deleted file mode 100644 index 8302bf8..0000000 --- a/match/btree.go +++ /dev/null @@ -1,185 +0,0 @@ -package match - -import ( - "fmt" - "unicode/utf8" -) - -type BTree struct { - Value Matcher - Left Matcher - Right Matcher - ValueLengthRunes int - LeftLengthRunes int - RightLengthRunes int - LengthRunes int -} - -func NewBTree(Value, Left, Right Matcher) (tree BTree) { - tree.Value = Value - tree.Left = Left - tree.Right = Right - - lenOk := true - if tree.ValueLengthRunes = Value.Len(); tree.ValueLengthRunes == -1 { - lenOk = false - } - - if Left != nil { - if tree.LeftLengthRunes = Left.Len(); tree.LeftLengthRunes == -1 { - lenOk = false - } - } - - if Right != nil { - if tree.RightLengthRunes = Right.Len(); tree.RightLengthRunes == -1 { - lenOk = false - } - } - - if lenOk { - tree.LengthRunes = tree.LeftLengthRunes + tree.ValueLengthRunes + tree.RightLengthRunes - } else { - tree.LengthRunes = -1 - } - - return tree -} - -func (self BTree) Len() int { - return self.LengthRunes -} - -// todo? -func (self BTree) Index(s string) (index int, segments []int) { - //inputLen := len(s) - //// try to cut unnecessary parts - //// by knowledge of length of right and left part - //offset, limit := self.offsetLimit(inputLen) - //for offset < limit { - // // search for matching part in substring - // vi, segments := self.Value.Index(s[offset:limit]) - // if index == -1 { - // return -1, nil - // } - // if self.Left == nil { - // if index != offset { - // return -1, nil - // } - // } else { - // left := s[:offset+vi] - // i := self.Left.IndexSuffix(left) - // if i == -1 { - // return -1, nil - // } - // index = i - // } - // if self.Right != nil { - // for _, seg := range segments { - // right := s[:offset+vi+seg] - // } - // } - - // l := s[:offset+index] - // var left bool - // if self.Left != nil { - // left = self.Left.Index(l) - // } else { - // left = l == "" - // } - //} - - return -1, nil -} - -func (self BTree) Match(s string) bool { - inputLen := len(s) - // try to cut unnecessary parts - // by knowledge of length of right and left part - offset, limit := self.offsetLimit(inputLen) - - for offset < limit { - // search for matching part in substring - index, segments := self.Value.Index(s[offset:limit]) - if index == -1 { - releaseSegments(segments) - return false - } - - l := s[:offset+index] - var left bool - if self.Left != nil { - left = self.Left.Match(l) - } else { - left = l == "" - } - - if left { - for i := len(segments) - 1; i >= 0; i-- { - length := segments[i] - - var right bool - var r string - // if there is no string for the right branch - if inputLen <= offset+index+length { - r = "" - } else { - r = s[offset+index+length:] - } - - if self.Right != nil { - right = self.Right.Match(r) - } else { - right = r == "" - } - - if right { - releaseSegments(segments) - return true - } - } - } - - _, step := utf8.DecodeRuneInString(s[offset+index:]) - offset += index + step - - releaseSegments(segments) - } - - return false -} - -func (self BTree) offsetLimit(inputLen int) (offset int, limit int) { - // self.Length, self.RLen and self.LLen are values meaning the length of runes for each part - // here we manipulating byte length for better optimizations - // but these checks still works, cause minLen of 1-rune string is 1 byte. - if self.LengthRunes != -1 && self.LengthRunes > inputLen { - return 0, 0 - } - if self.LeftLengthRunes >= 0 { - offset = self.LeftLengthRunes - } - if self.RightLengthRunes >= 0 { - limit = inputLen - self.RightLengthRunes - } else { - limit = inputLen - } - return offset, limit -} - -func (self BTree) String() string { - const n string = "" - var l, r string - if self.Left == nil { - l = n - } else { - l = self.Left.String() - } - if self.Right == nil { - r = n - } else { - r = self.Right.String() - } - - return fmt.Sprintf("%s]>", l, self.Value, r) -} diff --git a/match/btree_test.go b/match/btree_test.go deleted file mode 100644 index 3bd9ea5..0000000 --- a/match/btree_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package match - -import ( - "testing" -) - -func TestBTree(t *testing.T) { - for id, test := range []struct { - tree BTree - str string - exp bool - }{ - { - NewBTree(NewText("abc"), NewSuper(), NewSuper()), - "abc", - true, - }, - { - NewBTree(NewText("a"), NewSingle(nil), NewSingle(nil)), - "aaa", - true, - }, - { - NewBTree(NewText("b"), NewSingle(nil), nil), - "bbb", - false, - }, - { - NewBTree( - NewText("c"), - NewBTree( - NewSingle(nil), - NewSuper(), - nil, - ), - nil, - ), - "abc", - true, - }, - } { - act := test.tree.Match(test.str) - if act != test.exp { - t.Errorf("#%d match %q error: act: %t; exp: %t", id, test.str, act, test.exp) - continue - } - } -} - -type fakeMatcher struct { - len int - name string -} - -func (f *fakeMatcher) Match(string) bool { - return true -} - -var i = 3 - -func (f *fakeMatcher) Index(s string) (int, []int) { - seg := make([]int, 0, i) - for x := 0; x < i; x++ { - seg = append(seg, x) - } - return 0, seg -} -func (f *fakeMatcher) Len() int { - return f.len -} -func (f *fakeMatcher) String() string { - return f.name -} - -func BenchmarkMatchBTree(b *testing.B) { - l := &fakeMatcher{4, "left_fake"} - r := &fakeMatcher{4, "right_fake"} - v := &fakeMatcher{2, "value_fake"} - - // must be <= len(l + r + v) - fixture := "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij" - - bt := NewBTree(v, l, r) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - bt.Match(fixture) - } - }) -} diff --git a/match/contains.go b/match/contains.go deleted file mode 100644 index 0998e95..0000000 --- a/match/contains.go +++ /dev/null @@ -1,58 +0,0 @@ -package match - -import ( - "fmt" - "strings" -) - -type Contains struct { - Needle string - Not bool -} - -func NewContains(needle string, not bool) Contains { - return Contains{needle, not} -} - -func (self Contains) Match(s string) bool { - return strings.Contains(s, self.Needle) != self.Not -} - -func (self Contains) Index(s string) (int, []int) { - var offset int - - idx := strings.Index(s, self.Needle) - - if !self.Not { - if idx == -1 { - return -1, nil - } - - offset = idx + len(self.Needle) - if len(s) <= offset { - return 0, []int{offset} - } - s = s[offset:] - } else if idx != -1 { - s = s[:idx] - } - - segments := acquireSegments(len(s) + 1) - for i := range s { - segments = append(segments, offset+i) - } - - return 0, append(segments, offset+len(s)) -} - -func (self Contains) Len() int { - return lenNo -} - -func (self Contains) String() string { - var not string - if self.Not { - not = "!" - } - return fmt.Sprintf("", not, self.Needle) -} diff --git a/match/contains_test.go b/match/contains_test.go deleted file mode 100644 index 931322e..0000000 --- a/match/contains_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestContainsIndex(t *testing.T) { - for id, test := range []struct { - prefix string - not bool - fixture string - index int - segments []int - }{ - { - "ab", - false, - "abc", - 0, - []int{2, 3}, - }, - { - "ab", - false, - "fffabfff", - 0, - []int{5, 6, 7, 8}, - }, - { - "ab", - true, - "abc", - 0, - []int{0}, - }, - { - "ab", - true, - "fffabfff", - 0, - []int{0, 1, 2, 3}, - }, - } { - p := NewContains(test.prefix, test.not) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexContains(b *testing.B) { - m := NewContains(string(bench_separators), true) - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexContainsParallel(b *testing.B) { - m := NewContains(string(bench_separators), true) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/debug/debug.go b/match/debug/debug.go deleted file mode 100644 index 5c5dbc1..0000000 --- a/match/debug/debug.go +++ /dev/null @@ -1,55 +0,0 @@ -package debug - -import ( - "bytes" - "fmt" - "github.com/gobwas/glob/match" - "math/rand" -) - -func Graphviz(pattern string, m match.Matcher) string { - return fmt.Sprintf(`digraph G {graph[label="%s"];%s}`, pattern, graphviz_internal(m, fmt.Sprintf("%x", rand.Int63()))) -} - -func graphviz_internal(m match.Matcher, id string) string { - buf := &bytes.Buffer{} - - switch matcher := m.(type) { - case match.BTree: - fmt.Fprintf(buf, `"%s"[label="%s"];`, id, matcher.Value.String()) - for _, m := range []match.Matcher{matcher.Left, matcher.Right} { - switch n := m.(type) { - case nil: - rnd := rand.Int63() - fmt.Fprintf(buf, `"%x"[label=""];`, rnd) - fmt.Fprintf(buf, `"%s"->"%x";`, id, rnd) - - default: - sub := fmt.Sprintf("%x", rand.Int63()) - fmt.Fprintf(buf, `"%s"->"%s";`, id, sub) - fmt.Fprintf(buf, graphviz_internal(n, sub)) - } - } - - case match.AnyOf: - fmt.Fprintf(buf, `"%s"[label="AnyOf"];`, id) - for _, m := range matcher.Matchers { - rnd := rand.Int63() - fmt.Fprintf(buf, graphviz_internal(m, fmt.Sprintf("%x", rnd))) - fmt.Fprintf(buf, `"%s"->"%x";`, id, rnd) - } - - case match.EveryOf: - fmt.Fprintf(buf, `"%s"[label="EveryOf"];`, id) - for _, m := range matcher.Matchers { - rnd := rand.Int63() - fmt.Fprintf(buf, graphviz_internal(m, fmt.Sprintf("%x", rnd))) - fmt.Fprintf(buf, `"%s"->"%x";`, id, rnd) - } - - default: - fmt.Fprintf(buf, `"%s"[label="%s"];`, id, m.String()) - } - - return buf.String() -} diff --git a/match/every_of.go b/match/every_of.go deleted file mode 100644 index 7c968ee..0000000 --- a/match/every_of.go +++ /dev/null @@ -1,99 +0,0 @@ -package match - -import ( - "fmt" -) - -type EveryOf struct { - Matchers Matchers -} - -func NewEveryOf(m ...Matcher) EveryOf { - return EveryOf{Matchers(m)} -} - -func (self *EveryOf) Add(m Matcher) error { - self.Matchers = append(self.Matchers, m) - return nil -} - -func (self EveryOf) Len() (l int) { - for _, m := range self.Matchers { - if ml := m.Len(); l > 0 { - l += ml - } else { - return -1 - } - } - - return -} - -func (self EveryOf) Index(s string) (int, []int) { - var index int - var offset int - - // make `in` with cap as len(s), - // cause it is the maximum size of output segments values - next := acquireSegments(len(s)) - current := acquireSegments(len(s)) - - sub := s - for i, m := range self.Matchers { - idx, seg := m.Index(sub) - if idx == -1 { - releaseSegments(next) - releaseSegments(current) - return -1, nil - } - - if i == 0 { - // we use copy here instead of `current = seg` - // cause seg is a slice from reusable buffer `in` - // and it could be overwritten in next iteration - current = append(current, seg...) - } else { - // clear the next - next = next[:0] - - delta := index - (idx + offset) - for _, ex := range current { - for _, n := range seg { - if ex+delta == n { - next = append(next, n) - } - } - } - - if len(next) == 0 { - releaseSegments(next) - releaseSegments(current) - return -1, nil - } - - current = append(current[:0], next...) - } - - index = idx + offset - sub = s[index:] - offset += idx - } - - releaseSegments(next) - - return index, current -} - -func (self EveryOf) Match(s string) bool { - for _, m := range self.Matchers { - if !m.Match(s) { - return false - } - } - - return true -} - -func (self EveryOf) String() string { - return fmt.Sprintf("", self.Matchers) -} diff --git a/match/every_of_test.go b/match/every_of_test.go deleted file mode 100644 index eb83f86..0000000 --- a/match/every_of_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestEveryOfIndex(t *testing.T) { - for id, test := range []struct { - matchers Matchers - fixture string - index int - segments []int - }{ - { - Matchers{ - NewAny(nil), - NewText("b"), - NewText("c"), - }, - "dbc", - -1, - nil, - }, - { - Matchers{ - NewAny(nil), - NewPrefix("b"), - NewSuffix("c"), - }, - "abc", - 1, - []int{2}, - }, - } { - everyOf := NewEveryOf(test.matchers...) - index, segments := everyOf.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} diff --git a/match/list.go b/match/list.go deleted file mode 100644 index 7fd763e..0000000 --- a/match/list.go +++ /dev/null @@ -1,49 +0,0 @@ -package match - -import ( - "fmt" - "github.com/gobwas/glob/util/runes" - "unicode/utf8" -) - -type List struct { - List []rune - Not bool -} - -func NewList(list []rune, not bool) List { - return List{list, not} -} - -func (self List) Match(s string) bool { - r, w := utf8.DecodeRuneInString(s) - if len(s) > w { - return false - } - - inList := runes.IndexRune(self.List, r) != -1 - return inList == !self.Not -} - -func (self List) Len() int { - return lenOne -} - -func (self List) Index(s string) (int, []int) { - for i, r := range s { - if self.Not == (runes.IndexRune(self.List, r) == -1) { - return i, segmentsByRuneLength[utf8.RuneLen(r)] - } - } - - return -1, nil -} - -func (self List) String() string { - var not string - if self.Not { - not = "!" - } - - return fmt.Sprintf("", not, string(self.List)) -} diff --git a/match/list_test.go b/match/list_test.go deleted file mode 100644 index 10a5437..0000000 --- a/match/list_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestListIndex(t *testing.T) { - for id, test := range []struct { - list []rune - not bool - fixture string - index int - segments []int - }{ - { - []rune("ab"), - false, - "abc", - 0, - []int{1}, - }, - { - []rune("ab"), - true, - "fffabfff", - 0, - []int{1}, - }, - } { - p := NewList(test.list, test.not) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexList(b *testing.B) { - m := NewList([]rune("def"), false) - - for i := 0; i < b.N; i++ { - m.Index(bench_pattern) - } -} - -func BenchmarkIndexListParallel(b *testing.B) { - m := NewList([]rune("def"), false) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - m.Index(bench_pattern) - } - }) -} diff --git a/match/match.go b/match/match.go deleted file mode 100644 index f80e007..0000000 --- a/match/match.go +++ /dev/null @@ -1,81 +0,0 @@ -package match - -// todo common table of rune's length - -import ( - "fmt" - "strings" -) - -const lenOne = 1 -const lenZero = 0 -const lenNo = -1 - -type Matcher interface { - Match(string) bool - Index(string) (int, []int) - Len() int - String() string -} - -type Matchers []Matcher - -func (m Matchers) String() string { - var s []string - for _, matcher := range m { - s = append(s, fmt.Sprint(matcher)) - } - - return fmt.Sprintf("%s", strings.Join(s, ",")) -} - -// appendMerge merges and sorts given already SORTED and UNIQUE segments. -func appendMerge(target, sub []int) []int { - lt, ls := len(target), len(sub) - out := make([]int, 0, lt+ls) - - for x, y := 0, 0; x < lt || y < ls; { - if x >= lt { - out = append(out, sub[y:]...) - break - } - - if y >= ls { - out = append(out, target[x:]...) - break - } - - xValue := target[x] - yValue := sub[y] - - switch { - - case xValue == yValue: - out = append(out, xValue) - x++ - y++ - - case xValue < yValue: - out = append(out, xValue) - x++ - - case yValue < xValue: - out = append(out, yValue) - y++ - - } - } - - target = append(target[:0], out...) - - return target -} - -func reverseSegments(input []int) { - l := len(input) - m := l / 2 - - for i := 0; i < m; i++ { - input[i], input[l-i-1] = input[l-i-1], input[i] - } -} diff --git a/match/match_test.go b/match/match_test.go deleted file mode 100644 index 4c1b83c..0000000 --- a/match/match_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package match - -import ( - "reflect" - "testing" - "unicode/utf8" -) - -var bench_separators = []rune{'.'} - -const bench_pattern = "abcdefghijklmnopqrstuvwxyz0123456789" - -func TestAppendMerge(t *testing.T) { - for id, test := range []struct { - segments [2][]int - exp []int - }{ - { - [2][]int{ - {0, 6, 7}, - {0, 1, 3}, - }, - []int{0, 1, 3, 6, 7}, - }, - { - [2][]int{ - {0, 1, 3, 6, 7}, - {0, 1, 10}, - }, - []int{0, 1, 3, 6, 7, 10}, - }, - } { - act := appendMerge(test.segments[0], test.segments[1]) - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d merge sort segments unexpected:\nact: %v\nexp:%v", id, act, test.exp) - continue - } - } -} - -func BenchmarkAppendMerge(b *testing.B) { - s1 := []int{0, 1, 3, 6, 7} - s2 := []int{0, 1, 3} - - for i := 0; i < b.N; i++ { - appendMerge(s1, s2) - } -} - -func BenchmarkAppendMergeParallel(b *testing.B) { - s1 := []int{0, 1, 3, 6, 7} - s2 := []int{0, 1, 3} - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - appendMerge(s1, s2) - } - }) -} - -func BenchmarkReverse(b *testing.B) { - for i := 0; i < b.N; i++ { - reverseSegments([]int{1, 2, 3, 4}) - } -} - -func getTable() []int { - table := make([]int, utf8.MaxRune+1) - for i := 0; i <= utf8.MaxRune; i++ { - table[i] = utf8.RuneLen(rune(i)) - } - - return table -} - -var table = getTable() - -const runeToLen = 'q' - -func BenchmarkRuneLenFromTable(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = table[runeToLen] - } -} - -func BenchmarkRuneLenFromUTF8(b *testing.B) { - for i := 0; i < b.N; i++ { - _ = utf8.RuneLen(runeToLen) - } -} diff --git a/match/max.go b/match/max.go deleted file mode 100644 index d72f69e..0000000 --- a/match/max.go +++ /dev/null @@ -1,49 +0,0 @@ -package match - -import ( - "fmt" - "unicode/utf8" -) - -type Max struct { - Limit int -} - -func NewMax(l int) Max { - return Max{l} -} - -func (self Max) Match(s string) bool { - var l int - for range s { - l += 1 - if l > self.Limit { - return false - } - } - - return true -} - -func (self Max) Index(s string) (int, []int) { - segments := acquireSegments(self.Limit + 1) - segments = append(segments, 0) - var count int - for i, r := range s { - count++ - if count > self.Limit { - break - } - segments = append(segments, i+utf8.RuneLen(r)) - } - - return 0, segments -} - -func (self Max) Len() int { - return lenNo -} - -func (self Max) String() string { - return fmt.Sprintf("", self.Limit) -} diff --git a/match/max_test.go b/match/max_test.go deleted file mode 100644 index 2367628..0000000 --- a/match/max_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestMaxIndex(t *testing.T) { - for id, test := range []struct { - limit int - fixture string - index int - segments []int - }{ - { - 3, - "abc", - 0, - []int{0, 1, 2, 3}, - }, - { - 3, - "abcdef", - 0, - []int{0, 1, 2, 3}, - }, - } { - p := NewMax(test.limit) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexMax(b *testing.B) { - m := NewMax(10) - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexMaxParallel(b *testing.B) { - m := NewMax(10) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/min.go b/match/min.go deleted file mode 100644 index db57ac8..0000000 --- a/match/min.go +++ /dev/null @@ -1,57 +0,0 @@ -package match - -import ( - "fmt" - "unicode/utf8" -) - -type Min struct { - Limit int -} - -func NewMin(l int) Min { - return Min{l} -} - -func (self Min) Match(s string) bool { - var l int - for range s { - l += 1 - if l >= self.Limit { - return true - } - } - - return false -} - -func (self Min) Index(s string) (int, []int) { - var count int - - c := len(s) - self.Limit + 1 - if c <= 0 { - return -1, nil - } - - segments := acquireSegments(c) - for i, r := range s { - count++ - if count >= self.Limit { - segments = append(segments, i+utf8.RuneLen(r)) - } - } - - if len(segments) == 0 { - return -1, nil - } - - return 0, segments -} - -func (self Min) Len() int { - return lenNo -} - -func (self Min) String() string { - return fmt.Sprintf("", self.Limit) -} diff --git a/match/min_test.go b/match/min_test.go deleted file mode 100644 index ab854ae..0000000 --- a/match/min_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestMinIndex(t *testing.T) { - for id, test := range []struct { - limit int - fixture string - index int - segments []int - }{ - { - 1, - "abc", - 0, - []int{1, 2, 3}, - }, - { - 3, - "abcd", - 0, - []int{3, 4}, - }, - } { - p := NewMin(test.limit) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexMin(b *testing.B) { - m := NewMin(10) - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexMinParallel(b *testing.B) { - m := NewMin(10) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/nothing.go b/match/nothing.go deleted file mode 100644 index 0d4ecd3..0000000 --- a/match/nothing.go +++ /dev/null @@ -1,27 +0,0 @@ -package match - -import ( - "fmt" -) - -type Nothing struct{} - -func NewNothing() Nothing { - return Nothing{} -} - -func (self Nothing) Match(s string) bool { - return len(s) == 0 -} - -func (self Nothing) Index(s string) (int, []int) { - return 0, segments0 -} - -func (self Nothing) Len() int { - return lenZero -} - -func (self Nothing) String() string { - return fmt.Sprintf("") -} diff --git a/match/nothing_test.go b/match/nothing_test.go deleted file mode 100644 index 941c22d..0000000 --- a/match/nothing_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestNothingIndex(t *testing.T) { - for id, test := range []struct { - fixture string - index int - segments []int - }{ - { - "abc", - 0, - []int{0}, - }, - { - "", - 0, - []int{0}, - }, - } { - p := NewNothing() - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexNothing(b *testing.B) { - m := NewNothing() - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexNothingParallel(b *testing.B) { - m := NewNothing() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/prefix.go b/match/prefix.go deleted file mode 100644 index a734725..0000000 --- a/match/prefix.go +++ /dev/null @@ -1,50 +0,0 @@ -package match - -import ( - "fmt" - "strings" - "unicode/utf8" -) - -type Prefix struct { - Prefix string -} - -func NewPrefix(p string) Prefix { - return Prefix{p} -} - -func (self Prefix) Index(s string) (int, []int) { - idx := strings.Index(s, self.Prefix) - if idx == -1 { - return -1, nil - } - - length := len(self.Prefix) - var sub string - if len(s) > idx+length { - sub = s[idx+length:] - } else { - sub = "" - } - - segments := acquireSegments(len(sub) + 1) - segments = append(segments, length) - for i, r := range sub { - segments = append(segments, length+i+utf8.RuneLen(r)) - } - - return idx, segments -} - -func (self Prefix) Len() int { - return lenNo -} - -func (self Prefix) Match(s string) bool { - return strings.HasPrefix(s, self.Prefix) -} - -func (self Prefix) String() string { - return fmt.Sprintf("", self.Prefix) -} diff --git a/match/prefix_any.go b/match/prefix_any.go deleted file mode 100644 index 8ee58fe..0000000 --- a/match/prefix_any.go +++ /dev/null @@ -1,55 +0,0 @@ -package match - -import ( - "fmt" - "strings" - "unicode/utf8" - - sutil "github.com/gobwas/glob/util/strings" -) - -type PrefixAny struct { - Prefix string - Separators []rune -} - -func NewPrefixAny(s string, sep []rune) PrefixAny { - return PrefixAny{s, sep} -} - -func (self PrefixAny) Index(s string) (int, []int) { - idx := strings.Index(s, self.Prefix) - if idx == -1 { - return -1, nil - } - - n := len(self.Prefix) - sub := s[idx+n:] - i := sutil.IndexAnyRunes(sub, self.Separators) - if i > -1 { - sub = sub[:i] - } - - seg := acquireSegments(len(sub) + 1) - seg = append(seg, n) - for i, r := range sub { - seg = append(seg, n+i+utf8.RuneLen(r)) - } - - return idx, seg -} - -func (self PrefixAny) Len() int { - return lenNo -} - -func (self PrefixAny) Match(s string) bool { - if !strings.HasPrefix(s, self.Prefix) { - return false - } - return sutil.IndexAnyRunes(s[len(self.Prefix):], self.Separators) == -1 -} - -func (self PrefixAny) String() string { - return fmt.Sprintf("", self.Prefix, string(self.Separators)) -} diff --git a/match/prefix_any_test.go b/match/prefix_any_test.go deleted file mode 100644 index e6990e3..0000000 --- a/match/prefix_any_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestPrefixAnyIndex(t *testing.T) { - for id, test := range []struct { - prefix string - separators []rune - fixture string - index int - segments []int - }{ - { - "ab", - []rune{'.'}, - "ab", - 0, - []int{2}, - }, - { - "ab", - []rune{'.'}, - "abc", - 0, - []int{2, 3}, - }, - { - "ab", - []rune{'.'}, - "qw.abcd.efg", - 3, - []int{2, 3, 4}, - }, - } { - p := NewPrefixAny(test.prefix, test.separators) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} diff --git a/match/prefix_suffix.go b/match/prefix_suffix.go deleted file mode 100644 index 8208085..0000000 --- a/match/prefix_suffix.go +++ /dev/null @@ -1,62 +0,0 @@ -package match - -import ( - "fmt" - "strings" -) - -type PrefixSuffix struct { - Prefix, Suffix string -} - -func NewPrefixSuffix(p, s string) PrefixSuffix { - return PrefixSuffix{p, s} -} - -func (self PrefixSuffix) Index(s string) (int, []int) { - prefixIdx := strings.Index(s, self.Prefix) - if prefixIdx == -1 { - return -1, nil - } - - suffixLen := len(self.Suffix) - if suffixLen <= 0 { - return prefixIdx, []int{len(s) - prefixIdx} - } - - if (len(s) - prefixIdx) <= 0 { - return -1, nil - } - - segments := acquireSegments(len(s) - prefixIdx) - for sub := s[prefixIdx:]; ; { - suffixIdx := strings.LastIndex(sub, self.Suffix) - if suffixIdx == -1 { - break - } - - segments = append(segments, suffixIdx+suffixLen) - sub = sub[:suffixIdx] - } - - if len(segments) == 0 { - releaseSegments(segments) - return -1, nil - } - - reverseSegments(segments) - - return prefixIdx, segments -} - -func (self PrefixSuffix) Len() int { - return lenNo -} - -func (self PrefixSuffix) Match(s string) bool { - return strings.HasPrefix(s, self.Prefix) && strings.HasSuffix(s, self.Suffix) -} - -func (self PrefixSuffix) String() string { - return fmt.Sprintf("", self.Prefix, self.Suffix) -} diff --git a/match/prefix_suffix_test.go b/match/prefix_suffix_test.go deleted file mode 100644 index 79b17b2..0000000 --- a/match/prefix_suffix_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestPrefixSuffixIndex(t *testing.T) { - for id, test := range []struct { - prefix string - suffix string - fixture string - index int - segments []int - }{ - { - "a", - "c", - "abc", - 0, - []int{3}, - }, - { - "f", - "f", - "fffabfff", - 0, - []int{1, 2, 3, 6, 7, 8}, - }, - { - "ab", - "bc", - "abc", - 0, - []int{3}, - }, - } { - p := NewPrefixSuffix(test.prefix, test.suffix) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexPrefixSuffix(b *testing.B) { - m := NewPrefixSuffix("qew", "sqw") - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexPrefixSuffixParallel(b *testing.B) { - m := NewPrefixSuffix("qew", "sqw") - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/prefix_test.go b/match/prefix_test.go deleted file mode 100644 index 22a296e..0000000 --- a/match/prefix_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestPrefixIndex(t *testing.T) { - for id, test := range []struct { - prefix string - fixture string - index int - segments []int - }{ - { - "ab", - "abc", - 0, - []int{2, 3}, - }, - { - "ab", - "fffabfff", - 3, - []int{2, 3, 4, 5}, - }, - } { - p := NewPrefix(test.prefix) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexPrefix(b *testing.B) { - m := NewPrefix("qew") - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexPrefixParallel(b *testing.B) { - m := NewPrefix("qew") - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/range.go b/match/range.go deleted file mode 100644 index ce30245..0000000 --- a/match/range.go +++ /dev/null @@ -1,48 +0,0 @@ -package match - -import ( - "fmt" - "unicode/utf8" -) - -type Range struct { - Lo, Hi rune - Not bool -} - -func NewRange(lo, hi rune, not bool) Range { - return Range{lo, hi, not} -} - -func (self Range) Len() int { - return lenOne -} - -func (self Range) Match(s string) bool { - r, w := utf8.DecodeRuneInString(s) - if len(s) > w { - return false - } - - inRange := r >= self.Lo && r <= self.Hi - - return inRange == !self.Not -} - -func (self Range) Index(s string) (int, []int) { - for i, r := range s { - if self.Not != (r >= self.Lo && r <= self.Hi) { - return i, segmentsByRuneLength[utf8.RuneLen(r)] - } - } - - return -1, nil -} - -func (self Range) String() string { - var not string - if self.Not { - not = "!" - } - return fmt.Sprintf("", not, string(self.Lo), string(self.Hi)) -} diff --git a/match/range_test.go b/match/range_test.go deleted file mode 100644 index 0dddcfd..0000000 --- a/match/range_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestRangeIndex(t *testing.T) { - for id, test := range []struct { - lo, hi rune - not bool - fixture string - index int - segments []int - }{ - { - 'a', 'z', - false, - "abc", - 0, - []int{1}, - }, - { - 'a', 'c', - false, - "abcd", - 0, - []int{1}, - }, - { - 'a', 'c', - true, - "abcd", - 3, - []int{1}, - }, - } { - m := NewRange(test.lo, test.hi, test.not) - index, segments := m.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexRange(b *testing.B) { - m := NewRange('0', '9', false) - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexRangeParallel(b *testing.B) { - m := NewRange('0', '9', false) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/row.go b/match/row.go deleted file mode 100644 index 4379042..0000000 --- a/match/row.go +++ /dev/null @@ -1,77 +0,0 @@ -package match - -import ( - "fmt" -) - -type Row struct { - Matchers Matchers - RunesLength int - Segments []int -} - -func NewRow(len int, m ...Matcher) Row { - return Row{ - Matchers: Matchers(m), - RunesLength: len, - Segments: []int{len}, - } -} - -func (self Row) matchAll(s string) bool { - var idx int - for _, m := range self.Matchers { - length := m.Len() - - var next, i int - for next = range s[idx:] { - i++ - if i == length { - break - } - } - - if i < length || !m.Match(s[idx:idx+next+1]) { - return false - } - - idx += next + 1 - } - - return true -} - -func (self Row) lenOk(s string) bool { - var i int - for range s { - i++ - if i > self.RunesLength { - return false - } - } - return self.RunesLength == i -} - -func (self Row) Match(s string) bool { - return self.lenOk(s) && self.matchAll(s) -} - -func (self Row) Len() (l int) { - return self.RunesLength -} - -func (self Row) Index(s string) (int, []int) { - for i := range s { - if len(s[i:]) < self.RunesLength { - break - } - if self.matchAll(s[i:]) { - return i, self.Segments - } - } - return -1, nil -} - -func (self Row) String() string { - return fmt.Sprintf("", self.RunesLength, self.Matchers) -} diff --git a/match/row_test.go b/match/row_test.go deleted file mode 100644 index c9e65ef..0000000 --- a/match/row_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestRowIndex(t *testing.T) { - for id, test := range []struct { - matchers Matchers - length int - fixture string - index int - segments []int - }{ - { - Matchers{ - NewText("abc"), - NewText("def"), - NewSingle(nil), - }, - 7, - "qweabcdefghij", - 3, - []int{7}, - }, - { - Matchers{ - NewText("abc"), - NewText("def"), - NewSingle(nil), - }, - 7, - "abcd", - -1, - nil, - }, - } { - p := NewRow(test.length, test.matchers...) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkRowIndex(b *testing.B) { - m := NewRow( - 7, - Matchers{ - NewText("abc"), - NewText("def"), - NewSingle(nil), - }..., - ) - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexRowParallel(b *testing.B) { - m := NewRow( - 7, - Matchers{ - NewText("abc"), - NewText("def"), - NewSingle(nil), - }..., - ) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/segments.go b/match/segments.go deleted file mode 100644 index 9ea6f30..0000000 --- a/match/segments.go +++ /dev/null @@ -1,91 +0,0 @@ -package match - -import ( - "sync" -) - -type SomePool interface { - Get() []int - Put([]int) -} - -var segmentsPools [1024]sync.Pool - -func toPowerOfTwo(v int) int { - v-- - v |= v >> 1 - v |= v >> 2 - v |= v >> 4 - v |= v >> 8 - v |= v >> 16 - v++ - - return v -} - -const ( - cacheFrom = 16 - cacheToAndHigher = 1024 - cacheFromIndex = 15 - cacheToAndHigherIndex = 1023 -) - -var ( - segments0 = []int{0} - segments1 = []int{1} - segments2 = []int{2} - segments3 = []int{3} - segments4 = []int{4} -) - -var segmentsByRuneLength [5][]int = [5][]int{ - 0: segments0, - 1: segments1, - 2: segments2, - 3: segments3, - 4: segments4, -} - -func init() { - for i := cacheToAndHigher; i >= cacheFrom; i >>= 1 { - func(i int) { - segmentsPools[i-1] = sync.Pool{New: func() interface{} { - return make([]int, 0, i) - }} - }(i) - } -} - -func getTableIndex(c int) int { - p := toPowerOfTwo(c) - switch { - case p >= cacheToAndHigher: - return cacheToAndHigherIndex - case p <= cacheFrom: - return cacheFromIndex - default: - return p - 1 - } -} - -func acquireSegments(c int) []int { - // make []int with less capacity than cacheFrom - // is faster than acquiring it from pool - if c < cacheFrom { - return make([]int, 0, c) - } - - return segmentsPools[getTableIndex(c)].Get().([]int)[:0] -} - -func releaseSegments(s []int) { - c := cap(s) - - // make []int with less capacity than cacheFrom - // is faster than acquiring it from pool - if c < cacheFrom { - return - } - - segmentsPools[getTableIndex(c)].Put(s) -} diff --git a/match/segments_test.go b/match/segments_test.go deleted file mode 100644 index 1ce1123..0000000 --- a/match/segments_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package match - -import ( - "sync" - "testing" -) - -func benchPool(i int, b *testing.B) { - pool := sync.Pool{New: func() interface{} { - return make([]int, 0, i) - }} - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - s := pool.Get().([]int)[:0] - pool.Put(s) - } - }) -} - -func benchMake(i int, b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _ = make([]int, 0, i) - } - }) -} - -func BenchmarkSegmentsPool_1(b *testing.B) { - benchPool(1, b) -} -func BenchmarkSegmentsPool_2(b *testing.B) { - benchPool(2, b) -} -func BenchmarkSegmentsPool_4(b *testing.B) { - benchPool(4, b) -} -func BenchmarkSegmentsPool_8(b *testing.B) { - benchPool(8, b) -} -func BenchmarkSegmentsPool_16(b *testing.B) { - benchPool(16, b) -} -func BenchmarkSegmentsPool_32(b *testing.B) { - benchPool(32, b) -} -func BenchmarkSegmentsPool_64(b *testing.B) { - benchPool(64, b) -} -func BenchmarkSegmentsPool_128(b *testing.B) { - benchPool(128, b) -} -func BenchmarkSegmentsPool_256(b *testing.B) { - benchPool(256, b) -} - -func BenchmarkSegmentsMake_1(b *testing.B) { - benchMake(1, b) -} -func BenchmarkSegmentsMake_2(b *testing.B) { - benchMake(2, b) -} -func BenchmarkSegmentsMake_4(b *testing.B) { - benchMake(4, b) -} -func BenchmarkSegmentsMake_8(b *testing.B) { - benchMake(8, b) -} -func BenchmarkSegmentsMake_16(b *testing.B) { - benchMake(16, b) -} -func BenchmarkSegmentsMake_32(b *testing.B) { - benchMake(32, b) -} -func BenchmarkSegmentsMake_64(b *testing.B) { - benchMake(64, b) -} -func BenchmarkSegmentsMake_128(b *testing.B) { - benchMake(128, b) -} -func BenchmarkSegmentsMake_256(b *testing.B) { - benchMake(256, b) -} diff --git a/match/single.go b/match/single.go deleted file mode 100644 index ee6e395..0000000 --- a/match/single.go +++ /dev/null @@ -1,43 +0,0 @@ -package match - -import ( - "fmt" - "github.com/gobwas/glob/util/runes" - "unicode/utf8" -) - -// single represents ? -type Single struct { - Separators []rune -} - -func NewSingle(s []rune) Single { - return Single{s} -} - -func (self Single) Match(s string) bool { - r, w := utf8.DecodeRuneInString(s) - if len(s) > w { - return false - } - - return runes.IndexRune(self.Separators, r) == -1 -} - -func (self Single) Len() int { - return lenOne -} - -func (self Single) Index(s string) (int, []int) { - for i, r := range s { - if runes.IndexRune(self.Separators, r) == -1 { - return i, segmentsByRuneLength[utf8.RuneLen(r)] - } - } - - return -1, nil -} - -func (self Single) String() string { - return fmt.Sprintf("", string(self.Separators)) -} diff --git a/match/single_test.go b/match/single_test.go deleted file mode 100644 index a62d720..0000000 --- a/match/single_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestSingleIndex(t *testing.T) { - for id, test := range []struct { - separators []rune - fixture string - index int - segments []int - }{ - { - []rune{'.'}, - ".abc", - 1, - []int{1}, - }, - { - []rune{'.'}, - ".", - -1, - nil, - }, - } { - p := NewSingle(test.separators) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexSingle(b *testing.B) { - m := NewSingle(bench_separators) - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexSingleParallel(b *testing.B) { - m := NewSingle(bench_separators) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/suffix.go b/match/suffix.go deleted file mode 100644 index 85bea8c..0000000 --- a/match/suffix.go +++ /dev/null @@ -1,35 +0,0 @@ -package match - -import ( - "fmt" - "strings" -) - -type Suffix struct { - Suffix string -} - -func NewSuffix(s string) Suffix { - return Suffix{s} -} - -func (self Suffix) Len() int { - return lenNo -} - -func (self Suffix) Match(s string) bool { - return strings.HasSuffix(s, self.Suffix) -} - -func (self Suffix) Index(s string) (int, []int) { - idx := strings.Index(s, self.Suffix) - if idx == -1 { - return -1, nil - } - - return 0, []int{idx + len(self.Suffix)} -} - -func (self Suffix) String() string { - return fmt.Sprintf("", self.Suffix) -} diff --git a/match/suffix_any.go b/match/suffix_any.go deleted file mode 100644 index c5106f8..0000000 --- a/match/suffix_any.go +++ /dev/null @@ -1,43 +0,0 @@ -package match - -import ( - "fmt" - "strings" - - sutil "github.com/gobwas/glob/util/strings" -) - -type SuffixAny struct { - Suffix string - Separators []rune -} - -func NewSuffixAny(s string, sep []rune) SuffixAny { - return SuffixAny{s, sep} -} - -func (self SuffixAny) Index(s string) (int, []int) { - idx := strings.Index(s, self.Suffix) - if idx == -1 { - return -1, nil - } - - i := sutil.LastIndexAnyRunes(s[:idx], self.Separators) + 1 - - return i, []int{idx + len(self.Suffix) - i} -} - -func (self SuffixAny) Len() int { - return lenNo -} - -func (self SuffixAny) Match(s string) bool { - if !strings.HasSuffix(s, self.Suffix) { - return false - } - return sutil.IndexAnyRunes(s[:len(s)-len(self.Suffix)], self.Separators) == -1 -} - -func (self SuffixAny) String() string { - return fmt.Sprintf("", string(self.Separators), self.Suffix) -} diff --git a/match/suffix_any_test.go b/match/suffix_any_test.go deleted file mode 100644 index eed6e59..0000000 --- a/match/suffix_any_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestSuffixAnyIndex(t *testing.T) { - for id, test := range []struct { - suffix string - separators []rune - fixture string - index int - segments []int - }{ - { - "ab", - []rune{'.'}, - "ab", - 0, - []int{2}, - }, - { - "ab", - []rune{'.'}, - "cab", - 0, - []int{3}, - }, - { - "ab", - []rune{'.'}, - "qw.cdab.efg", - 3, - []int{4}, - }, - } { - p := NewSuffixAny(test.suffix, test.separators) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} diff --git a/match/suffix_test.go b/match/suffix_test.go deleted file mode 100644 index 4904763..0000000 --- a/match/suffix_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestSuffixIndex(t *testing.T) { - for id, test := range []struct { - prefix string - fixture string - index int - segments []int - }{ - { - "ab", - "abc", - 0, - []int{2}, - }, - { - "ab", - "fffabfff", - 0, - []int{5}, - }, - } { - p := NewSuffix(test.prefix) - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexSuffix(b *testing.B) { - m := NewSuffix("qwe") - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexSuffixParallel(b *testing.B) { - m := NewSuffix("qwe") - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/super.go b/match/super.go deleted file mode 100644 index 3875950..0000000 --- a/match/super.go +++ /dev/null @@ -1,33 +0,0 @@ -package match - -import ( - "fmt" -) - -type Super struct{} - -func NewSuper() Super { - return Super{} -} - -func (self Super) Match(s string) bool { - return true -} - -func (self Super) Len() int { - return lenNo -} - -func (self Super) Index(s string) (int, []int) { - segments := acquireSegments(len(s) + 1) - for i := range s { - segments = append(segments, i) - } - segments = append(segments, len(s)) - - return 0, segments -} - -func (self Super) String() string { - return fmt.Sprintf("") -} diff --git a/match/super_test.go b/match/super_test.go deleted file mode 100644 index 10418dc..0000000 --- a/match/super_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestSuperIndex(t *testing.T) { - for id, test := range []struct { - fixture string - index int - segments []int - }{ - { - "abc", - 0, - []int{0, 1, 2, 3}, - }, - { - "", - 0, - []int{0}, - }, - } { - p := NewSuper() - index, segments := p.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexSuper(b *testing.B) { - m := NewSuper() - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexSuperParallel(b *testing.B) { - m := NewSuper() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/match/text.go b/match/text.go deleted file mode 100644 index 0a17616..0000000 --- a/match/text.go +++ /dev/null @@ -1,45 +0,0 @@ -package match - -import ( - "fmt" - "strings" - "unicode/utf8" -) - -// raw represents raw string to match -type Text struct { - Str string - RunesLength int - BytesLength int - Segments []int -} - -func NewText(s string) Text { - return Text{ - Str: s, - RunesLength: utf8.RuneCountInString(s), - BytesLength: len(s), - Segments: []int{len(s)}, - } -} - -func (self Text) Match(s string) bool { - return self.Str == s -} - -func (self Text) Len() int { - return self.RunesLength -} - -func (self Text) Index(s string) (int, []int) { - index := strings.Index(s, self.Str) - if index == -1 { - return -1, nil - } - - return index, self.Segments -} - -func (self Text) String() string { - return fmt.Sprintf("", self.Str) -} diff --git a/match/text_test.go b/match/text_test.go deleted file mode 100644 index a3de40e..0000000 --- a/match/text_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package match - -import ( - "reflect" - "testing" -) - -func TestTextIndex(t *testing.T) { - for id, test := range []struct { - text string - fixture string - index int - segments []int - }{ - { - "b", - "abc", - 1, - []int{1}, - }, - { - "f", - "abcd", - -1, - nil, - }, - } { - m := NewText(test.text) - index, segments := m.Index(test.fixture) - if index != test.index { - t.Errorf("#%d unexpected index: exp: %d, act: %d", id, test.index, index) - } - if !reflect.DeepEqual(segments, test.segments) { - t.Errorf("#%d unexpected segments: exp: %v, act: %v", id, test.segments, segments) - } - } -} - -func BenchmarkIndexText(b *testing.B) { - m := NewText("foo") - - for i := 0; i < b.N; i++ { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } -} - -func BenchmarkIndexTextParallel(b *testing.B) { - m := NewText("foo") - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, s := m.Index(bench_pattern) - releaseSegments(s) - } - }) -} diff --git a/parse.go b/parse.go new file mode 100644 index 0000000..30114df --- /dev/null +++ b/parse.go @@ -0,0 +1,679 @@ +package glob + +import ( + "slices" + "unicode/utf8" + + "github.com/gobwas/glob/internal/debug" + "github.com/gobwas/glob/syntax" +) + +// compile parses the pattern into a matcher tree (see below), simplifies and +// specializes it, and computes the match-time hints and preconditions; see +// [simplify], [specialize], [annotateStars], [needsState], [minLength] and +// [requiredSuffix]. It is what [Compile] wraps. +func compile(str string, sep []rune) (*Pattern, error) { + if debug.Enabled { + debug.Printf("compiling %#q\n", str) + } + // The matchers keep sep and read it while matching, and the variadic slice + // may alias an array owned by the caller: give them a copy of their own. + // + // The pattern itself keeps the slice as given, to return it from + // Separators() without cloning. + var ( + sepCopy = slices.Clone(sep) + sepStr = string(sep) + ) + + type operator struct { + kind int + index int + } + const ( + opTerms = iota + opList + ) + /* + Stack-based parsing is a technique used to evaluate mathematical + expressions by leveraging the properties of the LIFO (Last-In, + First-Out) data structure, the stack. It involves using two stacks: one + for operands (numbers) and one for operators. By processing the + expression from left to right and strategically pushing and popping + elements from the stacks, the expression can be effectively evaluated. + + https://cp-algorithms.com/string/expression_parsing.html + + Here the operands are matchers and the only operators are the braces + and the commas inside them, so it goes as follows: + + - a leaf token (text, `?`, `*`, `**`, `[...]`) pushes its matcher + onto the stack; + + - `{` pushes two operators, both remembering the current stack + length: opTerms marks where the alternatives of the group will + be collected, opList marks where the terms of the current + alternative begin; + + - `,` pops the opList, collapses the terms above its index into a + single multiMatcher (or a voidMatcher when there are none, as in + `{,a}`), and pushes a fresh opList for the next alternative; + + - `}` pops the opList and collapses the last alternative the same + way, then pops the opTerms and collapses everything above its + index -- one matcher per alternative by now -- into an + altMatcher; + + - at the EOF whatever is left on the stack is the top-level + sequence; a leftover operator means an unclosed `{`. + + For example, `a{b*,c}d` goes like this (list@i is an opList with + index i, likewise terms@i): + + token stack operators + a "a" + { "a" terms@1 list@1 + b "a" "b" terms@1 list@1 + * "a" "b" * terms@1 list@1 + , "a" ["b"·*] terms@1 list@2 + c "a" ["b"·*] "c" terms@1 list@2 + } "a" ["b"·*] ["c"] terms@1 + "a" {["b"·*]|["c"]} + d "a" {["b"·*]|["c"]} "d" + EOF ["a"·{["b"·*]|["c"]}·"d"] + + The result is then simplified (["c"] becomes "c") and specialized; + see [simplify] and [specialize]. + */ + var ( + stack []matcher + operators []operator + ) + lex := syntax.NewLexer(str) +parsing: + for { + token := lex.Next() + if debug.Enabled { + debug.Printf("token: %s\n", token) + } + switch token.Type { + case syntax.EOF: + break parsing + + case syntax.Error: + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: token.Data, + } + + case syntax.Single: + stack = append(stack, &charMatcher{ + Sep: sepCopy, + }) + + case syntax.Text: + stack = append(stack, &textMatcher{ + Text: token.Data, + }) + + case syntax.RangeOpen: + m, err := parseRange(lex) + if err != nil { + return nil, err + } + stack = append(stack, m) + + case syntax.Any: + stack = append(stack, &starMatcher{ + Sep: sepCopy, + SepStr: sepStr, + }) + + case syntax.Super: + stack = append(stack, &starMatcher{ + Sep: nil, + }) + + case syntax.TermsOpen: + // Note that the `{` opens both the group and its first + // alternative: every alternative is delimited by an opList + // operator. This way TermsClose always collapses the trailing + // alternative into a single matcher first, even when the group + // has no commas at all, e.g. `{ab*}`. + operators = append(operators, + operator{kind: opTerms, index: len(stack)}, + operator{kind: opList, index: len(stack)}, + ) + if debug.Enabled { + debug.Printf("terms enter: %d\n", len(stack)) + } + + case syntax.TermSeparator: + k := len(operators) - 1 + if k < 0 { + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "unexpected `,`", + } + } + x := operators[k] + if x.kind == opList { + // Remove the most recent "comma" operator. + // Note that the previous one is the terms operator. + operators = operators[:k] + } + i := x.index + // Handle the `{,a}` case. + if i == len(stack) { + // Empty matchers. + stack = append(stack, &voidMatcher{}) + } else { + stack[i] = multiMatcher(slices.Clone(stack[i:])) + stack = stack[:i+1] + if debug.Enabled { + debug.Printf("terms next: %d: %s\n", i, stack[i]) + } + } + operators = append(operators, operator{ + kind: opList, + index: len(stack), + }) + if debug.Enabled { + debug.Printf("terms separator: %d\n", len(stack)) + } + + case syntax.TermsClose: + for { + k := len(operators) - 1 + if k < 0 { + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "unexpected `}`", + } + } + x := operators[k] + operators = operators[:k] + + i := x.index + c := slices.Clone(stack[i:]) + var m matcher + switch x.kind { + case opTerms: + m = altMatcher(c) + case opList: + m = multiMatcher(c) + } + // Handle the `{a,}` case. + if i == len(stack) { + stack = append(stack, m) + } else { + stack = stack[:i+1] + stack[i] = m + } + + if debug.Enabled { + debug.Printf( + "terms leave(%d): %d: %s\n", + x.kind, i, stack[i], + ) + } + if x.kind == opTerms { + break + } + } + + default: + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "unexpected token " + token.String(), + } + } + } + if len(operators) != 0 { + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "unclosed `{`", + } + } + m := simplify(multiMatcher(stack)) + m = specialize(m, true) + annotateStars(m, true) + if debug.Enabled { + debug.Printf("compiled %#q: %s\n", str, m) + } + p := &Pattern{ + str: str, + sep: sep, + m: m, + state: needsState(m), + } + if p.state { + p.minLen = minLength(m) + p.suffix = requiredSuffix(m) + } + return p, nil +} + +// parseRange parses a character class, called right after its opening `[` +// was read; it consumes the tokens up to and including the closing `]`. The +// class is either a range, `[a-c]`, or a set, `[abc]`, either possibly +// negated with a leading `!`; see [runeRangeMatcher] and [runeSetMatcher]. +func parseRange(lex *syntax.Lexer) (matcher, error) { + // -1 marks a range boundary as unset: any decoded rune, including + // U+0000, is non-negative. + var ( + not bool + lo, hi rune = -1, -1 + chars map[rune]struct{} + ) + for { + token := lex.Next() + switch token.Type { + case syntax.EOF: + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "unclosed `[`", + } + + case syntax.Error: + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: token.Data, + } + + case syntax.Not: + not = true + + case syntax.RangeLo: + r, w := utf8.DecodeRuneInString(token.Data) + if len(token.Data) > w { + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "unexpected length of range lo character", + } + } + lo = r + + case syntax.RangeBetween: + // The `-` between lo and hi: nothing to do. + + case syntax.RangeHi: + r, w := utf8.DecodeRuneInString(token.Data) + if len(token.Data) > w { + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "unexpected length of range hi character", + } + } + hi = r + + if hi < lo { + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "range hi character is less than lo", + } + } + + case syntax.Text: + chars = make(map[rune]struct{}) + for _, r := range token.Data { + chars[r] = struct{}{} + } + + case syntax.RangeClose: + isRange := lo >= 0 && hi >= 0 + isChars := chars != nil + + if isChars == isRange { + return nil, &SyntaxError{ + Offset: lex.Offset(), + Reason: "could not parse range", + } + } + if isRange { + return &runeRangeMatcher{ + Lo: lo, + Hi: hi, + Not: not, + }, nil + } + return &runeSetMatcher{ + Set: chars, + Not: not, + }, nil + } + } +} + +// simplify rewrites the freshly parsed tree into its canonical shape, bottom +// up: the sequences are normalized (see [normalizeSequence]), and a sequence +// or a group of alternatives with a single child is replaced by the child, +// with none -- by a void. +func simplify(m matcher) matcher { + var ( + ms []matcher + isMulti bool + ) + switch v := m.(type) { + case multiMatcher: + ms, isMulti = v, true + case altMatcher: + ms = v + default: + return m + } + for i, m := range ms { + ms[i] = simplify(m) + } + if isMulti { + ms = normalizeSequence(ms) + } + switch len(ms) { + case 0: + return &voidMatcher{} + case 1: + return ms[0] + } + if isMulti { + return multiMatcher(ms) + } + return altMatcher(ms) +} + +// normalizeSequence rewrites a sequence of (already simplified) matchers +// into a simpler equivalent one: +// +// ["a"·["b"·"c"]·"d"] => ["a"·"b"·"c"·"d"] inline the nested sequences +// ["a"·void] => ["a"] drop the void matchers +// ["a"·"b"] => ["ab"] merge the adjacent literals +// [*·**] => [**] coalesce the adjacent stars +// +// Longer literals also make better star jumps; see [annotateStars]. +func normalizeSequence(ms []matcher) []matcher { + if !needsNormalize(ms) { + // The common case: nothing to rewrite, no copy needed. + return ms + } + out := make([]matcher, 0, len(ms)) + var push func(m matcher) + push = func(m matcher) { + switch v := m.(type) { + case multiMatcher: + for _, c := range v { + push(c) + } + return + case *voidMatcher: + return + case *textMatcher: + if len(out) > 0 { + if prev, ok := out[len(out)-1].(*textMatcher); ok { + out[len(out)-1] = &textMatcher{Text: prev.Text + v.Text} + return + } + } + case *starMatcher: + if len(out) > 0 { + if prev, ok := out[len(out)-1].(*starMatcher); ok { + // Adjacent stars are equivalent to the most general + // of them: the one not limited by separators, if any. + if len(prev.Sep) > 0 && len(v.Sep) == 0 { + out[len(out)-1] = v + } + return + } + } + } + out = append(out, m) + } + for _, m := range ms { + push(m) + } + return out +} + +// needsNormalize reports whether [normalizeSequence] would change ms, so +// that the common case skips the copy. +func needsNormalize(ms []matcher) bool { + for i, m := range ms { + switch m.(type) { + case multiMatcher, *voidMatcher: + return true + case *textMatcher: + if i > 0 { + if _, ok := ms[i-1].(*textMatcher); ok { + return true + } + } + case *starMatcher: + if i > 0 { + if _, ok := ms[i-1].(*starMatcher); ok { + return true + } + } + } + } + return false +} + +// specialize rewrites the terminal sub-sequences of the simplified matcher +// tree into the shaped matchers -- [prefixMatcher], [suffixMatcher], +// [prefixSuffixMatcher] and [containsMatcher]; see [foldTail] for the +// rewrites. The tail flag tells whether nothing follows m in the pattern; +// only there the rewrites apply, since a shaped matcher consumes the whole +// remainder of the input. +func specialize(m matcher, tail bool) matcher { + switch v := m.(type) { + case altMatcher: + // Every alternative ends where the alt ends. + for i, c := range v { + v[i] = specialize(c, tail) + } + return v + + case multiMatcher: + for i, c := range v { + v[i] = specialize(c, tail && i == len(v)-1) + } + if !tail { + return v + } + ms := foldTail([]matcher(v)) + if len(ms) == 1 { + return ms[0] + } + return multiMatcher(ms) + } + return m +} + +// foldTail repeatedly folds the two trailing matchers of the terminal +// sequence ms into a shaped one, while possible: +// +// [..·"abc"·*] => [..·prefix("abc")] +// [..·*·"abc"] => [..·suffix("abc")] +// [..·"abc"·prefix("def")] => [..·prefix("abcdef")] +// [..·*·prefix("abc")] => [..·contains("abc")] (separator-free) +// [..·"abc"·suffix("def")] => [..·prefix_suffix("abc","def")] +// [..·*·contains("abc")] => [..·contains("abc")] (separator-free) +func foldTail(ms []matcher) []matcher { + for len(ms) >= 2 { + var ( + prev = ms[len(ms)-2] + folded matcher + ) + switch last := ms[len(ms)-1].(type) { + case *starMatcher: + if t, ok := prev.(*textMatcher); ok { + folded = &prefixMatcher{Text: t.Text, Sep: last.SepStr} + } + + case *textMatcher: + if star, ok := prev.(*starMatcher); ok { + folded = &suffixMatcher{Text: last.Text, Sep: star.SepStr} + } + + case *prefixMatcher: + switch p := prev.(type) { + case *textMatcher: + folded = &prefixMatcher{Text: p.Text + last.Text, Sep: last.Sep} + case *starMatcher: + if p.SepStr == "" && last.Sep == "" { + folded = &containsMatcher{Text: last.Text} + } + } + + case *suffixMatcher: + if t, ok := prev.(*textMatcher); ok { + folded = &prefixSuffixMatcher{ + Prefix: t.Text, + Suffix: last.Text, + Sep: last.Sep, + } + } + + case *containsMatcher: + if star, ok := prev.(*starMatcher); ok && star.SepStr == "" { + folded = last + } + } + if folded == nil { + break + } + ms = ms[:len(ms)-1] + ms[len(ms)-1] = folded + } + return ms +} + +// annotateStars computes the compile-time hints for the star matchers, in +// order to keep the number of restart points they store at match time low: +// +// - a star directly followed by a literal jumps between the literal +// occurrences instead of retrying at every rune (see +// [starMatcher.storeSkip]); +// +// - a star with nothing after it anywhere in the pattern (tail is true +// for m and the star closes it) consumes its whole reach at once and +// stores no restart points at all. +func annotateStars(m matcher, tail bool) { + switch v := m.(type) { + case multiMatcher: + for i, c := range v { + last := i == len(v)-1 + star, ok := c.(*starMatcher) + if !ok { + annotateStars(c, tail && last) + continue + } + star.Terminal = tail && last + if !last { + star.Next = leadingLiteral(v[i+1]) + } + } + case altMatcher: + for _, c := range v { + annotateStars(c, tail) + } + case *starMatcher: + v.Terminal = tail + } +} + +// leadingLiteral returns the literal the given matcher is guaranteed to +// begin its match with, if any. +func leadingLiteral(m matcher) string { + switch v := m.(type) { + case *textMatcher: + return v.Text + case *prefixMatcher: + return v.Text + case *prefixSuffixMatcher: + return v.Prefix + } + return "" +} + +// minLength returns the minimum length in bytes of a string m can match. +func minLength(m matcher) (n int) { + switch v := m.(type) { + case *textMatcher: + return len(v.Text) + case *charMatcher, *runeRangeMatcher, *runeSetMatcher: + return 1 + case *prefixMatcher: + return len(v.Text) + case *suffixMatcher: + return len(v.Text) + case *prefixSuffixMatcher: + return len(v.Prefix) + len(v.Suffix) + case *containsMatcher: + return len(v.Text) + case multiMatcher: + for _, c := range v { + n += minLength(c) + } + return n + case altMatcher: + n = minLength(v[0]) + for _, c := range v[1:] { + n = min(n, minLength(c)) + } + return n + } + return 0 // A star or a void. +} + +// requiredSuffix returns the literal every string m matches must end with. +func requiredSuffix(m matcher) string { + switch v := m.(type) { + case *textMatcher: + return v.Text + case *suffixMatcher: + return v.Text + case *prefixSuffixMatcher: + return v.Suffix + case multiMatcher: + return requiredSuffix(v[len(v)-1]) + case altMatcher: + s := requiredSuffix(v[0]) + for _, c := range v[1:] { + s = commonSuffix(s, requiredSuffix(c)) + if s == "" { + break + } + } + return s + } + return "" // A star, a single-character matcher or a void. +} + +// commonSuffix returns the longest common suffix of a and b, never splitting +// a multi-byte rune. +func commonSuffix(a, b string) string { + i := 0 + for i < len(a) && i < len(b) { + ra, wa := utf8.DecodeLastRuneInString(a[:len(a)-i]) + rb, wb := utf8.DecodeLastRuneInString(b[:len(b)-i]) + if ra != rb || wa != wb { + break + } + i += wa + } + return a[len(a)-i:] +} + +// needsState reports whether matching m may save a checkpoint. Only the +// alts and the non-terminal stars do; a pattern without them is matched +// with a plain call chain -- see [Pattern.Match]. +func needsState(m matcher) bool { + switch v := m.(type) { + case altMatcher: + return true + case multiMatcher: + return slices.ContainsFunc(v, needsState) + case *starMatcher: + return !v.Terminal + } + return false +} diff --git a/readme.md b/readme.md index 682f0d6..9f69257 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # glob.[go](https://golang.org) -[![GoDoc][godoc-image]][godoc-url] [![Build Status][travis-image]][travis-url] +[![GoDoc][godoc-image]][godoc-url] [![CI][ci-image]][ci-url] > Go Globbing Library. @@ -19,130 +19,225 @@ package main import "github.com/gobwas/glob" func main() { - var g glob.Glob - + var g *glob.Pattern + // create simple glob g = glob.MustCompile("*.github.com") g.Match("api.github.com") // true - - // quote meta characters and then create simple glob + + // quote meta characters and then create simple glob g = glob.MustCompile(glob.QuoteMeta("*.github.com")) g.Match("*.github.com") // true - + // create new glob with set of delimiters as ["."] g = glob.MustCompile("api.*.com", '.') g.Match("api.github.com") // true g.Match("api.gi.hub.com") // false - + // create new glob with set of delimiters as ["."] // but now with super wildcard g = glob.MustCompile("api.**.com", '.') g.Match("api.github.com") // true g.Match("api.gi.hub.com") // true - + // create glob with single symbol wildcard g = glob.MustCompile("?at") g.Match("cat") // true g.Match("fat") // true g.Match("at") // false - + // create glob with single symbol wildcard and delimiters ['f'] g = glob.MustCompile("?at", 'f') g.Match("cat") // true g.Match("fat") // false - g.Match("at") // false - - // create glob with character-list matchers + g.Match("at") // false + + // create glob with character-list matchers g = glob.MustCompile("[abc]at") g.Match("cat") // true g.Match("bat") // true g.Match("fat") // false g.Match("at") // false - - // create glob with character-list matchers + + // create glob with character-list matchers g = glob.MustCompile("[!abc]at") g.Match("cat") // false g.Match("bat") // false g.Match("fat") // true - g.Match("at") // false - - // create glob with character-range matchers + g.Match("at") // false + + // create glob with character-range matchers g = glob.MustCompile("[a-c]at") g.Match("cat") // true g.Match("bat") // true g.Match("fat") // false g.Match("at") // false - - // create glob with character-range matchers + + // create glob with character-range matchers g = glob.MustCompile("[!a-c]at") g.Match("cat") // false g.Match("bat") // false g.Match("fat") // true - g.Match("at") // false - - // create glob with pattern-alternatives list + g.Match("at") // false + + // create glob with pattern-alternatives list g = glob.MustCompile("{cat,bat,[fr]at}") g.Match("cat") // true g.Match("bat") // true g.Match("fat") // true g.Match("rat") // true - g.Match("at") // false - g.Match("zat") // false + g.Match("at") // false + g.Match("zat") // false } ``` +`Compile` reports malformed patterns with a `*glob.SyntaxError` carrying the +byte offset and the reason: + +```go +_, err := glob.Compile("{a,b") +// err: glob: syntax error at 4: unclosed `{` +``` + +A compiled `Pattern` captures what it was compiled from, so it can be passed +around instead of the raw arguments and inspected when needed (`String()` makes +it a `fmt.Stringer`, like `regexp.Regexp`): + +```go +g := glob.MustCompile("*.github.com", '.') +g.String() // "*.github.com" +g.Separators() // []rune{'.'} +``` + +## Syntax + +Syntax is inspired by [standard wildcards](http://tldp.org/LDP/GNU-Linux-Tools-Summary/html/x11655.htm), +with one addition: `**` (the "super-asterisk"), which matches any sequence +of characters *including* the separators, where `*` stops at them. Note that +it is just that -- a `*` that crosses separators -- and not the `**/` +"globstar" of shells and file globbers: `**/x` requires the literal `/`, so +it does not match `x`; use `{**/,}x` for that. The same applies to a +`**` between separators, e.g. `a/**/b` does not match `a/b`. + +``` +pattern: + { term } + +term: + `*` matches any sequence of non-separator characters + `**` matches any sequence of characters + `?` matches any single non-separator character + `[` [ `!` ] class `]` + character class; `!` negates it + `{` pattern-list `}` + pattern alternatives + c matches character c (c != `*`, `**`, `?`, `\`, `[`, `{`, `}`) + `\` c matches character c + +class: + lo `-` hi matches character c for lo <= c <= hi + { c } matches any of the listed characters (c != `\`, `]`; + `\` c matches c, `-` is literal here); must be non-empty + +pattern-list: + pattern { `,` pattern } + comma-separated (without spaces) patterns +``` + +### Escaping + +The backslash is the escape character: `\*` is a literal asterisk, and a +backslash itself is `\\`. Mind the Go string literals: `"foo\\bar"` is the +pattern `foo\bar`, which is the literal `foobar`, not `foo\bar`. To match a +backslash (e.g. in the Windows paths) write `"foo\\\\bar"` or `` `foo\\bar` ``, +or use `QuoteMeta` on the literal part. + +### Separators + +The separators are not part of the pattern syntax -- they are configured +once, at compilation time, as the extra arguments of `Compile`: + +```go +g := glob.MustCompile("api.*.com", '.', '/') +``` + +They only limit the wildcards: `*` and `?` never match a separator, while +`**` matches across them; the literals and the character classes are not +affected. With no separators given, `*` and `**` are equivalent. A compiled +`*glob.Pattern` keeps its separators for all matches -- to match the same +pattern with different separators, compile it again. + ## Performance -This library is created for compile-once patterns. This means, that compilation could take time, but -strings matching is done faster, than in case when always parsing template. +This library is created for compile-once patterns. This means, that +compilation could take time, but strings matching is done faster, than in +case when always parsing template. -If you will not use compiled `glob.Glob` object, and do `g := glob.MustCompile(pattern); g.Match(...)` every time, then your code will be much more slower. +If you will not use compiled `*glob.Pattern` object, and do +`g := glob.MustCompile(pattern); g.Match(...)` every time, then your code +will be much more slower. -Run `go test -bench=.` from source root to see the benchmarks: +`Match` performs zero allocations and is safe for concurrent use. Common +pattern shapes (literals, prefixes, suffixes, substrings) are recognized at +compile time and matched with plain string comparisons; the backtracking +engine behind the rest is differentially fuzzed against the `regexp` package +(see `FuzzMatchRegexp`). -Pattern | Fixture | Match | Speed (ns/op) ---------|---------|-------|-------------- -`[a-z][!a-x]*cat*[h][!b]*eyes*` | `my cat has very bright eyes` | `true` | 432 -`[a-z][!a-x]*cat*[h][!b]*eyes*` | `my dog has very bright eyes` | `false` | 199 -`https://*.google.*` | `https://account.google.com` | `true` | 96 -`https://*.google.*` | `https://google.com` | `false` | 66 -`{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}` | `http://yahoo.com` | `true` | 163 -`{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}` | `http://google.com` | `false` | 197 -`{https://*gobwas.com,http://exclude.gobwas.com}` | `https://safe.gobwas.com` | `true` | 22 -`{https://*gobwas.com,http://exclude.gobwas.com}` | `http://safe.gobwas.com` | `false` | 24 -`abc*` | `abcdef` | `true` | 8.15 -`abc*` | `af` | `false` | 5.68 -`*def` | `abcdef` | `true` | 8.84 -`*def` | `af` | `false` | 5.74 -`ab*ef` | `abcdef` | `true` | 15.2 -`ab*ef` | `af` | `false` | 10.4 - -The same things with `regexp` package: +Run `go test -bench=.` from source root to see the benchmarks (the numbers +below are from an Apple M4): Pattern | Fixture | Match | Speed (ns/op) --------|---------|-------|-------------- -`^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` | `my cat has very bright eyes` | `true` | 2553 -`^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` | `my dog has very bright eyes` | `false` | 1383 -`^https:\/\/.*\.google\..*$` | `https://account.google.com` | `true` | 1205 -`^https:\/\/.*\.google\..*$` | `https://google.com` | `false` | 767 -`^(https:\/\/.*\.google\..*\|.*yandex\..*\|.*yahoo\..*\|.*mail\.ru)$` | `http://yahoo.com` | `true` | 1435 -`^(https:\/\/.*\.google\..*\|.*yandex\..*\|.*yahoo\..*\|.*mail\.ru)$` | `http://google.com` | `false` | 1674 -`^(https:\/\/.*gobwas\.com\|http://exclude.gobwas.com)$` | `https://safe.gobwas.com` | `true` | 1039 -`^(https:\/\/.*gobwas\.com\|http://exclude.gobwas.com)$` | `http://safe.gobwas.com` | `false` | 272 -`^abc.*$` | `abcdef` | `true` | 237 -`^abc.*$` | `af` | `false` | 100 -`^.*def$` | `abcdef` | `true` | 464 -`^.*def$` | `af` | `false` | 265 -`^ab.*ef$` | `abcdef` | `true` | 375 -`^ab.*ef$` | `af` | `false` | 145 - -[godoc-image]: https://godoc.org/github.com/gobwas/glob?status.svg -[godoc-url]: https://godoc.org/github.com/gobwas/glob -[travis-image]: https://travis-ci.org/gobwas/glob.svg?branch=master -[travis-url]: https://travis-ci.org/gobwas/glob - -## Syntax - -Syntax is inspired by [standard wildcards](http://tldp.org/LDP/GNU-Linux-Tools-Summary/html/x11655.htm), -except that `**` is aka super-asterisk, that do not sensitive for separators. +`[a-z][!a-x]*cat*[h][!b]*eyes*` | `my cat has very bright eyes` | `true` | 141 +`[a-z][!a-x]*cat*[h][!b]*eyes*` | `my dog has very bright eyes` | `false` | 46 +`https://*.google.*` | `https://account.google.com` | `true` | 16 +`https://*.google.*` | `https://google.com` | `false` | 13 +`{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}` | `http://yahoo.com` | `true` | 61 +`{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}` | `http://google.com` | `false` | 70 +`{https://*gobwas.com,http://exclude.gobwas.com}` | `https://safe.gobwas.com` | `true` | 24 +`{https://*gobwas.com,http://exclude.gobwas.com}` | `http://safe.gobwas.com` | `false` | 32 +`google.com` | `google.com` | `true` | 5.0 +`google.com` | `gobwas.com` | `false` | 3.9 +`abc*` | `abcdef` | `true` | 4.1 +`abc*` | `af` | `false` | 3.0 +`*def` | `abcdef` | `true` | 4.1 +`*def` | `af` | `false` | 2.9 +`ab*ef` | `abcdef` | `true` | 6.0 +`ab*ef` | `af` | `false` | 3.0 + +The same things with the `regexp` package -- not to pick on it (it is a +general-purpose engine with much stronger guarantees), but as a reference +for how the glob-shaped specialization pays off per pattern. The regular +expressions are the exact equivalents: anchored, and with the `s` flag +where there is a `*`, since a `*` matches a newline like any other +character (see `BenchmarkCompareGlobAndRegexp`): + +Pattern | Fixture | Match | Speed (ns/op) | glob is +--------|---------|-------|---------------|-------- +`(?s)^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` | `my cat has very bright eyes` | `true` | 505 | 3.6x faster +`(?s)^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` | `my dog has very bright eyes` | `false` | 221 | 4.9x faster +`(?s)^https://.*\.google\..*$` | `https://account.google.com` | `true` | 251 | 16x faster +`(?s)^https://.*\.google\..*$` | `https://google.com` | `false` | 128 | 9.6x faster +`(?s)^(https://.*\.google\..*\|.*yandex\..*\|.*yahoo\..*\|.*mail\.ru)$` | `http://yahoo.com` | `true` | 396 | 6.5x faster +`(?s)^(https://.*\.google\..*\|.*yandex\..*\|.*yahoo\..*\|.*mail\.ru)$` | `http://google.com` | `false` | 558 | 8.0x faster +`(?s)^(https://.*gobwas\.com\|http://exclude\.gobwas\.com)$` | `https://safe.gobwas.com` | `true` | 210 | 8.8x faster +`(?s)^(https://.*gobwas\.com\|http://exclude\.gobwas\.com)$` | `http://safe.gobwas.com` | `false` | 46 | 1.4x faster +`^google\.com$` | `google.com` | `true` | 25 | 5.0x faster +`^google\.com$` | `gobwas.com` | `false` | 17 | 4.3x faster +`(?s)^abc.*$` | `abcdef` | `true` | 43 | 10x faster +`(?s)^abc.*$` | `af` | `false` | 1.5 | 2.0x slower +`(?s)^.*def$` | `abcdef` | `true` | 73 | 18x faster +`(?s)^.*def$` | `af` | `false` | 1.5 | 1.9x slower +`(?s)^ab.*ef$` | `abcdef` | `true` | 77 | 13x faster +`(?s)^ab.*ef$` | `af` | `false` | 1.5 | 2.0x slower + +(The three `slower` rows are the tiny-mismatch cases. Both engines reject +them with the same literal check; `regexp` just reaches it through less +call overhead. In absolute terms it is 1.5ns vs 3ns -- negligible either +way.) + +[godoc-image]: https://pkg.go.dev/badge/github.com/gobwas/glob.svg +[godoc-url]: https://pkg.go.dev/github.com/gobwas/glob +[ci-image]: https://github.com/gobwas/glob/actions/workflows/ci.yml/badge.svg?branch=master +[ci-url]: https://github.com/gobwas/glob/actions/workflows/ci.yml diff --git a/syntax/ast/ast.go b/syntax/ast/ast.go deleted file mode 100644 index 3220a69..0000000 --- a/syntax/ast/ast.go +++ /dev/null @@ -1,122 +0,0 @@ -package ast - -import ( - "bytes" - "fmt" -) - -type Node struct { - Parent *Node - Children []*Node - Value interface{} - Kind Kind -} - -func NewNode(k Kind, v interface{}, ch ...*Node) *Node { - n := &Node{ - Kind: k, - Value: v, - } - for _, c := range ch { - Insert(n, c) - } - return n -} - -func (a *Node) Equal(b *Node) bool { - if a.Kind != b.Kind { - return false - } - if a.Value != b.Value { - return false - } - if len(a.Children) != len(b.Children) { - return false - } - for i, c := range a.Children { - if !c.Equal(b.Children[i]) { - return false - } - } - return true -} - -func (a *Node) String() string { - var buf bytes.Buffer - buf.WriteString(a.Kind.String()) - if a.Value != nil { - buf.WriteString(" =") - buf.WriteString(fmt.Sprintf("%v", a.Value)) - } - if len(a.Children) > 0 { - buf.WriteString(" [") - for i, c := range a.Children { - if i > 0 { - buf.WriteString(", ") - } - buf.WriteString(c.String()) - } - buf.WriteString("]") - } - return buf.String() -} - -func Insert(parent *Node, children ...*Node) { - parent.Children = append(parent.Children, children...) - for _, ch := range children { - ch.Parent = parent - } -} - -type List struct { - Not bool - Chars string -} - -type Range struct { - Not bool - Lo, Hi rune -} - -type Text struct { - Text string -} - -type Kind int - -const ( - KindNothing Kind = iota - KindPattern - KindList - KindRange - KindText - KindAny - KindSuper - KindSingle - KindAnyOf -) - -func (k Kind) String() string { - switch k { - case KindNothing: - return "Nothing" - case KindPattern: - return "Pattern" - case KindList: - return "List" - case KindRange: - return "Range" - case KindText: - return "Text" - case KindAny: - return "Any" - case KindSuper: - return "Super" - case KindSingle: - return "Single" - case KindAnyOf: - return "AnyOf" - default: - return "" - } -} diff --git a/syntax/ast/parser.go b/syntax/ast/parser.go deleted file mode 100644 index 429b409..0000000 --- a/syntax/ast/parser.go +++ /dev/null @@ -1,157 +0,0 @@ -package ast - -import ( - "errors" - "fmt" - "github.com/gobwas/glob/syntax/lexer" - "unicode/utf8" -) - -type Lexer interface { - Next() lexer.Token -} - -type parseFn func(*Node, Lexer) (parseFn, *Node, error) - -func Parse(lexer Lexer) (*Node, error) { - var parser parseFn - - root := NewNode(KindPattern, nil) - - var ( - tree *Node - err error - ) - for parser, tree = parserMain, root; parser != nil; { - parser, tree, err = parser(tree, lexer) - if err != nil { - return nil, err - } - } - - return root, nil -} - -func parserMain(tree *Node, lex Lexer) (parseFn, *Node, error) { - for { - token := lex.Next() - switch token.Type { - case lexer.EOF: - return nil, tree, nil - - case lexer.Error: - return nil, tree, errors.New(token.Raw) - - case lexer.Text: - Insert(tree, NewNode(KindText, Text{token.Raw})) - return parserMain, tree, nil - - case lexer.Any: - Insert(tree, NewNode(KindAny, nil)) - return parserMain, tree, nil - - case lexer.Super: - Insert(tree, NewNode(KindSuper, nil)) - return parserMain, tree, nil - - case lexer.Single: - Insert(tree, NewNode(KindSingle, nil)) - return parserMain, tree, nil - - case lexer.RangeOpen: - return parserRange, tree, nil - - case lexer.TermsOpen: - a := NewNode(KindAnyOf, nil) - Insert(tree, a) - - p := NewNode(KindPattern, nil) - Insert(a, p) - - return parserMain, p, nil - - case lexer.Separator: - p := NewNode(KindPattern, nil) - Insert(tree.Parent, p) - - return parserMain, p, nil - - case lexer.TermsClose: - return parserMain, tree.Parent.Parent, nil - - default: - return nil, tree, fmt.Errorf("unexpected token: %s", token) - } - } - return nil, tree, fmt.Errorf("unknown error") -} - -func parserRange(tree *Node, lex Lexer) (parseFn, *Node, error) { - var ( - not bool - lo rune - hi rune - chars string - ) - for { - token := lex.Next() - switch token.Type { - case lexer.EOF: - return nil, tree, errors.New("unexpected end") - - case lexer.Error: - return nil, tree, errors.New(token.Raw) - - case lexer.Not: - not = true - - case lexer.RangeLo: - r, w := utf8.DecodeRuneInString(token.Raw) - if len(token.Raw) > w { - return nil, tree, fmt.Errorf("unexpected length of lo character") - } - lo = r - - case lexer.RangeBetween: - // - - case lexer.RangeHi: - r, w := utf8.DecodeRuneInString(token.Raw) - if len(token.Raw) > w { - return nil, tree, fmt.Errorf("unexpected length of lo character") - } - - hi = r - - if hi < lo { - return nil, tree, fmt.Errorf("hi character '%s' should be greater than lo '%s'", string(hi), string(lo)) - } - - case lexer.Text: - chars = token.Raw - - case lexer.RangeClose: - isRange := lo != 0 && hi != 0 - isChars := chars != "" - - if isChars == isRange { - return nil, tree, fmt.Errorf("could not parse range") - } - - if isRange { - Insert(tree, NewNode(KindRange, Range{ - Lo: lo, - Hi: hi, - Not: not, - })) - } else { - Insert(tree, NewNode(KindList, List{ - Chars: chars, - Not: not, - })) - } - - return parserMain, tree, nil - } - } -} diff --git a/syntax/ast/parser_test.go b/syntax/ast/parser_test.go deleted file mode 100644 index a469d38..0000000 --- a/syntax/ast/parser_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package ast - -import ( - "reflect" - "testing" - - "github.com/gobwas/glob/syntax/lexer" -) - -type stubLexer struct { - tokens []lexer.Token - pos int -} - -func (s *stubLexer) Next() (ret lexer.Token) { - if s.pos == len(s.tokens) { - return lexer.Token{lexer.EOF, ""} - } - ret = s.tokens[s.pos] - s.pos++ - return -} - -func TestParseString(t *testing.T) { - for id, test := range []struct { - tokens []lexer.Token - tree *Node - }{ - { - //pattern: "abc", - tokens: []lexer.Token{ - {lexer.Text, "abc"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "abc"}), - ), - }, - { - //pattern: "a*c", - tokens: []lexer.Token{ - {lexer.Text, "a"}, - {lexer.Any, "*"}, - {lexer.Text, "c"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "a"}), - NewNode(KindAny, nil), - NewNode(KindText, Text{Text: "c"}), - ), - }, - { - //pattern: "a**c", - tokens: []lexer.Token{ - {lexer.Text, "a"}, - {lexer.Super, "**"}, - {lexer.Text, "c"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "a"}), - NewNode(KindSuper, nil), - NewNode(KindText, Text{Text: "c"}), - ), - }, - { - //pattern: "a?c", - tokens: []lexer.Token{ - {lexer.Text, "a"}, - {lexer.Single, "?"}, - {lexer.Text, "c"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "a"}), - NewNode(KindSingle, nil), - NewNode(KindText, Text{Text: "c"}), - ), - }, - { - //pattern: "[!a-z]", - tokens: []lexer.Token{ - {lexer.RangeOpen, "["}, - {lexer.Not, "!"}, - {lexer.RangeLo, "a"}, - {lexer.RangeBetween, "-"}, - {lexer.RangeHi, "z"}, - {lexer.RangeClose, "]"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindRange, Range{Lo: 'a', Hi: 'z', Not: true}), - ), - }, - { - //pattern: "[az]", - tokens: []lexer.Token{ - {lexer.RangeOpen, "["}, - {lexer.Text, "az"}, - {lexer.RangeClose, "]"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindList, List{Chars: "az"}), - ), - }, - { - //pattern: "{a,z}", - tokens: []lexer.Token{ - {lexer.TermsOpen, "{"}, - {lexer.Text, "a"}, - {lexer.Separator, ","}, - {lexer.Text, "z"}, - {lexer.TermsClose, "}"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindAnyOf, nil, - NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "a"}), - ), - NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "z"}), - ), - ), - ), - }, - { - //pattern: "/{z,ab}*", - tokens: []lexer.Token{ - {lexer.Text, "/"}, - {lexer.TermsOpen, "{"}, - {lexer.Text, "z"}, - {lexer.Separator, ","}, - {lexer.Text, "ab"}, - {lexer.TermsClose, "}"}, - {lexer.Any, "*"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "/"}), - NewNode(KindAnyOf, nil, - NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "z"}), - ), - NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "ab"}), - ), - ), - NewNode(KindAny, nil), - ), - }, - { - //pattern: "{a,{x,y},?,[a-z],[!qwe]}", - tokens: []lexer.Token{ - {lexer.TermsOpen, "{"}, - {lexer.Text, "a"}, - {lexer.Separator, ","}, - {lexer.TermsOpen, "{"}, - {lexer.Text, "x"}, - {lexer.Separator, ","}, - {lexer.Text, "y"}, - {lexer.TermsClose, "}"}, - {lexer.Separator, ","}, - {lexer.Single, "?"}, - {lexer.Separator, ","}, - {lexer.RangeOpen, "["}, - {lexer.RangeLo, "a"}, - {lexer.RangeBetween, "-"}, - {lexer.RangeHi, "z"}, - {lexer.RangeClose, "]"}, - {lexer.Separator, ","}, - {lexer.RangeOpen, "["}, - {lexer.Not, "!"}, - {lexer.Text, "qwe"}, - {lexer.RangeClose, "]"}, - {lexer.TermsClose, "}"}, - {lexer.EOF, ""}, - }, - tree: NewNode(KindPattern, nil, - NewNode(KindAnyOf, nil, - NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "a"}), - ), - NewNode(KindPattern, nil, - NewNode(KindAnyOf, nil, - NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "x"}), - ), - NewNode(KindPattern, nil, - NewNode(KindText, Text{Text: "y"}), - ), - ), - ), - NewNode(KindPattern, nil, - NewNode(KindSingle, nil), - ), - NewNode(KindPattern, nil, - NewNode(KindRange, Range{Lo: 'a', Hi: 'z', Not: false}), - ), - NewNode(KindPattern, nil, - NewNode(KindList, List{Chars: "qwe", Not: true}), - ), - ), - ), - }, - } { - lexer := &stubLexer{tokens: test.tokens} - result, err := Parse(lexer) - if err != nil { - t.Errorf("[%d] unexpected error: %s", id, err) - } - if !reflect.DeepEqual(test.tree, result) { - t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree) - } - } -} diff --git a/syntax/lexer.go b/syntax/lexer.go new file mode 100644 index 0000000..4286080 --- /dev/null +++ b/syntax/lexer.go @@ -0,0 +1,381 @@ +// Package syntax implements the lexer of the glob pattern syntax. The parser +// lives in package glob; the syntax itself is described at [glob.Compile]. +package syntax + +import ( + "bytes" + "fmt" + "slices" + "unicode/utf8" +) + +// TokenType tells the kind of a [Token]. +type TokenType int + +const ( + // EOF marks the end of the input; the lexer returns it repeatedly. + EOF TokenType = iota + // Error carries an error message in Token.Data; the lexer keeps + // returning it once it happened. Note that the lexer catches only the + // errors local to a token (an invalid UTF-8 sequence, a malformed + // character class): the structural ones, like an unclosed `{`, are for + // the parser to detect. + Error + // Text is a run of literal characters, with the escapes resolved. + Text + // Any is the `*` wildcard. + Any + // Super is the `**` wildcard. + Super + // Single is the `?` wildcard. + Single + // Not is the `!` right after the `[` of a character class. + Not + // TermSeparator is the `,` between the alternatives of a `{...}` group. + // Outside of a group a comma is a plain Text character. + TermSeparator + // RangeOpen and RangeClose are the `[` and `]` of a character class. + // Between them the lexer produces either a Text token (a set of + // characters, `[abc]`) or a RangeLo, RangeBetween, RangeHi triple (a + // range, `[a-c]`), possibly preceded by Not. + RangeOpen + RangeClose + RangeLo + RangeHi + RangeBetween + // TermsOpen and TermsClose are the `{` and `}` of an alternatives group. + TermsOpen + TermsClose +) + +func (tt TokenType) String() string { + switch tt { + case EOF: + return "eof" + case Error: + return "error" + case Text: + return "text" + case Any: + return "any" + case Super: + return "super" + case Single: + return "single" + case Not: + return "not" + case TermSeparator: + return "separator" + case RangeOpen: + return "range_open" + case RangeClose: + return "range_close" + case RangeLo: + return "range_lo" + case RangeHi: + return "range_hi" + case RangeBetween: + return "range_between" + case TermsOpen: + return "terms_open" + case TermsClose: + return "terms_close" + default: + return "" + } +} + +// Token is a lexeme of the pattern: its kind and the source text it was +// read from (or the error message for Error, the literal characters with +// the escapes resolved for Text). +type Token struct { + Type TokenType + Data string +} + +func (t Token) String() string { + return fmt.Sprintf("%v<%q>", t.Type, t.Data) +} + +const ( + char_any = '*' + char_comma = ',' + char_single = '?' + char_escape = '\\' + char_range_open = '[' + char_range_close = ']' + char_terms_open = '{' + char_terms_close = '}' + char_range_not = '!' + char_range_between = '-' +) + +var specials = []byte{ + char_any, + char_single, + char_escape, + char_range_open, + char_range_close, + char_terms_open, + char_terms_close, +} + +// IsSpecial reports whether c is a glob meta character, that is, one that +// [glob.QuoteMeta] escapes. Note that `,`, `!` and `-` are not among them: +// they are special only inside `{...}` and `[...]` respectively, which are. +func IsSpecial(c byte) bool { + return bytes.IndexByte(specials, c) != -1 +} + +type tokens []Token + +func (i *tokens) shift() (ret Token) { + ret = (*i)[0] + copy(*i, (*i)[1:]) + *i = (*i)[:len(*i)-1] + return +} + +func (i *tokens) push(v Token) { + *i = append(*i, v) +} + +func (i *tokens) empty() bool { + return len(*i) == 0 +} + +// eof is the end-of-input sentinel. It must not collide with any rune that +// can appear in a valid pattern -- note that U+0000 can. +const eof rune = -1 + +// Lexer splits a pattern into tokens; see [Lexer.Next]. +type Lexer struct { + data string + pos int + err error + + tokens tokens + termsLevel int + + lastRune rune + lastRuneSize int + hasRune bool +} + +// NewLexer returns a lexer over the source pattern. +func NewLexer(source string) *Lexer { + l := &Lexer{ + data: source, + tokens: tokens(make([]Token, 0, 4)), + } + return l +} + +// Offset returns the byte offset in the source the lexer stopped at, that +// is, the position right after the most recently returned token. +func (l *Lexer) Offset() int { + return l.pos +} + +// Next returns the next token. Once the input is over it returns EOF, and +// once an error happened it returns that Error, repeatedly. +func (l *Lexer) Next() Token { + if l.err != nil { + return Token{Error, l.err.Error()} + } + if !l.tokens.empty() { + return l.tokens.shift() + } + + l.fetchItem() + return l.Next() +} + +func (l *Lexer) peek() (r rune, w int) { + if l.pos == len(l.data) { + return eof, 0 + } + + r, w = utf8.DecodeRuneInString(l.data[l.pos:]) + if r == utf8.RuneError && w == 1 { + // An invalid encoding: a valid U+FFFD decodes at its width of 3. + l.errorf("invalid UTF-8 sequence") + r = eof + w = 0 + } + + return +} + +func (l *Lexer) read() rune { + if l.hasRune { + l.hasRune = false + l.seek(l.lastRuneSize) + return l.lastRune + } + + r, s := l.peek() + l.seek(s) + + l.lastRune = r + l.lastRuneSize = s + + return r +} + +func (l *Lexer) seek(w int) { + l.pos += w +} + +func (l *Lexer) unread() { + if l.hasRune { + l.errorf("could not unread rune") + return + } + l.seek(-l.lastRuneSize) + l.hasRune = true +} + +func (l *Lexer) errorf(f string, v ...any) { + l.err = fmt.Errorf(f, v...) +} + +func (l *Lexer) inTerms() bool { + return l.termsLevel > 0 +} + +func (l *Lexer) termsEnter() { + l.termsLevel++ +} + +func (l *Lexer) termsLeave() { + l.termsLevel-- +} + +var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open} +var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma) + +func (l *Lexer) fetchItem() { + r := l.read() + switch { + case r == eof: + l.tokens.push(Token{EOF, ""}) + + case r == char_terms_open: + l.termsEnter() + l.tokens.push(Token{TermsOpen, string(r)}) + + case r == char_comma && l.inTerms(): + l.tokens.push(Token{TermSeparator, string(r)}) + + case r == char_terms_close && l.inTerms(): + l.tokens.push(Token{TermsClose, string(r)}) + l.termsLeave() + + case r == char_range_open: + l.tokens.push(Token{RangeOpen, string(r)}) + l.fetchRange() + + case r == char_single: + l.tokens.push(Token{Single, string(r)}) + + case r == char_any: + if l.read() == char_any { + l.tokens.push(Token{Super, string(r) + string(r)}) + } else { + l.unread() + l.tokens.push(Token{Any, string(r)}) + } + + default: + l.unread() + + var breakers []rune + if l.inTerms() { + breakers = inTermsBreakers + } else { + breakers = inTextBreakers + } + l.fetchText(breakers) + } +} + +func (l *Lexer) fetchRange() { + var wantHi bool + var wantClose bool + var seenNot bool + for { + r := l.read() + if r == eof { + l.errorf("unexpected end of input") + return + } + + if wantClose { + if r != char_range_close { + l.errorf("expected close range character") + } else { + l.tokens.push(Token{RangeClose, string(r)}) + } + return + } + + if wantHi { + l.tokens.push(Token{RangeHi, string(r)}) + wantClose = true + continue + } + + if !seenNot && r == char_range_not { + l.tokens.push(Token{Not, string(r)}) + seenNot = true + continue + } + + if n, w := l.peek(); n == char_range_between { + l.seek(w) + l.tokens.push(Token{RangeLo, string(r)}) + l.tokens.push(Token{RangeBetween, string(n)}) + wantHi = true + continue + } + + l.unread() // unread first peek and fetch as text + l.fetchText([]rune{char_range_close}) + wantClose = true + } +} + +func (l *Lexer) fetchText(breakers []rune) { + var data []rune + var escaped bool + +reading: + for { + r := l.read() + if r == eof { + if escaped { + l.errorf("trailing backslash") + } + break + } + + if !escaped { + if r == char_escape { + escaped = true + continue + } + if slices.Index(breakers, r) != -1 { + l.unread() + break reading + } + } + + escaped = false + data = append(data, r) + } + + if len(data) > 0 { + l.tokens.push(Token{Text, string(data)}) + } +} diff --git a/syntax/lexer/lexer.go b/syntax/lexer/lexer.go deleted file mode 100644 index a1c8d19..0000000 --- a/syntax/lexer/lexer.go +++ /dev/null @@ -1,273 +0,0 @@ -package lexer - -import ( - "bytes" - "fmt" - "github.com/gobwas/glob/util/runes" - "unicode/utf8" -) - -const ( - char_any = '*' - char_comma = ',' - char_single = '?' - char_escape = '\\' - char_range_open = '[' - char_range_close = ']' - char_terms_open = '{' - char_terms_close = '}' - char_range_not = '!' - char_range_between = '-' -) - -var specials = []byte{ - char_any, - char_single, - char_escape, - char_range_open, - char_range_close, - char_terms_open, - char_terms_close, -} - -func Special(c byte) bool { - return bytes.IndexByte(specials, c) != -1 -} - -type tokens []Token - -func (i *tokens) shift() (ret Token) { - ret = (*i)[0] - copy(*i, (*i)[1:]) - *i = (*i)[:len(*i)-1] - return -} - -func (i *tokens) push(v Token) { - *i = append(*i, v) -} - -func (i *tokens) empty() bool { - return len(*i) == 0 -} - -var eof rune = 0 - -type lexer struct { - data string - pos int - err error - - tokens tokens - termsLevel int - - lastRune rune - lastRuneSize int - hasRune bool -} - -func NewLexer(source string) *lexer { - l := &lexer{ - data: source, - tokens: tokens(make([]Token, 0, 4)), - } - return l -} - -func (l *lexer) Next() Token { - if l.err != nil { - return Token{Error, l.err.Error()} - } - if !l.tokens.empty() { - return l.tokens.shift() - } - - l.fetchItem() - return l.Next() -} - -func (l *lexer) peek() (r rune, w int) { - if l.pos == len(l.data) { - return eof, 0 - } - - r, w = utf8.DecodeRuneInString(l.data[l.pos:]) - if r == utf8.RuneError { - l.errorf("could not read rune") - r = eof - w = 0 - } - - return -} - -func (l *lexer) read() rune { - if l.hasRune { - l.hasRune = false - l.seek(l.lastRuneSize) - return l.lastRune - } - - r, s := l.peek() - l.seek(s) - - l.lastRune = r - l.lastRuneSize = s - - return r -} - -func (l *lexer) seek(w int) { - l.pos += w -} - -func (l *lexer) unread() { - if l.hasRune { - l.errorf("could not unread rune") - return - } - l.seek(-l.lastRuneSize) - l.hasRune = true -} - -func (l *lexer) errorf(f string, v ...interface{}) { - l.err = fmt.Errorf(f, v...) -} - -func (l *lexer) inTerms() bool { - return l.termsLevel > 0 -} - -func (l *lexer) termsEnter() { - l.termsLevel++ -} - -func (l *lexer) termsLeave() { - l.termsLevel-- -} - -var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open} -var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma) - -func (l *lexer) fetchItem() { - r := l.read() - switch { - case r == eof: - l.tokens.push(Token{EOF, ""}) - - case r == char_terms_open: - l.termsEnter() - l.tokens.push(Token{TermsOpen, string(r)}) - - case r == char_comma && l.inTerms(): - l.tokens.push(Token{Separator, string(r)}) - - case r == char_terms_close && l.inTerms(): - l.tokens.push(Token{TermsClose, string(r)}) - l.termsLeave() - - case r == char_range_open: - l.tokens.push(Token{RangeOpen, string(r)}) - l.fetchRange() - - case r == char_single: - l.tokens.push(Token{Single, string(r)}) - - case r == char_any: - if l.read() == char_any { - l.tokens.push(Token{Super, string(r) + string(r)}) - } else { - l.unread() - l.tokens.push(Token{Any, string(r)}) - } - - default: - l.unread() - - var breakers []rune - if l.inTerms() { - breakers = inTermsBreakers - } else { - breakers = inTextBreakers - } - l.fetchText(breakers) - } -} - -func (l *lexer) fetchRange() { - var wantHi bool - var wantClose bool - var seenNot bool - for { - r := l.read() - if r == eof { - l.errorf("unexpected end of input") - return - } - - if wantClose { - if r != char_range_close { - l.errorf("expected close range character") - } else { - l.tokens.push(Token{RangeClose, string(r)}) - } - return - } - - if wantHi { - l.tokens.push(Token{RangeHi, string(r)}) - wantClose = true - continue - } - - if !seenNot && r == char_range_not { - l.tokens.push(Token{Not, string(r)}) - seenNot = true - continue - } - - if n, w := l.peek(); n == char_range_between { - l.seek(w) - l.tokens.push(Token{RangeLo, string(r)}) - l.tokens.push(Token{RangeBetween, string(n)}) - wantHi = true - continue - } - - l.unread() // unread first peek and fetch as text - l.fetchText([]rune{char_range_close}) - wantClose = true - } -} - -func (l *lexer) fetchText(breakers []rune) { - var data []rune - var escaped bool - -reading: - for { - r := l.read() - if r == eof { - break - } - - if !escaped { - if r == char_escape { - escaped = true - continue - } - - if runes.IndexRune(breakers, r) != -1 { - l.unread() - break reading - } - } - - escaped = false - data = append(data, r) - } - - if len(data) > 0 { - l.tokens.push(Token{Text, string(data)}) - } -} diff --git a/syntax/lexer/token.go b/syntax/lexer/token.go deleted file mode 100644 index 2797c4e..0000000 --- a/syntax/lexer/token.go +++ /dev/null @@ -1,88 +0,0 @@ -package lexer - -import "fmt" - -type TokenType int - -const ( - EOF TokenType = iota - Error - Text - Char - Any - Super - Single - Not - Separator - RangeOpen - RangeClose - RangeLo - RangeHi - RangeBetween - TermsOpen - TermsClose -) - -func (tt TokenType) String() string { - switch tt { - case EOF: - return "eof" - - case Error: - return "error" - - case Text: - return "text" - - case Char: - return "char" - - case Any: - return "any" - - case Super: - return "super" - - case Single: - return "single" - - case Not: - return "not" - - case Separator: - return "separator" - - case RangeOpen: - return "range_open" - - case RangeClose: - return "range_close" - - case RangeLo: - return "range_lo" - - case RangeHi: - return "range_hi" - - case RangeBetween: - return "range_between" - - case TermsOpen: - return "terms_open" - - case TermsClose: - return "terms_close" - - default: - return "undef" - } -} - -type Token struct { - Type TokenType - Raw string -} - -func (t Token) String() string { - return fmt.Sprintf("%v<%q>", t.Type, t.Raw) -} diff --git a/syntax/lexer/lexer_test.go b/syntax/lexer_test.go similarity index 90% rename from syntax/lexer/lexer_test.go rename to syntax/lexer_test.go index ec35f81..9f2f208 100644 --- a/syntax/lexer/lexer_test.go +++ b/syntax/lexer_test.go @@ -1,4 +1,4 @@ -package lexer +package syntax import ( "testing" @@ -28,7 +28,7 @@ func TestLexGood(t *testing.T) { {Text, "/"}, {TermsOpen, "{"}, {Text, "rate"}, - {Separator, ","}, + {TermSeparator, ","}, {RangeOpen, "["}, {RangeLo, "0"}, {RangeBetween, "-"}, @@ -132,7 +132,7 @@ func TestLexGood(t *testing.T) { items: []Token{ {TermsOpen, "{"}, {Text, "a"}, - {Separator, ","}, + {TermSeparator, ","}, {Text, "b"}, {TermsClose, "}"}, {EOF, ""}, @@ -144,7 +144,7 @@ func TestLexGood(t *testing.T) { {Text, "/"}, {TermsOpen, "{"}, {Text, "z"}, - {Separator, ","}, + {TermSeparator, ","}, {Text, "ab"}, {TermsClose, "}"}, {Any, "*"}, @@ -161,16 +161,16 @@ func TestLexGood(t *testing.T) { {RangeBetween, "-"}, {RangeHi, "語"}, {RangeClose, "]"}, - {Separator, ","}, + {TermSeparator, ","}, {Any, "*"}, - {Separator, ","}, + {TermSeparator, ","}, {Single, "?"}, - {Separator, ","}, + {TermSeparator, ","}, {TermsOpen, "{"}, {Text, "a"}, - {Separator, ","}, + {TermSeparator, ","}, {Text, "b"}, - {Separator, ","}, + {TermSeparator, ","}, {Text, "c"}, {TermsClose, "}"}, {TermsClose, "}"}, @@ -184,8 +184,8 @@ func TestLexGood(t *testing.T) { if act.Type != exp.Type { t.Errorf("#%d %q: wrong %d-th item type: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Type, act.Type, exp, act) } - if act.Raw != exp.Raw { - t.Errorf("#%d %q: wrong %d-th item contents: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Raw, act.Raw, exp, act) + if act.Data != exp.Data { + t.Errorf("#%d %q: wrong %d-th item contents: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Data, act.Data, exp, act) } } } diff --git a/syntax/syntax.go b/syntax/syntax.go deleted file mode 100644 index 1d168b1..0000000 --- a/syntax/syntax.go +++ /dev/null @@ -1,14 +0,0 @@ -package syntax - -import ( - "github.com/gobwas/glob/syntax/ast" - "github.com/gobwas/glob/syntax/lexer" -) - -func Parse(s string) (*ast.Node, error) { - return ast.Parse(lexer.NewLexer(s)) -} - -func Special(b byte) bool { - return lexer.Special(b) -} diff --git a/testdata/fuzz/FuzzMatchRegexp/2ce462953d7dddb5 b/testdata/fuzz/FuzzMatchRegexp/2ce462953d7dddb5 new file mode 100644 index 0000000..e170fc0 --- /dev/null +++ b/testdata/fuzz/FuzzMatchRegexp/2ce462953d7dddb5 @@ -0,0 +1,4 @@ +go test fuzz v1 +string("*{*0,}") +string("1") +bool(false) diff --git a/testdata/fuzz/FuzzMatchRegexp/8b49bc47a9fab1f0 b/testdata/fuzz/FuzzMatchRegexp/8b49bc47a9fab1f0 new file mode 100644 index 0000000..a50f51e --- /dev/null +++ b/testdata/fuzz/FuzzMatchRegexp/8b49bc47a9fab1f0 @@ -0,0 +1,4 @@ +go test fuzz v1 +string("**\x00") +string("0") +bool(true) diff --git a/util/runes/runes.go b/util/runes/runes.go deleted file mode 100644 index a723556..0000000 --- a/util/runes/runes.go +++ /dev/null @@ -1,154 +0,0 @@ -package runes - -func Index(s, needle []rune) int { - ls, ln := len(s), len(needle) - - switch { - case ln == 0: - return 0 - case ln == 1: - return IndexRune(s, needle[0]) - case ln == ls: - if Equal(s, needle) { - return 0 - } - return -1 - case ln > ls: - return -1 - } - -head: - for i := 0; i < ls && ls-i >= ln; i++ { - for y := 0; y < ln; y++ { - if s[i+y] != needle[y] { - continue head - } - } - - return i - } - - return -1 -} - -func LastIndex(s, needle []rune) int { - ls, ln := len(s), len(needle) - - switch { - case ln == 0: - if ls == 0 { - return 0 - } - return ls - case ln == 1: - return IndexLastRune(s, needle[0]) - case ln == ls: - if Equal(s, needle) { - return 0 - } - return -1 - case ln > ls: - return -1 - } - -head: - for i := ls - 1; i >= 0 && i >= ln; i-- { - for y := ln - 1; y >= 0; y-- { - if s[i-(ln-y-1)] != needle[y] { - continue head - } - } - - return i - ln + 1 - } - - return -1 -} - -// IndexAny returns the index of the first instance of any Unicode code point -// from chars in s, or -1 if no Unicode code point from chars is present in s. -func IndexAny(s, chars []rune) int { - if len(chars) > 0 { - for i, c := range s { - for _, m := range chars { - if c == m { - return i - } - } - } - } - return -1 -} - -func Contains(s, needle []rune) bool { - return Index(s, needle) >= 0 -} - -func Max(s []rune) (max rune) { - for _, r := range s { - if r > max { - max = r - } - } - - return -} - -func Min(s []rune) rune { - min := rune(-1) - for _, r := range s { - if min == -1 { - min = r - continue - } - - if r < min { - min = r - } - } - - return min -} - -func IndexRune(s []rune, r rune) int { - for i, c := range s { - if c == r { - return i - } - } - return -1 -} - -func IndexLastRune(s []rune, r rune) int { - for i := len(s) - 1; i >= 0; i-- { - if s[i] == r { - return i - } - } - - return -1 -} - -func Equal(a, b []rune) bool { - if len(a) == len(b) { - for i := 0; i < len(a); i++ { - if a[i] != b[i] { - return false - } - } - - return true - } - - return false -} - -// HasPrefix tests whether the string s begins with prefix. -func HasPrefix(s, prefix []rune) bool { - return len(s) >= len(prefix) && Equal(s[0:len(prefix)], prefix) -} - -// HasSuffix tests whether the string s ends with suffix. -func HasSuffix(s, suffix []rune) bool { - return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) -} diff --git a/util/runes/runes_test.go b/util/runes/runes_test.go deleted file mode 100644 index 54498eb..0000000 --- a/util/runes/runes_test.go +++ /dev/null @@ -1,222 +0,0 @@ -package runes - -import ( - "strings" - "testing" -) - -type indexTest struct { - s []rune - sep []rune - out int -} - -type equalTest struct { - a []rune - b []rune - out bool -} - -func newIndexTest(s, sep string, out int) indexTest { - return indexTest{[]rune(s), []rune(sep), out} -} -func newEqualTest(s, sep string, out bool) equalTest { - return equalTest{[]rune(s), []rune(sep), out} -} - -var dots = "1....2....3....4" - -var indexTests = []indexTest{ - newIndexTest("", "", 0), - newIndexTest("", "a", -1), - newIndexTest("", "foo", -1), - newIndexTest("fo", "foo", -1), - newIndexTest("foo", "foo", 0), - newIndexTest("oofofoofooo", "f", 2), - newIndexTest("oofofoofooo", "foo", 4), - newIndexTest("barfoobarfoo", "foo", 3), - newIndexTest("foo", "", 0), - newIndexTest("foo", "o", 1), - newIndexTest("abcABCabc", "A", 3), - // cases with one byte strings - test special case in Index() - newIndexTest("", "a", -1), - newIndexTest("x", "a", -1), - newIndexTest("x", "x", 0), - newIndexTest("abc", "a", 0), - newIndexTest("abc", "b", 1), - newIndexTest("abc", "c", 2), - newIndexTest("abc", "x", -1), -} - -var lastIndexTests = []indexTest{ - newIndexTest("", "", 0), - newIndexTest("", "a", -1), - newIndexTest("", "foo", -1), - newIndexTest("fo", "foo", -1), - newIndexTest("foo", "foo", 0), - newIndexTest("foo", "f", 0), - newIndexTest("oofofoofooo", "f", 7), - newIndexTest("oofofoofooo", "foo", 7), - newIndexTest("barfoobarfoo", "foo", 9), - newIndexTest("foo", "", 3), - newIndexTest("foo", "o", 2), - newIndexTest("abcABCabc", "A", 3), - newIndexTest("abcABCabc", "a", 6), -} - -var indexAnyTests = []indexTest{ - newIndexTest("", "", -1), - newIndexTest("", "a", -1), - newIndexTest("", "abc", -1), - newIndexTest("a", "", -1), - newIndexTest("a", "a", 0), - newIndexTest("aaa", "a", 0), - newIndexTest("abc", "xyz", -1), - newIndexTest("abc", "xcz", 2), - newIndexTest("a☺b☻c☹d", "uvw☻xyz", 3), - newIndexTest("aRegExp*", ".(|)*+?^$[]", 7), - newIndexTest(dots+dots+dots, " ", -1), -} - -// Execute f on each test case. funcName should be the name of f; it's used -// in failure reports. -func runIndexTests(t *testing.T, f func(s, sep []rune) int, funcName string, testCases []indexTest) { - for _, test := range testCases { - actual := f(test.s, test.sep) - if actual != test.out { - t.Errorf("%s(%q,%q) = %v; want %v", funcName, test.s, test.sep, actual, test.out) - } - } -} - -func TestIndex(t *testing.T) { runIndexTests(t, Index, "Index", indexTests) } -func TestLastIndex(t *testing.T) { runIndexTests(t, LastIndex, "LastIndex", lastIndexTests) } -func TestIndexAny(t *testing.T) { runIndexTests(t, IndexAny, "IndexAny", indexAnyTests) } - -var equalTests = []equalTest{ - newEqualTest("a", "a", true), - newEqualTest("a", "b", false), - newEqualTest("a☺b☻c☹d", "uvw☻xyz", false), - newEqualTest("a☺b☻c☹d", "a☺b☻c☹d", true), -} - -func TestEqual(t *testing.T) { - for _, test := range equalTests { - actual := Equal(test.a, test.b) - if actual != test.out { - t.Errorf("Equal(%q,%q) = %v; want %v", test.a, test.b, actual, test.out) - } - } -} - -func BenchmarkLastIndexRunes(b *testing.B) { - r := []rune("abcdef") - n := []rune("cd") - - for i := 0; i < b.N; i++ { - LastIndex(r, n) - } -} -func BenchmarkLastIndexStrings(b *testing.B) { - r := "abcdef" - n := "cd" - - for i := 0; i < b.N; i++ { - strings.LastIndex(r, n) - } -} - -func BenchmarkIndexAnyRunes(b *testing.B) { - s := []rune("...b...") - c := []rune("abc") - - for i := 0; i < b.N; i++ { - IndexAny(s, c) - } -} -func BenchmarkIndexAnyStrings(b *testing.B) { - s := "...b..." - c := "abc" - - for i := 0; i < b.N; i++ { - strings.IndexAny(s, c) - } -} - -func BenchmarkIndexRuneRunes(b *testing.B) { - s := []rune("...b...") - r := 'b' - - for i := 0; i < b.N; i++ { - IndexRune(s, r) - } -} -func BenchmarkIndexRuneStrings(b *testing.B) { - s := "...b..." - r := 'b' - - for i := 0; i < b.N; i++ { - strings.IndexRune(s, r) - } -} - -func BenchmarkIndexRunes(b *testing.B) { - r := []rune("abcdef") - n := []rune("cd") - - for i := 0; i < b.N; i++ { - Index(r, n) - } -} -func BenchmarkIndexStrings(b *testing.B) { - r := "abcdef" - n := "cd" - - for i := 0; i < b.N; i++ { - strings.Index(r, n) - } -} - -func BenchmarkEqualRunes(b *testing.B) { - x := []rune("abc") - y := []rune("abc") - - for i := 0; i < b.N; i++ { - if Equal(x, y) { - continue - } - } -} - -func BenchmarkEqualStrings(b *testing.B) { - x := "abc" - y := "abc" - - for i := 0; i < b.N; i++ { - if x == y { - continue - } - } -} - -func BenchmarkNotEqualRunes(b *testing.B) { - x := []rune("abc") - y := []rune("abcd") - - for i := 0; i < b.N; i++ { - if Equal(x, y) { - continue - } - } -} - -func BenchmarkNotEqualStrings(b *testing.B) { - x := "abc" - y := "abcd" - - for i := 0; i < b.N; i++ { - if x == y { - continue - } - } -} diff --git a/util/strings/strings.go b/util/strings/strings.go deleted file mode 100644 index e8ee192..0000000 --- a/util/strings/strings.go +++ /dev/null @@ -1,39 +0,0 @@ -package strings - -import ( - "strings" - "unicode/utf8" -) - -func IndexAnyRunes(s string, rs []rune) int { - for _, r := range rs { - if i := strings.IndexRune(s, r); i != -1 { - return i - } - } - - return -1 -} - -func LastIndexAnyRunes(s string, rs []rune) int { - for _, r := range rs { - i := -1 - if 0 <= r && r < utf8.RuneSelf { - i = strings.LastIndexByte(s, byte(r)) - } else { - sub := s - for len(sub) > 0 { - j := strings.IndexRune(s, r) - if j == -1 { - break - } - i = j - sub = sub[i+1:] - } - } - if i != -1 { - return i - } - } - return -1 -}