From 0eb1796f71efab938981610df75ec620b8b7cf81 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Mon, 22 Jun 2026 16:58:53 +0200 Subject: [PATCH 1/6] Remove ASCII verification. --- fragmentation/encoding.go | 34 ++++++----------------------- fragmentation/fragmentation_test.go | 29 +++++++++++++++++++++++- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/fragmentation/encoding.go b/fragmentation/encoding.go index 9f9d0813..64d5cbd4 100644 --- a/fragmentation/encoding.go +++ b/fragmentation/encoding.go @@ -19,37 +19,17 @@ package fragmentation import ( - "os" + "errors" "unicode/utf8" ) -const lastASCIIchar = 127 +var errUnsupportedEncoding = errors.New("unsupported source encoding: expected UTF-8") -// IsEncodedAsText reports whether the file stored at filePath is encoded as a text. -// -// If file encoded in ASCII or UTF-8, it is meant to be a text file. -func IsEncodedAsText(filePath string) (bool, error) { - // Read the entire file into memory. - content, err := os.ReadFile(filePath) - if err != nil { - return false, err - } - - isUTF8Encoded := utf8.Valid(content) - isASCIIEncoded := areASCIIEncoded(content) - - return isUTF8Encoded || isASCIIEncoded, nil -} - -// Reports whether given bytes are ASCII-encoded. -// -// If all the characters fall within the ASCII range (0 to 127), it’s likely an ASCII-encoded file. -func areASCIIEncoded(bytes []byte) bool { - for _, char := range bytes { - if char > byte(lastASCIIchar) { - return false - } +// validateTextEncoding reports whether source content uses UTF-8 text encoding. +func validateTextEncoding(content []byte) error { + if !utf8.Valid(content) { + return errUnsupportedEncoding } - return true + return nil } diff --git a/fragmentation/fragmentation_test.go b/fragmentation/fragmentation_test.go index 742aaa47..24d7a6e5 100644 --- a/fragmentation/fragmentation_test.go +++ b/fragmentation/fragmentation_test.go @@ -23,6 +23,8 @@ import ( "embed-code/embed-code-go/fragmentation" _type "embed-code/embed-code-go/type" "fmt" + "os" + "path/filepath" "testing" . "github.com/onsi/ginkgo/v2" @@ -94,6 +96,31 @@ var _ = Describe("Fragmentation", func() { Expect(fragments).Should(HaveKey(fragmentation.DefaultFragmentName)) }) + It("should skip a non-UTF-8 source and use the next code root", func() { + invalidRoot := GinkgoT().TempDir() + validRoot := GinkgoT().TempDir() + fileName := "Example.java" + Expect(os.WriteFile(filepath.Join(invalidRoot, fileName), []byte{0xff}, 0600)).To(Succeed()) + Expect(os.WriteFile( + filepath.Join(validRoot, fileName), + []byte("class Example {}"), + 0600, + )).To(Succeed()) + config.CodeRoots = _type.NamedPathList{ + _type.NamedPath{Path: invalidRoot}, + _type.NamedPath{Path: validRoot}, + } + + content, err := fragmentation.ResolveContent( + fileName, + fragmentation.DefaultFragmentName, + config, + ) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(content).Should(Equal([]string{"class Example {}"})) + }) + It("should fail on an unopened fragment", func() { frag := buildTestFragmentation(unopenedFragmentFileName, config) @@ -221,7 +248,7 @@ func buildTestFragmentation(testFileName string, config configuration.Configuration) fragmentation.Fragmentation { codeRoot := config.CodeRoots[0] testFilePath := fmt.Sprintf("%s/org/example/%s", codeRoot.Path, testFileName) - frag, err := fragmentation.NewFragmentation(testFilePath, codeRoot, config) + frag, err := fragmentation.NewFragmentation(testFilePath) Expect(err).ShouldNot(HaveOccurred()) From 5ae2c7d41936f6cb7e50dd636f3bb6b9b64b24e7 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Tue, 23 Jun 2026 10:57:06 +0200 Subject: [PATCH 2/6] Simplify fragmentation path-choosing logic. --- fragmentation/fragmentation.go | 71 +++++++--------------------------- fragmentation/resolver.go | 61 ++++++++++++++--------------- indent/indent_test.go | 1 - 3 files changed, 45 insertions(+), 88 deletions(-) diff --git a/fragmentation/fragmentation.go b/fragmentation/fragmentation.go index b9627b2e..ed29c9c0 100644 --- a/fragmentation/fragmentation.go +++ b/fragmentation/fragmentation.go @@ -37,13 +37,10 @@ package fragmentation import ( "bufio" - "embed-code/embed-code-go/files" - _type "embed-code/embed-code-go/type" + "bytes" "fmt" "os" "path/filepath" - - config "embed-code/embed-code-go/configuration" ) // NamedPathPrefix the prefix before the named code source. @@ -51,46 +48,25 @@ const NamedPathPrefix = "$" // Fragmentation splits the given file into fragments. // -// Configuration — a configuration for embedding. -// -// SourcesRoot — a named source code path. -// // CodeFile — a full path of a file to fragment. type Fragmentation struct { - Configuration config.Configuration - SourcesRoot _type.NamedPath CodeFile string fragmentBuilders map[string]*FragmentBuilder } -// NewFragmentation builds Fragmentation from given codeFileRelative and config. +// NewFragmentation builds Fragmentation for the given code file. // -// codeFileRelative — a relative path to a code file to fragment. -// -// config — a configuration for embedding. -func NewFragmentation( - codeFileRelative string, - codeRoot _type.NamedPath, - config config.Configuration, -) (Fragmentation, error) { - fragmentation := Fragmentation{} - - fragmentation.SourcesRoot = codeRoot - _, err := filepath.Abs(codeRoot.Path) +// codeFile — a relative or absolute path to a code file to fragment. +func NewFragmentation(codeFile string) (Fragmentation, error) { + absoluteCodeFile, err := filepath.Abs(codeFile) if err != nil { return Fragmentation{}, err } - absoluteCodeFile, err := filepath.Abs(codeFileRelative) - if err != nil { - return Fragmentation{}, err - } - fragmentation.CodeFile = absoluteCodeFile - - fragmentation.Configuration = config - fragmentation.fragmentBuilders = make(map[string]*FragmentBuilder) - - return fragmentation, nil + return Fragmentation{ + CodeFile: absoluteCodeFile, + fragmentBuilders: make(map[string]*FragmentBuilder), + }, nil } // DoFragmentation splits the file into fragments. @@ -100,16 +76,15 @@ func NewFragmentation( func (f Fragmentation) DoFragmentation() ([]string, map[string]Fragment, error) { var contentToRender []string - file, err := os.Open(f.CodeFile) + content, err := os.ReadFile(f.CodeFile) if err != nil { return nil, nil, err } + if err := validateTextEncoding(content); err != nil { + return nil, nil, err + } - defer func(file *os.File) { - err = file.Close() - }(file) - - scanner := bufio.NewScanner(file) + scanner := bufio.NewScanner(bytes.NewReader(content)) lineNumber := 0 for scanner.Scan() { lineNumber++ @@ -122,7 +97,7 @@ func (f Fragmentation) DoFragmentation() ([]string, map[string]Fragment, error) ) } } - if err = scanner.Err(); err != nil { + if err := scanner.Err(); err != nil { return nil, nil, err } @@ -135,22 +110,6 @@ func (f Fragmentation) DoFragmentation() ([]string, map[string]Fragment, error) return contentToRender, fragments, nil } -// shouldDoFragmentation reports whether the file is valid to do fragmentation: -// - it exists by the given path -// - it is a file (not a dir) -// - it is textual-encoded. -func shouldDoFragmentation(filePath string) (bool, error) { - exists, err := files.IsFileExist(filePath) - if err != nil { - return false, err - } - if exists { - return IsEncodedAsText(filePath) - } - - return false, nil -} - // Parses a single line of input and performs the following actions: // - identifies fragment start and end markers within given line; // - updates fragmentBuilders based on the markers; diff --git a/fragmentation/resolver.go b/fragmentation/resolver.go index a6dff150..e4abe27d 100644 --- a/fragmentation/resolver.go +++ b/fragmentation/resolver.go @@ -19,12 +19,14 @@ package fragmentation import ( + "errors" "fmt" "log/slog" "path/filepath" "strings" config "embed-code/embed-code-go/configuration" + "embed-code/embed-code-go/files" "embed-code/embed-code-go/logging" _type "embed-code/embed-code-go/type" ) @@ -39,7 +41,7 @@ type fragmentedFile struct { } // resolverCache stores source fragmentations already resolved during the current run. -var resolverCache = newCache[resolvedPath, fragmentedFile]( +var resolverCache = newCache[string, fragmentedFile]( resolverCacheLimit, loadSourceFragments, ) @@ -76,8 +78,8 @@ func ResolveContent( fragment, found := content.fragments[fragmentName] if !found { - codeFileReference := logging.FileReference(source.absolutePath) - slog.Info(missingFragmentLogMessage(fragmentName, source.absolutePath)) + codeFileReference := logging.FileReference(source) + slog.Info(missingFragmentLogMessage(fragmentName, source)) return nil, fmt.Errorf("fragment `%s` from code file `%s` not found", fragmentName, codeFileReference) @@ -103,7 +105,7 @@ func ResolveCodeFileReference(codePath string, config config.Configuration) (str return "", err } if found { - return logging.FileReference(source.absolutePath), nil + return logging.FileReference(source), nil } return codeFileReference(codePath, config) @@ -114,15 +116,8 @@ func ClearResolverCache() { resolverCache.clear() } -// resolvedPath is a source file path resolved from a user-facing embedding path. -type resolvedPath struct { - root _type.NamedPath - relativePath string - absolutePath string -} - // resolveSource resolves the user-facing code path to the source file. -func resolveSource(codePath string, config config.Configuration) (resolvedPath, bool, error) { +func resolveSource(codePath string, config config.Configuration) (string, bool, error) { codeRootName, relativePath, named := splitNamedPath(codePath) for _, root := range config.CodeRoots { if named && strings.TrimSpace(root.Name) != codeRootName { @@ -131,20 +126,28 @@ func resolveSource(codePath string, config config.Configuration) (resolvedPath, source, err := sourceFromRoot(root, relativePath) if err != nil { - return resolvedPath{}, false, err + return "", false, err } - shouldFragment, err := shouldDoFragmentation(source.absolutePath) + exists, err := files.IsFileExist(source) if err != nil { - return resolvedPath{}, false, err + return "", false, err } - if !shouldFragment { + if !exists { continue } + _, err = cachedSourceFragments(source) + if errors.Is(err, errUnsupportedEncoding) { + continue + } + if err != nil { + return "", false, err + } + return source, true, nil } - return resolvedPath{}, false, nil + return "", false, nil } // splitNamedPath separates a named-code-root prefix from a code path. @@ -160,28 +163,24 @@ func splitNamedPath(codePath string) (string, string, bool) { return rootName, relativePath, true } -// sourceFromRoot builds a source path from a code root and a relative path. -func sourceFromRoot(root _type.NamedPath, relativePath string) (resolvedPath, error) { +// sourceFromRoot builds an absolute source path from a code root and a relative path. +func sourceFromRoot(root _type.NamedPath, relativePath string) (string, error) { rootAbs, err := filepath.Abs(root.Path) if err != nil { - return resolvedPath{}, err + return "", err } - return resolvedPath{ - root: root, - relativePath: filepath.FromSlash(relativePath), - absolutePath: filepath.Join(rootAbs, filepath.FromSlash(relativePath)), - }, nil + return filepath.Join(rootAbs, filepath.FromSlash(relativePath)), nil } -// cachedSourceFragments returns cached source fragmentation for a resolved source file. -func cachedSourceFragments(source resolvedPath) (fragmentedFile, error) { +// cachedSourceFragments returns cached source fragmentation for an absolute source path. +func cachedSourceFragments(source string) (fragmentedFile, error) { return resolverCache.get(source) } // loadSourceFragments reads and fragments the source file when it is not already cached. -func loadSourceFragments(source resolvedPath) (fragmentedFile, error) { - fragmentation, err := NewFragmentation(source.absolutePath, source.root, config.Configuration{}) +func loadSourceFragments(source string) (fragmentedFile, error) { + fragmentation, err := NewFragmentation(source) if err != nil { return fragmentedFile{}, err } @@ -240,10 +239,10 @@ func codeFileReference(codePath string, config config.Configuration) (string, er return "", err } if named { - return fmt.Sprintf("%s (%s)", codePath, logging.FileReference(source.absolutePath)), nil + return fmt.Sprintf("%s (%s)", codePath, logging.FileReference(source)), nil } if len(config.CodeRoots) == 1 { - return logging.FileReference(source.absolutePath), nil + return logging.FileReference(source), nil } } diff --git a/indent/indent_test.go b/indent/indent_test.go index f1374fa9..98165fc2 100644 --- a/indent/indent_test.go +++ b/indent/indent_test.go @@ -59,5 +59,4 @@ var _ = Describe("Indent", func() { Expect(changedLines).ShouldNot(Equal(testLines)) }) - }) From 65c6fa3918ef789a1b323bafe106732a17ca4cc0 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Tue, 23 Jun 2026 11:13:24 +0200 Subject: [PATCH 3/6] Add `absolutePath` type. --- fragmentation/resolver.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/fragmentation/resolver.go b/fragmentation/resolver.go index e4abe27d..2f6aee54 100644 --- a/fragmentation/resolver.go +++ b/fragmentation/resolver.go @@ -40,8 +40,11 @@ type fragmentedFile struct { fragments map[string]Fragment } +// absolutePath is a resolved absolute filesystem path. +type absolutePath string + // resolverCache stores source fragmentations already resolved during the current run. -var resolverCache = newCache[string, fragmentedFile]( +var resolverCache = newCache[absolutePath, fragmentedFile]( resolverCacheLimit, loadSourceFragments, ) @@ -78,7 +81,7 @@ func ResolveContent( fragment, found := content.fragments[fragmentName] if !found { - codeFileReference := logging.FileReference(source) + codeFileReference := logging.FileReference(string(source)) slog.Info(missingFragmentLogMessage(fragmentName, source)) return nil, fmt.Errorf("fragment `%s` from code file `%s` not found", @@ -89,8 +92,8 @@ func ResolveContent( } // missingFragmentLogMessage describes a missing fragment without exposing internal names. -func missingFragmentLogMessage(fragmentName string, sourcePath string) string { - sourceReference := logging.FileReference(sourcePath) +func missingFragmentLogMessage(fragmentName string, sourcePath absolutePath) string { + sourceReference := logging.FileReference(string(sourcePath)) if fragmentName == DefaultFragmentName { return fmt.Sprintf("Could not load source file `%s`.", sourceReference) } @@ -105,7 +108,7 @@ func ResolveCodeFileReference(codePath string, config config.Configuration) (str return "", err } if found { - return logging.FileReference(source), nil + return logging.FileReference(string(source)), nil } return codeFileReference(codePath, config) @@ -117,7 +120,7 @@ func ClearResolverCache() { } // resolveSource resolves the user-facing code path to the source file. -func resolveSource(codePath string, config config.Configuration) (string, bool, error) { +func resolveSource(codePath string, config config.Configuration) (absolutePath, bool, error) { codeRootName, relativePath, named := splitNamedPath(codePath) for _, root := range config.CodeRoots { if named && strings.TrimSpace(root.Name) != codeRootName { @@ -128,7 +131,7 @@ func resolveSource(codePath string, config config.Configuration) (string, bool, if err != nil { return "", false, err } - exists, err := files.IsFileExist(source) + exists, err := files.IsFileExist(string(source)) if err != nil { return "", false, err } @@ -164,23 +167,23 @@ func splitNamedPath(codePath string) (string, string, bool) { } // sourceFromRoot builds an absolute source path from a code root and a relative path. -func sourceFromRoot(root _type.NamedPath, relativePath string) (string, error) { +func sourceFromRoot(root _type.NamedPath, relativePath string) (absolutePath, error) { rootAbs, err := filepath.Abs(root.Path) if err != nil { return "", err } - return filepath.Join(rootAbs, filepath.FromSlash(relativePath)), nil + return absolutePath(filepath.Join(rootAbs, filepath.FromSlash(relativePath))), nil } // cachedSourceFragments returns cached source fragmentation for an absolute source path. -func cachedSourceFragments(source string) (fragmentedFile, error) { +func cachedSourceFragments(source absolutePath) (fragmentedFile, error) { return resolverCache.get(source) } // loadSourceFragments reads and fragments the source file when it is not already cached. -func loadSourceFragments(source string) (fragmentedFile, error) { - fragmentation, err := NewFragmentation(source) +func loadSourceFragments(source absolutePath) (fragmentedFile, error) { + fragmentation, err := NewFragmentation(string(source)) if err != nil { return fragmentedFile{}, err } @@ -239,10 +242,10 @@ func codeFileReference(codePath string, config config.Configuration) (string, er return "", err } if named { - return fmt.Sprintf("%s (%s)", codePath, logging.FileReference(source)), nil + return fmt.Sprintf("%s (%s)", codePath, logging.FileReference(string(source))), nil } if len(config.CodeRoots) == 1 { - return logging.FileReference(source), nil + return logging.FileReference(string(source)), nil } } From 4d2b9023b67c2f5b78be7ff02839e44228e6f40c Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Tue, 23 Jun 2026 12:17:45 +0200 Subject: [PATCH 4/6] Improve grammar. --- fragmentation/encoding.go | 4 +--- fragmentation/fragmentation.go | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fragmentation/encoding.go b/fragmentation/encoding.go index 64d5cbd4..c0bd2014 100644 --- a/fragmentation/encoding.go +++ b/fragmentation/encoding.go @@ -23,12 +23,10 @@ import ( "unicode/utf8" ) -var errUnsupportedEncoding = errors.New("unsupported source encoding: expected UTF-8") - // validateTextEncoding reports whether source content uses UTF-8 text encoding. func validateTextEncoding(content []byte) error { if !utf8.Valid(content) { - return errUnsupportedEncoding + return errors.New("unsupported source encoding: expected UTF-8") } return nil diff --git a/fragmentation/fragmentation.go b/fragmentation/fragmentation.go index ed29c9c0..066104dc 100644 --- a/fragmentation/fragmentation.go +++ b/fragmentation/fragmentation.go @@ -50,7 +50,8 @@ const NamedPathPrefix = "$" // // CodeFile — a full path of a file to fragment. type Fragmentation struct { - CodeFile string + CodeFile string + // fragmentBuilders collects fragment partitions by name while the source file is scanned. fragmentBuilders map[string]*FragmentBuilder } From de09cf08ede3e1d06462e566abbd0b40eb999083 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Tue, 23 Jun 2026 12:27:05 +0200 Subject: [PATCH 5/6] Make `Fragmentation.CodeFile` private. --- fragmentation/fragmentation.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fragmentation/fragmentation.go b/fragmentation/fragmentation.go index 066104dc..b16f7081 100644 --- a/fragmentation/fragmentation.go +++ b/fragmentation/fragmentation.go @@ -47,10 +47,9 @@ import ( const NamedPathPrefix = "$" // Fragmentation splits the given file into fragments. -// -// CodeFile — a full path of a file to fragment. type Fragmentation struct { - CodeFile string + // codeFile is the absolute path of the source file being fragmented. + codeFile string // fragmentBuilders collects fragment partitions by name while the source file is scanned. fragmentBuilders map[string]*FragmentBuilder } @@ -65,7 +64,7 @@ func NewFragmentation(codeFile string) (Fragmentation, error) { } return Fragmentation{ - CodeFile: absoluteCodeFile, + codeFile: absoluteCodeFile, fragmentBuilders: make(map[string]*FragmentBuilder), }, nil } @@ -77,7 +76,7 @@ func NewFragmentation(codeFile string) (Fragmentation, error) { func (f Fragmentation) DoFragmentation() ([]string, map[string]Fragment, error) { var contentToRender []string - content, err := os.ReadFile(f.CodeFile) + content, err := os.ReadFile(f.codeFile) if err != nil { return nil, nil, err } @@ -94,7 +93,7 @@ func (f Fragmentation) DoFragmentation() ([]string, map[string]Fragment, error) if err != nil { return nil, nil, fmt.Errorf( "failed to do fragmentation on file `file://%s:%d`: %w", - f.CodeFile, lineNumber, err, + f.codeFile, lineNumber, err, ) } } @@ -156,7 +155,7 @@ func (f Fragmentation) parseStartDocFragments(docFragments []string, cursor int) fragment, exists := f.fragmentBuilders[fragmentName] if !exists { builder := FragmentBuilder{ - CodeFilePath: f.CodeFile, + CodeFilePath: f.codeFile, Name: fragmentName, } f.fragmentBuilders[fragmentName] = &builder @@ -180,7 +179,7 @@ func (f Fragmentation) parseEndDocFragments(endDocFragments []string, cursor int } } else { return fmt.Errorf("cannot end the fragment `%s` of the file `file://%s` as it wasn't started", - fragmentName, f.CodeFile) + fragmentName, f.codeFile) } } From 5f411e2024a19d41e37d4daa2746e9d6c615eca1 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Tue, 23 Jun 2026 12:47:03 +0200 Subject: [PATCH 6/6] Extract `unsupportedEncodingError`. --- fragmentation/encoding.go | 11 +++++++++-- fragmentation/resolver.go | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/fragmentation/encoding.go b/fragmentation/encoding.go index c0bd2014..1b984fb3 100644 --- a/fragmentation/encoding.go +++ b/fragmentation/encoding.go @@ -19,14 +19,21 @@ package fragmentation import ( - "errors" "unicode/utf8" ) +// unsupportedEncodingError indicates that source content is not valid UTF-8. +type unsupportedEncodingError struct{} + +// Error describes the required source encoding. +func (*unsupportedEncodingError) Error() string { + return "unsupported source encoding: expected UTF-8" +} + // validateTextEncoding reports whether source content uses UTF-8 text encoding. func validateTextEncoding(content []byte) error { if !utf8.Valid(content) { - return errors.New("unsupported source encoding: expected UTF-8") + return &unsupportedEncodingError{} } return nil diff --git a/fragmentation/resolver.go b/fragmentation/resolver.go index 2f6aee54..6b21f43a 100644 --- a/fragmentation/resolver.go +++ b/fragmentation/resolver.go @@ -140,7 +140,8 @@ func resolveSource(codePath string, config config.Configuration) (absolutePath, } _, err = cachedSourceFragments(source) - if errors.Is(err, errUnsupportedEncoding) { + var encodingError *unsupportedEncodingError + if errors.As(err, &encodingError) { continue } if err != nil {