diff --git a/fragmentation/encoding.go b/fragmentation/encoding.go index 9f9d0813..1b984fb3 100644 --- a/fragmentation/encoding.go +++ b/fragmentation/encoding.go @@ -19,37 +19,22 @@ package fragmentation import ( - "os" "unicode/utf8" ) -const lastASCIIchar = 127 +// unsupportedEncodingError indicates that source content is not valid UTF-8. +type unsupportedEncodingError struct{} -// IsEncodedAsText reports whether the file stored at filePath is encoded as a text. -// -// If file encoded in ASCII or UTF-8, it is meant to be a text file. -func IsEncodedAsText(filePath string) (bool, error) { - // Read the entire file into memory. - content, err := os.ReadFile(filePath) - if err != nil { - return false, err - } - - isUTF8Encoded := utf8.Valid(content) - isASCIIEncoded := areASCIIEncoded(content) - - return isUTF8Encoded || isASCIIEncoded, nil +// Error describes the required source encoding. +func (*unsupportedEncodingError) Error() string { + return "unsupported source encoding: expected UTF-8" } -// Reports whether given bytes are ASCII-encoded. -// -// If all the characters fall within the ASCII range (0 to 127), it’s likely an ASCII-encoded file. -func areASCIIEncoded(bytes []byte) bool { - for _, char := range bytes { - if char > byte(lastASCIIchar) { - return false - } +// validateTextEncoding reports whether source content uses UTF-8 text encoding. +func validateTextEncoding(content []byte) error { + if !utf8.Valid(content) { + return &unsupportedEncodingError{} } - return true + return nil } diff --git a/fragmentation/fragmentation.go b/fragmentation/fragmentation.go index b9627b2e..b16f7081 100644 --- a/fragmentation/fragmentation.go +++ b/fragmentation/fragmentation.go @@ -37,60 +37,36 @@ package fragmentation import ( "bufio" - "embed-code/embed-code-go/files" - _type "embed-code/embed-code-go/type" + "bytes" "fmt" "os" "path/filepath" - - config "embed-code/embed-code-go/configuration" ) // NamedPathPrefix the prefix before the named code source. const NamedPathPrefix = "$" // Fragmentation splits the given file into fragments. -// -// Configuration — a configuration for embedding. -// -// SourcesRoot — a named source code path. -// -// CodeFile — a full path of a file to fragment. type Fragmentation struct { - Configuration config.Configuration - SourcesRoot _type.NamedPath - CodeFile string + // codeFile is the absolute path of the source file being fragmented. + codeFile string + // fragmentBuilders collects fragment partitions by name while the source file is scanned. fragmentBuilders map[string]*FragmentBuilder } -// NewFragmentation builds Fragmentation from given codeFileRelative and config. -// -// codeFileRelative — a relative path to a code file to fragment. +// NewFragmentation builds Fragmentation for the given code file. // -// config — a configuration for embedding. -func NewFragmentation( - codeFileRelative string, - codeRoot _type.NamedPath, - config config.Configuration, -) (Fragmentation, error) { - fragmentation := Fragmentation{} - - fragmentation.SourcesRoot = codeRoot - _, err := filepath.Abs(codeRoot.Path) +// codeFile — a relative or absolute path to a code file to fragment. +func NewFragmentation(codeFile string) (Fragmentation, error) { + absoluteCodeFile, err := filepath.Abs(codeFile) if err != nil { return Fragmentation{}, err } - absoluteCodeFile, err := filepath.Abs(codeFileRelative) - if err != nil { - return Fragmentation{}, err - } - fragmentation.CodeFile = absoluteCodeFile - - fragmentation.Configuration = config - fragmentation.fragmentBuilders = make(map[string]*FragmentBuilder) - - return fragmentation, nil + return Fragmentation{ + codeFile: absoluteCodeFile, + fragmentBuilders: make(map[string]*FragmentBuilder), + }, nil } // DoFragmentation splits the file into fragments. @@ -100,16 +76,15 @@ func NewFragmentation( func (f Fragmentation) DoFragmentation() ([]string, map[string]Fragment, error) { var contentToRender []string - file, err := os.Open(f.CodeFile) + content, err := os.ReadFile(f.codeFile) if err != nil { return nil, nil, err } + if err := validateTextEncoding(content); err != nil { + return nil, nil, err + } - defer func(file *os.File) { - err = file.Close() - }(file) - - scanner := bufio.NewScanner(file) + scanner := bufio.NewScanner(bytes.NewReader(content)) lineNumber := 0 for scanner.Scan() { lineNumber++ @@ -118,11 +93,11 @@ func (f Fragmentation) DoFragmentation() ([]string, map[string]Fragment, error) if err != nil { return nil, nil, fmt.Errorf( "failed to do fragmentation on file `file://%s:%d`: %w", - f.CodeFile, lineNumber, err, + f.codeFile, lineNumber, err, ) } } - if err = scanner.Err(); err != nil { + if err := scanner.Err(); err != nil { return nil, nil, err } @@ -135,22 +110,6 @@ func (f Fragmentation) DoFragmentation() ([]string, map[string]Fragment, error) return contentToRender, fragments, nil } -// shouldDoFragmentation reports whether the file is valid to do fragmentation: -// - it exists by the given path -// - it is a file (not a dir) -// - it is textual-encoded. -func shouldDoFragmentation(filePath string) (bool, error) { - exists, err := files.IsFileExist(filePath) - if err != nil { - return false, err - } - if exists { - return IsEncodedAsText(filePath) - } - - return false, nil -} - // Parses a single line of input and performs the following actions: // - identifies fragment start and end markers within given line; // - updates fragmentBuilders based on the markers; @@ -196,7 +155,7 @@ func (f Fragmentation) parseStartDocFragments(docFragments []string, cursor int) fragment, exists := f.fragmentBuilders[fragmentName] if !exists { builder := FragmentBuilder{ - CodeFilePath: f.CodeFile, + CodeFilePath: f.codeFile, Name: fragmentName, } f.fragmentBuilders[fragmentName] = &builder @@ -220,7 +179,7 @@ func (f Fragmentation) parseEndDocFragments(endDocFragments []string, cursor int } } else { return fmt.Errorf("cannot end the fragment `%s` of the file `file://%s` as it wasn't started", - fragmentName, f.CodeFile) + fragmentName, f.codeFile) } } diff --git a/fragmentation/fragmentation_test.go b/fragmentation/fragmentation_test.go index 742aaa47..24d7a6e5 100644 --- a/fragmentation/fragmentation_test.go +++ b/fragmentation/fragmentation_test.go @@ -23,6 +23,8 @@ import ( "embed-code/embed-code-go/fragmentation" _type "embed-code/embed-code-go/type" "fmt" + "os" + "path/filepath" "testing" . "github.com/onsi/ginkgo/v2" @@ -94,6 +96,31 @@ var _ = Describe("Fragmentation", func() { Expect(fragments).Should(HaveKey(fragmentation.DefaultFragmentName)) }) + It("should skip a non-UTF-8 source and use the next code root", func() { + invalidRoot := GinkgoT().TempDir() + validRoot := GinkgoT().TempDir() + fileName := "Example.java" + Expect(os.WriteFile(filepath.Join(invalidRoot, fileName), []byte{0xff}, 0600)).To(Succeed()) + Expect(os.WriteFile( + filepath.Join(validRoot, fileName), + []byte("class Example {}"), + 0600, + )).To(Succeed()) + config.CodeRoots = _type.NamedPathList{ + _type.NamedPath{Path: invalidRoot}, + _type.NamedPath{Path: validRoot}, + } + + content, err := fragmentation.ResolveContent( + fileName, + fragmentation.DefaultFragmentName, + config, + ) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(content).Should(Equal([]string{"class Example {}"})) + }) + It("should fail on an unopened fragment", func() { frag := buildTestFragmentation(unopenedFragmentFileName, config) @@ -221,7 +248,7 @@ func buildTestFragmentation(testFileName string, config configuration.Configuration) fragmentation.Fragmentation { codeRoot := config.CodeRoots[0] testFilePath := fmt.Sprintf("%s/org/example/%s", codeRoot.Path, testFileName) - frag, err := fragmentation.NewFragmentation(testFilePath, codeRoot, config) + frag, err := fragmentation.NewFragmentation(testFilePath) Expect(err).ShouldNot(HaveOccurred()) diff --git a/fragmentation/resolver.go b/fragmentation/resolver.go index a6dff150..6b21f43a 100644 --- a/fragmentation/resolver.go +++ b/fragmentation/resolver.go @@ -19,12 +19,14 @@ package fragmentation import ( + "errors" "fmt" "log/slog" "path/filepath" "strings" config "embed-code/embed-code-go/configuration" + "embed-code/embed-code-go/files" "embed-code/embed-code-go/logging" _type "embed-code/embed-code-go/type" ) @@ -38,8 +40,11 @@ type fragmentedFile struct { fragments map[string]Fragment } +// absolutePath is a resolved absolute filesystem path. +type absolutePath string + // resolverCache stores source fragmentations already resolved during the current run. -var resolverCache = newCache[resolvedPath, fragmentedFile]( +var resolverCache = newCache[absolutePath, fragmentedFile]( resolverCacheLimit, loadSourceFragments, ) @@ -76,8 +81,8 @@ func ResolveContent( fragment, found := content.fragments[fragmentName] if !found { - codeFileReference := logging.FileReference(source.absolutePath) - slog.Info(missingFragmentLogMessage(fragmentName, source.absolutePath)) + codeFileReference := logging.FileReference(string(source)) + slog.Info(missingFragmentLogMessage(fragmentName, source)) return nil, fmt.Errorf("fragment `%s` from code file `%s` not found", fragmentName, codeFileReference) @@ -87,8 +92,8 @@ func ResolveContent( } // missingFragmentLogMessage describes a missing fragment without exposing internal names. -func missingFragmentLogMessage(fragmentName string, sourcePath string) string { - sourceReference := logging.FileReference(sourcePath) +func missingFragmentLogMessage(fragmentName string, sourcePath absolutePath) string { + sourceReference := logging.FileReference(string(sourcePath)) if fragmentName == DefaultFragmentName { return fmt.Sprintf("Could not load source file `%s`.", sourceReference) } @@ -103,7 +108,7 @@ func ResolveCodeFileReference(codePath string, config config.Configuration) (str return "", err } if found { - return logging.FileReference(source.absolutePath), nil + return logging.FileReference(string(source)), nil } return codeFileReference(codePath, config) @@ -114,15 +119,8 @@ func ClearResolverCache() { resolverCache.clear() } -// resolvedPath is a source file path resolved from a user-facing embedding path. -type resolvedPath struct { - root _type.NamedPath - relativePath string - absolutePath string -} - // resolveSource resolves the user-facing code path to the source file. -func resolveSource(codePath string, config config.Configuration) (resolvedPath, bool, error) { +func resolveSource(codePath string, config config.Configuration) (absolutePath, bool, error) { codeRootName, relativePath, named := splitNamedPath(codePath) for _, root := range config.CodeRoots { if named && strings.TrimSpace(root.Name) != codeRootName { @@ -131,20 +129,29 @@ func resolveSource(codePath string, config config.Configuration) (resolvedPath, source, err := sourceFromRoot(root, relativePath) if err != nil { - return resolvedPath{}, false, err + return "", false, err } - shouldFragment, err := shouldDoFragmentation(source.absolutePath) + exists, err := files.IsFileExist(string(source)) if err != nil { - return resolvedPath{}, false, err + return "", false, err + } + if !exists { + continue } - if !shouldFragment { + + _, err = cachedSourceFragments(source) + var encodingError *unsupportedEncodingError + if errors.As(err, &encodingError) { continue } + if err != nil { + return "", false, err + } return source, true, nil } - return resolvedPath{}, false, nil + return "", false, nil } // splitNamedPath separates a named-code-root prefix from a code path. @@ -160,28 +167,24 @@ func splitNamedPath(codePath string) (string, string, bool) { return rootName, relativePath, true } -// sourceFromRoot builds a source path from a code root and a relative path. -func sourceFromRoot(root _type.NamedPath, relativePath string) (resolvedPath, error) { +// sourceFromRoot builds an absolute source path from a code root and a relative path. +func sourceFromRoot(root _type.NamedPath, relativePath string) (absolutePath, error) { rootAbs, err := filepath.Abs(root.Path) if err != nil { - return resolvedPath{}, err + return "", err } - return resolvedPath{ - root: root, - relativePath: filepath.FromSlash(relativePath), - absolutePath: filepath.Join(rootAbs, filepath.FromSlash(relativePath)), - }, nil + return absolutePath(filepath.Join(rootAbs, filepath.FromSlash(relativePath))), nil } -// cachedSourceFragments returns cached source fragmentation for a resolved source file. -func cachedSourceFragments(source resolvedPath) (fragmentedFile, error) { +// cachedSourceFragments returns cached source fragmentation for an absolute source path. +func cachedSourceFragments(source absolutePath) (fragmentedFile, error) { return resolverCache.get(source) } // loadSourceFragments reads and fragments the source file when it is not already cached. -func loadSourceFragments(source resolvedPath) (fragmentedFile, error) { - fragmentation, err := NewFragmentation(source.absolutePath, source.root, config.Configuration{}) +func loadSourceFragments(source absolutePath) (fragmentedFile, error) { + fragmentation, err := NewFragmentation(string(source)) if err != nil { return fragmentedFile{}, err } @@ -240,10 +243,10 @@ func codeFileReference(codePath string, config config.Configuration) (string, er return "", err } if named { - return fmt.Sprintf("%s (%s)", codePath, logging.FileReference(source.absolutePath)), nil + return fmt.Sprintf("%s (%s)", codePath, logging.FileReference(string(source))), nil } if len(config.CodeRoots) == 1 { - return logging.FileReference(source.absolutePath), nil + return logging.FileReference(string(source)), nil } } diff --git a/indent/indent_test.go b/indent/indent_test.go index a9bffd7e..41143c69 100644 --- a/indent/indent_test.go +++ b/indent/indent_test.go @@ -75,5 +75,4 @@ var _ = Describe("Indent", func() { "return;", })) }) - })