diff --git a/embedding/embedding_test.go b/embedding/embedding_test.go index 1a45a91f..8020ce14 100644 --- a/embedding/embedding_test.go +++ b/embedding/embedding_test.go @@ -84,18 +84,6 @@ var _ = Describe("Embedding", func() { Expect(processor.IsUpToDate()).Should(BeTrue()) }) - It("should have error as it has invalid transition map", func() { - docPath := fmt.Sprintf("%s/split-lines.md", config.DocumentationRoot) - - falseTransitions := parsing.TransitionMap{ - parsing.Start: {parsing.Finish, parsing.EmbedInstruction, parsing.RegularLine}, - parsing.RegularLine: {parsing.CodeFenceEnd}, - } - - falseProcessor := newProcessorWithTransitions(docPath, config, falseTransitions) - Expect(falseProcessor.Embed()).Error().Should(HaveOccurred()) - }) - It("should successfully embed with multi lined tag", func() { docPath := fmt.Sprintf("%s/multi-lined-tag.md", config.DocumentationRoot) processor := newProcessor(docPath, config) @@ -335,9 +323,10 @@ var _ = Describe("Embedding", func() { config.DocumentationRoot) processor := newProcessor(docPath, config) - _, err := embedding.EmbedAll(config) + result, err := embedding.EmbedAll(config) Expect(err).ShouldNot(HaveOccurred()) + Expect(result.UpdatedTargetFiles).Should(ContainElement(docPath)) Expect(processor.IsUpToDate()).Should(BeTrue()) }) @@ -376,18 +365,6 @@ func newProcessor( return processor } -func newProcessorWithTransitions( - docPath string, - config configuration.Configuration, - transitions parsing.TransitionMap, -) embedding.Processor { - processor, err := embedding.NewProcessorWithTransitions(docPath, config, transitions) - - Expect(err).ShouldNot(HaveOccurred()) - - return processor -} - func copyDirRecursive(sourceDirPath string, targetDirPath string) { info, err := os.Stat(sourceDirPath) if err != nil { diff --git a/embedding/orchestration.go b/embedding/orchestration.go new file mode 100644 index 00000000..a9e0108d --- /dev/null +++ b/embedding/orchestration.go @@ -0,0 +1,241 @@ +// Copyright 2026, TeamDev. All rights reserved. +// +// Redistribution and use in source and/or binary forms, with or without +// modification, must retain the above copyright notice and the following +// disclaimer. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package embedding + +import ( + "errors" + "fmt" + "log/slog" + "path/filepath" + "strings" + + "embed-code/embed-code-go/configuration" + "embed-code/embed-code-go/embedding/parsing" + "embed-code/embed-code-go/logging" + + "github.com/bmatcuk/doublestar/v4" +) + +// EmbedAllResult contains the result of an EmbedAll operation. +// +// TotalEmbeddings is the total number of embeddings found in the target documentation files. +// +// UpdatedTargetFiles is the list of updated target documentation files. +type EmbedAllResult struct { + TotalEmbeddings int + UpdatedTargetFiles []string +} + +// processorHandler applies one processing mode to a discovered documentation file. +type processorHandler func(docFilePath string, processor Processor) error + +// EmbedAll processes embedding for multiple documentation files based on provided config. +// +// Iterates over patterns in the configuration, finds documentation files matching those patterns, +// creates a Processor for each file, and embeds code fragments in them. +// +// config — a configuration for embedding. +func EmbedAll(config configuration.Configuration) (EmbedAllResult, error) { + totalEmbeddings := 0 + var updatedTargetFiles []string + requiredDocPaths, embeddingErrors := processRequiredDocs(config, func( + _ string, + processor Processor, + ) error { + context, err := processor.Embed() + if err != nil { + return err + } + totalEmbeddings += context.EmbeddingsCount() + if context.IsContentChanged() { + updatedTargetFiles = append(updatedTargetFiles, context.MarkdownFilePath) + } + + return nil + }) + if len(embeddingErrors) > 0 { + return EmbedAllResult{}, errors.Join(embeddingErrors...) + } + if totalEmbeddings > 0 { + slog.Info( + fmt.Sprintf( + "Processed %d documentation file(s) with %d embedding(s) in `%s`%s.", + len(requiredDocPaths), totalEmbeddings, + logging.FileReference(config.DocumentationRoot), + configNameLabel(config), + ), + ) + } else { + slog.Warn( + fmt.Sprintf("No embedding instructions were found in documentation folder `%s`%s.", + logging.FileReference(config.DocumentationRoot), configNameLabel(config)), + ) + } + + return EmbedAllResult{ + TotalEmbeddings: totalEmbeddings, + UpdatedTargetFiles: updatedTargetFiles, + }, nil +} + +// configNameLabel formats a configuration name for summary log messages. +// +// A non-empty label starts with a space so callers can append it directly. +func configNameLabel(config configuration.Configuration) string { + if config.Name == "" { + return "" + } + + return fmt.Sprintf(" for `%s` embedding setup", config.Name) +} + +// CheckUpToDate returns documentation files that are not up-to-date with code files. +// +// config — a configuration for embedding. +func CheckUpToDate(config configuration.Configuration) ([]string, error) { + changedFiles, checkErrors := findChangedFiles(config) + if len(checkErrors) > 0 { + return nil, errors.Join(checkErrors...) + } + + return changedFiles, nil +} + +// findChangedFiles returns documentation files that are not up-to-date with their code files. +// +// config — a configuration for embedding. +func findChangedFiles(config configuration.Configuration) ([]string, []error) { + var changedFiles []string + _, checkErrors := processRequiredDocs(config, func( + docFilePath string, + processor Processor, + ) error { + upToDate, err := processor.isUpToDate() + if err != nil { + return err + } + if !upToDate { + changedFiles = append(changedFiles, docFilePath) + } + + return nil + }) + + return changedFiles, checkErrors +} + +// processRequiredDocs applies a processing handler to every documentation file in config. +func processRequiredDocs( + config configuration.Configuration, + handle processorHandler, +) ([]string, []error) { + requiredDocPaths, err := requiredDocs(config) + if err != nil { + return nil, []error{err} + } + + var processingErrors []error + for _, doc := range requiredDocPaths { + processor := newProcessor(doc, config, parsing.Transitions, requiredDocPaths) + if err := handle(doc, processor); err != nil { + processingErrors = append(processingErrors, err) + } + } + + return requiredDocPaths, processingErrors +} + +// requiredDocs returns documentation files matched by includes minus excludes. +func requiredDocs(config configuration.Configuration) ([]string, error) { + documentationRoot := config.DocumentationRoot + includedPatterns := config.DocIncludes + excludedPatterns := config.DocExcludes + + includedDocs, err := getFilesByPatterns(documentationRoot, includedPatterns) + if err != nil { + return nil, err + } + + excludedDocs, err := getFilesByPatterns(documentationRoot, excludedPatterns) + if err != nil { + return nil, err + } + if len(excludedDocs) == 0 { + slog.Info(fmt.Sprintf( + "Found %d documentation file(s) from `%s` matching include pattern(s) %s.", + len(includedDocs), logging.FileReference(documentationRoot), + patternsLabel(includedPatterns), + )) + + return includedDocs, nil + } + + result := removeElements(includedDocs, excludedDocs) + slog.Info(fmt.Sprintf( + "Found %d documentation file(s) from `%s` matching include pattern(s) %s "+ + "and exclude pattern(s) %s.", + len(result), logging.FileReference(documentationRoot), patternsLabel(includedPatterns), + patternsLabel(excludedPatterns), + )) + + return result, nil +} + +// patternsLabel formats glob patterns for human-readable log messages. +func patternsLabel(patterns []string) string { + if len(patterns) == 0 { + return "nothing" + } + + return "`" + strings.Join(patterns, "`, `") + "`" +} + +// getFilesByPatterns expands documentation glob patterns relative to the given root. +func getFilesByPatterns(root string, patterns []string) ([]string, error) { + var result []string + for _, pattern := range patterns { + globString := filepath.Join(root, filepath.FromSlash(pattern)) + matches, err := doublestar.FilepathGlob(globString) + if err != nil { + return nil, err + } + for _, match := range matches { + result = append(result, filepath.ToSlash(match)) + } + } + + return result, nil +} + +// removeElements returns values from first that are not present in second. +func removeElements(first, second []string) []string { + secondMap := make(map[string]struct{}) + for _, value := range second { + secondMap[value] = struct{}{} + } + + var result []string + for _, value := range first { + if _, exists := secondMap[value]; !exists { + result = append(result, value) + } + } + + return result +} diff --git a/embedding/parsing/context.go b/embedding/parsing/context.go index c09c5dee..c037dfea 100644 --- a/embedding/parsing/context.go +++ b/embedding/parsing/context.go @@ -66,24 +66,16 @@ func (c *Context) EmbeddingsCount() int { return len(c.embeddings) } -// EmbeddingContext contains the information about the position in the source and the -// resulting Markdown files. +// EmbeddingContext contains an instruction and its position in the source Markdown file. // -// embeddingInstruction - an Instruction, containing all the needed embedding information. +// SourceStartIndex is the zero-based index of the first line after the opening code fence. // -// SourceStartIndex - an index of the StartState line in the original markdown file. -// -// SourceEndIndex - an index of the end line in the original markdown file. -// -// resultStartIndex - an index of the StartState line in the result markdown file. -// -// resultEndIndex - an index of the end line in the result markdown file. +// SourceEndIndex is the zero-based index of the closing code fence and the exclusive slice bound. type EmbeddingContext struct { + // embeddingInstruction contains the embedding parameters. embeddingInstruction Instruction SourceStartIndex int SourceEndIndex int - resultStartIndex int - resultEndIndex int } // NewContext Creates and returns a new Context struct with initial values for markdownFile, source, @@ -143,20 +135,6 @@ func (c *Context) IsContentChanged() bool { return false } -// FindChangedEmbeddings returns a list of changed embeddings. -func (c *Context) FindChangedEmbeddings() []Instruction { - var changedEmbeddings []Instruction - for _, embedding := range c.embeddings { - sourceContent := c.readEmbeddingSource(embedding) - resultContent := c.readEmbeddingResult(embedding) - if !isStringSlicesEqual(sourceContent, resultContent) { - changedEmbeddings = append(changedEmbeddings, embedding.embeddingInstruction) - } - } - - return changedEmbeddings -} - // IsContainsEmbedding reports whether the doc file contains an embedding. func (c *Context) IsContainsEmbedding() bool { return c.fileContainsEmbedding @@ -196,7 +174,6 @@ func (c *Context) StartEmbedding(instruction Instruction) { func (c *Context) FinishEmbedding() { currentEmbedding := c.CurrentEmbedding() currentEmbedding.SourceEndIndex = c.lineIndex - currentEmbedding.resultEndIndex = len(c.Result) c.EmbeddingInstruction = nil } @@ -206,7 +183,6 @@ func (c *Context) SetCodeStart() { if c.fileContainsEmbedding { lastEmbedding := c.CurrentEmbedding() lastEmbedding.SourceStartIndex = c.lineIndex - lastEmbedding.resultStartIndex = len(c.Result) } } @@ -236,11 +212,6 @@ func (c *Context) readEmbeddingSource(context EmbeddingContext) []string { return c.source[context.SourceStartIndex:context.SourceEndIndex] } -// readEmbeddingResult returns generated Markdown lines for one embedding. -func (c *Context) readEmbeddingResult(context EmbeddingContext) []string { - return c.Result[context.resultStartIndex:context.resultEndIndex] -} - // readLines returns the content of a file placed at filepath as a list of strings. func readLines(filepath string) ([]string, error) { bytes, err := os.ReadFile(filepath) @@ -252,16 +223,3 @@ func readLines(filepath string) ([]string, error) { return lines, nil } - -func isStringSlicesEqual(first, second []string) bool { - if len(first) != len(second) { - return false - } - for i := range first { - if first[i] != second[i] { - return false - } - } - - return true -} diff --git a/embedding/processor.go b/embedding/processor.go index 6155558a..0f385c48 100644 --- a/embedding/processor.go +++ b/embedding/processor.go @@ -23,7 +23,6 @@ import ( "fmt" "log/slog" "os" - "path/filepath" "slices" "strings" @@ -31,38 +30,22 @@ import ( "embed-code/embed-code-go/embedding/parsing" "embed-code/embed-code-go/files" "embed-code/embed-code-go/logging" - - "github.com/bmatcuk/doublestar/v4" ) -// Processor entity processes a single documentation file and embeds code snippets -// into it based on the provided configuration. -// -// DocFilePath — the path to the documentation file. -// -// Config — a configuration for embedding. +// Processor processes a single documentation file using the provided embedding configuration. type Processor struct { - DocFilePath string - Config configuration.Configuration - TransitionsMap parsing.TransitionMap - requiredDocPaths []string -} + // DocFilePath is the path to the documentation file. + DocFilePath string -// EmbedAllResult is result of the EmbedAll method. -// -// TargetFiles is the list of target documentation files. -// -// TotalEmbeddings is the total number of embeddings found in the target documentation files. -// -// UpdatedTargetFiles is the list of updated target documentation files. -type EmbedAllResult struct { - TargetFiles []string - TotalEmbeddings int - UpdatedTargetFiles []string -} + // Config contains the embedding settings. + Config configuration.Configuration + + // TransitionsMap defines valid parser state transitions. + TransitionsMap parsing.TransitionMap -// processorHandler applies one processing mode to a configured documentation processor. -type processorHandler func(processor Processor) error + // requiredDocPaths contains documentation files included by the configuration. + requiredDocPaths []string +} // NewProcessor creates and returns new Processor with given docFile and config. func NewProcessor(docFile string, config configuration.Configuration) (Processor, error) { @@ -74,18 +57,6 @@ func NewProcessor(docFile string, config configuration.Configuration) (Processor return newProcessor(docFile, config, parsing.Transitions, requiredDocPaths), nil } -// NewProcessorWithTransitions Creates and returns new Processor with given docFile, config -// and transitions. -func NewProcessorWithTransitions(docFile string, config configuration.Configuration, - transitions parsing.TransitionMap) (Processor, error) { - requiredDocPaths, err := requiredDocs(config) - if err != nil { - return Processor{}, err - } - - return newProcessor(docFile, config, transitions, requiredDocPaths), nil -} - // newProcessor creates a Processor with a precomputed documentation file list. func newProcessor( docFile string, @@ -137,22 +108,6 @@ func (p Processor) Embed() (*parsing.Context, error) { return &context, nil } -// FindChangedEmbeddings Returns the list of EmbeddingInstruction that are changed in the -// markdown file. -// -// If any problems during the embedding construction faced, an error is returned. -func (p Processor) FindChangedEmbeddings() ([]parsing.Instruction, error) { - if !slices.Contains(p.requiredDocPaths, p.DocFilePath) { - return nil, nil - } - context, err := p.fillEmbeddingContext() - if err != nil { - return nil, err - } - - return context.FindChangedEmbeddings(), nil -} - // IsUpToDate reports whether the embedding of the target markdown is up-to-date with the code file. func (p Processor) IsUpToDate() bool { upToDate, err := p.isUpToDate() @@ -188,74 +143,6 @@ func (p Processor) isUpToDate() (bool, error) { return upToDate, nil } -// EmbedAll processes embedding for multiple documentation files based on provided config. -// -// Iterates over patterns in the configuration, finds documentation files matching those patterns, -// creates an EmbeddingProcessor for each file, and embeds code fragments in them. -// -// config — a configuration for embedding. -func EmbedAll(config configuration.Configuration) (EmbedAllResult, error) { - totalEmbeddings := 0 - var updatedTargetFiles []string - requiredDocPaths, embeddingErrors := processRequiredDocs(config, func(processor Processor) error { - context, err := processor.Embed() - if err != nil { - return err - } - totalEmbeddings += context.EmbeddingsCount() - if context.IsContentChanged() { - updatedTargetFiles = append(updatedTargetFiles, processor.DocFilePath) - } - - return nil - }) - if len(embeddingErrors) > 0 { - return EmbedAllResult{}, errors.Join(embeddingErrors...) - } - if totalEmbeddings > 0 { - slog.Info( - fmt.Sprintf( - "Processed %d documentation file(s) with %d embedding(s) in `%s`%s.", - len(requiredDocPaths), totalEmbeddings, - logging.FileReference(config.DocumentationRoot), - configNameLabel(config), - ), - ) - } else { - slog.Warn( - fmt.Sprintf("No embedding instructions were found in documentation folder `%s`%s.", - logging.FileReference(config.DocumentationRoot), configNameLabel(config)), - ) - } - - return EmbedAllResult{ - TargetFiles: requiredDocPaths, - TotalEmbeddings: totalEmbeddings, - UpdatedTargetFiles: updatedTargetFiles, - }, nil -} - -// configNameLabel formats a configuration name for summary log messages. -func configNameLabel(config configuration.Configuration) string { - if config.Name == "" { - return "" - } - - return fmt.Sprintf(" for `%s` embedding setup", config.Name) -} - -// CheckUpToDate returns documentation files that are not up-to-date with code files. -// -// config — a configuration for embedding. -func CheckUpToDate(config configuration.Configuration) ([]string, error) { - changedFiles, checkErrors := findChangedFiles(config) - if len(checkErrors) > 0 { - return nil, errors.Join(checkErrors...) - } - - return changedFiles, nil -} - // Iterates through the doc file line by line considering them as a states of an embedding. // Such way, transits from the state to the next possible one until it reaches the end of a file. // By the transition process, fills the parsing.Context accordingly, so it is ready to retrieve @@ -359,123 +246,3 @@ func (p Processor) moveToNextState(state *parsing.State, context *parsing.Contex return false, state, nil } - -// findChangedFiles returns documentation files that are not up-to-date with their code files. -// -// config — a configuration for embedding. -func findChangedFiles(config configuration.Configuration) ([]string, []error) { - var changedFiles []string - _, checkErrors := processRequiredDocs(config, func(processor Processor) error { - upToDate, err := processor.isUpToDate() - if err != nil { - return err - } - if !upToDate { - changedFiles = append(changedFiles, processor.DocFilePath) - } - - return nil - }) - - return changedFiles, checkErrors -} - -// processRequiredDocs applies a processing handler to every documentation file in config. -func processRequiredDocs( - config configuration.Configuration, - handle processorHandler, -) ([]string, []error) { - requiredDocPaths, err := requiredDocs(config) - if err != nil { - return nil, []error{err} - } - - var processingErrors []error - for _, doc := range requiredDocPaths { - processor := newProcessor(doc, config, parsing.Transitions, requiredDocPaths) - if err := handle(processor); err != nil { - processingErrors = append(processingErrors, err) - } - } - - return requiredDocPaths, processingErrors -} - -// requiredDocs returns documentation files matched by includes minus excludes. -func requiredDocs(config configuration.Configuration) ([]string, error) { - documentationRoot := config.DocumentationRoot - includedPatterns := config.DocIncludes - excludedPatterns := config.DocExcludes - - includedDocs, err := getFilesByPatterns(documentationRoot, includedPatterns) - if err != nil { - return nil, err - } - - excludedDocs, err := getFilesByPatterns(documentationRoot, excludedPatterns) - if err != nil { - return nil, err - } - if len(excludedDocs) == 0 { - slog.Info(fmt.Sprintf( - "Found %d documentation file(s) from `%s` matching include pattern(s) %s.", - len(includedDocs), logging.FileReference(documentationRoot), - patternsLabel(includedPatterns), - )) - - return includedDocs, nil - } - - result := removeElements(includedDocs, excludedDocs) - slog.Info(fmt.Sprintf( - "Found %d documentation file(s) from `%s` matching include pattern(s) %s "+ - "and exclude pattern(s) %s.", - len(result), logging.FileReference(documentationRoot), patternsLabel(includedPatterns), - patternsLabel(excludedPatterns), - )) - - return result, nil -} - -// patternsLabel formats glob patterns for human-readable log messages. -func patternsLabel(patterns []string) string { - if len(patterns) == 0 { - return "nothing" - } - - return "`" + strings.Join(patterns, "`, `") + "`" -} - -// getFilesByPatterns expands documentation glob patterns relative to the given root. -func getFilesByPatterns(root string, patterns []string) ([]string, error) { - var result []string - for _, pattern := range patterns { - globString := filepath.Join(root, filepath.FromSlash(pattern)) - matches, err := doublestar.FilepathGlob(globString) - if err != nil { - return nil, err - } - for _, match := range matches { - result = append(result, filepath.ToSlash(match)) - } - } - - return result, nil -} - -// removeElements returns values from first that are not present in second. -func removeElements(first, second []string) []string { - secondMap := make(map[string]struct{}) - for _, value := range second { - secondMap[value] = struct{}{} - } - - var result []string - for _, value := range first { - if _, exists := secondMap[value]; !exists { - result = append(result, value) - } - } - - return result -}