implement pdf clean up

Author: aliamerj

.github/workflows/go.yml

@@ -17,11 +17,24 @@ jobs:
       with:
         go-version: '1.22.1'
 
-    - name: Build
-      run: go build -v ./...
-
-    - name: Clean
-      run: go clean -cache -testcache -modcache
-
+    - name: Install pkg-config and Dependencies
+      run: |
+          sudo apt-get update
+          sudo apt-get install -y pkg-config
+   
+    - name: Set up PDFium library (Linux)
+      run: |
+          # Download the prebuilt PDFium binary for Linux (x64)
+          sudo curl -L "https://github.com/bblanchon/pdfium-binaries/releases/download/chromium%2F${PDFIUM_VERSION}/pdfium-linux-x64.tgz" -o pdfium-linux-x64.tgz
+          sudo mkdir -p /opt/pdfium
+          sudo tar -C /opt/pdfium -xvf pdfium-linux-x64.tgz
+          sudo rm pdfium-linux-x64.tgz
+          # Copy the provided pkg-config file (ensure this file exists in your repo)
+          sudo cp ./.github/workflows/pdfium.pc /opt/pdfium/pdfium.pc
+    
     - name: Test
+      env:
+        RUNNING_TESTS: "true"
+        LD_LIBRARY_PATH: "/opt/pdfium/lib"
+        PKG_CONFIG_PATH: "/opt/pdfium"
       run: go test -v ./...

cmd/dcup/main.go

@@ -8,6 +8,7 @@ import (
 	"github.com/Dcup-dev/Dcup-lib/internal/html"
 	"github.com/Dcup-dev/Dcup-lib/internal/json"
 	"github.com/Dcup-dev/Dcup-lib/internal/md"
+	"github.com/Dcup-dev/Dcup-lib/internal/pdf"
 	"github.com/Dcup-dev/Dcup-lib/internal/pptx"
 	"github.com/Dcup-dev/Dcup-lib/internal/xlsx"
 )
@@ -20,6 +21,7 @@ type Dcup struct {
 	Markdown   *md.Client
 	PowerPoint *pptx.Client
 	Sheet      *xlsx.Client
+	Pdf        *pdf.Client
 }
 
 // Config is the user-facing configuration struct.
@@ -76,5 +78,6 @@ func Init(config Config) (*Dcup, error) {
 		Markdown:   md.NewClient(config),
 		PowerPoint: pptx.NewClient(config),
 		Sheet:      xlsx.NewClient(config),
+		Pdf:        pdf.NewClient(config),
 	}, nil
 }

examples/main.go

@@ -30,11 +30,12 @@ func main() {
 	}
 
 	schema := map[string]interface{}{
-		"title":  "string",
-		"auther": "string",
+		"user_name": "string",
+		"email":     "string",
+		"address":   "string",
 	}
 
-	res, err := client.Html.CleanUrl("https://www.kelche.co/blog/go/golang-environment-variables/", schema)
+	res, err := client.Pdf.CleanUlr("https://www.wmaccess.com/downloads/sample-invoice.pdf", schema)
 	if err != nil {
 		fmt.Println("Error:", err)
 		return

go.mod

@@ -3,7 +3,24 @@ module github.com/Dcup-dev/Dcup-lib
 go 1.22.1
 
 require (
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-hclog v1.6.3 // indirect
+	github.com/hashicorp/go-plugin v1.6.1 // indirect
+	github.com/hashicorp/yamux v0.1.1 // indirect
 	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/jolestar/go-commons-pool/v2 v2.1.2 // indirect
+	github.com/klippa-app/go-pdfium v1.13.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect
+	github.com/oklog/run v1.0.0 // indirect
 	github.com/thedatashed/xlsxreader v1.2.8 // indirect
 	golang.org/x/net v0.35.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/grpc v1.58.3 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
 )

go.sum

@@ -1,10 +1,63 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
+github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-plugin v1.6.1 h1:P7MR2UP6gNKGPp+y7EZw2kOiq4IR9WiqLvp0XOsVdwI=
+github.com/hashicorp/go-plugin v1.6.1/go.mod h1:XPHFku2tFo3o3QKFgSYo+cghcUhw1NA1hZyMK0PWAw0=
+github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
+github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/jolestar/go-commons-pool/v2 v2.1.2 h1:E+XGo58F23t7HtZiC/W6jzO2Ux2IccSH/yx4nD+J1CM=
+github.com/jolestar/go-commons-pool/v2 v2.1.2/go.mod h1:r4NYccrkS5UqP1YQI1COyTZ9UjPJAAGTUxzcsK1kqhY=
+github.com/klippa-app/go-pdfium v1.13.0 h1:Ow9+cPhhcmKdxp3GCwds79zWyOZHyJIJGGt70R/nUWc=
+github.com/klippa-app/go-pdfium v1.13.0/go.mod h1:eVVeeXJkk+W6KLaJy4fizsERYUaMqJulSXD9JQjx+8s=
+github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg=
+github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
+github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/thedatashed/xlsxreader v1.2.8 h1:8aGbkXIPEThQbA8KzUZqIa4v4oqFrJFKLQ36vWePI5U=
 github.com/thedatashed/xlsxreader v1.2.8/go.mod h1:wZyb/2xF1+rkZ2ujhC72tuuOWBY574QvcXHFls+5AXc=
 golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
 golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
+google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
+google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/pdf/clean.go

@@ -0,0 +1,77 @@
+package pdf
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/Dcup-dev/Dcup-lib/pdfium/renderer"
+	"github.com/klippa-app/go-pdfium/requests"
+)
+
+func clean(pdfBytes []byte) (string, error) {
+	var instance = renderer.Instance
+	// Open the PDF using PDFium
+	doc, err := instance.OpenDocument(&requests.OpenDocument{
+		File: &pdfBytes,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to open PDF document: %v", err)
+	}
+	defer instance.FPDF_CloseDocument(&requests.FPDF_CloseDocument{
+		Document: doc.Document,
+	})
+
+	// Get the page count
+	pageCount, err := instance.FPDF_GetPageCount(&requests.FPDF_GetPageCount{
+		Document: doc.Document,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to get page count: %v", err)
+	}
+
+	// Process pages concurrently
+	var allText strings.Builder
+	type PageContent struct {
+		content string
+		index   int
+		err     error
+	}
+
+	pagesChan := make(chan PageContent, pageCount.PageCount)
+
+	var pageWg sync.WaitGroup
+	for pageIndex := 0; pageIndex < pageCount.PageCount; pageIndex++ {
+		pageWg.Add(1)
+		go func(pageIndex int) {
+			defer pageWg.Done()
+			text, err := processPage(instance, doc.Document, pageIndex)
+			pagesChan <- PageContent{
+				content: text,
+				index:   pageIndex,
+				err:     err,
+			}
+		}(pageIndex)
+	}
+
+	// Wait for all goroutines to complete
+	go func() {
+		pageWg.Wait()
+		close(pagesChan)
+	}()
+
+	// Collect results and errors
+	pages := make([]PageContent, pageCount.PageCount)
+	for page := range pagesChan {
+		if page.err != nil {
+			return "", page.err
+		}
+		pages[page.index] = page
+	}
+
+	for _, page := range pages {
+		allText.WriteString(page.content + "\n")
+	}
+
+	return allText.String(), nil
+}

internal/pdf/client.go

@@ -0,0 +1,11 @@
+package pdf
+
+import "github.com/Dcup-dev/Dcup-lib/internal/core"
+
+type Client struct {
+	config core.ConfigProvider
+}
+
+func NewClient(config core.ConfigProvider) *Client {
+	return &Client{config: config}
+}

internal/pdf/file_cleaner.go

@@ -0,0 +1,31 @@
+package pdf
+
+import (
+	"bytes"
+	"fmt"
+	"mime/multipart"
+
+	"github.com/Dcup-dev/Dcup-lib/internal/core"
+)
+
+func (c Client) CleanFile(file multipart.FileHeader, schema map[string]interface{}) (map[string]interface{}, error) {
+
+	f, err := file.Open()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %s", file.Filename)
+	}
+	defer f.Close()
+
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(f); err != nil {
+		return nil, fmt.Errorf("failed to read uploaded file: %v", err)
+	}
+
+	pdfBytes := buf.Bytes()
+	pdfText, err := clean(pdfBytes)
+
+	if len(pdfText) == 0 {
+		return nil, fmt.Errorf("no text found in the file: %s", file.Filename)
+	}
+	return core.DataProcessing(c.config, core.CleanTextWithPreservation(pdfText), schema)
+}

internal/pdf/page_processer.go

@@ -0,0 +1,56 @@
+package pdf
+
+import (
+	"fmt"
+
+	"github.com/Dcup-dev/Dcup-lib/internal/core"
+	"github.com/klippa-app/go-pdfium"
+	"github.com/klippa-app/go-pdfium/references"
+	"github.com/klippa-app/go-pdfium/requests"
+)
+
+func processPage(instance pdfium.Pdfium, doc references.FPDF_DOCUMENT, pageIndex int) (string, error) {
+	// Load the page
+	page, err := instance.FPDF_LoadPage(&requests.FPDF_LoadPage{
+		Document: doc,
+		Index:    pageIndex,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to load page: %v", err)
+	}
+	defer instance.FPDF_ClosePage(&requests.FPDF_ClosePage{
+		Page: page.Page,
+	})
+
+	// Load text from the page
+	txt, err := instance.FPDFText_LoadPage(&requests.FPDFText_LoadPage{
+		Page: requests.Page{
+			ByReference: &page.Page,
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to load text page: %v", err)
+	}
+	defer instance.FPDFText_ClosePage(&requests.FPDFText_ClosePage{
+		TextPage: txt.TextPage,
+	})
+
+	// Count characters
+	countChar, err := instance.FPDFText_CountChars(&requests.FPDFText_CountChars{
+		TextPage: txt.TextPage,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to count characters: %v", err)
+	}
+
+	// Extract text
+	text, err := instance.FPDFText_GetText(&requests.FPDFText_GetText{
+		TextPage: txt.TextPage,
+		Count:    countChar.Count,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to extract text: %v", err)
+	}
+
+	return core.CleanTextWithPreservation(text.Text), nil
+}

internal/pdf/url_cleaner.go

@@ -0,0 +1,41 @@
+package pdf
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/Dcup-dev/Dcup-lib/internal/core"
+)
+
+func (c Client) CleanUlr(url string, schema map[string]interface{}) (map[string]interface{}, error) {
+	client := http.Client{
+		Timeout: 10 * time.Second,
+	}
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch pdf from URL: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Check if the URL returned a valid response
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to fetch pdf: URL returned status code %d", resp.StatusCode)
+	}
+
+	// Read the content into a buffer
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(resp.Body); err != nil {
+		return nil, fmt.Errorf("failed to read pdf content from URL: %v", err)
+	}
+
+	// Parse the pdf data
+	pdfBytes := buf.Bytes()
+	pdfText, err := clean(pdfBytes)
+
+	if len(pdfText) == 0 {
+		return nil, fmt.Errorf("no text found in the pdf")
+	}
+return core.DataProcessing(c.config,core.CleanTextWithPreservation(pdfText), schema)
+}