diff --git a/.env.example b/.env.example index a86896f17..aabdbd3d3 100644 --- a/.env.example +++ b/.env.example @@ -204,3 +204,9 @@ PLUGIN_RUNTIME_MAX_BUFFER_SIZE=5242880 DIFY_BACKWARDS_INVOCATION_WRITE_TIMEOUT=5000 # dify backwards invocation read timeout in milliseconds DIFY_BACKWARDS_INVOCATION_READ_TIMEOUT=240000 + + +TEMP_DIR = E:\\Temp +PYTHONIOENCODING = utf-8 +LANG=en_US.UTF-8 +LC_ALL=en_US.UTF-8 \ No newline at end of file diff --git a/go.mod b/go.mod index f434cd31f..feff53f3d 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 go.opentelemetry.io/otel/sdk v1.39.0 go.opentelemetry.io/otel/sdk/metric v1.39.0 + go.opentelemetry.io/otel/trace v1.39.0 golang.org/x/tools v0.38.0 gorm.io/driver/mysql v1.5.7 gorm.io/gorm v1.30.0 @@ -146,7 +147,6 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect golang.org/x/mod v0.29.0 // indirect golang.org/x/oauth2 v0.32.0 // indirect diff --git a/internal/core/local_runtime/setup_python_environment.go b/internal/core/local_runtime/setup_python_environment.go index eb5af5cfb..65bb0eeb8 100644 --- a/internal/core/local_runtime/setup_python_environment.go +++ b/internal/core/local_runtime/setup_python_environment.go @@ -10,6 +10,7 @@ import ( "os/exec" "path" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -18,6 +19,7 @@ import ( routinepkg "github.com/langgenius/dify-plugin-daemon/pkg/routine" "github.com/langgenius/dify-plugin-daemon/pkg/utils/log" "github.com/langgenius/dify-plugin-daemon/pkg/utils/routine" + "github.com/langgenius/dify-plugin-daemon/pkg/utils/system" gootel "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" @@ -139,6 +141,9 @@ func (p *LocalPluginRuntime) installDependencies( ) error { baseCtx, parent := p.startSpan("python.install_deps", attribute.String("plugin.identity", p.Config.Identity())) defer parent.End() + if runtime.GOOS == "windows" { + baseCtx = context.WithoutCancel(baseCtx) + } ctx, cancel := context.WithTimeout(baseCtx, 10*time.Minute) defer cancel() @@ -175,6 +180,10 @@ func (p *LocalPluginRuntime) installDependencies( if p.appConfig.NoProxy != "" { cmd.Env = append(cmd.Env, fmt.Sprintf("NO_PROXY=%s", p.appConfig.NoProxy)) } + if p.appConfig.TempDir != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("TEMP=%s", p.appConfig.TempDir)) + cmd.Env = append(cmd.Env, fmt.Sprintf("TMP=%s", p.appConfig.TempDir)) + } cmd.Dir = p.State.WorkingPath // get stdout and stderr @@ -299,12 +308,18 @@ const ( requirementsTxtFile PythonDependencyFileType = "requirements.txt" ) +var envPythonPath string +var envValidFlagFile string + const ( - envPath = ".venv" - envPythonPath = envPath + "/bin/python" - envValidFlagFile = envPath + "/dify/plugin.json" + envPath = ".venv" ) +func init() { + envPythonPath = system.GetEnvPythonPath(envPath) + envValidFlagFile = envPath + "/dify/plugin.json" +} + func (p *LocalPluginRuntime) checkPythonVirtualEnvironment() (*PythonVirtualEnvironment, error) { _, span := p.startSpan("python.check_venv") defer span.End() @@ -420,6 +435,9 @@ func (p *LocalPluginRuntime) preCompile( ) error { baseCtx, span := p.startSpan("python.precompile") defer span.End() + if runtime.GOOS == "windows" { + baseCtx = context.WithoutCancel(baseCtx) + } ctx, cancel := context.WithTimeout(baseCtx, 10*time.Minute) defer cancel() diff --git a/internal/core/plugin_manager/media_transport/installed_bucket.go b/internal/core/plugin_manager/media_transport/installed_bucket.go index 44305974e..0656a98a7 100644 --- a/internal/core/plugin_manager/media_transport/installed_bucket.go +++ b/internal/core/plugin_manager/media_transport/installed_bucket.go @@ -9,6 +9,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" "github.com/langgenius/dify-plugin-daemon/pkg/utils/log" + "github.com/langgenius/dify-plugin-daemon/pkg/utils/system" ) type InstalledBucket struct { @@ -73,9 +74,15 @@ func (b *InstalledBucket) List() ([]plugin_entities.PluginUniqueIdentifier, erro if strings.HasPrefix(path.Path, ".") { continue } + + convertPath := system.ConvertPath(path.Path) + // windows path start with "/" + if after, ok := strings.CutPrefix(convertPath, "/"); ok { + convertPath = after + } // remove prefix identifier, err := plugin_entities.NewPluginUniqueIdentifier( - strings.TrimPrefix(path.Path, b.installedPath), + strings.TrimPrefix(convertPath, b.installedPath), ) if err != nil { log.Error("failed to create PluginUniqueIdentifier from path", "path", path.Path, "error", err) diff --git a/internal/core/plugin_manager/media_transport/installed_bucket_test.go b/internal/core/plugin_manager/media_transport/installed_bucket_test.go new file mode 100644 index 000000000..ab4a8a042 --- /dev/null +++ b/internal/core/plugin_manager/media_transport/installed_bucket_test.go @@ -0,0 +1,39 @@ +package media_transport + +import ( + "runtime" + "testing" + + "github.com/langgenius/dify-cloud-kit/oss" + "github.com/langgenius/dify-cloud-kit/oss/factory" + "github.com/langgenius/dify-plugin-daemon/internal/types/app" + "github.com/stretchr/testify/assert" +) + +func TestPluginListWindonws(t *testing.T) { + if runtime.GOOS != "windows" { + return + } + config := &app.Config{ + PluginStorageLocalRoot: "testdata", + PluginInstalledPath: "plugin", + PluginStorageType: "local", + } + var storage oss.OSS + var err error + storage, err = factory.Load(config.PluginStorageType, oss.OSSArgs{ + Local: &oss.Local{ + Path: config.PluginStorageLocalRoot, + }, + }) + if err != nil { + t.Fatal("failed to create storage") + } + installedBucket := NewInstalledBucket(storage, config.PluginInstalledPath) + identifiers, err := installedBucket.List() + if err != nil { + t.Error(err) + } + assert.Equal(t, len(identifiers), 1) + assert.Equal(t, identifiers[0].String(), "langgenius/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122") +} diff --git a/internal/core/plugin_manager/media_transport/testdata/plugin/langgenius/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122 b/internal/core/plugin_manager/media_transport/testdata/plugin/langgenius/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122 new file mode 100644 index 000000000..636bfc721 Binary files /dev/null and b/internal/core/plugin_manager/media_transport/testdata/plugin/langgenius/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122 differ diff --git a/internal/types/app/config.go b/internal/types/app/config.go index 056388722..b54a387c6 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -187,6 +187,7 @@ type Config struct { PipPreferBinary bool `envconfig:"PIP_PREFER_BINARY" default:"true"` PipVerbose bool `envconfig:"PIP_VERBOSE" default:"true"` PipExtraArgs string `envconfig:"PIP_EXTRA_ARGS"` + TempDir string `envconfig:"TEMP_DIR"` // Runtime buffer configuration (applies to both local and serverless runtimes) // These are the new generic names that should be used going forward diff --git a/pkg/entities/plugin_entities/identity.go b/pkg/entities/plugin_entities/identity.go index 7934d86fe..1fee34ef1 100644 --- a/pkg/entities/plugin_entities/identity.go +++ b/pkg/entities/plugin_entities/identity.go @@ -8,6 +8,7 @@ import ( "github.com/go-playground/validator/v10" "github.com/google/uuid" "github.com/langgenius/dify-plugin-daemon/pkg/entities/manifest_entities" + "github.com/langgenius/dify-plugin-daemon/pkg/utils/system" "github.com/langgenius/dify-plugin-daemon/pkg/validators" ) @@ -20,7 +21,7 @@ var ( // for checksum, it must be a 32-character hexadecimal string. // the author part is optional, if not specified, it will be empty. pluginUniqueIdentifierRegexp = regexp.MustCompile( - `^(?:([a-z0-9_-]{1,64})\/)?([a-z0-9_-]{1,255}):([0-9]{1,4})(\.[0-9]{1,4}){1,3}(-\w{1,16})?@[a-f0-9]{32,64}$`, + `^(?:([a-z0-9_-]{1,64})\/)?([a-z0-9_-]{1,255})` + system.DelimiterFLag + `([0-9]{1,4})(\.[0-9]{1,4}){1,3}(-\w{1,16})?@[a-f0-9]{32,64}$`, ) ) @@ -33,7 +34,7 @@ func NewPluginUniqueIdentifier(identifier string) (PluginUniqueIdentifier, error func (p PluginUniqueIdentifier) PluginID() string { // try find : - split := strings.Split(p.String(), ":") + split := strings.Split(p.String(), system.DelimiterFLag) if len(split) == 2 { return split[0] } @@ -44,7 +45,7 @@ func (p PluginUniqueIdentifier) Version() manifest_entities.Version { // extract version part from the string split := strings.Split(p.String(), "@") if len(split) == 2 { - split = strings.Split(split[0], ":") + split = strings.Split(split[0], system.DelimiterFLag) if len(split) == 2 { return manifest_entities.Version(split[1]) } @@ -60,7 +61,7 @@ func (p PluginUniqueIdentifier) RemoteLike() bool { func (p PluginUniqueIdentifier) Author() string { // extract author part from the string - split := strings.Split(p.String(), ":") + split := strings.Split(p.String(), system.DelimiterFLag) if len(split) == 2 { split = strings.Split(split[0], "/") if len(split) == 2 { diff --git a/pkg/entities/plugin_entities/identity_test.go b/pkg/entities/plugin_entities/identity_test.go index 5f7eaf45b..2f6c02eac 100644 --- a/pkg/entities/plugin_entities/identity_test.go +++ b/pkg/entities/plugin_entities/identity_test.go @@ -2,10 +2,12 @@ package plugin_entities import ( "testing" + + "github.com/langgenius/dify-plugin-daemon/pkg/utils/system" ) func TestPluginUniqueIdentifier(t *testing.T) { - i, err := NewPluginUniqueIdentifier("langgenius/test:1.0.0@1234567890abcdef1234567890abcdef1234567890abcdef") + i, err := NewPluginUniqueIdentifier("langgenius/test" + system.DelimiterFLag + "1.0.0@1234567890abcdef1234567890abcdef1234567890abcdef") if err != nil { t.Fatalf("NewPluginUniqueIdentifier() returned an error: %v", err) } @@ -22,7 +24,7 @@ func TestPluginUniqueIdentifier(t *testing.T) { t.Fatalf("Checksum() = %s; want 1234567890abcdef1234567890abcdef1234567890abcdef", i.Checksum()) } - _, err = NewPluginUniqueIdentifier("test:1.0.0@1234567890abcdef1234567890abcdef1234567890abcdef") + _, err = NewPluginUniqueIdentifier("test" + system.DelimiterFLag + "1.0.0@1234567890abcdef1234567890abcdef1234567890abcdef") if err != nil { t.Fatalf("NewPluginUniqueIdentifier() returned an error: %v", err) } @@ -37,12 +39,12 @@ func TestPluginUniqueIdentifier(t *testing.T) { t.Fatalf("NewPluginUniqueIdentifier() returned nil error for invalid identifier") } - _, err = NewPluginUniqueIdentifier("langgenius/test:1.0.0@123456") + _, err = NewPluginUniqueIdentifier("langgenius/test" + system.DelimiterFLag + "1.0.0@123456") if err == nil { t.Fatalf("NewPluginUniqueIdentifier() returned nil error for invalid identifier") } - _, err = NewPluginUniqueIdentifier("langgenius/test:1.0.0") + _, err = NewPluginUniqueIdentifier("langgenius/test" + system.DelimiterFLag + "1.0.0") if err == nil { t.Fatalf("NewPluginUniqueIdentifier() returned nil error for invalid identifier") } diff --git a/pkg/plugin_packager/decoder/testdata/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122 b/pkg/plugin_packager/decoder/testdata/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122 new file mode 100644 index 000000000..636bfc721 Binary files /dev/null and b/pkg/plugin_packager/decoder/testdata/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122 differ diff --git a/pkg/plugin_packager/decoder/zip.go b/pkg/plugin_packager/decoder/zip.go index d991ccdc3..47f232740 100644 --- a/pkg/plugin_packager/decoder/zip.go +++ b/pkg/plugin_packager/decoder/zip.go @@ -16,6 +16,7 @@ import ( "github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities" "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/consts" "github.com/langgenius/dify-plugin-daemon/pkg/utils/parser" + "github.com/langgenius/dify-plugin-daemon/pkg/utils/system" ) type ZipPluginDecoder struct { @@ -304,7 +305,7 @@ func (z *ZipPluginDecoder) ExtractTo(dst string) error { return err } - bytes, err := z.ReadFile(filepath.Join(dir, filename)) + bytes, err := z.ReadFile(system.GetZipReadPath(dir, filename)) if err != nil { return err } diff --git a/pkg/plugin_packager/decoder/zip_test.go b/pkg/plugin_packager/decoder/zip_test.go new file mode 100644 index 000000000..c5906315a --- /dev/null +++ b/pkg/plugin_packager/decoder/zip_test.go @@ -0,0 +1,33 @@ +package decoder + +import ( + "os" + "path" + "testing" +) + +func TestExtractFile(t *testing.T) { + pluginFile, err := os.ReadFile("testdata/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122") + if err != nil { + t.Fatalf("read file error: %v", err) + } + decoder, err := NewZipPluginDecoder(pluginFile) + if err != nil { + t.Fatalf("create new zip decoder error: %v", err) + } + extractPath := "testdata/cwd" + err = os.Mkdir(extractPath, 0755) + if err != nil { + t.Fatalf("mk dir error: %v", err) + } + defer os.RemoveAll(extractPath) + + err = decoder.ExtractTo(extractPath) + if err != nil { + t.Fatalf("extract file error: %v", err) + } + _, err = os.Stat(path.Join(extractPath, "provider", "github.yaml")) + if err != nil { + t.Fatalf("extract file not exists: %v", err) + } +} diff --git a/pkg/utils/parser/identity.go b/pkg/utils/parser/identity.go index 8918c76a5..213ae2970 100644 --- a/pkg/utils/parser/identity.go +++ b/pkg/utils/parser/identity.go @@ -1,10 +1,14 @@ package parser -import "fmt" +import ( + "fmt" + + "github.com/langgenius/dify-plugin-daemon/pkg/utils/system" +) func MarshalPluginID(author string, name string, version string) string { if author == "" { - return fmt.Sprintf("%s:%s", name, version) + return fmt.Sprintf("%s%s%s", name, system.DelimiterFLag, version) } - return fmt.Sprintf("%s/%s:%s", author, name, version) + return fmt.Sprintf("%s/%s%s%s", author, name, system.DelimiterFLag, version) } diff --git a/pkg/utils/system/system.go b/pkg/utils/system/system.go new file mode 100644 index 000000000..1ea5b7e5e --- /dev/null +++ b/pkg/utils/system/system.go @@ -0,0 +1,39 @@ +package system + +import ( + "path" + "path/filepath" + "runtime" + "strings" +) + +var DelimiterFLag string + +func init() { + if runtime.GOOS == "windows" { + DelimiterFLag = "#" + } else { + DelimiterFLag = ":" + } +} + +func ConvertPath(input string) string { + if runtime.GOOS == "windows" { + return strings.ReplaceAll(input, "\\", "/") + } + return input +} + +func GetZipReadPath(dir string, filename string) string { + if runtime.GOOS == "windows" { + return path.Join(dir, filename) + } + return filepath.Join(dir, filename) +} + +func GetEnvPythonPath(envPath string) string { + if runtime.GOOS == "windows" { + return envPath + "/Scripts/python.exe" + } + return envPath + "/bin/python" +} diff --git a/pkg/utils/system/system_test.go b/pkg/utils/system/system_test.go new file mode 100644 index 000000000..55d9795e6 --- /dev/null +++ b/pkg/utils/system/system_test.go @@ -0,0 +1,25 @@ +package system + +import ( + "path" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFilePath(t *testing.T) { + filepathJoinRes := filepath.Join("foo", "bar.txt") + pathJoinRs := path.Join("foo", "bar.txt") + if runtime.GOOS == "windows" { + assert.Equal(t, "foo\\bar.txt", filepathJoinRes) + } + assert.Equal(t, "foo/bar.txt", pathJoinRs) +} + +func TestConvertPath(t *testing.T) { + filepathJoinRes := filepath.Join("foo", "bar.txt") + res := ConvertPath(filepathJoinRes) + assert.Equal(t, "foo/bar.txt", res) +}