diff --git a/.env.example b/.env.example index 48f742a52..211bd446e 100644 --- a/.env.example +++ b/.env.example @@ -188,3 +188,6 @@ PLUGIN_RUNTIME_MAX_BUFFER_SIZE=5242880 DIFY_BACKWARDS_INVOCATION_WRITE_TIMEOUT=5000 # dify backwards invocation read timeout in milliseconds DIFY_BACKWARDS_INVOCATION_READ_TIMEOUT=240000 + +PIP_MIRROR_URL= +PIP_EXTRA_INDEX_URL= diff --git a/README.md b/README.md index db18f0013..896ce263b 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,28 @@ Daemon uses `uv` to manage the dependencies of plugins, before you start the dae #### Interpreter There is a possibility that you have multiple python versions installed on your machine, a variable `PYTHON_INTERPRETER_PATH` is provided to specify the python interpreter path for you. +#### Speeding up Python dependency installation (uv/pip) +You can speed up plugin dependency installation by configuring Python package indexes via environment variables (set them in your shell or `.env`). The daemon reads these into its config on startup. + +Supported variables +- PIP_MIRROR_URL: Primary index for both uv sync and uv pip install +- PIP_EXTRA_INDEX_URL: One or more extra indexes (comma or space separated) + +Behavior +- Applies uniformly to both dependency styles: + - pyproject.toml (using `uv sync`) + - requirements.txt (using `uv pip install`) +- Trusted hosts are derived automatically from the configured URLs. + +Examples +```bash +# Use the official PyPI index +PIP_MIRROR_URL=https://pypi.org/simple + +# Add multiple mirrors (comma or space separated) +PIP_EXTRA_INDEX_URL="https://my.mirror/simple, https://another.mirror/simple" +``` + ## Deployment Currently, the daemon only supports Linux and MacOS, lots of adaptions are needed for Windows, feel free to contribute if you need it. diff --git a/internal/core/local_runtime/dependency_installation_test.go b/internal/core/local_runtime/dependency_installation_test.go index 44776c263..5d4ed1da3 100644 --- a/internal/core/local_runtime/dependency_installation_test.go +++ b/internal/core/local_runtime/dependency_installation_test.go @@ -200,6 +200,7 @@ func TestPreparePipArgs(t *testing.T) { "pip", "install", "-i", "https://pypi.tuna.tsinghua.edu.cn/simple", + "--trusted-host", "pypi.tuna.tsinghua.edu.cn", "-r", "requirements.txt", }, args) }) @@ -229,6 +230,7 @@ func TestPreparePipArgs(t *testing.T) { "pip", "install", "-i", "https://pypi.tuna.tsinghua.edu.cn/simple", + "--trusted-host", "pypi.tuna.tsinghua.edu.cn", "-r", "requirements.txt", "-vvv", "--no-cache", diff --git a/internal/core/local_runtime/setup_python_environment.go b/internal/core/local_runtime/setup_python_environment.go index 84433cb1c..294abef38 100644 --- a/internal/core/local_runtime/setup_python_environment.go +++ b/internal/core/local_runtime/setup_python_environment.go @@ -1,17 +1,22 @@ package local_runtime import ( + "bufio" "bytes" "context" "errors" "fmt" + "net/url" "os" "os/exec" "path" "path/filepath" + "regexp" + "sort" "strconv" "strings" "sync" + "sync/atomic" "time" routinepkg "github.com/langgenius/dify-plugin-daemon/pkg/routine" @@ -37,8 +42,15 @@ func (p *LocalPluginRuntime) prepareUV() (string, error) { func (p *LocalPluginRuntime) preparePipArgs() []string { args := []string{"install"} - if p.appConfig.PipMirrorUrl != "" { - args = append(args, "-i", p.appConfig.PipMirrorUrl) + // Determine index URL precedence for pip install: + indexURL := p.appConfig.PipMirrorUrl + // Extra index URLs (comma or space separated); fallback to UV extras + extra := p.appConfig.PipExtraIndexUrl + args = addIndexArgs(args, indexURL, extra) + + // Derive trusted-host from index/extra URLs + for _, h := range deriveTrustedHosts(indexURL, extra) { + args = append(args, "--trusted-host", h) } args = append(args, "-r", "requirements.txt") @@ -60,9 +72,11 @@ func (p *LocalPluginRuntime) preparePipArgs() []string { func (p *LocalPluginRuntime) prepareSyncArgs() []string { args := []string{"sync", "--no-dev"} - if p.appConfig.PipMirrorUrl != "" { - args = append(args, "-i", p.appConfig.PipMirrorUrl) - } + // Determine index URL precedence for uv sync: + indexURL := p.appConfig.PipMirrorUrl + // Extra index URLs; fallback to pip extras + extra := p.appConfig.PipExtraIndexUrl + args = addIndexArgs(args, indexURL, extra) if p.appConfig.PipVerbose { args = append(args, "-v") @@ -91,6 +105,50 @@ func (p *LocalPluginRuntime) detectDependencyFileType() (PythonDependencyFileTyp return "", fmt.Errorf("neither %s nor %s found in plugin directory", pyprojectTomlFile, requirementsTxtFile) } +// buildDependencyInstallEnv builds environment variables for dependency installation. +func (p *LocalPluginRuntime) buildDependencyInstallEnv(virtualEnvPath string) []string { + env := []string{ + "VIRTUAL_ENV=" + virtualEnvPath, + "PATH=" + os.Getenv("PATH"), + } + + // Provide PIP_TRUSTED_HOST (space-separated) for pip under uv + pipIndex := p.appConfig.PipMirrorUrl + pipExtra := p.appConfig.PipExtraIndexUrl + if hosts := deriveTrustedHosts(pipIndex, pipExtra); len(hosts) > 0 { + env = append(env, fmt.Sprintf("PIP_TRUSTED_HOST=%s", strings.Join(hosts, " "))) + } + + if p.appConfig.HttpProxy != "" { + env = append(env, fmt.Sprintf("HTTP_PROXY=%s", p.appConfig.HttpProxy)) + } + if p.appConfig.HttpsProxy != "" { + env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", p.appConfig.HttpsProxy)) + } + if p.appConfig.NoProxy != "" { + env = append(env, fmt.Sprintf("NO_PROXY=%s", p.appConfig.NoProxy)) + } + return env +} + +// withOpLogging wraps fn with standardized start/finish logging and duration measurement. +func (p *LocalPluginRuntime) withOpLogging(op string, kvs []any, fn func() error) error { + startAt := time.Now() + base := []any{"plugin", p.Config.Identity()} + log.Info("starting "+op, append(base, kvs...)...) + err := fn() + if err != nil { + fields := append(append([]any{}, base...), kvs...) + fields = append(fields, "duration", time.Since(startAt).String(), "error", err) + log.Error(op+" failed", fields...) + return err + } + fields := append(append([]any{}, base...), kvs...) + fields = append(fields, "duration", time.Since(startAt).String()) + log.Info(op+" finished", fields...) + return nil +} + func (p *LocalPluginRuntime) installDependencies( uvPath string, dependencyFileType PythonDependencyFileType, @@ -99,134 +157,161 @@ func (p *LocalPluginRuntime) installDependencies( defer cancel() var args []string + var methodLabel string switch dependencyFileType { case pyprojectTomlFile: args = p.prepareSyncArgs() - log.Info("installing plugin dependencies", "plugin", p.Config.Identity(), "method", "uv sync", "file", pyprojectTomlFile) + methodLabel = "uv sync" case requirementsTxtFile: args = p.preparePipArgs() - log.Info("installing plugin dependencies", "plugin", p.Config.Identity(), "method", "uv pip install", "file", requirementsTxtFile) + methodLabel = "uv pip install" default: return fmt.Errorf("unsupported dependency file type: %s", dependencyFileType) } virtualEnvPath := path.Join(p.State.WorkingPath, ".venv") - cmd := exec.CommandContext(ctx, uvPath, args...) - cmd.Env = append(cmd.Env, "VIRTUAL_ENV="+virtualEnvPath, "PATH="+os.Getenv("PATH")) - if p.appConfig.HttpProxy != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", p.appConfig.HttpProxy)) - } - if p.appConfig.HttpsProxy != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", p.appConfig.HttpsProxy)) - } - if p.appConfig.NoProxy != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("NO_PROXY=%s", p.appConfig.NoProxy)) - } - cmd.Dir = p.State.WorkingPath - - // get stdout and stderr - stdout, err := cmd.StdoutPipe() - if err != nil { - return fmt.Errorf("failed to get stdout: %s", err) - } - defer stdout.Close() - - stderr, err := cmd.StderrPipe() - if err != nil { - return fmt.Errorf("failed to get stderr: %s", err) - } - defer stderr.Close() - - // start command - if err := cmd.Start(); err != nil { - return fmt.Errorf("failed to start command: %s", err) - } - - defer func() { - if cmd.Process != nil { - cmd.Process.Kill() + sanitized := sanitizeArgs(args) + + return p.withOpLogging("dependency installation", []any{ + "method", methodLabel, + "args", strings.Join(sanitized, " "), + }, func() error { + cmd := exec.CommandContext(ctx, uvPath, args...) + cmd.Env = append(cmd.Env, p.buildDependencyInstallEnv(virtualEnvPath)...) + cmd.Dir = p.State.WorkingPath + + // get stdout and stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to get stdout: %s", err) } - }() - - var errMsg strings.Builder - var wg sync.WaitGroup - wg.Add(2) + defer stdout.Close() - lastActiveAt := time.Now() + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("failed to get stderr: %s", err) + } + defer stderr.Close() - routine.Submit(routinepkg.Labels{ - routinepkg.RoutineLabelKeyModule: "plugin_manager", - routinepkg.RoutineLabelKeyMethod: "InitPythonEnvironment", - }, func() { - defer wg.Done() - // read stdout - buf := make([]byte, 1024) - for { - n, err := stdout.Read(buf) - if err != nil { - break - } - // FIXME: move the log to separated layer - log.Info("installing plugin", "plugin", p.Config.Identity(), "output", string(buf[:n])) - lastActiveAt = time.Now() + // start command + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start command: %s", err) } - }) - routine.Submit(routinepkg.Labels{ - routinepkg.RoutineLabelKeyModule: "plugin_manager", - routinepkg.RoutineLabelKeyMethod: "InitPythonEnvironment", - }, func() { - defer wg.Done() - // read stderr - buf := make([]byte, 1024) - for { - n, err := stderr.Read(buf) - if err != nil && err != os.ErrClosed { - lastActiveAt = time.Now() - errMsg.WriteString(string(buf[:n])) - break - } else if err == os.ErrClosed { - break + defer func() { + if cmd.Process != nil { + cmd.Process.Kill() } - - if n > 0 { - errMsg.WriteString(string(buf[:n])) - lastActiveAt = time.Now() + }() + + var errMsg strings.Builder + var errMu sync.Mutex + var wg sync.WaitGroup + wg.Add(2) + + var lastActiveAt atomic.Int64 + lastActiveAt.Store(time.Now().UnixNano()) + + routine.Submit(routinepkg.Labels{ + routinepkg.RoutineLabelKeyModule: "plugin_manager", + routinepkg.RoutineLabelKeyMethod: "InitPythonEnvironment", + }, func() { + defer wg.Done() + // read stdout line by line + scanner := bufio.NewScanner(stdout) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 10*1024*1024) + for scanner.Scan() { + line := scanner.Text() + log.Info("install deps", "plugin", p.Config.Identity(), "stream", "stdout", "line", line) + lastActiveAt.Store(time.Now().UnixNano()) } - } - }) - - routine.Submit(routinepkg.Labels{ - routinepkg.RoutineLabelKeyModule: "plugin_manager", - routinepkg.RoutineLabelKeyMethod: "InitPythonEnvironment", - }, func() { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - for range ticker.C { - if cmd.ProcessState != nil && cmd.ProcessState.Exited() { - break + if err := scanner.Err(); err != nil { + errMu.Lock() + errMsg.WriteString("stdout scan error: ") + errMsg.WriteString(err.Error()) + errMsg.WriteString("\n") + errMu.Unlock() + log.Warn("install deps", "plugin", p.Config.Identity(), "stream", "stdout", "scanner_err", err.Error()) } + }) + + routine.Submit(routinepkg.Labels{ + routinepkg.RoutineLabelKeyModule: "plugin_manager", + routinepkg.RoutineLabelKeyMethod: "InitPythonEnvironment", + }, func() { + defer wg.Done() + // read stderr line by line + scanner := bufio.NewScanner(stderr) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 10*1024*1024) + for scanner.Scan() { + line := scanner.Text() + errMu.Lock() + errMsg.WriteString(line) + errMsg.WriteString("\n") + errMu.Unlock() + log.Warn("install deps", "plugin", p.Config.Identity(), "stream", "stderr", "line", line) + lastActiveAt.Store(time.Now().UnixNano()) + } + if err := scanner.Err(); err != nil { + errMu.Lock() + errMsg.WriteString("stderr scan error: ") + errMsg.WriteString(err.Error()) + errMsg.WriteString("\n") + errMu.Unlock() + log.Warn("install deps", "plugin", p.Config.Identity(), "stream", "stderr", "scanner_err", err.Error()) + } + }) + + routine.Submit(routinepkg.Labels{ + routinepkg.RoutineLabelKeyModule: "plugin_manager", + routinepkg.RoutineLabelKeyMethod: "InitPythonEnvironment", + }, func() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for range ticker.C { + if cmd.ProcessState != nil && cmd.ProcessState.Exited() { + break + } - if time.Since(lastActiveAt) > time.Duration( - p.appConfig.PythonEnvInitTimeout, - )*time.Second { - cmd.Process.Kill() - errMsg.WriteString(fmt.Sprintf( - "init process exited due to no activity for %d seconds", + if time.Since(time.Unix(0, lastActiveAt.Load())) > time.Duration( p.appConfig.PythonEnvInitTimeout, - )) - break + )*time.Second { + cmd.Process.Kill() + errMu.Lock() + errMsg.WriteString(fmt.Sprintf( + "init process exited due to no activity for %d seconds", + p.appConfig.PythonEnvInitTimeout, + )) + errMu.Unlock() + break + } } + }) + + wg.Wait() + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("failed to install dependencies: %s, output: %s", err, errMsg.String()) } + return nil }) +} - wg.Wait() +// sanitizeArgs redacts credentials in any URL-like arguments to avoid leaking secrets in logs. +func sanitizeArgs(args []string) []string { + // Match https://user:pass@ and https://user@ + reWithPass := regexp.MustCompile(`(https?://)[^/@:]+:[^/@]+@`) + reUserOnly := regexp.MustCompile(`(https?://)[^/@:]+@`) - if err := cmd.Wait(); err != nil { - return fmt.Errorf("failed to install dependencies: %s, output: %s", err, errMsg.String()) + out := make([]string, len(args)) + for i, a := range args { + s := reWithPass.ReplaceAllString(a, "${1}****:****@") + s = reUserOnly.ReplaceAllString(s, "${1}****:****@") + out[i] = s } - - return nil + return out } type PythonVirtualEnvironment struct { @@ -356,6 +441,69 @@ func (p *LocalPluginRuntime) markVirtualEnvironmentAsValid() error { return nil } +// splitByCommaOrSpace splits a list like "a,b c" into tokens. +func splitByCommaOrSpace(s string) []string { + // replace comma with space then split by spaces + s = strings.ReplaceAll(s, ",", " ") + fields := strings.Fields(s) + return fields +} + +// selectURL returns the first non-empty URL from the provided list. +func selectURL(urls ...string) string { + for _, u := range urls { + if u != "" { + return u + } + } + return "" +} + +// addIndexArgs appends index and extra-index URL arguments to args. +func addIndexArgs(args []string, indexURL string, extraIndexURL string) []string { + if indexURL != "" { + args = append(args, "-i", indexURL) + } + if extraIndexURL != "" { + for _, u := range splitByCommaOrSpace(extraIndexURL) { + if u != "" { + args = append(args, "--extra-index-url", u) + } + } + } + return args +} + +// deriveTrustedHosts parses hostnames from index/extra URLs and returns a de-duplicated list. +func deriveTrustedHosts(indexURL string, extraIndexURL string) []string { + set := map[string]struct{}{} + add := func(raw string) { + if strings.TrimSpace(raw) == "" { + return + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return + } + host := u.Host + if i := strings.Index(host, ":"); i >= 0 { + host = host[:i] + } + set[host] = struct{}{} + } + add(indexURL) + for _, raw := range splitByCommaOrSpace(extraIndexURL) { + add(raw) + } + out := make([]string, 0, len(set)) + for h := range set { + out = append(out, h) + } + // preserve deterministic order: sort hostnames + sort.Strings(out) + return out +} + func (p *LocalPluginRuntime) preCompile( pythonPath string, ) error { diff --git a/internal/core/local_runtime/setup_python_environment_test.go b/internal/core/local_runtime/setup_python_environment_test.go new file mode 100644 index 000000000..7ab5c2b92 --- /dev/null +++ b/internal/core/local_runtime/setup_python_environment_test.go @@ -0,0 +1,296 @@ +package local_runtime + +import ( + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/langgenius/dify-plugin-daemon/internal/types/app" + "github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/decoder" + "github.com/langgenius/dify-plugin-daemon/pkg/utils/routine" + "github.com/stretchr/testify/require" +) + +func envSliceToMap(in []string) map[string]string { + m := map[string]string{} + for _, kv := range in { + if kv == "" { + continue + } + parts := strings.SplitN(kv, "=", 2) + if len(parts) != 2 { + continue + } + m[parts[0]] = parts[1] + } + return m +} + +func TestSplitByCommaOrSpace(t *testing.T) { + cases := []struct { + in string + out []string + }{ + {"", []string{}}, + {"a", []string{"a"}}, + {"a,b", []string{"a", "b"}}, + {"a b", []string{"a", "b"}}, + {" a, b c ", []string{"a", "b", "c"}}, + {"https://a/simple, https://b/simple", []string{"https://a/simple", "https://b/simple"}}, + } + for _, c := range cases { + got := splitByCommaOrSpace(c.in) + require.Equal(t, c.out, got) + } +} + +func TestSanitizeArgs(t *testing.T) { + in := []string{"sync", "-i", "https://user:pass@example.com/simple", "--extra-index-url", "https://token@mirror.example/simple"} + got := sanitizeArgs(in) + joined := strings.Join(got, " ") + require.NotContains(t, joined, "user:pass@") + require.NotContains(t, joined, "token@") +} + +func TestPrepareSyncArgs_IndexAndExtras(t *testing.T) { + // Uses only PIP_MIRROR_URL and PIP_EXTRA_INDEX_URL + r := &LocalPluginRuntime{appConfig: &app.Config{PipMirrorUrl: "https://mirror.example/simple"}} + args := r.prepareSyncArgs() + require.Equal(t, []string{"sync", "--no-dev", "-i", "https://mirror.example/simple"}, args) +} + +func TestPreparePipArgs_IndexAndExtras(t *testing.T) { + // Index from PIP_MIRROR_URL + r := &LocalPluginRuntime{appConfig: &app.Config{PipMirrorUrl: "https://mirror.example/simple"}} + args := r.preparePipArgs() + joined := strings.Join(args, " ") + require.Contains(t, joined, "-i https://mirror.example/simple") + require.Contains(t, joined, "--trusted-host mirror.example") + + // Extra indexes from PIP_EXTRA_INDEX_URL + r = &LocalPluginRuntime{appConfig: &app.Config{PipExtraIndexUrl: "https://a/simple https://b/simple"}} + args = r.preparePipArgs() + joined = strings.Join(args, " ") + require.Contains(t, joined, "--extra-index-url https://a/simple") + require.Contains(t, joined, "--extra-index-url https://b/simple") + require.Contains(t, joined, "--trusted-host a") + require.Contains(t, joined, "--trusted-host b") +} + +// writeFakeUv creates a uv shim that records args and simulates `uv venv` by +// creating the expected virtualenv layout under the current working directory. +func writeFakeUv(t *testing.T, recordFile string) string { + t.Helper() + script := "#!/usr/bin/env bash\n" + + "set -euo pipefail\n" + + "echo \"$0 $@\" >> \"" + recordFile + "\"\n" + + "if [[ \"${1:-}\" == venv ]]; then\n" + + " envdir=\"${2:-.venv}\"\n" + + " mkdir -p \"$envdir/bin\"\n" + + " : > \"$envdir/bin/python\"\n" + + " chmod +x \"$envdir/bin/python\"\n" + + " exit 0\n" + + "fi\n" + + "exit 0\n" + p := path.Join(t.TempDir(), "uv") + require.NoError(t, os.WriteFile(p, []byte(script), 0755)) + return p +} + +func TestInstallDependencies_UvSync_WithIndexArgs(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + routine.InitPool(64) + record := path.Join(t.TempDir(), "uv_args.log") + fakeUv := writeFakeUv(t, record) + + pluginSourceDir := path.Join("testdata", "plugin-with-pyproject") + dec, err := decoder.NewFSPluginDecoder(pluginSourceDir) + require.NoError(t, err) + + appCfg := &app.Config{UvPath: fakeUv, PythonEnvInitTimeout: 60, PipMirrorUrl: "https://uv.example/simple", PipExtraIndexUrl: "https://a/simple https://b/simple"} + runtime, err := ConstructPluginRuntime(appCfg, dec) + require.NoError(t, err) + require.NoError(t, copyDir(pluginSourceDir, runtime.State.WorkingPath)) + _, err = runtime.createVirtualEnvironment(fakeUv) + require.NoError(t, err) + depType, err := runtime.detectDependencyFileType() + require.NoError(t, err) + require.NoError(t, runtime.installDependencies(fakeUv, depType)) + + b, err := os.ReadFile(record) + require.NoError(t, err) + out := string(b) + require.Contains(t, out, "sync --no-dev -i https://uv.example/simple") + require.Contains(t, out, "--extra-index-url https://a/simple") + require.Contains(t, out, "--extra-index-url https://b/simple") +} + +func TestInstallDependencies_UvPip_WithIndexArgs(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + routine.InitPool(64) + record := path.Join(t.TempDir(), "uv_args.log") + fakeUv := writeFakeUv(t, record) + + pluginSourceDir := path.Join("testdata", "plugin-with-requirements") + dec, err := decoder.NewFSPluginDecoder(pluginSourceDir) + require.NoError(t, err) + + appCfg := &app.Config{UvPath: fakeUv, PythonEnvInitTimeout: 60, PipMirrorUrl: "https://pypi.org/simple", PipExtraIndexUrl: "https://x/simple, https://y/simple"} + runtime, err := ConstructPluginRuntime(appCfg, dec) + require.NoError(t, err) + require.NoError(t, copyDir(pluginSourceDir, runtime.State.WorkingPath)) + _, err = runtime.createVirtualEnvironment(fakeUv) + require.NoError(t, err) + depType, err := runtime.detectDependencyFileType() + require.NoError(t, err) + require.NoError(t, runtime.installDependencies(fakeUv, depType)) + + b, err := os.ReadFile(record) + require.NoError(t, err) + out := string(b) + require.Contains(t, out, "pip install -i https://pypi.org/simple") + require.Contains(t, out, "--extra-index-url https://x/simple") + require.Contains(t, out, "--extra-index-url https://y/simple") + require.Contains(t, out, "-r requirements.txt") +} + +func TestBuildDependencyInstallEnv(t *testing.T) { + // Normalize PATH for deterministic assertion + t.Setenv("PATH", "/bin:/usr/bin") + r := &LocalPluginRuntime{appConfig: &app.Config{ + PipMirrorUrl: "https://uv.example/simple", + PipExtraIndexUrl: "https://b/simple https://a/simple", + HttpProxy: "http://proxy.internal:8080", + HttpsProxy: "https://secure-proxy.internal:8443", + NoProxy: "localhost,127.0.0.1", + }} + env := r.buildDependencyInstallEnv("/work/.venv") + m := envSliceToMap(env) + require.Equal(t, "/work/.venv", m["VIRTUAL_ENV"]) + require.Equal(t, "/bin:/usr/bin", m["PATH"]) + // hosts are sorted alphabetically in deriveTrustedHosts + require.Equal(t, "a b uv.example", m["PIP_TRUSTED_HOST"]) + require.Equal(t, "http://proxy.internal:8080", m["HTTP_PROXY"]) + require.Equal(t, "https://secure-proxy.internal:8443", m["HTTPS_PROXY"]) + require.Equal(t, "localhost,127.0.0.1", m["NO_PROXY"]) +} + +func TestBuildDependencyInstallEnv_NoTrustedHostsOrProxy(t *testing.T) { + r := &LocalPluginRuntime{appConfig: &app.Config{}} + env := r.buildDependencyInstallEnv("/venv") + m := envSliceToMap(env) + require.Equal(t, "/venv", m["VIRTUAL_ENV"]) + // PATH must always be present; value depends on test runner env, so just assert key exists + _, ok := m["PATH"] + require.True(t, ok) + require.NotContains(t, m, "PIP_TRUSTED_HOST") + require.NotContains(t, m, "HTTP_PROXY") + require.NotContains(t, m, "HTTPS_PROXY") + require.NotContains(t, m, "NO_PROXY") +} + +func shouldRunRealUV() bool { return os.Getenv("RUN_REAL_UV_TESTS") == "1" } + +func TestInstallDependencies_UvSync(t *testing.T) { + if testing.Short() || !shouldRunRealUV() { + t.Skip("Skipping real UV install test; set RUN_REAL_UV_TESTS=1 to enable") + } + routine.InitPool(256) + uvPath := findUVPath(t) + pythonPath := findPythonPath(t) + + pluginSourceDir := path.Join("testdata", "plugin-with-pyproject") + dec, err := decoder.NewFSPluginDecoder(pluginSourceDir) + require.NoError(t, err) + + appCfg := &app.Config{PythonInterpreterPath: pythonPath, UvPath: uvPath, PythonEnvInitTimeout: 180} + runtime, err := ConstructPluginRuntime(appCfg, dec) + require.NoError(t, err) + + require.NoError(t, copyDir(pluginSourceDir, runtime.State.WorkingPath)) + + venv, err := runtime.createVirtualEnvironment(uvPath) + require.NoError(t, err) + require.NotNil(t, venv) + + depType, err := runtime.detectDependencyFileType() + require.NoError(t, err) + require.Equal(t, pyprojectTomlFile, depType) + require.NoError(t, runtime.installDependencies(uvPath, depType)) + + // verify dify_plugin is installed in site-packages + venvRoot := path.Join(runtime.State.WorkingPath, ".venv") + entries, err := os.ReadDir(path.Join(venvRoot, "lib")) + require.NoError(t, err) + found := false + for _, e := range entries { + if strings.HasPrefix(e.Name(), "python") { + sp := path.Join(venvRoot, "lib", e.Name(), "site-packages") + if _, err := os.Stat(path.Join(sp, "dify_plugin")); err == nil { + found = true + break + } + matches, _ := filepath.Glob(path.Join(sp, "dify_plugin-*.dist-info")) + if len(matches) > 0 { + found = true + break + } + } + } + require.True(t, found, "dify_plugin not found in site-packages") +} + +func TestInstallDependencies_UvPip(t *testing.T) { + if testing.Short() || !shouldRunRealUV() { + t.Skip("Skipping real UV install test; set RUN_REAL_UV_TESTS=1 to enable") + } + routine.InitPool(256) + uvPath := findUVPath(t) + pythonPath := findPythonPath(t) + + pluginSourceDir := path.Join("testdata", "plugin-with-requirements") + dec, err := decoder.NewFSPluginDecoder(pluginSourceDir) + require.NoError(t, err) + + appCfg := &app.Config{PythonInterpreterPath: pythonPath, UvPath: uvPath, PythonEnvInitTimeout: 180} + runtime, err := ConstructPluginRuntime(appCfg, dec) + require.NoError(t, err) + + require.NoError(t, copyDir(pluginSourceDir, runtime.State.WorkingPath)) + + venv, err := runtime.createVirtualEnvironment(uvPath) + require.NoError(t, err) + require.NotNil(t, venv) + + depType, err := runtime.detectDependencyFileType() + require.NoError(t, err) + require.Equal(t, requirementsTxtFile, depType) + require.NoError(t, runtime.installDependencies(uvPath, depType)) + + // verify dify_plugin is installed in site-packages + venvRoot := path.Join(runtime.State.WorkingPath, ".venv") + entries, err := os.ReadDir(path.Join(venvRoot, "lib")) + require.NoError(t, err) + found := false + for _, e := range entries { + if strings.HasPrefix(e.Name(), "python") { + sp := path.Join(venvRoot, "lib", e.Name(), "site-packages") + if _, err := os.Stat(path.Join(sp, "dify_plugin")); err == nil { + found = true + break + } + matches, _ := filepath.Glob(path.Join(sp, "dify_plugin-*.dist-info")) + if len(matches) > 0 { + found = true + break + } + } + } + require.True(t, found, "dify_plugin not found in site-packages") +} diff --git a/internal/types/app/config.go b/internal/types/app/config.go index d9ea706eb..1a33dcf9e 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -183,9 +183,10 @@ type Config struct { PythonEnvInitTimeout int `envconfig:"PYTHON_ENV_INIT_TIMEOUT" validate:"required"` PythonCompileAllExtraArgs string `envconfig:"PYTHON_COMPILE_ALL_EXTRA_ARGS"` PipMirrorUrl string `envconfig:"PIP_MIRROR_URL"` - PipPreferBinary bool `envconfig:"PIP_PREFER_BINARY" default:"true"` - PipVerbose bool `envconfig:"PIP_VERBOSE" default:"true"` - PipExtraArgs string `envconfig:"PIP_EXTRA_ARGS"` + PipExtraIndexUrl string `envconfig:"PIP_EXTRA_INDEX_URL"` + PipPreferBinary bool `envconfig:"PIP_PREFER_BINARY" default:"true"` + PipVerbose bool `envconfig:"PIP_VERBOSE" default:"true"` + PipExtraArgs string `envconfig:"PIP_EXTRA_ARGS"` // Runtime buffer configuration (applies to both local and serverless runtimes) // These are the new generic names that should be used going forward