diff --git a/README.md b/README.md index eed645c..a04e13d 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,15 @@ Because: * Stores it in a local PostgreSQL database * Saves media files in your own S3-compatible bucket +## Features + +* **Transparent Proxy**: Redirects all requests through e6-cache, then to your chosen instance. +* **Passive Caching**: Automatically caches every post you view. +* **Local Storage**: Stores metadata in a local PostgreSQL database and media files in your own S3-compatible bucket. +* **Customizable**: Easily configurable to work with any e621-compatible instance. +* **Self-Hosted**: Runs on your own server, giving you full control over your data. +* **Authentication**: Supports authentication for secure access, even when exposed to the world. + ## Dev Setup ### Start DB and S3 Storage @@ -67,6 +76,7 @@ For The Wolf's Stash, it just reports the host not being supported. ### Other Clients Feel free to open a PR to add documentation for other clients. + ## Speed Comparison (Speed depends on your internet, and database speed) ### Image 1: diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index ccde4ce..8bb548e 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -29,13 +29,18 @@ services: image: minio/mc depends_on: - minio - entrypoint: > - /bin/sh -c " - sleep 5; # Waits for startup - /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin; - /usr/bin/mc mb --ignore-existing minio/e6cache-media; - exit 0; - " + entrypoint: + - /bin/sh + - -c + - | + echo "Waiting for MinIO to be ready..." + until /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin; do + echo "MinIO not ready yet, retrying in 2s..." + sleep 2 + done + /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb minio/e6cache-media --ignore-existing; + exit 0; volumes: dev_db_data: diff --git a/docker-compose.yml b/docker-compose.yml index fa1d485..ed29f13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,7 @@ services: # Proxy settings PROXY_URL: http://localhost:8080 # Set this to the Server IP / URL, as otherwise the proxy will not work. E6_BASE: https://e621.net + PROXY_AUTH: "" # Leave empty to disable proxy auth append like this to your username "Username:YourPassword" ports: - "8080:8080" # Point this to an Reverse Proxy and set the Proxy Url acordingly. @@ -49,13 +50,18 @@ services: image: minio/mc depends_on: - minio - entrypoint: > - /bin/sh -c " - sleep 5; # Waits for startup - /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin; - /usr/bin/mc mb --ignore-existing minio/e6cache-media; - exit 0; - " + entrypoint: + - /bin/sh + - -c + - | + echo "Waiting for MinIO to be ready..." + until /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin; do + echo "MinIO not ready yet, retrying in 2s..." + sleep 2 + done + /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb minio/e6cache-media --ignore-existing; + exit 0; volumes: db_data: minio_data: \ No newline at end of file diff --git a/src/.env.example b/src/.env.example index a2dc9bb..7954167 100644 --- a/src/.env.example +++ b/src/.env.example @@ -16,4 +16,5 @@ S3_REGION=us-east-1 # Proxy settings PROXY_URL=http://localhost:8080 -E6_BASE=https://e621.net \ No newline at end of file +E6_BASE=https://e621.net +PROXY_AUTH="" # Leave empty to disable proxy auth append like this to your username "Username:YourPassword" \ No newline at end of file diff --git a/src/__debug_bin372161703 b/src/__debug_bin372161703 new file mode 100755 index 0000000..ca53dcc Binary files /dev/null and b/src/__debug_bin372161703 differ diff --git a/src/http_calls.go b/src/http_calls.go index 48b257b..aa53411 100644 --- a/src/http_calls.go +++ b/src/http_calls.go @@ -53,6 +53,51 @@ func makeProxyLink(original string) string { } func proxyAndTransform(c *gin.Context) { + + logging.Debug("Headers: ", c.Request.Header) + + auth := c.Request.Header.Get("Authorization") + + requestUsername := "" + // if proxy auth is enabled, check for auth header + if PROXY_AUTH != "" { + if auth == "" { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"}) + return + } + + // an auth header should normaly look like this: base64 hashed username:password + // ours should look like this username:proxy_auth:password + // the idea is that the user enters their username and then a colon and then the proxy auth. Because most clients dont support colons in the api key, but will accept it in user names + auth = strings.TrimPrefix(auth, "Basic ") + decodedAuth, err := base64.StdEncoding.DecodeString(auth) + if err != nil { + logging.Error("Error decoding auth header: ", err) + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"}) + return + } + // parse it into username, password and proxy auth + authParts := strings.Split(string(decodedAuth), ":") + suppliedProxyAuth := authParts[1] // if you want to check the password, set it to 2 + if len(authParts) != 3 || suppliedProxyAuth != PROXY_AUTH { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"}) + return + } + + c.Request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(authParts[0]+":"+authParts[2]))) + } else { + + if auth == "" { + requestUsername = "anonymous" + } + + // parse the Authorization header + auth = strings.ReplaceAll(auth, "Basic ", "") + decodedAuth, _ := base64.URLEncoding.DecodeString(auth) + splitAuth := strings.Split(string(decodedAuth), ":") + requestUsername = splitAuth[0] // 0 is username, 1 is api key + } + // Construct full target URL originalURL := baseURL + c.Request.URL.Path if c.Request.URL.RawQuery != "" { @@ -77,10 +122,9 @@ func proxyAndTransform(c *gin.Context) { copyHeaders(c.Request.Header, req.Header) // useragent stuff - setUseragent(c, req) + setUseragent(requestUsername, req) logging.Debug("Host: ", c.Request.Host) - logging.Debug("Headers: ", c.Request.Header) logging.Debug("Proxied Headers: ", req.Header) // Perform request @@ -322,7 +366,9 @@ func proxyFile(c *gin.Context) { // download the image from the api req, _ := http.NewRequest("GET", string(url), nil) - setUseragent(c, req) + + // i dont think the username is required for downloading files + setUseragent("", req) client := &http.Client{} resp, err := client.Do(req) if err != nil { @@ -380,30 +426,15 @@ func ProcessPost(c *gin.Context, post *Post) { post.Sample.URL = makeProxyLink(post.Sample.URL) } -func setUseragent(c *gin.Context, req *http.Request) { - auth := c.Request.Header.Get("Authorization") - +func setUseragent(username string, req *http.Request) { var useragent string - - if auth == "" { - useragent = useragentBase - req.Header.Set("User-Agent", useragent) - return - } - - // parse the Authorization header - auth = strings.ReplaceAll(auth, "Basic ", "") - decodedAuth, _ := base64.URLEncoding.DecodeString(auth) - splitAuth := strings.Split(string(decodedAuth), ":") // 0 is username, 1 is api key - - if splitAuth[0] == "" { + if len(username) < 1 { // if it's too small, then forget about it useragent = useragentBase req.Header.Set("User-Agent", useragent) return } - //finally assemble the useragent - useragent = useragentBase + " (Request made on behalf of " + splitAuth[0] + ")" + useragent = useragentBase + " (Request made on behalf of " + username + ")" req.Header.Set("User-Agent", useragent) } diff --git a/src/main.go b/src/main.go index 2863db5..05f1285 100644 --- a/src/main.go +++ b/src/main.go @@ -21,7 +21,7 @@ func isDebug() bool { var ( debugMode string = "false" Database DB - useragentBase = "e6-cache" + useragentBase = "e6-cache (https://github.com/bugmaschine/e6-cache)" port = ":8080" Key []byte // gets randomly generated every launch, and used for signing the urls. maxCacheAge = 1 * time.Hour // idk what's a good value, but 1 hours seems enough @@ -45,8 +45,9 @@ var ( DB_PASS string // Proxy settings - PROXY_URL string - baseURL string + PROXY_URL string + baseURL string + PROXY_AUTH string ) func loadEnv() { @@ -74,6 +75,13 @@ func loadEnv() { // Proxy Settings PROXY_URL = os.Getenv("PROXY_URL") baseURL = os.Getenv("E6_BASE") + PROXY_AUTH = os.Getenv("PROXY_AUTH") + + if PROXY_AUTH != "" { + logging.Debug("Proxy auth is enabled with key: ", PROXY_AUTH) + } else { + logging.Debug("Proxy auth is disabled") + } }