diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3e336e..10169fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - name: Install dependencies - run: pip3 install --break-system-packages ruff pytest + run: pip3 install --break-system-packages ruff pytest -r server/requirements.txt - name: Lint with ruff run: python3 -m ruff check image_upload_transit.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6bf0df5..32e7d08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,8 +71,7 @@ jobs: def caveats <<~EOS - Requires 1Password CLI (installed via 1Password app or `brew install --cask 1password-cli`) - and access to the Transit 1Password Shared vault. + Run `image-upload-transit --login` to authenticate with your Transit Google account. EOS end diff --git a/.github/workflows/server-image.yml b/.github/workflows/server-image.yml new file mode 100644 index 0000000..23726f0 --- /dev/null +++ b/.github/workflows/server-image.yml @@ -0,0 +1,40 @@ +name: Server Image + +on: + push: + branches: [main] + paths: + - "server/**" + pull_request: + branches: [main] + paths: + - "server/**" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: transitapp/image-upload-server + +jobs: + build: + runs-on: [self-hosted, linux, ci-transitapp] + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: server + push: ${{ github.ref == 'refs/heads/main' }} + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest diff --git a/Formula/image-upload-transit.rb b/Formula/image-upload-transit.rb index e2d0926..6e3ac6c 100644 --- a/Formula/image-upload-transit.rb +++ b/Formula/image-upload-transit.rb @@ -11,8 +11,7 @@ def install def caveats <<~EOS - Requires 1Password CLI (installed via 1Password app or `brew install --cask 1password-cli`) - and access to the Transit 1Password Shared vault. + Run `image-upload-transit --login` to authenticate with your Transit Google account. EOS end diff --git a/image_upload_transit.py b/image_upload_transit.py index d42627e..9423515 100755 --- a/image_upload_transit.py +++ b/image_upload_transit.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 -"""CLI tool for uploading images/videos to GCS.""" +"""CLI tool for uploading images/videos to Transit's CDN.""" import argparse import base64 +import hashlib +import http.server import json import mimetypes import os +import secrets import stat -import subprocess import sys -import tempfile import time import urllib.error import urllib.parse @@ -17,94 +18,36 @@ import uuid from pathlib import Path -VERSION = "1.4.0" +VERSION = "2.0.0" CONFIG_DIR = Path.home() / ".config" / "image-upload-transit" +TOKEN_FILE = CONFIG_DIR / "token.json" IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".heic", ".bmp", ".tiff", ".tif", ".svg"} VIDEO_EXTENSIONS = {".mp4", ".mov", ".webm"} ALL_EXTENSIONS = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS +MAX_IMAGE_SIZE = 25 * 1024 * 1024 +MAX_VIDEO_SIZE = 100 * 1024 * 1024 -MAX_IMAGE_SIZE = 25 * 1024 * 1024 # 25 MB -MAX_VIDEO_SIZE = 100 * 1024 * 1024 # 100 MB +SERVER_URL = "https://images.office.transitapp.com" -BUCKET = "image-upload-cli-tool" -BASE_URL = f"https://storage.googleapis.com/{BUCKET}" +GOOGLE_CLIENT_ID = "267231886694-gskp4ng4vghd5pikqfaqp656nn66pkjc.apps.googleusercontent.com" +GOOGLE_CLIENT_SECRET = "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" +GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth" +GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token" -OP_ACCOUNT = "transit.1password.com" -OP_VAULT = "Shared" -OP_ITEM = "image-upload-transit Service Account (Production)" - -class CredentialsError(Exception): - """Raised when credentials cannot be obtained.""" +class AuthError(Exception): + """Raised when authentication fails.""" def error(msg: str) -> None: """Print red error message to stderr.""" - print(f"\033[91m\u2717 {msg}\033[0m", file=sys.stderr) + print(f"\033[91m✗ {msg}\033[0m", file=sys.stderr) def success(msg: str) -> None: """Print green success message to stderr.""" - print(f"\033[92m\u2713 {msg}\033[0m", file=sys.stderr) - - -def check_op_cli() -> None: - """Verify 1Password CLI is installed and user is signed in.""" - try: - subprocess.run(["op", "--version"], capture_output=True, check=True) - except FileNotFoundError: - raise CredentialsError("1Password CLI not found. Install with: brew install 1password-cli") - except subprocess.CalledProcessError: - raise CredentialsError("1Password CLI check failed") - - result = subprocess.run(["op", "account", "list", "--account", OP_ACCOUNT], capture_output=True) - if result.returncode != 0 or not result.stdout.strip(): - raise CredentialsError("Not signed in to 1Password. Run: op signin") - - -def get_credentials(force_refresh: bool = False) -> dict: - """Fetch credentials from 1Password, caching to file.""" - credentials_file = CONFIG_DIR / "credentials.json" - - if not force_refresh and credentials_file.exists(): - with open(credentials_file) as f: - return json.load(f) - - check_op_cli() - - result = subprocess.run( - ["op", "item", "get", OP_ITEM, "--vault", OP_VAULT, "--account", OP_ACCOUNT, "--format", "json"], - capture_output=True, - text=True, - ) - if result.returncode != 0: - raise CredentialsError(f"Failed to fetch credentials from 1Password: {result.stderr.strip()}") - - item = json.loads(result.stdout) - credentials = {} - - for field in item.get("fields", []): - label = field.get("label", "") - value = field.get("value", "") - if label == "client_email": - credentials["client_email"] = value - elif label == "private_key": - credentials["private_key"] = value - elif label == "token_uri": - credentials["token_uri"] = value - - required = ["client_email", "private_key", "token_uri"] - missing = [k for k in required if not credentials.get(k)] - if missing: - raise CredentialsError(f"Missing credential fields: {', '.join(missing)}") - - CONFIG_DIR.mkdir(parents=True, exist_ok=True) - with open(credentials_file, "w") as f: - json.dump(credentials, f, indent=2) - os.chmod(credentials_file, stat.S_IRUSR | stat.S_IWUSR) # chmod 600 - - return credentials + print(f"\033[92m✓ {msg}\033[0m", file=sys.stderr) def validate_file(filepath: str) -> tuple[Path, str]: @@ -138,63 +81,128 @@ def _base64url_encode(data: bytes) -> str: return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") -def get_access_token(credentials: dict) -> str: - """Create JWT, sign with openssl, exchange for OAuth token.""" - now = int(time.time()) - header = {"alg": "RS256", "typ": "JWT"} - payload = { - "iss": credentials["client_email"], - "scope": "https://www.googleapis.com/auth/devstorage.read_write", - "aud": credentials["token_uri"], - "iat": now, - "exp": now + 3600, - } - - header_b64 = _base64url_encode(json.dumps(header, separators=(",", ":")).encode()) - payload_b64 = _base64url_encode(json.dumps(payload, separators=(",", ":")).encode()) - unsigned_jwt = f"{header_b64}.{payload_b64}" +def login() -> str: + """Run Google OAuth PKCE flow. Opens browser, caches tokens. Returns id_token.""" + code_verifier = secrets.token_urlsafe(64) + challenge_bytes = hashlib.sha256(code_verifier.encode("ascii")).digest() + code_challenge = _base64url_encode(challenge_bytes) + + import socket + sock = socket.socket() + sock.bind(("localhost", 0)) + port = sock.getsockname()[1] + sock.close() + + redirect_uri = f"http://localhost:{port}" + params = urllib.parse.urlencode({ + "client_id": GOOGLE_CLIENT_ID, + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": "openid email", + "code_challenge": code_challenge, + "code_challenge_method": "S256", + "access_type": "offline", + "prompt": "consent", + }) + + auth_code = None + + class _CallbackHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + nonlocal auth_code + query = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + auth_code = query.get("code", [None])[0] + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(b"
You can close this tab.
") + + def log_message(self, *args): + pass + + import webbrowser + srv = http.server.HTTPServer(("localhost", port), _CallbackHandler) + webbrowser.open(f"{GOOGLE_AUTH_URL}?{params}") + srv.handle_request() + srv.server_close() + + if not auth_code: + raise AuthError("Login failed: no authorization code received") - with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as key_file: - key_file.write(credentials["private_key"]) - key_path = key_file.name + data = urllib.parse.urlencode({ + "grant_type": "authorization_code", + "code": auth_code, + "redirect_uri": redirect_uri, + "client_id": GOOGLE_CLIENT_ID, + "client_secret": GOOGLE_CLIENT_SECRET, + "code_verifier": code_verifier, + }).encode() + req = urllib.request.Request(GOOGLE_TOKEN_URL, data=data, method="POST") + req.add_header("Content-Type", "application/x-www-form-urlencoded") try: - result = subprocess.run( - ["openssl", "dgst", "-sha256", "-sign", key_path], - input=unsigned_jwt.encode(), - capture_output=True, - ) - if result.returncode != 0: - raise CredentialsError(f"Failed to sign JWT: {result.stderr.decode()}") - signature = _base64url_encode(result.stdout) - finally: - os.unlink(key_path) - - signed_jwt = f"{unsigned_jwt}.{signature}" + with urllib.request.urlopen(req, timeout=30) as resp: + tokens = json.loads(resp.read().decode()) + except urllib.error.URLError as e: + raise AuthError(f"Token exchange failed: {e}") + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + TOKEN_FILE.write_text(json.dumps({ + "id_token": tokens["id_token"], + "refresh_token": tokens["refresh_token"], + }, indent=2)) + os.chmod(TOKEN_FILE, stat.S_IRUSR | stat.S_IWUSR) + + return tokens["id_token"] + + +def _decode_jwt_payload(token: str) -> dict: + """Decode JWT payload without verification (for reading exp claim).""" + payload_b64 = token.split(".")[1] + payload_b64 += "=" * (4 - len(payload_b64) % 4) + return json.loads(base64.urlsafe_b64decode(payload_b64)) + + +def _refresh_id_token(refresh_token: str) -> str: + """Use refresh token to get a new id_token.""" data = urllib.parse.urlencode({ - "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", - "assertion": signed_jwt, + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "client_id": GOOGLE_CLIENT_ID, + "client_secret": GOOGLE_CLIENT_SECRET, }).encode() - - req = urllib.request.Request(credentials["token_uri"], data=data, method="POST") + req = urllib.request.Request(GOOGLE_TOKEN_URL, data=data, method="POST") req.add_header("Content-Type", "application/x-www-form-urlencoded") try: - with urllib.request.urlopen(req, timeout=30) as response: - result = json.loads(response.read().decode()) - return result["access_token"] + with urllib.request.urlopen(req, timeout=30) as resp: + tokens = json.loads(resp.read().decode()) except urllib.error.URLError as e: - raise CredentialsError(f"Failed to obtain access token: {e}") + raise AuthError(f"Token refresh failed: {e}. Run: image-upload-transit --login") + return tokens["id_token"] -def upload_file(filepath: str, credentials: dict) -> str: - """Validate file, generate short ID, upload to GCS, return URL.""" - path, ext = validate_file(filepath) - short_id = uuid.uuid4().hex[:8] - object_name = f"{short_id}{ext}" - access_token = get_access_token(credentials) +def get_token() -> str: + """Load cached token, auto-refreshing if expired.""" + if not TOKEN_FILE.exists(): + raise AuthError("Not logged in. Run: image-upload-transit --login") + + token_data = json.loads(TOKEN_FILE.read_text()) + id_token = token_data["id_token"] + payload = _decode_jwt_payload(id_token) + + if payload["exp"] < time.time(): + id_token = _refresh_id_token(token_data["refresh_token"]) + token_data["id_token"] = id_token + TOKEN_FILE.write_text(json.dumps(token_data, indent=2)) + + return id_token + + +def upload_file(filepath: str, token: str) -> str: + """Validate file, upload to server via HTTP POST, return public URL.""" + path, ext = validate_file(filepath) content_type, _ = mimetypes.guess_type(str(path)) if not content_type: @@ -203,56 +211,62 @@ def upload_file(filepath: str, credentials: dict) -> str: with open(path, "rb") as f: file_data = f.read() - upload_url = ( - f"https://storage.googleapis.com/upload/storage/v1/b/{BUCKET}/o" - f"?uploadType=media&name={urllib.parse.quote(object_name)}" - ) + boundary = uuid.uuid4().hex + body = ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="file"; filename="{path.name}"\r\n' + f"Content-Type: {content_type}\r\n" + f"\r\n" + ).encode() + file_data + f"\r\n--{boundary}--\r\n".encode() - req = urllib.request.Request(upload_url, data=file_data, method="POST") - req.add_header("Authorization", f"Bearer {access_token}") - req.add_header("Content-Type", content_type) - req.add_header("Content-Length", str(len(file_data))) + upload_url = f"{SERVER_URL}/upload" + req = urllib.request.Request(upload_url, data=body, method="POST") + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Content-Type", f"multipart/form-data; boundary={boundary}") try: - with urllib.request.urlopen(req, timeout=120) as response: - if response.status not in (200, 201): - raise ValueError(f"Upload failed with status {response.status}") + with urllib.request.urlopen(req, timeout=120) as resp: + result = json.loads(resp.read().decode()) + return result["url"] except urllib.error.HTTPError as e: + if e.code == 401: + raise AuthError("Authentication failed. Run: image-upload-transit --login") raise ValueError(f"Upload failed: {e.code} {e.reason}") except urllib.error.URLError as e: raise ValueError(f"Upload failed: {e}") - return f"{BASE_URL}/{object_name}" - def main() -> int: """Main entry point.""" + global SERVER_URL + parser = argparse.ArgumentParser( - description="Upload images/videos to GCS", + description="Upload images/videos to Transit's CDN", prog="image-upload-transit", - epilog=""" + epilog=f""" Examples: %(prog)s image.png Upload a single image %(prog)s *.jpg Upload multiple images - %(prog)s --json photo.jpg Output result as JSON (for scripting/agents) - -JSON Output Format (--json): - Success: {"file": "image.png", "url": "https://storage.googleapis.com/image-upload-cli-tool/abc123.png", "success": true} - Error: {"file": "bad.txt", "error": "Unsupported file type", "success": false} - Multiple files produce one JSON object per line (JSONL format). + %(prog)s --json photo.jpg Output result as JSON + %(prog)s --login Authenticate with Google Supported formats: images (jpg, png, gif, webp, avif, heic, bmp, tiff, svg) and videos (mp4, mov, webm). Size limits: 25MB for images, 100MB for videos. +Server: {SERVER_URL} """, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("files", nargs="*", help="Files to upload") parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {VERSION}") - parser.add_argument("--json", action="store_true", help="Output results as JSON (one object per line, for scripting/agents)") - parser.add_argument("--refresh-credentials", action="store_true", help="Force re-fetch credentials from 1Password") + parser.add_argument("--json", action="store_true", help="Output results as JSON (one object per line)") + parser.add_argument("--login", action="store_true", help="Authenticate with your Transit Google account") + parser.add_argument("--server", help="Override server URL") args = parser.parse_args() + if args.server: + SERVER_URL = args.server.rstrip("/") + def output_result(file: str, url: str = None, err: str = None) -> None: if args.json: result = {"file": file, "success": err is None} @@ -266,17 +280,18 @@ def output_result(file: str, url: str = None, err: str = None) -> None: else: success(f"{file} -> {url}") - if args.refresh_credentials and not args.files: + if args.login: try: - get_credentials(force_refresh=True) + login() if args.json: - print(json.dumps({"action": "refresh_credentials", "success": True})) + print(json.dumps({"action": "login", "success": True})) else: - success("Credentials refreshed") - return 0 - except CredentialsError as e: + success("Logged in successfully") + if not args.files: + return 0 + except AuthError as e: if args.json: - print(json.dumps({"action": "refresh_credentials", "error": str(e), "success": False})) + print(json.dumps({"action": "login", "error": str(e), "success": False})) else: error(str(e)) return 2 @@ -286,8 +301,8 @@ def output_result(file: str, url: str = None, err: str = None) -> None: return 1 try: - credentials = get_credentials(force_refresh=args.refresh_credentials) - except CredentialsError as e: + token = get_token() + except AuthError as e: if args.json: print(json.dumps({"error": str(e), "success": False})) else: @@ -297,12 +312,12 @@ def output_result(file: str, url: str = None, err: str = None) -> None: exit_code = 0 for filepath in args.files: try: - url = upload_file(filepath, credentials) + url = upload_file(filepath, token) output_result(filepath, url=url) except ValueError as e: output_result(filepath, err=str(e)) exit_code = 1 - except CredentialsError as e: + except AuthError as e: output_result(filepath, err=str(e)) return 2 diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 0000000..3227c2d --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12-slim + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY server.py . + +EXPOSE 8080 +CMD ["gunicorn", "-b", "0.0.0.0:8080", "-w", "1", "--access-logfile", "-", "server:app"] diff --git a/server/__init__.py b/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/requirements.txt b/server/requirements.txt new file mode 100644 index 0000000..f1c0a1b --- /dev/null +++ b/server/requirements.txt @@ -0,0 +1,3 @@ +flask==3.1.* +PyJWT[crypto]==2.10.* +gunicorn==23.* diff --git a/server/server.py b/server/server.py new file mode 100644 index 0000000..5b9c6e6 --- /dev/null +++ b/server/server.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Image upload server -- accepts authenticated uploads, serves files publicly.""" + +import logging +import mimetypes +import os +import threading +import time +import uuid +from pathlib import Path + +import jwt +from flask import Flask, request, jsonify, send_from_directory +from jwt import PyJWKClient + +app = Flask(__name__) +logger = logging.getLogger(__name__) + +UPLOAD_DIR = Path(os.environ.get("UPLOAD_DIR", "/data/uploads")) +GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_CLIENT_ID", "") +UPLOAD_BASE_URL = os.environ.get("UPLOAD_BASE_URL", "") +ALLOWED_DOMAIN = os.environ.get("ALLOWED_DOMAIN", "transit.app") +CLEANUP_MAX_AGE_DAYS = int(os.environ.get("CLEANUP_MAX_AGE_DAYS", "730")) + +IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".heic", ".bmp", ".tiff", ".tif", ".svg"} +VIDEO_EXTENSIONS = {".mp4", ".mov", ".webm"} +ALL_EXTENSIONS = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS +MAX_IMAGE_SIZE = 25 * 1024 * 1024 +MAX_VIDEO_SIZE = 100 * 1024 * 1024 + +app.config["MAX_CONTENT_LENGTH"] = MAX_VIDEO_SIZE + +GOOGLE_JWKS_URL = "https://www.googleapis.com/oauth2/v3/certs" +_jwks_client = None + + +def _get_jwks_client(): + global _jwks_client + if _jwks_client is None: + _jwks_client = PyJWKClient(GOOGLE_JWKS_URL, cache_keys=True, lifespan=3600) + return _jwks_client + + +def validate_google_token(token: str) -> dict: + """Validate a Google ID token. Returns the decoded claims.""" + signing_key = _get_jwks_client().get_signing_key_from_jwt(token) + payload = jwt.decode( + token, + signing_key.key, + algorithms=["RS256"], + audience=GOOGLE_CLIENT_ID, + options={"verify_iss": True}, + issuer=["accounts.google.com", "https://accounts.google.com"], + ) + if payload.get("hd") != ALLOWED_DOMAIN: + raise jwt.InvalidTokenError(f"Domain '{payload.get('hd')}' not allowed") + return payload + + +def _require_auth(): + """Extract and validate Bearer token from request. Returns claims or aborts.""" + auth = request.headers.get("Authorization", "") + if not auth.startswith("Bearer "): + return None, (jsonify(error="Missing or invalid Authorization header"), 401) + try: + return validate_google_token(auth.removeprefix("Bearer ")), None + except Exception as e: + return None, (jsonify(error=f"Invalid token: {e}"), 401) + + +@app.route("/upload", methods=["POST"]) +def upload(): + claims, err = _require_auth() + if err: + return err + + file = request.files.get("file") + if not file or not file.filename: + return jsonify(error="No file provided"), 400 + + ext = Path(file.filename).suffix.lower() + if ext not in ALL_EXTENSIONS: + return jsonify(error=f"Unsupported file type: {ext}"), 400 + + file.seek(0, 2) + size = file.tell() + file.seek(0) + max_size = MAX_VIDEO_SIZE if ext in VIDEO_EXTENSIONS else MAX_IMAGE_SIZE + if size > max_size: + return jsonify(error=f"File too large: {size} bytes (max {max_size})"), 400 + + filename = f"{uuid.uuid4().hex[:8]}{ext}" + UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + file.save(UPLOAD_DIR / filename) + + url = f"{UPLOAD_BASE_URL}/{filename}" + logger.info("Upload: %s by %s (%d bytes)", filename, claims.get("email"), size) + return jsonify(url=url, filename=filename) + + +@app.route("/