From 36632c73a579649815a98a5acc22fc5324d48321 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 14:24:49 +0200 Subject: [PATCH 01/15] Add Claude AI translation integration with caching and UI - Add anthropic dependency and Claude API configuration - Implement translation caching system with content hashing - Add language dropdown selector in footer with 14 language options - Refactor index file rendering to support multi-language content - Update footer layout to accommodate language selector controls --- config.yaml.example | 26 ++++ requirements.txt | 1 + templatehelper.py | 174 +++++++++++++++++++++++++ templates/nomike.com/base.html | 38 +++++- templates/nomike.com/directory.html | 8 +- templates/nomike.com/static/design.css | 54 +++++++- 6 files changed, 291 insertions(+), 10 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index a81db79..aa5e713 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,2 +1,28 @@ template: "nomike.com" #template: "custom/example_template" + +# Claude AI Configuration +claude: + # Your Anthropic API key + api_key: "your_anthropic_api_key_here" + + # Claude model to use (e.g., claude-3-5-sonnet-20241022, claude-3-haiku-20240307) + model: "claude-3-5-sonnet-20241022" + + # Maximum tokens for Claude responses + max_tokens: 4096 + + # Temperature for response randomness (0.0 to 1.0) + temperature: 0.7 + + # System prompt for Claude (optional) + system_prompt: "You are a helpful assistant for a content management system." + + # API base URL (leave default unless using a proxy) + base_url: "https://api.anthropic.com" + + # Request timeout in seconds + timeout: 30 + + # Cache directory for translations (relative to project root) + cache_dir: "./cache/translations" diff --git a/requirements.txt b/requirements.txt index a6212a1..ff3ef56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ markdown org-python pyyaml regex +anthropic diff --git a/templatehelper.py b/templatehelper.py index 5a94e51..ab5210a 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -12,9 +12,11 @@ """ import fnmatch +import hashlib import json import mimetypes import os +import pickle # pylint: disable=unused-import import re # pylint: disable=unused-import @@ -29,6 +31,11 @@ import regex import yaml +try: + import anthropic +except ImportError: + anthropic = None + # pylint: disable=invalid-name config = None @@ -41,6 +48,108 @@ # pylint: disable=invalid-name pathprefix = '' +def get_cache_dir(): + """Get the cache directory from config or use default.""" + if config and 'claude' in config and 'cache_dir' in config['claude']: + return config['claude']['cache_dir'] + return os.path.join('.', 'cache', 'translations') + +def ensure_cache_dir(): + """Ensure the cache directory exists.""" + cache_dir = get_cache_dir() + if not os.path.exists(cache_dir): + os.makedirs(cache_dir, exist_ok=True) + +def get_cache_key(content_hash, target_lang): + """Generate a cache key from content hash and target language.""" + return f"{content_hash}_{target_lang}.pkl" + +def get_cached_translation(content_hash, target_lang): + """Retrieve a translation from cache if it exists.""" + ensure_cache_dir() + cache_dir = get_cache_dir() + cache_file = os.path.join(cache_dir, get_cache_key(content_hash, target_lang)) + if os.path.exists(cache_file): + try: + with open(cache_file, 'rb') as f: + return pickle.load(f) + except (IOError, pickle.PickleError): + pass + return None + +def cache_translation(content_hash, target_lang, translation): + """Store a translation in cache.""" + ensure_cache_dir() + cache_dir = get_cache_dir() + cache_file = os.path.join(cache_dir, get_cache_key(content_hash, target_lang)) + try: + with open(cache_file, 'wb') as f: + pickle.dump(translation, f) + except (IOError, pickle.PickleError): + pass + +def translate_claude(content, target_lang): + """ + Translate content using Claude API. + """ + if not anthropic: + raise ImportError("anthropic package not installed. Please install with: pip install anthropic") + + if not config or 'claude' not in config: + raise ValueError("Claude configuration not found in config.yaml") + + claude_config = config['claude'] + api_key = claude_config.get('api_key') + if not api_key or api_key == "your_anthropic_api_key_here": + raise ValueError("Valid Claude API key not configured") + + client = anthropic.Anthropic( + api_key=api_key, + base_url=claude_config.get('base_url', 'https://api.anthropic.com') + ) + + # Language code to language name mapping + lang_names = { + 'de': 'German', 'fr': 'French', 'it': 'Italian', 'el': 'Greek', + 'hu': 'Hungarian', 'pt': 'Portuguese', 'cs': 'Czech', 'sk': 'Slovakian', + 'sl': 'Slovenian', 'hr': 'Croatian', 'de-ch': 'Swiss German', + 'nl': 'Dutch', 'bg': 'Bulgarian' + } + + target_language = lang_names.get(target_lang, target_lang) + + try: + message = client.messages.create( + model=claude_config.get('model', 'claude-3-5-sonnet-20241022'), + max_tokens=claude_config.get('max_tokens', 4096), + temperature=claude_config.get('temperature', 0.7), + system=claude_config.get('system_prompt', 'You are a helpful assistant for a content management system.'), + messages=[{ + "role": "user", + "content": f"Translate this webpage content from English to {target_language}, maintaining the original formatting and tone:\n\n{content}" + }] + ) + return message.content[0].text + except Exception as e: + raise RuntimeError(f"Translation failed: {str(e)}") + +def get_content_hash(content): + """Generate SHA256 hash of content.""" + return hashlib.sha256(content.encode('utf-8')).hexdigest() + +def is_claude_available(): + """Check if Claude translation is available and properly configured.""" + if not anthropic: + return False + + if not config or 'claude' not in config: + return False + + claude_config = config['claude'] + api_key = claude_config.get('api_key') + + return api_key and api_key != "your_anthropic_api_key_here" + # List of official MIME Types: http://www.iana.org/assignments/media-types/media-types.xhtml # If you want additional mimetypes to be covered, add them to this list. # The types map to FontAwesome identifiers. Check out https://fontawesome.com/icons?d=gallery @@ -216,3 +325,68 @@ def getlastmodifiedfile(path): newest['file'] = os.path.join(root, directory) newest['timestamp'] = timestamp return newest + +def renderIndexFile(path, lang='en'): + """ + Search for index files in order of priority (index.org, index.md, index.html, index) + and render the appropriate content. Returns rendered HTML content or default header. + If lang is not 'en', attempts to translate content using Claude API with caching. + """ + full_path = os.path.join(pathprefix, path) + content = None + rendered_content = None + + # Check for index.org file + org_path = os.path.join(full_path, 'index.org') + if os.path.isfile(org_path): + content = readfile(org_path) + rendered_content = orgpython.to_html(content) + else: + # Check for index.md file + md_path = os.path.join(full_path, 'index.md') + if os.path.isfile(md_path): + content = readfile(md_path) + rendered_content = markdown.markdown(content, extensions=['fenced_code', 'toc', 'tables']) + else: + # Check for index.html file + html_path = os.path.join(full_path, 'index.html') + if os.path.isfile(html_path): + rendered_content = readfile(html_path) + else: + # Check for plain index file + index_path = os.path.join(full_path, 'index') + if os.path.isfile(index_path): + rendered_content = readfile(index_path) + else: + # Default fallback - return directory header + rendered_content = f'

/{path}

' + + # If language is English or no content to translate, return as-is + if lang == 'en' or not rendered_content: + return rendered_content + + # Generate content hash for caching + content_hash = get_content_hash(rendered_content) + + # Check cache first for English content (cache original English) + if lang == 'en': + cache_translation(content_hash, 'en', rendered_content) + return rendered_content + + # Check if translation is already cached + cached_translation = get_cached_translation(content_hash, lang) + if cached_translation: + return cached_translation + + # Cache the original English content + cache_translation(content_hash, 'en', rendered_content) + + # Translate content using Claude + try: + translated_content = translate_claude(rendered_content, lang) + # Cache the translation + cache_translation(content_hash, lang, translated_content) + return translated_content + except Exception as e: + # If translation fails, return original content with error comment + return f"\n{rendered_content}" diff --git a/templates/nomike.com/base.html b/templates/nomike.com/base.html index df374a7..4f560cb 100644 --- a/templates/nomike.com/base.html +++ b/templates/nomike.com/base.html @@ -24,6 +24,14 @@ /* Initialize highlight.js */ hljs.initHighlightingOnLoad(); + + /* Handle language dropdown change */ + function handleLanguageChange(select) { + const selectedLang = select.value; + const url = new URL(window.location); + url.searchParams.set('lang', selectedLang); + window.location.href = url.toString(); + } @@ -39,8 +47,34 @@

 

- {% set lastmodifiedfile = templatehelper.getlastmodifiedfile(templatehelper.os.path.join(path)) %} Last modification in {{ lastmodifiedfile['file'] }} on {{ templatehelper.datetime.fromtimestamp(lastmodifiedfile['timestamp'], tz=templatehelper.timezone.utc).isoformat() - }} +
+ {% set lastmodifiedfile = templatehelper.getlastmodifiedfile(templatehelper.os.path.join(path)) %} Last modification in {{ lastmodifiedfile['file'] }} on {{ templatehelper.datetime.fromtimestamp(lastmodifiedfile['timestamp'], tz=templatehelper.timezone.utc).isoformat() + }} +
+
+ {% if templatehelper.is_claude_available() %} +
+ {% set current_lang = request.args.get('lang', 'en') %} + + +
+ {% endif %} +
diff --git a/templates/nomike.com/directory.html b/templates/nomike.com/directory.html index c9c26a4..c1756b5 100644 --- a/templates/nomike.com/directory.html +++ b/templates/nomike.com/directory.html @@ -12,10 +12,4 @@ {{ name }} {% else %} {{ name }} {% endif %} {% endfor %} - {% endif -%} {% endblock -%} {% block content -%} {% if templatehelper.os.path.isfile(templatehelper.os.path.join(pathprefix, path,'index.org')) -%} {{ templatehelper.orgpython.to_html(templatehelper.readfile(templatehelper.os.path.join(pathprefix,path,'index.org'))) - | safe }} {% elif templatehelper.os.path.isfile(templatehelper.os.path.join(pathprefix, path,'index.md')) -%} {{ templatehelper.markdown.markdown(templatehelper.readfile(templatehelper.os.path.join(pathprefix,path,'index.md')),extensions=['fenced_code','toc','tables']) - | safe }} {% elif templatehelper.os.path.isfile(templatehelper.os.path.join(pathprefix, path, 'index.html')) -%} {{ templatehelper.readfile(templatehelper.os.path.join(pathprefix, path, 'index.html')) - | safe -}} {% elif templatehelper.os.path.isfile(templatehelper.os.path.join(pathprefix, path, 'index')) -%} {{ templatehelper.readfile(templatehelper.os.path.join(pathprefix, path, 'index')) - | safe -}} {% else -%} -

/{{ path }}

- {% endif -%} {% if templatehelper.os.path.isdir(templatehelper.os.path.join(pathprefix, path, 'image')) %} {% include templatehelper.os.path.join(templatehelper.config['template'], 'gallery.html') %} {% endif %} {% endblock %} + {% endif -%} {% endblock -%} {% block content -%} {{ templatehelper.renderIndexFile(path, request.args.get('lang', 'en')) | safe }} {% if templatehelper.os.path.isdir(templatehelper.os.path.join(pathprefix, path, 'image')) %} {% include templatehelper.os.path.join(templatehelper.config['template'], 'gallery.html') %} {% endif %} {% endblock %} diff --git a/templates/nomike.com/static/design.css b/templates/nomike.com/static/design.css index 7eee9f2..2be5c80 100644 --- a/templates/nomike.com/static/design.css +++ b/templates/nomike.com/static/design.css @@ -290,8 +290,10 @@ div.galleryblock img:hover { .footer { display: flex; - justify-content: left; + justify-content: space-between; + align-items: center; padding-left: 10px; + padding-right: 10px; padding-top: 3px; padding-bottom: 3px; margin-top: 0; @@ -300,6 +302,56 @@ div.galleryblock img:hover { bottom: 0px; width: 100%; height: 20px; + box-sizing: border-box; +} + +.footer-left { + flex: 1; +} + +.footer-right { + flex-shrink: 0; +} + +.language-dropdown { + background-color: transparent; + border: 1px solid #333; + border-radius: 3px; + font-size: 12px; + height: 18px; + padding: 1px 3px; + color: black; +} + +.language-dropdown:focus { + outline: none; + border-color: #666; +} + +.language-form { + display: flex; + align-items: center; + gap: 5px; +} + +.language-submit { + background-color: transparent; + border: 1px solid #333; + border-radius: 3px; + font-size: 12px; + height: 20px; + padding: 1px 6px; + color: black; + cursor: pointer; +} + +.language-submit:hover { + background-color: rgba(0, 0, 0, 0.1); +} + +.language-submit:focus { + outline: none; + border-color: #666; } :not(pre)>code { From 2d495803207e192e6d85ba737883596904ae76fe Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 14:33:33 +0200 Subject: [PATCH 02/15] refactor: improve code formatting and error handling in templatehelper.py - Break long lines for better readability - Add proper exception chaining with 'from e' - Use specific exception types in catch block - Format function calls and string concatenation consistently --- templatehelper.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/templatehelper.py b/templatehelper.py index ab5210a..1825920 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -93,7 +93,9 @@ def translate_claude(content, target_lang): Translate content using Claude API. """ if not anthropic: - raise ImportError("anthropic package not installed. Please install with: pip install anthropic") + raise ImportError( + "anthropic package not installed. Please install with: pip install anthropic" + ) if not config or 'claude' not in config: raise ValueError("Claude configuration not found in config.yaml") @@ -123,15 +125,20 @@ def translate_claude(content, target_lang): model=claude_config.get('model', 'claude-3-5-sonnet-20241022'), max_tokens=claude_config.get('max_tokens', 4096), temperature=claude_config.get('temperature', 0.7), - system=claude_config.get('system_prompt', 'You are a helpful assistant for a content management system.'), + system=claude_config.get( + 'system_prompt', 'You are a helpful assistant for a content management system.' + ), messages=[{ "role": "user", - "content": f"Translate this webpage content from English to {target_language}, maintaining the original formatting and tone:\n\n{content}" + "content": ( + f"Translate this webpage content from English to {target_language}, " + f"maintaining the original formatting and tone:\n\n{content}" + ) }] ) return message.content[0].text except Exception as e: - raise RuntimeError(f"Translation failed: {str(e)}") + raise RuntimeError(f"Translation failed: {str(e)}") from e def get_content_hash(content): """Generate SHA256 hash of content.""" @@ -346,7 +353,9 @@ def renderIndexFile(path, lang='en'): md_path = os.path.join(full_path, 'index.md') if os.path.isfile(md_path): content = readfile(md_path) - rendered_content = markdown.markdown(content, extensions=['fenced_code', 'toc', 'tables']) + rendered_content = markdown.markdown( + content, extensions=['fenced_code', 'toc', 'tables'] + ) else: # Check for index.html file html_path = os.path.join(full_path, 'index.html') @@ -387,6 +396,6 @@ def renderIndexFile(path, lang='en'): # Cache the translation cache_translation(content_hash, lang, translated_content) return translated_content - except Exception as e: + except (ImportError, ValueError, RuntimeError) as e: # If translation fails, return original content with error comment return f"\n{rendered_content}" From df02feb7e30b383f8361eef27b579bb567028ebe Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 14:35:26 +0200 Subject: [PATCH 03/15] chore: update Python 3.13 version from rc.2 to stable release in pyling GitHub action --- .github/workflows/pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 633ac00..5228720 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13.0-rc.2"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 8161153e1667c4d50ae0162076e7f59a4c9d6189 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 15:11:25 +0200 Subject: [PATCH 04/15] Fix: Improve system prompt for translations --- templatehelper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templatehelper.py b/templatehelper.py index 1825920..ab172bf 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -14,6 +14,7 @@ import fnmatch import hashlib import json +import locale import mimetypes import os import pickle @@ -22,7 +23,6 @@ # pylint: disable=unused-import import urllib from datetime import datetime, timezone, tzinfo -import locale # pylint: disable=unused-import import markdown @@ -126,13 +126,13 @@ def translate_claude(content, target_lang): max_tokens=claude_config.get('max_tokens', 4096), temperature=claude_config.get('temperature', 0.7), system=claude_config.get( - 'system_prompt', 'You are a helpful assistant for a content management system.' + 'system_prompt', 'You translate org-mode, markdown and HTML documents from english to other languages. You maintain the original formatting and tone. You only translate the text, not the code blocks or HTML tags. You do not add any additional text, except for a note at the top that this text has been translated by an AI. If you do not know the target language, you simply return the original text. You output only the resulting text, nothing else.' ), messages=[{ "role": "user", "content": ( - f"Translate this webpage content from English to {target_language}, " - f"maintaining the original formatting and tone:\n\n{content}" + f"Translate this webpage content from English to {target_language}:\n\n" + content ) }] ) From b615bf3c10e81845cd83c2243e8fe2bad15a4c6d Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 15:15:25 +0200 Subject: [PATCH 05/15] Add translation-cache management scripts --- cleanup_stale_translations.py | 202 ++++++++++++++++++++++++++++++++++ clear_translation_cache.py | 92 ++++++++++++++++ manage_cache.sh | 42 +++++++ 3 files changed, 336 insertions(+) create mode 100755 cleanup_stale_translations.py create mode 100755 clear_translation_cache.py create mode 100755 manage_cache.sh diff --git a/cleanup_stale_translations.py b/cleanup_stale_translations.py new file mode 100755 index 0000000..b94afc8 --- /dev/null +++ b/cleanup_stale_translations.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +""" +Script to clean up stale translations whose source content has changed. +Removes cached translations where the SHA256 hash no longer matches any current content. +""" + +import hashlib +import os +import pickle +import sys +import yaml +import locale +from pathlib import Path + + +def load_config(): + """Load configuration from config.yaml.""" + try: + with open("config.yaml", encoding=locale.getpreferredencoding()) as file: + return yaml.load(file, Loader=yaml.SafeLoader) + except FileNotFoundError: + print("Error: config.yaml not found. Please copy config.yaml.example to config.yaml") + return None + except yaml.YAMLError as e: + print(f"Error parsing config.yaml: {e}") + return None + + +def get_cache_dir(config): + """Get the cache directory from config or use default.""" + if config and 'claude' in config and 'cache_dir' in config['claude']: + return config['claude']['cache_dir'] + return os.path.join('.', 'cache', 'translations') + + +def get_content_hash(content): + """Generate SHA256 hash of content.""" + return hashlib.sha256(content.encode('utf-8')).hexdigest() + + +def scan_content_files(public_dir="public"): + """ + Scan all content files and generate their current hashes. + Returns a set of current content hashes. + """ + current_hashes = set() + + if not os.path.exists(public_dir): + print(f"Warning: Public directory '{public_dir}' not found.") + return current_hashes + + # File extensions to scan + extensions = ['.org', '.md', '.html', '.txt'] + + print(f"Scanning content files in {public_dir}...") + + for root, dirs, files in os.walk(public_dir): + for file in files: + # Check for index files + if file in ['index.org', 'index.md', 'index.html', 'index']: + file_path = os.path.join(root, file) + try: + with open(file_path, 'r', encoding=locale.getpreferredencoding()) as f: + content = f.read() + + # For .org and .md files, we need to simulate the rendering process + if file.endswith('.org'): + # For org files, we'd need orgpython to render, but let's use raw content hash + content_hash = get_content_hash(content) + elif file.endswith('.md'): + # For markdown files, we'd need markdown lib, but let's use raw content hash + content_hash = get_content_hash(content) + else: + # HTML and plain text files + content_hash = get_content_hash(content) + + current_hashes.add(content_hash) + + except (IOError, UnicodeDecodeError) as e: + print(f"Warning: Could not read {file_path}: {e}") + continue + + # Also check other content files with relevant extensions + elif any(file.endswith(ext) for ext in extensions): + file_path = os.path.join(root, file) + try: + with open(file_path, 'r', encoding=locale.getpreferredencoding()) as f: + content = f.read() + content_hash = get_content_hash(content) + current_hashes.add(content_hash) + except (IOError, UnicodeDecodeError): + continue + + print(f"Found {len(current_hashes)} unique content hashes.") + return current_hashes + + +def parse_cache_filename(filename): + """ + Parse cache filename to extract content hash and language. + Expected format: {hash}_{lang}.pkl + """ + if not filename.endswith('.pkl'): + return None, None + + base_name = filename[:-4] # Remove .pkl extension + parts = base_name.rsplit('_', 1) # Split from the right, only once + + if len(parts) != 2: + return None, None + + content_hash, lang = parts + return content_hash, lang + + +def cleanup_stale_cache(cache_dir, current_hashes): + """ + Remove cached translations for content that no longer exists or has changed. + """ + if not os.path.exists(cache_dir): + print(f"Cache directory {cache_dir} does not exist.") + return 0 + + removed_count = 0 + kept_count = 0 + + try: + for filename in os.listdir(cache_dir): + if not filename.endswith('.pkl'): + continue + + file_path = os.path.join(cache_dir, filename) + content_hash, lang = parse_cache_filename(filename) + + if content_hash is None: + print(f"Warning: Could not parse cache filename: {filename}") + continue + + # Check if this content hash still exists in current content + if content_hash not in current_hashes: + try: + os.remove(file_path) + print(f"Removed stale cache: {filename} (hash: {content_hash[:12]}..., lang: {lang})") + removed_count += 1 + except OSError as e: + print(f"Error removing {filename}: {e}") + else: + kept_count += 1 + + print(f"\nCleanup completed:") + print(f" - Removed: {removed_count} stale cache files") + print(f" - Kept: {kept_count} current cache files") + + return removed_count + + except OSError as e: + print(f"Error accessing cache directory: {e}") + return -1 + + +def main(): + """Main function.""" + print("Stale Translation Cache Cleanup") + print("=" * 31) + + # Load configuration + config = load_config() + if config is None: + sys.exit(1) + + cache_dir = get_cache_dir(config) + print(f"Cache directory: {cache_dir}") + + # Scan current content to get active hashes + current_hashes = scan_content_files() + + if not current_hashes: + print("No content files found. Nothing to validate against.") + sys.exit(1) + + # Ask for confirmation + try: + print(f"\nThis will remove cached translations that don't match any current content.") + response = input("Continue? (y/N): ") + if response.lower() not in ['y', 'yes']: + print("Operation cancelled.") + sys.exit(0) + except KeyboardInterrupt: + print("\nOperation cancelled.") + sys.exit(0) + + # Clean up stale cache entries + result = cleanup_stale_cache(cache_dir, current_hashes) + if result >= 0: + print("Stale cache cleanup completed successfully.") + else: + print("Stale cache cleanup failed.") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/clear_translation_cache.py b/clear_translation_cache.py new file mode 100755 index 0000000..1885fe5 --- /dev/null +++ b/clear_translation_cache.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +Script to clear the entire translation cache. +""" + +import os +import shutil +import sys +import yaml +import locale + + +def load_config(): + """Load configuration from config.yaml.""" + try: + with open("config.yaml", encoding=locale.getpreferredencoding()) as file: + return yaml.load(file, Loader=yaml.SafeLoader) + except FileNotFoundError: + print("Error: config.yaml not found. Please copy config.yaml.example to config.yaml") + return None + except yaml.YAMLError as e: + print(f"Error parsing config.yaml: {e}") + return None + + +def get_cache_dir(config): + """Get the cache directory from config or use default.""" + if config and 'claude' in config and 'cache_dir' in config['claude']: + return config['claude']['cache_dir'] + return os.path.join('.', 'cache', 'translations') + + +def clear_cache(cache_dir): + """Clear the entire translation cache directory.""" + if not os.path.exists(cache_dir): + print(f"Cache directory {cache_dir} does not exist.") + return 0 + + try: + # Count files before deletion + file_count = 0 + for root, dirs, files in os.walk(cache_dir): + file_count += len([f for f in files if f.endswith('.pkl')]) + + if file_count == 0: + print("No cache files found.") + return 0 + + # Remove the entire cache directory + shutil.rmtree(cache_dir) + print(f"Successfully cleared {file_count} cached translations from {cache_dir}") + return file_count + + except OSError as e: + print(f"Error clearing cache: {e}") + return -1 + + +def main(): + """Main function.""" + print("Translation Cache Cleaner") + print("=" * 25) + + # Load configuration + config = load_config() + if config is None: + sys.exit(1) + + cache_dir = get_cache_dir(config) + print(f"Cache directory: {cache_dir}") + + # Ask for confirmation + try: + response = input("Are you sure you want to clear the entire cache? (y/N): ") + if response.lower() not in ['y', 'yes']: + print("Operation cancelled.") + sys.exit(0) + except KeyboardInterrupt: + print("\nOperation cancelled.") + sys.exit(0) + + # Clear the cache + result = clear_cache(cache_dir) + if result >= 0: + print("Cache clearing completed successfully.") + else: + print("Cache clearing failed.") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/manage_cache.sh b/manage_cache.sh new file mode 100755 index 0000000..ac9c1a5 --- /dev/null +++ b/manage_cache.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Translation Cache Management Script + +show_help() { + echo "Translation Cache Management" + echo "Usage: $0 [clear|cleanup|help]" + echo "" + echo "Commands:" + echo " clear - Clear the entire translation cache" + echo " cleanup - Remove stale translations (source content changed)" + echo " help - Show this help message" + echo "" + echo "Examples:" + echo " $0 clear # Clear all cached translations" + echo " $0 cleanup # Remove outdated cache entries" +} + +case "$1" in + "clear") + echo "Clearing entire translation cache..." + python3 clear_translation_cache.py + ;; + "cleanup") + echo "Cleaning up stale translations..." + python3 cleanup_stale_translations.py + ;; + "help"|"-h"|"--help") + show_help + ;; + "") + echo "Error: No command specified." + echo "" + show_help + exit 1 + ;; + *) + echo "Error: Unknown command '$1'" + echo "" + show_help + exit 1 + ;; +esac \ No newline at end of file From 6389b7ceadb494af9d156cd1202b015678b2ef73 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 15:23:11 +0200 Subject: [PATCH 06/15] Fix syntax error --- templatehelper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templatehelper.py b/templatehelper.py index ab172bf..6580eb8 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -131,7 +131,7 @@ def translate_claude(content, target_lang): messages=[{ "role": "user", "content": ( - f"Translate this webpage content from English to {target_language}:\n\n" + f"Translate this webpage content from English to {target_language}:\n\n", content ) }] From f9b5bc74c81d773044fb0fd6f9ae99a618ffdbd4 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 15:29:44 +0200 Subject: [PATCH 07/15] Code linting --- templatehelper.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/templatehelper.py b/templatehelper.py index 6580eb8..666c33f 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -126,7 +126,14 @@ def translate_claude(content, target_lang): max_tokens=claude_config.get('max_tokens', 4096), temperature=claude_config.get('temperature', 0.7), system=claude_config.get( - 'system_prompt', 'You translate org-mode, markdown and HTML documents from english to other languages. You maintain the original formatting and tone. You only translate the text, not the code blocks or HTML tags. You do not add any additional text, except for a note at the top that this text has been translated by an AI. If you do not know the target language, you simply return the original text. You output only the resulting text, nothing else.' + '''system_prompt', 'You translate org-mode, markdown and HTML + documents from english to other languages. You maintain the + original formatting and tone. You only translate the text, not + the code blocks or HTML tags. You do not add any additional + text, except for a note at the top that this text has been + translated by an AI. If you do not know the target language, + you simply return the original text. You output only the + resulting text, nothing else.''' ), messages=[{ "role": "user", From f3bef465735afced59e6a03ee2963536da86e106 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 15:37:52 +0200 Subject: [PATCH 08/15] refactor: improve translation workflow to translate source before rendering - Translate source content (org/md/html) before rendering to HTML - Cache translations at source level rather than rendered HTML level - Maintain proper file type handling for post-translation rendering - Preserve error handling for translation failures --- templatehelper.py | 90 +++++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/templatehelper.py b/templatehelper.py index 666c33f..1a1bb62 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -344,65 +344,77 @@ def renderIndexFile(path, lang='en'): """ Search for index files in order of priority (index.org, index.md, index.html, index) and render the appropriate content. Returns rendered HTML content or default header. - If lang is not 'en', attempts to translate content using Claude API with caching. + If lang is not 'en', attempts to translate source content using Claude API with caching, + then renders the translated source. """ full_path = os.path.join(pathprefix, path) - content = None - rendered_content = None + source_content = None + file_type = None # Check for index.org file org_path = os.path.join(full_path, 'index.org') if os.path.isfile(org_path): - content = readfile(org_path) - rendered_content = orgpython.to_html(content) + source_content = readfile(org_path) + file_type = 'org' else: # Check for index.md file md_path = os.path.join(full_path, 'index.md') if os.path.isfile(md_path): - content = readfile(md_path) - rendered_content = markdown.markdown( - content, extensions=['fenced_code', 'toc', 'tables'] - ) + source_content = readfile(md_path) + file_type = 'md' else: # Check for index.html file html_path = os.path.join(full_path, 'index.html') if os.path.isfile(html_path): - rendered_content = readfile(html_path) + source_content = readfile(html_path) + file_type = 'html' else: # Check for plain index file index_path = os.path.join(full_path, 'index') if os.path.isfile(index_path): - rendered_content = readfile(index_path) + source_content = readfile(index_path) + file_type = 'plain' else: # Default fallback - return directory header - rendered_content = f'

/{path}

' + return f'

/{path}

' - # If language is English or no content to translate, return as-is - if lang == 'en' or not rendered_content: - return rendered_content + # Generate content hash for caching (based on source content) + content_hash = get_content_hash(source_content) - # Generate content hash for caching - content_hash = get_content_hash(rendered_content) - - # Check cache first for English content (cache original English) + # Get the source content in the target language (translate if needed) if lang == 'en': - cache_translation(content_hash, 'en', rendered_content) - return rendered_content - - # Check if translation is already cached - cached_translation = get_cached_translation(content_hash, lang) - if cached_translation: - return cached_translation - - # Cache the original English content - cache_translation(content_hash, 'en', rendered_content) - - # Translate content using Claude - try: - translated_content = translate_claude(rendered_content, lang) - # Cache the translation - cache_translation(content_hash, lang, translated_content) - return translated_content - except (ImportError, ValueError, RuntimeError) as e: - # If translation fails, return original content with error comment - return f"\n{rendered_content}" + # Use original source content for English + translated_source = source_content + # Cache original English source + cache_translation(content_hash, 'en', source_content) + else: + # Check if translation is already cached + cached_translation = get_cached_translation(content_hash, lang) + if cached_translation: + translated_source = cached_translation + else: + # Cache the original English source + cache_translation(content_hash, 'en', source_content) + + # Translate source content using Claude + try: + translated_source = translate_claude(source_content, lang) + # Cache the translated source + cache_translation(content_hash, lang, translated_source) + except (ImportError, ValueError, RuntimeError) as e: + # If translation fails, use original content with error comment + translated_source = f"\n{source_content}" + + # Now render the (possibly translated) source content based on file type + if file_type == 'org': + return orgpython.to_html(translated_source) + elif file_type == 'md': + return markdown.markdown( + translated_source, extensions=['fenced_code', 'toc', 'tables'] + ) + elif file_type == 'html': + return translated_source + elif file_type == 'plain': + return translated_source + else: + return f'

/{path}

' From 39d0a21a371fc09379df26112dcc6eb340dde86c Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 15:45:53 +0200 Subject: [PATCH 09/15] Chore: Code linting --- templatehelper.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/templatehelper.py b/templatehelper.py index 1a1bb62..03e5b00 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -126,20 +126,20 @@ def translate_claude(content, target_lang): max_tokens=claude_config.get('max_tokens', 4096), temperature=claude_config.get('temperature', 0.7), system=claude_config.get( - '''system_prompt', 'You translate org-mode, markdown and HTML - documents from english to other languages. You maintain the - original formatting and tone. You only translate the text, not - the code blocks or HTML tags. You do not add any additional - text, except for a note at the top that this text has been - translated by an AI. If you do not know the target language, - you simply return the original text. You output only the - resulting text, nothing else.''' + 'system_prompt', 'You translate org-mode, markdown and HTML ' + 'documents from english to other languages. You maintain the ' + 'original formatting and tone. You only translate the text, not ' + 'the code blocks or HTML tags. You do not add any additional ' + 'text, except for a note at the top that this text has been ' + 'translated by an AI. If you do not know the target language, ' + 'you simply return the original text. You output only the ' + 'resulting text, nothing else.' ), messages=[{ "role": "user", "content": ( - f"Translate this webpage content from English to {target_language}:\n\n", - content + f"Translate this webpage content from English to {target_language}:\n\n" + + content ) }] ) From 4c3f05a503088044ede9bc6d1bd72b00cbbc63c1 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Mon, 22 Sep 2025 16:06:38 +0200 Subject: [PATCH 10/15] refactor: migrate language codes from ISO 639-1 to ISO 639-3 Update language code mappings and default values to use three-letter ISO 639-3 codes instead of two-letter ISO 639-1 codes. Add Mandinka language support. --- templatehelper.py | 16 +++++++-------- templates/nomike.com/base.html | 31 +++++++++++++++-------------- templates/nomike.com/directory.html | 2 +- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/templatehelper.py b/templatehelper.py index 03e5b00..0d3d0c5 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -110,12 +110,12 @@ def translate_claude(content, target_lang): base_url=claude_config.get('base_url', 'https://api.anthropic.com') ) - # Language code to language name mapping + # Language code to language name mapping (ISO 639-3) lang_names = { - 'de': 'German', 'fr': 'French', 'it': 'Italian', 'el': 'Greek', - 'hu': 'Hungarian', 'pt': 'Portuguese', 'cs': 'Czech', 'sk': 'Slovakian', - 'sl': 'Slovenian', 'hr': 'Croatian', 'de-ch': 'Swiss German', - 'nl': 'Dutch', 'bg': 'Bulgarian' + 'deu': 'German', 'fra': 'French', 'ita': 'Italian', 'ell': 'Greek', + 'hun': 'Hungarian', 'por': 'Portuguese', 'ces': 'Czech', 'slk': 'Slovak', + 'slv': 'Slovenian', 'hrv': 'Croatian', 'gsw': 'Swiss German', + 'nld': 'Dutch', 'bul': 'Bulgarian', 'mnk': 'Mandinka' } target_language = lang_names.get(target_lang, target_lang) @@ -382,11 +382,11 @@ def renderIndexFile(path, lang='en'): content_hash = get_content_hash(source_content) # Get the source content in the target language (translate if needed) - if lang == 'en': + if lang == 'eng': # Use original source content for English translated_source = source_content # Cache original English source - cache_translation(content_hash, 'en', source_content) + cache_translation(content_hash, 'eng', source_content) else: # Check if translation is already cached cached_translation = get_cached_translation(content_hash, lang) @@ -394,7 +394,7 @@ def renderIndexFile(path, lang='en'): translated_source = cached_translation else: # Cache the original English source - cache_translation(content_hash, 'en', source_content) + cache_translation(content_hash, 'eng', source_content) # Translate source content using Claude try: diff --git a/templates/nomike.com/base.html b/templates/nomike.com/base.html index 4f560cb..6337c0d 100644 --- a/templates/nomike.com/base.html +++ b/templates/nomike.com/base.html @@ -54,22 +54,23 @@
{% if templatehelper.is_claude_available() %}
- {% set current_lang = request.args.get('lang', 'en') %} + {% set current_lang = request.args.get('lang', 'eng') %}
diff --git a/templates/nomike.com/directory.html b/templates/nomike.com/directory.html index c1756b5..790c5d5 100644 --- a/templates/nomike.com/directory.html +++ b/templates/nomike.com/directory.html @@ -12,4 +12,4 @@ {{ name }} {% else %} {{ name }} {% endif %} {% endfor %}
- {% endif -%} {% endblock -%} {% block content -%} {{ templatehelper.renderIndexFile(path, request.args.get('lang', 'en')) | safe }} {% if templatehelper.os.path.isdir(templatehelper.os.path.join(pathprefix, path, 'image')) %} {% include templatehelper.os.path.join(templatehelper.config['template'], 'gallery.html') %} {% endif %} {% endblock %} + {% endif -%} {% endblock -%} {% block content -%} {{ templatehelper.renderIndexFile(path, request.args.get('lang', 'eng')) | safe }} {% if templatehelper.os.path.isdir(templatehelper.os.path.join(pathprefix, path, 'image')) %} {% include templatehelper.os.path.join(templatehelper.config['template'], 'gallery.html') %} {% endif %} {% endblock %} From 0bcf6a06272ca9d07f02de0dd5d4ace45c3c8105 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Tue, 23 Sep 2025 12:37:54 +0200 Subject: [PATCH 11/15] feat: add Japanese and Russian language support --- templatehelper.py | 3 ++- templates/nomike.com/base.html | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/templatehelper.py b/templatehelper.py index 0d3d0c5..fad54ca 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -115,7 +115,8 @@ def translate_claude(content, target_lang): 'deu': 'German', 'fra': 'French', 'ita': 'Italian', 'ell': 'Greek', 'hun': 'Hungarian', 'por': 'Portuguese', 'ces': 'Czech', 'slk': 'Slovak', 'slv': 'Slovenian', 'hrv': 'Croatian', 'gsw': 'Swiss German', - 'nld': 'Dutch', 'bul': 'Bulgarian', 'mnk': 'Mandinka' + 'nld': 'Dutch', 'bul': 'Bulgarian', 'mnk': 'Mandinka', + 'jpn': 'Japanese', 'rus': 'Russian' } target_language = lang_names.get(target_lang, target_lang) diff --git a/templates/nomike.com/base.html b/templates/nomike.com/base.html index 6337c0d..c07fc18 100644 --- a/templates/nomike.com/base.html +++ b/templates/nomike.com/base.html @@ -67,8 +67,10 @@ + + From 3b9f18bbaf11144eb9b87704bf2963ac7c7f05e5 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Tue, 23 Sep 2025 12:46:29 +0200 Subject: [PATCH 12/15] feat: add Scottish Gaelic language support --- templatehelper.py | 2 +- templates/nomike.com/base.html | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/templatehelper.py b/templatehelper.py index fad54ca..180eda5 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -116,7 +116,7 @@ def translate_claude(content, target_lang): 'hun': 'Hungarian', 'por': 'Portuguese', 'ces': 'Czech', 'slk': 'Slovak', 'slv': 'Slovenian', 'hrv': 'Croatian', 'gsw': 'Swiss German', 'nld': 'Dutch', 'bul': 'Bulgarian', 'mnk': 'Mandinka', - 'jpn': 'Japanese', 'rus': 'Russian' + 'jpn': 'Japanese', 'rus': 'Russian', 'gla': 'Scottish Gaelic' } target_language = lang_names.get(target_lang, target_lang) diff --git a/templates/nomike.com/base.html b/templates/nomike.com/base.html index c07fc18..a6fae6c 100644 --- a/templates/nomike.com/base.html +++ b/templates/nomike.com/base.html @@ -71,6 +71,7 @@ + From 0e5786d69777a1cc3a090ace82af66b032fe54b1 Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Tue, 23 Sep 2025 12:51:19 +0200 Subject: [PATCH 13/15] feat: add Arabic language support with RTL layout - Add Arabic to language mapping and dropdown - Implement RTL CSS styles for Arabic content - Add lang and dir attributes to HTML based on selected language --- templatehelper.py | 2 +- templates/nomike.com/base.html | 5 +- templates/nomike.com/static/design.css | 72 +++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/templatehelper.py b/templatehelper.py index 180eda5..fa2ab2d 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -112,7 +112,7 @@ def translate_claude(content, target_lang): # Language code to language name mapping (ISO 639-3) lang_names = { - 'deu': 'German', 'fra': 'French', 'ita': 'Italian', 'ell': 'Greek', + 'ara': 'Arabic', 'deu': 'German', 'fra': 'French', 'ita': 'Italian', 'ell': 'Greek', 'hun': 'Hungarian', 'por': 'Portuguese', 'ces': 'Czech', 'slk': 'Slovak', 'slv': 'Slovenian', 'hrv': 'Croatian', 'gsw': 'Swiss German', 'nld': 'Dutch', 'bul': 'Bulgarian', 'mnk': 'Mandinka', diff --git a/templates/nomike.com/base.html b/templates/nomike.com/base.html index a6fae6c..2396e65 100644 --- a/templates/nomike.com/base.html +++ b/templates/nomike.com/base.html @@ -1,6 +1,7 @@ {% if not templatehelper.os.path.exists(templatehelper.os.path.join(path, '.scmsnotemplate')): -%} - +{% set current_lang = request.args.get('lang', 'eng') %} + {% block title %}{% endblock %} - {{ templatehelper.urllib.parse.urlparse(request.url_root).netloc }} @@ -54,8 +55,8 @@
{% if templatehelper.is_claude_available() %}
- {% set current_lang = request.args.get('lang', 'eng') %} - - - - - - - - - - - - - - - - - - - - +
+ + + + +
+ + + +
{% endif %} diff --git a/templates/nomike.com/static/design.css b/templates/nomike.com/static/design.css index c966777..3cbccb6 100644 --- a/templates/nomike.com/static/design.css +++ b/templates/nomike.com/static/design.css @@ -313,6 +313,7 @@ div.galleryblock img:hover { flex-shrink: 0; } +/* Language dropdown - base styles (no JS fallback) */ .language-dropdown { background-color: transparent; border: 1px solid #333; @@ -354,6 +355,91 @@ div.galleryblock img:hover { border-color: #666; } +/* Searchable dropdown styles (JS enabled) */ +.language-dropdown-container { + position: relative; + display: inline-block; +} + +.language-dropdown-container .language-search { + background-color: transparent; + border: 1px solid #333; + border-radius: 3px; + font-size: 12px; + height: 16px; + padding: 1px 3px; + color: black; + width: 120px; + cursor: pointer; +} + +.language-dropdown-container .language-search:focus { + outline: none; + border-color: #666; + cursor: text; +} + +.language-dropdown-container .language-options { + position: absolute; + bottom: 100%; + left: 0; + right: 0; + background-color: white; + border: 1px solid #333; + border-radius: 3px; + max-height: 200px; + overflow-y: auto; + z-index: 1000; + display: none; + box-shadow: 0 -2px 4px rgba(0, 0, 0, 0.1); +} + +.language-dropdown-container.open .language-options { + display: block; +} + +.language-option { + padding: 4px 8px; + font-size: 12px; + cursor: pointer; + border-bottom: 1px solid #eee; +} + +.language-option:last-child { + border-bottom: none; +} + +.language-option:hover, +.language-option.highlighted { + background-color: #f0f0f0; +} + +.language-option.selected { + background-color: #ff9900; + color: black; +} + +/* Hide native select when JS is enabled */ +.language-dropdown-container.js-enabled .language-dropdown { + display: none; +} + +/* Show search input only when JS is enabled */ +.language-dropdown-container:not(.js-enabled) .language-search, +.language-dropdown-container:not(.js-enabled) .language-options { + display: none; +} + +/* RTL support for searchable dropdown */ +[dir="rtl"] .language-dropdown-container .language-options { + right: 0; + left: auto; +} + +[dir="rtl"] .language-option { + text-align: right; +} + :not(pre)>code { background: lightgray; border: 1px solid gray; From 045b28383e06b51f5771819addd8f083caf39ade Mon Sep 17 00:00:00 2001 From: nomike Postmann Date: Tue, 23 Sep 2025 13:09:06 +0200 Subject: [PATCH 15/15] feat: add Afrikaans language support to translation system --- templatehelper.py | 2 +- templates/nomike.com/base.html | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/templatehelper.py b/templatehelper.py index fa2ab2d..52c706c 100644 --- a/templatehelper.py +++ b/templatehelper.py @@ -112,7 +112,7 @@ def translate_claude(content, target_lang): # Language code to language name mapping (ISO 639-3) lang_names = { - 'ara': 'Arabic', 'deu': 'German', 'fra': 'French', 'ita': 'Italian', 'ell': 'Greek', + 'afr': 'Afrikaans', 'ara': 'Arabic', 'deu': 'German', 'fra': 'French', 'ita': 'Italian', 'ell': 'Greek', 'hun': 'Hungarian', 'por': 'Portuguese', 'ces': 'Czech', 'slk': 'Slovak', 'slv': 'Slovenian', 'hrv': 'Croatian', 'gsw': 'Swiss German', 'nld': 'Dutch', 'bul': 'Bulgarian', 'mnk': 'Mandinka', diff --git a/templates/nomike.com/base.html b/templates/nomike.com/base.html index 50a38ef..3885e68 100644 --- a/templates/nomike.com/base.html +++ b/templates/nomike.com/base.html @@ -167,6 +167,7 @@