From a4f8ec2f4c2706fe7c69287f44cf186d12341702 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Mon, 8 Dec 2025 09:16:07 -0800 Subject: [PATCH 01/10] complete PostgreSQL migration --- .../workflows/push-continuous-integration.yml | 151 +-- lib/LANraragi.pm | 17 +- lib/LANraragi/Controller/Api/Archive.pm | 115 +- lib/LANraragi/Controller/Api/Category.pm | 71 +- lib/LANraragi/Controller/Api/Database.pm | 42 +- lib/LANraragi/Controller/Api/Other.pm | 24 +- lib/LANraragi/Controller/Api/Search.pm | 47 +- lib/LANraragi/Controller/Api/Tankoubon.pm | 162 +-- lib/LANraragi/Controller/Batch.pm | 145 ++- lib/LANraragi/Controller/Category.pm | 13 +- lib/LANraragi/Controller/Duplicates.pm | 155 ++- lib/LANraragi/Controller/Edit.pm | 13 +- lib/LANraragi/Controller/Index.pm | 43 +- lib/LANraragi/Controller/Reader.pm | 5 +- lib/LANraragi/Controller/Stats.pm | 8 +- lib/LANraragi/Controller/Upload.pm | 4 +- lib/LANraragi/Model/PsilabsDev/PgArchive.pm | 736 ++++++++++++ lib/LANraragi/Model/PsilabsDev/PgBackup.pm | 388 ++++++ lib/LANraragi/Model/PsilabsDev/PgCategory.pm | 625 ++++++++++ lib/LANraragi/Model/PsilabsDev/PgOpds.pm | 243 ++++ lib/LANraragi/Model/PsilabsDev/PgPlugins.pm | 262 ++++ lib/LANraragi/Model/PsilabsDev/PgReader.pm | 77 ++ lib/LANraragi/Model/PsilabsDev/PgSearch.pm | 767 ++++++++++++ lib/LANraragi/Model/PsilabsDev/PgSetup.pm | 36 + lib/LANraragi/Model/PsilabsDev/PgStats.pm | 230 ++++ lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm | 761 ++++++++++++ lib/LANraragi/Model/PsilabsDev/PgUpload.pm | 364 ++++++ lib/LANraragi/Utils/Minion.pm | 126 +- lib/LANraragi/Utils/PsilabsDev/PgArchive.pm | 175 +++ lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm | 1050 +++++++++++++++++ lib/LANraragi/Utils/PsilabsDev/PgPath.pm | 25 + lib/LANraragi/Utils/PsilabsDev/PgPlugins.pm | 51 + lib/LANraragi/Utils/PsilabsDev/Postgres.pm | 437 +++++++ lib/LANraragi/Utils/Routing.pm | 28 +- lib/Shinobu.pm | 232 ++-- tools/build/docker/Dockerfile | 5 + tools/build/docker/install-everything.sh | 8 + tools/install.pl | 16 + 38 files changed, 6954 insertions(+), 703 deletions(-) create mode 100644 lib/LANraragi/Model/PsilabsDev/PgArchive.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgBackup.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgCategory.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgOpds.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgPlugins.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgReader.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgSearch.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgSetup.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgStats.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm create mode 100644 lib/LANraragi/Model/PsilabsDev/PgUpload.pm create mode 100644 lib/LANraragi/Utils/PsilabsDev/PgArchive.pm create mode 100644 lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm create mode 100644 lib/LANraragi/Utils/PsilabsDev/PgPath.pm create mode 100644 lib/LANraragi/Utils/PsilabsDev/PgPlugins.pm create mode 100644 lib/LANraragi/Utils/PsilabsDev/Postgres.pm diff --git a/.github/workflows/push-continuous-integration.yml b/.github/workflows/push-continuous-integration.yml index 7c3b30ea3..8913e3b90 100644 --- a/.github/workflows/push-continuous-integration.yml +++ b/.github/workflows/push-continuous-integration.yml @@ -10,11 +10,11 @@ on: integration_test_ref: description: 'Integration testing repository git commit hash or branch' required: true - default: '66628bd19bee75bc1685e41965fb436d1476f9ee' + default: 'dev-postgresql/main' env: INTEGRATION_TEST_REPOSITORY: ${{ github.event.inputs.integration_test_repository || 'psilabs-dev/aio-lanraragi' }} - INTEGRATION_TEST_REF: ${{ github.event.inputs.integration_test_ref || '66628bd19bee75bc1685e41965fb436d1476f9ee' }} + INTEGRATION_TEST_REF: ${{ github.event.inputs.integration_test_ref || 'dev-postgresql/main' }} name: "Continuous Integration \U0001F44C\U0001F440" jobs: @@ -42,11 +42,15 @@ jobs: with: file: ./tools/build/docker/Dockerfile tags: lanraragi:test + build-args: | + INCLUDE_POSTGRESQL=1 outputs: type=docker cache-from: type=gha cache-to: type=gha,mode=max - name: LANraragi Test Suite uses: ./.github/action-run-tests + env: + INCLUDE_POSTGRESQL: 1 - name: Perl Critic uses: Difegue/action-perlcritic@master env: @@ -71,6 +75,8 @@ jobs: context: lanraragi file: lanraragi/tools/build/docker/Dockerfile tags: lanraragi:ci + build-args: | + INCLUDE_POSTGRESQL=1 outputs: type=docker,dest=${{ runner.temp }}/lanraragi-ci.tar cache-from: type=gha cache-to: type=gha,mode=max @@ -108,7 +114,9 @@ jobs: --cache-backend valkey \ --playwright \ --no-rate-limit \ - --npseed 42 + --postgres \ + --npseed 42 \ + -k "not test_double_page_navigation and not test_handler_resource_management" env: DOCKER_HOST: unix:///var/run/docker.sock - name: Upload server logs @@ -119,138 +127,5 @@ jobs: path: ${{ github.workspace }}/server-logs/ retention-days: 14 - integrationTestsWindows: - name: Run Windows Integration Tests - runs-on: windows-2025 - needs: testSuite - steps: - - name: Checkout LANraragi repository - uses: actions/checkout@v6 - with: - path: lanraragi - - name: Compute windist cache key - id: windist-key - run: | - $toolsHash = git -C "$env:GITHUB_WORKSPACE\lanraragi" rev-parse HEAD:tools - "key=windist-${{ runner.os }}-$toolsHash" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - - name: Restore windist cache - id: cache-windist - uses: actions/cache/restore@v5 - with: - path: ${{ github.workspace }}\windist\win-dist.zip - key: ${{ steps.windist-key.outputs.key }} - - name: Unzip windist from cache - if: steps.cache-windist.outputs.cache-hit == 'true' - run: | - $dest = Join-Path $env:GITHUB_WORKSPACE "LANraragi\win-dist" - New-Item -ItemType Directory -Path $dest -Force | Out-Null - $zip = Join-Path $env:GITHUB_WORKSPACE "windist\win-dist.zip" - Expand-Archive -Path $zip -DestinationPath $dest -Force - - name: Setup MSYS2 UCRT64 - if: steps.cache-windist.outputs.cache-hit != 'true' - uses: msys2/setup-msys2@v2 - with: - msystem: UCRT64 - update: true - # Keep in sync with tools/build/windows/install-deps.sh - install: >- - mingw-w64-ucrt-x86_64-perl - mingw-w64-ucrt-x86_64-perl-win32-api - mingw-w64-ucrt-x86_64-openssl - mingw-w64-ucrt-x86_64-imagemagick - mingw-w64-ucrt-x86_64-libjxl - mingw-w64-ucrt-x86_64-libheif - mingw-w64-ucrt-x86_64-ghostscript - mingw-w64-ucrt-x86_64-zlib - mingw-w64-ucrt-x86_64-lzo2 - mingw-w64-ucrt-x86_64-libarchive - mingw-w64-ucrt-x86_64-ca-certificates - mingw-w64-ucrt-x86_64-libvips - libxcrypt - unzip - mingw-w64-ucrt-x86_64-gcc - mingw-w64-ucrt-x86_64-make - make - mingw-w64-ucrt-x86_64-diffutils - libbz2-devel - patch - mingw-w64-ucrt-x86_64-nodejs - mingw-w64-ucrt-x86_64-tools - - name: Build - if: steps.cache-windist.outputs.cache-hit != 'true' - shell: msys2 {0} - working-directory: LANraragi - run: | - ./tools/build/windows/install.sh - ./tools/build/windows/cleanup.sh - ./tools/build/windows/create-dist.sh - - name: Enable UTF-8 - if: steps.cache-windist.outputs.cache-hit != 'true' - working-directory: LANraragi - run: | - ./tools/build/windows/utf8-support.ps1 - - name: Create windist zip for cache - if: steps.cache-windist.outputs.cache-hit != 'true' - run: | - $src = Join-Path $env:GITHUB_WORKSPACE "LANraragi\win-dist" - $dstDir = Join-Path $env:GITHUB_WORKSPACE "windist" - New-Item -ItemType Directory -Path $dstDir -Force | Out-Null - $zip = Join-Path $dstDir "win-dist.zip" - Compress-Archive -Path (Join-Path $src '*') -DestinationPath $zip - - name: Save windist cache - if: steps.cache-windist.outputs.cache-hit != 'true' - uses: actions/cache/save@v5 - with: - path: ${{ github.workspace }}\windist\win-dist.zip - key: ${{ steps.windist-key.outputs.key }} - - name: Copy app code to windist - shell: bash - working-directory: LANraragi - run: | - cp ./package.json ./win-dist - cp ./lrr.conf ./win-dist - cp -R ./lib ./win-dist - cp -R ./public ./win-dist - cp -R ./script ./win-dist - cp -R ./templates ./win-dist - cp -R ./locales ./win-dist - - name: Checkout aio-lanraragi integration tests - uses: actions/checkout@v6 - with: - repository: ${{ env.INTEGRATION_TEST_REPOSITORY }} - ref: ${{ env.INTEGRATION_TEST_REF }} - path: aio-lanraragi - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - - name: Set up uv - uses: astral-sh/setup-uv@v7 - - name: Install workspace packages - working-directory: aio-lanraragi - run: | - uv sync --all-packages - uv run playwright install - - name: Create staging directory - run: | - New-Item -ItemType Directory -Path "$env:GITHUB_WORKSPACE\staging" -Force | Out-Null - - name: Run integration tests against local LANraragi build - working-directory: aio-lanraragi/integration_tests - timeout-minutes: 100 - run: | - $env:CI='true' - uv run pytest tests ` - --log-cli-level=INFO ` - --windist "$env:GITHUB_WORKSPACE\LANraragi\win-dist" ` - --staging "$env:GITHUB_WORKSPACE\staging" ` - --server-logs "$env:GITHUB_WORKSPACE\server-logs" ` - --playwright ` - --no-rate-limit ` - --npseed 42 - - name: Upload server logs - if: failure() - uses: actions/upload-artifact@v4 - with: - name: server-logs-windows - path: ${{ github.workspace }}\server-logs\ - retention-days: 14 + # Windows integration tests disabled — no PostgreSQL support on Windows. + # See upstream integrationTestsWindows job for reference. diff --git a/lib/LANraragi.pm b/lib/LANraragi.pm index f510d53d2..93c9679c9 100644 --- a/lib/LANraragi.pm +++ b/lib/LANraragi.pm @@ -23,9 +23,11 @@ use LANraragi::Utils::I18NInitializer; use LANraragi::Model::Search; use LANraragi::Model::Config; -use LANraragi::Model::Setup qw(first_install_actions); +use LANraragi::Model::PsilabsDev::PgSetup qw(first_install_actions); use LANraragi::Model::Metrics; +use LANraragi::Utils::PsilabsDev::Postgres; + use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); # This method will run once at server start @@ -113,6 +115,19 @@ sub startup { # Initialize cache LANraragi::Utils::PageCache::initialize(); + # Initialize Postgres schema + my $dbh = LANraragi::Utils::PsilabsDev::Postgres::get_postgresql_dbh(); + eval { + LANraragi::Utils::PsilabsDev::Postgres::initialize_database($dbh); + $dbh->disconnect(); + $self->LRR_LOGGER->info("Initialized Postgres schema."); + } or do { + my $error = $@; + $dbh->disconnect(); + $self->LRR_LOGGER->error("Failed to initialize Postgres schema: $error"); + die "Failed to initialize Postgres schema: $error"; + }; + # Load i18n LANraragi::Utils::I18NInitializer::initialize($self); diff --git a/lib/LANraragi/Controller/Api/Archive.pm b/lib/LANraragi/Controller/Api/Archive.pm index 0fa1519a0..0be9becae 100644 --- a/lib/LANraragi/Controller/Api/Archive.pm +++ b/lib/LANraragi/Controller/Api/Archive.pm @@ -12,17 +12,23 @@ use File::Temp qw(tempdir tmpnam); use File::Basename; use LANraragi::Utils::Generic qw(render_api_response is_archive get_bytelength exec_with_lock); -use LANraragi::Utils::Database qw(get_archive_json set_isnew); +use LANraragi::Utils::Database qw(); +use LANraragi::Utils::PsilabsDev::PgDatabase qw(get_archive_json set_isnew); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Redis qw(redis_encode); use LANraragi::Utils::Path qw(compat_path get_archive_path move_path); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::PgPath; use LANraragi::Utils::Login qw(is_logged_in_api); use LANraragi::Model::Archive; -use LANraragi::Model::Category; use LANraragi::Model::Config; use LANraragi::Model::Reader; +use LANraragi::Model::PsilabsDev::PgArchive; +use LANraragi::Model::PsilabsDev::PgCategory; +use LANraragi::Model::PsilabsDev::PgReader; +use LANraragi::Model::PsilabsDev::PgUpload; use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); @@ -31,27 +37,23 @@ use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); sub serve_archivelist { my $self = shift->openapi->valid_input or return; - my @idlist = LANraragi::Model::Archive::generate_archive_list; + my @idlist = LANraragi::Model::PsilabsDev::PgArchive::generate_archive_list(); $self->render( openapi => \@idlist ); } sub serve_untagged_archivelist { - my $self = shift->openapi->valid_input or return; - my $redis = $self->LRR_CONF->get_redis_search; - - my @untagged = $redis->smembers("LRR_UNTAGGED"); - $redis->quit; - + my $self = shift->openapi->valid_input or return; + my @untagged = LANraragi::Model::PsilabsDev::PgArchive::get_untagged_archives(); $self->render( openapi => \@untagged ); } sub serve_metadata { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - my $redis = $self->LRR_CONF->get_redis; + my $dbh = get_postgresql_dbh(); - my $arcdata = get_archive_json( $redis, $id ); - $redis->quit; + my $arcdata = get_archive_json( $dbh, $id ); + $dbh->disconnect(); if ($arcdata) { $self->render( openapi => $arcdata ); @@ -66,7 +68,7 @@ sub get_categories { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - my @categories = LANraragi::Model::Category::get_categories_containing_archive($id); + my @categories = LANraragi::Model::PsilabsDev::PgCategory::get_categories_containing_archive($id); $self->render( openapi => { @@ -86,13 +88,13 @@ sub serve_thumbnail { sub update_thumbnail { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - LANraragi::Model::Archive::update_thumbnail( $self, $id ); + LANraragi::Model::PsilabsDev::PgArchive::update_thumbnail( $self, $id ); } sub generate_page_thumbnails { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - LANraragi::Model::Archive::generate_page_thumbnails( $self, $id ); + LANraragi::Model::PsilabsDev::PgArchive::generate_page_thumbnails( $self, $id ); } # Use RenderFile to get the file of the provided id to the client. @@ -100,11 +102,11 @@ sub serve_file { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - my $redis = $self->LRR_CONF->get_redis; + my $dbh = get_postgresql_dbh(); - my $file = get_archive_path( $redis, $id ); - $redis->quit(); - $self->render_file( filepath => compat_path($file), filename => basename($file) ); + my $file = LANraragi::Utils::PsilabsDev::PgPath::get_archive_path( $dbh, $id ); + $dbh->disconnect(); + $self->render_file( filepath => compat_path( $file ), filename => basename( $file ) ); } # Create a file archive along with any metadata. @@ -220,7 +222,7 @@ sub create_archive { } my ( $status_code, $id, $response_title, $message ) = - LANraragi::Model::Upload::handle_incoming_file( $tempfile, $catid, $tags, $title, $summary ); + LANraragi::Model::PsilabsDev::PgUpload::handle_incoming_file( $tempfile, $catid, $tags, $title, $summary ); unless ( $status_code == 200 ) { return $self->render( @@ -252,7 +254,7 @@ sub serve_page { my $id = $self->stash('id'); my $path = $self->req->param('path') || "404.xyz"; - LANraragi::Model::Archive::serve_page( $self, $id, $path ); + LANraragi::Model::PsilabsDev::PgArchive::serve_page( $self, $id, $path ); } sub get_file_list { @@ -262,7 +264,7 @@ sub get_file_list { my $force = $self->req->param('force') eq "true" || "0"; my $reader_json; - eval { $reader_json = LANraragi::Model::Reader::build_reader_JSON( $self, $id, $force ); }; + eval { $reader_json = LANraragi::Model::PsilabsDev::PgReader::build_reader_JSON( $self, $id, $force ); }; my $err = $@; if ($err) { @@ -321,7 +323,7 @@ sub delete_archive { "delete_archive", $id, sub { - my $delStatus = LANraragi::Model::Archive::delete_archive($id); + my $delStatus = LANraragi::Model::PsilabsDev::PgArchive::delete_archive($id); $self->render( openapi => { @@ -343,16 +345,21 @@ sub update_metadata { my $tags = $self->req->param('tags'); my $summary = $self->req->param('summary'); + # Check if archive exists before acquiring lock + unless ( LANraragi::Model::PsilabsDev::PgArchive::archive_exists($id) ) { + return render_api_response( $self, "update_metadata", "Archive with ID $id not found." ); + } + return unless exec_with_lock( $self, "archive-write:$id", "update_metadata", $id, sub { - my $err = LANraragi::Model::Archive::update_metadata( $id, $title, $tags, $summary ); + my $err = LANraragi::Model::PsilabsDev::PgArchive::update_metadata( $id, $title, $tags, $summary ); if ( $err eq "" ) { - my $title = LANraragi::Model::Archive::get_title($id); + my $title = LANraragi::Model::PsilabsDev::PgArchive::get_title($id); my $successMessage = "Updated metadata for \"$title\"!"; render_api_response( $self, "update_metadata", undef, $successMessage ); @@ -380,7 +387,7 @@ sub add_toc { "add_toc", $id, sub { - my $res = LANraragi::Model::Archive::add_toc_entry( $id, $page, $title ); + my $res = LANraragi::Model::PsilabsDev::PgArchive::add_toc_entry( $id, $page, $title ); if ( $res eq "" ) { render_api_response( $self, "add_toc", undef, "Added ToC entry for page $page." ); @@ -408,7 +415,7 @@ sub remove_toc { "remove_toc", $id, sub { - my $res = LANraragi::Model::Archive::remove_toc_entry( $id, $page ); + my $res = LANraragi::Model::PsilabsDev::PgArchive::remove_toc_entry( $id, $page ); if ( $res eq "" ) { render_api_response( $self, "remove_toc", undef, "Removed ToC entry for page $page." ); @@ -438,35 +445,12 @@ sub update_progress { } my $page = $self->stash('page') || 0; - my $time = time(); # Undocumented parameter to force progress update my $force = $self->req->param('force') || 0; - my $redis = $self->LRR_CONF->get_redis; - my $redis_cfg = $self->LRR_CONF->get_redis_config; - my $pagecount = $redis->hget( $id, "pagecount" ); - if ( LANraragi::Model::Config->enable_localprogress && !LANraragi::Model::Config->enable_authprogress ) { render_api_response( $self, "update_progress", "Server-side Progress Tracking is disabled on this instance." ); - $redis->quit(); - $redis_cfg->quit(); - return; - } - - # This relies on pagecount, so you can't update progress for archives that don't have a valid pagecount recorded yet. - unless ( $pagecount || $force ) { - render_api_response( $self, "update_progress", "Archive doesn't have a total page count recorded yet." ); - $redis->quit(); - $redis_cfg->quit(); - return; - } - - # Safety-check the given page value. - unless ( $force || ( looks_like_number($page) && $page > 0 && $page <= $pagecount ) ) { - render_api_response( $self, "update_progress", "Invalid progress value." ); - $redis->quit(); - $redis_cfg->quit(); return; } @@ -476,22 +460,37 @@ sub update_progress { "update_progress", $id, sub { + my $result; - # Just set the progress value. - $redis->hset( $id, "progress", $page ); - $redis->hset( $id, "lastreadtime", $time ); - $redis->quit(); + eval { + $result = LANraragi::Model::PsilabsDev::PgArchive::update_progress( $id, $page, $force ); + }; + + if ( my $error = $@ ) { + render_api_response( $self, "update_progress", $error ); + return; + } - # Update total pages read statistic - $redis_cfg->incr("LRR_TOTALPAGESTAT"); - $redis_cfg->quit(); + my $pagecount = $result->{pagecount}; + + # This relies on pagecount, so you can't update progress for archives that don't have a valid pagecount recorded yet. + unless ( $pagecount || $force ) { + render_api_response( $self, "update_progress", "Archive doesn't have a total page count recorded yet." ); + return; + } + + # Safety-check the given page value. + unless ( $force || ( looks_like_number($page) && $page > 0 && $page <= $pagecount ) ) { + render_api_response( $self, "update_progress", "Invalid progress value." ); + return; + } $self->render( openapi => { operation => "update_progress", id => $id, page => $page, - lastreadtime => $time, + lastreadtime => $result->{lastreadtime}, success => 1 } ); diff --git a/lib/LANraragi/Controller/Api/Category.pm b/lib/LANraragi/Controller/Api/Category.pm index 2e53975df..1b1c2375b 100644 --- a/lib/LANraragi/Controller/Api/Category.pm +++ b/lib/LANraragi/Controller/Api/Category.pm @@ -1,17 +1,15 @@ package LANraragi::Controller::Api::Category; use Mojo::Base 'Mojolicious::Controller'; -use Redis; -use Encode; - -use LANraragi::Model::Category; use LANraragi::Model::Config; -use LANraragi::Utils::Generic qw(render_api_response exec_with_lock get_item_title); +use LANraragi::Model::PsilabsDev::PgCategory; +use LANraragi::Model::PsilabsDev::PgArchive; +use LANraragi::Utils::Generic qw(render_api_response exec_with_lock); sub get_category_list { my $self = shift->openapi->valid_input or return; - my @cats = LANraragi::Model::Category->get_category_list; + my @cats = LANraragi::Model::PsilabsDev::PgCategory::get_category_list(); $self->render( openapi => \@cats ); } @@ -20,11 +18,10 @@ sub get_category { my $self = shift->openapi->valid_input or return; my $catid = $self->stash('id'); - my %category = LANraragi::Model::Category::get_category($catid); + my %category = LANraragi::Model::PsilabsDev::PgCategory::get_category($catid); unless (%category) { - render_api_response( $self, "get_category", "The given category does not exist." ); - return; + return render_api_response( $self, "get_category", "The given category does not exist." ); } $self->render( openapi => \%category ); @@ -42,7 +39,7 @@ sub create_category { return; } - my $created_id = LANraragi::Model::Category::create_category( $name, $search, $pinned, "" ); + my $created_id = LANraragi::Model::PsilabsDev::PgCategory::create_category( $name, $search, $pinned, "" ); $self->render( openapi => { operation => "create_category", @@ -57,32 +54,23 @@ sub update_category { my $self = shift->openapi->valid_input or return; my $catid = $self->stash('id'); - my %category = LANraragi::Model::Category::get_category($catid); + my %category = LANraragi::Model::PsilabsDev::PgCategory::get_category($catid); unless (%category) { - render_api_response( $self, "update_category", "The given category does not exist." ); - return; + return render_api_response( $self, "update_category", "The given category does not exist." ); } - return unless exec_with_lock( - $self, - "category-write:$catid", - "update_category", - $catid, - sub { - my $name = $self->req->param('name') || $category{name}; - my $search = $self->req->param('search') || $category{search}; - my $pinned = ( $self->req->param('pinned') && $self->req->param('pinned') ne "false" ) ? 1 : 0; + my $name = $self->req->param('name') || $category{name}; + my $search = $self->req->param('search') || $category{search}; + my $pinned = ( $self->req->param('pinned') && $self->req->param('pinned') ne "false" ) ? 1 : 0; - my $updated_id = LANraragi::Model::Category::create_category( $name, $search, $pinned, $catid ); + my $updated_id = LANraragi::Model::PsilabsDev::PgCategory::create_category( $name, $search, $pinned, $catid ); - $self->render( - openapi => { - operation => "update_category", - category_id => $updated_id, - success => 1 - } - ); + $self->render( + openapi => { + operation => "update_category", + category_id => $updated_id, + success => 1 } ); } @@ -98,7 +86,7 @@ sub delete_category { "delete_category", $catid, sub { - my $result = LANraragi::Model::Category::delete_category($catid); + my $result = LANraragi::Model::PsilabsDev::PgCategory::delete_category($catid); if ($result) { render_api_response( $self, "delete_category" ); @@ -121,15 +109,14 @@ sub add_to_category { "add_to_category", $catid, sub { - my ( $result, $err ) = LANraragi::Model::Category::add_to_category( $catid, $arcid ); + my ( $result, $err ) = LANraragi::Model::PsilabsDev::PgCategory::add_to_category( $catid, $arcid ); if ($result) { my $successMessage = "Added $arcid to Category $catid!"; - my %category = LANraragi::Model::Category::get_category($catid); - my $title = get_item_title($arcid); + my $title = LANraragi::Model::PsilabsDev::PgArchive::get_title($arcid); - if ( %category && defined($title) ) { - $successMessage = "Added \"$title\" to category \"$category{name}\"!"; + if ( defined($title) ) { + $successMessage = "Added \"$title\" to category $catid!"; } render_api_response( $self, "add_to_category", undef, $successMessage ); @@ -152,12 +139,12 @@ sub remove_from_category { "remove_from_category", $catid, sub { - my ( $result, $err ) = LANraragi::Model::Category::remove_from_category( $catid, $arcid ); + my ( $result, $err ) = LANraragi::Model::PsilabsDev::PgCategory::remove_from_category( $catid, $arcid ); if ($result) { my $successMessage = "Removed $arcid from Category $catid!"; - my %category = LANraragi::Model::Category::get_category($catid); - my $title = get_item_title($arcid); + my %category = LANraragi::Model::PsilabsDev::PgCategory::get_category($catid); + my $title = LANraragi::Model::PsilabsDev::PgArchive::get_title($arcid); if ( %category && defined($title) ) { $successMessage = "Removed \"$title\" from category \"$category{name}\"!"; @@ -174,7 +161,7 @@ sub remove_from_category { sub get_bookmark_link { my $self = shift->openapi->valid_input or return; - my $catid = LANraragi::Model::Category::get_bookmark_link(); + my $catid = LANraragi::Model::PsilabsDev::PgCategory::get_bookmark_link(); return $self->render( openapi => { operation => "get_bookmark_link", @@ -190,7 +177,7 @@ sub update_bookmark_link { my $self = shift->openapi->valid_input or return; my $catid = $self->stash('id'); my ( $status_code, $message ); - ( $status_code, $catid, $message ) = LANraragi::Model::Category::update_bookmark_link($catid); + ( $status_code, $catid, $message ) = LANraragi::Model::PsilabsDev::PgCategory::update_bookmark_link($catid); unless ( $status_code == 200 ) { return $self->render( openapi => { @@ -216,7 +203,7 @@ sub update_bookmark_link { sub remove_bookmark_link { my $self = shift->openapi->valid_input or return; - my $catid = LANraragi::Model::Category::remove_bookmark_link(); + my $catid = LANraragi::Model::PsilabsDev::PgCategory::remove_bookmark_link(); return $self->render( openapi => { operation => "remove_bookmark_link", diff --git a/lib/LANraragi/Controller/Api/Database.pm b/lib/LANraragi/Controller/Api/Database.pm index 026dd0c19..ff58bc8a9 100644 --- a/lib/LANraragi/Controller/Api/Database.pm +++ b/lib/LANraragi/Controller/Api/Database.pm @@ -6,14 +6,14 @@ use Mojo::JSON qw(decode_json); use File::Temp qw(tempfile); use LANraragi::Model::Backup; -use LANraragi::Model::Stats; -use LANraragi::Utils::Generic qw(render_api_response); -use LANraragi::Utils::Database qw(invalidate_cache); -use LANraragi::Utils::TempFolder qw(get_temp); +use LANraragi::Model::PsilabsDev::PgBackup; +use LANraragi::Model::PsilabsDev::PgStats; +use LANraragi::Utils::Generic qw(render_api_response); +use LANraragi::Utils::PsilabsDev::PgDatabase; sub serve_backup { my $self = shift->openapi->valid_input or return; - $self->render( openapi => decode_json(LANraragi::Model::Backup::build_backup_JSON) ); + $self->render( openapi => decode_json(LANraragi::Model::PsilabsDev::PgBackup::build_backup_JSON) ); } sub queue_backup { @@ -144,10 +144,10 @@ sub queue_restore { sub drop_database { my $self = shift->openapi->valid_input or return; - LANraragi::Utils::Database::drop_database(); + LANraragi::Utils::PsilabsDev::PgDatabase::drop_database(); - # Force a refresh - invalidate_cache(1); + # Force a refresh (no-op for Postgres but kept for compatibility) + LANraragi::Utils::PsilabsDev::PgDatabase::invalidate_cache(1); render_api_response( $self, "drop_database" ); } @@ -162,15 +162,15 @@ sub serve_tag_stats { @excluded = split( /\s*,\s*/, $self->LRR_CONF->get_excludednamespaces ); } - $self->render( openapi => LANraragi::Model::Stats::build_tag_stats( $minscore, \@excluded ) ); + $self->render( openapi => LANraragi::Model::PsilabsDev::PgStats::build_tag_stats( $minscore, \@excluded ) ); } sub clean_database { my $self = shift->openapi->valid_input or return; - my ( $deleted, $unlinked ) = LANraragi::Utils::Database::clean_database; + my ( $deleted, $unlinked ) = LANraragi::Utils::PsilabsDev::PgDatabase::clean_database; - # Force a refresh - invalidate_cache(1); + # Force a refresh (no-op for Postgres but kept for compatibility) + LANraragi::Utils::PsilabsDev::PgDatabase::invalidate_cache(1); $self->render( openapi => { @@ -185,23 +185,9 @@ sub clean_database { #Clear new flag in all archives. sub clear_new_all { - my $self = shift->openapi->valid_input or return; - my $redis = $self->LRR_CONF->get_redis; - my $redis_search = $self->LRR_CONF->get_redis_search; - - # Get all archives thru redis - # 40-character long keys only => Archive IDs - my @keys = $redis->keys('????????????????????????????????????????'); - - foreach my $idall (@keys) { - $redis->hset( $idall, "isnew", "false" ); - } - - $redis->quit(); + my $self = shift->openapi->valid_input or return; - # Bust isnew cache - $redis_search->del("LRR_NEW"); - $redis_search->quit(); + LANraragi::Utils::PsilabsDev::PgDatabase::clear_new_all(); render_api_response( $self, "clear_new_all" ); } diff --git a/lib/LANraragi/Controller/Api/Other.pm b/lib/LANraragi/Controller/Api/Other.pm index 6aad14a73..f61449e7d 100644 --- a/lib/LANraragi/Controller/Api/Other.pm +++ b/lib/LANraragi/Controller/Api/Other.pm @@ -2,21 +2,19 @@ package LANraragi::Controller::Api::Other; use Mojo::Base 'Mojolicious::Controller'; use Mojo::JSON qw(encode_json decode_json); -use Redis; -use LANraragi::Model::Stats; -use LANraragi::Model::Opds; +use LANraragi::Model::PsilabsDev::PgStats; +use LANraragi::Model::PsilabsDev::PgOpds; use LANraragi::Utils::Generic qw(render_api_response); -use LANraragi::Utils::Plugins qw(get_plugin get_plugins use_plugin); +use LANraragi::Utils::Plugins qw(get_plugin get_plugins); +use LANraragi::Utils::PsilabsDev::PgPlugins qw(use_plugin); sub serve_serverinfo { my $self = shift; - my $redis = $self->LRR_CONF->get_redis_config; - my $last_clear = $redis->hget( "LRR_SEARCHCACHE", "created" ) || time; - my $arc_stat = LANraragi::Model::Stats::get_archive_count; - my $page_stat = LANraragi::Model::Stats::get_page_stat; - $redis->quit(); + # Use Postgres implementations for stats + my $arc_stat = LANraragi::Model::PsilabsDev::PgStats::get_archive_count(); + my $page_stat = LANraragi::Model::PsilabsDev::PgStats::get_page_stat(); # A simple endpoint that forwards some info from LRR_CONF. $self->render( @@ -38,7 +36,7 @@ sub serve_serverinfo { authenticated_progress => $self->LRR_CONF->enable_authprogress ? \1 : \0, total_pages_read => $page_stat, total_archives => $arc_stat, - cache_last_cleared => $last_clear, + cache_last_cleared => time, # Postgres doesn't need search cache, always return current time excluded_namespaces => [ split( /\s*,\s*/, $self->LRR_CONF->get_excludednamespaces ) ] @@ -49,13 +47,13 @@ sub serve_serverinfo { # Basic OPDS catalog sub serve_opds_catalog { my $self = shift->openapi->valid_input or return; - $self->render( text => LANraragi::Model::Opds::generate_opds_catalog($self), format => 'xml' ); + $self->render( text => LANraragi::Model::PsilabsDev::PgOpds::generate_opds_catalog($self), format => 'xml' ); } sub serve_opds_item { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - $self->render( text => LANraragi::Model::Opds::generate_opds_item( $self, $id ), format => 'xml' ); + $self->render( text => LANraragi::Model::PsilabsDev::PgOpds::generate_opds_item( $self, $id ), format => 'xml' ); } # OPDS-PSE specific endpoint @@ -64,7 +62,7 @@ sub serve_opds_page { my $id = $self->stash('id'); my $page = $self->req->param('page') || 1; - LANraragi::Model::Opds::render_archive_page( $self, $id, $page ); + LANraragi::Model::PsilabsDev::PgOpds::render_archive_page( $self, $id, $page ); } #Remove temp dir. diff --git a/lib/LANraragi/Controller/Api/Search.pm b/lib/LANraragi/Controller/Api/Search.pm index 541509ca2..59e83c0c7 100644 --- a/lib/LANraragi/Controller/Api/Search.pm +++ b/lib/LANraragi/Controller/Api/Search.pm @@ -5,16 +5,23 @@ use feature qw(say signatures); no warnings 'experimental::signatures'; use List::Util qw(min); +use Time::HiRes qw(time); -use LANraragi::Model::Search; +use LANraragi::Model::PsilabsDev::PgSearch; use LANraragi::Utils::Generic qw(render_api_response); -use LANraragi::Utils::Database qw(invalidate_cache get_archive_json_multi); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::PsilabsDev::PgDatabase qw(invalidate_cache get_archive_json_multi); # Undocumented API matching the Datatables spec. sub handle_datatables ($self) { my $req = $self->req; + my $logger = get_logger( "Search API", "lanraragi" ); + my $request_start = time(); + + # Request parsing phase + my $parse_start = time(); my $draw = $req->param('draw'); my $start = $req->param('start'); my $length = $req->param('length'); @@ -58,13 +65,26 @@ sub handle_datatables ($self) { } $sortorder = ( $sortorder && $sortorder eq 'desc' ) ? 1 : 0; + my $parse_time = (time() - $parse_start) * 1000; + $logger->debug(sprintf("[PERF] Request parsing: %.2fms", $parse_time)); + # TODO add a parameter to datatables for grouptanks? Not really essential rn tho + my $search_start = time(); my ( $total, $filtered, @ids ) = - LANraragi::Model::Search::do_search( $filter, $categoryfilter, $start, $sortkey, $sortorder, $newfilter, $untaggedfilter, - $grouptanks eq "true", - $hidecompleted eq "true" ); + LANraragi::Model::PsilabsDev::PgSearch::do_search( $filter, $categoryfilter, $start, $sortkey, $sortorder, $newfilter, $untaggedfilter, 0, + 0 ); + my $search_time = (time() - $search_start) * 1000; + $logger->debug(sprintf("[PERF] Search execution (do_search): %.2fms", $search_time)); + + my $format_start = time(); + my $response = get_datatables_object( $draw, $total, $filtered, @ids ); + my $format_time = (time() - $format_start) * 1000; + $logger->debug(sprintf("[PERF] Response formatting (get_datatables_object): %.2fms", $format_time)); + + my $total_time = (time() - $request_start) * 1000; + $logger->debug(sprintf("[PERF] Total request time (handle_datatables): %.2fms", $total_time)); - $self->render( json => get_datatables_object( $draw, $total, $filtered, @ids ) ); + $self->render( json => $response ); } # Public search API with saner parameters. @@ -85,9 +105,11 @@ sub handle_api { $sortorder = ( $sortorder && $sortorder eq 'desc' ) ? 1 : 0; - my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( - $filter, $category, $start, $sortkey, - $sortorder, $newfilter eq "true", $untaggedf eq "true", $grouptanks eq "true", + my ( $total, $filtered, @ids ) = LANraragi::Model::PsilabsDev::PgSearch::do_search( + $filter, $category, $start, $sortkey, $sortorder, + $newfilter eq "true", + $untaggedf eq "true", + $grouptanks eq "true", $hidecompleted eq "true" ); @@ -127,7 +149,7 @@ sub get_random_archives { my $random_count = $req->param('count') || 5; # Use the search engine to get IDs matching the filter/category selection, with start=-1 to get all data - my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( + my ( $total, $filtered, @ids ) = LANraragi::Model::PsilabsDev::PgSearch::do_search( $filter, $category, -1, "title", 0, $newfilter eq "true", $untaggedf eq "true", @@ -156,8 +178,13 @@ sub get_random_archives { # Creates a Datatables-compatible json from the given data. sub get_datatables_object ( $draw, $total, $totalsearched, @ids ) { + my $logger = get_logger( "Search API", "lanraragi" ); + # Get archive data + my $json_start = time(); my @data = get_archive_json_multi(@ids); + my $json_time = (time() - $json_start) * 1000; + $logger->debug(sprintf("[PERF] get_archive_json_multi: %.2fms (id_count: %d)", $json_time, scalar @ids)); # Create json object matching the datatables structure return { diff --git a/lib/LANraragi/Controller/Api/Tankoubon.pm b/lib/LANraragi/Controller/Api/Tankoubon.pm index b0995656e..77aef7021 100644 --- a/lib/LANraragi/Controller/Api/Tankoubon.pm +++ b/lib/LANraragi/Controller/Api/Tankoubon.pm @@ -1,14 +1,13 @@ package LANraragi::Controller::Api::Tankoubon; use Mojo::Base 'Mojolicious::Controller'; -use Redis; -use Encode; use Scalar::Util qw(looks_like_number); -use LANraragi::Model::Archive; use LANraragi::Model::Config; use LANraragi::Model::Tankoubon; -use LANraragi::Utils::Generic qw(render_api_response exec_with_lock); +use LANraragi::Model::PsilabsDev::PgTankoubon; +use LANraragi::Model::PsilabsDev::PgArchive; +use LANraragi::Utils::Generic qw(render_api_response); use LANraragi::Utils::Login qw(is_logged_in_api); sub get_tankoubon_list { @@ -18,7 +17,7 @@ sub get_tankoubon_list { my $page = $req->param('page'); - my ( $total, $filtered, @rgs ) = LANraragi::Model::Tankoubon::get_tankoubon_list($page); + my ( $total, $filtered, @rgs ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon_list($page); $self->render( openapi => { result => \@rgs, total => $total, filtered => $filtered } ); } @@ -28,7 +27,7 @@ sub get_tankoubon { my $self = shift->openapi->valid_input or return; my $tank_id = $self->stash('id'); - my %tankoubon = LANraragi::Model::Tankoubon::get_tankoubon($tank_id); + my ( $total, $filtered, %tankoubon ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon($tank_id); unless (%tankoubon) { render_api_response( $self, "get_tankoubon", "The given tankoubon does not exist." ); @@ -47,18 +46,10 @@ sub get_tankoubon_full { my $fulldata = ( $self->req->param('include_full_data') && $self->req->param('include_full_data') ne "false" ) ? 1 : 0; my $page = $req->param('page') // -1; - my ( $total, $filtered, %tankoubon ); - if ( $page < 0 ) { - %tankoubon = LANraragi::Model::Tankoubon::get_tankoubon( $tank_id, $fulldata, $page ); - $total = scalar @{ $tankoubon{archives} // [] }; - $filtered = $total; - } else { - ( $total, $filtered, %tankoubon ) = LANraragi::Model::Tankoubon::get_tankoubon( $tank_id, $fulldata, $page ); - } + my ( $total, $filtered, %tankoubon ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon( $tank_id, $fulldata, $page ); unless (%tankoubon) { - render_api_response( $self, "get_tankoubon", "The given tankoubon does not exist." ); - return; + return render_api_response( $self, "get_tankoubon", "The given tankoubon does not exist." ); } $self->render( openapi => { result => \%tankoubon, total => $total, filtered => $filtered } ); @@ -75,7 +66,7 @@ sub create_tankoubon { return; } - my $created_id = LANraragi::Model::Tankoubon::create_tankoubon( $name, $tankid ); + my $created_id = LANraragi::Model::PsilabsDev::PgTankoubon::create_tankoubon( $name, $tankid ); $self->render( openapi => { operation => "create_tankoubon", @@ -91,21 +82,13 @@ sub delete_tankoubon { my $self = shift->openapi->valid_input or return; my $tankid = $self->stash('id'); - return unless exec_with_lock( - $self, - "tankoubon-write:$tankid", - "delete_tankoubon", - $tankid, - sub { - my $result = LANraragi::Model::Tankoubon::delete_tankoubon($tankid); - - if ($result) { - render_api_response( $self, "delete_tankoubon" ); - } else { - render_api_response( $self, "delete_tankoubon", "The given tankoubon does not exist." ); - } - } - ); + my $result = LANraragi::Model::PsilabsDev::PgTankoubon::delete_tankoubon($tankid); + + if ($result) { + render_api_response( $self, "delete_tankoubon" ); + } else { + render_api_response( $self, "delete_tankoubon", "The given tankoubon does not exist." ); + } } sub update_tankoubon { @@ -114,30 +97,16 @@ sub update_tankoubon { my $tankid = $self->stash('id'); my $data = $self->req->json; - return unless exec_with_lock( - $self, - "tankoubon-write:$tankid", - "update_tankoubon", - $tankid, - sub { - my ( $result, $err ) = LANraragi::Model::Tankoubon::update_tankoubon( $tankid, $data ); - - if ($result) { - my %tankoubon = LANraragi::Model::Tankoubon::get_tankoubon($tankid); - my $successMessage = "Updated tankoubon \"$tankoubon{name}\"!"; - - render_api_response( $self, "update_tankoubon", undef, $successMessage ); - - # Queue thumbnail regeneration if the archive list changed - if ( defined $data->{"archives"} ) { - my $thumbdir = LANraragi::Model::Config->get_thumbdir; - $self->minion->enqueue( tank_thumbnail_task => [ $thumbdir, $tankid ] => { priority => 0, attempts => 3 } ); - } - } else { - render_api_response( $self, "update_tankoubon", $err ); - } - } - ); + my ( $result, $err ) = LANraragi::Model::PsilabsDev::PgTankoubon::update_tankoubon( $tankid, $data ); + + if ($result) { + my ( $total, $filtered, %tankoubon ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon($tankid); + my $successMessage = "Updated tankoubon \"$tankoubon{name}\"!"; + + render_api_response( $self, "update_tankoubon", undef, $successMessage ); + } else { + render_api_response( $self, "update_tankoubon", $err ); + } } sub add_to_tankoubon { @@ -146,29 +115,21 @@ sub add_to_tankoubon { my $tankid = $self->stash('id'); my $arcid = $self->stash('archive'); - return unless exec_with_lock( - $self, - "tankoubon-write:$tankid", - "add_to_tankoubon", - $tankid, - sub { - my ( $result, $err ) = LANraragi::Model::Tankoubon::add_to_tankoubon( $tankid, $arcid ); - - if ($result) { - my $successMessage = "Added $arcid to tankoubon $tankid!"; - my %tankoubon = LANraragi::Model::Tankoubon::get_tankoubon($tankid); - my $title = LANraragi::Model::Archive::get_title($arcid); - - if ( %tankoubon && defined($title) ) { - $successMessage = "Added \"$title\" to tankoubon \"$tankoubon{name}\"!"; - } - - render_api_response( $self, "add_to_tankoubon", undef, $successMessage ); - } else { - render_api_response( $self, "add_to_tankoubon", $err ); - } + my ( $result, $err ) = LANraragi::Model::PsilabsDev::PgTankoubon::add_to_tankoubon( $tankid, $arcid ); + + if ($result) { + my $successMessage = "Added $arcid to tankoubon $tankid!"; + my ( $total, $filtered, %tankoubon ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon($tankid); + my $title = LANraragi::Model::PsilabsDev::PgArchive::get_title($arcid); + + if ( %tankoubon && defined($title) ) { + $successMessage = "Added \"$title\" to tankoubon \"$tankoubon{name}\"!"; } - ); + + render_api_response( $self, "add_to_tankoubon", undef, $successMessage ); + } else { + render_api_response( $self, "add_to_tankoubon", $err ); + } } sub remove_from_tankoubon { @@ -177,36 +138,21 @@ sub remove_from_tankoubon { my $tankid = $self->stash('id'); my $arcid = $self->stash('archive'); - return unless exec_with_lock( - $self, - "tankoubon-write:$tankid", - "remove_from_tankoubon", - $tankid, - sub { - my ( $result, $err ) = LANraragi::Model::Tankoubon::remove_from_tankoubon( $tankid, $arcid ); - - if ($result) { - my $successMessage = "Removed $arcid from tankoubon $tankid!"; - my %tankoubon = LANraragi::Model::Tankoubon::get_tankoubon($tankid); - my $title = LANraragi::Model::Archive::get_title($arcid); - - if ( %tankoubon && defined($title) ) { - $successMessage = "Removed \"$title\" from tankoubon \"$tankoubon{name}\"!"; - } - - render_api_response( $self, "remove_from_tankoubon", undef, $successMessage ); - - # Queue thumbnail regeneration for the tankoubon if we removed the first archive - # (This technically messes up custom user thumbnails if they set one up, but we don't have a way to differentiate at the moment) - if ( $result == 1 ) { - my $thumbdir = LANraragi::Model::Config->get_thumbdir; - $self->minion->enqueue( tank_thumbnail_task => [ $thumbdir, $tankid ] => { priority => 0, attempts => 3 } ); - } - } else { - render_api_response( $self, "remove_from_tankoubon", $err ); - } + my ( $result, $err ) = LANraragi::Model::PsilabsDev::PgTankoubon::remove_from_tankoubon( $tankid, $arcid ); + + if ($result) { + my $successMessage = "Removed $arcid from tankoubon $tankid!"; + my ( $total, $filtered, %tankoubon ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon($tankid); + my $title = LANraragi::Model::PsilabsDev::PgArchive::get_title($arcid); + + if ( %tankoubon && defined($title) ) { + $successMessage = "Removed \"$title\" from tankoubon \"$tankoubon{name}\"!"; } - ); + + render_api_response( $self, "remove_from_tankoubon", undef, $successMessage ); + } else { + render_api_response( $self, "remove_from_tankoubon", $err ); + } } sub serve_tankoubon_thumbnail { @@ -279,7 +225,7 @@ sub get_tankoubons_file { return; } - my @tanks = LANraragi::Model::Tankoubon::get_tankoubons_containing_archive($arcid); + my @tanks = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubons_containing_archive($arcid); $self->render( openapi => { diff --git a/lib/LANraragi/Controller/Batch.pm b/lib/LANraragi/Controller/Batch.pm index 35691e55b..76bbc4fe8 100644 --- a/lib/LANraragi/Controller/Batch.pm +++ b/lib/LANraragi/Controller/Batch.pm @@ -1,16 +1,19 @@ package LANraragi::Controller::Batch; use Mojo::Base 'Mojolicious::Controller'; -use Redis; use Encode; use Mojo::JSON qw(decode_json); use LANraragi::Utils::Generic qw(generate_themes_header exec_with_lock_pure); use LANraragi::Utils::Tags qw(rewrite_tags build_tag_replace_hash split_tags_to_array restore_CRLF); -use LANraragi::Utils::Database qw(get_computed_tagrules set_tags set_title set_summary set_isnew invalidate_cache); +use LANraragi::Utils::Database qw(get_computed_tagrules); +use LANraragi::Utils::PsilabsDev::PgDatabase qw(set_tags set_title set_summary set_tags_with_dbh set_title_with_dbh set_summary_with_dbh set_isnew invalidate_cache); use LANraragi::Utils::Plugins qw(get_plugins get_plugin get_plugin_parameters); use LANraragi::Utils::Logging qw(get_logger); -use LANraragi::Utils::Redis qw(redis_decode); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Model::PsilabsDev::PgCategory; +use LANraragi::Utils::PsilabsDev::PgArchive qw(delete_archive); +use LANraragi::Model::PsilabsDev::PgPlugins qw(exec_metadata_plugin); # This action will render a template sub index { @@ -33,7 +36,7 @@ sub index { } # Get static category list - my @categories = LANraragi::Model::Category->get_static_category_list; + my @categories = LANraragi::Model::PsilabsDev::PgCategory::get_static_category_list(); $self->render( template => "batch", @@ -53,7 +56,6 @@ sub socket { my $self = shift; my $cancelled = 0; my $client = $self->tx; - my $redis = $self->LRR_CONF->get_redis; my $logger = get_logger( "Batch Tagging", "lanraragi" ); @@ -64,6 +66,22 @@ sub socket { my @rules = get_computed_tagrules(); my ( $rules, $hash_replace_rules ) = build_tag_replace_hash( \@rules ); + + # Prepare database connection and statement ONCE at WebSocket connection time + my $dbh = get_postgresql_dbh(); + my $tag_fetch_sth = $dbh->prepare(q{ + SELECT string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ) as tags + FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = ? + }); + $self->on( message => sub { my ( $self, $msg ) = @_; @@ -103,12 +121,12 @@ sub socket { $logger->debug("Overriding configured parameters"); if ( exists $args{customargs} ) { - # Decode user overrides - $args{customargs} = [ map { redis_decode($_) } @args_override ]; + # User overrides from JSON are already properly decoded + $args{customargs} = \@args_override; } else { my @keys = sort grep { $_ !~ m/^enabled$/ } keys %args; while ( my ( $idx, $key ) = each @keys ) { - $args{customargs}{$key} = redis_decode( $args_override[$idx] ); + $args{customargs}{$key} = $args_override[$idx]; } } @@ -134,7 +152,7 @@ sub socket { if ( $operation eq "addcat" ) { my $catid = $command->{"category"}; - my ( $catsucc, $caterr ) = LANraragi::Model::Category::add_to_category( $catid, $id ); + my ( $catsucc, $caterr ) = LANraragi::Model::PsilabsDev::PgCategory::add_to_category( $catid, $id ); $client->send( { json => { @@ -151,52 +169,61 @@ sub socket { if ( $operation eq "tagrules" ) { $logger->debug("Applying tag rules to $id..."); - my $tags = $redis->hget( $id, "tags" ); - $tags = redis_decode($tags); - my @tagarray = split_tags_to_array($tags); - @tagarray = rewrite_tags( \@tagarray, $rules, $hash_replace_rules ); + # Use WebSocket-level connection with explicit transaction + eval { + $dbh->begin_work; - # Merge array with commas - my $newtags = join( ', ', @tagarray ); - $logger->debug("New tags: $newtags"); - set_tags( $id, $newtags ); + # REUSE prepared statement instead of creating new connection per message + $tag_fetch_sth->execute($id); + my $row = $tag_fetch_sth->fetchrow_hashref; + my $tags = $row ? $row->{tags} : ""; - $client->send( - { json => { - id => $id, - success => 1, - tags => $newtags, - } - } - ); + my @tagarray = split_tags_to_array($tags); + @tagarray = rewrite_tags( \@tagarray, $rules, $hash_replace_rules ); - invalidate_cache(); + # Merge array with commas + my $newtags = join( ', ', @tagarray ); + $logger->debug("New tags: $newtags"); - return; - } + # Use _with_dbh variant to share connection + set_tags_with_dbh( $dbh, $id, $newtags, 0 ); - if ( $operation eq "delete" ) { - $logger->debug("Deleting $id..."); + $dbh->commit; - my ( $acquired, $delStatus ) = exec_with_lock_pure( - [ "archive-write:$id" ], - sub { LANraragi::Model::Archive::delete_archive($id) } - ); + $client->send( + { json => { + id => $id, + success => 1, + tags => $newtags, + } + } + ); + + invalidate_cache(); + }; - unless ($acquired) { + if ( my $error = $@ ) { + eval { $dbh->rollback }; + $logger->error("Failed to apply tag rules to $id: $error"); $client->send( { json => { - id => $id, - filename => "", - message => "Locked resource: $id.", - success => 0 + id => $id, + success => 0, + message => "Failed to apply tag rules: $error" } } ); - return; } + return; + } + + if ( $operation eq "delete" ) { + $logger->debug("Deleting $id..."); + + my $delStatus = delete_archive($id); + $client->send( { json => { id => $id, @@ -227,7 +254,10 @@ sub socket { finish => sub { $logger->info('Client disconnected, halting remaining operations'); $cancelled = 1; - $redis->quit(); + + # Clean up on WebSocket close + $tag_fetch_sth->finish if $tag_fetch_sth; + $dbh->disconnect if $dbh; } ); @@ -237,19 +267,38 @@ sub batch_plugin { my ( $id, $plugin, %args ) = @_; # Run plugin with args on id - my %plugin_result = LANraragi::Model::Plugins::exec_metadata_plugin( $plugin, $id, %args ); + my %plugin_result = exec_metadata_plugin( $plugin, $id, %args ); # If the plugin exec returned tags, add them unless ( exists $plugin_result{error} ) { - set_tags( $id, $plugin_result{new_tags}, 1 ); + # Wrap all metadata updates in a single transaction for atomicity + my $dbh = get_postgresql_dbh(); + $dbh->begin_work; + + eval { + # All metadata updates from this plugin in one transaction + if ( $plugin_result{new_tags} ) { + set_tags_with_dbh( $dbh, $id, $plugin_result{new_tags}, 1 ); + } - if ( exists $plugin_result{title} ) { - set_title( $id, $plugin_result{title} ); - } + if ( exists $plugin_result{title} ) { + set_title_with_dbh( $dbh, $id, $plugin_result{title} ); + } - if ( exists $plugin_result{summary} ) { - set_summary( $id, $plugin_result{summary} ); + if ( exists $plugin_result{summary} ) { + set_summary_with_dbh( $dbh, $id, $plugin_result{summary} ); + } + + $dbh->commit; + }; + + if ( my $error = $@ ) { + eval { $dbh->rollback }; + $dbh->disconnect; + die $error; } + + $dbh->disconnect; } return { diff --git a/lib/LANraragi/Controller/Category.pm b/lib/LANraragi/Controller/Category.pm index dc3e39e25..bdfa9f7af 100644 --- a/lib/LANraragi/Controller/Category.pm +++ b/lib/LANraragi/Controller/Category.pm @@ -3,26 +3,21 @@ use Mojo::Base 'Mojolicious::Controller'; use utf8; use URI::Escape; -use Redis; use Encode; use Mojo::Util qw(xml_escape); -use LANraragi::Model::Archive; -use LANraragi::Model::Tankoubon; use LANraragi::Utils::Generic qw(generate_themes_header); -use LANraragi::Utils::Redis qw(redis_decode); +use LANraragi::Model::PsilabsDev::PgArchive; +use LANraragi::Model::PsilabsDev::PgTankoubon; # Go through the archives in the content directory and build the template at the end. sub index { my $self = shift; - my $redis = $self->LRR_CONF->get_redis; my $userlogged = $self->LRR_CONF->enable_pass == 0 || $self->session('is_logged'); - $redis->quit(); - - my @idlist = LANraragi::Model::Archive::generate_archive_list; + my @idlist = LANraragi::Model::PsilabsDev::PgArchive::generate_archive_list(); #Parse the archive list and build
  • elements accordingly. my $arclist = ""; @@ -37,7 +32,7 @@ sub index { } # Build tank list - my ( $total, $filtered, @tanks ) = LANraragi::Model::Tankoubon::get_tankoubon_list(-1); + my ( $total, $filtered, @tanks ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon_list(-1); my $tanklist = ""; foreach my $tank (@tanks) { diff --git a/lib/LANraragi/Controller/Duplicates.pm b/lib/LANraragi/Controller/Duplicates.pm index 2cca0f08e..fea88a3c6 100644 --- a/lib/LANraragi/Controller/Duplicates.pm +++ b/lib/LANraragi/Controller/Duplicates.pm @@ -1,77 +1,134 @@ package LANraragi::Controller::Duplicates; use Mojo::Base 'Mojolicious::Controller'; use utf8; -use URI::Escape; -use Redis; use POSIX qw(strftime); -use Encode; use Mojo::JSON qw(decode_json encode_json); -use LANraragi::Utils::Redis qw(redis_decode); use LANraragi::Utils::Generic qw(generate_themes_header); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Model::Config; # Go through the archives in the content directory and build the template at the end. sub index { - my $self = shift; - my $redis_cfg = $self->LRR_CONF->get_redis_config; - my $redis = $self->LRR_CONF->get_redis; + my $self = shift; if ( $self->req->param('delete') ) { $self->LRR_LOGGER->debug("Cleared all detected duplicates!"); - $redis_cfg->del("LRR_DUPLICATE_GROUPS"); + eval { + my $redis = LANraragi::Model::Config->get_redis_config; + $redis->del("LRR_DUPLICATE_GROUPS"); + $redis->quit(); + }; + if (my $error = $@) { + $self->LRR_LOGGER->error("Error clearing duplicate groups: $error"); + } } - my %duplicate_groups = $redis_cfg->hgetall("LRR_DUPLICATE_GROUPS"); - my @duplicates; - - foreach my $key ( keys %duplicate_groups ) { - - # Decode the JSON-encoded array of IDs - my $deserialized = decode_json( $duplicate_groups{$key} ); - my @ids = @{$deserialized}; - - my @archives; - foreach my $id (@ids) { - my %archive = $redis->hgetall($id); - - my ( $name, $title, $tags ) = @archive{qw(name title tags)}; - ( $_ = redis_decode($_) ) for ( $name, $title, $tags ); - - # Check if archive still exists - if (%archive) { - $archive{'arcid'} = $id; - $archive{'group_key'} = $key; - $archive{'name'} = $name; - $archive{'title'} = $title; - $archive{'tags'} = $tags; - - if ( $tags =~ /date_added:(\d+)/ ) { - $archive{'date_added'} = strftime( "%Y-%m-%d %H:%M:%S", localtime($1) ); - } + my %duplicate_groups; + eval { + my $redis = LANraragi::Model::Config->get_redis_config; + if ( $redis->exists("LRR_DUPLICATE_GROUPS") ) { + %duplicate_groups = $redis->hgetall("LRR_DUPLICATE_GROUPS"); + } + $redis->quit(); + }; + if (my $error = $@) { + $self->LRR_LOGGER->error("Error fetching duplicate groups: $error"); + } - push @archives, \%archive; - } else { + my @duplicates; - # if dup size of group less than 2, its not a group anymore - if ( scalar @ids <= 2 ) { - my $size = scalar @ids; - $self->LRR_LOGGER->debug("group $key: too small ($size) - removing key"); - $redis_cfg->hdel( "LRR_DUPLICATE_GROUPS", $key ); + my $dbh; + eval { + $dbh = get_postgresql_dbh(); + + # Prepare statement once before loops for efficiency + my $sth = $dbh->prepare(q{ + SELECT arcid, filename, title, + COALESCE( + (SELECT string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ) + FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = ?), + '' + ) as tags + FROM lrr_archive + WHERE arcid = ? + }); + + foreach my $key ( keys %duplicate_groups ) { + + # Decode the JSON-encoded array of IDs + my $deserialized = decode_json( $duplicate_groups{$key} ); + my @ids = @{$deserialized}; + + my @archives; + foreach my $id (@ids) { + $sth->execute($id, $id); + my $row = $sth->fetchrow_hashref; + + # Check if archive still exists + if ($row) { + my %archive; + $archive{'arcid'} = $id; + $archive{'group_key'} = $key; + $archive{'name'} = $row->{filename}; + $archive{'title'} = $row->{title} // ""; + $archive{'tags'} = $row->{tags} // ""; + + if ( $archive{'tags'} =~ /date_added:(\d+)/ ) { + $archive{'date_added'} = strftime( "%Y-%m-%d %H:%M:%S", localtime($1) ); + } + + push @archives, \%archive; } else { - # archive vanished -> remove from dupes - @ids = grep { $_ ne $id } @ids; - $self->LRR_LOGGER->debug("group $key: archive $id vanished - removing from group"); - $redis_cfg->hset( "LRR_DUPLICATE_GROUPS", $key, encode_json( \@ids ) ); + # if dup size of group less than 2, its not a group anymore + if ( scalar @ids <= 2 ) { + my $size = scalar @ids; + $self->LRR_LOGGER->debug("group $key: too small ($size) - removing key"); + eval { + my $redis = LANraragi::Model::Config->get_redis_config; + $redis->hdel( "LRR_DUPLICATE_GROUPS", $key ); + $redis->quit(); + }; + if (my $error = $@) { + $self->LRR_LOGGER->error("Error deleting duplicate group $key: $error"); + } + } else { + + # archive vanished -> remove from dupes + @ids = grep { $_ ne $id } @ids; + $self->LRR_LOGGER->debug("group $key: archive $id vanished - removing from group"); + eval { + my $redis = LANraragi::Model::Config->get_redis_config; + $redis->hset( "LRR_DUPLICATE_GROUPS", $key, encode_json( \@ids ) ); + $redis->quit(); + }; + if (my $error = $@) { + $self->LRR_LOGGER->error("Error updating duplicate group $key: $error"); + } + } } } + push @duplicates, \@archives; } - push @duplicates, \@archives; - } - $redis->quit(); + $sth->finish; + $dbh->disconnect(); + }; + if ( my $error = $@ ) { + $self->LRR_LOGGER->error("Database error in duplicates endpoint: $error"); + $dbh->disconnect() if $dbh; + } $self->render( template => "duplicates", diff --git a/lib/LANraragi/Controller/Edit.pm b/lib/LANraragi/Controller/Edit.pm index 4b3817b6c..0694d61d4 100644 --- a/lib/LANraragi/Controller/Edit.pm +++ b/lib/LANraragi/Controller/Edit.pm @@ -2,15 +2,14 @@ package LANraragi::Controller::Edit; use Mojo::Base 'Mojolicious::Controller'; use File::Basename; -use Redis; use Encode; use Template; use Mojo::Util qw(xml_escape); use LANraragi::Utils::Generic qw(generate_themes_header); -use LANraragi::Utils::Redis qw(redis_decode); use LANraragi::Utils::Plugins qw(get_plugins); use LANraragi::Model::Tankoubon; +use LANraragi::Utils::PsilabsDev::PgDatabase qw(get_archive); sub index { my $self = shift; @@ -23,20 +22,14 @@ sub index { return $self->edit_tankoubon($id); } - my $redis = $self->LRR_CONF->get_redis; - - if ( $redis->exists($id) ) { - my %hash = $redis->hgetall($id); + my %hash = get_archive($id); + if ( %hash ) { my ( $name, $title, $tags, $summary, $file, $thumbhash ) = @hash{qw(name title tags summary file thumbhash)}; - ( $_ = redis_decode($_) ) for ( $name, $title, $tags, $summary ); - #Build plugin listing my @pluginlist = get_plugins("metadata"); - $redis->quit(); - $self->render( template => "edit", id => $id, diff --git a/lib/LANraragi/Controller/Index.pm b/lib/LANraragi/Controller/Index.pm index b374acbbc..5607b56f1 100644 --- a/lib/LANraragi/Controller/Index.pm +++ b/lib/LANraragi/Controller/Index.pm @@ -3,43 +3,30 @@ use Mojo::Base 'Mojolicious::Controller'; use utf8; use URI::Escape; -use Redis; use Encode; use File::Basename; use Authen::Passphrase; use LANraragi::Utils::Generic qw(generate_themes_header); -use LANraragi::Utils::Path qw(get_archive_path); +use LANraragi::Model::PsilabsDev::PgCategory; +use LANraragi::Model::PsilabsDev::PgArchive; # This endpoint is technically superseded by /api/search/random, but it's still useful in the Reader. sub random_archive { - my $self = shift; - my $archive = ""; - my $archiveexists = 0; - - my $redis = $self->LRR_CONF->get_redis; - - # We get a random archive ID. - # We check for the length to (sort-of) avoid not getting an archive ID. - # TODO: This will loop infinitely if there are zero archives in store. - until ($archiveexists) { - $archive = $redis->randomkey(); - - $self->LRR_LOGGER->debug("Found key $archive"); + my $self = shift; - #We got a key, but does the matching archive still exist on the server? - if ( length($archive) == 40 - && $redis->type($archive) eq "hash" - && $redis->hexists( $archive, "file" ) ) { - my $arclocation = get_archive_path( $redis, $archive ); - if ( -e $arclocation ) { $archiveexists = 1; } - } + # Get a random archive ID from Postgres + my $archive = LANraragi::Model::PsilabsDev::PgArchive::get_random_archive(); + + if ( defined($archive) ) { + $self->LRR_LOGGER->debug("Found random archive: $archive"); + # Redirect to the reader with the archive ID as parameter + $self->redirect_to( '/reader?id=' . $archive ); + } else { + # No archives available or none have valid files + $self->LRR_LOGGER->warn("No random archive found"); + $self->redirect_to('/'); } - - $redis->quit(); - - #We redirect to the reader, with the key as parameter. - $self->redirect_to( '/reader?id=' . $archive ); } # Render the index template with a few prefilled arguments. @@ -55,7 +42,7 @@ sub index { my $userlogged = $self->LRR_CONF->enable_pass == 0 || $self->session('is_logged'); # Get static category list to populate the right-click menu - my @categories = LANraragi::Model::Category->get_static_category_list; + my @categories = LANraragi::Model::PsilabsDev::PgCategory::get_static_category_list(); $self->render( template => "index", diff --git a/lib/LANraragi/Controller/Reader.pm b/lib/LANraragi/Controller/Reader.pm index 3ce449493..a97a5a758 100644 --- a/lib/LANraragi/Controller/Reader.pm +++ b/lib/LANraragi/Controller/Reader.pm @@ -5,6 +5,7 @@ use Mojo::URL; use Encode; use LANraragi::Utils::Generic qw(generate_themes_header); +use LANraragi::Model::PsilabsDev::PgCategory; # This action will render a template sub index { @@ -19,8 +20,8 @@ sub index { } # Allow adding to static categories - my @categories = LANraragi::Model::Category->get_static_category_list; - my @arc_categories = LANraragi::Model::Category::get_categories_containing_archive( $self->req->param('id') ); + my @categories = LANraragi::Model::PsilabsDev::PgCategory::get_static_category_list(); + my @arc_categories = LANraragi::Model::PsilabsDev::PgCategory::get_categories_containing_archive( $self->req->param('id') ); # Get query string from referrer URL, if there's one my $referrer = $self->req->headers->referrer; diff --git a/lib/LANraragi/Controller/Stats.pm b/lib/LANraragi/Controller/Stats.pm index 255cfbf79..8e35e3c4b 100644 --- a/lib/LANraragi/Controller/Stats.pm +++ b/lib/LANraragi/Controller/Stats.pm @@ -1,7 +1,7 @@ package LANraragi::Controller::Stats; use Mojo::Base 'Mojolicious::Controller'; -use LANraragi::Model::Stats; +use LANraragi::Model::PsilabsDev::PgStats; use LANraragi::Utils::Generic qw(generate_themes_header); # This action will render a template @@ -13,9 +13,9 @@ sub index { title => $self->LRR_CONF->get_htmltitle, descstr => $self->LRR_DESC, csshead => generate_themes_header($self), - archivecount => LANraragi::Model::Stats::get_archive_count, - arcsize => LANraragi::Model::Stats::compute_content_size, - pagestat => LANraragi::Model::Stats::get_page_stat, + archivecount => LANraragi::Model::PsilabsDev::PgStats::get_archive_count, + arcsize => LANraragi::Model::PsilabsDev::PgStats::compute_content_size, + pagestat => LANraragi::Model::PsilabsDev::PgStats::get_page_stat, version => $self->LRR_VERSION ); } diff --git a/lib/LANraragi/Controller/Upload.pm b/lib/LANraragi/Controller/Upload.pm index a42a84aed..5d4527f40 100644 --- a/lib/LANraragi/Controller/Upload.pm +++ b/lib/LANraragi/Controller/Upload.pm @@ -1,7 +1,6 @@ package LANraragi::Controller::Upload; use Mojo::Base 'Mojolicious::Controller'; -use Redis; use Encode; use File::Temp qw(tempdir tmpnam); use File::Find; @@ -9,6 +8,7 @@ use File::Basename; use LANraragi::Utils::Generic qw(generate_themes_header is_archive get_bytelength); use LANraragi::Utils::Path qw(move_path); +use LANraragi::Model::PsilabsDev::PgCategory; sub process_upload { my $self = shift; @@ -81,7 +81,7 @@ sub index { my $self = shift; # Allow adding to category on direct uploads - my @categories = LANraragi::Model::Category->get_static_category_list; + my @categories = LANraragi::Model::PsilabsDev::PgCategory::get_static_category_list(); $self->render( template => "upload", diff --git a/lib/LANraragi/Model/PsilabsDev/PgArchive.pm b/lib/LANraragi/Model/PsilabsDev/PgArchive.pm new file mode 100644 index 000000000..b520fa093 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgArchive.pm @@ -0,0 +1,736 @@ +package LANraragi::Model::PsilabsDev::PgArchive; + +use v5.36; +use experimental 'try'; + +use strict; +use warnings; +use utf8; + +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Generic qw(render_api_response); +use LANraragi::Utils::String qw(trim trim_CRLF); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::PgDatabase; +use LANraragi::Utils::Path qw(create_path unlink_path); +use LANraragi::Utils::Archive qw(extract_single_file); +use LANraragi::Utils::PageCache qw(fetch put); +use LANraragi::Utils::PsilabsDev::PgPath; +use LANraragi::Model::Config; +use LANraragi::Model::Reader; + +use File::Basename; +use File::Path qw(remove_tree); + +# replaces LANraragi::Model::Archive::generate_archive_list +# Generates an array of all the archive JSONs in the database that have existing files. +sub generate_archive_list { + + my $logger = get_logger( "PgArchive", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + my @archives; + + eval { + # Query all archives with their tags + my $sql = q{ + SELECT + a.arcid, + a.filename, + a.title, + a.summary, + a.isnew, + a.progress, + a.pagecount, + a.lastreadtime, + a.arcsize, + a.extension, + COALESCE( + string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ), + '' + ) as tags + FROM lrr_archive a + LEFT JOIN lrr_archive_to_tag_map atm ON a.arcid = atm.arcid + LEFT JOIN lrr_tag t ON atm.tagid = t.tagid + GROUP BY a.arcid, a.filename, a.title, a.summary, a.isnew, + a.progress, a.pagecount, a.lastreadtime, a.arcsize, a.extension + ORDER BY a.title + }; + + my $sth = $dbh->prepare($sql); + $sth->execute(); + + while (my $row = $sth->fetchrow_hashref) { + # Check if file exists on filesystem (matches Redis implementation in Database.pm:239-240) + my $file = create_path($row->{filename}); + next unless ( defined($file) && -e $file ); + + # Handle whitespace-only title (matches Redis implementation in Database.pm:246-248) + my $title = $row->{title}; + if ( !defined($title) || $title =~ /^\s*$/ ) { + $title = $row->{filename}; + } + + my $arcdata = { + arcid => $row->{arcid}, + title => $title, + filename => $row->{filename}, + tags => $row->{tags} // '', + summary => $row->{summary} // '', + isnew => $row->{isnew} ? 'true' : 'false', + extension => $row->{extension} // '', + progress => $row->{progress} ? int($row->{progress}) : 0, + pagecount => $row->{pagecount} ? int($row->{pagecount}) : 0, + lastreadtime => $row->{lastreadtime} ? int($row->{lastreadtime}) : 0, + size => $row->{arcsize} ? int($row->{arcsize}) : 0 + }; + + push @archives, $arcdata; + } + + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error generating archive list: $error"); + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); + + return @archives; +} + +# replaces serve_untagged_archivelist logic from LANraragi::Controller::Api::Archive +# Returns a list of archive IDs that have no "meaningful" tags. +# Archives are considered untagged if they have NO tags at all, OR only have tags +# from "basic" namespaces that don't count as tagged: artist, parody, series, +# language, event, group, date_added, timestamp, source +sub get_untagged_archives { + + my $logger = get_logger( "PgArchive", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + my @untagged; + + eval { + # Query archives that either have no tags, or only have tags from excluded namespaces + # This matches the Redis logic in LANraragi::Model::Stats.pm line 149 + my $sql = q{ + SELECT a.arcid + FROM lrr_archive a + WHERE NOT EXISTS ( + SELECT 1 + FROM lrr_archive_to_tag_map atm + INNER JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND t.namespace NOT IN ('artist', 'parody', 'series', 'language', 'event', 'group', 'date_added', 'timestamp', 'source') + ) + ORDER BY a.arcid + }; + + my $sth = $dbh->prepare($sql); + $sth->execute(); + + while (my $row = $sth->fetchrow_hashref) { + push @untagged, $row->{arcid}; + } + + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error retrieving untagged archives: $error"); + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); + + return @untagged; +} + +# replaces LANraragi::Controller::Index::random_archive logic +# Returns a random archive ID from the database that has an existing file on disk. +# Returns undef if no archives exist or none have valid files. +sub get_random_archive { + + my $logger = get_logger( "PgArchive", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + my $arcid; + + eval { + # Get a random archive ID using Postgres ORDER BY RANDOM() + # This is efficient enough for typical use cases + my $sql = q{ + SELECT arcid, filename + FROM lrr_archive + ORDER BY RANDOM() + LIMIT 1 + }; + + my $sth = $dbh->prepare($sql); + $sth->execute(); + + while (my $row = $sth->fetchrow_hashref) { + # Verify the file exists on disk before returning + my $file = create_path($row->{filename}); + if ( defined($file) && -e $file ) { + $arcid = $row->{arcid}; + last; + } + # If file doesn't exist, we could try again, but for simplicity + # we return undef. The controller can handle this. + } + + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error getting random archive: $error"); + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); + + return $arcid; +} + +# replaces LANraragi::Model::Archive::generate_page_thumbnails +sub generate_page_thumbnails { + + my ( $self, $id ) = @_; + + my $force = $self->req->param('force'); + $force = ( $force && $force eq "true" ) || "0"; # Prevent undef warnings by checking the variable first + + my $logger = get_logger( "PgArchive", "lanraragi" ); + my $thumbdir = LANraragi::Model::Config->get_thumbdir; + my $use_hq = LANraragi::Model::Config->get_hqthumbpages; + my $use_jxl = LANraragi::Model::Config->get_jxlthumbpages; + my $format = $use_jxl ? 'jxl' : 'jpg'; + + # Get the number of pages in the archive from Postgres + my $dbh = get_postgresql_dbh(); + my $pages; + + eval { + my $sql = q{SELECT pagecount FROM lrr_archive WHERE arcid = ?}; + my $sth = $dbh->prepare($sql); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $pages = $row->{pagecount} if $row; + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error retrieving pagecount for archive $id: $error"); + $dbh->disconnect(); + LANraragi::Utils::Generic::render_api_response( $self, "generate_page_thumbnails", $error ); + return; + } + + unless ($pages) { + $dbh->disconnect(); + LANraragi::Utils::Generic::render_api_response( $self, "generate_page_thumbnails", "Archive has no pagecount." ); + return; + } + + my $subfolder = substr( $id, 0, 2 ); + my $thumbname = "$thumbdir/$subfolder/$id.$format"; + + my $should_queue_job = 0; + + for ( my $page = 1; $page <= $pages; $page++ ) { + my $thumbname = "$thumbdir/$subfolder/$id/$page.$format"; + + unless ( $force == 0 && -e $thumbname ) { + $logger->debug("Thumbnail for page $page doesn't exist (path: $thumbname or force=$force), queueing job."); + $should_queue_job = 1; + last; + } + } + + # Done with Postgres + $dbh->disconnect(); + + if ($should_queue_job) { + + # Use Redis for ephemeral thumbjob tracking (caching operation) + my $redis = LANraragi::Model::Config->get_redis; + + # Check if a job is already queued for this archive + if ( $redis->hexists( $id, "thumbjob" ) ) { + + my $job_id = $redis->hget( $id, "thumbjob" ); + + # If the job is pending or running, don't queue a new job and just return this one + my $job_state = $self->minion->job($job_id)->info->{state}; + if ( $job_state eq "active" || $job_state eq "inactive" ) { + $self->render( + json => { + operation => "generate_page_thumbnails", + success => 1, + job => $job_id + }, + status => 202 # 202 Accepted + ); + $redis->quit; + return; + } + } + + # Queue a minion job to generate the thumbnails. Clients can check on its progress through the job ID. + my $job_id = $self->minion->enqueue( page_thumbnails => [ $id, $force ] => { priority => 0, attempts => 3 } ); + + # Save job in Redis so we can check on it if this endpoint is called again + $redis->hset( $id, "thumbjob", $job_id ); + $self->render( + json => { + operation => "generate_page_thumbnails", + success => 1, + job => $job_id + }, + status => 202 # 202 Accepted + ); + $redis->quit; + } else { + $self->render( + json => { + operation => "generate_page_thumbnails", + success => 1, + message => "No job queued, all thumbnails already exist." + }, + status => 200 # 200 OK + ); + } +} + +# replaces LANraragi::Controller::Api::Archive::update_progress (partial - the Redis metadata operations) +# Updates the reading progress for an archive. +# Returns a hashref with pagecount and lastreadtime values on success, or dies on error. +sub update_progress { + my ($id, $page, $force) = @_; + + my $logger = get_logger("PgArchive", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + my $time = time(); + my $pagecount; + + eval { + # Get the current pagecount for validation + my $sth = $dbh->prepare('SELECT pagecount FROM lrr_archive WHERE arcid = ?'); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + + unless ($row) { + die "Archive with ID $id not found"; + } + + $pagecount = $row->{pagecount}; + + # Update progress and lastreadtime + my $update_sth = $dbh->prepare('UPDATE lrr_archive SET progress = ?, lastreadtime = ? WHERE arcid = ?'); + $update_sth->execute($page, $time, $id); + $update_sth->finish; + + $logger->debug("Updated progress for archive $id to page $page"); + }; + + if (my $error = $@) { + $logger->error("Error updating progress for archive $id: $error"); + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); + + # Increment total pages read counter in Redis (configuration database) + # Configuration exception: LRR_TOTALPAGESTAT remains in Redis Database 2 + my $redis = LANraragi::Model::Config->get_redis_config; + $redis->incr("LRR_TOTALPAGESTAT"); + $redis->quit(); + + # Return pagecount and lastreadtime for the caller + return { + pagecount => $pagecount, + lastreadtime => $time + }; +} + +# replaces LANraragi::Model::Archive::get_page_data +sub get_page_data ($id, $path) { + my $cachekey = "page/$id/$path"; + my $content = fetch($cachekey); + if ( !defined($content) ) { + # Extract the file from the parent archive if it doesn't exist + my $dbh = get_postgresql_dbh(); + my $archive = LANraragi::Utils::PsilabsDev::PgPath::get_archive_path( $dbh, $id ); + $dbh->disconnect(); + $content = extract_single_file($archive, $path); + put($cachekey, $content); + } + return $content; +} + +# replaces LANraragi::Model::Archive::serve_page +sub serve_page { + my ( $self, $id, $path ) = @_; + + my $logger = get_logger( "File Serving", "lanraragi" ); + + $logger->debug("Page /$id/$path was requested"); + + # Apply resizing transformation if set in Settings + if ( LANraragi::Model::Config->enable_resize ) { + + # Store resized files in a subfolder of the ID's temp folder, keyed by quality + my $threshold = LANraragi::Model::Config->get_threshold; + my $quality = LANraragi::Model::Config->get_readquality; + + my $cachekey = "resize_page/$id/$path/$threshold/$quality"; + my $content = fetch($cachekey); + if ( !defined($content)) { + $content = LANraragi::Model::Reader::resize_image(get_page_data($id, $path), $quality, $threshold); + put($cachekey, $content); + } + + # LANraragi::Model::Reader::resize_image always converts the image to jpg + $self->render_file( + data => $content, + content_disposition => "inline", + format => "jpg" + ); + } else { + + # Get the file extension to report content-type properly + my ( $n, $p, $file_ext ) = fileparse( $path, qr/\.[^.]*/ ); + my $content = get_page_data($id, $path); + $logger->debug("Data size: " . length($content)); + # Serve extracted file directly + $self->render_file( + data => $content, + content_disposition => "inline", + format => substr( $file_ext, 1 ) + ); + } +} + +# archive_exists(id) +# Returns 1 if the archive exists in the database, 0 otherwise. +sub archive_exists ($id) { + + my $logger = get_logger( "PgArchive", "lanraragi" ); + + if ( $id eq "" ) { + $logger->debug("No archive ID provided."); + return 0; + } + + my $dbh = get_postgresql_dbh(); + my $exists = 0; + + eval { + my $sql = 'SELECT 1 FROM lrr_archive WHERE arcid = ? LIMIT 1'; + my $sth = $dbh->prepare($sql); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $exists = 1 if $row; + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error checking existence for archive $id: $error"); + } + + $dbh->disconnect(); + + return $exists; +} + +# replaces LANraragi::Model::Archive::get_title +# get_title(id) +# Returns the title for the archive matching the given id. +# Returns undef if the id doesn't exist. +sub get_title ($id) { + + my $logger = get_logger( "PgArchive", "lanraragi" ); + + if ( $id eq "" ) { + $logger->debug("No archive ID provided."); + return (); + } + + my $dbh = get_postgresql_dbh(); + my $title; + + eval { + my $sql = 'SELECT title FROM lrr_archive WHERE arcid = ?'; + my $sth = $dbh->prepare($sql); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $title = $row->{title} if $row; + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error retrieving title for archive $id: $error"); + } + + $dbh->disconnect(); + + return $title; +} + +# replaces LANraragi::Model::Archive::update_thumbnail +sub update_thumbnail { + + my ( $self, $id ) = @_; + + my $page = $self->req->param('page'); + $page = 1 unless $page; + + my $thumbdir = LANraragi::Model::Config->get_thumbdir; + my $use_jxl = LANraragi::Model::Config->get_jxlthumbpages; + my $format = $use_jxl ? 'jxl' : 'jpg'; + + # Thumbnails are stored in the content directory, thumb subfolder. + # Another subfolder with the first two characters of the id is used for FS optimization. + my $subfolder = substr( $id, 0, 2 ); + my $thumbname = "$thumbdir/$subfolder/$id.$format"; # Path to main thumbnail + + my $newthumb = ""; + + # Get the required thumbnail we want to make the main one + no warnings 'experimental::try'; + try { + $newthumb = LANraragi::Utils::PsilabsDev::PgArchive::extract_thumbnail( $thumbdir, $id, $page, 1, 1 ) + } catch ($e) { + LANraragi::Utils::Generic::render_api_response( $self, "update_thumbnail", $e ); + return; + } + + if ( !$newthumb ) { + LANraragi::Utils::Generic::render_api_response( $self, "update_thumbnail", "Thumbnail not generated." ); + } else { + $self->render( + json => { + operation => "update_thumbnail", + new_thumbnail => $newthumb, + success => 1 + } + ); + } + +} + +# replaces LANraragi::Model::Archive::update_metadata +sub update_metadata { + my ( $id, $title, $tags, $summary ) = @_; + + my $logger = get_logger( "PgArchive", "lanraragi" ); + + unless ( defined $title || defined $tags ) { + return "No metadata parameters (Please supply title, tags or summary)"; + } + + # Clean up the user's inputs. + ( $_ = LANraragi::Utils::String::trim($_) ) for ( $title, $tags ); + ( $_ = LANraragi::Utils::String::trim_CRLF($_) ) for ( $title, $tags ); + + # Use a transaction to ensure all metadata updates are atomic + my $dbh = get_postgresql_dbh(); + + eval { + $dbh->begin_work; + + if ( defined $title ) { + LANraragi::Utils::PsilabsDev::PgDatabase::set_title_with_dbh( $dbh, $id, $title ); + } + + if ( defined $tags ) { + LANraragi::Utils::PsilabsDev::PgDatabase::set_tags_with_dbh( $dbh, $id, $tags ); + } + + if ( defined $summary ) { + LANraragi::Utils::PsilabsDev::PgDatabase::set_summary_with_dbh( $dbh, $id, $summary ); + } + + $dbh->commit; + }; + + my $error = $@; + if ($error) { + $logger->error("Error updating metadata for archive $id: $error"); + eval { $dbh->rollback }; + $dbh->disconnect; + return $error; + } + + $dbh->disconnect; + + # No errors. + return ""; +} + +# replaces LANraragi::Model::Archive::delete_archive +# Deletes the archive with the given id from Postgres, and the matching archive file/thumbnail. +sub delete_archive ($id) { + + my $logger = get_logger( "PgArchive", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + my $filename; + + eval { + # Get archive filename before deletion + my $sql = 'SELECT filename FROM lrr_archive WHERE arcid = ?'; + my $sth = $dbh->prepare($sql); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + + unless ($row) { + $logger->warn("Archive $id doesn't exist in the database"); + die "Archive not found"; + } + + $filename = $row->{filename}; + + # Start transaction for multi-step deletion + $dbh->begin_work; + + # Delete from junction tables manually (schema has no CASCADE) + # Note: We delete junction tables in the same transaction for atomicity + + # Delete archive-to-tag mappings + my $del_tags_sql = 'DELETE FROM lrr_archive_to_tag_map WHERE arcid = ?'; + my $del_tags_sth = $dbh->prepare($del_tags_sql); + $del_tags_sth->execute($id); + $del_tags_sth->finish; + $logger->debug("Deleted tag mappings for archive $id"); + + # Delete archive-to-category mappings + my $del_cats_sql = 'DELETE FROM lrr_category_to_archive_map WHERE arcid = ?'; + my $del_cats_sth = $dbh->prepare($del_cats_sql); + $del_cats_sth->execute($id); + $del_cats_sth->finish; + $logger->debug("Deleted category mappings for archive $id"); + + # Delete archive-to-tank mappings + my $del_tanks_sql = 'DELETE FROM lrr_tank_to_archive_map WHERE arcid = ?'; + my $del_tanks_sth = $dbh->prepare($del_tanks_sql); + $del_tanks_sth->execute($id); + $del_tanks_sth->finish; + $logger->debug("Deleted tankoubon mappings for archive $id"); + + # Delete ToC entries + my $del_toc_sth = $dbh->prepare('DELETE FROM lrr_toc WHERE arcid = ?'); + $del_toc_sth->execute($id); + $del_toc_sth->finish; + $logger->debug("Deleted ToC entries for archive $id"); + + # Finally, delete the archive itself + my $del_arc_sql = 'DELETE FROM lrr_archive WHERE arcid = ?'; + my $del_arc_sth = $dbh->prepare($del_arc_sql); + $del_arc_sth->execute($id); + $del_arc_sth->finish; + + # Commit transaction + $dbh->commit; + + $logger->info("Deleted archive $id from database"); + }; + + if (my $error = $@) { + $logger->error("Error deleting archive $id: $error"); + eval { $dbh->rollback }; + $dbh->disconnect(); + return "0"; + } + + $dbh->disconnect(); + + # Delete the physical file and thumbnails + if ( defined($filename) && $filename ne "" ) { + my $file = create_path($filename); + + if ( defined($file) && -e $file ) { + my $status = unlink_path( $file ); + + my $thumbdir = LANraragi::Model::Config->get_thumbdir; + my $subfolder = substr( $id, 0, 2 ); + + my $jpg_thumbname = "$thumbdir/$subfolder/$id.jpg"; + unlink $jpg_thumbname; + + my $jxl_thumbname = "$thumbdir/$subfolder/$id.jxl"; + unlink $jxl_thumbname; + + # Delete the thumbpages folder + remove_tree("$thumbdir/$subfolder/$id/"); + + return $status ? $filename : "0"; + } + } + + return "0"; +} + +# replaces LANraragi::Model::Archive::add_toc_entry +sub add_toc_entry ( $id, $page, $title ) { + my $logger = get_logger( "PgArchive", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + eval { + my $sth = $dbh->prepare(q{ + INSERT INTO lrr_toc (arcid, page, title) + VALUES (?, ?, ?) + ON CONFLICT (arcid, page) DO UPDATE SET title = EXCLUDED.title + }); + $sth->execute( $id, $page, $title ); + $sth->finish; + }; + + if ( my $error = $@ ) { + $logger->error("Error adding ToC entry for archive $id page $page: $error"); + $dbh->disconnect(); + return $error; + } + + $dbh->disconnect(); + return ""; +} + +# replaces LANraragi::Model::Archive::remove_toc_entry +sub remove_toc_entry ( $id, $page ) { + my $logger = get_logger( "PgArchive", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + eval { + my $sth = $dbh->prepare('DELETE FROM lrr_toc WHERE arcid = ? AND page = ?'); + $sth->execute( $id, $page ); + $sth->finish; + }; + + if ( my $error = $@ ) { + $logger->error("Error removing ToC entry for archive $id page $page: $error"); + $dbh->disconnect(); + return $error; + } + + $dbh->disconnect(); + return ""; +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgBackup.pm b/lib/LANraragi/Model/PsilabsDev/PgBackup.pm new file mode 100644 index 000000000..949776c0a --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgBackup.pm @@ -0,0 +1,388 @@ +package LANraragi::Model::PsilabsDev::PgBackup; + +use strict; +use warnings; +use utf8; + +use Mojo::JSON qw(encode_json decode_json); + +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Model::PsilabsDev::PgTankoubon; +use LANraragi::Utils::PsilabsDev::PgDatabase qw(clean_categories_and_tanks); + +# replaces LANraragi::Model::Backup::build_backup_JSON +# build_backup_JSON() +# Goes through the Postgres database and builds a JSON string containing archive metadata. +sub build_backup_JSON { + my $dbh = get_postgresql_dbh(); + my $logger = get_logger("Backup/Restore", "lanraragi"); + + # Basic structure of the backup object + my %backup = ( + categories => [], + tankoubons => [], + archives => [] + ); + + # Backup categories first + my $cat_sql = <<'SQL'; + SELECT catid, name, COALESCE(search, '') as search + FROM lrr_category + ORDER BY catid +SQL + + my $cat_sth = $dbh->prepare($cat_sql); + $cat_sth->execute(); + + # Prepare archive statement ONCE before the category loop + my $cat_arc_sql = <<'SQL'; + SELECT arcid + FROM lrr_category_to_archive_map + WHERE catid = ? + ORDER BY arcid +SQL + my $cat_arc_sth = $dbh->prepare($cat_arc_sql); + + while (my $cat_row = $cat_sth->fetchrow_hashref) { + my $catid = $cat_row->{catid}; + + eval { + # Fetch archives for this category + $cat_arc_sth->execute($catid); + + my @archives; + while (my $arc_row = $cat_arc_sth->fetchrow_hashref) { + push @archives, $arc_row->{arcid}; + } + + # Build category hash matching Redis backup format + my %category = ( + catid => $catid, + name => $cat_row->{name}, + search => $cat_row->{search}, + archives => \@archives + ); + + push @{ $backup{categories} }, \%category; + }; + + $logger->trace("Backing up category $catid: $@"); + } + $cat_arc_sth->finish; + $cat_sth->finish; + + # Backup tanks + my ($total, $filtered, @tanks) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon_list(-1); + foreach my $tank (@tanks) { + my $tank_id = $tank->{id}; + my $tank_title = $tank->{name}; + my @tank_archives = @{ $tank->{archives} }; + + my %tank = ( + tankid => $tank_id, + name => $tank_title, + archives => \@tank_archives + ); + + push @{ $backup{tankoubons} }, \%tank; + } + + # Backup archives themselves next + my $arc_sql = <<'SQL'; + SELECT arcid, filename, title, summary, thumbhash + FROM lrr_archive + ORDER BY arcid +SQL + + my $arc_sth = $dbh->prepare($arc_sql); + $arc_sth->execute(); + + # Prepare tag statement ONCE before the archive loop + my $tag_sql = <<'SQL'; + SELECT CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END as tag + FROM lrr_archive_to_tag_map m + JOIN lrr_tag t ON m.tagid = t.tagid + WHERE m.arcid = ? + ORDER BY t.tagid +SQL + my $tag_sth = $dbh->prepare($tag_sql); + + while (my $arc_row = $arc_sth->fetchrow_hashref) { + my $id = $arc_row->{arcid}; + + eval { + # Get tags for this archive + $tag_sth->execute($id); + + my @tags; + while (my $tag_row = $tag_sth->fetchrow_hashref) { + push @tags, $tag_row->{tag}; + } + + my $tags_str = join(', ', @tags); + + # Backup all user-generated metadata, alongside the unique ID + my %arc = ( + arcid => $id, + title => $arc_row->{title} // '', + tags => $tags_str, + summary => $arc_row->{summary} // '', + thumbhash => $arc_row->{thumbhash} // '', + filename => $arc_row->{filename} + ); + + push @{ $backup{archives} }, \%arc; + }; + + $logger->trace("Backing up archive $id: $@"); + } + $tag_sth->finish; + $arc_sth->finish; + + $dbh->disconnect(); + return encode_json \%backup; +} + +# replaces LANraragi::Model::Backup::restore_from_JSON +# restore_from_JSON(backupJSON) +# Restores metadata from a JSON to the Postgres database, for existing IDs. +sub restore_from_JSON { + my $json_str = shift; + my $logger = get_logger("Backup/Restore", "lanraragi"); + my $json = decode_json($json_str); + + $logger->info("Received a JSON backup to restore."); + + my $dbh = get_postgresql_dbh(); + + # Clean categories and tankoubons before restoring from JSON + eval { + LANraragi::Utils::PsilabsDev::PgDatabase::clean_categories_and_tanks(); + }; + + my $clean_error = $@; + if ($clean_error) { + $logger->error("Failed to clean categories and tankoubons: $clean_error"); + $dbh->disconnect(); + return; + } + + # Prepare archive existence check statement ONCE for all restore operations + my $check_arc_sth = $dbh->prepare('SELECT arcid FROM lrr_archive WHERE arcid = ?'); + + # Restore categories + foreach my $category (@{ $json->{categories} }) { + my $cat_id = $category->{"catid"}; + $logger->info("Restoring Category $cat_id..."); + + my $name = $category->{"name"}; + my $search = $category->{"search"} // ''; + my @archives = @{ $category->{"archives"} // [] }; + + eval { + $dbh->begin_work(); + + # Create category + my $insert_cat_sql = <<'SQL'; + INSERT INTO lrr_category (catid, name, pinned, search) + VALUES (?, ?, ?, ?) + ON CONFLICT (catid) DO UPDATE + SET name = EXCLUDED.name, search = EXCLUDED.search +SQL + + my $insert_cat_sth = $dbh->prepare($insert_cat_sql); + $insert_cat_sth->execute($cat_id, $name, 0, $search); + $insert_cat_sth->finish; + + # Add archives to category + foreach my $arcid (@archives) { + # Check if archive exists + $check_arc_sth->execute($arcid); + my $arc_exists = $check_arc_sth->fetchrow_hashref; + + if ($arc_exists) { + my $insert_map_sql = <<'SQL'; + INSERT INTO lrr_category_to_archive_map (catid, arcid, update_date) + VALUES (?, ?, CURRENT_DATE) + ON CONFLICT DO NOTHING +SQL + + my $insert_map_sth = $dbh->prepare($insert_map_sql); + $insert_map_sth->execute($cat_id, $arcid); + $insert_map_sth->finish; + } + } + + $dbh->commit(); + }; + + my $cat_error = $@; + if ($cat_error) { + $logger->error("Failed to restore category $cat_id: $cat_error"); + $dbh->rollback(); + } + } + + # Restore tankoubons + foreach my $tank (@{ $json->{tankoubons} // [] }) { + my $tank_id = $tank->{"tankid"}; + $logger->info("Restoring Tankoubon $tank_id..."); + + my $name = $tank->{"name"}; + my @archives = @{ $tank->{"archives"} // [] }; + + eval { + $dbh->begin_work(); + + # Create tankoubon + my $insert_tank_sql = <<'SQL'; + INSERT INTO lrr_tank (tankid, name, summary, tags) + VALUES (?, ?, '', '') + ON CONFLICT (tankid) DO UPDATE + SET name = EXCLUDED.name +SQL + + my $insert_tank_sth = $dbh->prepare($insert_tank_sql); + $insert_tank_sth->execute($tank_id, $name); + $insert_tank_sth->finish; + + # Add archives to tankoubon with position + my $position = 1; + foreach my $arcid (@archives) { + # Check if archive exists + $check_arc_sth->execute($arcid); + my $arc_exists = $check_arc_sth->fetchrow_hashref; + + if ($arc_exists) { + my $insert_map_sql = <<'SQL'; + INSERT INTO lrr_tank_to_archive_map (tankid, arcid, position, update_date) + VALUES (?, ?, ?, CURRENT_DATE) + ON CONFLICT DO NOTHING +SQL + + my $insert_map_sth = $dbh->prepare($insert_map_sql); + $insert_map_sth->execute($tank_id, $arcid, $position); + $insert_map_sth->finish; + $position++; + } + } + + $dbh->commit(); + }; + + my $tank_error = $@; + if ($tank_error) { + $logger->error("Failed to restore tankoubon $tank_id: $tank_error"); + $dbh->rollback(); + } + } + + # Prepare SQL statements ONCE before loops (performance optimization) + my $insert_tag_sth = $dbh->prepare(<<'SQL'); + INSERT INTO lrr_tag (namespace, value) + VALUES (?, ?) + ON CONFLICT (namespace, value) DO NOTHING +SQL + + my $select_tagid_sth = $dbh->prepare(<<'SQL'); + SELECT tagid FROM lrr_tag WHERE namespace = ? AND value = ? +SQL + + my $insert_map_sth = $dbh->prepare(<<'SQL'); + INSERT INTO lrr_archive_to_tag_map (arcid, tagid, namespace, value, update_date) + VALUES (?, ?, ?, ?, CURRENT_DATE) + ON CONFLICT DO NOTHING +SQL + + # Restore archive metadata + foreach my $archive (@{ $json->{archives} }) { + my $id = $archive->{"arcid"}; + + eval { + # Check if archive exists + $check_arc_sth->execute($id); + my $exists = $check_arc_sth->fetchrow_hashref; + + if ($exists) { + $logger->info("Restoring metadata for Archive $id..."); + + $dbh->begin_work(); + + # Update archive metadata + my $update_sql = <<'SQL'; + UPDATE lrr_archive + SET title = ?, summary = ?, thumbhash = ? + WHERE arcid = ? +SQL + + my $update_sth = $dbh->prepare($update_sql); + $update_sth->execute( + $archive->{"title"} // '', + $archive->{"summary"} // '', + $archive->{"thumbhash"} // '', + $id + ); + $update_sth->finish; + + # Parse and insert tags + my $tags_str = $archive->{"tags"} // ''; + if ($tags_str) { + # Delete existing tags for this archive + my $delete_tags_sql = 'DELETE FROM lrr_archive_to_tag_map WHERE arcid = ?'; + my $delete_tags_sth = $dbh->prepare($delete_tags_sql); + $delete_tags_sth->execute($id); + $delete_tags_sth->finish; + + # Split tags by comma + my @tags = split(/,\s*/, $tags_str); + foreach my $tag (@tags) { + next unless $tag; + + my ($namespace, $value); + if ($tag =~ /^([^:]+):(.+)$/) { + $namespace = $1; + $value = $2; + } else { + $namespace = ''; + $value = $tag; + } + + # Insert or do nothing if exists + $insert_tag_sth->execute($namespace, $value); + + # Get the tagid (works whether tag was just inserted or already existed) + $select_tagid_sth->execute($namespace, $value); + my $tag_row = $select_tagid_sth->fetchrow_hashref; + my $tagid = $tag_row->{tagid}; + + # Link tag to archive (using pre-prepared statement) + $insert_map_sth->execute($id, $tagid, $namespace, $value); + } + } + + $dbh->commit(); + } + }; + + my $arc_error = $@; + if ($arc_error) { + $logger->error("Failed to restore archive $id: $arc_error"); + $dbh->rollback() if $dbh->{AutoCommit} == 0; + } + } + + # Finish prepared statements after all loops complete + $check_arc_sth->finish; + $insert_tag_sth->finish; + $select_tagid_sth->finish; + $insert_map_sth->finish; + + $dbh->disconnect(); + $logger->info("Backup restore completed."); +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgCategory.pm b/lib/LANraragi/Model/PsilabsDev/PgCategory.pm new file mode 100644 index 000000000..826652835 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgCategory.pm @@ -0,0 +1,625 @@ +package LANraragi::Model::PsilabsDev::PgCategory; + +use strict; +use warnings; +use utf8; + +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Model::Config; + +# replaces: LANraragi::Model::Category::get_category_list +# get_category_list() +# Returns a list of all the category objects. +sub get_category_list { + my $logger = get_logger("PgCategory", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + # Query for all categories + my $cat_sql = <<'SQL'; + SELECT catid, name, pinned, COALESCE(search, '') as search + FROM lrr_category + ORDER BY catid +SQL + + my $cat_sth = $dbh->prepare($cat_sql); + $cat_sth->execute(); + + # Prepare archive fetching statement once before loop + my $arc_sql = <<'SQL'; + SELECT arcid + FROM lrr_category_to_archive_map + WHERE catid = ? + ORDER BY arcid +SQL + my $arc_sth = $dbh->prepare($arc_sql); + + my @result; + + while (my $cat_row = $cat_sth->fetchrow_hashref) { + my $catid = $cat_row->{catid}; + my $search = $cat_row->{search}; + + my @archives; + + # Only fetch archives for static categories (search is empty) + if ($search eq '') { + $arc_sth->execute($catid); + + while (my $arc_row = $arc_sth->fetchrow_hashref) { + push @archives, $arc_row->{arcid}; + } + # Don't finish here - allows statement handle reuse + } + + # Build category hash matching Redis implementation format + my %category = ( + id => $catid, + name => $cat_row->{name}, + search => $search, + pinned => $cat_row->{pinned} ? 1 : 0, # Convert boolean to 1/0 + archives => \@archives + ); + + push @result, \%category; + } + $cat_sth->finish; + $arc_sth->finish; + + $dbh->disconnect(); + + $logger->debug("Found " . scalar(@result) . " categories"); + + return @result; +} + +# replaces: LANraragi::Model::Category::get_category +# get_category(categoryid) +# Returns the category with the given ID. +# Returns an empty hash if the category doesn't exist. +sub get_category { + my $cat_id = $_[0]; + my $logger = get_logger("PgCategory", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + if ( $cat_id eq "" ) { + $logger->debug("No category ID provided."); + $dbh->disconnect(); + return (); + } + + # Check if category exists and fetch its data + my $cat_sql = 'SELECT catid, name, pinned, COALESCE(search, \'\') as search FROM lrr_category WHERE catid = ?'; + my $cat_sth = $dbh->prepare($cat_sql); + $cat_sth->execute($cat_id); + my $cat_row = $cat_sth->fetchrow_hashref; + $cat_sth->finish; + + unless ($cat_row) { + $logger->warn("$cat_id doesn't exist in the database!"); + $dbh->disconnect(); + return (); + } + + my %category = ( + id => $cat_id, + name => $cat_row->{name}, + search => $cat_row->{search}, + pinned => $cat_row->{pinned} ? 1 : 0, # Convert boolean to 1/0 + ); + + # For static categories, fetch the archives list + # For dynamic categories, return an empty array + if ( $category{search} eq "" ) { + my $arc_sql = <<'SQL'; + SELECT arcid + FROM lrr_category_to_archive_map + WHERE catid = ? + ORDER BY arcid +SQL + + my $arc_sth = $dbh->prepare($arc_sql); + $arc_sth->execute($cat_id); + + my @archives; + while (my $arc_row = $arc_sth->fetchrow_hashref) { + push @archives, $arc_row->{arcid}; + } + $arc_sth->finish; + + $category{archives} = \@archives; + } else { + # Dynamic category - return empty archives array + $category{archives} = []; + } + + $dbh->disconnect(); + + return %category; +} + +# replaces: LANraragi::Model::Category::get_static_category_list +# get_static_category_list() +# Returns a list of all the static category objects. +sub get_static_category_list { + my $logger = get_logger("PgCategory", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + # Query for static categories (where search is NULL or empty string) + my $cat_sql = <<'SQL'; + SELECT catid, name, pinned, COALESCE(search, '') as search + FROM lrr_category + WHERE search IS NULL OR search = '' + ORDER BY catid +SQL + + my $cat_sth = $dbh->prepare($cat_sql); + $cat_sth->execute(); + + # Prepare archive fetching statement once before loop + my $arc_sql = <<'SQL'; + SELECT arcid + FROM lrr_category_to_archive_map + WHERE catid = ? + ORDER BY arcid +SQL + my $arc_sth = $dbh->prepare($arc_sql); + + my @result; + + while (my $cat_row = $cat_sth->fetchrow_hashref) { + my $catid = $cat_row->{catid}; + + # Fetch archives for this category + $arc_sth->execute($catid); + + my @archives; + while (my $arc_row = $arc_sth->fetchrow_hashref) { + push @archives, $arc_row->{arcid}; + } + # Don't finish here - allows statement handle reuse + + # Build category hash matching Redis implementation format + my %category = ( + id => $catid, + name => $cat_row->{name}, + search => $cat_row->{search}, + pinned => $cat_row->{pinned} ? 1 : 0, # Convert boolean to 1/0 + archives => \@archives + ); + + push @result, \%category; + } + $cat_sth->finish; + $arc_sth->finish; + + $dbh->disconnect(); + + $logger->debug("Found " . scalar(@result) . " static categories"); + + return @result; +} + +# replaces: LANraragi::Model::Category::get_categories_containing_archive +# get_categories_containing_archive(id) +# Returns a list of all the categories that contain the given archive. +sub get_categories_containing_archive { + my $archive_id = shift; + + my $logger = get_logger("PgCategory", "lanraragi"); + $logger->debug("Finding categories containing $archive_id"); + + my $dbh = get_postgresql_dbh(); + + # Query for static categories containing the archive + my $sql = <<'SQL'; + SELECT c.catid, c.name, c.pinned, COALESCE(c.search, '') as search + FROM lrr_category c + INNER JOIN lrr_category_to_archive_map m ON c.catid = m.catid + WHERE m.arcid = ? + AND (c.search IS NULL OR c.search = '') + ORDER BY c.catid +SQL + + my $cat_sth = $dbh->prepare($sql); + $cat_sth->execute($archive_id); + + # Prepare archive fetching statement once before loop + my $arc_sql = <<'SQL'; + SELECT arcid + FROM lrr_category_to_archive_map + WHERE catid = ? + ORDER BY arcid +SQL + my $arc_sth = $dbh->prepare($arc_sql); + + my @result; + + while (my $cat_row = $cat_sth->fetchrow_hashref) { + my $catid = $cat_row->{catid}; + + $logger->debug("$archive_id is in '" . $cat_row->{name} . "'"); + + # Fetch all archives for this category to match Redis format + $arc_sth->execute($catid); + + my @archives; + while (my $arc_row = $arc_sth->fetchrow_hashref) { + push @archives, $arc_row->{arcid}; + } + # Don't finish here - allows statement handle reuse + + # Build category hash matching Redis implementation format + my %category = ( + id => $catid, + name => $cat_row->{name}, + search => $cat_row->{search}, + pinned => $cat_row->{pinned} ? 1 : 0, # Convert boolean to 1/0 + archives => \@archives + ); + + push @result, \%category; + } + $cat_sth->finish; + $arc_sth->finish; + + $dbh->disconnect(); + + return @result; +} + +# replaces: LANraragi::Model::Category::add_to_category +# add_to_category(categoryid, arcid) +# Adds the given archive ID to the given category. +# Only valid if the category is Static. +# Returns 1 on success, 0 on failure alongside an error message. +sub add_to_category { + my ( $cat_id, $arc_id ) = @_; + my $logger = get_logger("PgCategory", "lanraragi"); + my $dbh = get_postgresql_dbh(); + my $err = ""; + + # Check if category exists + my $cat_check_sql = 'SELECT catid, search FROM lrr_category WHERE catid = ?'; + my $cat_sth = $dbh->prepare($cat_check_sql); + $cat_sth->execute($cat_id); + my $cat_row = $cat_sth->fetchrow_hashref; + $cat_sth->finish; + + if (!$cat_row) { + $err = "$cat_id doesn't exist in the database!"; + $logger->warn($err); + $dbh->disconnect(); + return (0, $err); + } + + # Check if category is static (search field is NULL or empty) + my $search = $cat_row->{search} // ''; + unless ($search eq '') { + $err = "$cat_id is a favorite search/dynamic category, can't add archives to it."; + $logger->error($err); + $dbh->disconnect(); + return (0, $err); + } + + # Check if archive exists + my $arc_check_sql = 'SELECT arcid FROM lrr_archive WHERE arcid = ?'; + my $arc_sth = $dbh->prepare($arc_check_sql); + $arc_sth->execute($arc_id); + my $arc_row = $arc_sth->fetchrow_hashref; + $arc_sth->finish; + + if (!$arc_row) { + $err = "$arc_id does not exist in the database."; + $logger->error($err); + $dbh->disconnect(); + return (0, $err); + } + + # Check if archive is already in the category + my $check_sql = 'SELECT 1 FROM lrr_category_to_archive_map WHERE catid = ? AND arcid = ?'; + my $check_sth = $dbh->prepare($check_sql); + $check_sth->execute($cat_id, $arc_id); + my $exists = $check_sth->fetchrow_hashref; + $check_sth->finish; + + if ($exists) { + $err = "$arc_id already present in category $cat_id, doing nothing."; + $logger->warn($err); + $dbh->disconnect(); + return (1, $err); + } + + # Add archive to category + my $insert_sql = <<'SQL'; + INSERT INTO lrr_category_to_archive_map (catid, arcid, update_date) + VALUES (?, ?, CURRENT_DATE) +SQL + + my $insert_sth = $dbh->prepare($insert_sql); + eval { + $insert_sth->execute($cat_id, $arc_id); + }; + + my $insert_error = $@; + if ($insert_error) { + $err = "Failed to add $arc_id to category $cat_id: $insert_error"; + $logger->error($err); + $dbh->disconnect(); + return (0, $err); + } + + $insert_sth->finish; + $dbh->disconnect(); + + $logger->debug("Added $arc_id to category $cat_id"); + + # Postgres doesn't need cache invalidation + return (1, $err); +} + +# replaces: LANraragi::Model::Category::create_category +# create_category(name, favtag, pinned, existing_id) +# Create a Category. +# If the "favtag" argument is supplied, the category will be Dynamic. +# Otherwise, it'll be Static. +# If an existing category ID is supplied, said category will be updated with the given parameters. +# Returns the ID of the created/updated Category. +sub create_category { + my ( $name, $favtag, $pinned, $cat_id ) = @_; + my $logger = get_logger("PgCategory", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + # Set all fields of the category object + unless ( length($cat_id) ) { + $cat_id = "SET_" . time(); + + my $isnewkey = 0; + # Prepare statement ONCE before the loop + my $check_sth = $dbh->prepare('SELECT catid FROM lrr_category WHERE catid = ?'); + + until ($isnewkey) { + # Check if the category ID exists, move timestamp further if it does + $check_sth->execute($cat_id); + my $exists = $check_sth->fetchrow_hashref; + + if ($exists) { + $cat_id = "SET_" . ( time() + 1 ); + } else { + $isnewkey = 1; + } + } + + $check_sth->finish; # Finish AFTER loop + } + + # Check if category exists + my $exists_sth = $dbh->prepare('SELECT catid FROM lrr_category WHERE catid = ?'); + $exists_sth->execute($cat_id); + my $existing = $exists_sth->fetchrow_hashref; + $exists_sth->finish; + + # Convert pinned to boolean + my $pinned_bool = $pinned ? 1 : 0; + + if ($existing) { + # Update existing category + my $update_sth = $dbh->prepare( + 'UPDATE lrr_category SET name = ?, search = ?, pinned = ? WHERE catid = ?' + ); + $update_sth->execute($name, $favtag, $pinned_bool, $cat_id); + $update_sth->finish; + $logger->debug("Updated category $cat_id"); + } else { + # Insert new category + my $insert_sth = $dbh->prepare( + 'INSERT INTO lrr_category (catid, name, search, pinned) VALUES (?, ?, ?, ?)' + ); + $insert_sth->execute($cat_id, $name, $favtag, $pinned_bool); + $insert_sth->finish; + $logger->debug("Created new category $cat_id"); + } + + $dbh->disconnect(); + return $cat_id; +} + +# replaces: LANraragi::Model::Category::get_bookmark_link +# get_bookmark_link() +# Gets the ID of the category that is linked to the bookmark button. +# If no such ID exists, returns an empty string. +sub get_bookmark_link { + my $redis = LANraragi::Model::Config->get_redis_config(); + my $catid = $redis->hget('LRR_CONFIG', 'bookmark_link') || ""; + $redis->quit(); + return $catid; +} + +# replaces: LANraragi::Model::Category::update_bookmark_link +# update_bookmark_link(cat_id) +# Links the bookmark button to a static category. +# Returns an HTTP status code, category ID, and response message. +sub update_bookmark_link { + my $cat_id = shift; + my $logger = get_logger("PgCategory", "lanraragi"); + + unless (defined $cat_id && $cat_id =~ /^SET_\d{10}$/) { + return (400, $cat_id, "Input category ID is invalid."); + } + + my $dbh = get_postgresql_dbh(); + + # Check if category exists using Postgres + my $check_sql = 'SELECT catid, search FROM lrr_category WHERE catid = ?'; + my $check_sth = $dbh->prepare($check_sql); + $check_sth->execute($cat_id); + my $cat_row = $check_sth->fetchrow_hashref; + $check_sth->finish; + + unless ($cat_row) { + $dbh->disconnect(); + return (404, $cat_id, "Category does not exist!"); + } + + # Check if category is static (search field is NULL or empty) + my $search = $cat_row->{search} // ''; + unless ($search eq '') { + $dbh->disconnect(); + return (400, $cat_id, "Cannot link bookmark to a dynamic category."); + } + + $dbh->disconnect(); + + # Store bookmark_link in Redis config + my $redis = LANraragi::Model::Config->get_redis_config(); + $redis->hset('LRR_CONFIG', 'bookmark_link', $cat_id); + $redis->quit(); + + $logger->info("Updated bookmark link to category $cat_id"); + return (200, $cat_id, "success"); +} + +# replaces: LANraragi::Model::Category::remove_bookmark_link +# remove_bookmark_link() +# Unlinks the bookmark from its current category and returns the category ID. +sub remove_bookmark_link { + my $logger = get_logger("PgCategory", "lanraragi"); + my $redis = LANraragi::Model::Config->get_redis_config(); + my $cat_id = $redis->hget('LRR_CONFIG', 'bookmark_link') || ""; + $redis->hdel('LRR_CONFIG', 'bookmark_link'); + $redis->quit(); + $logger->info("Removed bookmark link from category " . ($cat_id || "(none)")); + return $cat_id; +} + +# replaces: LANraragi::Model::Category::remove_from_category +# remove_from_category(categoryid, arcid) +# Removes the given archive ID from the given category. +# Only valid if the category is a Static category. +# Returns 1 on success, 0 on failure alongside an error message. +sub remove_from_category { + my ( $cat_id, $arc_id ) = @_; + my $logger = get_logger("PgCategory", "lanraragi"); + my $dbh = get_postgresql_dbh(); + my $err = ""; + + # Check if category exists + my $cat_check_sql = 'SELECT catid, search FROM lrr_category WHERE catid = ?'; + my $cat_sth = $dbh->prepare($cat_check_sql); + $cat_sth->execute($cat_id); + my $cat_row = $cat_sth->fetchrow_hashref; + $cat_sth->finish; + + if (!$cat_row) { + $err = "$cat_id doesn't exist in the database!"; + $logger->warn($err); + $dbh->disconnect(); + return (0, $err); + } + + # Check if category is static (search field is NULL or empty) + my $search = $cat_row->{search} // ''; + unless ($search eq '') { + $err = "$cat_id is a favorite search, it doesn't contain archives."; + $logger->error($err); + $dbh->disconnect(); + return (0, $err); + } + + # Remove archive from category + my $delete_sql = 'DELETE FROM lrr_category_to_archive_map WHERE catid = ? AND arcid = ?'; + my $delete_sth = $dbh->prepare($delete_sql); + eval { + $delete_sth->execute($cat_id, $arc_id); + }; + + my $delete_error = $@; + if ($delete_error) { + $err = "Failed to remove $arc_id from category $cat_id: $delete_error"; + $logger->error($err); + $dbh->disconnect(); + return (0, $err); + } + + $delete_sth->finish; + $dbh->disconnect(); + + $logger->debug("Removed $arc_id from category $cat_id"); + + # Postgres doesn't need cache invalidation + return (1, $err); +} + +# replaces: LANraragi::Model::Category::delete_category +# delete_category(id) +# Deletes the category with the given ID. +# If bookmark is linked to the category, remove the link. +# Returns 0 if the given ID isn't a category ID, 1 otherwise +sub delete_category { + my $cat_id = $_[0]; + my $logger = get_logger("PgCategory", "lanraragi"); + + if ( length($cat_id) != 14 ) { + # Probably not a category ID + $logger->error("$cat_id is not a category ID, doing nothing."); + return 0; + } + + my $dbh = get_postgresql_dbh(); + + # Check if category exists + my $check_sql = 'SELECT catid FROM lrr_category WHERE catid = ?'; + my $check_sth = $dbh->prepare($check_sql); + $check_sth->execute($cat_id); + my $exists = $check_sth->fetchrow_hashref; + $check_sth->finish; + + if ($exists) { + # Check if bookmark is linked to this category and remove if so + my $redis = LANraragi::Model::Config->get_redis_config(); + my $bookmark_catid = $redis->hget('LRR_CONFIG', 'bookmark_link') || ""; + + if ($bookmark_catid eq $cat_id) { + $redis->hdel('LRR_CONFIG', 'bookmark_link'); + $logger->info("Removed link from bookmark to category $cat_id."); + } + $redis->quit(); + + # Delete the category + # First, delete all archive mappings for this category + # Then delete the category itself + $dbh->begin_work(); + eval { + # Delete archive mappings + my $delete_map_sql = 'DELETE FROM lrr_category_to_archive_map WHERE catid = ?'; + my $delete_map_sth = $dbh->prepare($delete_map_sql); + $delete_map_sth->execute($cat_id); + $delete_map_sth->finish; + + # Delete the category + my $delete_cat_sql = 'DELETE FROM lrr_category WHERE catid = ?'; + my $delete_cat_sth = $dbh->prepare($delete_cat_sql); + $delete_cat_sth->execute($cat_id); + $delete_cat_sth->finish; + + $dbh->commit(); + }; + + if ( my $error = $@ ) { + $dbh->rollback(); + $logger->error("Error deleting category $cat_id: $error"); + $dbh->disconnect(); + return 0; + } + + $dbh->disconnect(); + $logger->info("Deleted category $cat_id"); + return 1; + } else { + $logger->warn("$cat_id doesn't exist in the database!"); + $dbh->disconnect(); + return 1; + } +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgOpds.pm b/lib/LANraragi/Model/PsilabsDev/PgOpds.pm new file mode 100644 index 000000000..969edba94 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgOpds.pm @@ -0,0 +1,243 @@ +package LANraragi::Model::PsilabsDev::PgOpds; + +use strict; +use warnings; +use utf8; + +use POSIX qw(strftime); +use Mojo::Util qw(xml_escape); +use File::Basename; + +use LANraragi::Utils::Generic qw(get_tag_with_namespace); +use LANraragi::Utils::Archive qw(get_filelist extract_single_file); +use LANraragi::Utils::PageCache qw(fetch put); +use LANraragi::Utils::Resizer qw(get_resizer); +use LANraragi::Utils::PsilabsDev::PgDatabase qw(get_archive_json); +use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); + +use LANraragi::Model::PsilabsDev::PgCategory; +use LANraragi::Model::PsilabsDev::PgSearch; + +# replaces LANraragi::Model::Opds::generate_opds_catalog +sub generate_opds_catalog { + + my $mojo = shift; + my $cat_id = $mojo->req->param('category') || ""; + my $start = $mojo->req->param('start') || 0; + + # If the user authentified to this via an API key, we need to carry it over to the OPDS links. + my $api_key = $mojo->req->param('key'); + my @cats = LANraragi::Model::PsilabsDev::PgCategory::get_category_list(); + + # Use the Postgres search engine to get the list of archives to show in the catalog. + # TODO Add tankgroup support to opds? + my ( $total, $filtered, @keys ) = LANraragi::Model::PsilabsDev::PgSearch::do_search( "", $cat_id, $start, "title", 0, 0, 0, 0 ); + + my @list = (); + + foreach my $id (@keys) { + my $arcdata = get_opds_data($id); + push @list, $arcdata if $arcdata; + } + + foreach my $cat (@cats) { + + for ( values %{$cat} ) { $_ = xml_escape($_); } + + # If the category doesn't have a search string, we can add the total count of archives to the entry. + if ( $cat->{search} eq "" ) { + $cat->{count} = scalar @{ $cat->{archives} }; + } + + if ( $cat->{id} eq $cat_id ) { + $cat->{active} = 1; + } + } + + # Sort lists to get reproducible results + @list = sort { lc( $a->{title} ) cmp lc( $b->{title} ) } @list; + @cats = sort { lc( $a->{name} ) cmp lc( $b->{name} ) } @cats; + + return $mojo->render_to_string( + template => "opds", + arclist => \@list, + catlist => \@cats, + nocat => $cat_id eq "", + nextpage => $start + scalar @list, + title => $mojo->LRR_CONF->get_htmltitle, + motd => $mojo->LRR_CONF->get_motd, + version => $mojo->LRR_VERSION, + api_key_query => $api_key ? "?key=" . $api_key : "", + api_key_and => $api_key ? "&key=" . $api_key : "" + ); +} + +# replaces LANraragi::Model::Opds::generate_opds_item +sub generate_opds_item { + + my ( $mojo, $id ) = @_; + + # If the user authentified to this via an API key, we need to carry it over to the OPDS links. + my $api_key = $mojo->req->param('key'); + + # Detailed pages just return a single entry instead of all the archives. + my $arcdata = get_opds_data($id); + + return $mojo->render_to_string( + template => "opds_entry", + arc => $arcdata, + title => $mojo->LRR_CONF->get_htmltitle, + motd => $mojo->LRR_CONF->get_motd, + version => $mojo->LRR_VERSION, + api_key_query => $api_key ? "?key=" . $api_key : "", + api_key_and => $api_key ? "&key=" . $api_key : "" + ); +} + +# replaces LANraragi::Model::Opds::get_opds_data +sub get_opds_data { + + my $id = shift; + my $dbh = get_postgresql_dbh(); + + my $file = get_archive_path( $dbh, $id ); + unless ( -e $file ) { + $dbh->disconnect(); + return; + } + + my $arcdata = get_archive_json( $dbh, $id ); + unless ($arcdata) { + $dbh->disconnect(); + return; + } + + my $tags = $arcdata->{tags}; + + # Parse date from the date_added tag, and convert from unix time to ISO 8601. + my $date = get_tag_with_namespace( "date_added", $tags, "0" ); + $arcdata->{dateadded} = strftime( "%Y-%m-%dT%H:%M:%SZ", gmtime($date) ); + + # Infer a few OPDS-related fields from the tags + $arcdata->{author} = get_tag_with_namespace( "artist", $tags, "" ); + $arcdata->{language} = get_tag_with_namespace( "language", $tags, "" ); + $arcdata->{circle} = get_tag_with_namespace( "group", $tags, "" ); + $arcdata->{event} = get_tag_with_namespace( "event", $tags, "" ); + + # Application/zip is universally hated by all readers so it's better to use x-cbz and x-cbr here. + if ( $file =~ /^(.*\/)*.+\.(pdf)$/ ) { + $arcdata->{mimetype} = "application/pdf"; + } elsif ( $file =~ /^(.*\/)*.+\.(rar|cbr)$/ ) { + $arcdata->{mimetype} = "application/x-cbr"; + } elsif ( $file =~ /^(.*\/)*.+\.(epub)$/ ) { + $arcdata->{mimetype} = "application/epub+zip"; + } else { + $arcdata->{mimetype} = "application/x-cbz"; + } + + if ( $arcdata->{lastreadtime} > 0 ) { + $arcdata->{lastreaddate} = strftime( "%Y-%m-%dT%H:%M:%SZ", gmtime( $arcdata->{lastreadtime} ) ); + } + + for ( values %{$arcdata} ) { $_ = xml_escape($_); } + + $dbh->disconnect(); + return $arcdata; +} + +# replaces LANraragi::Model::Opds::render_archive_page +sub render_archive_page { + + my ( $mojo, $id, $page ) = @_; + + my $logger = LANraragi::Utils::Logging::get_logger( "OPDS Page Serving", "lanraragi" ); + + my $dbh = get_postgresql_dbh(); + my $archive = get_archive_path( $dbh, $id ); + $dbh->disconnect(); + + # Parse archive to get its list of images + my @images = get_filelist($archive, $id); + + # If the page number is invalid, use the first page. + if ( $page > scalar @images ) { + $page = 1; + } + + # If the page number is valid, render the page. + my $path = $images[ $page - 1 ]; + + $logger->debug("Page /$id/$path was requested"); + + # Implement page serving logic directly (replaces Archive::serve_page) + # This avoids the Redis metadata access in Archive::serve_page -> get_page_data -> get_archive_path + + # Apply resizing transformation if set in Settings + if ( LANraragi::Model::Config->enable_resize ) { + + # Store resized files in a subfolder of the ID's temp folder, keyed by quality + my $threshold = LANraragi::Model::Config->get_threshold; + my $quality = LANraragi::Model::Config->get_readquality; + + my $cachekey = "resize_page/$id/$path/$threshold/$quality"; + my $content = fetch($cachekey); + if ( !defined($content) ) { + # Get page data from cache or extract from archive + my $page_cachekey = "page/$id/$path"; + my $page_content = fetch($page_cachekey); + if ( !defined($page_content) ) { + # Extract the file from the parent archive using the Postgres-retrieved path + $page_content = extract_single_file( $archive, $path ); + put( $page_cachekey, $page_content ); + } + + # Inline resize_image logic (replaces LANraragi::Model::Reader::resize_image) + # Is the file size higher than the threshold? + if ( ( ( length($page_content) / 1024 * 10 ) / 10 ) > $threshold ) { + my $resizer = get_resizer(); + my $resized = $resizer->resize_page( $page_content, $quality, "jpg" ); + if ( defined($resized) ) { + $content = $resized; + } else { + $content = $page_content; + } + } else { + $content = $page_content; + } + + put( $cachekey, $content ); + } + + # resize_image always converts the image to jpg + $mojo->render_file( + data => $content, + content_disposition => "inline", + format => "jpg" + ); + } else { + + # Get the file extension to report content-type properly + my ( $n, $p, $file_ext ) = fileparse( $path, qr/\.[^.]*/ ); + + # Get page data from cache or extract from archive + my $cachekey = "page/$id/$path"; + my $content = fetch($cachekey); + if ( !defined($content) ) { + # Extract the file from the parent archive using the Postgres-retrieved path + $content = extract_single_file( $archive, $path ); + put( $cachekey, $content ); + } + + $logger->debug( "Data size:" . length($content) ); + + # Serve extracted file directly + $mojo->render_file( + data => $content, + content_disposition => "inline", + format => substr( $file_ext, 1 ) + ); + } +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgPlugins.pm b/lib/LANraragi/Model/PsilabsDev/PgPlugins.pm new file mode 100644 index 000000000..ba2625521 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgPlugins.pm @@ -0,0 +1,262 @@ +package LANraragi::Model::PsilabsDev::PgPlugins; + +use v5.36; +use experimental 'try'; + +use strict; +use warnings; +use utf8; +use feature 'fc'; + +use Mojo::JSON qw(decode_json encode_json); +use Mojo::UserAgent; + +use LANraragi::Utils::String qw(trim); +use LANraragi::Utils::PsilabsDev::PgDatabase qw(set_tags set_title set_summary set_tags_with_dbh set_title_with_dbh set_summary_with_dbh); +use LANraragi::Utils::PsilabsDev::PgArchive qw(extract_thumbnail_with_dbh); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Tags qw(rewrite_tags split_tags_to_array); +use LANraragi::Utils::Plugins qw(get_plugin_parameters get_plugin); +use LANraragi::Utils::Path qw(create_path); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); + +use Exporter 'import'; +our @EXPORT_OK = qw(exec_metadata_plugin exec_enabled_plugins_on_file); + +# replaces LANraragi::Model::Plugins::exec_metadata_plugin +# Execute a specified plugin on a file, described through its archive ID. +sub exec_metadata_plugin ( $plugin, $id, %args ) { + + no warnings 'experimental::try'; + + my $logger = get_logger( "Plugin System", "lanraragi" ); + + if ( !$id ) { + return ( error => "Tried to call a metadata plugin without providing an id." ); + } + + # Get archive metadata from Postgres instead of Redis + my $dbh = get_postgresql_dbh(); + my $sth = $dbh->prepare(q{ + SELECT + filename, + title, + COALESCE( + (SELECT string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ) + FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid), + '' + ) as tags, + thumbhash + FROM lrr_archive a + WHERE arcid = ? + }); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + + unless ($row) { + $dbh->disconnect; + return ( error => "Archive with ID $id not found in database." ); + } + + my $name = $row->{filename} // ""; + my $title = $row->{title} // ""; + my $tags = $row->{tags} // ""; + my $thumbhash = $row->{thumbhash} // ""; + + # Get file path from Postgres (filename column stores the full path) + require LANraragi::Utils::PsilabsDev::PgPath; + LANraragi::Utils::PsilabsDev::PgPath->import('get_archive_path'); + my $file = get_archive_path($dbh, $id); + + # If the thumbnail hash is empty or undefined, we'll generate it here. + unless ( length $thumbhash ) { + $logger->info("Thumbnail hash invalid, regenerating."); + my $thumbdir = LANraragi::Model::Config->get_thumbdir; + $thumbhash = ""; + + try { + extract_thumbnail_with_dbh( $dbh, $thumbdir, $id, 1, 1, 1 ); + + # Re-fetch the thumbhash after generation + my $hash_sth = $dbh->prepare('SELECT thumbhash FROM lrr_archive WHERE arcid = ?'); + $hash_sth->execute($id); + my $hash_row = $hash_sth->fetchrow_hashref; + $thumbhash = $hash_row->{thumbhash} // ""; + $hash_sth->finish; + } catch ($e) { + $logger->warn("Error building thumbnail: $e"); + } + } + + $dbh->disconnect; + + my %returnhash; + try { + # Hand it off to the plugin here. + # If the plugin requires a login, execute that first to get a UserAgent + my %pluginfo = $plugin->plugin_info(); + my $ua = LANraragi::Model::Plugins::exec_login_plugin( $pluginfo{login_from} ); + + # Bundle all the potentially interesting info in a hash + my %infohash = ( + archive_id => $id, + archive_title => $title, + existing_tags => $tags, + thumbnail_hash => $thumbhash, + file_path => create_path( $file ), + user_agent => $ua, + oneshot_param => $args{'oneshot'} # for old style plugins compatibility + ); + + my %newmetadata; + + if ( LANraragi::Model::Plugins::has_old_style_params(%args) ) { + %newmetadata = $plugin->get_tags( \%infohash, @{ $args{customargs} } ); + } else { + %newmetadata = $plugin->get_tags( \%infohash, \%args ); + } + + # Error checking + if ( exists $newmetadata{error} ) { + return %newmetadata; + } + + my @tagarray = split_tags_to_array( $newmetadata{tags} ); + my $newtags = ""; + + # Process new metadata. + if ( LANraragi::Model::Config->enable_tagrules ) { + $logger->info("Applying tag rules..."); + my @rules = LANraragi::Utils::Database::get_computed_tagrules(); + @tagarray = rewrite_tags( \@tagarray, \@rules ); + } + + foreach my $tagtoadd (@tagarray) { + + # Only proceed if the tag isn't already in the existing tags + unless ( index( uc($tags), uc($tagtoadd) ) != -1 ) { + $newtags .= " $tagtoadd,"; + } + } + + # Strip last comma and return processed tags in a hash + chop($newtags); + %returnhash = ( new_tags => $newtags ); + + # Indicate a title change, if the plugin reports one + if ( exists $newmetadata{title} && LANraragi::Model::Config->can_replacetitles ) { + my $newtitle = $newmetadata{title}; + $newtitle = trim($newtitle); + $returnhash{title} = $newtitle; + } + + # Include updated summary data in response + if ( exists $newmetadata{summary} ) { + $returnhash{summary} = $newmetadata{summary}; + } + + } catch ($e) { + return ( error => $e ); + } + + return %returnhash; +} + +# replaces LANraragi::Model::Plugins::exec_enabled_plugins_on_file +# Sub used by Auto-Plugin. +sub exec_enabled_plugins_on_file ($id) { + + my $logger = get_logger( "Auto-Plugin", "lanraragi" ); + + $logger->info("Executing enabled metadata plugins on archive with id $id."); + + my $successes = 0; + my $failures = 0; + my $addedtags = 0; + my $newtitle = ""; + + my @plugins = LANraragi::Utils::Plugins::get_enabled_plugins("metadata"); + + # If the regex plugin is in the list, make sure it's ran first. + # TODO: Make plugin exec order configurable + foreach my $plugin (@plugins) { + if ( $plugin->{namespace} eq "regexplugin" ) { + my $regex_plugin = $plugin; + + # Remove element from array + @plugins = grep { $_->{namespace} ne "regexplugin" } @plugins; + unshift @plugins, $regex_plugin; + last; + } + } + + foreach my $pluginfo (@plugins) { + my $name = $pluginfo->{namespace}; + my %args = get_plugin_parameters($name); + my $plugin = get_plugin($name); + my %plugin_result; + + my %pluginfo = $plugin->plugin_info(); + + %plugin_result = exec_metadata_plugin( $plugin, $id, %args ); + + if ( exists $plugin_result{error} ) { + $failures++; + $logger->error( $plugin_result{error} ); + next; + } + + $successes++; + + # Create shared database handle for this plugin's atomic updates + my $dbh = get_postgresql_dbh(); + $dbh->begin_work; + + eval { + # All metadata updates from this plugin in one transaction + if ( $plugin_result{new_tags} ) { + set_tags_with_dbh( $dbh, $id, $plugin_result{new_tags}, 1 ); + } + + if ( exists $plugin_result{title} ) { + set_title_with_dbh( $dbh, $id, $plugin_result{title} ); + $newtitle = $plugin_result{title}; + $logger->debug("Changing title to $newtitle."); + } + + if ( exists $plugin_result{summary} ) { + set_summary_with_dbh( $dbh, $id, $plugin_result{summary} ); + $logger->debug("Summary has been changed."); + } + + $dbh->commit; + }; + + if ( my $error = $@ ) { + $logger->error("Error updating metadata for plugin $name: $error"); + eval { $dbh->rollback }; + $dbh->disconnect; + $failures++; + next; + } + + $dbh->disconnect; + + # Sum up added tags + my @added_tags = split( ',', $plugin_result{new_tags} ); + $addedtags += @added_tags; + } + + return ( $successes, $failures, $addedtags, $newtitle ); +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgReader.pm b/lib/LANraragi/Model/PsilabsDev/PgReader.pm new file mode 100644 index 000000000..530a93053 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgReader.pm @@ -0,0 +1,77 @@ +package LANraragi::Model::PsilabsDev::PgReader; + +use v5.36; +use experimental 'try'; + +use strict; +use warnings; +use utf8; + +use File::Basename; +use Mojo::JSON qw(encode_json); +use Data::Dumper; +use URI::Escape; + +use LANraragi::Utils::Generic qw(is_image); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Archive qw(get_filelist); +use LANraragi::Utils::Redis qw(redis_decode); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::PgPath; + +# replaces LANraragi::Model::Reader::build_reader_JSON +# build_reader_JSON(mojo, id, forceReload) +# Opens the archive specified by its ID, and returns a json containing the page names. +sub build_reader_JSON ( $self, $id, $force ) { + + # Get the path from Postgres. + # Filenames are stored as they are on the OS, so no decoding! + my $dbh = get_postgresql_dbh(); + my $archive = LANraragi::Utils::PsilabsDev::PgPath::get_archive_path( $dbh, $id ); + + # Parse archive to get its list of images + my @images = get_filelist($archive, $id); + + $self->LRR_LOGGER->debug( "Files found in archive (encoding might be incorrect): \n " . Dumper @images ); + + # Build a browser-compliant filepath array from @images + my @images_browser; + + foreach my $imgpath (@images) { + + # Since we're using uri_escape_utf8 for escaping, we need to make sure the path is valid UTF8. + # The good ole' redis_decode allows us to make sure of that. + $imgpath = redis_decode($imgpath); + + # We need to sanitize the image's path, in case the folder contains illegal characters, + # but uri_escape would also nuke the / needed for navigation. Let's solve this with a quick regex search&replace. + # First, we encode all HTML characters... + $imgpath = uri_escape_utf8($imgpath); + + # Then we bring the slashes back. + $imgpath =~ s!%2F!/!g; + + # Bundle this path into an API call which will be used by the browser + push @images_browser, $self->url_for("/api/archives/$id/page?path=$imgpath")->path_query; + } + + # Update pagecount in Postgres + eval { + my $sth = $dbh->prepare('UPDATE lrr_archive SET pagecount = ? WHERE arcid = ?'); + $sth->execute(scalar @images, $id); + $sth->finish; + }; + + if (my $error = $@) { + my $logger = get_logger("PgReader", "lanraragi"); + $logger->error("Error updating pagecount for archive $id: $error"); + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); + + return { pages => \@images_browser, }; +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgSearch.pm b/lib/LANraragi/Model/PsilabsDev/PgSearch.pm new file mode 100644 index 000000000..374f2c666 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgSearch.pm @@ -0,0 +1,767 @@ +package LANraragi::Model::PsilabsDev::PgSearch; + +use feature qw(signatures); +no warnings 'experimental::signatures'; + +use strict; +use warnings; +use utf8; + +use List::Util qw(min); +use Time::HiRes qw(time); +use LANraragi::Utils::Generic qw(intersect_arrays); +use LANraragi::Utils::String qw(trim); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Model::Config; +use LANraragi::Model::PsilabsDev::PgCategory; + +# replaces LANraragi::Model::Search::do_search +# Performs a search on the Postgres database. +# Returns ($total, $filtered, @ids) +sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks ) { + + my $logger = get_logger( "PgSearch Engine", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + my ( $total, $filtered, @ids ); + my $start_time = time(); + + eval { + # Get total count of archives + my $count_start = time(); + if ($grouptanks) { + # When grouping tanks: count standalone archives + tank count + # Standalone archives = archives not in any tank + my $archive_count_sth = $dbh->prepare( + 'SELECT COUNT(*) FROM lrr_archive WHERE NOT EXISTS (SELECT 1 FROM lrr_tank_to_archive_map WHERE arcid = lrr_archive.arcid)' + ); + $archive_count_sth->execute(); + my ($archive_count) = $archive_count_sth->fetchrow_array; + $archive_count_sth->finish; + + my $tank_count_sth = $dbh->prepare('SELECT COUNT(*) FROM lrr_tank'); + $tank_count_sth->execute(); + my ($tank_count) = $tank_count_sth->fetchrow_array; + $tank_count_sth->finish; + + $total = ($archive_count || 0) + ($tank_count || 0); + } else { + # When not grouping tanks: count all archives (excluding tanks themselves) + my $count_sth = $dbh->prepare('SELECT COUNT(*) FROM lrr_archive'); + $count_sth->execute(); + my ($count) = $count_sth->fetchrow_array; + $count_sth->finish; + $total = $count || 0; + } + my $count_time = (time() - $count_start) * 1000; + $logger->debug(sprintf("[PERF] Total count: %.2fms (grouptanks: %s)", $count_time, $grouptanks ? 'true' : 'false')); + + # Determine pagination parameters + my $keysperpage = LANraragi::Model::Config->get_pagesize; + my $use_pagination = ( $start != -1 ); + + # Perform the search with SQL-level pagination + my $search_start = time(); + ( $filtered, @ids ) = search_postgres_with_dbh( + $dbh, $category_id, $filter, $sortkey, $sortorder, + $newonly, $untaggedonly, $grouptanks, + $use_pagination ? $start : undef, + $use_pagination ? $keysperpage : undef + ); + my $search_time = (time() - $search_start) * 1000; + $logger->debug(sprintf("[PERF] search_postgres total: %.2fms", $search_time)); + }; + + if ( my $error = $@ ) { + $logger->error("Search error: $error"); + $dbh->disconnect(); + return ( -1, -1, () ); + } + + $dbh->disconnect(); + + my $total_time = (time() - $start_time) * 1000; + $logger->debug(sprintf("[PERF] do_search total: %.2fms", $total_time)); + + return ( $total, $filtered, @ids ); +} + +# Main search logic using Postgres +# Returns ($filtered_count, @ids) +sub search_postgres_with_dbh ( $dbh, $category_id, $filter, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks, $start, $keysperpage ) { + + my $logger = get_logger( "PgSearch Core", "lanraragi" ); + + # Compute search filters + my $token_start = time(); + my @tokens = compute_search_filter($filter); + my $token_time = (time() - $token_start) * 1000; + $logger->debug(sprintf("[PERF] Token computation: %.2fms (token_count: %d)", $token_time, scalar @tokens)); + + # Build the SQL query + my @where_clauses = (); + my @params = (); + my @lateral_params = (); # Separate array for LATERAL JOIN parameters + + # Tank grouping: When grouptanks=true, we want to return tank IDs and standalone archives. + # When grouptanks=false, we want to return individual archives excluding those in tanks. + if ($grouptanks) { + # When grouping tanks, exclude archives that are members of tanks + push @where_clauses, "NOT EXISTS (SELECT 1 FROM lrr_tank_to_archive_map WHERE arcid = a.arcid)"; + } + + # Category filter + if ( $category_id && $category_id ne "" ) { + my $cat_start = time(); + my %category = LANraragi::Model::PsilabsDev::PgCategory::get_category($category_id); + my $cat_time = (time() - $cat_start) * 1000; + $logger->debug(sprintf("[PERF] Category lookup: %.2fms", $cat_time)); + + if (%category) { + if ( $category{search} && $category{search} ne "" ) { + # Dynamic category - add its search predicate to tokens + my @cat_tokens = compute_search_filter( $category{search} ); + push @tokens, @cat_tokens; + } else { + # Static category - filter by category membership + push @where_clauses, "EXISTS (SELECT 1 FROM lrr_category_to_archive_map WHERE catid = ? AND arcid = a.arcid)"; + push @params, $category_id; + } + } + } + + # New filter + if ($newonly) { + push @where_clauses, "a.isnew = TRUE"; + } + + # Untagged filter - archives with no "meaningful" tags + # Excludes basic metadata namespaces that don't count as "tagged" + # Uses denormalized namespace on tag map — no join to lrr_tag needed. + if ($untaggedonly) { + push @where_clauses, "NOT EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + WHERE atm.arcid = a.arcid + AND atm.namespace NOT IN ('artist', 'parody', 'series', 'language', 'event', 'group', 'date_added', 'timestamp', 'source') + )"; + } + + # Process search tokens + foreach my $token (@tokens) { + my $tag = $token->{tag}; + my $isneg = $token->{isneg}; + my $isexact = $token->{isexact}; + + $logger->debug("Searching for $tag, isneg=$isneg, isexact=$isexact"); + + # Handle pagecount/read searches: pages:20, pages:>20, read:>0, etc. + if ( $tag =~ /^(read|pages):(>|<|>=|<=)?(\d+)$/ ) { + my $col = $1; + my $operator = $2 || "="; + my $value = $3; + + # Map column names + $col = $col eq "pages" ? "pagecount" : "progress"; + + my $clause = "a.$col $operator ?"; + if ($isneg) { + $clause = "NOT ($clause)"; + } + push @where_clauses, $clause; + push @params, $value; + next; + } + + # Tag-based search + # For exact matches, we need exact namespace:value match + # For non-exact, we use ILIKE for partial matching + + my ($namespace, $value); + if ( $tag =~ /^([^:]+):(.*)$/ ) { + $namespace = $1; + $value = $2; + } else { + $namespace = undef; + $value = $tag; + } + + # Check if this looks like an archive ID (40-char hex string) + # Archive IDs need direct matching since FTS tokenizes them poorly + if (!defined $namespace && $value =~ /^[a-f0-9]{40}$/i) { + my $tag_clause = "a.arcid = ?"; + if ($isneg) { + $tag_clause = "a.arcid != ?"; + } + push @where_clauses, $tag_clause; + push @params, lc($value); # arcids are lowercase in DB + next; + } + + # Convert wildcards: ? to _, * to % + if (defined $namespace) { + $namespace =~ s/\?/_/g; + $namespace =~ s/\*/%/g; + } + $value =~ s/\?/_/g; + $value =~ s/\*/%/g; + + my $tag_clause; + if ($isexact) { + # Exact match + if (defined $namespace) { + # Handle namespace-only search (e.g., "date_uploaded:") + if ($value eq "") { + # Search for ANY tag with this namespace + $tag_clause = "EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND t.namespace = ? + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, $namespace; + } else { + # Search for specific namespace:value + $tag_clause = "EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND t.namespace = ? + AND t.value = ? + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, $namespace, $value; + } + } else { + # No namespace - match tag value or archive title + # Check if value contains wildcards (already converted from ? to _ and * to %) + if ($value =~ /[%_]/) { + # Wildcard exact search - use ILIKE for case-insensitive pattern matching, include title + $tag_clause = "( + a.title ILIKE ? + OR EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND t.value ILIKE ? + ) + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, $value, $value; + } else { + # No wildcards - exact match on tag value or title (case-insensitive) + $tag_clause = "( + LOWER(a.title) = ? + OR EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND t.value = ? + ) + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, $value, $value; + } + } + } else { + # Partial match using ILIKE + if (defined $namespace) { + # Handle namespace-only search (e.g., "date_uploaded:") + if ($value eq "") { + # Search for ANY tag with this namespace + $tag_clause = "EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND t.namespace ILIKE ? + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, $namespace; + } else { + # Search for namespace (exact unless wildcards) and value (fuzzy) + $tag_clause = "EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND t.namespace ILIKE ? + AND t.value ILIKE ? + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, $namespace, "%$value%"; + } + } else { + # No namespace - use FTS for simple searches, ILIKE for wildcards + # Check if value contains wildcards (before they were converted to SQL LIKE patterns) + # We already converted ? to _ and * to %, so check for those + if ($value =~ /[%_]/) { + # Wildcard search - use ILIKE (fallback for pattern matching) + $tag_clause = "( + a.title ILIKE ? + OR a.arcid ILIKE ? + OR EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND (t.namespace ILIKE ? OR t.value ILIKE ?) + ) + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, "%$value%", "%$value%", "%$value%", "%$value%"; + } else { + # Simple word search - use ILIKE for substring matching + # This handles CJK, short tokens, and numeric values correctly + $tag_clause = "( + a.title ILIKE ? + OR EXISTS ( + SELECT 1 FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = a.arcid + AND (t.namespace ILIKE ? OR t.value ILIKE ?) + ) + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, "%$value%", "%$value%", "%$value%"; + } + } + } + } + + # Build WHERE clause + my $where_sql = ""; + if (@where_clauses) { + $where_sql = "WHERE " . join( " AND ", @where_clauses ); + } + + # Build JOIN for tag-based sorting + # Using DISTINCT ON to efficiently get one tag value per archive + # This is much faster than LATERAL JOIN which executes for every archive row + my $lateral_join_sql = ""; + my $use_lateral_sort = 0; + if ( $sortkey && $sortkey ne "title" && $sortkey ne "lastread" ) { + # Validate sortkey to prevent SQL injection (whitelist alphanumeric, underscore, hyphen) + if ( $sortkey !~ /^[a-zA-Z0-9_-]+$/ ) { + $logger->warn("Invalid sortkey: $sortkey. Falling back to title sort."); + } else { + # Single-table sort subquery using denormalized (namespace, value) on the tag map. + # No join to lrr_tag needed — the covering index (namespace, arcid) INCLUDE (value) + # services the entire subquery from a single index-only scan. + $lateral_join_sql = "LEFT JOIN ( + SELECT atm.arcid, MAX(atm.value) as sort_value + FROM lrr_archive_to_tag_map atm + WHERE atm.namespace = ? + GROUP BY atm.arcid + ) sort_tag ON sort_tag.arcid = a.arcid"; + push @lateral_params, $sortkey; # Add to lateral_params instead of params + $use_lateral_sort = 1; + } + } + + # Build ORDER BY clause + my $order_sql = ""; + if ( !$sortkey || $sortkey eq "title" ) { + $order_sql = "ORDER BY a.title" . ( $sortorder ? " DESC" : " ASC" ); + } elsif ( $sortkey eq "lastread" ) { + # Partition: archives with lastreadtime > 0 first, unread archives last + $order_sql = "ORDER BY CASE WHEN a.lastreadtime IS NULL OR a.lastreadtime = 0 THEN 1 ELSE 0 END, a.lastreadtime" + . ( $sortorder ? " DESC" : " ASC" ); + } else { + # Sort by tag namespace value (using LATERAL JOIN result) + if ($use_lateral_sort) { + # Partition: keyed archives (have sort namespace tag) first, unkeyed last + $order_sql = "ORDER BY CASE WHEN sort_tag.sort_value IS NULL THEN 1 ELSE 0 END, sort_tag.sort_value" + . ( $sortorder ? " DESC" : " ASC" ) . ", a.title ASC"; + } else { + # Fallback to title sort if sortkey was invalid + $order_sql = "ORDER BY a.title" . ( $sortorder ? " DESC" : " ASC" ); + } + } + + # Build LIMIT/OFFSET clause + my $limit_sql = ""; + my @limit_params = (); + if ( defined $start && defined $keysperpage && $keysperpage > 0 ) { + $limit_sql = "LIMIT ? OFFSET ?"; + push @limit_params, $keysperpage, $start; + } + + # Single query: SELECT with COUNT(*) OVER() to get filtered count and paginated results together. + # This avoids running the filter twice (once for COUNT, once for SELECT). + my $sql = "SELECT a.arcid, COUNT(*) OVER() as filtered_total FROM lrr_archive a $lateral_join_sql $where_sql $order_sql $limit_sql"; + + $logger->debug("SQL: $sql"); + $logger->debug("LATERAL params: " . join(", ", @lateral_params)); + $logger->debug("WHERE params: " . join(", ", @params)); + $logger->debug("LIMIT params: " . join(", ", @limit_params)); + + my $select_start = time(); + my $sth = $dbh->prepare($sql); + $sth->execute(@lateral_params, @params, @limit_params); + + my @ids; + my $archive_filtered_count = 0; + while ( my $row = $sth->fetchrow_hashref ) { + $archive_filtered_count = $row->{filtered_total} unless $archive_filtered_count; + push @ids, $row->{arcid}; + } + $sth->finish; + + # When OFFSET exceeds result count, no rows are returned and filtered_total is unknown. + # Fall back to a COUNT query only in this edge case (paginated query with 0 results). + if ( !$archive_filtered_count && !@ids && defined $start ) { + my $count_sql = "SELECT COUNT(*) as total FROM lrr_archive a $where_sql"; + my $count_sth = $dbh->prepare($count_sql); + $count_sth->execute(@params); + $archive_filtered_count = $count_sth->fetchrow_hashref->{total} || 0; + $count_sth->finish; + } + + my $select_time = (time() - $select_start) * 1000; + + # Determine sort type for logging + my $sort_desc = $sortkey || "title"; + if ($sortkey && $sortkey ne "title" && $sortkey ne "lastread" && $use_lateral_sort) { + $sort_desc = "tag:$sortkey"; + } + + $logger->debug(sprintf("[PERF] Results SELECT query: %.2fms (sort: %s, lateral_join: %s, limit: %s, offset: %s)", + $select_time, + $sort_desc, + $use_lateral_sort ? 'true' : 'false', + defined $keysperpage ? $keysperpage : 'none', + defined $start ? $start : 'none')); + + # EXPLAIN ANALYZE for diagnostic purposes (debug mode only) + if ($select_time > 50) { + eval { + my $explain_sql = "EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT) $sql"; + my $explain_sth = $dbh->prepare($explain_sql); + $explain_sth->execute(@lateral_params, @params, @limit_params); + my @plan_lines; + while ( my @row = $explain_sth->fetchrow_array ) { + push @plan_lines, $row[0]; + } + $explain_sth->finish; + $logger->info("[EXPLAIN] Query: $sql"); + $logger->info("[EXPLAIN] Params: " . join(", ", @lateral_params, @params, @limit_params)); + foreach my $line (@plan_lines) { + $logger->info("[EXPLAIN] $line"); + } + }; + if ($@) { + $logger->info("[EXPLAIN] Failed to run EXPLAIN ANALYZE: $@"); + } + } + + # When grouptanks=true, we also need to fetch tank IDs that match the search criteria + # and prepend them to the results (tanks typically come first) + my $tank_filtered_count = 0; + if ($grouptanks) { + my $tank_start = time(); + my ( $tank_count, @tank_ids ) = search_tanks_postgres_with_dbh($dbh, $category_id, $filter, $sortkey, $sortorder, $start, $keysperpage); + my $tank_time = (time() - $tank_start) * 1000; + $logger->debug(sprintf("[PERF] Tank search: %.2fms", $tank_time)); + $tank_filtered_count = $tank_count; + if (@tank_ids) { + $logger->debug( "Found " . scalar @tank_ids . " tank results (paginated)" ); + # Prepend tank IDs to archive IDs + unshift @ids, @tank_ids; + } + } + + my $total_filtered = $archive_filtered_count + $tank_filtered_count; + $logger->debug( "Found $total_filtered total filtered results, returning " . scalar @ids . " paginated results" ); + + return ( $total_filtered, @ids ); +} + +# Search for tanks matching the given criteria +# Returns ($filtered_count, @tank_ids) +sub search_tanks_postgres_with_dbh ( $dbh, $category_id, $filter, $sortkey, $sortorder, $start, $keysperpage ) { + + my $logger = get_logger( "PgSearch Tank", "lanraragi" ); + + # Compute search filters + my @tokens = compute_search_filter($filter); + + # Build the SQL query for tanks + my @where_clauses = (); + my @params = (); + + # Category filter for tanks + # Note: Tanks can be in categories through lrr_category_to_archive_map using their tankid + if ( $category_id && $category_id ne "" ) { + my %category = LANraragi::Model::PsilabsDev::PgCategory::get_category($category_id); + + if (%category) { + if ( $category{search} && $category{search} ne "" ) { + # Dynamic category - add its search predicate to tokens + my @cat_tokens = compute_search_filter( $category{search} ); + push @tokens, @cat_tokens; + } else { + # Static category - filter by category membership + # Tanks can be in categories directly + push @where_clauses, "EXISTS (SELECT 1 FROM lrr_category_to_archive_map WHERE catid = ? AND arcid = t.tankid)"; + push @params, $category_id; + } + } + } + + # Process search tokens for tanks + foreach my $token (@tokens) { + my $tag = $token->{tag}; + my $isneg = $token->{isneg}; + my $isexact = $token->{isexact}; + + $logger->debug("Tank search for $tag, isneg=$isneg, isexact=$isexact"); + + # Skip page/read count searches for tanks (they don't have pagecount/progress) + if ( $tag =~ /^(read|pages):/ ) { + next; + } + + # Tag-based search for tanks + my ($namespace, $value); + if ( $tag =~ /^([^:]+):(.*)$/ ) { + $namespace = $1; + $value = $2; + } else { + $namespace = undef; + $value = $tag; + } + + # Convert wildcards: ? to _, * to % + if (defined $namespace) { + $namespace =~ s/\?/_/g; + $namespace =~ s/\*/%/g; + } + $value =~ s/\?/_/g; + $value =~ s/\*/%/g; + + my $tag_clause; + if ($isexact) { + # Exact match - search in tank name or tags field + if (defined $namespace) { + # For tanks, tags are stored as a text field, so we search within it + # Handle namespace-only search (e.g., "date_uploaded:") + if ($value eq "") { + # Search for ANY tag with this namespace + $tag_clause = "t.tags LIKE ?"; + if ($isneg) { + $tag_clause = "NOT ($tag_clause)"; + } + push @where_clauses, $tag_clause; + push @params, "%$namespace:%"; + } else { + # Search for specific namespace:value + $tag_clause = "t.tags LIKE ?"; + if ($isneg) { + $tag_clause = "NOT ($tag_clause)"; + } + push @where_clauses, $tag_clause; + push @params, "%$namespace:$value%"; + } + } else { + # No namespace - match in name or tags (tank IDs searchable via partial search) + $tag_clause = "(t.name = ? OR t.tags LIKE ?)"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, $value, "%$value%"; + } + } else { + # Partial match using ILIKE + if (defined $namespace) { + # Handle namespace-only search (e.g., "date_uploaded:") + if ($value eq "") { + # Search for ANY tag with this namespace (partial namespace match) + $tag_clause = "t.tags ILIKE ?"; + if ($isneg) { + $tag_clause = "NOT ($tag_clause)"; + } + push @where_clauses, $tag_clause; + push @params, "%$namespace:%"; + } else { + # Search for namespace and value (both partial match) + $tag_clause = "t.tags ILIKE ?"; + if ($isneg) { + $tag_clause = "NOT ($tag_clause)"; + } + push @where_clauses, $tag_clause; + push @params, "%$namespace%$value%"; + } + } else { + # No namespace - search in tank ID, name, summary, or tags + $tag_clause = "( + t.tankid ILIKE ? + OR t.name ILIKE ? + OR t.summary ILIKE ? + OR t.tags ILIKE ? + )"; + if ($isneg) { + $tag_clause = "NOT $tag_clause"; + } + push @where_clauses, $tag_clause; + push @params, "%$value%", "%$value%", "%$value%", "%$value%"; + } + } + } + + # Build WHERE clause + my $where_sql = ""; + if (@where_clauses) { + $where_sql = "WHERE " . join( " AND ", @where_clauses ); + } + + # Build ORDER BY clause for tanks + my $order_sql = ""; + if ( !$sortkey || $sortkey eq "title" ) { + $order_sql = "ORDER BY t.name" . ( $sortorder ? " DESC" : " ASC" ); + } elsif ( $sortkey eq "lastread" ) { + # Tanks don't have lastreadtime, so we just order by name + $order_sql = "ORDER BY t.name" . ( $sortorder ? " DESC" : " ASC" ); + } else { + # Sort by tag namespace - search within tags field + # Since tanks store tags as text, we can't sort by specific namespace easily + # Just fall back to name sorting + $order_sql = "ORDER BY t.name" . ( $sortorder ? " DESC" : " ASC" ); + } + + # Build LIMIT/OFFSET clause + my $limit_sql = ""; + my @limit_params = (); + if ( defined $start && defined $keysperpage && $keysperpage > 0 ) { + $limit_sql = "LIMIT ? OFFSET ?"; + push @limit_params, $keysperpage, $start; + } + + # Single query with COUNT(*) OVER() to get filtered count and paginated results together. + my $sql = "SELECT t.tankid, COUNT(*) OVER() as filtered_total FROM lrr_tank t $where_sql $order_sql $limit_sql"; + + $logger->debug("Tank SQL: $sql"); + $logger->debug("Tank Params: " . join(", ", @params)); + $logger->debug("Tank LIMIT params: " . join(", ", @limit_params)); + + my $select_start = time(); + my $sth = $dbh->prepare($sql); + $sth->execute(@params, @limit_params); + + my @tank_ids; + my $tank_filtered_count = 0; + while ( my $row = $sth->fetchrow_hashref ) { + $tank_filtered_count = $row->{filtered_total} unless $tank_filtered_count; + push @tank_ids, $row->{tankid}; + } + $sth->finish; + my $select_time = (time() - $select_start) * 1000; + $logger->debug(sprintf("[PERF] Tank SELECT query: %.2fms (limit: %s, offset: %s)", + $select_time, + defined $keysperpage ? $keysperpage : 'none', + defined $start ? $start : 'none')); + + return ( $tank_filtered_count, @tank_ids ); +} + +# replaces LANraragi::Model::Search::compute_search_filter +# Transform the search engine syntax into a list of tokens. +sub compute_search_filter ($filter) { + + my $logger = get_logger( "PgSearch Core", "lanraragi" ); + my @tokens = (); + if ( !$filter ) { $filter = ""; } + + # Special characters: + # "" for exact search (or $, but is that one really useful now?) + # ?/_ for any character + # * % for multiple characters + # - to exclude the next tag + + my $b = reverse($filter); + while ( $b ne "" ) { + + my $char = chop $b; + my $isneg = 0; + + # Skip spaces + while ( $char eq " " && $b ne "" ) { + $char = chop $b; + } + + if ( $char eq "-" ) { + $isneg = 1; + $char = chop $b; + } + + # Get characters until the next comma, or the next " if the following char is " + my $delimiter = ','; + if ( $char eq '"' ) { + $delimiter = '"'; + $char = chop $b; + } + + my $tag = ""; + my $isexact = 0; + TAGBUILD: while (1) { + if ( $char eq $delimiter || $char eq "" ) { last TAGBUILD; } + $tag = $tag . $char; # Add characters in reverse order since we used reverse earlier on + $char = chop $b; + } + + # If last char is $ or delimiter was ", enable isexact + if ( $delimiter eq '"' ) { + $isexact = 1; + + # Quotes then $ is an accepted syntax, even though it does nothing + $char = chop $b; + unless ( $char eq "\$" ) { + $b = $b . $char; + } + } else { + $char = chop $tag; + if ( $char eq "\$" ) { + $isexact = 1; + } else { + $tag = $tag . $char; + } + } + + $logger->debug("Pre-trim tag: $tag"); + $tag = trim($tag); + + if ( $tag ne "" ) { # Blank tokens shouldn't be added as they'll slow down search + push @tokens, + { tag => lc($tag), + isneg => $isneg, + isexact => $isexact + }; + } + } + return @tokens; +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgSetup.pm b/lib/LANraragi/Model/PsilabsDev/PgSetup.pm new file mode 100644 index 000000000..8579d2912 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgSetup.pm @@ -0,0 +1,36 @@ +package LANraragi::Model::PsilabsDev::PgSetup; +use strict; +use warnings; +use utf8; + +use LANraragi::Model::Config qw(get_redis_config); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Model::PsilabsDev::PgCategory; + +use Exporter 'import'; +our @EXPORT_OK = qw(first_install_actions); + +# replaces LANraragi::Model::Setup::first_install_actions +# first_install_actions() +# Setup tasks for first-time installations. New installs are checked by confirming updated +# user settings. On first installation, create default 'Favorites' category link it to the bookmark +# button. Returns 1 if is first-time installation, else 0. +sub first_install_actions { + my $redis = LANraragi::Model::Config::get_redis_config(); + my $logger = get_logger( "Config", "lanraragi" ); + unless ( $redis->hexists('LRR_CONFIG', 'htmltitle') ) { + $logger->info("First-time installation detected!"); + $redis->hset('LRR_CONFIG', 'htmltitle', 'LANraragi'); + + $logger->debug("Creating first category..."); + my $default_category_id = LANraragi::Model::PsilabsDev::PgCategory::create_category("🔖 Favorites", "", 0, ""); + LANraragi::Model::PsilabsDev::PgCategory::update_bookmark_link($default_category_id); + $logger->info("Created default Favorites category."); + $redis->quit(); + return 1; + } + $redis->quit(); + return 0; +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgStats.pm b/lib/LANraragi/Model/PsilabsDev/PgStats.pm new file mode 100644 index 000000000..2b8703265 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgStats.pm @@ -0,0 +1,230 @@ +package LANraragi::Model::PsilabsDev::PgStats; + +use strict; +use warnings; +use utf8; + +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Model::Config; + +# replaces: LANraragi::Model::Stats::get_archive_count +# get_archive_count() +# Returns the total number of archives in the database. +# In Redis, this counts from LRR_TANKGROUPED which contains: +# - Tank IDs (for non-empty tanks) +# - Archive IDs that are NOT in any tank +# In Postgres, we replicate this by counting: +# - Non-empty tanks (tanks with at least one archive) +# - All archives that are not in any tank +sub get_archive_count { + my $logger = get_logger("PgStats", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + # Count non-empty tanks + archives not in any tank (matching Redis LRR_TANKGROUPED semantics) + my $sql = <<'SQL'; + SELECT + (SELECT COUNT(DISTINCT tankid) FROM lrr_tank_to_archive_map) + + (SELECT COUNT(*) FROM lrr_archive a + WHERE NOT EXISTS ( + SELECT 1 FROM lrr_tank_to_archive_map m + WHERE m.arcid = a.arcid + ) + ) as count +SQL + + my $sth = $dbh->prepare($sql); + $sth->execute(); + + my $row = $sth->fetchrow_hashref; + my $count = $row->{count} || 0; + + $sth->finish; + $dbh->disconnect(); + + $logger->debug("Archive count: $count"); + + return $count + 0; # Return as integer +} + +# replaces: LANraragi::Model::Stats::get_page_stat +# get_page_stat() +# Returns the cumulative total number of pages read across all archives. +# Reads from LRR_TOTALPAGESTAT in Redis Database 2 (configuration database). +# This counter is incremented each time a user reads a page. +# Configuration exception: This remains in Redis as it's stored in the config database. +sub get_page_stat { + my $logger = get_logger("PgStats", "lanraragi"); + + my $redis = LANraragi::Model::Config->get_redis_config; + my $stat = ($redis->get("LRR_TOTALPAGESTAT") || 0) + 0; + $redis->quit(); + + $logger->debug("Total pages read: $stat"); + + return $stat; +} + +# replaces: LANraragi::Model::Stats::compute_content_size +# compute_content_size() +# Computes the total size of all archives in the database. +# In Redis, this sums the arcsize field from all archive hashes. +# In Postgres, this sums the arcsize column from lrr_archive table. +# Returns the size in GB (as a decimal number). +sub compute_content_size { + my $logger = get_logger("PgStats", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + # Sum all archive sizes + my $sql = <<'SQL'; + SELECT COALESCE(SUM(arcsize), 0) as total_size + FROM lrr_archive +SQL + + my $sth = $dbh->prepare($sql); + $sth->execute(); + + my $row = $sth->fetchrow_hashref; + my $size = $row->{total_size} || 0; + + $sth->finish; + $dbh->disconnect(); + + # Convert to GB (matching Redis implementation) + my $size_gb = int( $size / 1073741824 * 100 ) / 100; + + $logger->debug("Total content size: $size_gb GB"); + + return $size_gb; +} + +# replaces: LANraragi::Model::Stats::is_url_recorded +# is_url_recorded($url) +# Checks if a URL has already been recorded in the database. +# In Redis, this checks the LRR_URLMAP hash. +# In Postgres, this queries for archives with a source: tag matching the URL. +# Returns the archive ID if found, 0 otherwise. +sub is_url_recorded { + my $url = shift; + + my $logger = get_logger("PgStats", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + $logger->debug("Checking if url $url is in the database."); + + # Trim last slash from url if it's present (matching Redis implementation) + use LANraragi::Utils::String qw(trim_url); + $url = trim_url($url); + + # Query for archives with a source: tag matching this URL + my $sql = <<'SQL'; + SELECT a.arcid + FROM lrr_archive a + INNER JOIN lrr_archive_to_tag_map atm ON a.arcid = atm.arcid + INNER JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE t.namespace = 'source' AND t.value = ? + LIMIT 1 +SQL + + my $sth = $dbh->prepare($sql); + $sth->execute($url); + + my $id = 0; + if (my $row = $sth->fetchrow_hashref) { + $id = $row->{arcid}; + $logger->debug("Found! id $id."); + } + + $sth->finish; + $dbh->disconnect(); + + return $id; +} + +# replaces: LANraragi::Model::Stats::build_tag_stats +# build_tag_stats($minscore) +# Builds tag statistics for display in the tag cloud. +# In Redis, this queries the LRR_STATS sorted set. +# In Postgres, this counts tag occurrences across all archives. +# Returns a reference to an array of tag objects with text, namespace, and weight fields. +sub build_tag_stats { + my ( $minscore, $excluded ) = @_; + my $logger = get_logger("PgStats", "lanraragi"); + + $logger->debug("Serving tag statistics with a minimum weight of $minscore"); + + my $dbh = get_postgresql_dbh(); + + # Count tag occurrences across all archives, filtering by minimum score + # Join lrr_tag with lrr_archive_to_tag_map to count how many times each tag appears + # Lowercase tags during grouping to match Redis behavior (e.g., "Artist:John" and "artist:john" counted together) + my $sql = <<'SQL'; + SELECT LOWER(t.namespace) as namespace, LOWER(t.value) as value, COUNT(*) as weight + FROM lrr_tag t + INNER JOIN lrr_archive_to_tag_map atm ON t.tagid = atm.tagid + GROUP BY LOWER(t.namespace), LOWER(t.value) + HAVING COUNT(*) >= ? + ORDER BY weight DESC +SQL + + my $sth = $dbh->prepare($sql); + $sth->execute($minscore); + + # Build the array of tag objects + my @tags; + while (my $row = $sth->fetchrow_hashref) { + my $namespace = $row->{namespace} || ""; + my $value = $row->{value} || ""; + my $weight = $row->{weight} || 0; + + # Skip empty values + next if $value eq ""; + + # Skip excluded namespaces if provided + next if @$excluded && grep { lc($_) eq $namespace } @$excluded; + + my $j = { + text => $value, + namespace => $namespace, + weight => $weight + 0 # Ensure it's treated as a number + }; + push(@tags, $j); + } + + $sth->finish; + $dbh->disconnect(); + + $logger->debug("Returning " . scalar(@tags) . " tags with minimum weight $minscore"); + + return \@tags; +} + +# replaces: LANraragi::Model::Stats::build_stat_hashes +# build_stat_hashes() +# In Redis, this rebuilds multiple search indexes: +# - LRR_URL_MAP (URL to archive ID mapping) +# - LRR_STATS (tag statistics/counts) +# - LRR_UNTAGGED (set of untagged archives) +# - LRR_NEW (set of new archives) +# - LRR_TITLES (lexicographically sorted titles) +# - LRR_TANKGROUPED (tank IDs and archives not in tanks) +# - INDEX_ (individual tag search indexes) +# +# In Postgres, these indexes are maintained automatically: +# - Tag statistics are computed on-the-fly via SQL queries +# - URL lookups use JOIN queries on lrr_tag table +# - Untagged/new filtering uses WHERE clauses +# +# This function is a no-op for Postgres but must exist for compatibility. +sub build_stat_hashes { + my $logger = get_logger("PgStats", "lanraragi"); + + $logger->info("build_stat_hashes called - no-op for Postgres (indexes maintained automatically)"); + + # No manual index building is required for Postgres. + # This function exists only for compatibility with the Minion task system. + + return; +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm b/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm new file mode 100644 index 000000000..f1db05307 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm @@ -0,0 +1,761 @@ +package LANraragi::Model::PsilabsDev::PgTankoubon; + +use strict; +use warnings; +use utf8; + +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Model::Config; + +# replaces LANraragi::Model::Tankoubon::get_tankoubon_list +# get_tankoubon_list(page) +# Returns a list of all the Tankoubon objects. +sub get_tankoubon_list { + my $page = shift // 0; + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + # Get all tankoubons + my $tank_sql = <<'SQL'; + SELECT tankid, name, summary, tags + FROM lrr_tank + ORDER BY tankid +SQL + + my $tank_sth = $dbh->prepare($tank_sql); + $tank_sth->execute(); + + # Prepare archive query ONCE outside the loop + my $arc_sql = <<'SQL'; + SELECT arcid + FROM lrr_tank_to_archive_map + WHERE tankid = ? + ORDER BY position +SQL + my $arc_sth = $dbh->prepare($arc_sql); + + my @result; + + while (my $tank_row = $tank_sth->fetchrow_hashref) { + my $tankid = $tank_row->{tankid}; + + # Execute the prepared statement for this tankoubon + $arc_sth->execute($tankid); + + my @archives; + while (my $arc_row = $arc_sth->fetchrow_hashref) { + push @archives, $arc_row->{arcid}; + } + + # Build tankoubon hash matching Redis implementation format + my %tankoubon = ( + id => $tankid, + name => $tank_row->{name}, + summary => $tank_row->{summary} // '', + tags => $tank_row->{tags} // '', + archives => \@archives + ); + + push @result, \%tankoubon; + } + $tank_sth->finish; + $arc_sth->finish; + + $dbh->disconnect(); + + my $total = scalar(@result); + + # Handle pagination + if ($page < 0) { + # Return all results + return ($total, $total, @result); + } else { + # Get page size from config + my $keysperpage = LANraragi::Model::Config->get_pagesize; + + my $start = $page * $keysperpage; + my $end = $start + $keysperpage - 1; + + if ($end > $#result) { + $end = $#result; + } + + if ($start > $#result) { + # No results for this page + return ($total, $total); + } + + my @page_results = @result[$start .. $end]; + return ($total, $total, @page_results); + } +} + +# replaces: LANraragi::Model::Tankoubon::create_tankoubon +# create_tankoubon(name, existing_id) +# Create a Tankoubon. +# If an existing Tankoubon ID is supplied, said Tankoubon will be updated with the given parameters. +# Returns the ID of the created/updated Tankoubon. +sub create_tankoubon { + my ( $name, $tank_id ) = @_; + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + # Set all fields of the tank object + unless ( length($tank_id) ) { + $tank_id = "TANK_" . time(); + + my $isnewkey = 0; + # Prepare statement once before loop + my $check_sth = $dbh->prepare('SELECT tankid FROM lrr_tank WHERE tankid = ?'); + until ($isnewkey) { + # Check if the tank ID exists, move timestamp further if it does + $check_sth->execute($tank_id); + my $exists = $check_sth->fetchrow_hashref; + + if ($exists) { + $tank_id = "TANK_" . ( time() + 1 ); + } else { + $isnewkey = 1; + } + } + $check_sth->finish; + } + + # Check if tank exists + my $exists_sth = $dbh->prepare('SELECT tankid FROM lrr_tank WHERE tankid = ?'); + $exists_sth->execute($tank_id); + my $existing = $exists_sth->fetchrow_hashref; + $exists_sth->finish; + + if ($existing) { + # Update existing tank + my $update_sth = $dbh->prepare( + 'UPDATE lrr_tank SET name = ? WHERE tankid = ?' + ); + $update_sth->execute($name, $tank_id); + $update_sth->finish; + $logger->debug("Updated tankoubon $tank_id"); + } else { + # Insert new tank with default values + my $insert_sth = $dbh->prepare( + 'INSERT INTO lrr_tank (tankid, name, summary, tags) VALUES (?, ?, ?, ?)' + ); + $insert_sth->execute($tank_id, $name, '', ''); + $insert_sth->finish; + $logger->debug("Created new tankoubon $tank_id"); + } + + $dbh->disconnect(); + return $tank_id; +} + +# replaces: LANraragi::Model::Tankoubon::get_tankoubon +# get_tankoubon(tankoubonid, fulldata, page) +# Returns the Tankoubon matching the given id. +# Returns undef if the id doesn't exist. +sub get_tankoubon { + my ( $tank_id, $fulldata, $page ) = @_; + + $fulldata //= 0; + $page //= 0; + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + if ( $tank_id eq "" ) { + $logger->debug("No Tankoubon ID provided."); + $dbh->disconnect(); + return (); + } + + # Check if tank exists and get metadata + my $tank_sql = 'SELECT tankid, name, summary, tags FROM lrr_tank WHERE tankid = ?'; + my $tank_sth = $dbh->prepare($tank_sql); + $tank_sth->execute($tank_id); + my $tank_row = $tank_sth->fetchrow_hashref; + $tank_sth->finish; + + unless ($tank_row) { + $logger->warn("$tank_id doesn't exist in the database!"); + $dbh->disconnect(); + return (); + } + + # Build base metadata hash + my %tank = ( + id => $tank_id, + name => $tank_row->{name}, + summary => $tank_row->{summary} // '', + tags => $tank_row->{tags} // '' + ); + + # Get total count of archives in this tankoubon + my $count_sql = 'SELECT COUNT(*) as total FROM lrr_tank_to_archive_map WHERE tankid = ?'; + my $count_sth = $dbh->prepare($count_sql); + $count_sth->execute($tank_id); + my $count_row = $count_sth->fetchrow_hashref; + my $total = $count_row->{total} || 0; + $count_sth->finish; + + # Fetch archives with pagination + my @archives; + my $arc_sql; + my $arc_sth; + + if ( $page < 0 ) { + # Get all archives + $arc_sql = 'SELECT arcid FROM lrr_tank_to_archive_map WHERE tankid = ? ORDER BY position'; + $arc_sth = $dbh->prepare($arc_sql); + $arc_sth->execute($tank_id); + } else { + # Get paginated archives + my $keysperpage = LANraragi::Model::Config->get_pagesize; + my $offset = $page * $keysperpage; + + $arc_sql = 'SELECT arcid FROM lrr_tank_to_archive_map WHERE tankid = ? ORDER BY position LIMIT ? OFFSET ?'; + $arc_sth = $dbh->prepare($arc_sql); + $arc_sth->execute($tank_id, $keysperpage, $offset); + } + + while (my $arc_row = $arc_sth->fetchrow_hashref) { + push @archives, $arc_row->{arcid}; + } + $arc_sth->finish; + + # If fulldata is requested, fetch complete archive information + if ($fulldata) { + my @full_data; + + # Batch query optimization: fetch all archive data in a single query + if (@archives) { + my $placeholders = join(',', ('?') x @archives); + my $batch_sql = qq{ + SELECT + a.arcid, + a.filename, + a.title, + a.summary, + a.isnew, + a.progress, + a.pagecount, + a.lastreadtime, + a.arcsize, + a.extension, + COALESCE( + string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ), + '' + ) as tags + FROM lrr_archive a + LEFT JOIN lrr_archive_to_tag_map atm ON a.arcid = atm.arcid + LEFT JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE a.arcid IN ($placeholders) + GROUP BY a.arcid, a.filename, a.title, a.summary, a.isnew, + a.progress, a.pagecount, a.lastreadtime, a.arcsize, a.extension + }; + + my $batch_sth = $dbh->prepare($batch_sql); + $batch_sth->execute(@archives); + + # Build hash map of results keyed by arcid + my %archive_data; + while (my $arc_row = $batch_sth->fetchrow_hashref) { + $archive_data{$arc_row->{arcid}} = $arc_row; + } + $batch_sth->finish; + + # Build response in original order, preserving tankoubon sequence + foreach my $arc_id (@archives) { + if (exists $archive_data{$arc_id}) { + my $arc_row = $archive_data{$arc_id}; + + # Check if the file exists on disk (matching Redis behavior) + my $filename = $arc_row->{filename}; + next unless (defined($filename) && -e $filename); + + # Handle whitespace-only title + my $title = $arc_row->{title}; + if ( !defined($title) || $title =~ /^\s*$/ ) { + $title = $arc_row->{filename}; + } + + my $arcdata = { + arcid => $arc_row->{arcid}, + title => $title, + filename => $arc_row->{filename}, + tags => $arc_row->{tags} // '', + summary => $arc_row->{summary} // '', + isnew => $arc_row->{isnew} ? 'true' : 'false', + extension => $arc_row->{extension} // '', + progress => $arc_row->{progress} ? int($arc_row->{progress}) : 0, + pagecount => $arc_row->{pagecount} ? int($arc_row->{pagecount}) : 0, + lastreadtime => $arc_row->{lastreadtime} ? int($arc_row->{lastreadtime}) : 0, + size => $arc_row->{arcsize} ? int($arc_row->{arcsize}) : 0 + }; + + push @full_data, $arcdata; + } + # If archive doesn't exist in results, gracefully skip it (missing archive) + } + } + + $tank{archives} = \@archives; + $tank{full_data} = \@full_data; + } else { + $tank{archives} = \@archives; + } + + $dbh->disconnect(); + + my $filtered = scalar(@archives); + return ( $total, $filtered, %tank ); +} + +# replaces: LANraragi::Model::Tankoubon::update_metadata +# update_metadata(tankoubonid, data) +# Updates the metadata in the Tankoubon. +# Returns 1 on success, 0 on failure alongside an error message. +sub update_metadata { + my ( $tank_id, $data ) = @_; + + if ( not defined $data->{"metadata"} ) { + return ( 1, "" ); + } + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + my $err = ""; + my $name = $data->{"metadata"}->{"name"} || undef; + my $summary = exists $data->{"metadata"}->{"summary"} ? $data->{"metadata"}->{"summary"} : undef; + my $tags = exists $data->{"metadata"}->{"tags"} ? $data->{"metadata"}->{"tags"} : undef; + + # Check if tank exists + my $tank_check_sth = $dbh->prepare('SELECT tankid FROM lrr_tank WHERE tankid = ?'); + $tank_check_sth->execute($tank_id); + my $tank_exists = $tank_check_sth->fetchrow_hashref; + $tank_check_sth->finish; + + unless ($tank_exists) { + $err = "$tank_id doesn't exist in the database!"; + $logger->warn($err); + $dbh->disconnect(); + return ( 0, $err ); + } + + # Build dynamic UPDATE statement + my @set_clauses; + my @values; + + if ( defined $name ) { + push @set_clauses, "name = ?"; + push @values, $name; + } + + if ( defined $summary ) { + push @set_clauses, "summary = ?"; + push @values, $summary; + } + + if ( defined $tags ) { + push @set_clauses, "tags = ?"; + push @values, $tags; + } + + if ( @set_clauses ) { + my $update_sql = "UPDATE lrr_tank SET " . join(", ", @set_clauses) . " WHERE tankid = ?"; + push @values, $tank_id; + + my $update_sth = $dbh->prepare($update_sql); + $update_sth->execute(@values); + $update_sth->finish; + $logger->debug("Updated metadata for tankoubon $tank_id"); + } + + $dbh->disconnect(); + return ( 1, $err ); +} + +# replaces: LANraragi::Model::Tankoubon::update_archive_list +# update_archive_list(tankoubonid, data) +# Updates the archives list in a Tankoubon. +# Returns 1 on success, 0 on failure alongside an error message. +sub update_archive_list { + my ( $tank_id, $data ) = @_; + + if ( not defined $data->{"archives"} ) { + return ( 1, "" ); + } + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + my $err = ""; + my @tank_archives = @{ $data->{"archives"} }; + + # Check if tank exists + my $tank_check_sth = $dbh->prepare('SELECT tankid FROM lrr_tank WHERE tankid = ?'); + $tank_check_sth->execute($tank_id); + my $tank_exists = $tank_check_sth->fetchrow_hashref; + $tank_check_sth->finish; + + unless ($tank_exists) { + $err = "$tank_id doesn't exist in the database!"; + $logger->warn($err); + $dbh->disconnect(); + return ( 0, $err ); + } + + # Verify all archives exist + my $arc_check_sth = $dbh->prepare('SELECT arcid FROM lrr_archive WHERE arcid = ?'); + foreach my $arc_id (@tank_archives) { + $arc_check_sth->execute($arc_id); + my $arc_exists = $arc_check_sth->fetchrow_hashref; + + unless ($arc_exists) { + $err = "$arc_id does not exist in the database."; + $logger->error($err); + $arc_check_sth->finish; + $dbh->disconnect(); + return ( 0, $err ); + } + } + $arc_check_sth->finish; + + # Begin transaction + $dbh->begin_work; + + eval { + # Delete all existing archive mappings for this tank + my $delete_sth = $dbh->prepare('DELETE FROM lrr_tank_to_archive_map WHERE tankid = ?'); + $delete_sth->execute($tank_id); + $delete_sth->finish; + + # Insert new archive mappings with positions + my $insert_sth = $dbh->prepare( + 'INSERT INTO lrr_tank_to_archive_map (tankid, arcid, position, update_date) VALUES (?, ?, ?, CURRENT_DATE)' + ); + + for ( my $i = 0; $i < scalar(@tank_archives); $i++ ) { + # Position is 1-indexed to match Redis implementation (scores start at 1) + $insert_sth->execute($tank_id, $tank_archives[$i], $i + 1); + } + $insert_sth->finish; + + $logger->debug("Updated archive list for tankoubon $tank_id with " . scalar(@tank_archives) . " archives"); + }; + + my $update_error = $@; + if ($update_error) { + $err = "Failed to update archive list for tankoubon $tank_id: $update_error"; + $logger->error($err); + eval { $dbh->rollback }; + $dbh->disconnect(); + return ( 0, $err ); + } + + $dbh->commit; + $dbh->disconnect(); + + # Postgres doesn't need cache invalidation + return ( 1, $err ); +} + +# replaces: LANraragi::Model::Tankoubon::update_tankoubon +# update_tankoubon(tankoubonid, data) +# Updates metadata and archive list. +# Returns 1 on success, 0 on failure alongside an error message. +sub update_tankoubon { + my ( $tank_id, $data ) = @_; + + my ( $result, $err ) = update_metadata( $tank_id, $data ); + if ($result) { + ( $result, $err ) = update_archive_list( $tank_id, $data ); + } + + return ( $result, $err ); +} + +# replaces: LANraragi::Model::Tankoubon::get_tankoubons_containing_archive +# get_tankoubons_containing_archive(arcid) +# Gets a list of Tankoubons where archive ID is contained. +# Returns an array of tank IDs. +sub get_tankoubons_containing_archive { + my ($arcid) = @_; + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + # Check if archive exists + my $arc_check_sth = $dbh->prepare('SELECT arcid FROM lrr_archive WHERE arcid = ?'); + $arc_check_sth->execute($arcid); + my $arc_exists = $arc_check_sth->fetchrow_hashref; + $arc_check_sth->finish; + + unless ($arc_exists) { + my $err = "$arcid does not exist in the database."; + $logger->error($err); + $dbh->disconnect(); + return (); + } + + # Query for all tankoubons containing this archive + my $tank_sql = <<'SQL'; + SELECT tankid + FROM lrr_tank_to_archive_map + WHERE arcid = ? + ORDER BY tankid +SQL + + my $tank_sth = $dbh->prepare($tank_sql); + $tank_sth->execute($arcid); + + my @tankoubons; + while (my $tank_row = $tank_sth->fetchrow_hashref) { + push @tankoubons, $tank_row->{tankid}; + } + $tank_sth->finish; + + $dbh->disconnect(); + return @tankoubons; +} + +# replaces LANraragi::Model::Tankoubon::delete_tankoubon +# delete_tankoubon(tankoubonid) +# Deletes the Tankoubon with the given ID. +# Returns 0 if the given ID isn't a Tankoubon ID, 1 otherwise +sub delete_tankoubon { + my ($tank_id) = @_; + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + if ( length($tank_id) != 15 ) { + # Probably not a Tankoubon ID + $logger->error("$tank_id is not a Tankoubon ID, doing nothing."); + $dbh->disconnect(); + return 0; + } + + # Check if tank exists + my $tank_check_sth = $dbh->prepare('SELECT tankid FROM lrr_tank WHERE tankid = ?'); + $tank_check_sth->execute($tank_id); + my $tank_exists = $tank_check_sth->fetchrow_hashref; + $tank_check_sth->finish; + + unless ($tank_exists) { + $logger->warn("$tank_id doesn't exist in the database!"); + $dbh->disconnect(); + return 1; + } + + # Begin transaction for multi-step delete + $dbh->begin_work; + + eval { + # First delete from lrr_tank_to_archive_map (no CASCADE in schema) + my $delete_map_sth = $dbh->prepare('DELETE FROM lrr_tank_to_archive_map WHERE tankid = ?'); + $delete_map_sth->execute($tank_id); + $delete_map_sth->finish; + + # Then delete from lrr_tank + my $delete_tank_sth = $dbh->prepare('DELETE FROM lrr_tank WHERE tankid = ?'); + $delete_tank_sth->execute($tank_id); + $delete_tank_sth->finish; + + $logger->debug("Deleted tankoubon $tank_id"); + }; + + if ( my $error = $@ ) { + $logger->error("Failed to delete tankoubon $tank_id: $error"); + eval { $dbh->rollback }; + $dbh->disconnect(); + return 0; + } + + $dbh->commit; + $dbh->disconnect(); + + return 1; +} + +# replaces LANraragi::Model::Tankoubon::add_to_tankoubon +# add_to_tankoubon(tankoubonid, arcid) +# Adds the given archive ID to the given Tankoubon. +# Returns 1 on success, 0 on failure alongside an error message. +sub add_to_tankoubon { + my ( $tank_id, $arc_id ) = @_; + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + my $err = ""; + + # Check if tank exists + my $tank_check_sth = $dbh->prepare('SELECT tankid FROM lrr_tank WHERE tankid = ?'); + $tank_check_sth->execute($tank_id); + my $tank_exists = $tank_check_sth->fetchrow_hashref; + $tank_check_sth->finish; + + unless ($tank_exists) { + $err = "$tank_id doesn't exist in the database!"; + $logger->warn($err); + $dbh->disconnect(); + return ( 0, $err ); + } + + # Check if archive exists + my $arc_check_sth = $dbh->prepare('SELECT arcid FROM lrr_archive WHERE arcid = ?'); + $arc_check_sth->execute($arc_id); + my $arc_exists = $arc_check_sth->fetchrow_hashref; + $arc_check_sth->finish; + + unless ($arc_exists) { + $err = "$arc_id does not exist in the database."; + $logger->error($err); + $dbh->disconnect(); + return ( 0, $err ); + } + + # Check if archive is already in the tankoubon + my $check_mapping_sth = $dbh->prepare( + 'SELECT tankid FROM lrr_tank_to_archive_map WHERE tankid = ? AND arcid = ?' + ); + $check_mapping_sth->execute($tank_id, $arc_id); + my $already_present = $check_mapping_sth->fetchrow_hashref; + $check_mapping_sth->finish; + + if ($already_present) { + $err = "$arc_id already present in category $tank_id, doing nothing."; + $logger->warn($err); + $dbh->disconnect(); + return ( 1, $err ); + } + + # Get the next position (current max position + 1) + # This matches Redis's zcard behavior which returns the number of elements + my $position_sth = $dbh->prepare( + 'SELECT COALESCE(MAX(position), 0) + 1 as next_position FROM lrr_tank_to_archive_map WHERE tankid = ?' + ); + $position_sth->execute($tank_id); + my $position_row = $position_sth->fetchrow_hashref; + my $next_position = $position_row->{next_position}; + $position_sth->finish; + + # Insert the archive into the tankoubon + eval { + my $insert_sth = $dbh->prepare( + 'INSERT INTO lrr_tank_to_archive_map (tankid, arcid, position, update_date) VALUES (?, ?, ?, CURRENT_DATE)' + ); + $insert_sth->execute($tank_id, $arc_id, $next_position); + $insert_sth->finish; + + $logger->debug("Added archive $arc_id to tankoubon $tank_id at position $next_position"); + }; + + if ( my $error = $@ ) { + $err = "Failed to add archive to tankoubon: $error"; + $logger->error($err); + $dbh->disconnect(); + return ( 0, $err ); + } + + $dbh->disconnect(); + return ( 1, $err ); +} + +# replaces LANraragi::Model::Tankoubon::remove_from_tankoubon +# remove_from_tankoubon(tankoubonid, arcid) +# Removes the given archive ID from the given Tankoubon. +# Returns 1 on success, 0 on failure alongside an error message. +sub remove_from_tankoubon { + my ( $tank_id, $arcid ) = @_; + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_postgresql_dbh(); + my $err = ""; + + # Check if tank exists + my $tank_check_sth = $dbh->prepare('SELECT tankid FROM lrr_tank WHERE tankid = ?'); + $tank_check_sth->execute($tank_id); + my $tank_exists = $tank_check_sth->fetchrow_hashref; + $tank_check_sth->finish; + + unless ($tank_exists) { + $err = "$tank_id doesn't exist in the database!"; + $logger->warn($err); + $dbh->disconnect(); + return ( 0, $err ); + } + + # Check if archive exists + my $arc_check_sth = $dbh->prepare('SELECT arcid FROM lrr_archive WHERE arcid = ?'); + $arc_check_sth->execute($arcid); + my $arc_exists = $arc_check_sth->fetchrow_hashref; + $arc_check_sth->finish; + + unless ($arc_exists) { + $err = "$arcid does not exist in the database."; + $logger->error($err); + $dbh->disconnect(); + return ( 0, $err ); + } + + # Check if archive is in the tankoubon and get its position + my $position_check_sth = $dbh->prepare( + 'SELECT position FROM lrr_tank_to_archive_map WHERE tankid = ? AND arcid = ?' + ); + $position_check_sth->execute($tank_id, $arcid); + my $position_row = $position_check_sth->fetchrow_hashref; + $position_check_sth->finish; + + unless ($position_row) { + $err = "$arcid not in tankoubon $tank_id, doing nothing."; + $logger->warn($err); + $dbh->disconnect(); + return ( 1, $err ); + } + + my $removed_position = $position_row->{position}; + + # Begin transaction to remove archive and update positions atomically + $dbh->begin_work; + + eval { + # Remove the archive from the tankoubon + my $delete_sth = $dbh->prepare( + 'DELETE FROM lrr_tank_to_archive_map WHERE tankid = ? AND arcid = ?' + ); + $delete_sth->execute($tank_id, $arcid); + $delete_sth->finish; + + # Update positions of all archives that came after the removed one + # This matches Redis behavior where scores are decremented by 1 for all elements after removal + my $update_sth = $dbh->prepare( + 'UPDATE lrr_tank_to_archive_map SET position = position - 1 WHERE tankid = ? AND position > ?' + ); + $update_sth->execute($tank_id, $removed_position); + $update_sth->finish; + + $logger->debug("Removed archive $arcid from tankoubon $tank_id and updated positions"); + }; + + if ( my $error = $@ ) { + $err = "Failed to remove archive from tankoubon: $error"; + $logger->error($err); + eval { $dbh->rollback }; + $dbh->disconnect(); + return ( 0, $err ); + } + + $dbh->commit; + $dbh->disconnect(); + + return ( 1, $err ); +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgUpload.pm b/lib/LANraragi/Model/PsilabsDev/PgUpload.pm new file mode 100644 index 000000000..7f5dbdc56 --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgUpload.pm @@ -0,0 +1,364 @@ +package LANraragi::Model::PsilabsDev::PgUpload; + +use v5.36; + +use strict; +use warnings; + +use Config; +use Encode; +use File::Basename; +use Exporter 'import'; + +our @EXPORT_OK = qw(add_timestamp_tag add_pagecount add_arcsize add_timestamp_tag_with_dbh add_pagecount_with_dbh add_arcsize_with_dbh add_archive_to_postgres); + +use LANraragi::Utils::Database qw(compute_id); +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Generic qw(is_archive); +use LANraragi::Utils::String qw(trim); +use LANraragi::Utils::Path qw(create_path rename_path move_path unlink_path date_modified); + +use LANraragi::Model::Config qw(get_userdir); +use LANraragi::Model::PsilabsDev::PgCategory; +use LANraragi::Model::PsilabsDev::PgPlugins; +use LANraragi::Utils::PsilabsDev::PgArchive qw(extract_thumbnail); +use LANraragi::Utils::PsilabsDev::PgDatabase qw(set_tags set_title set_summary invalidate_cache); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); + +use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); + +# replaces: LANraragi::Model::Upload::handle_incoming_file +# Process a file. +# First argument is the filepath, preferably in a temp directory, +# as we'll copy it to the content folder and delete the original at the end. +# +# The file will be added to a category, if its ID is specified. +# You can also specify tags to add to the metadata for the processed file before autoplugin is ran. (if it's enabled) +# +# Returns an HTTP status code, the ID and title of the file, and a status message. +sub handle_incoming_file ( $tempfile, $catid, $tags, $title, $summary ) { + + my ( $filename, $dirs, $suffix ) = fileparse( $tempfile, qr/\.[^.]*/ ); + $filename = $filename . $suffix; + my $logger = get_logger( "File Upload/Download", "lanraragi" ); + + # Check if file is an archive + unless ( is_archive($filename) ) { + $logger->debug("$filename is not an archive, halting upload process."); + return ( 415, "deadbeef", $filename, "Unsupported File Extension ($filename)" ); + } + + # Compute an ID here + my $id = compute_id($tempfile); + $logger->debug("ID of uploaded file $filename is $id"); + + # Future home of the file + my $userdir = LANraragi::Model::Config->get_userdir; + my $output_file = create_path( $userdir . '/' . $filename ); + + # Check if the ID is already in the database, and + # that the file it references still exists on the filesystem + my $dbh = get_postgresql_dbh(); + my $replace_dupe = LANraragi::Model::Config->get_replacedupe; + + my $check_sql = 'SELECT arcid, filename FROM lrr_archive WHERE arcid = ?'; + my $check_sth = $dbh->prepare($check_sql); + $check_sth->execute($id); + my $existing_row = $check_sth->fetchrow_hashref; + $check_sth->finish; + + my $isdupe = 0; + if ($existing_row) { + # Check if the file still exists on filesystem + my $existing_file = $existing_row->{filename}; + $isdupe = -e $existing_file; + } + + # Stop here if file is a dupe and replacement is turned off. + if ( ( -e $output_file || $isdupe ) && !$replace_dupe ) { + + # Trash temporary file + unlink_path $tempfile; + + # The file already exists + my $suffix = " Enable replace duplicated archive in config to replace old ones."; + my $msg = + $isdupe + ? "This file already exists in the Library." . $suffix + : "A file with the same name is present in the Library." . $suffix; + + $dbh->disconnect(); + return ( 409, $id, $filename, $msg ); + } + + # If we are replacing an existing one, just remove the old one first. + if ($replace_dupe) { + $logger->debug("Delete archive $id before replacing it."); + LANraragi::Utils::PsilabsDev::PgArchive::delete_archive($id); + } + + # Add the file to the database ourselves + # This allows autoplugin to be ran ASAP. + my $name = add_archive_to_postgres( $id, (IS_UNIX ? encode_utf8( $output_file ) : $output_file), $dbh ); + + # If additional tags were given to the sub, add them now. + if ($tags) { + set_tags( $id, $tags ); + } + + # Set title + if ($title) { + set_title( $id, $title ); + } + + # Set summary + if ($summary) { + set_summary( $id, $summary ); + } + + $dbh->disconnect(); + + # Move the file to the content folder. + # Move to a .upload first in case copy to the content folder takes a while... + move_path( $tempfile, $output_file . ".upload" ) + or return ( 500, $id, $name, "The file couldn't be moved to your content folder: $!" ); + + # Then rename inside the content folder itself to proc Shinobu's filemap update. + rename_path( $output_file . ".upload", $output_file ) + or return ( 500, $id, $name, "The file couldn't be renamed in your content folder: $!" ); + + # If the move didn't signal an error, but still doesn't exist, something is quite spooky indeed! + # Really funky permissions that prevents viewing folder contents? + unless ( -e $output_file ) { + return ( 500, $id, $name, "The file couldn't be moved to your content folder!" ); + } + + # Now that the file has been copied, we can add the timestamp tag and calculate pagecount. + # (The file being physically present is necessary in case last modified time is used) + # Reuse database connection and fetch filename once to avoid redundant queries + my $metadata_dbh = get_postgresql_dbh(); + add_timestamp_tag_with_dbh( $metadata_dbh, $id, $output_file ); + add_pagecount_with_dbh( $metadata_dbh, $id, $output_file ); + add_arcsize_with_dbh( $metadata_dbh, $id, $output_file ); + $metadata_dbh->disconnect(); + + # Generate thumbnail + my $thumbdir = LANraragi::Model::Config->get_thumbdir; + extract_thumbnail( $thumbdir, $id, 1, 1, 1 ); + + $logger->debug("Running autoplugin on newly uploaded file $id..."); + + my ( $succ, $fail, $addedtags, $newtitle ) = LANraragi::Model::PsilabsDev::PgPlugins::exec_enabled_plugins_on_file($id); + my $successmsg = "$succ Plugins used successfully, $fail Plugins failed, $addedtags tags added. "; + + if ( $newtitle ne "" ) { + $name = $newtitle; + } + + if ($catid) { + $logger->debug("Adding uploaded file to category $catid"); + + my ( $catsucc, $caterr ) = LANraragi::Model::PsilabsDev::PgCategory::add_to_category( $catid, $id ); + if ($catsucc) { + $successmsg .= "Added to Category!"; + } else { + $successmsg .= "Couldn't add to Category: $caterr"; + } + } + + # Postgres doesn't need cache invalidation + invalidate_cache(); + + return ( 200, $id, $name, $successmsg ); +} + +# Helper function: add_archive_to_postgres +# Creates a DB entry for a file path with the given ID. +# This is the Postgres equivalent of add_archive_to_redis. +sub add_archive_to_postgres ( $id, $file, $dbh ) { + + my $logger = get_logger( "Archive", "lanraragi" ); + my ( $name, $path, $suffix ) = fileparse( $file, qr/\.[^.]*/ ); + + # Initialize Postgres entry for the added file + $logger->debug("Pushing to Postgres on ID $id:"); + $logger->debug("File Name: $name"); + $logger->debug("Filesystem Path: $file"); + + my $arcsize = 0; + if ( defined($file) && -e $file ) { + $arcsize = -s $file; + } + + # Insert the archive into the database + my $sql = <<'SQL'; + INSERT INTO lrr_archive (arcid, filename, extension, isnew, lastreadtime, pagecount, progress, title, summary, thumbhash, arcsize) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (arcid) DO UPDATE SET + filename = EXCLUDED.filename, + extension = EXCLUDED.extension, + isnew = EXCLUDED.isnew, + lastreadtime = EXCLUDED.lastreadtime, + pagecount = EXCLUDED.pagecount, + progress = EXCLUDED.progress, + title = EXCLUDED.title, + arcsize = EXCLUDED.arcsize +SQL + + my $sth = $dbh->prepare($sql); + $sth->execute( + $id, + $file, + $suffix, + 1, # isnew = true + 0, # lastreadtime = 0 + 0, # pagecount = 0 + 0, # progress = 0 + $name, # title = filename without extension + '', # summary = empty + '', # thumbhash = empty + $arcsize # arcsize + ); + $sth->finish; + + return $name; +} + +# Helper function: add_timestamp_tag +# Adds a timestamp tag to the given ID. +sub add_timestamp_tag ( $id ) { + my $dbh = get_postgresql_dbh(); + my $result = add_timestamp_tag_with_dbh( $dbh, $id ); + $dbh->disconnect(); + return $result; +} + +# Helper function: add_timestamp_tag_with_dbh +# Adds a timestamp tag to the given ID, using provided database handle. +# Optional third parameter: filepath (avoids redundant database query if provided) +sub add_timestamp_tag_with_dbh ( $dbh, $id, $filepath = undef ) { + + my $logger = get_logger( "Archive", "lanraragi" ); + + # Initialize tags to the current date if the matching pref is enabled + if ( LANraragi::Model::Config->enable_dateadded eq "1" ) { + + $logger->debug("Adding timestamp tag..."); + my $date; + + if ( LANraragi::Model::Config->use_lastmodified eq "1" ) { + $logger->debug("Using file date"); + + # Get the file path from database only if not provided + unless ($filepath) { + my $sql = 'SELECT filename FROM lrr_archive WHERE arcid = ?'; + my $sth = $dbh->prepare($sql); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $filepath = $row ? $row->{filename} : ''; + $sth->finish; + } + + if ($filepath && -e $filepath) { + $date = date_modified($filepath); + } else { + $date = time(); + } + } else { + $logger->debug("Using current date"); + $date = time(); + } + + # Set date_added tag + set_tags( $id, "date_added:$date", 1 ); + } +} + +# Helper function: add_pagecount +# Adds pagecount to the archive metadata. +sub add_pagecount ( $id ) { + my $dbh = get_postgresql_dbh(); + my $result = add_pagecount_with_dbh( $dbh, $id ); + $dbh->disconnect(); + return $result; +} + +# Helper function: add_pagecount_with_dbh +# Adds pagecount to the archive metadata, using provided database handle. +# Optional third parameter: filepath (avoids redundant database query if provided) +sub add_pagecount_with_dbh ( $dbh, $id, $filepath = undef ) { + + my $logger = get_logger( "Archive", "lanraragi" ); + + # Get the file path from database only if not provided + unless ($filepath) { + my $sql = 'SELECT filename FROM lrr_archive WHERE arcid = ?'; + my $sth = $dbh->prepare($sql); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $filepath = $row ? $row->{filename} : ''; + $sth->finish; + } + + if (!$filepath || !-e $filepath) { + $logger->warn("Cannot calculate pagecount for $id - file not found"); + return; + } + + # Calculate pagecount using Archive utility + use LANraragi::Utils::Archive qw(get_filelist); + my @files = get_filelist($filepath, $id); + my $pagecount = scalar @files; + + # Update pagecount in database + my $update_sql = 'UPDATE lrr_archive SET pagecount = ? WHERE arcid = ?'; + my $update_sth = $dbh->prepare($update_sql); + $update_sth->execute($pagecount, $id); + $update_sth->finish; + + $logger->debug("Set pagecount for $id to $pagecount"); +} + +# Helper function: add_arcsize +# Adds archive size to the archive metadata. +sub add_arcsize ( $id ) { + my $dbh = get_postgresql_dbh(); + my $result = add_arcsize_with_dbh( $dbh, $id ); + $dbh->disconnect(); + return $result; +} + +# Helper function: add_arcsize_with_dbh +# Adds archive size to the archive metadata, using provided database handle. +# Optional third parameter: filepath (avoids redundant database query if provided) +sub add_arcsize_with_dbh ( $dbh, $id, $filepath = undef ) { + + my $logger = get_logger( "Archive", "lanraragi" ); + + # Get the file path from database only if not provided + unless ($filepath) { + my $sql = 'SELECT filename FROM lrr_archive WHERE arcid = ?'; + my $sth = $dbh->prepare($sql); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $filepath = $row ? $row->{filename} : ''; + $sth->finish; + } + + if (!$filepath || !-e $filepath) { + $logger->warn("Cannot calculate arcsize for $id - file not found"); + return; + } + + my $arcsize = -s $filepath; + + # Update arcsize in the database + my $update_sql = 'UPDATE lrr_archive SET arcsize = ? WHERE arcid = ?'; + my $update_sth = $dbh->prepare($update_sql); + $update_sth->execute($arcsize, $id); + $update_sth->finish; + + $logger->debug("Set arcsize for $id to $arcsize bytes"); +} + +1; diff --git a/lib/LANraragi/Utils/Minion.pm b/lib/LANraragi/Utils/Minion.pm index 666a8077f..cd09809b4 100644 --- a/lib/LANraragi/Utils/Minion.pm +++ b/lib/LANraragi/Utils/Minion.pm @@ -14,16 +14,19 @@ use MCE::Shared; use Config; use LANraragi::Utils::Logging qw(get_logger); -use LANraragi::Utils::Redis qw(redis_decode); -use LANraragi::Utils::Archive qw(extract_thumbnail); -use LANraragi::Utils::Plugins qw(get_downloader_for_url get_plugin get_plugin_parameters use_plugin); +use LANraragi::Utils::Plugins qw(get_downloader_for_url get_plugin get_plugin_parameters); +use LANraragi::Utils::PsilabsDev::PgPlugins qw(use_plugin); use LANraragi::Utils::String qw(trim_url); use LANraragi::Utils::TempFolder qw(get_temp); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::PgArchive qw(extract_thumbnail extract_thumbnail_with_dbh); use LANraragi::Model::Upload; use LANraragi::Model::Config; use LANraragi::Model::Stats; use LANraragi::Model::Backup; +use LANraragi::Model::PsilabsDev::PgStats; +use LANraragi::Model::PsilabsDev::PgUpload; use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); @@ -43,7 +46,7 @@ sub add_tasks { my $thumbname = ""; # Take a shortcut here - Minion jobs can keep the old basic behavior of page 0 = cover. - eval { $thumbname = extract_thumbnail( $thumbdir, $id, $page, $page eq 0, $use_hq ); }; + eval { $thumbname = LANraragi::Utils::PsilabsDev::PgArchive::extract_thumbnail( $thumbdir, $id, $page, $page eq 0, $use_hq ); }; if ($@) { my $msg = "Error building thumbnail: $@"; $logger->error($msg); @@ -109,9 +112,34 @@ sub add_tasks { my $logger = get_logger( "Minion", "minion" ); $logger->debug("Generating page thumbnails for archive $id..."); - # Get the number of pages in the archive - my $redis = LANraragi::Model::Config->get_redis; - my $pages = $redis->hget( $id, "pagecount" ); + # Get the number of pages in the archive from Postgres + my $dbh = get_postgresql_dbh(); + my $pages; + + eval { + my $sql = q{SELECT pagecount FROM lrr_archive WHERE arcid = ?}; + my $sth = $dbh->prepare($sql); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $pages = $row->{pagecount} if $row; + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error retrieving pagecount for archive $id: $error"); + $dbh->disconnect(); + $job->fail({ error => "Failed to retrieve pagecount: $error" }); + return; + } + + unless ($pages) { + $logger->error("Archive $id has no pagecount in database"); + $dbh->disconnect(); + $job->fail({ error => "Archive has no pagecount" }); + return; + } + + $dbh->disconnect(); my $use_hq = LANraragi::Model::Config->get_hqthumbpages; my $thumbdir = LANraragi::Model::Config->get_thumbdir; @@ -137,7 +165,7 @@ sub add_tasks { my $thumbname = "$thumbdir/$subfolder/$id/$i.$format"; unless ( $force == 0 && -e $thumbname ) { $logger->debug("Generating thumbnail for page $i... ($thumbname)"); - eval { $thumbname = extract_thumbnail( $thumbdir, $id, $i, 0, $use_hq ); }; + eval { $thumbname = LANraragi::Utils::PsilabsDev::PgArchive::extract_thumbnail( $thumbdir, $id, $i, 0, $use_hq ); }; if ($@) { $logger->warn("Error while generating thumbnail: $@"); $errors->push($@); @@ -164,9 +192,6 @@ sub add_tasks { } }; - $redis->hdel( $id, "thumbjob" ); - $redis->quit; - my @err = $errors->values; $job->finish( { errors => \@err } ); @@ -183,9 +208,14 @@ sub add_tasks { my ( $thumbdir, $force ) = @args; my $logger = get_logger( "Minion", "minion" ); - my $redis = LANraragi::Model::Config->get_redis; - my @keys = $redis->keys('????????????????????????????????????????'); - $redis->quit(); + my $dbh = get_postgresql_dbh(); + my $sth = $dbh->prepare('SELECT arcid FROM lrr_archive'); + $sth->execute(); + my @keys; + while (my $row = $sth->fetchrow_hashref) { + push @keys, $row->{arcid}; + } + $sth->finish; $logger->info("Starting thumbnail regen job (force = $force)"); my $errors = MCE::Shared->array; @@ -194,6 +224,9 @@ sub add_tasks { my $sub = sub { my (@keys) = @_; + # Each thread/process needs its own database connection + my $dbh_worker = get_postgresql_dbh(); + foreach my $id (@keys) { my $use_jxl = LANraragi::Model::Config->get_jxlthumbpages; @@ -204,7 +237,7 @@ sub add_tasks { unless ( $force == 0 && -e $thumbname ) { eval { $logger->debug("Regenerating for $id..."); - extract_thumbnail( $thumbdir, $id, 0, 1, 1 ); + extract_thumbnail_with_dbh( $dbh_worker, $thumbdir, $id, 0, 1, 1 ); }; if ($@) { @@ -213,6 +246,8 @@ sub add_tasks { } } } + + $dbh_worker->disconnect; }; eval { @@ -229,6 +264,8 @@ sub add_tasks { } }; + $dbh->disconnect; + my @err = $errors->values; # Crashes on Windows so don't run it there @@ -281,18 +318,33 @@ sub add_tasks { my ($threshold) = @args; my $logger = get_logger( "Minion", "minion" ); - my $redis = LANraragi::Model::Config->get_redis; - my @keys = $redis->keys('????????????????????????????????????????'); $logger->info("Starting find duplicate job (threshold = $threshold)"); - # Gather thumbhashes + # Gather thumbhashes from Postgres instead of Redis + my $dbh = get_postgresql_dbh(); my %thumbhashes; - foreach my $id (@keys) { - my $thumbhash = $redis->hget( $id, "thumbhash" ); - $thumbhashes{$id} = $thumbhash if $thumbhash; + + eval { + my $sql = 'SELECT arcid, thumbhash FROM lrr_archive WHERE thumbhash IS NOT NULL'; + my $sth = $dbh->prepare($sql); + $sth->execute(); + + while (my $row = $sth->fetchrow_hashref) { + $thumbhashes{$row->{arcid}} = $row->{thumbhash}; + } + + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error fetching thumbhashes from Postgres: $error"); + $dbh->disconnect(); + $job->fail( { errors => [$error] } ); + return; } - $redis->quit(); + + $dbh->disconnect(); # Prepare to track visited nodes my $visited = MCE::Shared->hash; @@ -301,8 +353,6 @@ sub add_tasks { my $sub = sub { my (@keys) = @_; - my $redis = LANraragi::Model::Config->get_redis_config; - foreach my $id (@keys) { # Skip if this ID has already been processed in another thread @@ -338,18 +388,24 @@ sub add_tasks { } } - # Add the discovered group to redis - # to avoid redudnant groups in different orders - sort and composite key + # Add the discovered group to Redis (config database) + # to avoid redundant groups in different orders - sort and composite key if ( @group && scalar @group >= 2 ) { @group = sort @group; my $composite_key = join '', map { substr( $_, 0, 10 ) } @group; my $group_json = encode_json( \@group ); $logger->debug("duplicate group '$composite_key': $group_json"); - $redis->hset( "LRR_DUPLICATE_GROUPS", "dupgp_$composite_key", $group_json ); + + eval { + my $redis = LANraragi::Model::Config->get_redis_config; + $redis->hset( "LRR_DUPLICATE_GROUPS", "dupgp_$composite_key", $group_json ); + $redis->quit(); + }; + if (my $error = $@) { + $logger->error("Error saving duplicate group: $error"); + } } } - - $redis->quit(); }; eval { @@ -376,7 +432,7 @@ sub add_tasks { $minion->add_task( build_stat_hashes => sub { my ( $job, @args ) = @_; - LANraragi::Model::Stats->build_stat_hashes; + LANraragi::Model::PsilabsDev::PgStats::build_stat_hashes(); $job->finish; } ); @@ -396,13 +452,13 @@ sub add_tasks { # Since we already have a file, this goes straight to handle_incoming_file. my ( $status_code, $id, $title, $message ) = - LANraragi::Model::Upload::handle_incoming_file( $file, $catid, "", "", "" ); + LANraragi::Model::PsilabsDev::PgUpload::handle_incoming_file( $file, $catid, "", "", "" ); my $status = $status_code == 200 ? 1 : 0; $job->finish( { success => $status, id => $id, category => $catid, - title => redis_decode($title), # Fix display issues in the response + title => $title, message => $message } ); @@ -423,7 +479,7 @@ sub add_tasks { $og_url = trim_url($og_url); # If the URL is already recorded, abort the download - my $recorded_id = LANraragi::Model::Stats::is_url_recorded($og_url); + my $recorded_id = LANraragi::Model::PsilabsDev::PgStats::is_url_recorded($og_url); if ($recorded_id) { $job->finish( { success => 0, @@ -470,7 +526,7 @@ sub add_tasks { # Hand off the result to handle_incoming_file my ( $status_code, $id, $title, $message ) = - LANraragi::Model::Upload::handle_incoming_file( $tempfile, $catid, $tag, "", "" ); + LANraragi::Model::PsilabsDev::PgUpload::handle_incoming_file( $tempfile, $catid, $tag, "", "" ); my $status = $status_code == 200 ? 1 : 0; $job->finish( @@ -504,7 +560,7 @@ sub add_tasks { # Hand off the result to handle_incoming_file my ( $status_code, $id, $title, $message ) = - LANraragi::Model::Upload::handle_incoming_file( $tempfile, $catid, $tag, "", "" ); + LANraragi::Model::PsilabsDev::PgUpload::handle_incoming_file( $tempfile, $catid, $tag, "", "" ); my $status = $status_code == 200 ? 1 : 0; eval { diff --git a/lib/LANraragi/Utils/PsilabsDev/PgArchive.pm b/lib/LANraragi/Utils/PsilabsDev/PgArchive.pm new file mode 100644 index 000000000..8bafb2327 --- /dev/null +++ b/lib/LANraragi/Utils/PsilabsDev/PgArchive.pm @@ -0,0 +1,175 @@ +package LANraragi::Utils::PsilabsDev::PgArchive; + +use v5.36; +use experimental 'try'; + +use strict; +use warnings; +use utf8; + +use File::Path qw(make_path remove_tree); + +use Exporter 'import'; +our @EXPORT_OK = qw(extract_thumbnail extract_thumbnail_with_dbh delete_archive); + +use LANraragi::Model::Config; +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Generic qw(shasum_str); +use LANraragi::Utils::Archive qw(get_filelist extract_single_file generate_thumbnail); +use LANraragi::Utils::Path qw(unlink_path); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); + +# replaces LANraragi::Model::Archive::delete_archive +# Deletes the archive with the given id from Postgres, and the matching archive file/thumbnail. +sub delete_archive ($id) { + + my $logger = get_logger( "Archive", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + my $filename; # Declare outside transaction + + $dbh->begin_work; + + eval { + # Get the filename INSIDE transaction for consistency + $filename = get_archive_path( $dbh, $id ); + + # Delete from tank-to-archive mappings + my $tank_sql = 'DELETE FROM lrr_tank_to_archive_map WHERE arcid = ?'; + my $tank_sth = $dbh->prepare($tank_sql); + $tank_sth->execute($id); + $tank_sth->finish; + + # Delete from category-to-archive mappings + my $cat_sql = 'DELETE FROM lrr_category_to_archive_map WHERE arcid = ?'; + my $cat_sth = $dbh->prepare($cat_sql); + $cat_sth->execute($id); + $cat_sth->finish; + + # Delete from archive-to-tag mappings + my $tag_sql = 'DELETE FROM lrr_archive_to_tag_map WHERE arcid = ?'; + my $tag_sth = $dbh->prepare($tag_sql); + $tag_sth->execute($id); + $tag_sth->finish; + + # Delete the archive entry itself + my $arc_sql = 'DELETE FROM lrr_archive WHERE arcid = ?'; + my $arc_sth = $dbh->prepare($arc_sql); + $arc_sth->execute($id); + $arc_sth->finish; + + $dbh->commit; + $logger->debug("Successfully deleted archive $id from database"); + }; + + if ( my $error = $@ ) { + $logger->error("Error deleting archive $id: $error"); + eval { $dbh->rollback }; + $dbh->disconnect(); + return "0"; + } + + $dbh->disconnect(); + + # Delete the file from filesystem if it exists + if ( $filename && -e $filename ) { + my $status = unlink_path( $filename ); + + my $thumbdir = LANraragi::Model::Config->get_thumbdir; + my $subfolder = substr( $id, 0, 2 ); + + my $jpg_thumbname = "$thumbdir/$subfolder/$id.jpg"; + unlink $jpg_thumbname; + + my $jxl_thumbname = "$thumbdir/$subfolder/$id.jxl"; + unlink $jxl_thumbname; + + # Delete the thumbpages folder + remove_tree("$thumbdir/$subfolder/$id/"); + + return $status ? $filename : "0"; + } + + return "0"; +} + +# replaces LANraragi::Utils::Archive::extract_thumbnail +# Wrapper function with original signature that manages its own database connection +sub extract_thumbnail ( $thumbdir, $id, $page, $set_cover, $use_hq ) { + my $dbh = get_postgresql_dbh(); + my $result; + eval { + $result = extract_thumbnail_with_dbh( $dbh, $thumbdir, $id, $page, $set_cover, $use_hq ); + }; + my $error = $@; + $dbh->disconnect(); + die $error if $error; + return $result; +} + +# For transactional contexts where endpoint manages $dbh +sub extract_thumbnail_with_dbh ( $dbh, $thumbdir, $id, $page, $set_cover, $use_hq ) { + + my $logger = get_logger( "Archive", "lanraragi" ); + + # JPG is used for thumbnails by default + my $use_jxl = LANraragi::Model::Config->get_jxlthumbpages; + my $format = $use_jxl ? 'jxl' : 'jpg'; + + # Another subfolder with the first two characters of the id is used for FS optimization. + my $subfolder = substr( $id, 0, 2 ); + make_path("$thumbdir/$subfolder"); + + my $file = get_archive_path( $dbh, $id ); + + # Get first image from archive using filelist + my @filelist = get_filelist($file, $id); + my $requested_image = $filelist[ $page > 0 ? $page - 1 : 0 ]; + + die "Requested image not found: $id page $page" unless $requested_image; + $logger->debug("Extracting thumbnail for $id page $page from $requested_image"); + + # Extract requested image to temp dir if it doesn't already exist + my $arcimg = extract_single_file( $file, $requested_image ); + + my $thumbname; + unless ($set_cover) { + + # Non-cover thumbnails land in a dedicated folder. + $thumbname = "$thumbdir/$subfolder/$id/$page.$format"; + make_path("$thumbdir/$subfolder/$id"); + } else { + + $thumbname = "$thumbdir/$subfolder/$id.$format"; + + # For cover thumbnails, grab the SHA-1 hash for tag research. + # That way, no need to repeat a costly extraction later. + my $shasum = shasum_str( $arcimg, 1 ); + $logger->debug("Setting thumbnail hash: $shasum"); + + # Update thumbhash + eval { + my $sth = $dbh->prepare('UPDATE lrr_archive SET thumbhash = ? WHERE arcid = ?'); + $sth->execute($shasum, $id); + $sth->finish; + }; + if ( my $error = $@ ) { + $logger->error("Error updating thumbhash for $id: $error"); + die $error; + } + } + + # Thumbnail generation + no warnings 'experimental::try'; + try { + generate_thumbnail( $arcimg, $thumbname, $use_hq, $use_jxl ); + } catch ($e) { + $logger->error("Thumbnail generation failed for archive '$file' entry '$requested_image' -> '$thumbname': $e"); + die $e; + } + + return $thumbname; +} + +1; diff --git a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm new file mode 100644 index 000000000..d62e20795 --- /dev/null +++ b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm @@ -0,0 +1,1050 @@ +package LANraragi::Utils::PsilabsDev::PgDatabase; + +use strict; +use warnings; +use utf8; + +use feature qw(signatures); +no warnings 'experimental::signatures'; + +use File::Basename; +use Cwd qw(getcwd); +use Redis; +use Time::HiRes qw(time); +use LANraragi::Model::Config; +use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Tags qw(split_tags_to_array join_tags_to_string); +use LANraragi::Utils::String qw(trim trim_CRLF); +use List::MoreUtils qw(uniq); +use List::Util qw(max); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::Path; +use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); +use LANraragi::Utils::PsilabsDev::PgArchive; +use LANraragi::Model::PsilabsDev::PgBackup; +use LANraragi::Model::PsilabsDev::PgCategory; +use LANraragi::Model::PsilabsDev::PgTankoubon; + +# Functions for interacting with Postgres. +use Exporter 'import'; +our @EXPORT_OK = qw( + get_archive get_archive_json get_archive_json_multi set_tags set_tags_with_dbh set_title set_title_with_dbh set_summary set_summary_with_dbh set_isnew clear_new_all invalidate_cache clean_database clean_categories_and_tanks change_archive_id change_archive_id_with_dbh drop_database +); + +# replaces LANraragi::Utils::Database::get_archive +# Retrieves archive metadata from Postgres and returns it as a hash +# similar to the Redis hgetall structure for compatibility +sub get_archive ($id) { + my $logger = get_logger( "PgDatabase", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + eval { + # Get archive data + my $sth = $dbh->prepare(q{ + SELECT arcid, filename, title, summary, thumbhash + FROM lrr_archive + WHERE arcid = ? + }); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + + unless ($row) { + $dbh->disconnect(); + return (); + } + + # Get tags as a comma-separated string + my $tag_sth = $dbh->prepare(q{ + SELECT string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ) as tags + FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = ? + }); + $tag_sth->execute($id); + my $tag_row = $tag_sth->fetchrow_hashref; + my $tags = $tag_row->{tags} // ""; + $tag_sth->finish; + + $dbh->disconnect(); + + # Extract name from filename + my ( $name, $path, $suffix ) = fileparse( $row->{filename}, qr/\.[^.]*/ ); + + # Build hash compatible with Redis version + my %hash = ( + name => $name, + title => $row->{title} // "", + tags => $tags, + summary => $row->{summary} // "", + file => $row->{filename}, + thumbhash => $row->{thumbhash} // "" + ); + + return %hash; + }; + + if ( my $error = $@ ) { + $logger->error("Error retrieving archive $id: $error"); + $dbh->disconnect(); + return (); + } +} + +# Internal function for building an archive JSON from Postgres data. +# This is similar to LANraragi::Utils::Database::build_json but does NOT apply +# redis_decode to text fields. Postgres data with client_encoding=UTF8 is already +# properly UTF-8 encoded, while Redis stores binary data that requires decoding. +# Applying redis_decode to already-decoded Postgres data causes double-decoding corruption. +sub build_json_pg ( $id, %hash ) { + + # Grab all metadata from the hash + my ( $name, $title, $tags, $summary, $file, $isnew, $progress, $pagecount, $lastreadtime, $arcsize, $toc ) = + @hash{qw(name title tags summary file isnew progress pagecount lastreadtime arcsize toc)}; + + $file = LANraragi::Utils::Path::create_path($file); + + # Return undef if the file doesn't exist. + return unless ( defined($file) && -e $file ); + + # NOTE: Unlike Database::build_json, we do NOT call redis_decode here. + # Postgres data is already UTF-8 encoded. + + # Workaround if title was incorrectly parsed as blank + if ( !defined($title) || $title =~ /^\s*$/ ) { + $title = $name; + } + + # Build ToC chapters array from rows (already fetched as arrayrefs of [page, title]) + my @chapters = (); + if ( defined $toc ) { + foreach my $entry (@$toc) { + push @chapters, { page => $entry->[0] + 0, name => $entry->[1] }; + } + } + + my $arcdata = { + arcid => $id, + title => $title, + filename => $name, + tags => $tags, + summary => $summary, + isnew => $isnew ? $isnew : "false", + extension => lc( ( split( /\./, $file ) )[-1] ), + progress => $progress ? int($progress) : 0, + pagecount => $pagecount ? int($pagecount) : 0, + lastreadtime => $lastreadtime ? int($lastreadtime) : 0, + size => $arcsize ? int($arcsize) : 0, + toc => \@chapters + }; + + return $arcdata; +} + +# replaces LANraragi::Utils::Database::get_archive_json +# Builds a JSON object for an archive registered in the database and returns it. +sub get_archive_json ( $dbh, $id ) { + my $logger = get_logger( "PgDatabase", "lanraragi" ); + + my $arcdata; + + eval { + # Check if this is a tank ID + if ( $id =~ /^TANK/ ) { + $arcdata = build_tank_json_pg($id); + } else { + # Check if archive exists + my $check_sth = $dbh->prepare('SELECT arcid FROM lrr_archive WHERE arcid = ?'); + $check_sth->execute($id); + my $exists = $check_sth->fetchrow_hashref; + $check_sth->finish; + + return unless $exists; + + # Get full archive data + my $sth = $dbh->prepare(q{ + SELECT arcid, filename, title, summary, thumbhash, isnew, progress, pagecount, lastreadtime, arcsize + FROM lrr_archive + WHERE arcid = ? + }); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + + # Get tags as a comma-separated string + my $tag_sth = $dbh->prepare(q{ + SELECT string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ) as tags + FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = ? + }); + $tag_sth->execute($id); + my $tag_row = $tag_sth->fetchrow_hashref; + my $tags = $tag_row->{tags} // ""; + $tag_sth->finish; + + # Get ToC entries + my $toc_sth = $dbh->prepare('SELECT page, title FROM lrr_toc WHERE arcid = ? ORDER BY page'); + $toc_sth->execute($id); + my @toc_rows; + while ( my @row_data = $toc_sth->fetchrow_array ) { + push @toc_rows, [ @row_data ]; + } + $toc_sth->finish; + + # Extract name from filename + my ( $name, $path, $suffix ) = fileparse( $row->{filename}, qr/\.[^.]*/ ); + + # Build hash for build_json_pg + my %hash = ( + name => $name, + title => $row->{title} // "", + tags => $tags, + summary => $row->{summary} // "", + file => $row->{filename}, + isnew => $row->{isnew} ? "true" : "false", + progress => $row->{progress} // 0, + pagecount => $row->{pagecount} // 0, + lastreadtime => $row->{lastreadtime} // 0, + arcsize => $row->{arcsize} // 0, + toc => \@toc_rows + ); + + # Use Postgres-specific build_json_pg which doesn't apply redis_decode + $arcdata = build_json_pg( $id, %hash ); + } + }; + + if ( my $error = $@ ) { + $logger->error("Error in get_archive_json for $id: $error"); + return; + } + + return $arcdata; +} + +# Internal helper for building a tank JSON (Postgres version). +# NOTE: Cannot directly reuse LANraragi::Utils::Database::build_tank_json because +# it calls LANraragi::Model::Tankoubon::get_tankoubon (Redis version) instead of +# LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon (Postgres version). +# The aggregation logic below is identical to Database::build_tank_json. +sub build_tank_json_pg ($id) { + my ( $total, $count, %tank ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon( $id, 1 ); + + # Aggregate data of all archives in the tank + my $aggregate_tags = ""; + my $aggregate_names = ""; + my $aggregate_isnew = 0; + my $aggregate_progress = 0; + my $aggregate_pagecount = 0; + my $latest_readtime = 0; + my $aggregate_size = 0; + + foreach my $archive_info ( @{ $tank{full_data} } ) { + $aggregate_tags .= %$archive_info{tags} . ","; + $aggregate_names .= %$archive_info{title} . ","; + $aggregate_isnew = $aggregate_isnew || %$archive_info{isnew}; + $aggregate_progress = $aggregate_progress + %$archive_info{progress}; + $aggregate_pagecount = $aggregate_pagecount + %$archive_info{pagecount}; + $aggregate_size = $aggregate_size + %$archive_info{size}; + $latest_readtime = max( $latest_readtime, %$archive_info{lastreadtime} ); + } + + chop $aggregate_tags; + chop $aggregate_names; + + my $arcdata = { + arcid => $id, + title => $tank{name}, + filename => "", + tags => $aggregate_tags, + summary => "Tankoubon containing: $aggregate_names", + isnew => $aggregate_isnew ? $aggregate_isnew : "false", + extension => ".tank", + progress => $aggregate_progress, + pagecount => $aggregate_pagecount, + lastreadtime => $latest_readtime, + size => $aggregate_size + }; + + return $arcdata; +} + +# replaces LANraragi::Utils::Database::get_archive_json_multi +# Builds JSON objects for multiple archives and returns them as an array. +sub get_archive_json_multi (@ids) { + my $logger = get_logger( "PgDatabase", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + my @archives; + + eval { + # Return empty array if no IDs provided + return unless @ids; + + # Separate tank IDs from archive IDs + my @tank_ids; + my @archive_ids; + foreach my $id (@ids) { + if ( $id =~ /^TANK/ ) { + push @tank_ids, $id; + } else { + push @archive_ids, $id; + } + } + + # Process tank IDs individually (they require complex aggregation logic) + if (@tank_ids) { + my $tank_start = time(); + foreach my $tank_id (@tank_ids) { + my $arcdata = build_tank_json_pg($tank_id); + if ($arcdata) { + push @archives, $arcdata; + } + } + my $tank_time = (time() - $tank_start) * 1000; + $logger->debug(sprintf("[PERF] Tank JSON building: %.2fms (tank_count: %d)", + $tank_time, scalar @tank_ids)); + } + + # Batch process archive IDs with a single query + if (@archive_ids) { + # Use batch query to fetch all archives and their tags at once + my $query_start = time(); + my $placeholders = join(',', ('?') x @archive_ids); + my $sth = $dbh->prepare(qq{ + SELECT a.arcid, a.filename, a.title, a.summary, a.thumbhash, + a.isnew, a.progress, a.pagecount, a.lastreadtime, a.arcsize, + COALESCE(string_agg( + CASE WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value END, + ', ' + ), '') as tags + FROM lrr_archive a + LEFT JOIN lrr_archive_to_tag_map atm ON a.arcid = atm.arcid + LEFT JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE a.arcid IN ($placeholders) + GROUP BY a.arcid, a.filename, a.title, a.summary, a.thumbhash, + a.isnew, a.progress, a.pagecount, a.lastreadtime, a.arcsize + }); + $sth->execute(@archive_ids); + + # Batch-fetch ToC entries for all archives + my $toc_sth = $dbh->prepare(qq{ + SELECT arcid, page, title FROM lrr_toc + WHERE arcid IN ($placeholders) + ORDER BY arcid, page + }); + $toc_sth->execute(@archive_ids); + my %toc_by_arcid; + while ( my $toc_row = $toc_sth->fetchrow_hashref ) { + push @{ $toc_by_arcid{ $toc_row->{arcid} } }, [ $toc_row->{page}, $toc_row->{title} ]; + } + $toc_sth->finish; + + # Store results in a hash for quick lookup + my %archive_data; + while (my $row = $sth->fetchrow_hashref) { + # Extract name from filename + my ( $name, $path, $suffix ) = fileparse( $row->{filename}, qr/\.[^.]*/ ); + + # Build hash for build_json_pg + my %hash = ( + name => $name, + title => $row->{title} // "", + tags => $row->{tags} // "", + summary => $row->{summary} // "", + file => $row->{filename}, + isnew => $row->{isnew} ? "true" : "false", + progress => $row->{progress} // 0, + pagecount => $row->{pagecount} // 0, + lastreadtime => $row->{lastreadtime} // 0, + arcsize => $row->{arcsize} // 0, + toc => $toc_by_arcid{ $row->{arcid} } + ); + + # Use Postgres-specific build_json_pg which doesn't apply redis_decode + my $arcdata = build_json_pg( $row->{arcid}, %hash ); + if ($arcdata) { + $archive_data{$row->{arcid}} = $arcdata; + } + } + $sth->finish; + my $query_time = (time() - $query_start) * 1000; + $logger->debug(sprintf("[PERF] Archive batch query execution: %.2fms (archive_count: %d)", + $query_time, scalar @archive_ids)); + + # Add archives to result array in the original order (preserving input order) + my $transform_start = time(); + foreach my $id (@archive_ids) { + if (exists $archive_data{$id}) { + push @archives, $archive_data{$id}; + } + } + my $transform_time = (time() - $transform_start) * 1000; + $logger->debug(sprintf("[PERF] JSON transformation: %.2fms", $transform_time)); + } + }; + + if ( my $error = $@ ) { + $logger->error("Error in get_archive_json_multi: $error"); + } + + $dbh->disconnect(); + return @archives; +} + +# replaces LANraragi::Utils::Database::set_title +sub set_title ( $id, $newtitle ) { + my $dbh = get_postgresql_dbh(); + $dbh->begin_work; + + eval { + set_title_with_dbh( $dbh, $id, $newtitle ); + }; + + my $error = $@; + if ($error) { + eval { $dbh->rollback }; + $dbh->disconnect; + die $error; + } + + $dbh->commit; + $dbh->disconnect; +} + +sub set_title_with_dbh ( $dbh, $id, $newtitle ) { + + my $logger = get_logger( "PgDatabase", "lanraragi" ); + + if ( $newtitle ne "" ) { + # NO begin_work - assume already in transaction + + # Update title in archive table + my $sth = $dbh->prepare('UPDATE lrr_archive SET title = ? WHERE arcid = ?'); + $sth->execute($newtitle, $id); + $sth->finish; + + # NO commit - caller manages transaction + + $logger->debug("Updated title for archive $id to: $newtitle"); + } +} + +# replaces LANraragi::Utils::Database::set_tags +# Set $tags for the archive with id $id. +# Set $append to 1 if you want to append the tags instead of replacing them. +sub set_tags ( $id, $newtags, $append = 0 ) { + my $dbh = get_postgresql_dbh(); + $dbh->begin_work; + + eval { + set_tags_with_dbh( $dbh, $id, $newtags, $append ); + }; + + my $error = $@; + if ($error) { + eval { $dbh->rollback }; + $dbh->disconnect; + die $error; + } + + $dbh->commit; + $dbh->disconnect; +} + +sub set_tags_with_dbh ( $dbh, $id, $newtags, $append = 0 ) { + + my $logger = get_logger( "PgDatabase", "lanraragi" ); + + # Get existing tags if we're appending + my $oldtags = ""; + if ($append) { + my $sth = $dbh->prepare(q{ + SELECT string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ) as tags + FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = ? + }); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $oldtags = $row->{tags} // ""; + $sth->finish; + + # If the new tags are empty, don't do anything + unless ( length $newtags ) { + return; + } + + if ($oldtags) { + $oldtags = trim($oldtags); + if ( $oldtags ne "" ) { + $newtags = $oldtags . "," . $newtags; + } + } + } + + # Normalize tags: split, unique, rejoin + $newtags = join_tags_to_string( uniq( split_tags_to_array($newtags) ) ); + + $logger->debug("Setting tags for archive $id: $newtags"); + + # NO begin_work - assume already in transaction + + # Remove existing tag mappings for this archive + my $delete_sth = $dbh->prepare('DELETE FROM lrr_archive_to_tag_map WHERE arcid = ?'); + $delete_sth->execute($id); + $delete_sth->finish; + + # Parse and insert new tags + my @tag_array = split_tags_to_array($newtags); + + # Prepare all statements ONCE before the loop + my $tag_sth = $dbh->prepare(q{ + INSERT INTO lrr_tag (namespace, value) + VALUES (?, ?) + ON CONFLICT (namespace, value) DO NOTHING + }); + my $tagid_sth = $dbh->prepare(q{ + SELECT tagid FROM lrr_tag WHERE namespace = ? AND value = ? + }); + my $map_sth = $dbh->prepare(q{ + INSERT INTO lrr_archive_to_tag_map (arcid, tagid, namespace, value, update_date) + VALUES (?, ?, ?, ?, CURRENT_DATE) + }); + + foreach my $tag (@tag_array) { + next unless $tag; + + # Parse namespace and value + my ($namespace, $value); + if ( $tag =~ /^([^:]+):(.+)$/ ) { + $namespace = $1; + $value = $2; + } else { + $namespace = ''; + $value = $tag; + } + + # Trim whitespace + $namespace = trim($namespace); + $value = trim($value); + + # Insert tag if it doesn't exist (ON CONFLICT DO NOTHING) + $tag_sth->execute($namespace, $value); + + # Get the tag ID + $tagid_sth->execute($namespace, $value); + my $row = $tagid_sth->fetchrow_hashref; + my $tagid = $row->{tagid}; + + # Insert mapping + $map_sth->execute($id, $tagid, $namespace, $value); + } + + # Finish all statements AFTER the loop + $tag_sth->finish; + $tagid_sth->finish; + $map_sth->finish; + + # NO commit - caller manages transaction + + $logger->debug("Successfully updated tags for archive $id"); +} + +# replaces LANraragi::Utils::Database::set_summary +sub set_summary ( $id, $summary ) { + my $dbh = get_postgresql_dbh(); + $dbh->begin_work; + + eval { + set_summary_with_dbh( $dbh, $id, $summary ); + }; + + my $error = $@; + if ($error) { + eval { $dbh->rollback }; + $dbh->disconnect; + die $error; + } + + $dbh->commit; + $dbh->disconnect; +} + +sub set_summary_with_dbh ( $dbh, $id, $summary ) { + + my $logger = get_logger( "PgDatabase", "lanraragi" ); + + eval { + my $sth = $dbh->prepare('UPDATE lrr_archive SET summary = ? WHERE arcid = ?'); + $sth->execute($summary, $id); + $sth->finish; + + $logger->debug("Updated summary for archive $id"); + }; + + if ( my $error = $@ ) { + $logger->error("Error setting summary for archive $id: $error"); + die $error; + } +} + +# replaces LANraragi::Utils::Database::set_isnew +sub set_isnew ( $id, $isnew ) { + my $logger = get_logger( "PgDatabase", "lanraragi" ); + + # Convert "false" to false boolean, everything else to true + my $newval = $isnew ne "false" ? 1 : 0; + + my $dbh = get_postgresql_dbh(); + + eval { + my $sth = $dbh->prepare('UPDATE lrr_archive SET isnew = ? WHERE arcid = ?'); + $sth->execute($newval, $id); + $sth->finish; + + $logger->debug("Updated isnew for archive $id to: $newval"); + }; + + if ( my $error = $@ ) { + $logger->error("Error setting isnew for archive $id: $error"); + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); +} + +# replaces LANraragi::Controller::Api::Database::clear_new_all (endpoint logic) +# Clear the new flag in all archives. +sub clear_new_all { + my $logger = get_logger( "PgDatabase", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + eval { + # Set isnew to false for all archives + my $sth = $dbh->prepare('UPDATE lrr_archive SET isnew = FALSE'); + $sth->execute(); + my $rows_affected = $sth->rows; + $sth->finish; + + $logger->info("Cleared new flag for all archives (affected $rows_affected rows)"); + }; + + if ( my $error = $@ ) { + $logger->error("Error clearing new flag for all archives: $error"); + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); +} + +# replaces LANraragi::Utils::Database::invalidate_cache +# In Postgres, there's no separate search cache to invalidate. +sub invalidate_cache ( $rebuild_indexes = 0 ) { + return; +} + +# replaces LANraragi::Utils::Database::change_archive_id +# Changes an archive's ID from $old_id to $new_id in the database. +# This updates the archive record and all references in categories and tankoubons. +# Also updates the filemap in Redis (still used by Shinobu for file tracking). +sub change_archive_id ( $old_id, $new_id ) { + my $logger = get_logger( "PgDatabase", "lanraragi" ); + my $dbh = get_postgresql_dbh(); + + $logger->debug("Changing ID $old_id to $new_id"); + + my $file_for_redis; + eval { + $dbh->begin_work; + + # Check if old ID exists in archive table + my $check_sth = $dbh->prepare('SELECT arcid, filename FROM lrr_archive WHERE arcid = ?'); + $check_sth->execute($old_id); + my $row = $check_sth->fetchrow_hashref; + $check_sth->finish; + + if ($row) { + # Update the archive ID + my $update_arc_sth = $dbh->prepare('UPDATE lrr_archive SET arcid = ? WHERE arcid = ?'); + $update_arc_sth->execute($new_id, $old_id); + $update_arc_sth->finish; + + # Update archive size based on file + my $file = LANraragi::Utils::Path::create_path($row->{filename}); + if (defined($file) && -e $file) { + my $arcsize = -s $file; + my $update_size_sth = $dbh->prepare('UPDATE lrr_archive SET arcsize = ? WHERE arcid = ?'); + $update_size_sth->execute($arcsize, $new_id); + $update_size_sth->finish; + } + + # Update tag mappings + my $update_tag_sth = $dbh->prepare('UPDATE lrr_archive_to_tag_map SET arcid = ? WHERE arcid = ?'); + $update_tag_sth->execute($new_id, $old_id); + $update_tag_sth->finish; + + # Update category mappings + my $update_cat_sth = $dbh->prepare('UPDATE lrr_category_to_archive_map SET arcid = ? WHERE arcid = ?'); + $update_cat_sth->execute($new_id, $old_id); + $update_cat_sth->finish; + + # Update tankoubon mappings + my $update_tank_sth = $dbh->prepare('UPDATE lrr_tank_to_archive_map SET arcid = ? WHERE arcid = ?'); + $update_tank_sth->execute($new_id, $old_id); + $update_tank_sth->finish; + + $logger->debug("Updated archive and all references from $old_id to $new_id"); + + # Get file path for Redis update before committing + $file_for_redis = get_archive_path($dbh, $new_id); + } + + $dbh->commit; + }; + + if (my $error = $@) { + $logger->error("Error changing archive ID from $old_id to $new_id: $error"); + eval { $dbh->rollback }; + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); + + # Update the filemap in Redis (still used by Shinobu) + if (defined($file_for_redis) && $file_for_redis ne "") { + my $redis_config = LANraragi::Model::Config->get_redis_config; + $redis_config->hset( "LRR_FILEMAP", $file_for_redis, $new_id ); + $redis_config->quit; + } +} + +# Cleans the database by clearing all categories, tankoubons, and their mappings. +# This is used before restoring from a backup. +# Note: Archives themselves are NOT deleted - only their metadata relationships. +sub clean_categories_and_tanks { + my $logger = get_logger("PgDatabase", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + $logger->info("Cleaning categories and tankoubons before restore..."); + + eval { + $dbh->begin_work; + + # Delete category to archive mappings + $dbh->do('DELETE FROM lrr_category_to_archive_map'); + $logger->debug("Cleared category to archive mappings"); + + # Delete all categories + $dbh->do('DELETE FROM lrr_category'); + $logger->debug("Cleared categories"); + + # Delete tankoubon to archive mappings + $dbh->do('DELETE FROM lrr_tank_to_archive_map'); + $logger->debug("Cleared tankoubon to archive mappings"); + + # Delete all tankoubons + $dbh->do('DELETE FROM lrr_tank'); + $logger->debug("Cleared tankoubons"); + + $dbh->commit; + $logger->info("Categories and tankoubons cleaned successfully"); + }; + + if ($@) { + my $error = $@; + $logger->error("Error cleaning categories and tankoubons: $error"); + eval { $dbh->rollback }; + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); + return; +} + +# Helper for change_archive_id that accepts a database handle +# Used internally by clean_database to avoid creating new connections mid-operation +sub change_archive_id_with_dbh ( $dbh, $old_id, $new_id ) { + my $logger = get_logger( "PgDatabase", "lanraragi" ); + + $logger->debug("Changing ID $old_id to $new_id"); + + my $file_for_redis; + + # NO begin_work - assume already in transaction + + # Check if old ID exists in archive table + my $check_sth = $dbh->prepare('SELECT arcid, filename FROM lrr_archive WHERE arcid = ?'); + $check_sth->execute($old_id); + my $row = $check_sth->fetchrow_hashref; + $check_sth->finish; + + if ($row) { + # Update the archive ID + my $update_arc_sth = $dbh->prepare('UPDATE lrr_archive SET arcid = ? WHERE arcid = ?'); + $update_arc_sth->execute($new_id, $old_id); + $update_arc_sth->finish; + + # Update archive size based on file + my $file = LANraragi::Utils::Path::create_path($row->{filename}); + if (defined($file) && -e $file) { + my $arcsize = -s $file; + my $update_size_sth = $dbh->prepare('UPDATE lrr_archive SET arcsize = ? WHERE arcid = ?'); + $update_size_sth->execute($arcsize, $new_id); + $update_size_sth->finish; + } + + # Update tag mappings + my $update_tag_sth = $dbh->prepare('UPDATE lrr_archive_to_tag_map SET arcid = ? WHERE arcid = ?'); + $update_tag_sth->execute($new_id, $old_id); + $update_tag_sth->finish; + + # Update category mappings + my $update_cat_sth = $dbh->prepare('UPDATE lrr_category_to_archive_map SET arcid = ? WHERE arcid = ?'); + $update_cat_sth->execute($new_id, $old_id); + $update_cat_sth->finish; + + # Update tankoubon mappings + my $update_tank_sth = $dbh->prepare('UPDATE lrr_tank_to_archive_map SET arcid = ? WHERE arcid = ?'); + $update_tank_sth->execute($new_id, $old_id); + $update_tank_sth->finish; + + $logger->debug("Updated archive and all references from $old_id to $new_id"); + + # Get file path for Redis update + $file_for_redis = get_archive_path($dbh, $new_id); + } + + # NO commit - caller manages transaction + + return $file_for_redis; +} + +# replaces LANraragi::Utils::Database::clean_database +# Remove entries from the database that don't have a matching archive on the filesystem. +# Returns the number of entries deleted/unlinked. +sub clean_database { + my $logger = get_logger("PgDatabase", "lanraragi"); + + eval { + # Save an autobackup somewhere before cleaning + my $outfile = getcwd() . "/autobackup.json"; + $logger->info("Saving automatic backup to $outfile"); + open( my $fh, '>', $outfile ); + print $fh LANraragi::Model::PsilabsDev::PgBackup::build_backup_JSON(); + close $fh; + }; + + if ($@) { + $logger->warn("Unable to open a file to save backup before cleaning database! $@"); + } + + # Get the filemap from Redis for ID checks later down the line + # This is still needed because Shinobu uses Redis to track file changes + my $redis_config = LANraragi::Model::Config->get_redis_config; + my @filemapids = $redis_config->exists("LRR_FILEMAP") ? $redis_config->hvals("LRR_FILEMAP") : (); + my %filemap = map { $_ => 1 } @filemapids; + + my $dbh = get_postgresql_dbh(); + my $deleted_arcs = 0; + my $unlinked_arcs = 0; + + eval { + # Get all archive IDs + my $sql = 'SELECT arcid, filename FROM lrr_archive'; + my $sth = $dbh->prepare($sql); + $sth->execute(); + + while (my $row = $sth->fetchrow_hashref) { + my $id = $row->{arcid}; + my $file = LANraragi::Utils::Path::create_path($row->{filename}); + + # Check if the linked file exists + unless ( defined($file) && -e $file ) { + $logger->debug("Archive $id file does not exist: " . ($row->{filename} // "undefined")); + + # Delete the archive using PgArchive + LANraragi::Utils::PsilabsDev::PgArchive::delete_archive($id); + $deleted_arcs++; + next; + } + + # If the linked file exists, check if its ID is in the filemap + # This handles cases where archive files are modified and get new IDs + unless ( $file eq "" || exists $filemap{$id} ) { + $logger->warn("File exists but its ID is no longer $id!"); + $logger->warn("Trying to find its new ID in the Shinobu filemap..."); + + if ( $redis_config->hexists( "LRR_FILEMAP", $file ) ) { + my $newid = $redis_config->hget( "LRR_FILEMAP", $file ); + $logger->warn("Found $newid in the filemap! Changing ID from $id to it."); + + # Check if the new ID already exists as a separate entry + my $check_sth = $dbh->prepare('SELECT arcid FROM lrr_archive WHERE arcid = ?'); + $check_sth->execute($newid); + my $exists = $check_sth->fetchrow_hashref; + $check_sth->finish; + + if ( $exists ) { + $logger->warn("ID $newid already exists in the database! Unlinking old ID."); + # Clear the filename to unlink the old entry (with transaction) + eval { + $dbh->begin_work; + my $unlink_sth = $dbh->prepare('UPDATE lrr_archive SET filename = ? WHERE arcid = ?'); + $unlink_sth->execute("", $id); + $unlink_sth->finish; + $dbh->commit; + }; + if (my $err = $@) { + eval { $dbh->rollback }; + die $err; + } + # NOTE: Do NOT increment $unlinked_arcs here, matching Redis behavior at line 381 + } else { + # Use the transactional version of change_archive_id + my $file_for_redis; + eval { + $dbh->begin_work; + $file_for_redis = change_archive_id_with_dbh( $dbh, $id, $newid ); + $dbh->commit; + }; + if (my $err = $@) { + eval { $dbh->rollback }; + die $err; + } + # Update Redis filemap + if (defined($file_for_redis) && $file_for_redis ne "") { + $redis_config->hset( "LRR_FILEMAP", $file_for_redis, $newid ); + } + } + + } else { + $logger->warn("File $file not found in the filemap! Removing file reference in the database entry for $id."); + # Clear the filename to unlink the entry (with transaction) + eval { + $dbh->begin_work; + my $unlink_sth = $dbh->prepare('UPDATE lrr_archive SET filename = ? WHERE arcid = ?'); + $unlink_sth->execute("", $id); + $unlink_sth->finish; + $dbh->commit; + }; + if (my $err = $@) { + eval { $dbh->rollback }; + die $err; + } + $unlinked_arcs++; + } + } + } + + $sth->finish; + }; + + if (my $error = $@) { + $logger->error("Error during clean_database: $error"); + $dbh->disconnect(); + $redis_config->quit; + die $error; + } + + $dbh->disconnect(); + $redis_config->quit; + return ( $deleted_arcs, $unlinked_arcs ); +} + +# replaces LANraragi::Utils::Database::drop_database +# Drops the entire database by deleting all data from all tables. +# This is extremely dangerous and cannot be undone. +sub drop_database { + my $logger = get_logger("PgDatabase", "lanraragi"); + my $dbh = get_postgresql_dbh(); + + $logger->warn("Dropping entire database - all Postgres and Redis data will be lost!"); + + # Drop Postgres tables + eval { + $dbh->begin_work; + + # Delete in order to respect foreign key constraints + # First delete all mapping tables (they have foreign keys to other tables) + $dbh->do('DELETE FROM lrr_archive_to_tag_map'); + $logger->debug("Cleared archive to tag mappings"); + + $dbh->do('DELETE FROM lrr_category_to_archive_map'); + $logger->debug("Cleared category to archive mappings"); + + $dbh->do('DELETE FROM lrr_tank_to_archive_map'); + $logger->debug("Cleared tankoubon to archive mappings"); + + $dbh->do('DELETE FROM lrr_toc'); + $logger->debug("Cleared ToC entries"); + + # Then delete the main tables + $dbh->do('DELETE FROM lrr_tag'); + $logger->debug("Cleared tags"); + + $dbh->do('DELETE FROM lrr_category'); + $logger->debug("Cleared categories"); + + $dbh->do('DELETE FROM lrr_tank'); + $logger->debug("Cleared tankoubons"); + + $dbh->do('DELETE FROM lrr_archive'); + $logger->debug("Cleared archives"); + + $dbh->commit; + $logger->info("Postgres database dropped successfully"); + }; + + if (my $error = $@) { + $logger->error("Error dropping Postgres database: $error"); + eval { $dbh->rollback }; + $dbh->disconnect(); + die $error; + } + + $dbh->disconnect(); + + # Drop all Redis databases (config, minion, search cache) + # This matches the original behavior of flushall() in the Redis implementation + my $redis; + eval { + $redis = LANraragi::Model::Config->get_redis; + $redis->flushall(); + $logger->info("All Redis databases cleared successfully"); + }; + + if (my $error = $@) { + $logger->error("Error clearing Redis databases: $error"); + eval { $redis->quit if $redis }; + die $error; + } + + $redis->quit(); +} + +1; diff --git a/lib/LANraragi/Utils/PsilabsDev/PgPath.pm b/lib/LANraragi/Utils/PsilabsDev/PgPath.pm new file mode 100644 index 000000000..23098ae95 --- /dev/null +++ b/lib/LANraragi/Utils/PsilabsDev/PgPath.pm @@ -0,0 +1,25 @@ +package LANraragi::Utils::PsilabsDev::PgPath; + +use v5.36; + +use strict; +use warnings; +use utf8; + +use Exporter 'import'; +our @EXPORT_OK = qw(get_archive_path); + +use LANraragi::Utils::Path qw(create_path); + +# replaces LANraragi::Utils::Path::get_archive_path +sub get_archive_path ( $dbh, $id ) { + my $sth = $dbh->prepare('SELECT filename FROM lrr_archive WHERE arcid = ?'); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + + return unless $row; + return create_path( $row->{filename} ); +} + +1; diff --git a/lib/LANraragi/Utils/PsilabsDev/PgPlugins.pm b/lib/LANraragi/Utils/PsilabsDev/PgPlugins.pm new file mode 100644 index 000000000..8d24e88bd --- /dev/null +++ b/lib/LANraragi/Utils/PsilabsDev/PgPlugins.pm @@ -0,0 +1,51 @@ +package LANraragi::Utils::PsilabsDev::PgPlugins; + +use strict; +use warnings; +use utf8; + +use LANraragi::Utils::Plugins qw(get_plugin get_plugin_parameters); +use LANraragi::Utils::Redis qw(redis_decode); +use LANraragi::Utils::Logging qw(get_logger); + +use Exporter 'import'; +our @EXPORT_OK = qw(use_plugin); + +# replaces LANraragi::Utils::Plugins::use_plugin +# Shorthand method to use a plugin by name. +sub use_plugin { + + my ( $plugname, $id, $input ) = @_; + + my $plugin = get_plugin($plugname); + my %plugin_result; + my %pluginfo; + + if ( !$plugin ) { + $plugin_result{error} = "Plugin not found on system."; + } else { + %pluginfo = $plugin->plugin_info(); + + # Get the plugin settings in Redis (plugin settings are still in Redis) + my %settings = get_plugin_parameters($plugname); + $settings{oneshot} = $input; + + # Execute the plugin using Postgres implementations + if ( $pluginfo{type} eq "script" ) { + %plugin_result = LANraragi::Model::Plugins::exec_script_plugin( $plugin, %settings ); + } elsif ( $pluginfo{type} eq "metadata" ) { + # Use Postgres implementation for metadata plugins + require LANraragi::Model::PsilabsDev::PgPlugins; + %plugin_result = LANraragi::Model::PsilabsDev::PgPlugins::exec_metadata_plugin( $plugin, $id, %settings ); + } + + # Decode the error value if there's one to avoid garbled characters + if ( exists $plugin_result{error} ) { + $plugin_result{error} = redis_decode( $plugin_result{error} ); + } + } + + return ( \%pluginfo, \%plugin_result ); +} + +1; diff --git a/lib/LANraragi/Utils/PsilabsDev/Postgres.pm b/lib/LANraragi/Utils/PsilabsDev/Postgres.pm new file mode 100644 index 000000000..922efb8ce --- /dev/null +++ b/lib/LANraragi/Utils/PsilabsDev/Postgres.pm @@ -0,0 +1,437 @@ +package LANraragi::Utils::PsilabsDev::Postgres; + +use strict; +use warnings; +use utf8; + +use Exporter 'import'; +our @EXPORT_OK = qw(get_postgresql_dbh initialize_database); + +use DBD::Pg; +use DBI; +use LANraragi::Utils::Logging qw(get_logger); + +# Get the PostgreSQL database connection. +sub get_postgresql_dbh { + + # required variables. + # TODO: this needs to be populated but for now we can just go with this. + my $dbname = $ENV{LRR_POSTGRES_DB} // 'postgres'; + my $host = $ENV{LRR_POSTGRES_HOST} // 'postgres'; + my $port = $ENV{LRR_POSTGRES_PORT} // 5432; + # my $options = ''; # TODO: figure out if options is needed. + my $username = $ENV{LRR_POSTGRES_USER} // 'postgres'; + my $password = $ENV{LRR_POSTGRES_PASSWORD} // 'postgres'; + + my $dbh = DBI->connect( + "dbi:Pg:dbname=$dbname;host=$host;port=$port", + $username, + $password, + { + AutoCommit => 1, + RaiseError => 1, + PrintError => 0, + pg_enable_utf8 => 1, + } + ); + + return $dbh; +} + +sub initialize_database { + my $dbh = shift; + my $logger = get_logger("Postgres Utils", "lanraragi"); + my $rv; + my $sql; + + $logger->info("Initializing PostgreSQL database..."); + + # Natural sort collation: numeric sequences sort by value, not lexicographically. + my $collate_clause = ""; + $sql = <<'SQL'; +CREATE COLLATION IF NOT EXISTS natural_sort (provider = icu, locale = 'en-u-kn-true') +SQL + eval { $dbh->do($sql); }; + if ($@) { + $logger->warn("Failed to create natural_sort collation (ICU may not be available): $@"); + $logger->warn("Falling back to default collation — title sort will be lexicographic"); + } else { + $logger->info("Created collation: natural_sort"); + $collate_clause = "COLLATE natural_sort"; + } + + $sql = <<"SQL"; +CREATE TABLE IF NOT EXISTS lrr_archive ( + arcid VARCHAR(255) PRIMARY KEY, + filename VARCHAR(255) NOT NULL, + extension VARCHAR(255), + isnew BOOLEAN NOT NULL, + lastreadtime INTEGER NOT NULL, + pagecount INTEGER NOT NULL, + progress INTEGER NOT NULL, + title VARCHAR(255) $collate_clause NOT NULL, + summary TEXT, + thumbhash VARCHAR(255), + arcsize BIGINT +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_archive table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_archive"); + + $sql = <<'SQL'; +CREATE TABLE IF NOT EXISTS lrr_category ( + catid VARCHAR(255) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + pinned BOOLEAN NOT NULL, + search VARCHAR(255) +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_category table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_category"); + + $sql = <<"SQL"; +CREATE TABLE IF NOT EXISTS lrr_tank ( + tankid VARCHAR(255) PRIMARY KEY, + name VARCHAR(255) $collate_clause NOT NULL, + summary TEXT, + tags TEXT +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_tank table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_tank"); + + $sql = <<'SQL'; +CREATE TABLE IF NOT EXISTS lrr_tag ( + tagid INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + namespace VARCHAR(255) NOT NULL DEFAULT '', + value VARCHAR(255) NOT NULL, + UNIQUE (namespace, value) +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_tag table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_tag"); + + $sql = <<'SQL'; +CREATE TABLE IF NOT EXISTS lrr_category_to_archive_map ( + catid VARCHAR(255) NOT NULL, + arcid VARCHAR(255) NOT NULL, + update_date DATE, + FOREIGN KEY (arcid) REFERENCES lrr_archive (arcid), + FOREIGN KEY (catid) REFERENCES lrr_category (catid) +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_category_to_archive_map table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_category_to_archive_map"); + + $sql = <<'SQL'; +CREATE TABLE IF NOT EXISTS lrr_archive_to_tag_map ( + arcid VARCHAR(255) NOT NULL, + tagid INTEGER NOT NULL, + namespace VARCHAR(255) NOT NULL DEFAULT '', + value VARCHAR(255) NOT NULL DEFAULT '', + update_date DATE, + FOREIGN KEY (arcid) REFERENCES lrr_archive (arcid), + FOREIGN KEY (tagid) REFERENCES lrr_tag (tagid) +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_archive_to_tag_map table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_archive_to_tag_map"); + + $sql = <<'SQL'; +CREATE TABLE IF NOT EXISTS lrr_tank_to_archive_map ( + tankid VARCHAR(255) NOT NULL, + arcid VARCHAR(255) NOT NULL, + position INTEGER NOT NULL, + update_date DATE, + FOREIGN KEY (arcid) REFERENCES lrr_archive (arcid), + FOREIGN KEY (tankid) REFERENCES lrr_tank (tankid) +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_tank_to_archive_map table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_tank_to_archive_map"); + + $sql = <<'SQL'; +CREATE TABLE IF NOT EXISTS lrr_toc ( + arcid VARCHAR(255) NOT NULL, + page INTEGER NOT NULL, + title VARCHAR(255) NOT NULL, + FOREIGN KEY (arcid) REFERENCES lrr_archive (arcid), + UNIQUE (arcid, page) +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_toc table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_toc"); + + $sql = <<'SQL'; +CREATE EXTENSION IF NOT EXISTS pg_trgm +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create pg_trgm extension: $errorcode - $errorstr"; + } + $logger->info("Created extension: pg_trgm"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_title_trgm ON lrr_archive USING gin (title gin_trgm_ops) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_title_trgm index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_title_trgm"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_tag_value_trgm ON lrr_tag USING gin (value gin_trgm_ops) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_tag_value_trgm index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_tag_value_trgm"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_to_tag_arcid ON lrr_archive_to_tag_map (arcid) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_to_tag_arcid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_to_tag_arcid"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_to_tag_tagid ON lrr_archive_to_tag_map (tagid) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_to_tag_tagid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_to_tag_tagid"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_isnew ON lrr_archive (isnew) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_isnew index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_isnew"); + + # P7: Category to archive mapping indexes + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_category_to_archive_catid ON lrr_category_to_archive_map (catid) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_category_to_archive_catid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_category_to_archive_catid"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_category_to_archive_arcid ON lrr_category_to_archive_map (arcid) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_category_to_archive_arcid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_category_to_archive_arcid"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_category_to_archive_catid_arcid ON lrr_category_to_archive_map (catid, arcid) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_category_to_archive_catid_arcid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_category_to_archive_catid_arcid"); + + # P7: Tank to archive mapping indexes + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_tank_to_archive_tankid ON lrr_tank_to_archive_map (tankid) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_tank_to_archive_tankid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_tank_to_archive_tankid"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_tank_to_archive_arcid ON lrr_tank_to_archive_map (arcid) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_tank_to_archive_arcid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_tank_to_archive_arcid"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_tank_to_archive_tankid_position ON lrr_tank_to_archive_map (tankid, position) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_tank_to_archive_tankid_position index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_tank_to_archive_tankid_position"); + + # P8: Tag namespace trigram index + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_tag_namespace_trgm ON lrr_tag USING gin (namespace gin_trgm_ops) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_tag_namespace_trgm index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_tag_namespace_trgm"); + + # P9: Sort optimization index + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_lastreadtime ON lrr_archive (lastreadtime DESC) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_lastreadtime index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_lastreadtime"); + + # P14: Filter optimization indexes + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_pagecount ON lrr_archive (pagecount) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_pagecount index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_pagecount"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_progress ON lrr_archive (progress) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_progress index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_progress"); + + # P2-Lite: Composite indexes for EXISTS subquery optimization + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_tag_namespace_value ON lrr_tag (namespace, value) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_tag_namespace_value index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_tag_namespace_value"); + + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_to_tag_arcid_tagid ON lrr_archive_to_tag_map (arcid, tagid) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_to_tag_arcid_tagid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_to_tag_arcid_tagid"); + + # P15: Covering index for LATERAL JOIN optimization (tag-based sorting) + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_tag_tagid_namespace_value ON lrr_tag (tagid, namespace) INCLUDE (value) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_tag_tagid_namespace_value index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_tag_tagid_namespace_value"); + + # P16: Namespace-based sort index on tag map (denormalized namespace enables direct namespace lookups + # without joining to lrr_tag, critical for namespace sort queries at scale) + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_archive_to_tag_namespace_arcid ON lrr_archive_to_tag_map (namespace, arcid) INCLUDE (tagid, value) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_archive_to_tag_namespace_arcid index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_archive_to_tag_namespace_arcid"); + + $logger->info("PostgreSQL database initialized successfully"); +} + +1; \ No newline at end of file diff --git a/lib/LANraragi/Utils/Routing.pm b/lib/LANraragi/Utils/Routing.pm index 3ff4b6049..f4afa96ab 100644 --- a/lib/LANraragi/Utils/Routing.pm +++ b/lib/LANraragi/Utils/Routing.pm @@ -102,11 +102,11 @@ sub apply_routes { $public_routes = $logged_in; } - $public_routes->get('/')->to('index#index'); - $public_routes->get('/index')->to('index#index'); - $public_routes->get('/random')->to('index#random_archive'); - $public_routes->get('/reader')->to('reader#index'); - $public_routes->get('/stats')->to('stats#index'); + $public_routes->get('/')->to('index#index'); # TODO: require postgres migration (done) + $public_routes->get('/index')->to('index#index'); # TODO: require postgres migration (done) + $public_routes->get('/random')->to('index#random_archive'); # TODO: require postgres migration (done) + $public_routes->get('/reader')->to('reader#index'); # TODO: require postgres migration (done) + $public_routes->get('/stats')->to('stats#index'); # TODO: require postgres migration (done) $public_routes->get('/js/i18n.js')->to('i18_n#index'); # Minion Admin UI @@ -130,17 +130,17 @@ sub apply_routes { $logged_in->post('/config/plugins')->to('plugins#save_config'); $logged_in->post('/config/plugins/upload')->to('plugins#process_upload'); - $logged_in->get('/config/categories')->to('category#index'); + $logged_in->get('/config/categories')->to('category#index'); # TODO: require postgres migration (done) - $logged_in->get('/batch')->to('batch#index'); - $logged_in->websocket('/batch/socket')->to('batch#socket'); + $logged_in->get('/batch')->to('batch#index'); # TODO: require postgres migration (done) + $logged_in->websocket('/batch/socket')->to('batch#socket'); # TODO: require postgres migration (done) - $logged_in->get('/edit')->to('edit#index'); + $logged_in->get('/edit')->to('edit#index'); # TODO: require postgres migration (done) - $logged_in->get('/backup')->to('backup#index'); + $logged_in->get('/backup')->to('backup#index'); # TODO: require postgres migration (done) - $logged_in->get('/upload')->to('upload#index'); - $logged_in->post('/upload')->to('upload#process_upload'); + $logged_in->get('/upload')->to('upload#index'); # TODO: require postgres migration (done) + $logged_in->post('/upload')->to('upload#process_upload'); # TODO: require postgres migration (done) $logged_in->get('/logs')->to('logging#index'); $logged_in->get('/logs/general')->to('logging#print_general'); @@ -149,14 +149,14 @@ sub apply_routes { $logged_in->get('/logs/mojo')->to('logging#print_mojo'); $logged_in->get('/logs/redis')->to('logging#print_redis'); - $logged_in->get('/duplicates')->to('duplicates#index'); + $logged_in->get('/duplicates')->to('duplicates#index'); # TODO: require postgres migration (done) # Metrics API (not part of OpenAPI spec, serves Prometheus format) if ( $self->LRR_CONF->enable_metrics ) { $logged_in_api->get('/api/info/metrics')->to('api-metrics#serve_metrics'); } - $search_api->get('/search')->to('api-search#handle_datatables'); + $search_api->get('/search')->to('api-search#handle_datatables'); # TODO: require postgres migration } diff --git a/lib/Shinobu.pm b/lib/Shinobu.pm index c67a8de0e..721ffcb7c 100644 --- a/lib/Shinobu.pm +++ b/lib/Shinobu.pm @@ -28,18 +28,21 @@ use File::ChangeNotify; use File::Basename; use Encode; -use LANraragi::Utils::Archive qw(extract_thumbnail); -use LANraragi::Utils::Database qw(invalidate_cache compute_id change_archive_id get_arcsize add_timestamp_tag add_archive_to_redis add_arcsize add_pagecount); +use LANraragi::Utils::PsilabsDev::PgArchive qw(extract_thumbnail); +use LANraragi::Utils::Database qw(invalidate_cache compute_id); use LANraragi::Utils::Logging qw(get_logger); -use LANraragi::Utils::Generic qw(is_archive exec_with_lock_pure); -use LANraragi::Utils::Redis qw(redis_encode); -use LANraragi::Utils::Path qw(create_path open_path find_path get_archive_path); +use LANraragi::Utils::Generic qw(is_archive); +use LANraragi::Utils::Path qw(create_path open_path find_path); use LANraragi::Model::Config; -use LANraragi::Model::Plugins; use LANraragi::Model::Metrics; -use LANraragi::Utils::Plugins; # Needed here since Shinobu doesn't inherit from the main LRR package -use LANraragi::Model::Search; # idem + +# Postgres-specific imports +use LANraragi::Model::PsilabsDev::PgUpload qw(add_timestamp_tag add_pagecount add_arcsize add_timestamp_tag_with_dbh add_pagecount_with_dbh add_arcsize_with_dbh add_archive_to_postgres); +use LANraragi::Model::PsilabsDev::PgPlugins; +use LANraragi::Utils::PsilabsDev::PgDatabase qw(change_archive_id); +use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); @@ -175,14 +178,19 @@ sub update_filemap { } }; - if ($@) { - $logger->error("Error while scanning content folder: $@"); + if ( my $error = $@ ) { + $logger->error("Error while scanning content folder: $error"); } } sub add_to_filemap ( $redis_cfg, $file ) { - my $redis_arc = LANraragi::Model::Config->get_redis; + my $dbh = get_postgresql_dbh(); + unless ($dbh) { + $logger->error("Failed to connect to PostgreSQL database"); + return; + } + if ( is_archive($file) ) { $logger->debug("Adding $file to Shinobu filemap."); @@ -213,107 +221,112 @@ sub add_to_filemap ( $redis_cfg, $file ) { if ($compute_error && -e $file) { $logger->error("Couldn't open $file for ID computation: $compute_error"); $logger->error("Giving up on adding it to the filemap."); + $dbh->disconnect(); return; } elsif ($compute_error) { $logger->warn("File $file no longer exists; giving up on adding it to the filemap."); + $dbh->disconnect(); return; } - # Acquire exclusive metadata and file write access for archive by ID with 1m timeout - my ($acquired, $is_new) = exec_with_lock_pure( - [ "archive-write:$id" ], - sub { update_filemap_entry( $logger, $id, $file, $redis_cfg, $redis_arc ) }, - undef, 60 - ); - - if ( !$acquired ) { - $logger->warn("Write lock already acquired for archive $file with ID $id, skipping."); - } - - # New file handling runs outside the lock so auto-plugin can acquire its own lock. - if ( $acquired && $is_new ) { - add_new_file( $id, $file ); - invalidate_cache(); - } - - } else { - $logger->debug("$file not recognized as archive, skipping."); - } - $redis_arc->quit; -} - -sub update_filemap_entry ( $logger, $id, $file, $redis_cfg, $redis_arc ) { + $logger->debug("Computed ID is $id."); - $logger->debug("Computed ID is $id."); - unless ( -e $file ) { - # A TOCTOU check, if the file was deleted after ID computation but before a lock was acquired. - $logger->warn("File does not exist; giving up on adding it to the filemap: $file"); - return; - } + # If the id already exists on the server, throw a warning about duplicates + if ( $redis_cfg->hexists( "LRR_FILEMAP", $file ) ) { - # If the id already exists on the server, throw a warning about duplicates - if ( $redis_cfg->hexists( "LRR_FILEMAP", $file ) ) { + my $filemap_id = $redis_cfg->hget( "LRR_FILEMAP", $file ); - my $filemap_id = $redis_cfg->hget( "LRR_FILEMAP", $file ); + $logger->debug("$file was logged but is already in the filemap!"); - $logger->debug("$file was logged but is already in the filemap!"); + if ( $filemap_id ne $id ) { + $logger->debug("$file has a different ID than the one in the filemap! ($filemap_id)"); + $logger->info("$file has been modified, updating its ID from $filemap_id to $id."); - if ( $filemap_id ne $id ) { - $logger->debug("$file has a different ID than the one in the filemap! ($filemap_id)"); - $logger->info("$file has been modified, updating its ID from $filemap_id to $id."); + change_archive_id( $filemap_id, $id ); - change_archive_id( $filemap_id, $id ); + # Don't forget to update the filemap, later operations will behave incorrectly otherwise + $redis_cfg->hset( "LRR_FILEMAP", $file, $id ); + } else { + $logger->debug( + "$file has the same ID as the one in the filemap. Duplicate inotify events? Cleaning cache just to make sure"); + invalidate_cache(); + } - # Don't forget to update the filemap, later operations will behave incorrectly otherwise - $redis_cfg->hset( "LRR_FILEMAP", $file, $id ); - } else { - $logger->debug( - "$file has the same ID as the one in the filemap. Duplicate inotify events? Cleaning cache just to make sure"); - invalidate_cache(); + $dbh->disconnect(); + return; } - return; + # Filename sanity check - check if archive exists in Postgres + my $check_sth = $dbh->prepare('SELECT arcid, filename FROM lrr_archive WHERE arcid = ?'); + $check_sth->execute($id); + my $existing = $check_sth->fetchrow_hashref; + $check_sth->finish; + + if ( $existing ) { + + my $filecheck = $existing->{filename}; + + #Update the real file path and title if they differ from the saved one + #This is meant to always track the current filename for the OS. + unless ( $file eq $filecheck ) { + $logger->debug("File name discrepancy detected between DB and filesystem!"); + $logger->debug("Filesystem: $file"); + $logger->debug("Database: $filecheck"); + my ( $name, $path, $suffix ) = fileparse( $file, qr/\.[^.]*/ ); + + # Wrap multi-statement update in transaction + eval { + $dbh->begin_work; + + # Update filename and title in Postgres + my $update_sth = $dbh->prepare('UPDATE lrr_archive SET filename = ?, title = ? WHERE arcid = ?'); + $update_sth->execute($file, $name, $id); + $update_sth->finish; + + $dbh->commit; + + invalidate_cache(); + }; + + if ( my $error = $@ ) { + $logger->error("Error updating filename/title for $id: $error"); + eval { $dbh->rollback }; + } + } + + # Check and set arcsize if not already set + my $arcsize_check_sth = $dbh->prepare('SELECT arcsize FROM lrr_archive WHERE arcid = ?'); + $arcsize_check_sth->execute($id); + my $arcsize_row = $arcsize_check_sth->fetchrow_hashref; + $arcsize_check_sth->finish; + + unless ( $arcsize_row && $arcsize_row->{arcsize} ) { + $logger->debug("arcsize is not set for $id, storing now!"); + add_arcsize_with_dbh( $dbh, $id ); + } + + # Set pagecount in case it's not already there + my $pagecount_check_sth = $dbh->prepare('SELECT pagecount FROM lrr_archive WHERE arcid = ?'); + $pagecount_check_sth->execute($id); + my $pagecount_row = $pagecount_check_sth->fetchrow_hashref; + $pagecount_check_sth->finish; + + unless ( $pagecount_row && $pagecount_row->{pagecount} ) { + $logger->debug("Pagecount not calculated for $id, doing it now!"); + add_pagecount_with_dbh( $dbh, $id ); + } - } else { - $redis_cfg->hset( "LRR_FILEMAP", $file, $id ); # raw FS path so no encoding/decoding whatsoever - } + } else { - # Filename sanity check - if ( $redis_arc->exists($id) ) { - - my $filecheck = get_archive_path( $redis_arc, $id ); - - #Update the real file path and title if they differ from the saved one - #This is meant to always track the current filename for the OS. - unless ( $file eq $filecheck ) { - $logger->debug("File name discrepancy detected between DB and filesystem!"); - $logger->debug("Filesystem: $file"); - $logger->debug("Database: $filecheck"); - my ( $name, $path, $suffix ) = fileparse( $file, qr/\.[^.]*/ ); - $redis_arc->hset( $id, "file", $file ); - $redis_arc->hset( $id, "name", redis_encode($name) ); - $redis_arc->wait_all_responses; + # Add to Postgres if not present beforehand + add_new_file( $id, $file ); invalidate_cache(); } - unless ( get_arcsize( $redis_arc, $id ) ) { - $logger->debug("arcsize is not set for $id, storing now!"); - add_arcsize( $redis_arc, $id ); - } - - # Set pagecount in case it's not already there - unless ( $redis_arc->hget( $id, "pagecount" ) ) { - $logger->debug("Pagecount not calculated for $id, doing it now!"); - add_pagecount( $redis_arc, $id ); - } - } else { - - # Signal that this is a new file; caller will handle add_new_file outside the lock. - return 1; + $logger->debug("$file not recognized as archive, skipping."); } - - return 0; + $dbh->disconnect(); } # Only handle new files. As per the ChangeNotify doc, it @@ -327,8 +340,8 @@ sub new_file_callback ($name) { eval { add_to_filemap( $redis, $name ); }; $redis->quit(); - if ($@) { - $logger->error("Error while handling new file: $@"); + if ( my $error = $@ ) { + $logger->error("Error while handling new file: $error"); } } } @@ -360,8 +373,8 @@ sub add_new_files (@files) { # Individual files are also eval'd so we can keep scanning eval { add_to_filemap( $redis, $file ); }; - if ($@) { - $logger->error("Error scanning $file: $@"); + if ( my $error = $@ ) { + $logger->error("Error scanning $file: $error"); } } @@ -371,28 +384,39 @@ sub add_new_files (@files) { sub add_new_file ( $id, $file ) { - my $redis = LANraragi::Model::Config->get_redis; - my $redis_search = LANraragi::Model::Config->get_redis_search; + my $dbh = get_postgresql_dbh(); + unless ($dbh) { + $logger->error("Failed to connect to PostgreSQL database"); + return; + } + $logger->info("Adding new file $file with ID $id"); eval { - add_archive_to_redis( $id, $file, $redis, $redis_search ); - add_timestamp_tag( $redis, $id ); - add_pagecount( $redis, $id ); + # Add archive to Postgres (equivalent to add_archive_to_redis) + add_archive_to_postgres( $id, $file, $dbh ); + + # Add timestamp tag (reuse $dbh) + add_timestamp_tag_with_dbh( $dbh, $id ); + + # Add pagecount (reuse $dbh) + add_pagecount_with_dbh( $dbh, $id ); + + # Add arcsize (reuse $dbh) + add_arcsize_with_dbh( $dbh, $id ); # Generate thumbnail my $thumbdir = LANraragi::Model::Config->get_thumbdir; extract_thumbnail( $thumbdir, $id, 1, 1, 1 ); # AutoTagging using enabled plugins goes here! - LANraragi::Model::Plugins::exec_enabled_plugins_on_file($id); + LANraragi::Model::PsilabsDev::PgPlugins::exec_enabled_plugins_on_file($id); }; - if ($@) { - $logger->error("Error while adding file: $@"); + if ( my $error = $@ ) { + $logger->error("Error while adding file: $error"); } - $redis->quit; - $redis_search->quit; + $dbh->disconnect(); } __PACKAGE__->initialize_from_new_process unless caller; diff --git a/tools/build/docker/Dockerfile b/tools/build/docker/Dockerfile index 99d846355..4198ea929 100644 --- a/tools/build/docker/Dockerfile +++ b/tools/build/docker/Dockerfile @@ -3,6 +3,11 @@ FROM alpine:3.23 LABEL git="https://github.com/Difegue/LANraragi" ARG INSTALL_PARAMETER +# Include SQL dependencies +# include with build arguments: `---build-arg INCLUDE_POSTGRESQL=1` or `--build-arg INCLUDE_SQLITE=1`. +ARG INCLUDE_POSTGRESQL=1 +ARG INCLUDE_SQLITE=0 + ENV S6_KEEP_ENV=1 # warn if we can't run stage2 (fix-attrs/cont-init) diff --git a/tools/build/docker/install-everything.sh b/tools/build/docker/install-everything.sh index eddd31a2c..465485eff 100755 --- a/tools/build/docker/install-everything.sh +++ b/tools/build/docker/install-everything.sh @@ -26,6 +26,14 @@ apk add --no-cache ghostscript apk add --no-cache shadow s6 s6-overlay s6-portable-utils procps-ng apk add --no-cache g++ make pkgconf wget curl nodejs npm perl-dev libarchive-dev linux-headers patch +# Add SQL dependencies. +if [ $INCLUDE_POSTGRESQL ]; then + apk add --no-cache postgresql-dev +fi +if [ $INCLUDE_SQLITE ]; then + apk add --no-cache sqlite-dev +fi + # Run it with unlimited jobs to improve performance export MAKEFLAGS="-j" diff --git a/tools/install.pl b/tools/install.pl index 2c894b816..75103c77f 100755 --- a/tools/install.pl +++ b/tools/install.pl @@ -12,6 +12,9 @@ use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); +use constant INCLUDE_POSTGRESQL => ( $ENV{INCLUDE_POSTGRESQL} ); +use constant INCLUDE_SQLITE => ( $ENV{INCLUDE_SQLITE}); + #Vendor dependencies my @vendor_css = ( "/blueimp-file-upload/css/jquery.fileupload.css", @@ -172,6 +175,19 @@ install_package( "Win32API::File", $cpanopt ); } + # Build and install SQL dependencies. + if ( INCLUDE_POSTGRESQL ) { + say("Installing dependencies for PostgreSQL... (This will do nothing if the package is there already)"); + install_package( "DBD::Pg", $cpanopt ); + install_package( "DBI", $cpanopt ); + + } + if ( INCLUDE_SQLITE ) { + say("Installing dependencies for Sqlite... (This will do nothing if the package is there already)"); + install_package( "DBD::Sqlite", $cpanopt ); + install_package( "DBI", $cpanopt ); + } + if ( system( "cpanm --installdeps ./tools/. --notest" . $cpanopt ) != 0 ) { die "Something went wrong while installing Perl modules - Bailing out."; } From 3c8814e6d8cc6ff7fb884047263dbc9a58b372a0 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Mon, 16 Mar 2026 23:03:05 -0700 Subject: [PATCH 02/10] add postgres flag --- lib/LANraragi/Controller/Api/Category.pm | 5 +++-- lib/LANraragi/Model/PsilabsDev/PgSearch.pm | 8 ++++---- lib/LANraragi/Utils/PsilabsDev/Postgres.pm | 17 +++++++++++++++-- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Category.pm b/lib/LANraragi/Controller/Api/Category.pm index 1b1c2375b..f4b1b0657 100644 --- a/lib/LANraragi/Controller/Api/Category.pm +++ b/lib/LANraragi/Controller/Api/Category.pm @@ -113,10 +113,11 @@ sub add_to_category { if ($result) { my $successMessage = "Added $arcid to Category $catid!"; + my %category = LANraragi::Model::PsilabsDev::PgCategory::get_category($catid); my $title = LANraragi::Model::PsilabsDev::PgArchive::get_title($arcid); - if ( defined($title) ) { - $successMessage = "Added \"$title\" to category $catid!"; + if ( %category && defined($title) ) { + $successMessage = "Added \"$title\" to category \"$category{name}\"!"; } render_api_response( $self, "add_to_category", undef, $successMessage ); diff --git a/lib/LANraragi/Model/PsilabsDev/PgSearch.pm b/lib/LANraragi/Model/PsilabsDev/PgSearch.pm index 374f2c666..e43082b2e 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgSearch.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgSearch.pm @@ -217,7 +217,7 @@ sub search_postgres_with_dbh ( $dbh, $category_id, $filter, $sortkey, $sortorder SELECT 1 FROM lrr_archive_to_tag_map atm JOIN lrr_tag t ON atm.tagid = t.tagid WHERE atm.arcid = a.arcid - AND t.namespace = ? + AND LOWER(t.namespace) = ? )"; if ($isneg) { $tag_clause = "NOT $tag_clause"; @@ -230,8 +230,8 @@ sub search_postgres_with_dbh ( $dbh, $category_id, $filter, $sortkey, $sortorder SELECT 1 FROM lrr_archive_to_tag_map atm JOIN lrr_tag t ON atm.tagid = t.tagid WHERE atm.arcid = a.arcid - AND t.namespace = ? - AND t.value = ? + AND LOWER(t.namespace) = ? + AND LOWER(t.value) = ? )"; if ($isneg) { $tag_clause = "NOT $tag_clause"; @@ -266,7 +266,7 @@ sub search_postgres_with_dbh ( $dbh, $category_id, $filter, $sortkey, $sortorder SELECT 1 FROM lrr_archive_to_tag_map atm JOIN lrr_tag t ON atm.tagid = t.tagid WHERE atm.arcid = a.arcid - AND t.value = ? + AND LOWER(t.value) = ? ) )"; if ($isneg) { diff --git a/lib/LANraragi/Utils/PsilabsDev/Postgres.pm b/lib/LANraragi/Utils/PsilabsDev/Postgres.pm index 922efb8ce..5d4982253 100644 --- a/lib/LANraragi/Utils/PsilabsDev/Postgres.pm +++ b/lib/LANraragi/Utils/PsilabsDev/Postgres.pm @@ -69,7 +69,7 @@ CREATE TABLE IF NOT EXISTS lrr_archive ( lastreadtime INTEGER NOT NULL, pagecount INTEGER NOT NULL, progress INTEGER NOT NULL, - title VARCHAR(255) $collate_clause NOT NULL, + title TEXT $collate_clause NOT NULL, summary TEXT, thumbhash VARCHAR(255), arcsize BIGINT @@ -189,7 +189,7 @@ SQL CREATE TABLE IF NOT EXISTS lrr_toc ( arcid VARCHAR(255) NOT NULL, page INTEGER NOT NULL, - title VARCHAR(255) NOT NULL, + title TEXT NOT NULL, FOREIGN KEY (arcid) REFERENCES lrr_archive (arcid), UNIQUE (arcid, page) ) @@ -395,6 +395,19 @@ SQL } $logger->info("Created index: idx_lrr_tag_namespace_value"); + # Functional index for case-insensitive exact match (LOWER() in PgSearch). + # Matches Redis behavior where INDEX keys are lowercased via lc(). + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_tag_lower_namespace_value ON lrr_tag (LOWER(namespace), LOWER(value)) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_tag_lower_namespace_value index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_tag_lower_namespace_value"); + $sql = <<'SQL'; CREATE INDEX IF NOT EXISTS idx_lrr_archive_to_tag_arcid_tagid ON lrr_archive_to_tag_map (arcid, tagid) SQL From 75fa1262945891bd8365200a90faf50cc8d2f485 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 18 Mar 2026 23:30:48 -0700 Subject: [PATCH 03/10] refactor for upcoming split --- lib/LANraragi.pm | 15 ++-- lib/LANraragi/Controller/Api/Archive.pm | 6 +- lib/LANraragi/Controller/Batch.pm | 28 ++----- lib/LANraragi/Controller/Duplicates.pm | 29 +------ lib/LANraragi/Model/PsilabsDev/PgArchive.pm | 26 +++---- lib/LANraragi/Model/PsilabsDev/PgBackup.pm | 6 +- lib/LANraragi/Model/PsilabsDev/PgCategory.pm | 20 ++--- lib/LANraragi/Model/PsilabsDev/PgOpds.pm | 6 +- lib/LANraragi/Model/PsilabsDev/PgPlugins.pm | 6 +- lib/LANraragi/Model/PsilabsDev/PgReader.pm | 4 +- lib/LANraragi/Model/PsilabsDev/PgSearch.pm | 4 +- lib/LANraragi/Model/PsilabsDev/PgStats.pm | 10 +-- lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm | 20 ++--- lib/LANraragi/Model/PsilabsDev/PgUpload.pm | 12 +-- lib/LANraragi/Utils/Minion.pm | 10 +-- lib/LANraragi/Utils/PsilabsDev/Database.pm | 41 ++++++++++ lib/LANraragi/Utils/PsilabsDev/PgArchive.pm | 6 +- lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm | 76 +++++++++++++++---- lib/Shinobu.pm | 6 +- 19 files changed, 191 insertions(+), 140 deletions(-) create mode 100644 lib/LANraragi/Utils/PsilabsDev/Database.pm diff --git a/lib/LANraragi.pm b/lib/LANraragi.pm index 93c9679c9..60acde268 100644 --- a/lib/LANraragi.pm +++ b/lib/LANraragi.pm @@ -26,7 +26,7 @@ use LANraragi::Model::Config; use LANraragi::Model::PsilabsDev::PgSetup qw(first_install_actions); use LANraragi::Model::Metrics; -use LANraragi::Utils::PsilabsDev::Postgres; +use LANraragi::Utils::PsilabsDev::Database; use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); @@ -115,17 +115,14 @@ sub startup { # Initialize cache LANraragi::Utils::PageCache::initialize(); - # Initialize Postgres schema - my $dbh = LANraragi::Utils::PsilabsDev::Postgres::get_postgresql_dbh(); + # Initialize database schema eval { - LANraragi::Utils::PsilabsDev::Postgres::initialize_database($dbh); - $dbh->disconnect(); - $self->LRR_LOGGER->info("Initialized Postgres schema."); + LANraragi::Utils::PsilabsDev::Database::initialize(); + $self->LRR_LOGGER->info("Initialized database schema."); } or do { my $error = $@; - $dbh->disconnect(); - $self->LRR_LOGGER->error("Failed to initialize Postgres schema: $error"); - die "Failed to initialize Postgres schema: $error"; + $self->LRR_LOGGER->error("Failed to initialize database schema: $error"); + die "Failed to initialize database schema: $error"; }; # Load i18n diff --git a/lib/LANraragi/Controller/Api/Archive.pm b/lib/LANraragi/Controller/Api/Archive.pm index 0be9becae..913f78afb 100644 --- a/lib/LANraragi/Controller/Api/Archive.pm +++ b/lib/LANraragi/Controller/Api/Archive.pm @@ -17,7 +17,7 @@ use LANraragi::Utils::PsilabsDev::PgDatabase qw(get_archive_json set_isnew); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Redis qw(redis_encode); use LANraragi::Utils::Path qw(compat_path get_archive_path move_path); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::PsilabsDev::PgPath; use LANraragi::Utils::Login qw(is_logged_in_api); @@ -50,7 +50,7 @@ sub serve_untagged_archivelist { sub serve_metadata { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $arcdata = get_archive_json( $dbh, $id ); $dbh->disconnect(); @@ -102,7 +102,7 @@ sub serve_file { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $file = LANraragi::Utils::PsilabsDev::PgPath::get_archive_path( $dbh, $id ); $dbh->disconnect(); diff --git a/lib/LANraragi/Controller/Batch.pm b/lib/LANraragi/Controller/Batch.pm index 76bbc4fe8..e13726945 100644 --- a/lib/LANraragi/Controller/Batch.pm +++ b/lib/LANraragi/Controller/Batch.pm @@ -7,10 +7,10 @@ use Mojo::JSON qw(decode_json); use LANraragi::Utils::Generic qw(generate_themes_header exec_with_lock_pure); use LANraragi::Utils::Tags qw(rewrite_tags build_tag_replace_hash split_tags_to_array restore_CRLF); use LANraragi::Utils::Database qw(get_computed_tagrules); -use LANraragi::Utils::PsilabsDev::PgDatabase qw(set_tags set_title set_summary set_tags_with_dbh set_title_with_dbh set_summary_with_dbh set_isnew invalidate_cache); +use LANraragi::Utils::PsilabsDev::PgDatabase qw(set_tags set_title set_summary set_tags_with_dbh set_title_with_dbh set_summary_with_dbh set_isnew invalidate_cache get_tags_string_with_dbh); use LANraragi::Utils::Plugins qw(get_plugins get_plugin get_plugin_parameters); use LANraragi::Utils::Logging qw(get_logger); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Model::PsilabsDev::PgCategory; use LANraragi::Utils::PsilabsDev::PgArchive qw(delete_archive); use LANraragi::Model::PsilabsDev::PgPlugins qw(exec_metadata_plugin); @@ -67,20 +67,8 @@ sub socket { my @rules = get_computed_tagrules(); my ( $rules, $hash_replace_rules ) = build_tag_replace_hash( \@rules ); - # Prepare database connection and statement ONCE at WebSocket connection time - my $dbh = get_postgresql_dbh(); - my $tag_fetch_sth = $dbh->prepare(q{ - SELECT string_agg( - CASE - WHEN t.namespace = '' THEN t.value - ELSE t.namespace || ':' || t.value - END, - ', ' - ) as tags - FROM lrr_archive_to_tag_map atm - JOIN lrr_tag t ON atm.tagid = t.tagid - WHERE atm.arcid = ? - }); + # Open database connection at WebSocket connection time (customs border) + my $dbh = get_dbh(); $self->on( message => sub { @@ -174,10 +162,7 @@ sub socket { eval { $dbh->begin_work; - # REUSE prepared statement instead of creating new connection per message - $tag_fetch_sth->execute($id); - my $row = $tag_fetch_sth->fetchrow_hashref; - my $tags = $row ? $row->{tags} : ""; + my $tags = get_tags_string_with_dbh($dbh, $id); my @tagarray = split_tags_to_array($tags); @tagarray = rewrite_tags( \@tagarray, $rules, $hash_replace_rules ); @@ -256,7 +241,6 @@ sub socket { $cancelled = 1; # Clean up on WebSocket close - $tag_fetch_sth->finish if $tag_fetch_sth; $dbh->disconnect if $dbh; } ); @@ -272,7 +256,7 @@ sub batch_plugin { # If the plugin exec returned tags, add them unless ( exists $plugin_result{error} ) { # Wrap all metadata updates in a single transaction for atomicity - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $dbh->begin_work; eval { diff --git a/lib/LANraragi/Controller/Duplicates.pm b/lib/LANraragi/Controller/Duplicates.pm index fea88a3c6..c0fb429f8 100644 --- a/lib/LANraragi/Controller/Duplicates.pm +++ b/lib/LANraragi/Controller/Duplicates.pm @@ -6,7 +6,8 @@ use POSIX qw(strftime); use Mojo::JSON qw(decode_json encode_json); use LANraragi::Utils::Generic qw(generate_themes_header); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); +use LANraragi::Utils::PsilabsDev::PgDatabase qw(get_archive_summary_with_dbh); use LANraragi::Model::Config; # Go through the archives in the content directory and build the template at the end. @@ -42,27 +43,7 @@ sub index { my $dbh; eval { - $dbh = get_postgresql_dbh(); - - # Prepare statement once before loops for efficiency - my $sth = $dbh->prepare(q{ - SELECT arcid, filename, title, - COALESCE( - (SELECT string_agg( - CASE - WHEN t.namespace = '' THEN t.value - ELSE t.namespace || ':' || t.value - END, - ', ' - ) - FROM lrr_archive_to_tag_map atm - JOIN lrr_tag t ON atm.tagid = t.tagid - WHERE atm.arcid = ?), - '' - ) as tags - FROM lrr_archive - WHERE arcid = ? - }); + $dbh = get_dbh(); foreach my $key ( keys %duplicate_groups ) { @@ -72,8 +53,7 @@ sub index { my @archives; foreach my $id (@ids) { - $sth->execute($id, $id); - my $row = $sth->fetchrow_hashref; + my $row = get_archive_summary_with_dbh($dbh, $id); # Check if archive still exists if ($row) { @@ -122,7 +102,6 @@ sub index { push @duplicates, \@archives; } - $sth->finish; $dbh->disconnect(); }; if ( my $error = $@ ) { diff --git a/lib/LANraragi/Model/PsilabsDev/PgArchive.pm b/lib/LANraragi/Model/PsilabsDev/PgArchive.pm index b520fa093..409c66373 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgArchive.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgArchive.pm @@ -10,7 +10,7 @@ use utf8; use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Generic qw(render_api_response); use LANraragi::Utils::String qw(trim trim_CRLF); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::PsilabsDev::PgDatabase; use LANraragi::Utils::Path qw(create_path unlink_path); use LANraragi::Utils::Archive qw(extract_single_file); @@ -27,7 +27,7 @@ use File::Path qw(remove_tree); sub generate_archive_list { my $logger = get_logger( "PgArchive", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my @archives; @@ -116,7 +116,7 @@ sub generate_archive_list { sub get_untagged_archives { my $logger = get_logger( "PgArchive", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my @untagged; @@ -163,7 +163,7 @@ sub get_untagged_archives { sub get_random_archive { my $logger = get_logger( "PgArchive", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $arcid; @@ -220,7 +220,7 @@ sub generate_page_thumbnails { my $format = $use_jxl ? 'jxl' : 'jpg'; # Get the number of pages in the archive from Postgres - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $pages; eval { @@ -322,7 +322,7 @@ sub update_progress { my ($id, $page, $force) = @_; my $logger = get_logger("PgArchive", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $time = time(); my $pagecount; @@ -375,7 +375,7 @@ sub get_page_data ($id, $path) { my $content = fetch($cachekey); if ( !defined($content) ) { # Extract the file from the parent archive if it doesn't exist - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $archive = LANraragi::Utils::PsilabsDev::PgPath::get_archive_path( $dbh, $id ); $dbh->disconnect(); $content = extract_single_file($archive, $path); @@ -438,7 +438,7 @@ sub archive_exists ($id) { return 0; } - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $exists = 0; eval { @@ -472,7 +472,7 @@ sub get_title ($id) { return (); } - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $title; eval { @@ -550,7 +550,7 @@ sub update_metadata { ( $_ = LANraragi::Utils::String::trim_CRLF($_) ) for ( $title, $tags ); # Use a transaction to ensure all metadata updates are atomic - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); eval { $dbh->begin_work; @@ -589,7 +589,7 @@ sub update_metadata { sub delete_archive ($id) { my $logger = get_logger( "PgArchive", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $filename; eval { @@ -690,7 +690,7 @@ sub delete_archive ($id) { # replaces LANraragi::Model::Archive::add_toc_entry sub add_toc_entry ( $id, $page, $title ) { my $logger = get_logger( "PgArchive", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); eval { my $sth = $dbh->prepare(q{ @@ -715,7 +715,7 @@ sub add_toc_entry ( $id, $page, $title ) { # replaces LANraragi::Model::Archive::remove_toc_entry sub remove_toc_entry ( $id, $page ) { my $logger = get_logger( "PgArchive", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); eval { my $sth = $dbh->prepare('DELETE FROM lrr_toc WHERE arcid = ? AND page = ?'); diff --git a/lib/LANraragi/Model/PsilabsDev/PgBackup.pm b/lib/LANraragi/Model/PsilabsDev/PgBackup.pm index 949776c0a..404396256 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgBackup.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgBackup.pm @@ -6,7 +6,7 @@ use utf8; use Mojo::JSON qw(encode_json decode_json); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Model::PsilabsDev::PgTankoubon; use LANraragi::Utils::PsilabsDev::PgDatabase qw(clean_categories_and_tanks); @@ -15,7 +15,7 @@ use LANraragi::Utils::PsilabsDev::PgDatabase qw(clean_categories_and_tanks); # build_backup_JSON() # Goes through the Postgres database and builds a JSON string containing archive metadata. sub build_backup_JSON { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $logger = get_logger("Backup/Restore", "lanraragi"); # Basic structure of the backup object @@ -157,7 +157,7 @@ sub restore_from_JSON { $logger->info("Received a JSON backup to restore."); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Clean categories and tankoubons before restoring from JSON eval { diff --git a/lib/LANraragi/Model/PsilabsDev/PgCategory.pm b/lib/LANraragi/Model/PsilabsDev/PgCategory.pm index 826652835..3f09c7c05 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgCategory.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgCategory.pm @@ -4,7 +4,7 @@ use strict; use warnings; use utf8; -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Model::Config; @@ -13,7 +13,7 @@ use LANraragi::Model::Config; # Returns a list of all the category objects. sub get_category_list { my $logger = get_logger("PgCategory", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Query for all categories my $cat_sql = <<'SQL'; @@ -80,7 +80,7 @@ SQL sub get_category { my $cat_id = $_[0]; my $logger = get_logger("PgCategory", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); if ( $cat_id eq "" ) { $logger->debug("No category ID provided."); @@ -143,7 +143,7 @@ SQL # Returns a list of all the static category objects. sub get_static_category_list { my $logger = get_logger("PgCategory", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Query for static categories (where search is NULL or empty string) my $cat_sql = <<'SQL'; @@ -209,7 +209,7 @@ sub get_categories_containing_archive { my $logger = get_logger("PgCategory", "lanraragi"); $logger->debug("Finding categories containing $archive_id"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Query for static categories containing the archive my $sql = <<'SQL'; @@ -276,7 +276,7 @@ SQL sub add_to_category { my ( $cat_id, $arc_id ) = @_; my $logger = get_logger("PgCategory", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $err = ""; # Check if category exists @@ -368,7 +368,7 @@ SQL sub create_category { my ( $name, $favtag, $pinned, $cat_id ) = @_; my $logger = get_logger("PgCategory", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Set all fields of the category object unless ( length($cat_id) ) { @@ -447,7 +447,7 @@ sub update_bookmark_link { return (400, $cat_id, "Input category ID is invalid."); } - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Check if category exists using Postgres my $check_sql = 'SELECT catid, search FROM lrr_category WHERE catid = ?'; @@ -500,7 +500,7 @@ sub remove_bookmark_link { sub remove_from_category { my ( $cat_id, $arc_id ) = @_; my $logger = get_logger("PgCategory", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $err = ""; # Check if category exists @@ -565,7 +565,7 @@ sub delete_category { return 0; } - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Check if category exists my $check_sql = 'SELECT catid FROM lrr_category WHERE catid = ?'; diff --git a/lib/LANraragi/Model/PsilabsDev/PgOpds.pm b/lib/LANraragi/Model/PsilabsDev/PgOpds.pm index 969edba94..2be918526 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgOpds.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgOpds.pm @@ -14,7 +14,7 @@ use LANraragi::Utils::PageCache qw(fetch put); use LANraragi::Utils::Resizer qw(get_resizer); use LANraragi::Utils::PsilabsDev::PgDatabase qw(get_archive_json); use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Model::PsilabsDev::PgCategory; use LANraragi::Model::PsilabsDev::PgSearch; @@ -99,7 +99,7 @@ sub generate_opds_item { sub get_opds_data { my $id = shift; - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $file = get_archive_path( $dbh, $id ); unless ( -e $file ) { @@ -153,7 +153,7 @@ sub render_archive_page { my $logger = LANraragi::Utils::Logging::get_logger( "OPDS Page Serving", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $archive = get_archive_path( $dbh, $id ); $dbh->disconnect(); diff --git a/lib/LANraragi/Model/PsilabsDev/PgPlugins.pm b/lib/LANraragi/Model/PsilabsDev/PgPlugins.pm index ba2625521..789ba1e61 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgPlugins.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgPlugins.pm @@ -18,7 +18,7 @@ use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Tags qw(rewrite_tags split_tags_to_array); use LANraragi::Utils::Plugins qw(get_plugin_parameters get_plugin); use LANraragi::Utils::Path qw(create_path); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use Exporter 'import'; our @EXPORT_OK = qw(exec_metadata_plugin exec_enabled_plugins_on_file); @@ -36,7 +36,7 @@ sub exec_metadata_plugin ( $plugin, $id, %args ) { } # Get archive metadata from Postgres instead of Redis - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $sth = $dbh->prepare(q{ SELECT filename, @@ -218,7 +218,7 @@ sub exec_enabled_plugins_on_file ($id) { $successes++; # Create shared database handle for this plugin's atomic updates - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $dbh->begin_work; eval { diff --git a/lib/LANraragi/Model/PsilabsDev/PgReader.pm b/lib/LANraragi/Model/PsilabsDev/PgReader.pm index 530a93053..ced1a11eb 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgReader.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgReader.pm @@ -16,7 +16,7 @@ use LANraragi::Utils::Generic qw(is_image); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Archive qw(get_filelist); use LANraragi::Utils::Redis qw(redis_decode); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::PsilabsDev::PgPath; # replaces LANraragi::Model::Reader::build_reader_JSON @@ -26,7 +26,7 @@ sub build_reader_JSON ( $self, $id, $force ) { # Get the path from Postgres. # Filenames are stored as they are on the OS, so no decoding! - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $archive = LANraragi::Utils::PsilabsDev::PgPath::get_archive_path( $dbh, $id ); # Parse archive to get its list of images diff --git a/lib/LANraragi/Model/PsilabsDev/PgSearch.pm b/lib/LANraragi/Model/PsilabsDev/PgSearch.pm index e43082b2e..b05863440 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgSearch.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgSearch.pm @@ -12,7 +12,7 @@ use Time::HiRes qw(time); use LANraragi::Utils::Generic qw(intersect_arrays); use LANraragi::Utils::String qw(trim); use LANraragi::Utils::Logging qw(get_logger); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Model::Config; use LANraragi::Model::PsilabsDev::PgCategory; @@ -22,7 +22,7 @@ use LANraragi::Model::PsilabsDev::PgCategory; sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks ) { my $logger = get_logger( "PgSearch Engine", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my ( $total, $filtered, @ids ); my $start_time = time(); diff --git a/lib/LANraragi/Model/PsilabsDev/PgStats.pm b/lib/LANraragi/Model/PsilabsDev/PgStats.pm index 2b8703265..19a376496 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgStats.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgStats.pm @@ -4,7 +4,7 @@ use strict; use warnings; use utf8; -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Model::Config; @@ -19,7 +19,7 @@ use LANraragi::Model::Config; # - All archives that are not in any tank sub get_archive_count { my $logger = get_logger("PgStats", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Count non-empty tanks + archives not in any tank (matching Redis LRR_TANKGROUPED semantics) my $sql = <<'SQL'; @@ -73,7 +73,7 @@ sub get_page_stat { # Returns the size in GB (as a decimal number). sub compute_content_size { my $logger = get_logger("PgStats", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Sum all archive sizes my $sql = <<'SQL'; @@ -108,7 +108,7 @@ sub is_url_recorded { my $url = shift; my $logger = get_logger("PgStats", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $logger->debug("Checking if url $url is in the database."); @@ -153,7 +153,7 @@ sub build_tag_stats { $logger->debug("Serving tag statistics with a minimum weight of $minscore"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Count tag occurrences across all archives, filtering by minimum score # Join lrr_tag with lrr_archive_to_tag_map to count how many times each tag appears diff --git a/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm b/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm index f1db05307..9d41009d4 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm @@ -4,7 +4,7 @@ use strict; use warnings; use utf8; -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Model::Config; @@ -15,7 +15,7 @@ sub get_tankoubon_list { my $page = shift // 0; my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Get all tankoubons my $tank_sql = <<'SQL'; @@ -100,7 +100,7 @@ SQL sub create_tankoubon { my ( $name, $tank_id ) = @_; my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Set all fields of the tank object unless ( length($tank_id) ) { @@ -162,7 +162,7 @@ sub get_tankoubon { $page //= 0; my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); if ( $tank_id eq "" ) { $logger->debug("No Tankoubon ID provided."); @@ -330,7 +330,7 @@ sub update_metadata { } my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $err = ""; my $name = $data->{"metadata"}->{"name"} || undef; my $summary = exists $data->{"metadata"}->{"summary"} ? $data->{"metadata"}->{"summary"} : undef; @@ -394,7 +394,7 @@ sub update_archive_list { } my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $err = ""; my @tank_archives = @{ $data->{"archives"} }; @@ -489,7 +489,7 @@ sub get_tankoubons_containing_archive { my ($arcid) = @_; my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); # Check if archive exists my $arc_check_sth = $dbh->prepare('SELECT arcid FROM lrr_archive WHERE arcid = ?'); @@ -533,7 +533,7 @@ sub delete_tankoubon { my ($tank_id) = @_; my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); if ( length($tank_id) != 15 ) { # Probably not a Tankoubon ID @@ -592,7 +592,7 @@ sub add_to_tankoubon { my ( $tank_id, $arc_id ) = @_; my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $err = ""; # Check if tank exists @@ -676,7 +676,7 @@ sub remove_from_tankoubon { my ( $tank_id, $arcid ) = @_; my $logger = get_logger("PgTankoubon", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $err = ""; # Check if tank exists diff --git a/lib/LANraragi/Model/PsilabsDev/PgUpload.pm b/lib/LANraragi/Model/PsilabsDev/PgUpload.pm index 7f5dbdc56..89f0572f7 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgUpload.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgUpload.pm @@ -23,7 +23,7 @@ use LANraragi::Model::PsilabsDev::PgCategory; use LANraragi::Model::PsilabsDev::PgPlugins; use LANraragi::Utils::PsilabsDev::PgArchive qw(extract_thumbnail); use LANraragi::Utils::PsilabsDev::PgDatabase qw(set_tags set_title set_summary invalidate_cache); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); @@ -58,7 +58,7 @@ sub handle_incoming_file ( $tempfile, $catid, $tags, $title, $summary ) { # Check if the ID is already in the database, and # that the file it references still exists on the filesystem - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $replace_dupe = LANraragi::Model::Config->get_replacedupe; my $check_sql = 'SELECT arcid, filename FROM lrr_archive WHERE arcid = ?'; @@ -136,7 +136,7 @@ sub handle_incoming_file ( $tempfile, $catid, $tags, $title, $summary ) { # Now that the file has been copied, we can add the timestamp tag and calculate pagecount. # (The file being physically present is necessary in case last modified time is used) # Reuse database connection and fetch filename once to avoid redundant queries - my $metadata_dbh = get_postgresql_dbh(); + my $metadata_dbh = get_dbh(); add_timestamp_tag_with_dbh( $metadata_dbh, $id, $output_file ); add_pagecount_with_dbh( $metadata_dbh, $id, $output_file ); add_arcsize_with_dbh( $metadata_dbh, $id, $output_file ); @@ -227,7 +227,7 @@ SQL # Helper function: add_timestamp_tag # Adds a timestamp tag to the given ID. sub add_timestamp_tag ( $id ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $result = add_timestamp_tag_with_dbh( $dbh, $id ); $dbh->disconnect(); return $result; @@ -277,7 +277,7 @@ sub add_timestamp_tag_with_dbh ( $dbh, $id, $filepath = undef ) { # Helper function: add_pagecount # Adds pagecount to the archive metadata. sub add_pagecount ( $id ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $result = add_pagecount_with_dbh( $dbh, $id ); $dbh->disconnect(); return $result; @@ -322,7 +322,7 @@ sub add_pagecount_with_dbh ( $dbh, $id, $filepath = undef ) { # Helper function: add_arcsize # Adds archive size to the archive metadata. sub add_arcsize ( $id ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $result = add_arcsize_with_dbh( $dbh, $id ); $dbh->disconnect(); return $result; diff --git a/lib/LANraragi/Utils/Minion.pm b/lib/LANraragi/Utils/Minion.pm index cd09809b4..ad4341f8e 100644 --- a/lib/LANraragi/Utils/Minion.pm +++ b/lib/LANraragi/Utils/Minion.pm @@ -18,7 +18,7 @@ use LANraragi::Utils::Plugins qw(get_downloader_for_url get_plugin get_plugin use LANraragi::Utils::PsilabsDev::PgPlugins qw(use_plugin); use LANraragi::Utils::String qw(trim_url); use LANraragi::Utils::TempFolder qw(get_temp); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::PsilabsDev::PgArchive qw(extract_thumbnail extract_thumbnail_with_dbh); use LANraragi::Model::Upload; @@ -113,7 +113,7 @@ sub add_tasks { $logger->debug("Generating page thumbnails for archive $id..."); # Get the number of pages in the archive from Postgres - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $pages; eval { @@ -208,7 +208,7 @@ sub add_tasks { my ( $thumbdir, $force ) = @args; my $logger = get_logger( "Minion", "minion" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $sth = $dbh->prepare('SELECT arcid FROM lrr_archive'); $sth->execute(); my @keys; @@ -225,7 +225,7 @@ sub add_tasks { my (@keys) = @_; # Each thread/process needs its own database connection - my $dbh_worker = get_postgresql_dbh(); + my $dbh_worker = get_dbh(); foreach my $id (@keys) { @@ -322,7 +322,7 @@ sub add_tasks { $logger->info("Starting find duplicate job (threshold = $threshold)"); # Gather thumbhashes from Postgres instead of Redis - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my %thumbhashes; eval { diff --git a/lib/LANraragi/Utils/PsilabsDev/Database.pm b/lib/LANraragi/Utils/PsilabsDev/Database.pm new file mode 100644 index 000000000..55a05ffdc --- /dev/null +++ b/lib/LANraragi/Utils/PsilabsDev/Database.pm @@ -0,0 +1,41 @@ +package LANraragi::Utils::PsilabsDev::Database; + +use strict; +use warnings; +use utf8; + +use Exporter 'import'; +our @EXPORT_OK = qw(get_dbh initialize BACKEND); + +use constant BACKEND => ( $ENV{LRR_DATABASE_BACKEND} // 'postgres' ); + +# Conditionally load only the selected backend — its dependencies +# (DBD::Pg, DBD::SQLite, etc.) are only pulled in here. +my $_get_dbh; +my $_initialize; + +BEGIN { + if ( BACKEND eq 'postgres' ) { + require LANraragi::Utils::PsilabsDev::Postgres; + $_get_dbh = \&LANraragi::Utils::PsilabsDev::Postgres::get_postgresql_dbh; + $_initialize = \&LANraragi::Utils::PsilabsDev::Postgres::initialize_database; + } else { + die "Unknown database backend: " . BACKEND; + } +} + +sub get_dbh { return $_get_dbh->(); } + +sub initialize { + my $dbh = get_dbh(); + eval { + $_initialize->($dbh); + $dbh->disconnect(); + } or do { + my $error = $@; + $dbh->disconnect(); + die $error; + }; +} + +1; diff --git a/lib/LANraragi/Utils/PsilabsDev/PgArchive.pm b/lib/LANraragi/Utils/PsilabsDev/PgArchive.pm index 8bafb2327..f1e87f20c 100644 --- a/lib/LANraragi/Utils/PsilabsDev/PgArchive.pm +++ b/lib/LANraragi/Utils/PsilabsDev/PgArchive.pm @@ -17,7 +17,7 @@ use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Generic qw(shasum_str); use LANraragi::Utils::Archive qw(get_filelist extract_single_file generate_thumbnail); use LANraragi::Utils::Path qw(unlink_path); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); # replaces LANraragi::Model::Archive::delete_archive @@ -25,7 +25,7 @@ use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); sub delete_archive ($id) { my $logger = get_logger( "Archive", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $filename; # Declare outside transaction @@ -97,7 +97,7 @@ sub delete_archive ($id) { # replaces LANraragi::Utils::Archive::extract_thumbnail # Wrapper function with original signature that manages its own database connection sub extract_thumbnail ( $thumbdir, $id, $page, $set_cover, $use_hq ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $result; eval { $result = extract_thumbnail_with_dbh( $dbh, $thumbdir, $id, $page, $set_cover, $use_hq ); diff --git a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm index d62e20795..dfd989b38 100644 --- a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm +++ b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm @@ -17,7 +17,7 @@ use LANraragi::Utils::Tags qw(split_tags_to_array join_tags_to_string); use LANraragi::Utils::String qw(trim trim_CRLF); use List::MoreUtils qw(uniq); use List::Util qw(max); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::Path; use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); use LANraragi::Utils::PsilabsDev::PgArchive; @@ -28,7 +28,7 @@ use LANraragi::Model::PsilabsDev::PgTankoubon; # Functions for interacting with Postgres. use Exporter 'import'; our @EXPORT_OK = qw( - get_archive get_archive_json get_archive_json_multi set_tags set_tags_with_dbh set_title set_title_with_dbh set_summary set_summary_with_dbh set_isnew clear_new_all invalidate_cache clean_database clean_categories_and_tanks change_archive_id change_archive_id_with_dbh drop_database + get_archive get_archive_json get_archive_json_multi get_tags_string_with_dbh get_archive_summary_with_dbh set_tags set_tags_with_dbh set_title set_title_with_dbh set_summary set_summary_with_dbh set_isnew clear_new_all invalidate_cache clean_database clean_categories_and_tanks change_archive_id change_archive_id_with_dbh drop_database ); # replaces LANraragi::Utils::Database::get_archive @@ -36,7 +36,7 @@ our @EXPORT_OK = qw( # similar to the Redis hgetall structure for compatibility sub get_archive ($id) { my $logger = get_logger( "PgDatabase", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); eval { # Get archive data @@ -97,6 +97,56 @@ sub get_archive ($id) { } } +# Returns the comma-separated tag string for the given archive ID. +# Accepts an existing $dbh from the caller (customs border pattern). +sub get_tags_string_with_dbh ( $dbh, $id ) { + my $sth = $dbh->prepare(q{ + SELECT COALESCE(string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ), '') as tags + FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = ? + }); + $sth->execute($id); + my $row = $sth->fetchrow_hashref; + my $tags = $row->{tags} // ""; + $sth->finish; + return $tags; +} + +# Returns { arcid, filename, title, tags } for the given archive ID, +# suitable for display/summary purposes (e.g. duplicates page). +# Accepts an existing $dbh from the caller (customs border pattern). +sub get_archive_summary_with_dbh ( $dbh, $id ) { + my $sth = $dbh->prepare(q{ + SELECT arcid, filename, title, + COALESCE( + (SELECT string_agg( + CASE + WHEN t.namespace = '' THEN t.value + ELSE t.namespace || ':' || t.value + END, + ', ' + ) + FROM lrr_archive_to_tag_map atm + JOIN lrr_tag t ON atm.tagid = t.tagid + WHERE atm.arcid = ?), + '' + ) as tags + FROM lrr_archive + WHERE arcid = ? + }); + $sth->execute($id, $id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + return $row; +} + # Internal function for building an archive JSON from Postgres data. # This is similar to LANraragi::Utils::Database::build_json but does NOT apply # redis_decode to text fields. Postgres data with client_encoding=UTF8 is already @@ -286,7 +336,7 @@ sub build_tank_json_pg ($id) { # Builds JSON objects for multiple archives and returns them as an array. sub get_archive_json_multi (@ids) { my $logger = get_logger( "PgDatabase", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my @archives; @@ -408,7 +458,7 @@ sub get_archive_json_multi (@ids) { # replaces LANraragi::Utils::Database::set_title sub set_title ( $id, $newtitle ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $dbh->begin_work; eval { @@ -448,7 +498,7 @@ sub set_title_with_dbh ( $dbh, $id, $newtitle ) { # Set $tags for the archive with id $id. # Set $append to 1 if you want to append the tags instead of replacing them. sub set_tags ( $id, $newtags, $append = 0 ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $dbh->begin_work; eval { @@ -573,7 +623,7 @@ sub set_tags_with_dbh ( $dbh, $id, $newtags, $append = 0 ) { # replaces LANraragi::Utils::Database::set_summary sub set_summary ( $id, $summary ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $dbh->begin_work; eval { @@ -616,7 +666,7 @@ sub set_isnew ( $id, $isnew ) { # Convert "false" to false boolean, everything else to true my $newval = $isnew ne "false" ? 1 : 0; - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); eval { my $sth = $dbh->prepare('UPDATE lrr_archive SET isnew = ? WHERE arcid = ?'); @@ -639,7 +689,7 @@ sub set_isnew ( $id, $isnew ) { # Clear the new flag in all archives. sub clear_new_all { my $logger = get_logger( "PgDatabase", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); eval { # Set isnew to false for all archives @@ -672,7 +722,7 @@ sub invalidate_cache ( $rebuild_indexes = 0 ) { # Also updates the filemap in Redis (still used by Shinobu for file tracking). sub change_archive_id ( $old_id, $new_id ) { my $logger = get_logger( "PgDatabase", "lanraragi" ); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $logger->debug("Changing ID $old_id to $new_id"); @@ -747,7 +797,7 @@ sub change_archive_id ( $old_id, $new_id ) { # Note: Archives themselves are NOT deleted - only their metadata relationships. sub clean_categories_and_tanks { my $logger = get_logger("PgDatabase", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $logger->info("Cleaning categories and tankoubons before restore..."); @@ -869,7 +919,7 @@ sub clean_database { my @filemapids = $redis_config->exists("LRR_FILEMAP") ? $redis_config->hvals("LRR_FILEMAP") : (); my %filemap = map { $_ => 1 } @filemapids; - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); my $deleted_arcs = 0; my $unlinked_arcs = 0; @@ -981,7 +1031,7 @@ sub clean_database { # This is extremely dangerous and cannot be undone. sub drop_database { my $logger = get_logger("PgDatabase", "lanraragi"); - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); $logger->warn("Dropping entire database - all Postgres and Redis data will be lost!"); diff --git a/lib/Shinobu.pm b/lib/Shinobu.pm index 721ffcb7c..7f35865b9 100644 --- a/lib/Shinobu.pm +++ b/lib/Shinobu.pm @@ -41,7 +41,7 @@ use LANraragi::Model::Metrics; use LANraragi::Model::PsilabsDev::PgUpload qw(add_timestamp_tag add_pagecount add_arcsize add_timestamp_tag_with_dbh add_pagecount_with_dbh add_arcsize_with_dbh add_archive_to_postgres); use LANraragi::Model::PsilabsDev::PgPlugins; use LANraragi::Utils::PsilabsDev::PgDatabase qw(change_archive_id); -use LANraragi::Utils::PsilabsDev::Postgres qw(get_postgresql_dbh); +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); @@ -185,7 +185,7 @@ sub update_filemap { sub add_to_filemap ( $redis_cfg, $file ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); unless ($dbh) { $logger->error("Failed to connect to PostgreSQL database"); return; @@ -384,7 +384,7 @@ sub add_new_files (@files) { sub add_new_file ( $id, $file ) { - my $dbh = get_postgresql_dbh(); + my $dbh = get_dbh(); unless ($dbh) { $logger->error("Failed to connect to PostgreSQL database"); return; From f8bc98bc5bb96397b178aec72ba4445f10c0059a Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Fri, 20 Mar 2026 13:12:49 -0700 Subject: [PATCH 04/10] cache tag stats with redis otherwise a tag stats call would take 10s on 100k archives/tags --- lib/LANraragi/Controller/Api/Database.pm | 9 +- lib/LANraragi/Model/PsilabsDev/PgStats.pm | 99 ++++++++++++++------ lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm | 5 +- 3 files changed, 82 insertions(+), 31 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Database.pm b/lib/LANraragi/Controller/Api/Database.pm index ff58bc8a9..53333dcf2 100644 --- a/lib/LANraragi/Controller/Api/Database.pm +++ b/lib/LANraragi/Controller/Api/Database.pm @@ -162,7 +162,14 @@ sub serve_tag_stats { @excluded = split( /\s*,\s*/, $self->LRR_CONF->get_excludednamespaces ); } - $self->render( openapi => LANraragi::Model::PsilabsDev::PgStats::build_tag_stats( $minscore, \@excluded ) ); + my ( $result, $is_cached ) = LANraragi::Model::PsilabsDev::PgStats::build_tag_stats( $minscore, \@excluded ); + + if ($is_cached) { + # Raw JSON from Redis — render directly without Perl decode+re-encode + $self->render( data => $result, format => 'json' ); + } else { + $self->render( openapi => $result ); + } } sub clean_database { diff --git a/lib/LANraragi/Model/PsilabsDev/PgStats.pm b/lib/LANraragi/Model/PsilabsDev/PgStats.pm index 19a376496..f0e4bf401 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgStats.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgStats.pm @@ -4,8 +4,11 @@ use strict; use warnings; use utf8; +use Mojo::JSON qw(encode_json decode_json); + use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Redis qw(redis_decode); use LANraragi::Model::Config; # replaces: LANraragi::Model::Stats::get_archive_count @@ -142,61 +145,101 @@ SQL } # replaces: LANraragi::Model::Stats::build_tag_stats -# build_tag_stats($minscore) +# build_tag_stats($minscore, $excluded) # Builds tag statistics for display in the tag cloud. -# In Redis, this queries the LRR_STATS sorted set. -# In Postgres, this counts tag occurrences across all archives. -# Returns a reference to an array of tag objects with text, namespace, and weight fields. +# Results are cached as raw JSON in Redis (search DB) under LRR_TAG_STATS:* +# keyed by query parameters. Cache is invalidated by invalidate_tag_stats_cache(), +# called from PgDatabase::invalidate_cache. +# +# Returns ($json_string, 1) on cache hit — caller renders the raw JSON directly. +# Returns ($arrayref, 0) on cache miss — caller renders via openapi. sub build_tag_stats { my ( $minscore, $excluded ) = @_; my $logger = get_logger("PgStats", "lanraragi"); $logger->debug("Serving tag statistics with a minimum weight of $minscore"); + # Build a cache key from parameters + my $excl_key = join( ',', sort map { lc($_) } @$excluded ); + my $cache_key = "LRR_TAG_STATS:${minscore}:${excl_key}"; + + # Check Redis cache — return raw JSON string on hit to avoid decode+re-encode + my $redis = LANraragi::Model::Config->get_redis_search; + my $cached = $redis->get($cache_key); + $redis->quit(); + + if ($cached) { + $logger->debug("Tag stats cache hit for $cache_key"); + return ( redis_decode($cached), 1 ); + } + + # Cache miss — compute from Postgres my $dbh = get_dbh(); - # Count tag occurrences across all archives, filtering by minimum score - # Join lrr_tag with lrr_archive_to_tag_map to count how many times each tag appears - # Lowercase tags during grouping to match Redis behavior (e.g., "Artist:John" and "artist:john" counted together) - my $sql = <<'SQL'; - SELECT LOWER(t.namespace) as namespace, LOWER(t.value) as value, COUNT(*) as weight - FROM lrr_tag t - INNER JOIN lrr_archive_to_tag_map atm ON t.tagid = atm.tagid - GROUP BY LOWER(t.namespace), LOWER(t.value) + my @params; + + my $exclude_clause = ""; + if ( @$excluded ) { + my @placeholders = map { '?' } @$excluded; + $exclude_clause = "WHERE LOWER(atm.namespace) NOT IN (" . join( ', ', @placeholders ) . ")"; + push @params, map { lc($_) } @$excluded; + } + + # HAVING ? comes after WHERE ? in bind order + push @params, $minscore; + + my $sql = <= ? ORDER BY weight DESC SQL my $sth = $dbh->prepare($sql); - $sth->execute($minscore); + $sth->execute(@params); - # Build the array of tag objects my @tags; while (my $row = $sth->fetchrow_hashref) { - my $namespace = $row->{namespace} || ""; my $value = $row->{value} || ""; - my $weight = $row->{weight} || 0; - - # Skip empty values next if $value eq ""; - # Skip excluded namespaces if provided - next if @$excluded && grep { lc($_) eq $namespace } @$excluded; - - my $j = { - text => $value, - namespace => $namespace, - weight => $weight + 0 # Ensure it's treated as a number + push @tags, { + text => $value, + namespace => $row->{namespace} || "", + weight => $row->{weight} + 0 }; - push(@tags, $j); } $sth->finish; $dbh->disconnect(); - $logger->debug("Returning " . scalar(@tags) . " tags with minimum weight $minscore"); + # Store rendered JSON in Redis cache + my $json = encode_json( \@tags ); + $redis = LANraragi::Model::Config->get_redis_search; + $redis->set( $cache_key, $json ); + $redis->quit(); + + $logger->debug("Returning " . scalar(@tags) . " tags (cached under $cache_key)"); - return \@tags; + return ( \@tags, 0 ); +} + +# invalidate_tag_stats_cache() +# Deletes all LRR_TAG_STATS:* keys from the Redis search DB. +# Called by PgDatabase::invalidate_cache on tag writes. +sub invalidate_tag_stats_cache { + my $redis = LANraragi::Model::Config->get_redis_search; + my @keys = $redis->keys("LRR_TAG_STATS:*"); + + if (@keys) { + $redis->del(@keys); + my $logger = get_logger("PgStats", "lanraragi"); + $logger->debug("Invalidated " . scalar(@keys) . " tag stats cache entries"); + } + + $redis->quit(); } # replaces: LANraragi::Model::Stats::build_stat_hashes diff --git a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm index dfd989b38..3418b555e 100644 --- a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm +++ b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm @@ -23,6 +23,7 @@ use LANraragi::Utils::PsilabsDev::PgPath qw(get_archive_path); use LANraragi::Utils::PsilabsDev::PgArchive; use LANraragi::Model::PsilabsDev::PgBackup; use LANraragi::Model::PsilabsDev::PgCategory; +use LANraragi::Model::PsilabsDev::PgStats; use LANraragi::Model::PsilabsDev::PgTankoubon; # Functions for interacting with Postgres. @@ -711,9 +712,9 @@ sub clear_new_all { } # replaces LANraragi::Utils::Database::invalidate_cache -# In Postgres, there's no separate search cache to invalidate. +# Invalidates cached tag statistics so the next call recomputes from Postgres. sub invalidate_cache ( $rebuild_indexes = 0 ) { - return; + LANraragi::Model::PsilabsDev::PgStats::invalidate_tag_stats_cache(); } # replaces LANraragi::Utils::Database::change_archive_id From afe4227ef3d831c3ce65b2f76d9a0b907d189db7 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sat, 21 Mar 2026 00:55:47 -0700 Subject: [PATCH 05/10] purge all redis cache-related logic The cache and cache invalidation logic is incorrect and not well-planned. --- lib/LANraragi/Controller/Api/Database.pm | 10 +--- lib/LANraragi/Model/PsilabsDev/PgArchive.pm | 3 ++ lib/LANraragi/Model/PsilabsDev/PgStats.pm | 52 ++------------------ lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm | 3 +- 4 files changed, 9 insertions(+), 59 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Database.pm b/lib/LANraragi/Controller/Api/Database.pm index 53333dcf2..16987f87a 100644 --- a/lib/LANraragi/Controller/Api/Database.pm +++ b/lib/LANraragi/Controller/Api/Database.pm @@ -162,14 +162,8 @@ sub serve_tag_stats { @excluded = split( /\s*,\s*/, $self->LRR_CONF->get_excludednamespaces ); } - my ( $result, $is_cached ) = LANraragi::Model::PsilabsDev::PgStats::build_tag_stats( $minscore, \@excluded ); - - if ($is_cached) { - # Raw JSON from Redis — render directly without Perl decode+re-encode - $self->render( data => $result, format => 'json' ); - } else { - $self->render( openapi => $result ); - } + my $result = LANraragi::Model::PsilabsDev::PgStats::build_tag_stats( $minscore, \@excluded ); + $self->render( openapi => $result ); } sub clean_database { diff --git a/lib/LANraragi/Model/PsilabsDev/PgArchive.pm b/lib/LANraragi/Model/PsilabsDev/PgArchive.pm index 409c66373..0594532e8 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgArchive.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgArchive.pm @@ -580,6 +580,9 @@ sub update_metadata { $dbh->disconnect; + # Bust cache + LANraragi::Utils::PsilabsDev::PgDatabase::invalidate_cache(); + # No errors. return ""; } diff --git a/lib/LANraragi/Model/PsilabsDev/PgStats.pm b/lib/LANraragi/Model/PsilabsDev/PgStats.pm index f0e4bf401..60c1d1926 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgStats.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgStats.pm @@ -4,12 +4,8 @@ use strict; use warnings; use utf8; -use Mojo::JSON qw(encode_json decode_json); - use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::Logging qw(get_logger); -use LANraragi::Utils::Redis qw(redis_decode); -use LANraragi::Model::Config; # replaces: LANraragi::Model::Stats::get_archive_count # get_archive_count() @@ -147,33 +143,13 @@ SQL # replaces: LANraragi::Model::Stats::build_tag_stats # build_tag_stats($minscore, $excluded) # Builds tag statistics for display in the tag cloud. -# Results are cached as raw JSON in Redis (search DB) under LRR_TAG_STATS:* -# keyed by query parameters. Cache is invalidated by invalidate_tag_stats_cache(), -# called from PgDatabase::invalidate_cache. -# -# Returns ($json_string, 1) on cache hit — caller renders the raw JSON directly. -# Returns ($arrayref, 0) on cache miss — caller renders via openapi. +# Always queries Postgres directly — no caching layer. sub build_tag_stats { my ( $minscore, $excluded ) = @_; my $logger = get_logger("PgStats", "lanraragi"); $logger->debug("Serving tag statistics with a minimum weight of $minscore"); - # Build a cache key from parameters - my $excl_key = join( ',', sort map { lc($_) } @$excluded ); - my $cache_key = "LRR_TAG_STATS:${minscore}:${excl_key}"; - - # Check Redis cache — return raw JSON string on hit to avoid decode+re-encode - my $redis = LANraragi::Model::Config->get_redis_search; - my $cached = $redis->get($cache_key); - $redis->quit(); - - if ($cached) { - $logger->debug("Tag stats cache hit for $cache_key"); - return ( redis_decode($cached), 1 ); - } - - # Cache miss — compute from Postgres my $dbh = get_dbh(); my @params; @@ -215,31 +191,9 @@ SQL $sth->finish; $dbh->disconnect(); - # Store rendered JSON in Redis cache - my $json = encode_json( \@tags ); - $redis = LANraragi::Model::Config->get_redis_search; - $redis->set( $cache_key, $json ); - $redis->quit(); - - $logger->debug("Returning " . scalar(@tags) . " tags (cached under $cache_key)"); - - return ( \@tags, 0 ); -} + $logger->debug("Returning " . scalar(@tags) . " tags"); -# invalidate_tag_stats_cache() -# Deletes all LRR_TAG_STATS:* keys from the Redis search DB. -# Called by PgDatabase::invalidate_cache on tag writes. -sub invalidate_tag_stats_cache { - my $redis = LANraragi::Model::Config->get_redis_search; - my @keys = $redis->keys("LRR_TAG_STATS:*"); - - if (@keys) { - $redis->del(@keys); - my $logger = get_logger("PgStats", "lanraragi"); - $logger->debug("Invalidated " . scalar(@keys) . " tag stats cache entries"); - } - - $redis->quit(); + return \@tags; } # replaces: LANraragi::Model::Stats::build_stat_hashes diff --git a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm index 3418b555e..d86a01bdc 100644 --- a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm +++ b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm @@ -712,9 +712,8 @@ sub clear_new_all { } # replaces LANraragi::Utils::Database::invalidate_cache -# Invalidates cached tag statistics so the next call recomputes from Postgres. +# No-op for Postgres — no search caching layer to invalidate. sub invalidate_cache ( $rebuild_indexes = 0 ) { - LANraragi::Model::PsilabsDev::PgStats::invalidate_tag_stats_cache(); } # replaces LANraragi::Utils::Database::change_archive_id From 8d21d1c8f652c09eea4dfe53d60a8a4fefb71077 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sat, 21 Mar 2026 22:53:30 -0700 Subject: [PATCH 06/10] remove all Routing.pm comments --- lib/LANraragi/Utils/Routing.pm | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/LANraragi/Utils/Routing.pm b/lib/LANraragi/Utils/Routing.pm index f4afa96ab..3ff4b6049 100644 --- a/lib/LANraragi/Utils/Routing.pm +++ b/lib/LANraragi/Utils/Routing.pm @@ -102,11 +102,11 @@ sub apply_routes { $public_routes = $logged_in; } - $public_routes->get('/')->to('index#index'); # TODO: require postgres migration (done) - $public_routes->get('/index')->to('index#index'); # TODO: require postgres migration (done) - $public_routes->get('/random')->to('index#random_archive'); # TODO: require postgres migration (done) - $public_routes->get('/reader')->to('reader#index'); # TODO: require postgres migration (done) - $public_routes->get('/stats')->to('stats#index'); # TODO: require postgres migration (done) + $public_routes->get('/')->to('index#index'); + $public_routes->get('/index')->to('index#index'); + $public_routes->get('/random')->to('index#random_archive'); + $public_routes->get('/reader')->to('reader#index'); + $public_routes->get('/stats')->to('stats#index'); $public_routes->get('/js/i18n.js')->to('i18_n#index'); # Minion Admin UI @@ -130,17 +130,17 @@ sub apply_routes { $logged_in->post('/config/plugins')->to('plugins#save_config'); $logged_in->post('/config/plugins/upload')->to('plugins#process_upload'); - $logged_in->get('/config/categories')->to('category#index'); # TODO: require postgres migration (done) + $logged_in->get('/config/categories')->to('category#index'); - $logged_in->get('/batch')->to('batch#index'); # TODO: require postgres migration (done) - $logged_in->websocket('/batch/socket')->to('batch#socket'); # TODO: require postgres migration (done) + $logged_in->get('/batch')->to('batch#index'); + $logged_in->websocket('/batch/socket')->to('batch#socket'); - $logged_in->get('/edit')->to('edit#index'); # TODO: require postgres migration (done) + $logged_in->get('/edit')->to('edit#index'); - $logged_in->get('/backup')->to('backup#index'); # TODO: require postgres migration (done) + $logged_in->get('/backup')->to('backup#index'); - $logged_in->get('/upload')->to('upload#index'); # TODO: require postgres migration (done) - $logged_in->post('/upload')->to('upload#process_upload'); # TODO: require postgres migration (done) + $logged_in->get('/upload')->to('upload#index'); + $logged_in->post('/upload')->to('upload#process_upload'); $logged_in->get('/logs')->to('logging#index'); $logged_in->get('/logs/general')->to('logging#print_general'); @@ -149,14 +149,14 @@ sub apply_routes { $logged_in->get('/logs/mojo')->to('logging#print_mojo'); $logged_in->get('/logs/redis')->to('logging#print_redis'); - $logged_in->get('/duplicates')->to('duplicates#index'); # TODO: require postgres migration (done) + $logged_in->get('/duplicates')->to('duplicates#index'); # Metrics API (not part of OpenAPI spec, serves Prometheus format) if ( $self->LRR_CONF->enable_metrics ) { $logged_in_api->get('/api/info/metrics')->to('api-metrics#serve_metrics'); } - $search_api->get('/search')->to('api-search#handle_datatables'); # TODO: require postgres migration + $search_api->get('/search')->to('api-search#handle_datatables'); } From 3303c8e15e2c3131044d5e7706481b4ccc25467c Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sat, 21 Mar 2026 23:30:44 -0700 Subject: [PATCH 07/10] remove stale TODO comments --- lib/LANraragi/Utils/PsilabsDev/Postgres.pm | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/LANraragi/Utils/PsilabsDev/Postgres.pm b/lib/LANraragi/Utils/PsilabsDev/Postgres.pm index 5d4982253..37d64608b 100644 --- a/lib/LANraragi/Utils/PsilabsDev/Postgres.pm +++ b/lib/LANraragi/Utils/PsilabsDev/Postgres.pm @@ -14,12 +14,9 @@ use LANraragi::Utils::Logging qw(get_logger); # Get the PostgreSQL database connection. sub get_postgresql_dbh { - # required variables. - # TODO: this needs to be populated but for now we can just go with this. my $dbname = $ENV{LRR_POSTGRES_DB} // 'postgres'; my $host = $ENV{LRR_POSTGRES_HOST} // 'postgres'; my $port = $ENV{LRR_POSTGRES_PORT} // 5432; - # my $options = ''; # TODO: figure out if options is needed. my $username = $ENV{LRR_POSTGRES_USER} // 'postgres'; my $password = $ENV{LRR_POSTGRES_PASSWORD} // 'postgres'; From 9a4dfcd5555b7119a193486f13ad0c2e7d1b2327 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 29 Apr 2026 18:16:07 -0700 Subject: [PATCH 08/10] PgSearch: honor hidecompleted parameter (mirror upstream #547) --- lib/LANraragi/Model/PsilabsDev/PgSearch.pm | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/LANraragi/Model/PsilabsDev/PgSearch.pm b/lib/LANraragi/Model/PsilabsDev/PgSearch.pm index b05863440..1a5b497af 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgSearch.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgSearch.pm @@ -19,7 +19,7 @@ use LANraragi::Model::PsilabsDev::PgCategory; # replaces LANraragi::Model::Search::do_search # Performs a search on the Postgres database. # Returns ($total, $filtered, @ids) -sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks ) { +sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks, $hidecompleted ) { my $logger = get_logger( "PgSearch Engine", "lanraragi" ); my $dbh = get_dbh(); @@ -65,7 +65,7 @@ sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $ my $search_start = time(); ( $filtered, @ids ) = search_postgres_with_dbh( $dbh, $category_id, $filter, $sortkey, $sortorder, - $newonly, $untaggedonly, $grouptanks, + $newonly, $untaggedonly, $grouptanks, $hidecompleted, $use_pagination ? $start : undef, $use_pagination ? $keysperpage : undef ); @@ -89,7 +89,7 @@ sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $ # Main search logic using Postgres # Returns ($filtered_count, @ids) -sub search_postgres_with_dbh ( $dbh, $category_id, $filter, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks, $start, $keysperpage ) { +sub search_postgres_with_dbh ( $dbh, $category_id, $filter, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks, $hidecompleted, $start, $keysperpage ) { my $logger = get_logger( "PgSearch Core", "lanraragi" ); @@ -136,6 +136,11 @@ sub search_postgres_with_dbh ( $dbh, $category_id, $filter, $sortkey, $sortorder push @where_clauses, "a.isnew = TRUE"; } + # Hide completed archives — match upstream's >85% threshold (Model::Search::search_uncached) + if ($hidecompleted) { + push @where_clauses, "NOT (a.pagecount > 0 AND a.progress::float / a.pagecount > 0.85)"; + } + # Untagged filter - archives with no "meaningful" tags # Excludes basic metadata namespaces that don't count as "tagged" # Uses denormalized namespace on tag map — no join to lrr_tag needed. From 1341826d3421575c0cef9d73e420f226d3ceaed7 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Wed, 29 Apr 2026 18:16:22 -0700 Subject: [PATCH 09/10] PgOpds: pass hidecompleted=0 to PgSearch (mirror upstream a3c98b12) --- lib/LANraragi/Model/PsilabsDev/PgOpds.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/LANraragi/Model/PsilabsDev/PgOpds.pm b/lib/LANraragi/Model/PsilabsDev/PgOpds.pm index 2be918526..340d1e0b5 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgOpds.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgOpds.pm @@ -31,8 +31,8 @@ sub generate_opds_catalog { my @cats = LANraragi::Model::PsilabsDev::PgCategory::get_category_list(); # Use the Postgres search engine to get the list of archives to show in the catalog. - # TODO Add tankgroup support to opds? - my ( $total, $filtered, @keys ) = LANraragi::Model::PsilabsDev::PgSearch::do_search( "", $cat_id, $start, "title", 0, 0, 0, 0 ); + # TODO Add tankgroup/hidecompleted support to opds? + my ( $total, $filtered, @keys ) = LANraragi::Model::PsilabsDev::PgSearch::do_search( "", $cat_id, $start, "title", 0, 0, 0, 0, 0 ); my @list = (); From 789384c41a90a30f11cf9dfab48df9966f5c9bc9 Mon Sep 17 00:00:00 2001 From: psilabs-dev <113860476+psilabs-dev@users.noreply.github.com> Date: Sun, 7 Jun 2026 23:34:27 -0700 Subject: [PATCH 10/10] Migrate sync-introduced endpoints to Postgres --- lib/LANraragi/Controller/Api/Archive.pm | 3 - lib/LANraragi/Controller/Api/Database.pm | 3 - lib/LANraragi/Controller/Api/Minion.pm | 3 - lib/LANraragi/Controller/Api/Stamp.pm | 21 +- lib/LANraragi/Controller/Api/Tankoubon.pm | 4 +- lib/LANraragi/Controller/Edit.pm | 17 +- lib/LANraragi/Controller/Logging.pm | 1 - lib/LANraragi/Model/PsilabsDev/PgArchive.pm | 6 + lib/LANraragi/Model/PsilabsDev/PgStamp.pm | 261 ++++++++++++++++++ lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm | 134 ++++++++- lib/LANraragi/Utils/Minion.pm | 33 +-- lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm | 3 + lib/LANraragi/Utils/PsilabsDev/Postgres.pm | 34 ++- 13 files changed, 463 insertions(+), 60 deletions(-) create mode 100644 lib/LANraragi/Model/PsilabsDev/PgStamp.pm diff --git a/lib/LANraragi/Controller/Api/Archive.pm b/lib/LANraragi/Controller/Api/Archive.pm index 913f78afb..9d2b031a8 100644 --- a/lib/LANraragi/Controller/Api/Archive.pm +++ b/lib/LANraragi/Controller/Api/Archive.pm @@ -12,10 +12,8 @@ use File::Temp qw(tempdir tmpnam); use File::Basename; use LANraragi::Utils::Generic qw(render_api_response is_archive get_bytelength exec_with_lock); -use LANraragi::Utils::Database qw(); use LANraragi::Utils::PsilabsDev::PgDatabase qw(get_archive_json set_isnew); use LANraragi::Utils::Logging qw(get_logger); -use LANraragi::Utils::Redis qw(redis_encode); use LANraragi::Utils::Path qw(compat_path get_archive_path move_path); use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); use LANraragi::Utils::PsilabsDev::PgPath; @@ -24,7 +22,6 @@ use LANraragi::Utils::Login qw(is_logged_in_api); use LANraragi::Model::Archive; use LANraragi::Model::Config; -use LANraragi::Model::Reader; use LANraragi::Model::PsilabsDev::PgArchive; use LANraragi::Model::PsilabsDev::PgCategory; use LANraragi::Model::PsilabsDev::PgReader; diff --git a/lib/LANraragi/Controller/Api/Database.pm b/lib/LANraragi/Controller/Api/Database.pm index 16987f87a..0ecc9c940 100644 --- a/lib/LANraragi/Controller/Api/Database.pm +++ b/lib/LANraragi/Controller/Api/Database.pm @@ -1,11 +1,8 @@ package LANraragi::Controller::Api::Database; use Mojo::Base 'Mojolicious::Controller'; -use Redis; use Mojo::JSON qw(decode_json); -use File::Temp qw(tempfile); -use LANraragi::Model::Backup; use LANraragi::Model::PsilabsDev::PgBackup; use LANraragi::Model::PsilabsDev::PgStats; use LANraragi::Utils::Generic qw(render_api_response); diff --git a/lib/LANraragi/Controller/Api/Minion.pm b/lib/LANraragi/Controller/Api/Minion.pm index e9f02c4ba..2e5cd3de3 100644 --- a/lib/LANraragi/Controller/Api/Minion.pm +++ b/lib/LANraragi/Controller/Api/Minion.pm @@ -2,11 +2,8 @@ package LANraragi::Controller::Api::Minion; use Mojo::Base 'Mojolicious::Controller'; use Mojo::JSON qw(encode_json decode_json); -use Redis; -use LANraragi::Model::Stats; use LANraragi::Utils::Generic qw(render_api_response); -use LANraragi::Utils::Plugins qw(get_plugin get_plugins use_plugin); # Returns basic info for the given Minion job id. sub minion_job_status { diff --git a/lib/LANraragi/Controller/Api/Stamp.pm b/lib/LANraragi/Controller/Api/Stamp.pm index 44aa3da30..a8d462efe 100644 --- a/lib/LANraragi/Controller/Api/Stamp.pm +++ b/lib/LANraragi/Controller/Api/Stamp.pm @@ -1,10 +1,7 @@ package LANraragi::Controller::Api::Stamp; use Mojo::Base 'Mojolicious::Controller'; -use Redis; -use Encode; - -use LANraragi::Model::Stamp; +use LANraragi::Model::PsilabsDev::PgStamp; use LANraragi::Utils::Generic qw(render_api_response exec_with_lock exec_with_lock_pure); @@ -13,7 +10,7 @@ sub get_stamp { my $self = shift->openapi->valid_input or return; my $stamp_id = $self->stash('id'); - my ( $stamp, $err ) = LANraragi::Model::Stamp::get_stamp($stamp_id); + my ( $stamp, $err ) = LANraragi::Model::PsilabsDev::PgStamp::get_stamp($stamp_id); unless ($stamp) { render_api_response($self, "get_stamp", "The given stamp does not exist."); @@ -29,7 +26,7 @@ sub get_stamps_by_page { my $id = $self->stash('id'); my $index = $self->stash('index'); - my ( $stamps, $err ) = LANraragi::Model::Stamp::get_stamps_by_page($id, $index); + my ( $stamps, $err ) = LANraragi::Model::PsilabsDev::PgStamp::get_stamps_by_page($id, $index); $self->render( openapi => { result => $stamps } ); } @@ -39,7 +36,7 @@ sub get_stamped_pages { my $self = shift->openapi->valid_input or return; my $id = $self->stash('id'); - my ( $indexes, $err ) = LANraragi::Model::Stamp::get_stamped_pages( $id ); + my ( $indexes, $err ) = LANraragi::Model::PsilabsDev::PgStamp::get_stamped_pages( $id ); $self->render( openapi => { result => $indexes } ); } @@ -62,7 +59,7 @@ sub add_stamp { "update_archive", $id, sub { - my ( $created_id, $err ) = LANraragi::Model::Stamp::add_stamp( $id, $index, $content, $position ); + my ( $created_id, $err ) = LANraragi::Model::PsilabsDev::PgStamp::add_stamp( $id, $index, $content, $position ); if ($created_id) { $self->render( @@ -99,10 +96,10 @@ sub update_stamp { "update_stamp", $stamp_id, sub { - my ( $result, $err ) = LANraragi::Model::Stamp::update_stamp( $stamp_id, $content, $position ); + my ( $result, $err ) = LANraragi::Model::PsilabsDev::PgStamp::update_stamp( $stamp_id, $content, $position ); if ($result) { - my %stamp = LANraragi::Model::Stamp::get_stamp( $stamp_id ); + my %stamp = LANraragi::Model::PsilabsDev::PgStamp::get_stamp( $stamp_id ); my $successMessage = "Updated stamp \"$stamp_id\"!"; render_api_response( $self, "update_stamp", undef, $successMessage ); @@ -119,13 +116,13 @@ sub delete_stamp { my $self = shift->openapi->valid_input or return; my $stamp_id = $self->stash('id'); - my ( $result, $id ) = LANraragi::Model::Stamp::get_stamp_archive_id($stamp_id); + my ( $result, $id ) = LANraragi::Model::PsilabsDev::PgStamp::get_stamp_archive_id($stamp_id); if ( $result ) { my ( $acquired, $response ) = exec_with_lock_pure( [ "archive-write:$id", "stamp-write:$stamp_id" ], sub { - my ( $result, $err ) = LANraragi::Model::Stamp::remove_stamp($stamp_id); + my ( $result, $err ) = LANraragi::Model::PsilabsDev::PgStamp::remove_stamp($stamp_id); if ( $result ) { render_api_response( $self, "delete_stamp" ); diff --git a/lib/LANraragi/Controller/Api/Tankoubon.pm b/lib/LANraragi/Controller/Api/Tankoubon.pm index 77aef7021..0e6e3e928 100644 --- a/lib/LANraragi/Controller/Api/Tankoubon.pm +++ b/lib/LANraragi/Controller/Api/Tankoubon.pm @@ -164,7 +164,7 @@ sub serve_tankoubon_thumbnail { sub update_tankoubon_thumbnail { my $self = shift->openapi->valid_input or return; my $tank_id = $self->stash('id'); - LANraragi::Model::Tankoubon::update_tankoubon_thumbnail( $self, $tank_id ); + LANraragi::Model::PsilabsDev::PgTankoubon::update_tankoubon_thumbnail( $self, $tank_id ); } sub update_tank_progress { @@ -198,7 +198,7 @@ sub update_tank_progress { return; } - my ( $result, $err ) = LANraragi::Model::Tankoubon::update_tank_progress( $tank_id, $page ); + my ( $result, $err ) = LANraragi::Model::PsilabsDev::PgTankoubon::update_tank_progress( $tank_id, $page ); if ($result) { $self->render( diff --git a/lib/LANraragi/Controller/Edit.pm b/lib/LANraragi/Controller/Edit.pm index 0694d61d4..6ea9eff68 100644 --- a/lib/LANraragi/Controller/Edit.pm +++ b/lib/LANraragi/Controller/Edit.pm @@ -8,7 +8,7 @@ use Mojo::Util qw(xml_escape); use LANraragi::Utils::Generic qw(generate_themes_header); use LANraragi::Utils::Plugins qw(get_plugins); -use LANraragi::Model::Tankoubon; +use LANraragi::Model::PsilabsDev::PgTankoubon; use LANraragi::Utils::PsilabsDev::PgDatabase qw(get_archive); sub index { @@ -54,22 +54,21 @@ sub index { sub edit_tankoubon { my ( $self, $id ) = @_; - my %metadata = LANraragi::Model::Tankoubon::fetch_metadata_fields($id); + # full_data is used to get the archive titles for the edit page. get_tankoubon also returns the + # tank's own name/summary/tags, so a separate metadata fetch is unnecessary on the Postgres side. + my ( $total, $filtered, %tank ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon( $id, 1 ); - unless (%metadata) { + unless (%tank) { $self->redirect_to('index'); return; } - # Note: We only use full_data here to get the archive titles. If this proves to be too slow for large tanks, - # we could discard this and do separate API calls to get the titles in the client. - my %tank = LANraragi::Model::Tankoubon::get_tankoubon( $id, 1 ); my @archives = @{ $tank{archives} // [] }; my @full_data = @{ $tank{full_data} // [] }; - my $name = $metadata{name} // ""; - my $tags = $metadata{tags} // ""; - my $summary = $metadata{summary} // ""; + my $name = $tank{name} // ""; + my $tags = $tank{tags} // ""; + my $summary = $tank{summary} // ""; $self->render( template => "edit", diff --git a/lib/LANraragi/Controller/Logging.pm b/lib/LANraragi/Controller/Logging.pm index aa7e0c514..44be1e6ef 100644 --- a/lib/LANraragi/Controller/Logging.pm +++ b/lib/LANraragi/Controller/Logging.pm @@ -1,7 +1,6 @@ package LANraragi::Controller::Logging; use Mojo::Base 'Mojolicious::Controller'; -use Redis; use Encode; use LANraragi::Utils::Generic qw(generate_themes_header); diff --git a/lib/LANraragi/Model/PsilabsDev/PgArchive.pm b/lib/LANraragi/Model/PsilabsDev/PgArchive.pm index 0594532e8..e150166f9 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgArchive.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgArchive.pm @@ -643,6 +643,12 @@ sub delete_archive ($id) { $del_toc_sth->finish; $logger->debug("Deleted ToC entries for archive $id"); + # Delete stamps (no CASCADE -- must precede the archive row delete) + my $del_stamp_sth = $dbh->prepare('DELETE FROM lrr_stamp WHERE arcid = ?'); + $del_stamp_sth->execute($id); + $del_stamp_sth->finish; + $logger->debug("Deleted stamps for archive $id"); + # Finally, delete the archive itself my $del_arc_sql = 'DELETE FROM lrr_archive WHERE arcid = ?'; my $del_arc_sth = $dbh->prepare($del_arc_sql); diff --git a/lib/LANraragi/Model/PsilabsDev/PgStamp.pm b/lib/LANraragi/Model/PsilabsDev/PgStamp.pm new file mode 100644 index 000000000..0f5182eee --- /dev/null +++ b/lib/LANraragi/Model/PsilabsDev/PgStamp.pm @@ -0,0 +1,261 @@ +package LANraragi::Model::PsilabsDev::PgStamp; + +use strict; +use warnings; +use utf8; + +use Time::HiRes qw(time); + +use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); +use LANraragi::Utils::Logging qw(get_logger); + +# Postgres-side equivalent of LANraragi::Model::Stamp. Stamps are per-page annotations stored in the +# lrr_stamp table (stampid, arcid, page, content, position). The stampid keeps the upstream +# "STAMPS__" key format for API compatibility. content/position are stored as +# native UTF-8 (DBD::Pg with pg_enable_utf8) -- no redis_encode/redis_decode on this path. + +# replaces LANraragi::Model::Stamp::get_stamp +# get_stamp(stamp_id) +# Returns (\%stamp, $err) with keys content/position/id, or an empty list if the stamp doesn't exist. +sub get_stamp { + my ($stamp_id) = @_; + + my $logger = get_logger( "PgStamp", "lanraragi" ); + + if ( $stamp_id eq "" ) { + $logger->debug("No stamp ID provided."); + return (); + } + + my $dbh = get_dbh(); + my $sth = $dbh->prepare('SELECT content, position FROM lrr_stamp WHERE stampid = ?'); + $sth->execute($stamp_id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + $dbh->disconnect(); + + unless ($row) { + $logger->warn("$stamp_id doesn't exist in the database!"); + return (); + } + + my %stamp = ( + content => $row->{content} // "", + position => $row->{position} // "", + id => $stamp_id + ); + + return ( \%stamp, "" ); +} + +# replaces LANraragi::Model::Stamp::get_stamps_by_page +# get_stamps_by_page(archive_id, page) +# Returns (\@stamps, $err) -- the stamps attached to the given archive page. +sub get_stamps_by_page { + my ( $archive_id, $index ) = @_; + + my $logger = get_logger( "PgStamp", "lanraragi" ); + my $dbh = get_dbh(); + + unless ( _archive_exists( $dbh, $archive_id ) ) { + $dbh->disconnect(); + my $err = "$archive_id does not exist in the database."; + $logger->error($err); + return ( 0, $err ); + } + + my $sth = $dbh->prepare('SELECT stampid, content, position FROM lrr_stamp WHERE arcid = ? AND page = ?'); + $sth->execute( $archive_id, $index ); + + my @stamps; + while ( my $row = $sth->fetchrow_hashref ) { + push @stamps, + { + content => $row->{content} // "", + position => $row->{position} // "", + id => $row->{stampid} + }; + } + $sth->finish; + $dbh->disconnect(); + + return ( \@stamps, "" ); +} + +# replaces LANraragi::Model::Stamp::get_stamped_pages +# get_stamped_pages(archive_id) +# Returns (\@pages, $err) -- the distinct page numbers that have at least one stamp. +sub get_stamped_pages { + my ($archive_id) = @_; + + my $logger = get_logger( "PgStamp", "lanraragi" ); + my $dbh = get_dbh(); + + unless ( _archive_exists( $dbh, $archive_id ) ) { + $dbh->disconnect(); + my $err = "$archive_id does not exist in the database."; + $logger->error($err); + return ( 0, $err ); + } + + my $sth = $dbh->prepare('SELECT DISTINCT page FROM lrr_stamp WHERE arcid = ?'); + $sth->execute($archive_id); + + my @pages; + while ( my $row = $sth->fetchrow_arrayref ) { + push @pages, $row->[0]; + } + $sth->finish; + $dbh->disconnect(); + + return ( \@pages, "" ); +} + +# replaces LANraragi::Model::Stamp::add_stamp +# add_stamp(archive_id, index, content, position) +# Adds a stamp to the given archive page. Returns ($stampid, $err); ($err is empty on success). +sub add_stamp { + my ( $archive_id, $index, $content, $position ) = @_; + + my $logger = get_logger( "PgStamp", "lanraragi" ); + my $dbh = get_dbh(); + + my $sth = $dbh->prepare('SELECT pagecount FROM lrr_archive WHERE arcid = ?'); + $sth->execute($archive_id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + + unless ($row) { + $dbh->disconnect(); + my $err = "$archive_id does not exist in the database."; + $logger->error($err); + return ( 0, $err ); + } + + my $pagecount = $row->{pagecount} || 0; + unless ( int($index) <= int($pagecount) && int($index) > 0 ) { + $dbh->disconnect(); + my $err = "Page $index out of range."; + $logger->error($err); + return ( 0, $err ); + } + + my $key = "STAMPS_" . $index . "_" . int( time() * 1000 ); + my $check = $dbh->prepare('SELECT 1 FROM lrr_stamp WHERE stampid = ?'); + my $isnewkey = 0; + until ($isnewkey) { + $check->execute($key); + if ( $check->fetchrow_arrayref ) { + $key = "STAMPS_" . $index . "_" . int( time() * 1000 + 1 ); + } else { + $isnewkey = 1; + } + } + $check->finish; + + my $ins = $dbh->prepare('INSERT INTO lrr_stamp (stampid, arcid, page, content, position) VALUES (?, ?, ?, ?, ?)'); + $ins->execute( $key, $archive_id, int($index), $content, $position ); + $ins->finish; + $dbh->disconnect(); + + return ( $key, "" ); +} + +# replaces LANraragi::Model::Stamp::update_stamp +# update_stamp(stamp_id, content, position) +# Updates the stamp's content and/or position (only the defined fields). Returns (1, "") on success, +# (0, error) if the stamp doesn't exist. +sub update_stamp { + my ( $stamp_id, $content, $position ) = @_; + + my $logger = get_logger( "PgStamp", "lanraragi" ); + my $dbh = get_dbh(); + + my $chk = $dbh->prepare('SELECT 1 FROM lrr_stamp WHERE stampid = ?'); + $chk->execute($stamp_id); + my $exists = $chk->fetchrow_arrayref; + $chk->finish; + + unless ($exists) { + $dbh->disconnect(); + my $err = "$stamp_id doesn't exist in the database!"; + $logger->warn($err); + return ( 0, $err ); + } + + if ( defined $position ) { + my $sth = $dbh->prepare('UPDATE lrr_stamp SET position = ? WHERE stampid = ?'); + $sth->execute( $position, $stamp_id ); + $sth->finish; + } + + if ( defined $content ) { + my $sth = $dbh->prepare('UPDATE lrr_stamp SET content = ? WHERE stampid = ?'); + $sth->execute( $content, $stamp_id ); + $sth->finish; + } + + $dbh->disconnect(); + return ( 1, "" ); +} + +# replaces LANraragi::Model::Stamp::remove_stamp +# remove_stamp(stamp_id) +# Removes the stamp. Returns (1, "") on success, (0, error) if the stamp doesn't exist. +sub remove_stamp { + my ($key) = @_; + + my $logger = get_logger( "PgStamp", "lanraragi" ); + my $dbh = get_dbh(); + + my $del = $dbh->prepare('DELETE FROM lrr_stamp WHERE stampid = ?'); + my $rows = $del->execute($key); + $del->finish; + $dbh->disconnect(); + + unless ( $rows && $rows > 0 ) { + my $err = "$key doesn't exist in the database!"; + $logger->warn($err); + return ( 0, $err ); + } + + return ( 1, "" ); +} + +# replaces LANraragi::Model::Stamp::get_stamp_archive_id +# get_stamp_archive_id(stamp_id) +# Returns (1, archive_id) if the stamp exists, otherwise (0, error message). +sub get_stamp_archive_id { + my ($key) = @_; + + my $logger = get_logger( "PgStamp", "lanraragi" ); + my $dbh = get_dbh(); + + my $sth = $dbh->prepare('SELECT arcid FROM lrr_stamp WHERE stampid = ?'); + $sth->execute($key); + my $row = $sth->fetchrow_hashref; + $sth->finish; + $dbh->disconnect(); + + unless ($row) { + my $err = "$key doesn't exist in the database!"; + $logger->warn($err); + return ( 0, $err ); + } + + return ( 1, $row->{arcid} ); +} + +# Returns true if the archive exists. Uses the caller's $dbh (does not disconnect it). +sub _archive_exists { + my ( $dbh, $archive_id ) = @_; + + my $sth = $dbh->prepare('SELECT 1 FROM lrr_archive WHERE arcid = ?'); + $sth->execute($archive_id); + my $exists = $sth->fetchrow_arrayref; + $sth->finish; + + return $exists ? 1 : 0; +} + +1; diff --git a/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm b/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm index 9d41009d4..b1f3ccca9 100644 --- a/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm +++ b/lib/LANraragi/Model/PsilabsDev/PgTankoubon.pm @@ -4,8 +4,13 @@ use strict; use warnings; use utf8; +use File::Path qw(make_path); +use File::Copy qw(copy); + use LANraragi::Utils::PsilabsDev::Database qw(get_dbh); +use LANraragi::Utils::PsilabsDev::PgArchive; use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Generic qw(render_api_response); use LANraragi::Model::Config; # replaces LANraragi::Model::Tankoubon::get_tankoubon_list @@ -19,7 +24,7 @@ sub get_tankoubon_list { # Get all tankoubons my $tank_sql = <<'SQL'; - SELECT tankid, name, summary, tags + SELECT tankid, name, summary, tags, progress FROM lrr_tank ORDER BY tankid SQL @@ -55,6 +60,7 @@ SQL name => $tank_row->{name}, summary => $tank_row->{summary} // '', tags => $tank_row->{tags} // '', + progress => int( $tank_row->{progress} // 0 ), archives => \@archives ); @@ -171,7 +177,7 @@ sub get_tankoubon { } # Check if tank exists and get metadata - my $tank_sql = 'SELECT tankid, name, summary, tags FROM lrr_tank WHERE tankid = ?'; + my $tank_sql = 'SELECT tankid, name, summary, tags, progress FROM lrr_tank WHERE tankid = ?'; my $tank_sth = $dbh->prepare($tank_sql); $tank_sth->execute($tank_id); my $tank_row = $tank_sth->fetchrow_hashref; @@ -185,10 +191,11 @@ sub get_tankoubon { # Build base metadata hash my %tank = ( - id => $tank_id, - name => $tank_row->{name}, - summary => $tank_row->{summary} // '', - tags => $tank_row->{tags} // '' + id => $tank_id, + name => $tank_row->{name}, + summary => $tank_row->{summary} // '', + tags => $tank_row->{tags} // '', + progress => int( $tank_row->{progress} // 0 ) ); # Get total count of archives in this tankoubon @@ -318,6 +325,121 @@ sub get_tankoubon { return ( $total, $filtered, %tank ); } +# replaces: LANraragi::Model::Tankoubon::update_tank_progress +# update_tank_progress(tankoubonid, page) +# Saves the given page number as the Tankoubon's reading progress. +# Returns (1, "") on success, (0, error) if the tankoubon does not exist. +sub update_tank_progress { + my ( $tank_id, $page ) = @_; + + my $logger = get_logger("PgTankoubon", "lanraragi"); + my $dbh = get_dbh(); + + my $update_sth = $dbh->prepare('UPDATE lrr_tank SET progress = ? WHERE tankid = ?'); + my $rows = $update_sth->execute( int($page), $tank_id ); + $update_sth->finish; + + $dbh->disconnect(); + + unless ( $rows && $rows > 0 ) { + my $err = "$tank_id doesn't exist in the database!"; + $logger->warn($err); + return ( 0, $err ); + } + + $logger->debug("Updated progress for tankoubon $tank_id to page $page"); + return ( 1, "" ); +} + +# get_first_archive_of_tank(tankoubonid) +# Returns the arcid of the first archive (lowest position) in the Tankoubon, or undef if empty. +sub get_first_archive_of_tank { + my ($tank_id) = @_; + + my $dbh = get_dbh(); + my $sth = $dbh->prepare('SELECT arcid FROM lrr_tank_to_archive_map WHERE tankid = ? ORDER BY position LIMIT 1'); + $sth->execute($tank_id); + my $row = $sth->fetchrow_hashref; + $sth->finish; + $dbh->disconnect(); + + return $row ? $row->{arcid} : undef; +} + +# replaces: LANraragi::Model::Tankoubon::translate_global_page +# translate_global_page(tankoubonid, global_page) +# Translates a tank-global page number to (arcid, local_page) by walking member archives in order. +# Returns an empty list if the page is out of range or if member archives have no pagecount data. +sub translate_global_page { + my ( $tank_id, $global_page ) = @_; + + my $dbh = get_dbh(); + my $sth = $dbh->prepare( + 'SELECT m.arcid AS arcid, a.pagecount AS pagecount FROM lrr_tank_to_archive_map m ' + . 'JOIN lrr_archive a ON a.arcid = m.arcid WHERE m.tankid = ? ORDER BY m.position' ); + $sth->execute($tank_id); + + my $offset = 0; + while ( my $row = $sth->fetchrow_hashref ) { + my $pagecount = $row->{pagecount} || 0; + if ( $global_page <= $offset + $pagecount ) { + $sth->finish; + $dbh->disconnect(); + return ( $row->{arcid}, $global_page - $offset ); + } + $offset += $pagecount; + } + $sth->finish; + $dbh->disconnect(); + + return (); +} + +# replaces: LANraragi::Model::Tankoubon::update_tankoubon_thumbnail +# update_tankoubon_thumbnail(self, tankoubonid) +# Sets the Tankoubon's cover thumbnail from a global page number (read from the request) spanning +# its member archives. +sub update_tankoubon_thumbnail { + my ( $self, $tank_id ) = @_; + + my $page = $self->req->param('page'); + $page = 1 unless $page; + + my $logger = get_logger( "PgTankoubon", "lanraragi" ); + my $thumbdir = LANraragi::Model::Config->get_thumbdir; + my $use_jxl = LANraragi::Model::Config->get_jxlthumbpages; + my $format = $use_jxl ? 'jxl' : 'jpg'; + + my ( $arc_id, $local_page ) = translate_global_page( $tank_id, $page ); + + unless ( defined $arc_id ) { + render_api_response( $self, "update_tankoubon_thumbnail", "Page $page is out of range for this tankoubon." ); + return; + } + + my $tank_thumb = "$thumbdir/TA/$tank_id.$format"; + + eval { + my $newthumb = LANraragi::Utils::PsilabsDev::PgArchive::extract_thumbnail( $thumbdir, $arc_id, $local_page, 0, 1 ); + make_path("$thumbdir/TA") unless -d "$thumbdir/TA"; + copy( $newthumb, $tank_thumb ); + }; + if ($@) { + render_api_response( $self, "update_tankoubon_thumbnail", $@ ); + return; + } + + $logger->debug("Set tank $tank_id thumbnail from archive $arc_id page $local_page"); + + $self->render( + openapi => { + operation => "update_tankoubon_thumbnail", + new_thumbnail => $tank_thumb, + success => 1 + } + ); +} + # replaces: LANraragi::Model::Tankoubon::update_metadata # update_metadata(tankoubonid, data) # Updates the metadata in the Tankoubon. diff --git a/lib/LANraragi/Utils/Minion.pm b/lib/LANraragi/Utils/Minion.pm index ad4341f8e..b66033c3c 100644 --- a/lib/LANraragi/Utils/Minion.pm +++ b/lib/LANraragi/Utils/Minion.pm @@ -23,10 +23,10 @@ use LANraragi::Utils::PsilabsDev::PgArchive qw(extract_thumbnail extract_thumbna use LANraragi::Model::Upload; use LANraragi::Model::Config; -use LANraragi::Model::Stats; -use LANraragi::Model::Backup; +use LANraragi::Model::PsilabsDev::PgBackup; use LANraragi::Model::PsilabsDev::PgStats; use LANraragi::Model::PsilabsDev::PgUpload; +use LANraragi::Model::PsilabsDev::PgTankoubon; use constant IS_UNIX => ( $Config{osname} ne 'MSWin32' ); @@ -65,17 +65,13 @@ sub add_tasks { my $logger = get_logger( "Minion", "minion" ); - my $redis = LANraragi::Model::Config->get_redis; - my @archives = $redis->zrangebyscore( $tank_id, 1, "+inf", "LIMIT", 0, 1 ); - $redis->quit; + my $first_arc = LANraragi::Model::PsilabsDev::PgTankoubon::get_first_archive_of_tank($tank_id); - unless (@archives) { + unless ($first_arc) { $logger->info("Tank $tank_id has no archives, skipping thumbnail generation."); $job->finish("No archives in tank."); return; } - - my $first_arc = $archives[0]; my $thumbname = ""; eval { @@ -274,13 +270,12 @@ sub add_tasks { } # Regen thumbnails for all tankoubons (sequential - fewer items than archives) - my $redis_tank = LANraragi::Model::Config->get_redis; - my @tank_keys = $redis_tank->keys('TANK_??????????'); - $redis_tank->quit(); + my ( $tank_total, $tank_filtered, @tanks ) = LANraragi::Model::PsilabsDev::PgTankoubon::get_tankoubon_list(-1); - $logger->info("Regenerating thumbnails for " . scalar(@tank_keys) . " tankoubons..."); + $logger->info("Regenerating thumbnails for " . scalar(@tanks) . " tankoubons..."); - foreach my $tank_id (@tank_keys) { + foreach my $tank (@tanks) { + my $tank_id = $tank->{id}; my $use_jxl = LANraragi::Model::Config->get_jxlthumbpages; my $format = $use_jxl ? 'jxl' : 'jpg'; my $subfolder = substr( $tank_id, 0, 2 ); @@ -290,12 +285,10 @@ sub add_tasks { eval { $logger->debug("Regenerating tank thumbnail for $tank_id..."); - my $redis_t = LANraragi::Model::Config->get_redis; - my @archives = $redis_t->zrangebyscore( $tank_id, 1, "+inf", "LIMIT", 0, 1 ); - $redis_t->quit; + my $first_arc = $tank->{archives}[0]; - if (@archives) { - my $src = extract_thumbnail( $thumbdir, $archives[0], 0, 0, 1 ); + if ($first_arc) { + my $src = extract_thumbnail( $thumbdir, $first_arc, 0, 0, 1 ); make_path("$thumbdir/$subfolder"); copy( $src, $thumbname ) or die "Could not copy tank thumbnail: $!"; } @@ -621,7 +614,7 @@ sub add_tasks { eval { # Generate the backup JSON with progress reporting - my $json = LANraragi::Model::Backup::build_backup_JSON($job); + my $json = LANraragi::Model::PsilabsDev::PgBackup::build_backup_JSON(); # Write JSON to temp file my $tempdir = get_temp(); @@ -661,7 +654,7 @@ sub add_tasks { eval { # Restore from JSON with progress reporting - LANraragi::Model::Backup::restore_from_JSON( $json_data, $job ); + LANraragi::Model::PsilabsDev::PgBackup::restore_from_JSON($json_data); $logger->info("Backup restored successfully"); diff --git a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm index d86a01bdc..4b0d67ddb 100644 --- a/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm +++ b/lib/LANraragi/Utils/PsilabsDev/PgDatabase.pm @@ -1053,6 +1053,9 @@ sub drop_database { $dbh->do('DELETE FROM lrr_toc'); $logger->debug("Cleared ToC entries"); + $dbh->do('DELETE FROM lrr_stamp'); + $logger->debug("Cleared stamps"); + # Then delete the main tables $dbh->do('DELETE FROM lrr_tag'); $logger->debug("Cleared tags"); diff --git a/lib/LANraragi/Utils/PsilabsDev/Postgres.pm b/lib/LANraragi/Utils/PsilabsDev/Postgres.pm index 37d64608b..cf354bd16 100644 --- a/lib/LANraragi/Utils/PsilabsDev/Postgres.pm +++ b/lib/LANraragi/Utils/PsilabsDev/Postgres.pm @@ -101,7 +101,8 @@ CREATE TABLE IF NOT EXISTS lrr_tank ( tankid VARCHAR(255) PRIMARY KEY, name VARCHAR(255) $collate_clause NOT NULL, summary TEXT, - tags TEXT + tags TEXT, + progress INTEGER NOT NULL DEFAULT 0 ) SQL $rv = $dbh->do($sql); @@ -199,6 +200,24 @@ SQL } $logger->info("Created table: lrr_toc"); + $sql = <<'SQL'; +CREATE TABLE IF NOT EXISTS lrr_stamp ( + stampid VARCHAR(255) PRIMARY KEY, + arcid VARCHAR(255) NOT NULL, + page INTEGER NOT NULL, + content TEXT, + position TEXT, + FOREIGN KEY (arcid) REFERENCES lrr_archive (arcid) +) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create lrr_stamp table: $errorcode - $errorstr"; + } + $logger->info("Created table: lrr_stamp"); + $sql = <<'SQL'; CREATE EXTENSION IF NOT EXISTS pg_trgm SQL @@ -441,6 +460,19 @@ SQL } $logger->info("Created index: idx_lrr_archive_to_tag_namespace_arcid"); + # Stamp lookups are always scoped to an archive (get_stamps_by_page, get_stamped_pages); archive + # delete in PgArchive also filters by arcid. + $sql = <<'SQL'; +CREATE INDEX IF NOT EXISTS idx_lrr_stamp_arcid_page ON lrr_stamp (arcid, page) +SQL + $rv = $dbh->do($sql); + unless ( defined $rv ) { + my $errorcode = $dbh->err // ''; + my $errorstr = $dbh->errstr // ''; + die "Failed to create idx_lrr_stamp_arcid_page index: $errorcode - $errorstr"; + } + $logger->info("Created index: idx_lrr_stamp_arcid_page"); + $logger->info("PostgreSQL database initialized successfully"); }