SQLite3 as the database

.gitignore

@@ -19,3 +19,6 @@ package-lock.json
 _yardoc
 doc/
 .DS_Store
+
+# SQLite database
+/storage/*

Dockerfile

@@ -7,8 +7,7 @@ WORKDIR /app
 # Install base packages
 RUN apt-get update -qq && \
     apt-get install --no-install-recommends -y \
-    curl \
-    postgresql-client
+    curl
 
 # Set production environment
 ENV BUNDLE_DEPLOYMENT="1" \
@@ -23,7 +22,6 @@ FROM base as build
 RUN apt-get install --no-install-recommends -y \
     build-essential \
     git \
-    libpq-dev \
     pkg-config
 
 # Install application gems

Gemfile

@@ -4,8 +4,6 @@ source 'https://rubygems.org/'
 ruby file: '.ruby-version'
 
 gem 'sequel', "5.104.0"
-gem 'pg'
-gem 'sequel_pg', require: 'sequel'
 gem 'sinatra'
 gem 'sinatra-contrib'
 gem 'rack-flash3', require: 'rack-flash'
@@ -28,6 +26,7 @@ gem 'zeitwerk'
 gem 'rubocop', '~> 1.79.2', require: false
 gem 'dyno_metadata'
 gem 'localhost'
+gem 'sqlite3'
 
 group :test do
   gem 'climate_control'

Gemfile.lock

@@ -63,11 +63,6 @@ GEM
     parser (3.3.9.0)
       ast (~> 2.4.1)
       racc
-    pg (1.6.3)
-    pg (1.6.3-aarch64-linux)
-    pg (1.6.3-arm64-darwin)
-    pg (1.6.3-x86_64-darwin)
-    pg (1.6.3-x86_64-linux)
     prism (1.4.0)
     public_suffix (7.0.5)
     puma (7.2.0)
@@ -125,9 +120,6 @@ GEM
       concurrent-ruby (~> 1.0, >= 1.0.2)
     sequel (5.104.0)
       bigdecimal
-    sequel_pg (1.18.2)
-      pg (>= 0.18.0, != 1.2.0)
-      sequel (>= 4.38.0)
     sinatra (4.2.1)
       logger (>= 1.6.0)
       mustermann (~> 3.0)
@@ -143,6 +135,10 @@ GEM
       tilt (~> 2.0)
     spinels-rack-ssl-enforcer (1.0.0)
       rack (>= 3.0.0)
+    sqlite3 (2.9.4-aarch64-linux-gnu)
+    sqlite3 (2.9.4-arm64-darwin)
+    sqlite3 (2.9.4-x86_64-darwin)
+    sqlite3 (2.9.4-x86_64-linux-gnu)
     temple (0.10.3)
     thor (1.4.0)
     tilt (2.4.0)
@@ -181,7 +177,6 @@ DEPENDENCIES
   m
   minitest
   octokit
-  pg
   puma
   rack-flash3
   rack-test
@@ -191,10 +186,10 @@ DEPENDENCIES
   rubocop (~> 1.79.2)
   sentry-ruby
   sequel (= 5.104.0)
-  sequel_pg
   sinatra
   sinatra-contrib
   spinels-rack-ssl-enforcer
+  sqlite3
   wait_for_it
   warning
   webmock
@@ -239,11 +234,6 @@ CHECKSUMS
   octokit (10.0.0) sha256=82e99a539b7637b7e905e6d277bb0c1a4bed56735935cc33db6da7eae49a24e8
   parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130
   parser (3.3.9.0) sha256=94d6929354b1a6e3e1f89d79d4d302cc8f5aa814431a6c9c7e0623335d7687f2
-  pg (1.6.3) sha256=1388d0563e13d2758c1089e35e973a3249e955c659592d10e5b77c468f628a99
-  pg (1.6.3-aarch64-linux) sha256=0698ad563e02383c27510b76bf7d4cd2de19cd1d16a5013f375dd473e4be72ea
-  pg (1.6.3-arm64-darwin) sha256=7240330b572e6355d7c75a7de535edb5dfcbd6295d9c7777df4d9dddfb8c0e5f
-  pg (1.6.3-x86_64-darwin) sha256=ee2e04a17c0627225054ffeb43e31a95be9d7e93abda2737ea3ce4a62f2729d6
-  pg (1.6.3-x86_64-linux) sha256=5d9e188c8f7a0295d162b7b88a768d8452a899977d44f3274d1946d67920ae8d
   prism (1.4.0) sha256=dc0e3e00e93160213dc2a65519d9002a4a1e7b962db57d444cf1a71565bb703e
   public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623
   puma (7.2.0) sha256=bf8ef4ab514a4e6d4554cb4326b2004eba5036ae05cf765cfe51aba9706a72a8
@@ -270,10 +260,13 @@ CHECKSUMS
   selma (0.4.11-x86_64-linux) sha256=e6eb4087c304b915b2f9dbda590677e1cf5dd1194f1b0caddd06a4830ae1e8b8
   sentry-ruby (5.26.0) sha256=37b82d4849683f7172cbdb0867ee6697e0d8ad8fe9c242b5df0edf690894554b
   sequel (5.104.0) sha256=1521819b9204aa7f5c25b29fbb2a609ba3f6a82d2271f34f718e1169d31bf03e
-  sequel_pg (1.18.2) sha256=6bdf0f5f13e062ef0e4b1fd2ddafbe6a13d1aab579473c9627ef4e88da431953
   sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080
   sinatra-contrib (4.2.1) sha256=10d091c944d268aa910c618ea40a3c3ebe0533e6e32990d84af92235a3d26b4a
   spinels-rack-ssl-enforcer (1.0.0) sha256=1d6b155a847a97622155f7d459d3145f9a50aedcd6b2b5cd87c469566e36244c
+  sqlite3 (2.9.4-aarch64-linux-gnu) sha256=ecabed721e6eaad54601d2685f09029d90025efc8d931040dc89cb3f8a2080ec
+  sqlite3 (2.9.4-arm64-darwin) sha256=1d5aad413a815d236e96d43f05a1acc600b6cd086800770342a3f9c2877499ff
+  sqlite3 (2.9.4-x86_64-darwin) sha256=f280c476e360b73e86165a5e59b72801385b4a6c3a47f8af5ecefb9d90bec17f
+  sqlite3 (2.9.4-x86_64-linux-gnu) sha256=537a3eda71b1df1336d0055cbebe55a7317c34870c192c7b6b9d8d0be6871847
   temple (0.10.3) sha256=df3145fe6577af1e25387eb7f7122d32ed51bdb6f2e7bb0f4fbf07b66151913b
   thor (1.4.0) sha256=8763e822ccb0f1d7bee88cde131b19a65606657b847cc7b7b4b82e772bcd8a3d
   tilt (2.4.0) sha256=df74f29a451daed26591a85e8e0cebb198892cb75b6573394303acda273fba4d

README.md

@@ -13,23 +13,20 @@ These instructions assume you are using OS X.
 
 Install prerequisites
 
-    brew install postgresql
     gem install overman
     bundle install
 
 Ruby gems are vendored into `vendor/cache`, you should always check in the gems when changing gems. The caching is set up with [`bundle package --all`](https://bundler.io/man/bundle-package.1.html).
 
 ### Database setup
 
-Make sure PostgreSQL is running
-
-    postgres
+The app uses SQLite. The database file lives at `${DATABASE_DIR}/${DATABASE_NAME}.db`, defaulting to `./storage/wiki.db`. `bin/dev` runs migrations on startup.
 
 #### Import production database
 
     dotenv bin/download_database_backup
-    createdb prod-wikimum
-    psql postgres://localhost/prod-wikimum < starkast_wiki_backup_2024-06-15_215012.sql
+    mkdir -p storage
+    mv starkast_wiki_backup_*.sqlite3 storage/wiki.db
 
 ### Start the app
 
@@ -43,12 +40,12 @@ Go to [http://wikimum.127.0.0.1.nip.io:8080](http://wikimum.127.0.0.1.nip.io:808
 
 ```bash
 # Development and production
-DATABASE_URL=postgres://
+DATABASE_DIR=./storage  # directory holding the SQLite file
+DATABASE_NAME=wiki      # the file is "${DATABASE_NAME}.db"
 GITHUB_BASIC_CLIENT_ID=
 GITHUB_BASIC_SECRET_ID=
 BACKUP_USER=
 BACKUP_PASSWORD=
-PGGSSENCMODE=disable # https://github.com/ged/ruby-pg/issues/311#issuecomment-1609970533
 SESSION_SECRET= # generate with: ruby -rsecurerandom -e 'p SecureRandom.hex(32)'
 # Production
 SENTRY_DSN=
@@ -74,17 +71,17 @@ Run single test with [`m`](https://github.com/qrush/m):
 
 ### [Migrations][sequel-migrations]
 
-To migrate to the latest version, run:
+`bin/dev` runs migrations on startup. To run them manually:
 
-    bin/dev_db_bootstrap
+    bundle exec rake db:migrate
 
-This Rake task takes an optional argument specifying the target version. To migrate to version 42, run:
+To migrate to a specific version, e.g. 42:
 
     bundle exec rake db:migrate[42]
 
-Manually:
+Or via the `sequel` CLI:
 
-    sequel -E -m migrations -M <n> postgres://localhost/wikimum
+    sequel -E -m migrations -M <n> sqlite://storage/wiki.db
 
 ## Deployment
 
@@ -105,8 +102,6 @@ _TODO_
 
 ### Preview
 
-_TODO: the staging database is not online._
-
 Fly.io app `wikimum-preview` exist to test changes before production. It can be reached at https://wikimum-preview.fly.dev/
 
 The first deploy was manual:
@@ -115,19 +110,16 @@ The first deploy was manual:
 flyctl deploy --ha=false --build-arg "RUBY_VERSION=$(cat .ruby-version)" --app=wikimum-preview
 ```
 
-The preview app uses a PostgreSQL database from Aiven, deployed in AWS eu-north-1.
-
-These are the secrets configured:
+The SQLite database lives on a Fly volume named `storage` mounted at `/storage`. Create one before the first deploy:
 
 ```bash
-fly secrets set --detach --app wikimum-preview DATABASE_URL="$(pbpaste)"
-fly secrets set --detach --app wikimum-preview SESSION_SECRET="$(pbpaste)"
+flyctl volumes create storage --app=wikimum-preview --region=arn --size=1
 ```
 
-To keep the preview database (at Aiven) from being removed prematurely, GitHub Actions has been configured with the `cron.yml` workflow and this secret:
+These are the secrets configured:
 
 ```bash
-gh secret --repo Starkast/wikimum set PREVIEW_DATABASE_URL --body "$(pbpaste)"
+fly secrets set --detach --app wikimum-preview SESSION_SECRET="$(pbpaste)"
 ```
 
 ## Code Scanning

Rakefile

@@ -88,14 +88,16 @@ namespace(:db) do
   desc "Run migrations"
   task :migrate, [:version] do |t, args|
     require 'sequel'
+    require_relative 'config/database' # creates DB
+
     Sequel.extension(:migration)
-    db = Sequel.connect(ENV.fetch('DATABASE_URL', 'postgres://localhost/wikimum'))
+
     if args[:version]
       puts "Migrating to version #{args[:version]}"
-      Sequel::Migrator.run(db, 'migrations', target: args[:version].to_i)
+      Sequel::Migrator.run(DB, 'migrations', target: args[:version].to_i)
     else
       puts "Migrating to latest"
-      Sequel::Migrator.run(db, 'migrations')
+      Sequel::Migrator.run(DB, 'migrations')
     end
   end
 end

bin/dev

@@ -3,4 +3,6 @@
 set -e
 set -u
 
+bundle exec rake db:migrate
+
 overman start -e default.env

bin/download_database_backup

@@ -9,5 +9,5 @@ curl \
   --location \
   --request POST \
   --user "${BACKUP_USER}":"${BACKUP_PASSWORD}" \
-  --output "starkast_wiki_backup_$(date "+%Y-%m-%d_%H%M%S").sql" \
+  --output "starkast_wiki_backup_$(date "+%Y-%m-%d_%H%M%S").sqlite3" \
   ${BACKUP_URL:-"https://starkast.wiki/.backup"}

bin/setup

@@ -17,9 +17,6 @@ FileUtils.chdir APP_ROOT do
   puts "\n== Installing dependencies =="
   system("bundle check") || system!("bundle install")
 
-  puts "\n== Preparing database =="
-  system! "bin/dev_db_bootstrap"
-
   unless ARGV.include?("--skip-server")
     puts "\n== Starting development server =="
     STDOUT.flush # flush the output before exec(2) so that it displays

config/app.rb

@@ -16,16 +16,7 @@
 loader.enable_reloading if App.development?
 loader.setup
 
-# https://starkast.wiki/ruby_homebrew_postgres
-# https://github.com/ged/ruby-pg/issues/311#issuecomment-1609970533
-ENV["PGGSSENCMODE"] = "disable" if App.macos?
-
-DB = Sequel.connect(ENV.fetch('DATABASE_URL', 'postgres://localhost/wikimum'))
-
-# https://github.com/Starkast/wikimum/issues/412
-# https://sequel.jeremyevans.net/rdoc-plugins/files/lib/sequel/extensions/connection_validator_rb.html
-DB.extension(:connection_validator)
-DB.pool.connection_validation_timeout = 60 * 5
+require_relative 'database'
 
 if App.development?
   require 'logger'

config/brakeman.ignore

@@ -8,9 +8,9 @@
       "check_name": "SendFile",
       "message": "Parameter value used in file name",
       "file": "lib/controllers/backup_controller.rb",
-      "line": 37,
+      "line": 32,
       "link": "https://brakemanscanner.org/docs/warning_types/file_access/",
-      "code": "send_file(File.join(backup_tmpdir, Base64.urlsafe_decode64(params.fetch(\"encoded_path\"))), :type => \"text/plain\")",
+      "code": "send_file(File.join(backup_tmpdir, Base64.urlsafe_decode64(params.fetch(\"encoded_path\"))), :type => \"application/vnd.sqlite3\")",
       "render_path": null,
       "location": {
         "type": "method",

config/database.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+database_dir  = ENV.fetch("DATABASE_DIR",  "./storage")
+database_name = ENV.fetch("DATABASE_NAME", "wiki")
+
+DB = Sequel.connect("sqlite://#{database_dir}/#{database_name}.db")

docker-compose.yml

@@ -10,23 +10,6 @@ services:
     env_file:
       - ./default.env
     environment:
-      DATABASE_URL: postgres://postgres:postgres@db/db
       PORT: 3000
-    depends_on:
-      db:
-        condition: service_healthy
-  db:
-    image: postgres
-    restart: always
-    # set shared memory limit when using docker-compose
-    # https://github.com/docker-library/postgres/issues/416
-    shm_size: 128mb
-    environment:
-      POSTGRES_DB: db
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready --dbname=db --username=postgres"]
-      interval: 2s
-      retries: 10
-      start_period: 20s
+    volumes:
+      - ./storage:/app/storage

fly.toml

@@ -9,6 +9,11 @@ primary_region = "arn"
 [env]
   PORT = "8080"
   RACK_ENV = "production"
+  DATABASE_DIR = "/storage"
+
+[[mounts]]
+  source = "storage"
+  destination = "/storage"
 
 [[services]]
   processes = ["web"]

lib/app.rb

@@ -101,11 +101,6 @@ def redirect_to_https?
     def thruthy?(value)
       %w(1 on true).include?(value.to_s)
     end
-
-    def macos?
-      # e.g. "arm64-darwin23"
-      RUBY_PLATFORM.split("-").last.include?("darwin")
-    end
   end
 
   if production?