feat(db): Postgres backend + dialect-portable queries + data migration (Phase 5)
- db/pg.js: the pg backend (prepare/exec/tx/init) — ?→$N translation, BIGINT parsed as Number (matches sqlite; else expires_at<Date.now() compares string<number), transactions on one pooled client, init() applies schema.pg.sql. Same interface as db/sqlite.js, so repos are unchanged. - repos.js: the ~7 SQLite-only queries rewritten to run on BOTH engines — audit.add @named→positional; email lookups COLLATE NOCASE→LOWER()=LOWER(); INSERT OR IGNORE→ON CONFLICT DO NOTHING (addMember/poll vote/favorite); mergeInto's UPDATE OR IGNORE→UPDATE…WHERE NOT EXISTS/NOT IN and INSERT OR REPLACE→ON CONFLICT DO UPDATE. Re-validated on sqlite: db-smoke still 22/22. - server.js: boot now `await db.init()` before listening (pg creates tables; sqlite no-op), so the first request can't hit a missing table. - db/migrate-sqlite-to-pg.js: one-shot row copy in FK order (bulk insert, TRUNCATE first so re-runnable). audit_log id left to PG's identity. - package.json: add pg ^8.13.1. Next: validate DB_BACKEND=pg smoke against a real Postgres on the server, then merge. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
// One-shot data migration: copy every row from the SQLite data.db into Postgres. Run ONCE at cutover,
|
||||
// with the app stopped, BEFORE switching DB_BACKEND to pg.
|
||||
//
|
||||
// DB_PATH=/data/data.db DATABASE_URL=postgres://user:pass@host/db node db/migrate-sqlite-to-pg.js
|
||||
//
|
||||
// It applies the Postgres schema first, TRUNCATEs the target tables (so a re-run re-copies cleanly), then
|
||||
// bulk-inserts in FK-dependency order. audit_log.id is a GENERATED identity, so its id is not copied (PG
|
||||
// assigns fresh ones — nothing references audit_log.id). Timestamps/flags are plain integers on both sides.
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const { Pool } = require('pg');
|
||||
|
||||
const SQLITE = process.env.DB_PATH || path.join(__dirname, '..', 'data.db');
|
||||
if (!process.env.DATABASE_URL) { console.error('DATABASE_URL is required'); process.exit(1); }
|
||||
|
||||
const src = new DatabaseSync(SQLITE);
|
||||
const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 4 });
|
||||
|
||||
// Parents before children (users→teams, sessions_auth→users, machines→teams); the rest have no FKs.
|
||||
const ORDER = [
|
||||
'teams', 'users', 'machines', 'sessions_auth', 'audit_log', 'sessions_log', 'refresh_tokens',
|
||||
'api_keys', 'webhooks', 'messages', 'message_reactions', 'attachments', 'conversations',
|
||||
'conversation_members', 'call_history', 'user_aliases', 'polls', 'poll_votes', 'scheduled_meetings',
|
||||
'recordings', 'push_subscriptions', 'device_tokens', 'app_installs', 'favorites',
|
||||
];
|
||||
|
||||
async function main() {
|
||||
await pool.query(fs.readFileSync(path.join(__dirname, 'schema.pg.sql'), 'utf8')); // ensure schema exists
|
||||
await pool.query('TRUNCATE ' + ORDER.map((t) => '"' + t + '"').join(', ') + ' RESTART IDENTITY CASCADE');
|
||||
|
||||
const totals = {};
|
||||
for (const table of ORDER) {
|
||||
let rows = [];
|
||||
try { rows = src.prepare('SELECT * FROM ' + table).all(); } catch (e) { totals[table] = 'skip(' + e.message + ')'; continue; }
|
||||
if (!rows.length) { totals[table] = 0; continue; }
|
||||
let cols = Object.keys(rows[0]);
|
||||
if (table === 'audit_log') cols = cols.filter((c) => c !== 'id'); // GENERATED — let PG assign
|
||||
const colList = cols.map((c) => '"' + c + '"').join(',');
|
||||
const CHUNK = 400; // keep param count well under Postgres' 65535 limit even for wide tables
|
||||
for (let i = 0; i < rows.length; i += CHUNK) {
|
||||
const batch = rows.slice(i, i + CHUNK);
|
||||
const values = []; const params = [];
|
||||
batch.forEach((r, ri) => {
|
||||
values.push('(' + cols.map((c, ci) => '$' + (ri * cols.length + ci + 1)).join(',') + ')');
|
||||
cols.forEach((c) => params.push(r[c] === undefined ? null : r[c]));
|
||||
});
|
||||
await pool.query('INSERT INTO "' + table + '" (' + colList + ') VALUES ' + values.join(','), params);
|
||||
}
|
||||
totals[table] = rows.length;
|
||||
}
|
||||
console.log('MIGRATED rows:', JSON.stringify(totals, null, 0));
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((e) => { console.error('MIGRATION FAILED:', e && e.message); process.exit(1); });
|
||||
@@ -0,0 +1,59 @@
|
||||
// PostgreSQL backend for the async DB adapter. Same interface as db/sqlite.js — prepare(sql).{get,all,run},
|
||||
// exec(sql), tx(fn), init() — so repos and app code are engine-agnostic. Selected by DB_BACKEND=pg;
|
||||
// connection string from DATABASE_URL.
|
||||
const { Pool, types } = require('pg');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// BIGINT (int8, OID 20) defaults to STRING in node-postgres to avoid precision loss. Every BIGINT here is
|
||||
// an epoch-ms timestamp or a byte size — all far below Number.MAX_SAFE_INTEGER — so parse them as numbers
|
||||
// to match the SQLite backend. Otherwise `expires_at < Date.now()` would compare a string to a number.
|
||||
types.setTypeParser(20, (v) => (v === null ? null : parseInt(v, 10)));
|
||||
|
||||
const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 10 });
|
||||
|
||||
// Repos use '?' placeholders (SQLite style); Postgres wants $1,$2,… — replace positionally. Safe because
|
||||
// no literal '?' appears inside any SQL string literal in this codebase.
|
||||
function toPg(sql) { let i = 0; return sql.replace(/\?/g, () => '$' + (++i)); }
|
||||
|
||||
function prepare(sql) {
|
||||
const q = toPg(sql);
|
||||
return {
|
||||
get: (...p) => pool.query(q, p).then((r) => r.rows[0]),
|
||||
all: (...p) => pool.query(q, p).then((r) => r.rows),
|
||||
run: (...p) => pool.query(q, p).then((r) => ({ changes: r.rowCount, lastInsertRowid: undefined })),
|
||||
};
|
||||
}
|
||||
|
||||
function exec(sql) { return pool.query(sql).then(() => {}); }
|
||||
|
||||
// Transaction on ONE pooled client (a pool would scatter BEGIN/COMMIT across connections). Same runner
|
||||
// shape the sqlite backend's tx() exposes, so repos.mergeInto is identical on both engines.
|
||||
async function tx(fn) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
const t = {
|
||||
run: (sql, ...p) => client.query(toPg(sql), p).then((r) => ({ changes: r.rowCount })),
|
||||
get: (sql, ...p) => client.query(toPg(sql), p).then((r) => r.rows[0]),
|
||||
all: (sql, ...p) => client.query(toPg(sql), p).then((r) => r.rows),
|
||||
};
|
||||
const out = await fn(t);
|
||||
await client.query('COMMIT');
|
||||
return out;
|
||||
} catch (e) {
|
||||
try { await client.query('ROLLBACK'); } catch (_) {}
|
||||
throw e;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
// Apply the schema (all CREATE ... IF NOT EXISTS — idempotent). Multi-statement, no params, so it runs via
|
||||
// the simple-query protocol in one call. MUST be awaited before serving (server.js boot).
|
||||
async function init() {
|
||||
const sql = fs.readFileSync(path.join(__dirname, 'schema.pg.sql'), 'utf8');
|
||||
await pool.query(sql);
|
||||
}
|
||||
|
||||
module.exports = { prepare, exec, tx, init, name: 'pg', _pool: pool };
|
||||
Reference in New Issue
Block a user