Add versioned DB migration system with automatic backup
migrations.py - schema_migrations table tracks applied versions (version, description, applied_at) - MIGRATIONS list is append-only; each entry is (version, description, sql) - backup() dumps all user-data tables to a timestamped JSON file in backups/ before any schema changes so data can be recovered if something goes wrong - run_migrations() is idempotent: already-applied versions are skipped Integration - app.py calls _run_startup_migrations() at module load so every restart applies any pending migrations (no-op if schema is current) - setup_routes.py calls run_migrations() after the initial setup form is submitted so all tables exist before the user hits the main page for the first time - notes.py and admin.py: removed all per-request CREATE TABLE DDL; schema is now owned entirely by the migration system Docker - docker-compose.dev.yml: add backups-data volume so JSON backups survive container restarts and rebuilds - Dockerfile: pre-create /app/backend/logs and /app/backend/backups so the directories exist even before volumes are mounted Adding future schema changes - Append a new (version, description, sql) tuple to MIGRATIONS in migrations.py - The next restart will detect it as pending, back up first, then apply it Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Nirodan
@@ -8,10 +8,6 @@ admin_bp = Blueprint("admin", __name__)
|
||||
|
||||
_VALID_ROLES = {"user", "admin"}
|
||||
|
||||
# Module-level flag: DDL runs at most once per process lifetime.
|
||||
# Resets automatically on worker restart, which re-triggers the check.
|
||||
_tables_initialized = False
|
||||
|
||||
|
||||
def _require_admin():
|
||||
user = verify_token()
|
||||
@@ -23,33 +19,6 @@ def _require_admin():
|
||||
return user, None
|
||||
|
||||
|
||||
def _ensure_tables(cur):
|
||||
global _tables_initialized
|
||||
if _tables_initialized:
|
||||
return
|
||||
cur.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
username VARCHAR(50) NOT NULL UNIQUE,
|
||||
password VARCHAR(255) NOT NULL,
|
||||
role VARCHAR(20) NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
cur.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS websites (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
url VARCHAR(255) NOT NULL,
|
||||
description VARCHAR(255) DEFAULT ''
|
||||
)
|
||||
"""
|
||||
)
|
||||
_tables_initialized = True
|
||||
|
||||
|
||||
@admin_bp.route("/api/admin/users", methods=["GET"])
|
||||
def list_users():
|
||||
_, err = _require_admin()
|
||||
@@ -59,7 +28,6 @@ def list_users():
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor(dictionary=True)
|
||||
_ensure_tables(cur)
|
||||
cur.execute("SELECT id, username, role FROM users ORDER BY username ASC")
|
||||
users = cur.fetchall()
|
||||
cur.close()
|
||||
@@ -88,7 +56,6 @@ def create_user():
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor(dictionary=True)
|
||||
_ensure_tables(cur)
|
||||
cur.execute("SELECT id FROM users WHERE username=%s", (username,))
|
||||
if cur.fetchone():
|
||||
cur.close()
|
||||
@@ -125,7 +92,6 @@ def update_user(user_id):
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
_ensure_tables(cur)
|
||||
if role:
|
||||
cur.execute("UPDATE users SET role=%s WHERE id=%s", (role, user_id))
|
||||
if password:
|
||||
@@ -153,7 +119,6 @@ def delete_user(user_id):
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
_ensure_tables(cur)
|
||||
cur.execute("SELECT username FROM users WHERE id=%s", (user_id,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
@@ -186,7 +151,6 @@ def list_websites_admin():
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor(dictionary=True)
|
||||
_ensure_tables(cur)
|
||||
cur.execute("SELECT id, name, url, description FROM websites ORDER BY name ASC")
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
@@ -213,7 +177,6 @@ def create_website():
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
_ensure_tables(cur)
|
||||
cur.execute(
|
||||
"INSERT INTO websites (name, url, description) VALUES (%s, %s, %s)",
|
||||
(name, url, description),
|
||||
@@ -244,7 +207,6 @@ def update_website(item_id):
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
_ensure_tables(cur)
|
||||
cur.execute(
|
||||
"UPDATE websites SET name=%s, url=%s, description=%s WHERE id=%s",
|
||||
(name, url, description, item_id),
|
||||
@@ -271,7 +233,6 @@ def delete_website(item_id):
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
_ensure_tables(cur)
|
||||
cur.execute("DELETE FROM websites WHERE id=%s", (item_id,))
|
||||
conn.commit()
|
||||
affected = cur.rowcount
|
||||
@@ -297,7 +258,6 @@ def list_websites_public():
|
||||
conn = get_connection()
|
||||
try:
|
||||
cur = conn.cursor(dictionary=True)
|
||||
_ensure_tables(cur)
|
||||
cur.execute("SELECT id, name, url, description FROM websites ORDER BY name ASC")
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
|
||||
@@ -93,6 +93,26 @@ def serve_frontend(path):
|
||||
return send_from_directory(dist_dir, 'index.html')
|
||||
|
||||
|
||||
def _run_startup_migrations():
|
||||
"""Run pending DB migrations if the database is already configured."""
|
||||
if not is_configured():
|
||||
logger.info("[startup] DB not yet configured — skipping migrations.")
|
||||
return
|
||||
try:
|
||||
from util.db_pool import get_connection
|
||||
from util.migrations import run_migrations
|
||||
conn = get_connection()
|
||||
try:
|
||||
run_migrations(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"[startup] Migration error: {e}")
|
||||
|
||||
|
||||
_run_startup_migrations()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
os.makedirs("config", exist_ok=True)
|
||||
app.run(host='0.0.0.0', port=5000, debug=True)
|
||||
|
||||
+4
-35
@@ -6,35 +6,6 @@ from auth.token import verify_token
|
||||
|
||||
notes_blueprint = Blueprint('notes_tool', __name__)
|
||||
|
||||
# Module-level flag: DDL runs at most once per process lifetime.
|
||||
# Resets automatically on worker restart, which re-triggers the check.
|
||||
_table_ready = False
|
||||
|
||||
|
||||
def _ensure_table():
|
||||
global _table_ready
|
||||
if _table_ready:
|
||||
return
|
||||
conn = get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS notes (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
user_id INT NOT NULL,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
content TEXT,
|
||||
language VARCHAR(50) DEFAULT 'text',
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
cursor.close()
|
||||
_table_ready = True
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@notes_blueprint.route('/api/notes', methods=['GET'])
|
||||
def get_notes():
|
||||
@@ -42,12 +13,12 @@ def get_notes():
|
||||
if not user:
|
||||
return jsonify({"message": "Nicht autorisiert"}), 401
|
||||
try:
|
||||
_ensure_table()
|
||||
conn = get_connection()
|
||||
try:
|
||||
cursor = conn.cursor(dictionary=True)
|
||||
cursor.execute(
|
||||
"SELECT id, title, content, language, created_at, updated_at FROM notes WHERE user_id = %s ORDER BY updated_at DESC",
|
||||
"SELECT id, title, content, language, created_at, updated_at "
|
||||
"FROM notes WHERE user_id = %s ORDER BY updated_at DESC",
|
||||
(user['id'],)
|
||||
)
|
||||
notes = cursor.fetchall()
|
||||
@@ -71,7 +42,6 @@ def create_note():
|
||||
if not user:
|
||||
return jsonify({"message": "Nicht autorisiert"}), 401
|
||||
try:
|
||||
_ensure_table()
|
||||
data = request.get_json(silent=True) or {}
|
||||
title = data.get("title", "Neue Notiz").strip() or "Neue Notiz"
|
||||
content = data.get("content", "")
|
||||
@@ -103,7 +73,6 @@ def update_note(note_id):
|
||||
if not user:
|
||||
return jsonify({"message": "Nicht autorisiert"}), 401
|
||||
try:
|
||||
_ensure_table()
|
||||
data = request.get_json(silent=True) or {}
|
||||
title = data.get("title", "").strip() or "Neue Notiz"
|
||||
content = data.get("content", "")
|
||||
@@ -116,7 +85,8 @@ def update_note(note_id):
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"UPDATE notes SET title=%s, content=%s, language=%s, updated_at=%s WHERE id=%s AND user_id=%s",
|
||||
"UPDATE notes SET title=%s, content=%s, language=%s, updated_at=%s "
|
||||
"WHERE id=%s AND user_id=%s",
|
||||
(title, content, language, now, note_id, user['id'])
|
||||
)
|
||||
conn.commit()
|
||||
@@ -139,7 +109,6 @@ def delete_note(note_id):
|
||||
if not user:
|
||||
return jsonify({"message": "Nicht autorisiert"}), 401
|
||||
try:
|
||||
_ensure_table()
|
||||
conn = get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Schema migration system.
|
||||
|
||||
Rules for maintainers
|
||||
---------------------
|
||||
- NEVER modify or delete an existing entry in MIGRATIONS.
|
||||
- ALWAYS append new entries at the end with an incremented version number.
|
||||
- Use ALTER TABLE to change an existing table, not a new CREATE TABLE.
|
||||
- Each entry must be a single, complete SQL statement.
|
||||
|
||||
How it works
|
||||
------------
|
||||
On startup (and after the initial setup form) run_migrations() is called.
|
||||
It creates a `schema_migrations` table the first time, checks which versions
|
||||
have already been applied, and runs any that are missing. A full JSON backup
|
||||
of all user-data tables is written to the backups/ directory before any
|
||||
schema changes are made so data can be recovered if something goes wrong.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from util.logger import logger
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Migration registry — append-only
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MIGRATIONS = [
|
||||
(1, "Create users table", """
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
username VARCHAR(50) NOT NULL UNIQUE,
|
||||
password VARCHAR(255) NOT NULL,
|
||||
role VARCHAR(20) NOT NULL
|
||||
)
|
||||
"""),
|
||||
(2, "Create websites table", """
|
||||
CREATE TABLE IF NOT EXISTS websites (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
url VARCHAR(255) NOT NULL,
|
||||
description VARCHAR(255) DEFAULT ''
|
||||
)
|
||||
"""),
|
||||
(3, "Create notes table", """
|
||||
CREATE TABLE IF NOT EXISTS notes (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
user_id INT NOT NULL,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
content TEXT,
|
||||
language VARCHAR(50) DEFAULT 'text',
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
)
|
||||
"""),
|
||||
# ── add new migrations below this line ──────────────────────────────────
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internals
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BACKUP_DIR = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "..", "backups"
|
||||
)
|
||||
|
||||
|
||||
def _ensure_migration_table(conn):
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version INT PRIMARY KEY,
|
||||
description VARCHAR(255) NOT NULL,
|
||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
cur.close()
|
||||
|
||||
|
||||
def _applied_versions(conn):
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT version FROM schema_migrations")
|
||||
versions = {row[0] for row in cur.fetchall()}
|
||||
cur.close()
|
||||
return versions
|
||||
|
||||
|
||||
def backup(conn):
|
||||
"""
|
||||
Dump every user-data table to a timestamped JSON file.
|
||||
Returns the path of the written file.
|
||||
Datetime values are serialised as ISO-8601 strings.
|
||||
"""
|
||||
os.makedirs(_BACKUP_DIR, exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
path = os.path.join(_BACKUP_DIR, f"backup_{ts}.json")
|
||||
|
||||
cur = conn.cursor(dictionary=True)
|
||||
cur.execute("SHOW TABLES")
|
||||
tables = [list(row.values())[0] for row in cur.fetchall()]
|
||||
|
||||
snapshot = {}
|
||||
for table in tables:
|
||||
if table == "schema_migrations":
|
||||
continue
|
||||
cur.execute(f"SELECT * FROM `{table}`")
|
||||
snapshot[table] = [
|
||||
{
|
||||
k: v.isoformat() if hasattr(v, "isoformat") else v
|
||||
for k, v in row.items()
|
||||
}
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
cur.close()
|
||||
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(snapshot, f, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.info(f"[migrations] Backup written → {path}")
|
||||
return path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def run_migrations(conn):
|
||||
"""
|
||||
Apply every pending migration in version order.
|
||||
A backup is created automatically before the first schema change.
|
||||
Safe to call on every startup — already-applied migrations are skipped.
|
||||
"""
|
||||
_ensure_migration_table(conn)
|
||||
applied = _applied_versions(conn)
|
||||
pending = [(v, d, sql) for v, d, sql in MIGRATIONS if v not in applied]
|
||||
|
||||
if not pending:
|
||||
logger.info("[migrations] Schema is up to date.")
|
||||
return
|
||||
|
||||
backup_path = backup(conn)
|
||||
logger.info(
|
||||
f"[migrations] {len(pending)} pending migration(s). "
|
||||
f"Backup: {backup_path}"
|
||||
)
|
||||
|
||||
cur = conn.cursor()
|
||||
for version, description, sql in pending:
|
||||
logger.info(f"[migrations] Applying v{version}: {description}")
|
||||
cur.execute(sql.strip())
|
||||
cur.execute(
|
||||
"INSERT INTO schema_migrations (version, description) VALUES (%s, %s)",
|
||||
(version, description),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
cur.close()
|
||||
logger.info("[migrations] All migrations applied.")
|
||||
@@ -43,6 +43,18 @@ def setup():
|
||||
reset_pool()
|
||||
if test_connection(db_config):
|
||||
initialize_admin_user(db_config)
|
||||
# Apply schema migrations immediately so all tables exist before
|
||||
# the user lands on the main page.
|
||||
try:
|
||||
from util.db_pool import get_connection
|
||||
from util.migrations import run_migrations
|
||||
conn = get_connection()
|
||||
try:
|
||||
run_migrations(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"[setup] Post-setup migration error: {e}")
|
||||
return redirect('/')
|
||||
else:
|
||||
return "Verbindung fehlgeschlagen. Bitte zurück und prüfen.", 500
|
||||
|
||||
Reference in New Issue
Block a user