From 9db922703b0f7fe791a7ce12f79bb1b792511b3e Mon Sep 17 00:00:00 2001 From: Nirodan Date: Wed, 6 May 2026 10:27:11 +0200 Subject: [PATCH] Add versioned DB migration system with automatic backup migrations.py - schema_migrations table tracks applied versions (version, description, applied_at) - MIGRATIONS list is append-only; each entry is (version, description, sql) - backup() dumps all user-data tables to a timestamped JSON file in backups/ before any schema changes so data can be recovered if something goes wrong - run_migrations() is idempotent: already-applied versions are skipped Integration - app.py calls _run_startup_migrations() at module load so every restart applies any pending migrations (no-op if schema is current) - setup_routes.py calls run_migrations() after the initial setup form is submitted so all tables exist before the user hits the main page for the first time - notes.py and admin.py: removed all per-request CREATE TABLE DDL; schema is now owned entirely by the migration system Docker - docker-compose.dev.yml: add backups-data volume so JSON backups survive container restarts and rebuilds - Dockerfile: pre-create /app/backend/logs and /app/backend/backups so the directories exist even before volumes are mounted Adding future schema changes - Append a new (version, description, sql) tuple to MIGRATIONS in migrations.py - The next restart will detect it as pending, back up first, then apply it Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 3 + backend/admin.py | 40 --------- backend/app.py | 20 +++++ backend/tools/notes.py | 39 +-------- backend/util/migrations.py | 161 +++++++++++++++++++++++++++++++++++ backend/util/setup_routes.py | 12 +++ docker-compose.dev.yml | 2 + 7 files changed, 202 insertions(+), 75 deletions(-) create mode 100644 backend/util/migrations.py diff --git a/Dockerfile b/Dockerfile index 3f7f7cd..a4d561f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,9 @@ COPY backend/config /config COPY backend/requirements.txt ./requirements.txt COPY backend/entrypoint.sh ./entrypoint.sh +# Persistent directories (overridden by Docker volumes at runtime) +RUN mkdir -p /app/backend/logs /app/backend/backups + # Frontend aus Build-Stage übernehmen COPY --from=frontend-build /app/frontend/dist ./frontend/dist diff --git a/backend/admin.py b/backend/admin.py index 73ce10b..e7ab4c6 100644 --- a/backend/admin.py +++ b/backend/admin.py @@ -8,10 +8,6 @@ admin_bp = Blueprint("admin", __name__) _VALID_ROLES = {"user", "admin"} -# Module-level flag: DDL runs at most once per process lifetime. -# Resets automatically on worker restart, which re-triggers the check. -_tables_initialized = False - def _require_admin(): user = verify_token() @@ -23,33 +19,6 @@ def _require_admin(): return user, None -def _ensure_tables(cur): - global _tables_initialized - if _tables_initialized: - return - cur.execute( - """ - CREATE TABLE IF NOT EXISTS users ( - id INT AUTO_INCREMENT PRIMARY KEY, - username VARCHAR(50) NOT NULL UNIQUE, - password VARCHAR(255) NOT NULL, - role VARCHAR(20) NOT NULL - ) - """ - ) - cur.execute( - """ - CREATE TABLE IF NOT EXISTS websites ( - id INT AUTO_INCREMENT PRIMARY KEY, - name VARCHAR(100) NOT NULL, - url VARCHAR(255) NOT NULL, - description VARCHAR(255) DEFAULT '' - ) - """ - ) - _tables_initialized = True - - @admin_bp.route("/api/admin/users", methods=["GET"]) def list_users(): _, err = _require_admin() @@ -59,7 +28,6 @@ def list_users(): conn = get_connection() try: cur = conn.cursor(dictionary=True) - _ensure_tables(cur) cur.execute("SELECT id, username, role FROM users ORDER BY username ASC") users = cur.fetchall() cur.close() @@ -88,7 +56,6 @@ def create_user(): conn = get_connection() try: cur = conn.cursor(dictionary=True) - _ensure_tables(cur) cur.execute("SELECT id FROM users WHERE username=%s", (username,)) if cur.fetchone(): cur.close() @@ -125,7 +92,6 @@ def update_user(user_id): conn = get_connection() try: cur = conn.cursor() - _ensure_tables(cur) if role: cur.execute("UPDATE users SET role=%s WHERE id=%s", (role, user_id)) if password: @@ -153,7 +119,6 @@ def delete_user(user_id): conn = get_connection() try: cur = conn.cursor() - _ensure_tables(cur) cur.execute("SELECT username FROM users WHERE id=%s", (user_id,)) row = cur.fetchone() if not row: @@ -186,7 +151,6 @@ def list_websites_admin(): conn = get_connection() try: cur = conn.cursor(dictionary=True) - _ensure_tables(cur) cur.execute("SELECT id, name, url, description FROM websites ORDER BY name ASC") rows = cur.fetchall() cur.close() @@ -213,7 +177,6 @@ def create_website(): conn = get_connection() try: cur = conn.cursor() - _ensure_tables(cur) cur.execute( "INSERT INTO websites (name, url, description) VALUES (%s, %s, %s)", (name, url, description), @@ -244,7 +207,6 @@ def update_website(item_id): conn = get_connection() try: cur = conn.cursor() - _ensure_tables(cur) cur.execute( "UPDATE websites SET name=%s, url=%s, description=%s WHERE id=%s", (name, url, description, item_id), @@ -271,7 +233,6 @@ def delete_website(item_id): conn = get_connection() try: cur = conn.cursor() - _ensure_tables(cur) cur.execute("DELETE FROM websites WHERE id=%s", (item_id,)) conn.commit() affected = cur.rowcount @@ -297,7 +258,6 @@ def list_websites_public(): conn = get_connection() try: cur = conn.cursor(dictionary=True) - _ensure_tables(cur) cur.execute("SELECT id, name, url, description FROM websites ORDER BY name ASC") rows = cur.fetchall() cur.close() diff --git a/backend/app.py b/backend/app.py index 5d188f4..d715563 100644 --- a/backend/app.py +++ b/backend/app.py @@ -93,6 +93,26 @@ def serve_frontend(path): return send_from_directory(dist_dir, 'index.html') +def _run_startup_migrations(): + """Run pending DB migrations if the database is already configured.""" + if not is_configured(): + logger.info("[startup] DB not yet configured — skipping migrations.") + return + try: + from util.db_pool import get_connection + from util.migrations import run_migrations + conn = get_connection() + try: + run_migrations(conn) + finally: + conn.close() + except Exception as e: + logger.error(f"[startup] Migration error: {e}") + + +_run_startup_migrations() + + if __name__ == '__main__': os.makedirs("config", exist_ok=True) app.run(host='0.0.0.0', port=5000, debug=True) diff --git a/backend/tools/notes.py b/backend/tools/notes.py index 4f97ef6..657d529 100644 --- a/backend/tools/notes.py +++ b/backend/tools/notes.py @@ -6,35 +6,6 @@ from auth.token import verify_token notes_blueprint = Blueprint('notes_tool', __name__) -# Module-level flag: DDL runs at most once per process lifetime. -# Resets automatically on worker restart, which re-triggers the check. -_table_ready = False - - -def _ensure_table(): - global _table_ready - if _table_ready: - return - conn = get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - CREATE TABLE IF NOT EXISTS notes ( - id INT AUTO_INCREMENT PRIMARY KEY, - user_id INT NOT NULL, - title VARCHAR(255) NOT NULL, - content TEXT, - language VARCHAR(50) DEFAULT 'text', - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP - ) - """) - conn.commit() - cursor.close() - _table_ready = True - finally: - conn.close() - @notes_blueprint.route('/api/notes', methods=['GET']) def get_notes(): @@ -42,12 +13,12 @@ def get_notes(): if not user: return jsonify({"message": "Nicht autorisiert"}), 401 try: - _ensure_table() conn = get_connection() try: cursor = conn.cursor(dictionary=True) cursor.execute( - "SELECT id, title, content, language, created_at, updated_at FROM notes WHERE user_id = %s ORDER BY updated_at DESC", + "SELECT id, title, content, language, created_at, updated_at " + "FROM notes WHERE user_id = %s ORDER BY updated_at DESC", (user['id'],) ) notes = cursor.fetchall() @@ -71,7 +42,6 @@ def create_note(): if not user: return jsonify({"message": "Nicht autorisiert"}), 401 try: - _ensure_table() data = request.get_json(silent=True) or {} title = data.get("title", "Neue Notiz").strip() or "Neue Notiz" content = data.get("content", "") @@ -103,7 +73,6 @@ def update_note(note_id): if not user: return jsonify({"message": "Nicht autorisiert"}), 401 try: - _ensure_table() data = request.get_json(silent=True) or {} title = data.get("title", "").strip() or "Neue Notiz" content = data.get("content", "") @@ -116,7 +85,8 @@ def update_note(note_id): try: cursor = conn.cursor() cursor.execute( - "UPDATE notes SET title=%s, content=%s, language=%s, updated_at=%s WHERE id=%s AND user_id=%s", + "UPDATE notes SET title=%s, content=%s, language=%s, updated_at=%s " + "WHERE id=%s AND user_id=%s", (title, content, language, now, note_id, user['id']) ) conn.commit() @@ -139,7 +109,6 @@ def delete_note(note_id): if not user: return jsonify({"message": "Nicht autorisiert"}), 401 try: - _ensure_table() conn = get_connection() try: cursor = conn.cursor() diff --git a/backend/util/migrations.py b/backend/util/migrations.py new file mode 100644 index 0000000..3f4831f --- /dev/null +++ b/backend/util/migrations.py @@ -0,0 +1,161 @@ +""" +Schema migration system. + +Rules for maintainers +--------------------- +- NEVER modify or delete an existing entry in MIGRATIONS. +- ALWAYS append new entries at the end with an incremented version number. +- Use ALTER TABLE to change an existing table, not a new CREATE TABLE. +- Each entry must be a single, complete SQL statement. + +How it works +------------ +On startup (and after the initial setup form) run_migrations() is called. +It creates a `schema_migrations` table the first time, checks which versions +have already been applied, and runs any that are missing. A full JSON backup +of all user-data tables is written to the backups/ directory before any +schema changes are made so data can be recovered if something goes wrong. +""" + +import json +import os +from datetime import datetime + +from util.logger import logger + +# --------------------------------------------------------------------------- +# Migration registry — append-only +# --------------------------------------------------------------------------- + +MIGRATIONS = [ + (1, "Create users table", """ + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(50) NOT NULL UNIQUE, + password VARCHAR(255) NOT NULL, + role VARCHAR(20) NOT NULL + ) + """), + (2, "Create websites table", """ + CREATE TABLE IF NOT EXISTS websites ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100) NOT NULL, + url VARCHAR(255) NOT NULL, + description VARCHAR(255) DEFAULT '' + ) + """), + (3, "Create notes table", """ + CREATE TABLE IF NOT EXISTS notes ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT NOT NULL, + title VARCHAR(255) NOT NULL, + content TEXT, + language VARCHAR(50) DEFAULT 'text', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) + """), + # ── add new migrations below this line ────────────────────────────────── +] + +# --------------------------------------------------------------------------- +# Internals +# --------------------------------------------------------------------------- + +_BACKUP_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "backups" +) + + +def _ensure_migration_table(conn): + cur = conn.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version INT PRIMARY KEY, + description VARCHAR(255) NOT NULL, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.commit() + cur.close() + + +def _applied_versions(conn): + cur = conn.cursor() + cur.execute("SELECT version FROM schema_migrations") + versions = {row[0] for row in cur.fetchall()} + cur.close() + return versions + + +def backup(conn): + """ + Dump every user-data table to a timestamped JSON file. + Returns the path of the written file. + Datetime values are serialised as ISO-8601 strings. + """ + os.makedirs(_BACKUP_DIR, exist_ok=True) + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + path = os.path.join(_BACKUP_DIR, f"backup_{ts}.json") + + cur = conn.cursor(dictionary=True) + cur.execute("SHOW TABLES") + tables = [list(row.values())[0] for row in cur.fetchall()] + + snapshot = {} + for table in tables: + if table == "schema_migrations": + continue + cur.execute(f"SELECT * FROM `{table}`") + snapshot[table] = [ + { + k: v.isoformat() if hasattr(v, "isoformat") else v + for k, v in row.items() + } + for row in cur.fetchall() + ] + cur.close() + + with open(path, "w", encoding="utf-8") as f: + json.dump(snapshot, f, indent=2, ensure_ascii=False) + + logger.info(f"[migrations] Backup written → {path}") + return path + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def run_migrations(conn): + """ + Apply every pending migration in version order. + A backup is created automatically before the first schema change. + Safe to call on every startup — already-applied migrations are skipped. + """ + _ensure_migration_table(conn) + applied = _applied_versions(conn) + pending = [(v, d, sql) for v, d, sql in MIGRATIONS if v not in applied] + + if not pending: + logger.info("[migrations] Schema is up to date.") + return + + backup_path = backup(conn) + logger.info( + f"[migrations] {len(pending)} pending migration(s). " + f"Backup: {backup_path}" + ) + + cur = conn.cursor() + for version, description, sql in pending: + logger.info(f"[migrations] Applying v{version}: {description}") + cur.execute(sql.strip()) + cur.execute( + "INSERT INTO schema_migrations (version, description) VALUES (%s, %s)", + (version, description), + ) + + conn.commit() + cur.close() + logger.info("[migrations] All migrations applied.") diff --git a/backend/util/setup_routes.py b/backend/util/setup_routes.py index 06f9142..4af8ee2 100644 --- a/backend/util/setup_routes.py +++ b/backend/util/setup_routes.py @@ -43,6 +43,18 @@ def setup(): reset_pool() if test_connection(db_config): initialize_admin_user(db_config) + # Apply schema migrations immediately so all tables exist before + # the user lands on the main page. + try: + from util.db_pool import get_connection + from util.migrations import run_migrations + conn = get_connection() + try: + run_migrations(conn) + finally: + conn.close() + except Exception as e: + logger.warning(f"[setup] Post-setup migration error: {e}") return redirect('/') else: return "Verbindung fehlgeschlagen. Bitte zurück und prüfen.", 500 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d99d87d..b49167a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -10,8 +10,10 @@ services: volumes: - config-data:/config - logs-data:/app/backend/logs + - backups-data:/app/backend/backups working_dir: /app/backend command: python app.py volumes: config-data: logs-data: + backups-data: