Add versioned DB migration system with automatic backup

migrations.py
- schema_migrations table tracks applied versions (version, description, applied_at)
- MIGRATIONS list is append-only; each entry is (version, description, sql)
- backup() dumps all user-data tables to a timestamped JSON file in backups/
  before any schema changes so data can be recovered if something goes wrong
- run_migrations() is idempotent: already-applied versions are skipped

Integration
- app.py calls _run_startup_migrations() at module load so every restart
  applies any pending migrations (no-op if schema is current)
- setup_routes.py calls run_migrations() after the initial setup form is
  submitted so all tables exist before the user hits the main page for the
  first time
- notes.py and admin.py: removed all per-request CREATE TABLE DDL; schema is
  now owned entirely by the migration system

Docker
- docker-compose.dev.yml: add backups-data volume so JSON backups survive
  container restarts and rebuilds
- Dockerfile: pre-create /app/backend/logs and /app/backend/backups so the
  directories exist even before volumes are mounted

Adding future schema changes
- Append a new (version, description, sql) tuple to MIGRATIONS in migrations.py
- The next restart will detect it as pending, back up first, then apply it

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Dockerfile

@@ -21,6 +21,9 @@ COPY backend/config /config
 COPY backend/requirements.txt ./requirements.txt
 COPY backend/entrypoint.sh ./entrypoint.sh
 
+# Persistent directories (overridden by Docker volumes at runtime)
+RUN mkdir -p /app/backend/logs /app/backend/backups
+
 # Frontend aus Build-Stage übernehmen
 COPY --from=frontend-build /app/frontend/dist ./frontend/dist
 

backend/admin.py

@@ -8,10 +8,6 @@ admin_bp = Blueprint("admin", __name__)
 
 _VALID_ROLES = {"user", "admin"}
 
-# Module-level flag: DDL runs at most once per process lifetime.
-# Resets automatically on worker restart, which re-triggers the check.
-_tables_initialized = False
-
 
 def _require_admin():
     user = verify_token()
@@ -23,33 +19,6 @@ def _require_admin():
     return user, None
 
 
-def _ensure_tables(cur):
-    global _tables_initialized
-    if _tables_initialized:
-        return
-    cur.execute(
-        """
-        CREATE TABLE IF NOT EXISTS users (
-            id INT AUTO_INCREMENT PRIMARY KEY,
-            username VARCHAR(50) NOT NULL UNIQUE,
-            password VARCHAR(255) NOT NULL,
-            role VARCHAR(20) NOT NULL
-        )
-        """
-    )
-    cur.execute(
-        """
-        CREATE TABLE IF NOT EXISTS websites (
-            id INT AUTO_INCREMENT PRIMARY KEY,
-            name VARCHAR(100) NOT NULL,
-            url VARCHAR(255) NOT NULL,
-            description VARCHAR(255) DEFAULT ''
-        )
-        """
-    )
-    _tables_initialized = True
-
-
 @admin_bp.route("/api/admin/users", methods=["GET"])
 def list_users():
     _, err = _require_admin()
@@ -59,7 +28,6 @@ def list_users():
         conn = get_connection()
         try:
             cur = conn.cursor(dictionary=True)
-            _ensure_tables(cur)
             cur.execute("SELECT id, username, role FROM users ORDER BY username ASC")
             users = cur.fetchall()
             cur.close()
@@ -88,7 +56,6 @@ def create_user():
         conn = get_connection()
         try:
             cur = conn.cursor(dictionary=True)
-            _ensure_tables(cur)
             cur.execute("SELECT id FROM users WHERE username=%s", (username,))
             if cur.fetchone():
                 cur.close()
@@ -125,7 +92,6 @@ def update_user(user_id):
         conn = get_connection()
         try:
             cur = conn.cursor()
-            _ensure_tables(cur)
             if role:
                 cur.execute("UPDATE users SET role=%s WHERE id=%s", (role, user_id))
             if password:
@@ -153,7 +119,6 @@ def delete_user(user_id):
         conn = get_connection()
         try:
             cur = conn.cursor()
-            _ensure_tables(cur)
             cur.execute("SELECT username FROM users WHERE id=%s", (user_id,))
             row = cur.fetchone()
             if not row:
@@ -186,7 +151,6 @@ def list_websites_admin():
         conn = get_connection()
         try:
             cur = conn.cursor(dictionary=True)
-            _ensure_tables(cur)
             cur.execute("SELECT id, name, url, description FROM websites ORDER BY name ASC")
             rows = cur.fetchall()
             cur.close()
@@ -213,7 +177,6 @@ def create_website():
         conn = get_connection()
         try:
             cur = conn.cursor()
-            _ensure_tables(cur)
             cur.execute(
                 "INSERT INTO websites (name, url, description) VALUES (%s, %s, %s)",
                 (name, url, description),
@@ -244,7 +207,6 @@ def update_website(item_id):
         conn = get_connection()
         try:
             cur = conn.cursor()
-            _ensure_tables(cur)
             cur.execute(
                 "UPDATE websites SET name=%s, url=%s, description=%s WHERE id=%s",
                 (name, url, description, item_id),
@@ -271,7 +233,6 @@ def delete_website(item_id):
         conn = get_connection()
         try:
             cur = conn.cursor()
-            _ensure_tables(cur)
             cur.execute("DELETE FROM websites WHERE id=%s", (item_id,))
             conn.commit()
             affected = cur.rowcount
@@ -297,7 +258,6 @@ def list_websites_public():
         conn = get_connection()
         try:
             cur = conn.cursor(dictionary=True)
-            _ensure_tables(cur)
             cur.execute("SELECT id, name, url, description FROM websites ORDER BY name ASC")
             rows = cur.fetchall()
             cur.close()

backend/app.py

@@ -93,6 +93,26 @@ def serve_frontend(path):
     return send_from_directory(dist_dir, 'index.html')
 
 
+def _run_startup_migrations():
+    """Run pending DB migrations if the database is already configured."""
+    if not is_configured():
+        logger.info("[startup] DB not yet configured — skipping migrations.")
+        return
+    try:
+        from util.db_pool import get_connection
+        from util.migrations import run_migrations
+        conn = get_connection()
+        try:
+            run_migrations(conn)
+        finally:
+            conn.close()
+    except Exception as e:
+        logger.error(f"[startup] Migration error: {e}")
+
+
+_run_startup_migrations()
+
+
 if __name__ == '__main__':
     os.makedirs("config", exist_ok=True)
     app.run(host='0.0.0.0', port=5000, debug=True)

backend/tools/notes.py

@@ -6,35 +6,6 @@ from auth.token import verify_token
 
 notes_blueprint = Blueprint('notes_tool', __name__)
 
-# Module-level flag: DDL runs at most once per process lifetime.
-# Resets automatically on worker restart, which re-triggers the check.
-_table_ready = False
-
-
-def _ensure_table():
-    global _table_ready
-    if _table_ready:
-        return
-    conn = get_connection()
-    try:
-        cursor = conn.cursor()
-        cursor.execute("""
-            CREATE TABLE IF NOT EXISTS notes (
-                id INT AUTO_INCREMENT PRIMARY KEY,
-                user_id INT NOT NULL,
-                title VARCHAR(255) NOT NULL,
-                content TEXT,
-                language VARCHAR(50) DEFAULT 'text',
-                created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-                updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
-            )
-        """)
-        conn.commit()
-        cursor.close()
-        _table_ready = True
-    finally:
-        conn.close()
-
 
 @notes_blueprint.route('/api/notes', methods=['GET'])
 def get_notes():
@@ -42,12 +13,12 @@ def get_notes():
     if not user:
         return jsonify({"message": "Nicht autorisiert"}), 401
     try:
-        _ensure_table()
         conn = get_connection()
         try:
             cursor = conn.cursor(dictionary=True)
             cursor.execute(
-                "SELECT id, title, content, language, created_at, updated_at FROM notes WHERE user_id = %s ORDER BY updated_at DESC",
+                "SELECT id, title, content, language, created_at, updated_at "
+                "FROM notes WHERE user_id = %s ORDER BY updated_at DESC",
                 (user['id'],)
             )
             notes = cursor.fetchall()
@@ -71,7 +42,6 @@ def create_note():
     if not user:
         return jsonify({"message": "Nicht autorisiert"}), 401
     try:
-        _ensure_table()
         data = request.get_json(silent=True) or {}
         title = data.get("title", "Neue Notiz").strip() or "Neue Notiz"
         content = data.get("content", "")
@@ -103,7 +73,6 @@ def update_note(note_id):
     if not user:
         return jsonify({"message": "Nicht autorisiert"}), 401
     try:
-        _ensure_table()
         data = request.get_json(silent=True) or {}
         title = data.get("title", "").strip() or "Neue Notiz"
         content = data.get("content", "")
@@ -116,7 +85,8 @@ def update_note(note_id):
         try:
             cursor = conn.cursor()
             cursor.execute(
-                "UPDATE notes SET title=%s, content=%s, language=%s, updated_at=%s WHERE id=%s AND user_id=%s",
+                "UPDATE notes SET title=%s, content=%s, language=%s, updated_at=%s "
+                "WHERE id=%s AND user_id=%s",
                 (title, content, language, now, note_id, user['id'])
             )
             conn.commit()
@@ -139,7 +109,6 @@ def delete_note(note_id):
     if not user:
         return jsonify({"message": "Nicht autorisiert"}), 401
     try:
-        _ensure_table()
         conn = get_connection()
         try:
             cursor = conn.cursor()

backend/util/migrations.py

@@ -0,0 +1,161 @@
+"""
+Schema migration system.
+
+Rules for maintainers
+---------------------
+- NEVER modify or delete an existing entry in MIGRATIONS.
+- ALWAYS append new entries at the end with an incremented version number.
+- Use ALTER TABLE to change an existing table, not a new CREATE TABLE.
+- Each entry must be a single, complete SQL statement.
+
+How it works
+------------
+On startup (and after the initial setup form) run_migrations() is called.
+It creates a `schema_migrations` table the first time, checks which versions
+have already been applied, and runs any that are missing.  A full JSON backup
+of all user-data tables is written to the backups/ directory before any
+schema changes are made so data can be recovered if something goes wrong.
+"""
+
+import json
+import os
+from datetime import datetime
+
+from util.logger import logger
+
+# ---------------------------------------------------------------------------
+# Migration registry — append-only
+# ---------------------------------------------------------------------------
+
+MIGRATIONS = [
+    (1, "Create users table", """
+        CREATE TABLE IF NOT EXISTS users (
+            id INT AUTO_INCREMENT PRIMARY KEY,
+            username VARCHAR(50) NOT NULL UNIQUE,
+            password VARCHAR(255) NOT NULL,
+            role VARCHAR(20) NOT NULL
+        )
+    """),
+    (2, "Create websites table", """
+        CREATE TABLE IF NOT EXISTS websites (
+            id INT AUTO_INCREMENT PRIMARY KEY,
+            name VARCHAR(100) NOT NULL,
+            url VARCHAR(255) NOT NULL,
+            description VARCHAR(255) DEFAULT ''
+        )
+    """),
+    (3, "Create notes table", """
+        CREATE TABLE IF NOT EXISTS notes (
+            id INT AUTO_INCREMENT PRIMARY KEY,
+            user_id INT NOT NULL,
+            title VARCHAR(255) NOT NULL,
+            content TEXT,
+            language VARCHAR(50) DEFAULT 'text',
+            created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+            updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+        )
+    """),
+    # ── add new migrations below this line ──────────────────────────────────
+]
+
+# ---------------------------------------------------------------------------
+# Internals
+# ---------------------------------------------------------------------------
+
+_BACKUP_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "backups"
+)
+
+
+def _ensure_migration_table(conn):
+    cur = conn.cursor()
+    cur.execute("""
+        CREATE TABLE IF NOT EXISTS schema_migrations (
+            version     INT          PRIMARY KEY,
+            description VARCHAR(255) NOT NULL,
+            applied_at  DATETIME     DEFAULT CURRENT_TIMESTAMP
+        )
+    """)
+    conn.commit()
+    cur.close()
+
+
+def _applied_versions(conn):
+    cur = conn.cursor()
+    cur.execute("SELECT version FROM schema_migrations")
+    versions = {row[0] for row in cur.fetchall()}
+    cur.close()
+    return versions
+
+
+def backup(conn):
+    """
+    Dump every user-data table to a timestamped JSON file.
+    Returns the path of the written file.
+    Datetime values are serialised as ISO-8601 strings.
+    """
+    os.makedirs(_BACKUP_DIR, exist_ok=True)
+    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+    path = os.path.join(_BACKUP_DIR, f"backup_{ts}.json")
+
+    cur = conn.cursor(dictionary=True)
+    cur.execute("SHOW TABLES")
+    tables = [list(row.values())[0] for row in cur.fetchall()]
+
+    snapshot = {}
+    for table in tables:
+        if table == "schema_migrations":
+            continue
+        cur.execute(f"SELECT * FROM `{table}`")
+        snapshot[table] = [
+            {
+                k: v.isoformat() if hasattr(v, "isoformat") else v
+                for k, v in row.items()
+            }
+            for row in cur.fetchall()
+        ]
+    cur.close()
+
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(snapshot, f, indent=2, ensure_ascii=False)
+
+    logger.info(f"[migrations] Backup written → {path}")
+    return path
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def run_migrations(conn):
+    """
+    Apply every pending migration in version order.
+    A backup is created automatically before the first schema change.
+    Safe to call on every startup — already-applied migrations are skipped.
+    """
+    _ensure_migration_table(conn)
+    applied = _applied_versions(conn)
+    pending = [(v, d, sql) for v, d, sql in MIGRATIONS if v not in applied]
+
+    if not pending:
+        logger.info("[migrations] Schema is up to date.")
+        return
+
+    backup_path = backup(conn)
+    logger.info(
+        f"[migrations] {len(pending)} pending migration(s). "
+        f"Backup: {backup_path}"
+    )
+
+    cur = conn.cursor()
+    for version, description, sql in pending:
+        logger.info(f"[migrations] Applying v{version}: {description}")
+        cur.execute(sql.strip())
+        cur.execute(
+            "INSERT INTO schema_migrations (version, description) VALUES (%s, %s)",
+            (version, description),
+        )
+
+    conn.commit()
+    cur.close()
+    logger.info("[migrations] All migrations applied.")

backend/util/setup_routes.py

@@ -43,6 +43,18 @@ def setup():
         reset_pool()
         if test_connection(db_config):
             initialize_admin_user(db_config)
+            # Apply schema migrations immediately so all tables exist before
+            # the user lands on the main page.
+            try:
+                from util.db_pool import get_connection
+                from util.migrations import run_migrations
+                conn = get_connection()
+                try:
+                    run_migrations(conn)
+                finally:
+                    conn.close()
+            except Exception as e:
+                logger.warning(f"[setup] Post-setup migration error: {e}")
             return redirect('/')
         else:
             return "Verbindung fehlgeschlagen. Bitte zurück und prüfen.", 500

docker-compose.dev.yml

@@ -10,8 +10,10 @@ services:
     volumes:
       - config-data:/config
       - logs-data:/app/backend/logs
+      - backups-data:/app/backend/backups
     working_dir: /app/backend
     command: python app.py
 volumes:
   config-data:
   logs-data:
+  backups-data: