2.0.0 Beta : Self-hosted Release

2026-08-09 02:03:29 +08:00
parent 63a0751aba
commit 37cb05eb70
33 changed files with 810 additions and 31 deletions
@@ -22,6 +22,9 @@ This directory separates four different responsibilities:
 8. Missing metrics are stored as `NULL`, not fabricated zero values.
 9. `Admin -> Data Integrity` is the operational source of truth.
 10. Web schema changes increment `Config.WEB_SCHEMA_VERSION`.
+11. `YRTV_DATA_DIR` owns runtime databases; code schemas always stay in Git.
+12. Container and production deployments must keep runtime data outside the
+    image and source checkout.

 ## Entry Points

@@ -31,11 +34,16 @@ make l2          # Rebuild normalized facts
 make l3          # Rebuild active-roster features
 make pipeline    # Run L1 -> L2 -> L3 with backup and validation
 make check       # Compile and run tests
+make bootstrap   # Initialize an empty runtime data root
+make prepare-data # Copy legacy data into runtime-data
 ```

 ## Directory Policy

- `L1/`, `L2/`, `L3/`, `Web/`: active code, schema and database.
+- Without `YRTV_DATA_DIR`, `L1/`, `L2/`, `L3/`, `Web/` remain the compatible
+  development runtime.
+- With `YRTV_DATA_DIR`, those folders provide code and schemas while databases
+  live under the configured external directory.
 - `backups/`: generated rollback snapshots; ignored by Git.
 - `schema_bkp/`: historical schema research only; not used at runtime.
 - `L1B/`: reserved demo-parser integration; not used at runtime.
@@ -0,0 +1,66 @@
+import sqlite3
+
+from database.paths import (
+    L1_DB,
+    L2_DB,
+    L2_SCHEMA,
+    L3_DB,
+    L3_SCHEMA,
+    RUNTIME_DATA_ROOT,
+    ensure_runtime_directories,
+)
+
+
+def _apply_schema(database_path, schema_path):
+    db = sqlite3.connect(str(database_path))
+    try:
+        db.execute('PRAGMA foreign_keys = ON')
+        db.executescript(schema_path.read_text(encoding='utf-8'))
+        result = db.execute('PRAGMA quick_check').fetchone()[0]
+        if result != 'ok':
+            raise RuntimeError(
+                f'{database_path.name} quick_check failed: {result}'
+            )
+        db.commit()
+    finally:
+        db.close()
+
+
+def bootstrap_runtime():
+    ensure_runtime_directories()
+
+    l1 = sqlite3.connect(str(L1_DB))
+    try:
+        l1.execute(
+            """
+            CREATE TABLE IF NOT EXISTS raw_iframe_network (
+                match_id TEXT PRIMARY KEY,
+                content TEXT NOT NULL,
+                processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+            """
+        )
+        l1.commit()
+    finally:
+        l1.close()
+
+    _apply_schema(L2_DB, L2_SCHEMA)
+    _apply_schema(L3_DB, L3_SCHEMA)
+
+    from web.database import initialize_web_db
+    initialize_web_db()
+
+    return {
+        'data_root': str(RUNTIME_DATA_ROOT),
+        'l1': str(L1_DB),
+        'l2': str(L2_DB),
+        'l3': str(L3_DB),
+    }
+
+
+if __name__ == '__main__':
+    result = bootstrap_runtime()
+    print('YRTV runtime initialized')
+    for key, value in result.items():
+        print(f'  {key}: {value}')
+
@@ -0,0 +1,107 @@
+import argparse
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sqlite3
+
+from database.paths import DATABASE_CODE_ROOT, PROJECT_ROOT
+
+
+DATABASE_LAYOUT = {
+    'L1': ('L1', 'L1.db'),
+    'L2': ('L2', 'L2.db'),
+    'L3': ('L3', 'L3.db'),
+    'Web': ('Web', 'Web_App.sqlite'),
+}
+
+
+def _sha256(path):
+    digest = hashlib.sha256()
+    with path.open('rb') as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b''):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _sqlite_copy(source, target):
+    target.parent.mkdir(parents=True, exist_ok=True)
+    source_db = sqlite3.connect(str(source))
+    target_db = sqlite3.connect(str(target))
+    try:
+        source_db.backup(target_db)
+        result = target_db.execute('PRAGMA quick_check').fetchone()[0]
+        if result != 'ok':
+            raise RuntimeError(f'{target.name} quick_check failed: {result}')
+    finally:
+        source_db.close()
+        target_db.close()
+
+
+def migrate_data(source_root, target_root, include_imports=False):
+    source_root = Path(source_root).expanduser().resolve()
+    target_root = Path(target_root).expanduser().resolve()
+    if source_root == target_root:
+        raise ValueError('Source and target data roots must differ')
+
+    target_root.mkdir(parents=True, exist_ok=True)
+    manifest = {
+        'created_at': datetime.now(timezone.utc).isoformat(),
+        'source_root': str(source_root),
+        'target_root': str(target_root),
+        'databases': {},
+    }
+    for name, (directory, filename) in DATABASE_LAYOUT.items():
+        source = source_root / directory / filename
+        target = target_root / directory / filename
+        if not source.exists():
+            raise FileNotFoundError(f'Missing source database: {source}')
+        _sqlite_copy(source, target)
+        manifest['databases'][name] = {
+            'path': str(target),
+            'size_bytes': target.stat().st_size,
+            'sha256': _sha256(target),
+            'quick_check': 'ok',
+        }
+
+    if include_imports:
+        legacy_imports = PROJECT_ROOT / 'output_arena'
+        if legacy_imports.exists():
+            destination = target_root / 'imports'
+            shutil.copytree(
+                legacy_imports,
+                destination,
+                dirs_exist_ok=True,
+            )
+            manifest['imports'] = str(destination)
+
+    manifest_path = target_root / 'migration-manifest.json'
+    manifest_path.write_text(
+        json.dumps(manifest, ensure_ascii=True, indent=2),
+        encoding='utf-8',
+    )
+    return manifest_path
+
+
+def _parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--source',
+        default=str(DATABASE_CODE_ROOT),
+        help='Existing runtime data root',
+    )
+    parser.add_argument('--target', required=True)
+    parser.add_argument('--include-imports', action='store_true')
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = _parse_args()
+    path = migrate_data(
+        args.source,
+        args.target,
+        include_imports=args.include_imports,
+    )
+    print(f'Migration completed: {path}')
+
@@ -1,26 +1,43 @@
+import os
 from pathlib import Path


 PROJECT_ROOT = Path(__file__).resolve().parent.parent
-DATABASE_ROOT = PROJECT_ROOT / 'database'
+DATABASE_CODE_ROOT = PROJECT_ROOT / 'database'

-L1_DIR = DATABASE_ROOT / 'L1'
-L2_DIR = DATABASE_ROOT / 'L2'
-L3_DIR = DATABASE_ROOT / 'L3'
-WEB_DIR = DATABASE_ROOT / 'Web'
+_runtime_override = os.environ.get('YRTV_DATA_DIR')
+RUNTIME_DATA_ROOT = (
+    Path(_runtime_override).expanduser().resolve()
+    if _runtime_override
+    else DATABASE_CODE_ROOT
+)
+
+L1_DIR = RUNTIME_DATA_ROOT / 'L1'
+L2_DIR = RUNTIME_DATA_ROOT / 'L2'
+L3_DIR = RUNTIME_DATA_ROOT / 'L3'
+WEB_DIR = RUNTIME_DATA_ROOT / 'Web'

 L1_DB = L1_DIR / 'L1.db'
 L2_DB = L2_DIR / 'L2.db'
 L3_DB = L3_DIR / 'L3.db'
 WEB_DB = WEB_DIR / 'Web_App.sqlite'

-L2_SCHEMA = L2_DIR / 'schema.sql'
-L3_SCHEMA = L3_DIR / 'schema.sql'
-WEB_SCHEMA = WEB_DIR / 'schema.sql'
+L2_SCHEMA = DATABASE_CODE_ROOT / 'L2' / 'schema.sql'
+L3_SCHEMA = DATABASE_CODE_ROOT / 'L3' / 'schema.sql'
+WEB_SCHEMA = DATABASE_CODE_ROOT / 'Web' / 'schema.sql'

-OUTPUT_ARENA = PROJECT_ROOT / 'output_arena'
-BACKUP_ROOT = DATABASE_ROOT / 'backups'
-PIPELINE_LOCK = DATABASE_ROOT / '.pipeline.lock'
+_import_override = os.environ.get('YRTV_IMPORT_DIR')
+OUTPUT_ARENA = (
+    Path(_import_override).expanduser().resolve()
+    if _import_override
+    else (
+        RUNTIME_DATA_ROOT / 'imports'
+        if _runtime_override
+        else PROJECT_ROOT / 'output_arena'
+    )
+)
+BACKUP_ROOT = RUNTIME_DATA_ROOT / 'backups'
+PIPELINE_LOCK = RUNTIME_DATA_ROOT / '.pipeline.lock'


 def ensure_runtime_directories():
@@ -33,4 +50,3 @@ def ensure_runtime_directories():
        BACKUP_ROOT,
    ):
        path.mkdir(parents=True, exist_ok=True)
-