2.0.0 Alpha: Data Refinery

database/L1/L1_Builder.py

@@ -14,13 +14,18 @@ import os
 import json
 import sqlite3
 import glob
-import argparse  # Added
+import argparse
+import sys
 
-# Paths
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-OUTPUT_ARENA_DIR = os.path.join(BASE_DIR, 'output_arena')
-DB_DIR = os.path.join(BASE_DIR, 'database', 'L1')
-DB_PATH = os.path.join(DB_DIR, 'L1.db')
+if BASE_DIR not in sys.path:
+    sys.path.insert(0, BASE_DIR)
+
+from database.paths import L1_DB, L1_DIR, OUTPUT_ARENA
+
+OUTPUT_ARENA_DIR = str(OUTPUT_ARENA)
+DB_DIR = str(L1_DIR)
+DB_PATH = str(L1_DB)
 
 def init_db():
     if not os.path.exists(DB_DIR):
@@ -65,6 +70,7 @@ def process_files():
     
     count = 0
     skipped = 0
+    errors = 0
     
     for file_path in files:
         try:
@@ -92,11 +98,14 @@ def process_files():
                 conn.commit()
                 
         except Exception as e:
+            errors += 1
             print(f"Error processing {file_path}: {e}")
             
     conn.commit()
     conn.close()
-    print(f"Finished. Processed: {count}, Skipped: {skipped}.")
+    print(f"Finished. Processed: {count}, Skipped: {skipped}, Errors: {errors}.")
+    if errors:
+        raise RuntimeError(f"L1 ingestion failed for {errors} file(s)")
 
 if __name__ == '__main__':
-    process_files()
+    process_files()

database/L1/README.md

@@ -1,16 +1,25 @@
-L1A 5eplay平台网页爬虫原始数据。
+# L1 Raw Match Store
 
-## ETL Step 1:
-从原始json数据库提取到L1A级数据库中。
-`output_arena/*/iframe_network.json` -> `database/L1A/L1A.sqlite`
+L1 stores one complete 5E network capture per match without transforming its
+payload.
 
-### 脚本说明
-- **脚本位置**: `ETL/L1A.py`
-- **功能**: 自动遍历 `output_arena` 目录下所有的 `iframe_network.json` 文件，提取原始内容并以 `match_id` (文件夹名) 为主键存入 `L1A.sqlite` 数据库的 `raw_iframe_network` 表中。
+## Runtime Files
 
-### 运行方式
-使用项目指定的 Python 环境运行脚本：
+- Database: `database/L1/L1.db`
+- Builder: `database/L1/L1_Builder.py`
+- Input: `output_arena/<match_id>/iframe_network.json`
+- Primary key: `raw_iframe_network.match_id`
+
+## Commands
 
 ```bash
-C:/ProgramData/anaconda3/python.exe ETL/L1A.py
+make l1
+make pipeline
 ```
+
+Normal ingestion is incremental. `--force` re-reads every capture currently
+present in `output_arena`.
+
+`L1A.db` and the historical `database/L1A/L1A.sqlite` path are retired. L1B is
+reserved for a future demo-parser source and is not part of the runtime
+pipeline.