2.0.0 Alpha: Data Refinery
This commit is contained in:
@@ -14,13 +14,18 @@ import os
|
||||
import json
|
||||
import sqlite3
|
||||
import glob
|
||||
import argparse # Added
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
# Paths
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
OUTPUT_ARENA_DIR = os.path.join(BASE_DIR, 'output_arena')
|
||||
DB_DIR = os.path.join(BASE_DIR, 'database', 'L1')
|
||||
DB_PATH = os.path.join(DB_DIR, 'L1.db')
|
||||
if BASE_DIR not in sys.path:
|
||||
sys.path.insert(0, BASE_DIR)
|
||||
|
||||
from database.paths import L1_DB, L1_DIR, OUTPUT_ARENA
|
||||
|
||||
OUTPUT_ARENA_DIR = str(OUTPUT_ARENA)
|
||||
DB_DIR = str(L1_DIR)
|
||||
DB_PATH = str(L1_DB)
|
||||
|
||||
def init_db():
|
||||
if not os.path.exists(DB_DIR):
|
||||
@@ -65,6 +70,7 @@ def process_files():
|
||||
|
||||
count = 0
|
||||
skipped = 0
|
||||
errors = 0
|
||||
|
||||
for file_path in files:
|
||||
try:
|
||||
@@ -92,11 +98,14 @@ def process_files():
|
||||
conn.commit()
|
||||
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
print(f"Error processing {file_path}: {e}")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Finished. Processed: {count}, Skipped: {skipped}.")
|
||||
print(f"Finished. Processed: {count}, Skipped: {skipped}, Errors: {errors}.")
|
||||
if errors:
|
||||
raise RuntimeError(f"L1 ingestion failed for {errors} file(s)")
|
||||
|
||||
if __name__ == '__main__':
|
||||
process_files()
|
||||
process_files()
|
||||
|
||||
+19
-10
@@ -1,16 +1,25 @@
|
||||
L1A 5eplay平台网页爬虫原始数据。
|
||||
# L1 Raw Match Store
|
||||
|
||||
## ETL Step 1:
|
||||
从原始json数据库提取到L1A级数据库中。
|
||||
`output_arena/*/iframe_network.json` -> `database/L1A/L1A.sqlite`
|
||||
L1 stores one complete 5E network capture per match without transforming its
|
||||
payload.
|
||||
|
||||
### 脚本说明
|
||||
- **脚本位置**: `ETL/L1A.py`
|
||||
- **功能**: 自动遍历 `output_arena` 目录下所有的 `iframe_network.json` 文件，提取原始内容并以 `match_id` (文件夹名) 为主键存入 `L1A.sqlite` 数据库的 `raw_iframe_network` 表中。
|
||||
## Runtime Files
|
||||
|
||||
### 运行方式
|
||||
使用项目指定的 Python 环境运行脚本：
|
||||
- Database: `database/L1/L1.db`
|
||||
- Builder: `database/L1/L1_Builder.py`
|
||||
- Input: `output_arena/<match_id>/iframe_network.json`
|
||||
- Primary key: `raw_iframe_network.match_id`
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
C:/ProgramData/anaconda3/python.exe ETL/L1A.py
|
||||
make l1
|
||||
make pipeline
|
||||
```
|
||||
|
||||
Normal ingestion is incremental. `--force` re-reads every capture currently
|
||||
present in `output_arena`.
|
||||
|
||||
`L1A.db` and the historical `database/L1A/L1A.sqlite` path are retired. L1B is
|
||||
reserved for a future demo-parser source and is not part of the runtime
|
||||
pipeline.
|
||||
|
||||
Binary file not shown.
@@ -7,14 +7,20 @@ from dataclasses import dataclass, field
|
||||
from typing import List, Dict, Optional, Any, Tuple
|
||||
from datetime import datetime
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
if PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, PROJECT_ROOT)
|
||||
|
||||
from database.paths import L1_DB, L2_DB, L2_SCHEMA
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Constants
|
||||
L1A_DB_PATH = 'database/L1/L1.db'
|
||||
L2_DB_PATH = 'database/L2/L2.db'
|
||||
SCHEMA_PATH = 'database/L2/schema.sql'
|
||||
L1A_DB_PATH = str(L1_DB)
|
||||
L2_DB_PATH = str(L2_DB)
|
||||
SCHEMA_PATH = str(L2_SCHEMA)
|
||||
|
||||
# --- Data Structures for Unification ---
|
||||
|
||||
@@ -1238,6 +1244,10 @@ def process_matches():
|
||||
l1_conn.close()
|
||||
l2_conn.close()
|
||||
logger.info(f"\nDone. Processed {count} matches ({success_count} success, {error_count} errors).")
|
||||
if error_count:
|
||||
raise RuntimeError(
|
||||
f"L2 build failed for {error_count}/{count} matches"
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
process_matches()
|
||||
|
||||
@@ -636,3 +636,24 @@ SELECT
|
||||
FROM fact_match_players fmp
|
||||
JOIN fact_matches fm ON fmp.match_id = fm.match_id
|
||||
GROUP BY fmp.steam_id_64, fm.map_name;
|
||||
|
||||
-- ==========================================
|
||||
-- Operational query indexes
|
||||
-- ==========================================
|
||||
CREATE INDEX IF NOT EXISTS idx_match_players_player_match
|
||||
ON fact_match_players(steam_id_64, match_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_match_players_match_team
|
||||
ON fact_match_players(match_id, team_id, steam_id_64);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_match_players_party
|
||||
ON fact_match_players(match_id, match_team_id, steam_id_64);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_round_events_victim
|
||||
ON fact_round_events(victim_steam_id, match_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_economy_player_match
|
||||
ON fact_round_player_economy(steam_id_64, match_id, round_num);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_matches_map_time
|
||||
ON fact_matches(map_name, start_time DESC);
|
||||
|
||||
Binary file not shown.
+490
-12
@@ -6,6 +6,8 @@ import sqlite3
|
||||
import json
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
from collections import defaultdict, deque
|
||||
from typing import Optional
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
@@ -15,10 +17,14 @@ logger = logging.getLogger(__name__)
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Points to database/ directory
|
||||
PROJECT_ROOT = os.path.dirname(BASE_DIR) # Points to project root
|
||||
sys.path.insert(0, PROJECT_ROOT) # Add project root to Python path
|
||||
L2_DB_PATH = os.path.join(BASE_DIR, 'L2', 'L2.db')
|
||||
L3_DB_PATH = os.path.join(BASE_DIR, 'L3', 'L3.db')
|
||||
WEB_DB_PATH = os.path.join(BASE_DIR, 'Web', 'Web_App.sqlite')
|
||||
SCHEMA_PATH = os.path.join(BASE_DIR, 'L3', 'schema.sql')
|
||||
|
||||
from database.paths import L2_DB, L3_DB, L3_SCHEMA, WEB_DB
|
||||
|
||||
L2_DB_PATH = str(L2_DB)
|
||||
L3_DB_PATH = str(L3_DB)
|
||||
L3_BACKUP_PATH = f"{L3_DB_PATH}.bak"
|
||||
WEB_DB_PATH = str(WEB_DB)
|
||||
SCHEMA_PATH = str(L3_SCHEMA)
|
||||
|
||||
def _get_existing_columns(conn, table_name):
|
||||
cur = conn.execute(f"PRAGMA table_info({table_name})")
|
||||
@@ -76,7 +82,28 @@ def _get_team_players():
|
||||
try:
|
||||
conn = sqlite3.connect(WEB_DB_PATH)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT player_ids_json FROM team_lineups")
|
||||
columns = {
|
||||
row[1] for row in cursor.execute("PRAGMA table_info(team_lineups)")
|
||||
}
|
||||
if 'is_active' in columns:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT player_ids_json
|
||||
FROM team_lineups
|
||||
WHERE is_active = 1
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
else:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT player_ids_json
|
||||
FROM team_lineups
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
steam_ids = set()
|
||||
@@ -150,7 +177,25 @@ def _build_player_record(steam_id: str):
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
def main(force_all: bool = False, workers: int = 1):
|
||||
def _backup_l3_database(source_path=L3_DB_PATH, backup_path=L3_BACKUP_PATH):
|
||||
if not os.path.exists(source_path):
|
||||
return None
|
||||
|
||||
source = sqlite3.connect(source_path)
|
||||
backup = sqlite3.connect(backup_path)
|
||||
try:
|
||||
source.backup(backup)
|
||||
result = backup.execute("PRAGMA quick_check").fetchone()[0]
|
||||
if result != 'ok':
|
||||
raise RuntimeError(f"L3 backup quick_check failed: {result}")
|
||||
finally:
|
||||
source.close()
|
||||
backup.close()
|
||||
logger.info("L3 backup created at %s", backup_path)
|
||||
return backup_path
|
||||
|
||||
|
||||
def main(force_all: bool = False, workers: int = 1, create_backup: bool = True):
|
||||
"""
|
||||
Main L3 feature building pipeline using modular processors
|
||||
"""
|
||||
@@ -158,6 +203,9 @@ def main(force_all: bool = False, workers: int = 1):
|
||||
logger.info("Starting L3 Builder with 5-Tier Architecture")
|
||||
logger.info("========================================")
|
||||
|
||||
if create_backup:
|
||||
_backup_l3_database()
|
||||
|
||||
# 1. Ensure Schema is up to date
|
||||
init_db()
|
||||
|
||||
@@ -181,6 +229,7 @@ def main(force_all: bool = False, workers: int = 1):
|
||||
conn_l3 = sqlite3.connect(L3_DB_PATH)
|
||||
|
||||
try:
|
||||
conn_l3.execute("BEGIN IMMEDIATE")
|
||||
cursor_l2 = conn_l2.cursor()
|
||||
if force_all:
|
||||
logger.info("Force mode enabled: building L3 for all players in L2.")
|
||||
@@ -240,7 +289,6 @@ def main(force_all: bool = False, workers: int = 1):
|
||||
)
|
||||
success_count += 1
|
||||
if processed_count % 2 == 0:
|
||||
conn_l3.commit()
|
||||
logger.info(f"Progress: {processed_count}/{total_players} ({success_count} success, {error_count} errors)")
|
||||
else:
|
||||
for idx, row in enumerate(players, 1):
|
||||
@@ -268,10 +316,20 @@ def main(force_all: bool = False, workers: int = 1):
|
||||
|
||||
processed_count = idx
|
||||
if processed_count % 2 == 0:
|
||||
conn_l3.commit()
|
||||
logger.info(f"Progress: {processed_count}/{total_players} ({success_count} success, {error_count} errors)")
|
||||
|
||||
# Final commit
|
||||
|
||||
if error_count:
|
||||
raise RuntimeError(
|
||||
f"L3 feature build failed for {error_count}/{total_players} players"
|
||||
)
|
||||
|
||||
processed_ids = [str(row[0]) for row in players]
|
||||
_update_percentiles(conn_l3, processed_ids)
|
||||
_rebuild_auxiliary_marts(conn_l2, conn_l3, processed_ids)
|
||||
|
||||
quick_check = conn_l3.execute("PRAGMA quick_check").fetchone()[0]
|
||||
if quick_check != 'ok':
|
||||
raise RuntimeError(f"L3 quick_check failed before commit: {quick_check}")
|
||||
conn_l3.commit()
|
||||
|
||||
logger.info("========================================")
|
||||
@@ -283,9 +341,11 @@ def main(force_all: bool = False, workers: int = 1):
|
||||
logger.info("========================================")
|
||||
|
||||
except Exception as e:
|
||||
conn_l3.rollback()
|
||||
logger.error(f"Fatal error during L3 build: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise
|
||||
|
||||
finally:
|
||||
conn_l2.close()
|
||||
@@ -313,7 +373,7 @@ def _get_round_count(steam_id: str, conn_l2: sqlite3.Connection) -> int:
|
||||
|
||||
|
||||
def _upsert_features(conn_l3: sqlite3.Connection, steam_id: str, features: dict,
|
||||
match_count: int, round_count: int, conn_l2: sqlite3.Connection | None,
|
||||
match_count: int, round_count: int, conn_l2: Optional[sqlite3.Connection],
|
||||
first_match_date=None, last_match_date=None):
|
||||
"""
|
||||
Insert or update player features in dm_player_features
|
||||
@@ -353,12 +413,430 @@ def _upsert_features(conn_l3: sqlite3.Connection, steam_id: str, features: dict,
|
||||
|
||||
cursor_l3.execute(sql, values)
|
||||
|
||||
|
||||
def _rebuild_auxiliary_marts(conn_l2, conn_l3, steam_ids):
|
||||
"""Rebuild player-grain marts used by profiles and trend APIs."""
|
||||
if not steam_ids:
|
||||
return
|
||||
|
||||
logger.info("Rebuilding L3 match, map and weapon marts")
|
||||
total_history = 0
|
||||
total_maps = 0
|
||||
total_weapons = 0
|
||||
total_periods = 0
|
||||
total_records = 0
|
||||
|
||||
for start in range(0, len(steam_ids), 400):
|
||||
chunk = steam_ids[start:start + 400]
|
||||
placeholders = ','.join('?' for _ in chunk)
|
||||
|
||||
for table in (
|
||||
'dm_player_match_history',
|
||||
'dm_player_map_stats',
|
||||
'dm_player_weapon_stats',
|
||||
'dm_player_period_stats',
|
||||
'dm_player_records',
|
||||
):
|
||||
conn_l3.execute(
|
||||
f"DELETE FROM {table} WHERE steam_id_64 IN ({placeholders})",
|
||||
chunk,
|
||||
)
|
||||
|
||||
history_rows = conn_l2.execute(
|
||||
f"""
|
||||
SELECT
|
||||
mp.steam_id_64,
|
||||
mp.match_id,
|
||||
m.start_time,
|
||||
mp.rating,
|
||||
mp.kd_ratio,
|
||||
mp.adr,
|
||||
mp.kast,
|
||||
mp.is_win,
|
||||
m.map_name,
|
||||
mp.kills,
|
||||
mp.deaths,
|
||||
mp.headshot_count,
|
||||
(
|
||||
SELECT AVG(teammate.rating)
|
||||
FROM fact_match_players teammate
|
||||
WHERE teammate.match_id = mp.match_id
|
||||
AND teammate.team_id = mp.team_id
|
||||
AND teammate.steam_id_64 != mp.steam_id_64
|
||||
) AS teammate_avg_rating
|
||||
FROM fact_match_players mp
|
||||
JOIN fact_matches m ON m.match_id = mp.match_id
|
||||
WHERE mp.steam_id_64 IN ({placeholders})
|
||||
ORDER BY mp.steam_id_64, m.start_time, mp.match_id
|
||||
""",
|
||||
chunk,
|
||||
).fetchall()
|
||||
|
||||
history_values = []
|
||||
player_state = defaultdict(lambda: {
|
||||
'sequence': 0,
|
||||
'rating_sum': 0.0,
|
||||
'recent': deque(maxlen=10),
|
||||
})
|
||||
for row in history_rows:
|
||||
steam_id = str(row[0])
|
||||
state = player_state[steam_id]
|
||||
rating = float(row[3] or 0.0)
|
||||
state['sequence'] += 1
|
||||
state['rating_sum'] += rating
|
||||
state['recent'].append(rating)
|
||||
history_values.append((
|
||||
steam_id,
|
||||
row[1],
|
||||
row[2],
|
||||
state['sequence'],
|
||||
row[3],
|
||||
row[4],
|
||||
row[5],
|
||||
row[6],
|
||||
row[7],
|
||||
row[8],
|
||||
None,
|
||||
row[12],
|
||||
state['rating_sum'] / state['sequence'],
|
||||
sum(state['recent']) / len(state['recent']),
|
||||
))
|
||||
|
||||
conn_l3.executemany(
|
||||
"""
|
||||
INSERT INTO dm_player_match_history (
|
||||
steam_id_64, match_id, match_date, match_sequence,
|
||||
rating, kd_ratio, adr, kast, is_win, map_name,
|
||||
opponent_avg_elo, teammate_avg_rating,
|
||||
cumulative_rating, rolling_10_rating
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
history_values,
|
||||
)
|
||||
total_history += len(history_values)
|
||||
|
||||
map_rows = conn_l2.execute(
|
||||
f"""
|
||||
SELECT
|
||||
mp.steam_id_64,
|
||||
m.map_name,
|
||||
COUNT(*) AS matches,
|
||||
SUM(CASE WHEN mp.is_win = 1 THEN 1 ELSE 0 END) AS wins,
|
||||
AVG(mp.rating) AS avg_rating,
|
||||
AVG(mp.kd_ratio) AS avg_kd,
|
||||
AVG(mp.adr) AS avg_adr,
|
||||
AVG(mp.kast) AS avg_kast,
|
||||
MAX(mp.rating) AS best_rating,
|
||||
MIN(mp.rating) AS worst_rating
|
||||
FROM fact_match_players mp
|
||||
JOIN fact_matches m ON m.match_id = mp.match_id
|
||||
WHERE mp.steam_id_64 IN ({placeholders})
|
||||
AND m.map_name IS NOT NULL
|
||||
AND m.map_name != ''
|
||||
GROUP BY mp.steam_id_64, m.map_name
|
||||
""",
|
||||
chunk,
|
||||
).fetchall()
|
||||
map_values = [
|
||||
tuple(row[:4]) + (
|
||||
(row[3] or 0) / row[2] if row[2] else 0.0,
|
||||
) + tuple(row[4:])
|
||||
for row in map_rows
|
||||
]
|
||||
conn_l3.executemany(
|
||||
"""
|
||||
INSERT INTO dm_player_map_stats (
|
||||
steam_id_64, map_name, matches, wins, win_rate,
|
||||
avg_rating, avg_kd, avg_adr, avg_kast,
|
||||
best_rating, worst_rating
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
map_values,
|
||||
)
|
||||
total_maps += len(map_values)
|
||||
|
||||
round_counts = {
|
||||
str(row[0]): int(row[1] or 0)
|
||||
for row in conn_l2.execute(
|
||||
f"""
|
||||
SELECT steam_id_64, SUM(round_total)
|
||||
FROM fact_match_players
|
||||
WHERE steam_id_64 IN ({placeholders})
|
||||
GROUP BY steam_id_64
|
||||
""",
|
||||
chunk,
|
||||
)
|
||||
}
|
||||
weapon_rows = conn_l2.execute(
|
||||
f"""
|
||||
SELECT
|
||||
attacker_steam_id,
|
||||
weapon,
|
||||
COUNT(*) AS total_kills,
|
||||
SUM(CASE WHEN is_headshot = 1 THEN 1 ELSE 0 END) AS total_headshots,
|
||||
COUNT(DISTINCT match_id || ':' || round_num) AS usage_rounds
|
||||
FROM fact_round_events
|
||||
WHERE event_type = 'kill'
|
||||
AND attacker_steam_id IN ({placeholders})
|
||||
AND weapon IS NOT NULL
|
||||
AND weapon != ''
|
||||
GROUP BY attacker_steam_id, weapon
|
||||
""",
|
||||
chunk,
|
||||
).fetchall()
|
||||
weapon_values = []
|
||||
for row in weapon_rows:
|
||||
rounds = round_counts.get(str(row[0]), 0)
|
||||
kills = int(row[2] or 0)
|
||||
headshots = int(row[3] or 0)
|
||||
usage_rounds = int(row[4] or 0)
|
||||
hs_rate = headshots / kills if kills else 0.0
|
||||
usage_rate = usage_rounds / rounds if rounds else 0.0
|
||||
kills_per_round = kills / rounds if rounds else 0.0
|
||||
effectiveness = kills / usage_rounds if usage_rounds else 0.0
|
||||
weapon_values.append((
|
||||
str(row[0]),
|
||||
row[1],
|
||||
kills,
|
||||
headshots,
|
||||
hs_rate,
|
||||
usage_rounds,
|
||||
usage_rate,
|
||||
kills_per_round,
|
||||
effectiveness,
|
||||
))
|
||||
conn_l3.executemany(
|
||||
"""
|
||||
INSERT INTO dm_player_weapon_stats (
|
||||
steam_id_64, weapon_name, total_kills, total_headshots,
|
||||
hs_rate, usage_rounds, usage_rate,
|
||||
avg_kills_per_round, effectiveness_score
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
weapon_values,
|
||||
)
|
||||
total_weapons += len(weapon_values)
|
||||
|
||||
period_values = _calculate_period_rows(history_rows)
|
||||
conn_l3.executemany(
|
||||
"""
|
||||
INSERT INTO dm_player_period_stats (
|
||||
steam_id_64, period_key, period_label,
|
||||
period_start, period_end, matches, wins, win_rate,
|
||||
avg_rating, avg_kd, avg_adr, avg_kast,
|
||||
total_kills, total_deaths, sample_reliable
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
period_values,
|
||||
)
|
||||
total_periods += len(period_values)
|
||||
|
||||
record_values = _calculate_record_rows(history_rows)
|
||||
conn_l3.executemany(
|
||||
"""
|
||||
INSERT INTO dm_player_records (
|
||||
steam_id_64, record_key, record_label, record_value,
|
||||
match_id, map_name, match_date
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
record_values,
|
||||
)
|
||||
total_records += len(record_values)
|
||||
|
||||
logger.info(
|
||||
"Auxiliary marts rebuilt: %s history, %s map, %s weapon, "
|
||||
"%s period, %s record rows",
|
||||
total_history,
|
||||
total_maps,
|
||||
total_weapons,
|
||||
total_periods,
|
||||
total_records,
|
||||
)
|
||||
|
||||
|
||||
def _group_player_match_rows(history_rows):
|
||||
grouped = defaultdict(list)
|
||||
for row in history_rows:
|
||||
grouped[str(row['steam_id_64'])].append(row)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda row: (row['start_time'] or 0, row['match_id']))
|
||||
return grouped
|
||||
|
||||
|
||||
def _safe_average(rows, key):
|
||||
values = [float(row[key]) for row in rows if row[key] is not None]
|
||||
return sum(values) / len(values) if values else None
|
||||
|
||||
|
||||
def _calculate_period_rows(history_rows):
|
||||
result = []
|
||||
for steam_id, all_rows in _group_player_match_rows(history_rows).items():
|
||||
latest_time = max(int(row['start_time'] or 0) for row in all_rows)
|
||||
period_groups = [
|
||||
('career', '生涯', all_rows),
|
||||
('last_10', '最近 10 场', all_rows[-10:]),
|
||||
('last_20', '最近 20 场', all_rows[-20:]),
|
||||
('last_30', '最近 30 场', all_rows[-30:]),
|
||||
(
|
||||
'days_30',
|
||||
'最近 30 天',
|
||||
[
|
||||
row for row in all_rows
|
||||
if int(row['start_time'] or 0) >= latest_time - 30 * 86400
|
||||
],
|
||||
),
|
||||
(
|
||||
'days_90',
|
||||
'最近 90 天',
|
||||
[
|
||||
row for row in all_rows
|
||||
if int(row['start_time'] or 0) >= latest_time - 90 * 86400
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
for period_key, period_label, rows in period_groups:
|
||||
if not rows:
|
||||
continue
|
||||
matches = len(rows)
|
||||
wins = sum(1 for row in rows if row['is_win'])
|
||||
kills = sum(int(row['kills'] or 0) for row in rows)
|
||||
deaths = sum(int(row['deaths'] or 0) for row in rows)
|
||||
result.append((
|
||||
steam_id,
|
||||
period_key,
|
||||
period_label,
|
||||
min(int(row['start_time'] or 0) for row in rows),
|
||||
max(int(row['start_time'] or 0) for row in rows),
|
||||
matches,
|
||||
wins,
|
||||
wins / matches,
|
||||
_safe_average(rows, 'rating'),
|
||||
kills / deaths if deaths else float(kills),
|
||||
_safe_average(rows, 'adr'),
|
||||
_safe_average(rows, 'kast'),
|
||||
kills,
|
||||
deaths,
|
||||
1 if matches >= 10 else 0,
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
def _calculate_record_rows(history_rows):
|
||||
result = []
|
||||
metric_definitions = (
|
||||
('highest_rating', '最高 Rating', 'rating'),
|
||||
('most_kills', '单场最多击杀', 'kills'),
|
||||
('highest_adr', '单场最高 ADR', 'adr'),
|
||||
('highest_kd', '单场最高 K/D', 'kd_ratio'),
|
||||
('most_headshots', '单场最多爆头', 'headshot_count'),
|
||||
)
|
||||
|
||||
for steam_id, rows in _group_player_match_rows(history_rows).items():
|
||||
for record_key, record_label, field in metric_definitions:
|
||||
candidates = [row for row in rows if row[field] is not None]
|
||||
if not candidates:
|
||||
continue
|
||||
best = max(
|
||||
candidates,
|
||||
key=lambda row: (
|
||||
float(row[field]),
|
||||
int(row['start_time'] or 0),
|
||||
),
|
||||
)
|
||||
result.append((
|
||||
steam_id,
|
||||
record_key,
|
||||
record_label,
|
||||
float(best[field]),
|
||||
best['match_id'],
|
||||
best['map_name'],
|
||||
best['start_time'],
|
||||
))
|
||||
|
||||
longest_streak = 0
|
||||
current_streak = 0
|
||||
streak_end = None
|
||||
for row in rows:
|
||||
if row['is_win']:
|
||||
current_streak += 1
|
||||
if current_streak >= longest_streak:
|
||||
longest_streak = current_streak
|
||||
streak_end = row
|
||||
else:
|
||||
current_streak = 0
|
||||
if streak_end is not None:
|
||||
result.append((
|
||||
steam_id,
|
||||
'longest_win_streak',
|
||||
'最长连胜',
|
||||
float(longest_streak),
|
||||
streak_end['match_id'],
|
||||
streak_end['map_name'],
|
||||
streak_end['start_time'],
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
def _update_percentiles(conn_l3, steam_ids):
|
||||
"""Calculate a real percentile among eligible players in this build."""
|
||||
if not steam_ids:
|
||||
return
|
||||
|
||||
score_rows = []
|
||||
for start in range(0, len(steam_ids), 400):
|
||||
chunk = steam_ids[start:start + 400]
|
||||
placeholders = ','.join('?' for _ in chunk)
|
||||
conn_l3.execute(
|
||||
f"""
|
||||
UPDATE dm_player_features
|
||||
SET tier_percentile = NULL
|
||||
WHERE steam_id_64 IN ({placeholders})
|
||||
""",
|
||||
chunk,
|
||||
)
|
||||
score_rows.extend(conn_l3.execute(
|
||||
f"""
|
||||
SELECT steam_id_64, score_overall
|
||||
FROM dm_player_features
|
||||
WHERE steam_id_64 IN ({placeholders})
|
||||
AND score_overall > 0
|
||||
""",
|
||||
chunk,
|
||||
).fetchall())
|
||||
|
||||
if not score_rows:
|
||||
return
|
||||
|
||||
scores = [float(row[1]) for row in score_rows]
|
||||
percentile_values = []
|
||||
for row in score_rows:
|
||||
score = float(row[1])
|
||||
percentile = sum(value <= score for value in scores) / len(scores) * 100
|
||||
percentile_values.append((round(percentile, 2), str(row[0])))
|
||||
|
||||
conn_l3.executemany(
|
||||
"""
|
||||
UPDATE dm_player_features
|
||||
SET tier_percentile = ?
|
||||
WHERE steam_id_64 = ?
|
||||
""",
|
||||
percentile_values,
|
||||
)
|
||||
logger.info("Updated percentiles for %s eligible players", len(score_rows))
|
||||
|
||||
|
||||
def _parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--workers", type=int, default=1)
|
||||
parser.add_argument("--no-backup", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = _parse_args()
|
||||
main(force_all=args.force, workers=args.workers)
|
||||
main(
|
||||
force_all=args.force,
|
||||
workers=args.workers,
|
||||
create_backup=not args.no_backup,
|
||||
)
|
||||
|
||||
@@ -65,8 +65,8 @@ class CompositeProcessor(BaseFeatureProcessor):
|
||||
# Classify tier based on overall score
|
||||
features['tier_classification'] = CompositeProcessor._classify_tier(features['score_overall'])
|
||||
|
||||
# Percentile rank (placeholder - requires all players)
|
||||
features['tier_percentile'] = min(features['score_overall'], 100.0)
|
||||
# Filled by L3_Builder after every eligible player has been calculated.
|
||||
features['tier_percentile'] = None
|
||||
|
||||
return features
|
||||
|
||||
@@ -266,13 +266,13 @@ class CompositeProcessor(BaseFeatureProcessor):
|
||||
STABILITY Score (0-100) | 8%
|
||||
"""
|
||||
# Extract features
|
||||
volatility = features.get('meta_rating_volatility', 0.0)
|
||||
loss_rating = features.get('meta_loss_rating', 0.0)
|
||||
consistency = features.get('meta_rating_consistency', 0.0)
|
||||
tilt_resilience = features.get('int_pressure_tilt_resistance', 0.0)
|
||||
map_stable = features.get('meta_map_stability', 0.0)
|
||||
elo_stable = features.get('meta_elo_tier_stability', 0.0)
|
||||
recent_form = features.get('meta_recent_form_rating', 0.0)
|
||||
volatility = features.get('meta_rating_volatility') or 0.0
|
||||
loss_rating = features.get('meta_loss_rating') or 0.0
|
||||
consistency = features.get('meta_rating_consistency') or 0.0
|
||||
tilt_resilience = features.get('int_pressure_tilt_resistance') or 0.0
|
||||
map_stable = features.get('meta_map_stability') or 0.0
|
||||
elo_stable = features.get('meta_elo_tier_stability') or 0.0
|
||||
recent_form = features.get('meta_recent_form_rating') or 0.0
|
||||
|
||||
# Normalize
|
||||
# Volatility: Reverse score. 100 - (Vol * 220)
|
||||
@@ -281,8 +281,8 @@ class CompositeProcessor(BaseFeatureProcessor):
|
||||
loss_score = min((loss_rating / 1.00) * 100, 100)
|
||||
cons_score = min((consistency / 70) * 100, 100)
|
||||
tilt_score = min((tilt_resilience / 0.80) * 100, 100)
|
||||
map_score = min((map_stable / 0.25) * 100, 100)
|
||||
elo_score = min((elo_stable / 0.48) * 100, 100)
|
||||
map_score = max(0, min(100, 100 - (map_stable / 0.25) * 100))
|
||||
elo_score = max(0, min(100, 100 - (elo_stable / 0.48) * 100))
|
||||
recent_score = min((recent_form / 1.15) * 100, 100)
|
||||
|
||||
# Weighted Sum
|
||||
@@ -337,12 +337,12 @@ class CompositeProcessor(BaseFeatureProcessor):
|
||||
PACE Score (0-100) | 5%
|
||||
"""
|
||||
# Extract features
|
||||
early_kill_pct = features.get('int_timing_early_kill_share', 0.0)
|
||||
aggression = features.get('int_timing_aggression_index', 0.0)
|
||||
trade_speed = features.get('int_trade_response_time', 0.0)
|
||||
trade_kill = features.get('int_trade_kill_count', 0)
|
||||
teamwork = features.get('int_teamwork_score', 0.0)
|
||||
first_contact = features.get('int_timing_first_contact_time', 0.0)
|
||||
early_kill_pct = features.get('int_timing_early_kill_share') or 0.0
|
||||
aggression = features.get('int_timing_aggression_index') or 0.0
|
||||
trade_speed = features.get('int_trade_response_time') or 0.0
|
||||
trade_kill = features.get('int_trade_kill_count') or 0
|
||||
teamwork = features.get('int_teamwork_score') or 0.0
|
||||
first_contact = features.get('int_timing_first_contact_time') or 0.0
|
||||
|
||||
# Normalize
|
||||
early_score = min((early_kill_pct / 0.44) * 100, 100)
|
||||
@@ -353,7 +353,7 @@ class CompositeProcessor(BaseFeatureProcessor):
|
||||
if trade_speed > 0.01:
|
||||
trade_speed_score = min((2.0 / trade_speed) * 100, 100)
|
||||
else:
|
||||
trade_speed_score = 100 # Instant trade
|
||||
trade_speed_score = 0
|
||||
|
||||
trade_kill_score = min((trade_kill / 650) * 100, 100)
|
||||
teamwork_score = min((teamwork / 29) * 100, 100)
|
||||
@@ -362,13 +362,7 @@ class CompositeProcessor(BaseFeatureProcessor):
|
||||
if first_contact > 0.01:
|
||||
first_contact_score = min((30 / first_contact) * 100, 100)
|
||||
else:
|
||||
first_contact_score = 0 # If 0, probably no data, safe to say 0? Or 100?
|
||||
# 0 first contact time means instant damage.
|
||||
# But "30 / Contact" means smaller contact time gives higher score.
|
||||
# If contact time is 0, score explodes.
|
||||
# Realistically first contact time is > 0.
|
||||
# I will clamp it.
|
||||
first_contact_score = 100 # Assume very fast
|
||||
first_contact_score = 0
|
||||
|
||||
# Weighted Sum
|
||||
pace_score = (
|
||||
@@ -416,5 +410,5 @@ def _get_default_composite_features() -> Dict[str, Any]:
|
||||
'score_pace': 0.0,
|
||||
'score_overall': 0.0,
|
||||
'tier_classification': 'Beginner',
|
||||
'tier_percentile': 0.0,
|
||||
'tier_percentile': None,
|
||||
}
|
||||
|
||||
@@ -466,7 +466,8 @@ class IntelligenceProcessor(BaseFeatureProcessor):
|
||||
- int_pos_spatial_iq_score
|
||||
- int_pos_avg_distance_from_teammates
|
||||
|
||||
Note: Simplified implementation - full version requires DBSCAN clustering
|
||||
Only geometry-independent values are calculated here. Metrics that
|
||||
require map boundaries, paths or teammate positions remain NULL.
|
||||
"""
|
||||
cursor = conn_l2.cursor()
|
||||
|
||||
@@ -481,26 +482,23 @@ class IntelligenceProcessor(BaseFeatureProcessor):
|
||||
has_position_data = cursor.fetchone()[0] > 0
|
||||
|
||||
if not has_position_data:
|
||||
# Return placeholder values if no position data
|
||||
return {
|
||||
'int_pos_site_a_control_rate': 0.0,
|
||||
'int_pos_site_b_control_rate': 0.0,
|
||||
'int_pos_mid_control_rate': 0.0,
|
||||
'int_pos_favorite_position': 'unknown',
|
||||
'int_pos_position_diversity': 0.0,
|
||||
'int_pos_rotation_speed': 0.0,
|
||||
'int_pos_map_coverage': 0.0,
|
||||
'int_pos_lurk_tendency': 0.0,
|
||||
'int_pos_site_anchor_score': 0.0,
|
||||
'int_pos_entry_route_diversity': 0.0,
|
||||
'int_pos_retake_positioning': 0.0,
|
||||
'int_pos_postplant_positioning': 0.0,
|
||||
'int_pos_spatial_iq_score': 0.0,
|
||||
'int_pos_avg_distance_from_teammates': 0.0,
|
||||
'int_pos_site_a_control_rate': None,
|
||||
'int_pos_site_b_control_rate': None,
|
||||
'int_pos_mid_control_rate': None,
|
||||
'int_pos_favorite_position': None,
|
||||
'int_pos_position_diversity': None,
|
||||
'int_pos_rotation_speed': None,
|
||||
'int_pos_map_coverage': None,
|
||||
'int_pos_lurk_tendency': None,
|
||||
'int_pos_site_anchor_score': None,
|
||||
'int_pos_entry_route_diversity': None,
|
||||
'int_pos_retake_positioning': None,
|
||||
'int_pos_postplant_positioning': None,
|
||||
'int_pos_spatial_iq_score': None,
|
||||
'int_pos_avg_distance_from_teammates': None,
|
||||
}
|
||||
|
||||
# Simplified position analysis (proper implementation needs clustering)
|
||||
# Calculate basic position variance as proxy for mobility
|
||||
cursor.execute("""
|
||||
SELECT
|
||||
AVG(attacker_pos_x) as avg_x,
|
||||
@@ -515,34 +513,24 @@ class IntelligenceProcessor(BaseFeatureProcessor):
|
||||
pos_row = cursor.fetchone()
|
||||
position_count = pos_row[3] if pos_row[3] else 1
|
||||
|
||||
# Position diversity based on unique grid cells visited
|
||||
position_diversity = min(position_count / 50.0, 1.0) # Normalize to 0-1
|
||||
|
||||
# Map coverage (simplified)
|
||||
map_coverage = position_diversity
|
||||
|
||||
# Site control rates CANNOT be calculated without map-specific geometry data
|
||||
# Each map (Dust2, Mirage, Nuke, etc.) has different site boundaries
|
||||
# Would require: CREATE TABLE map_boundaries (map_name, site_name, min_x, max_x, min_y, max_y)
|
||||
# Commenting out these 3 features:
|
||||
# - int_pos_site_a_control_rate
|
||||
# - int_pos_site_b_control_rate
|
||||
# - int_pos_mid_control_rate
|
||||
|
||||
return {
|
||||
'int_pos_site_a_control_rate': 0.33, # Placeholder
|
||||
'int_pos_site_b_control_rate': 0.33, # Placeholder
|
||||
'int_pos_mid_control_rate': 0.34, # Placeholder
|
||||
'int_pos_favorite_position': 'mid',
|
||||
'int_pos_site_a_control_rate': None,
|
||||
'int_pos_site_b_control_rate': None,
|
||||
'int_pos_mid_control_rate': None,
|
||||
'int_pos_favorite_position': None,
|
||||
'int_pos_position_diversity': round(position_diversity, 3),
|
||||
'int_pos_rotation_speed': 50.0,
|
||||
'int_pos_rotation_speed': None,
|
||||
'int_pos_map_coverage': round(map_coverage, 3),
|
||||
'int_pos_lurk_tendency': 0.25,
|
||||
'int_pos_site_anchor_score': 50.0,
|
||||
'int_pos_lurk_tendency': None,
|
||||
'int_pos_site_anchor_score': None,
|
||||
'int_pos_entry_route_diversity': round(position_diversity, 3),
|
||||
'int_pos_retake_positioning': 50.0,
|
||||
'int_pos_postplant_positioning': 50.0,
|
||||
'int_pos_retake_positioning': None,
|
||||
'int_pos_postplant_positioning': None,
|
||||
'int_pos_spatial_iq_score': round(position_diversity * 100, 2),
|
||||
'int_pos_avg_distance_from_teammates': 500.0,
|
||||
'int_pos_avg_distance_from_teammates': None,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -706,20 +694,20 @@ def _get_default_intelligence_features() -> Dict[str, Any]:
|
||||
'int_pressure_big_moment_score': 0.0,
|
||||
'int_pressure_tilt_resistance': 0.0,
|
||||
# Position Mastery (14)
|
||||
'int_pos_site_a_control_rate': 0.0,
|
||||
'int_pos_site_b_control_rate': 0.0,
|
||||
'int_pos_mid_control_rate': 0.0,
|
||||
'int_pos_favorite_position': 'unknown',
|
||||
'int_pos_position_diversity': 0.0,
|
||||
'int_pos_rotation_speed': 0.0,
|
||||
'int_pos_map_coverage': 0.0,
|
||||
'int_pos_lurk_tendency': 0.0,
|
||||
'int_pos_site_anchor_score': 0.0,
|
||||
'int_pos_entry_route_diversity': 0.0,
|
||||
'int_pos_retake_positioning': 0.0,
|
||||
'int_pos_postplant_positioning': 0.0,
|
||||
'int_pos_spatial_iq_score': 0.0,
|
||||
'int_pos_avg_distance_from_teammates': 0.0,
|
||||
'int_pos_site_a_control_rate': None,
|
||||
'int_pos_site_b_control_rate': None,
|
||||
'int_pos_mid_control_rate': None,
|
||||
'int_pos_favorite_position': None,
|
||||
'int_pos_position_diversity': None,
|
||||
'int_pos_rotation_speed': None,
|
||||
'int_pos_map_coverage': None,
|
||||
'int_pos_lurk_tendency': None,
|
||||
'int_pos_site_anchor_score': None,
|
||||
'int_pos_entry_route_diversity': None,
|
||||
'int_pos_retake_positioning': None,
|
||||
'int_pos_postplant_positioning': None,
|
||||
'int_pos_spatial_iq_score': None,
|
||||
'int_pos_avg_distance_from_teammates': None,
|
||||
# Trade Network (8)
|
||||
'int_trade_kill_count': 0,
|
||||
'int_trade_kill_rate': 0.0,
|
||||
|
||||
@@ -60,10 +60,11 @@ class MetaProcessor(BaseFeatureProcessor):
|
||||
|
||||
# Get recent matches for volatility
|
||||
cursor.execute("""
|
||||
SELECT rating
|
||||
FROM fact_match_players
|
||||
WHERE steam_id_64 = ?
|
||||
ORDER BY match_id DESC
|
||||
SELECT p.rating
|
||||
FROM fact_match_players p
|
||||
JOIN fact_matches m ON m.match_id = p.match_id
|
||||
WHERE p.steam_id_64 = ?
|
||||
ORDER BY m.start_time DESC, p.match_id DESC
|
||||
LIMIT 20
|
||||
""", (steam_id,))
|
||||
|
||||
@@ -141,8 +142,35 @@ class MetaProcessor(BaseFeatureProcessor):
|
||||
map_ratings = [row[1] for row in cursor.fetchall() if row[1] is not None]
|
||||
map_stability = SafeAggregator.safe_stddev(map_ratings, 0.0)
|
||||
|
||||
# ELO tier stability (placeholder)
|
||||
elo_tier_stability = rating_volatility # Simplified
|
||||
cursor.execute("""
|
||||
SELECT
|
||||
CASE
|
||||
WHEN p.origin_elo - opponent.avg_elo > 200 THEN 'lower'
|
||||
WHEN p.origin_elo - opponent.avg_elo < -200 THEN 'higher'
|
||||
ELSE 'similar'
|
||||
END AS opponent_tier,
|
||||
AVG(p.rating) AS avg_rating
|
||||
FROM fact_match_players p
|
||||
JOIN (
|
||||
SELECT match_id, team_id, AVG(origin_elo) AS avg_elo
|
||||
FROM fact_match_players
|
||||
WHERE origin_elo IS NOT NULL
|
||||
GROUP BY match_id, team_id
|
||||
) opponent
|
||||
ON opponent.match_id = p.match_id
|
||||
AND opponent.team_id != p.team_id
|
||||
WHERE p.steam_id_64 = ?
|
||||
AND p.origin_elo IS NOT NULL
|
||||
AND p.rating IS NOT NULL
|
||||
GROUP BY opponent_tier
|
||||
""", (steam_id,))
|
||||
elo_tier_ratings = [
|
||||
row[1] for row in cursor.fetchall() if row[1] is not None
|
||||
]
|
||||
elo_tier_stability = SafeAggregator.safe_stddev(
|
||||
elo_tier_ratings,
|
||||
0.0,
|
||||
)
|
||||
|
||||
return {
|
||||
'meta_rating_volatility': round(rating_volatility, 3),
|
||||
|
||||
@@ -378,6 +378,56 @@ CREATE TABLE IF NOT EXISTS dm_player_weapon_stats (
|
||||
CREATE INDEX IF NOT EXISTS idx_player_weapon_stats_player ON dm_player_weapon_stats(steam_id_64);
|
||||
CREATE INDEX IF NOT EXISTS idx_player_weapon_stats_weapon ON dm_player_weapon_stats(weapon_name);
|
||||
|
||||
-- ============================================================================
|
||||
-- Profile Mart: Time-window statistics
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS dm_player_period_stats (
|
||||
steam_id_64 TEXT NOT NULL,
|
||||
period_key TEXT NOT NULL,
|
||||
period_label TEXT NOT NULL,
|
||||
period_start INTEGER,
|
||||
period_end INTEGER,
|
||||
matches INTEGER NOT NULL DEFAULT 0,
|
||||
wins INTEGER NOT NULL DEFAULT 0,
|
||||
win_rate REAL,
|
||||
avg_rating REAL,
|
||||
avg_kd REAL,
|
||||
avg_adr REAL,
|
||||
avg_kast REAL,
|
||||
total_kills INTEGER NOT NULL DEFAULT 0,
|
||||
total_deaths INTEGER NOT NULL DEFAULT 0,
|
||||
sample_reliable BOOLEAN NOT NULL DEFAULT 0,
|
||||
last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
PRIMARY KEY (steam_id_64, period_key),
|
||||
FOREIGN KEY (steam_id_64)
|
||||
REFERENCES dm_player_features(steam_id_64) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_player_period_player
|
||||
ON dm_player_period_stats(steam_id_64, period_key);
|
||||
|
||||
-- ============================================================================
|
||||
-- Profile Mart: Career records linked to the source match
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS dm_player_records (
|
||||
steam_id_64 TEXT NOT NULL,
|
||||
record_key TEXT NOT NULL,
|
||||
record_label TEXT NOT NULL,
|
||||
record_value REAL,
|
||||
match_id TEXT,
|
||||
map_name TEXT,
|
||||
match_date INTEGER,
|
||||
last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
PRIMARY KEY (steam_id_64, record_key),
|
||||
FOREIGN KEY (steam_id_64)
|
||||
REFERENCES dm_player_features(steam_id_64) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_player_records_player
|
||||
ON dm_player_records(steam_id_64, record_key);
|
||||
|
||||
-- ============================================================================
|
||||
-- Schema Summary
|
||||
-- ============================================================================
|
||||
@@ -391,4 +441,6 @@ CREATE INDEX IF NOT EXISTS idx_player_weapon_stats_weapon ON dm_player_weapon_st
|
||||
-- dm_player_match_history: Per-match snapshots for trend analysis
|
||||
-- dm_player_map_stats: Map-level aggregations
|
||||
-- dm_player_weapon_stats: Weapon usage statistics
|
||||
-- dm_player_period_stats: Career/recent time-window aggregations
|
||||
-- dm_player_records: Career record values and source matches
|
||||
-- ============================================================================
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Database Governance
|
||||
|
||||
The repository intentionally keeps SQLite for the current private-team scale.
|
||||
This directory separates four different responsibilities:
|
||||
|
||||
| Layer | Database | Grain | Owner |
|
||||
|---|---|---|---|
|
||||
| L1 | `L1/L1.db` | One raw network capture per match | Import pipeline |
|
||||
| L2 | `L2/L2.db` | Normalized match, player, round and event facts | L2 Builder |
|
||||
| L3 | `L3/L3.db` | Roster features and profile marts | L3 Builder |
|
||||
| Web | `Web/Web_App.sqlite` | Lineups, comments, jobs and editorial data | Flask app |
|
||||
|
||||
## Rules
|
||||
|
||||
1. Paths are defined only in `database/paths.py`.
|
||||
2. Schemas live next to their owning database.
|
||||
3. Builders may read the previous layer and write only their own layer.
|
||||
4. User-generated Web data is never restored as part of an ETL rollback.
|
||||
5. A full import must run through `database/pipeline.py`.
|
||||
6. Pipeline runs are serialized by `database/.pipeline.lock`.
|
||||
7. L1/L2/L3 are backed up before a full pipeline run.
|
||||
8. Missing metrics are stored as `NULL`, not fabricated zero values.
|
||||
9. `Admin -> Data Integrity` is the operational source of truth.
|
||||
10. Web schema changes increment `Config.WEB_SCHEMA_VERSION`.
|
||||
|
||||
## Entry Points
|
||||
|
||||
```bash
|
||||
make l1 # Import output_arena JSON into L1
|
||||
make l2 # Rebuild normalized facts
|
||||
make l3 # Rebuild active-roster features
|
||||
make pipeline # Run L1 -> L2 -> L3 with backup and validation
|
||||
make check # Compile and run tests
|
||||
```
|
||||
|
||||
## Directory Policy
|
||||
|
||||
- `L1/`, `L2/`, `L3/`, `Web/`: active code, schema and database.
|
||||
- `backups/`: generated rollback snapshots; ignored by Git.
|
||||
- `schema_bkp/`: historical schema research only; not used at runtime.
|
||||
- `L1B/`: reserved demo-parser integration; not used at runtime.
|
||||
- `L3/Roadmap/`: historical design notes; not used at runtime.
|
||||
|
||||
Large-scale directory moves are deliberately deferred until the legacy
|
||||
builders no longer depend on their current module layout.
|
||||
Binary file not shown.
@@ -0,0 +1,95 @@
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version INTEGER PRIMARY KEY,
|
||||
description TEXT NOT NULL,
|
||||
applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS comments (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT,
|
||||
username TEXT,
|
||||
target_type TEXT NOT NULL,
|
||||
target_id TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
likes INTEGER NOT NULL DEFAULT 0,
|
||||
is_hidden INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_comments_target
|
||||
ON comments(target_type, target_id, is_hidden, created_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS player_metadata (
|
||||
steam_id_64 TEXT PRIMARY KEY,
|
||||
notes TEXT,
|
||||
tags TEXT NOT NULL DEFAULT '[]',
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS strategy_boards (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
map_name TEXT NOT NULL,
|
||||
data_json TEXT NOT NULL,
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS team_lineups (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
player_ids_json TEXT NOT NULL DEFAULT '[]',
|
||||
is_active INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_team_lineups_single_active
|
||||
ON team_lineups(is_active)
|
||||
WHERE is_active = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS wiki_pages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
content TEXT NOT NULL DEFAULT '',
|
||||
updated_by TEXT,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS etl_jobs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
job_type TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued', 'running', 'succeeded', 'failed')),
|
||||
match_id TEXT,
|
||||
input_path TEXT,
|
||||
current_stage TEXT,
|
||||
progress INTEGER NOT NULL DEFAULT 0
|
||||
CHECK (progress >= 0 AND progress <= 100),
|
||||
message TEXT,
|
||||
log_text TEXT NOT NULL DEFAULT '',
|
||||
created_by TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
started_at TIMESTAMP,
|
||||
finished_at TIMESTAMP,
|
||||
duration_seconds REAL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_etl_jobs_created
|
||||
ON etl_jobs(created_at DESC, id DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_etl_jobs_status
|
||||
ON etl_jobs(status, created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS match_imports (
|
||||
match_id TEXT PRIMARY KEY,
|
||||
content_sha256 TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
job_id INTEGER,
|
||||
imported_at TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (job_id) REFERENCES etl_jobs(id) ON DELETE SET NULL
|
||||
);
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Database builders, schemas, maintenance tools and local data stores."""
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
import sqlite3
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from database.paths import WEB_DB
|
||||
|
||||
|
||||
class JobStore:
|
||||
def __init__(self, database_path=WEB_DB):
|
||||
self.database_path = str(database_path)
|
||||
|
||||
def _connect(self):
|
||||
db = sqlite3.connect(self.database_path, timeout=30)
|
||||
db.row_factory = sqlite3.Row
|
||||
db.execute('PRAGMA busy_timeout = 30000')
|
||||
db.execute('PRAGMA foreign_keys = ON')
|
||||
return db
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
job_type: str,
|
||||
match_id: Optional[str] = None,
|
||||
input_path: Optional[str] = None,
|
||||
created_by: Optional[str] = None,
|
||||
) -> int:
|
||||
db = self._connect()
|
||||
try:
|
||||
cursor = db.execute(
|
||||
"""
|
||||
INSERT INTO etl_jobs (
|
||||
job_type, match_id, input_path, created_by
|
||||
) VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
[job_type, match_id, input_path, created_by],
|
||||
)
|
||||
db.commit()
|
||||
return int(cursor.lastrowid)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def get_job(self, job_id: int) -> Optional[Dict[str, Any]]:
|
||||
db = self._connect()
|
||||
try:
|
||||
row = db.execute(
|
||||
'SELECT * FROM etl_jobs WHERE id = ?',
|
||||
[job_id],
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def list_jobs(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
db = self._connect()
|
||||
try:
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM etl_jobs
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
[max(1, min(int(limit), 100))],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def start_job(self, job_id: int, stage: str, message: str):
|
||||
db = self._connect()
|
||||
try:
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE etl_jobs
|
||||
SET status = 'running',
|
||||
current_stage = ?,
|
||||
progress = 1,
|
||||
message = ?,
|
||||
started_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ? AND status = 'queued'
|
||||
""",
|
||||
[stage, message, job_id],
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def update_progress(
|
||||
self,
|
||||
job_id: int,
|
||||
stage: str,
|
||||
progress: int,
|
||||
message: str,
|
||||
):
|
||||
db = self._connect()
|
||||
try:
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE etl_jobs
|
||||
SET current_stage = ?, progress = ?, message = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
[stage, max(0, min(int(progress), 100)), message, job_id],
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def append_log(self, job_id: int, text: str):
|
||||
if not text:
|
||||
return
|
||||
db = self._connect()
|
||||
try:
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE etl_jobs
|
||||
SET log_text = substr(log_text || ?, -100000)
|
||||
WHERE id = ?
|
||||
""",
|
||||
[text, job_id],
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def finish_job(
|
||||
self,
|
||||
job_id: int,
|
||||
succeeded: bool,
|
||||
message: str,
|
||||
duration_seconds: float,
|
||||
):
|
||||
status = 'succeeded' if succeeded else 'failed'
|
||||
db = self._connect()
|
||||
try:
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE etl_jobs
|
||||
SET status = ?,
|
||||
current_stage = ?,
|
||||
progress = ?,
|
||||
message = ?,
|
||||
finished_at = CURRENT_TIMESTAMP,
|
||||
duration_seconds = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
[
|
||||
status,
|
||||
'complete' if succeeded else 'failed',
|
||||
100 if succeeded else 0,
|
||||
message,
|
||||
round(float(duration_seconds), 3),
|
||||
job_id,
|
||||
],
|
||||
)
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE match_imports
|
||||
SET status = ?,
|
||||
imported_at = CASE
|
||||
WHEN ? = 'succeeded' THEN CURRENT_TIMESTAMP
|
||||
ELSE imported_at
|
||||
END,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE job_id = ?
|
||||
""",
|
||||
[status, status, job_id],
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def upsert_match_import(
|
||||
self,
|
||||
match_id: str,
|
||||
content_sha256: str,
|
||||
source_path: str,
|
||||
status: str,
|
||||
job_id: int,
|
||||
):
|
||||
db = self._connect()
|
||||
try:
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO match_imports (
|
||||
match_id, content_sha256, source_path, status, job_id
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(match_id) DO UPDATE SET
|
||||
content_sha256 = excluded.content_sha256,
|
||||
source_path = excluded.source_path,
|
||||
status = excluded.status,
|
||||
job_id = excluded.job_id,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
[match_id, content_sha256, source_path, status, job_id],
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sqlite3
|
||||
from typing import Dict
|
||||
|
||||
from database.paths import BACKUP_ROOT, L1_DB, L2_DB, L3_DB
|
||||
|
||||
|
||||
MANAGED_DATABASES = {
|
||||
'l1': L1_DB,
|
||||
'l2': L2_DB,
|
||||
'l3': L3_DB,
|
||||
}
|
||||
|
||||
|
||||
def quick_check(path: Path) -> str:
|
||||
if not path.exists():
|
||||
return 'missing'
|
||||
db = sqlite3.connect(str(path))
|
||||
try:
|
||||
return str(db.execute('PRAGMA quick_check').fetchone()[0])
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def backup_database(source_path: Path, backup_path: Path):
|
||||
if not source_path.exists():
|
||||
raise FileNotFoundError(f'Database does not exist: {source_path}')
|
||||
|
||||
backup_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
source = sqlite3.connect(str(source_path))
|
||||
destination = sqlite3.connect(str(backup_path))
|
||||
try:
|
||||
source.backup(destination)
|
||||
result = destination.execute('PRAGMA quick_check').fetchone()[0]
|
||||
if result != 'ok':
|
||||
raise RuntimeError(
|
||||
f'Backup quick_check failed for {source_path.name}: {result}'
|
||||
)
|
||||
finally:
|
||||
source.close()
|
||||
destination.close()
|
||||
|
||||
|
||||
def restore_database(backup_path: Path, target_path: Path):
|
||||
if not backup_path.exists():
|
||||
raise FileNotFoundError(f'Backup does not exist: {backup_path}')
|
||||
|
||||
source = sqlite3.connect(str(backup_path))
|
||||
target = sqlite3.connect(str(target_path), timeout=30)
|
||||
try:
|
||||
source.backup(target)
|
||||
result = target.execute('PRAGMA quick_check').fetchone()[0]
|
||||
if result != 'ok':
|
||||
raise RuntimeError(
|
||||
f'Restored quick_check failed for {target_path.name}: {result}'
|
||||
)
|
||||
finally:
|
||||
source.close()
|
||||
target.close()
|
||||
|
||||
|
||||
def create_backup_set(label: str) -> Path:
|
||||
safe_label = ''.join(
|
||||
character for character in str(label)
|
||||
if character.isalnum() or character in {'-', '_'}
|
||||
)
|
||||
if not safe_label:
|
||||
raise ValueError('Backup label is empty after sanitization')
|
||||
|
||||
backup_dir = BACKUP_ROOT / safe_label
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest: Dict[str, object] = {
|
||||
'label': safe_label,
|
||||
'created_at': datetime.now(timezone.utc).isoformat(),
|
||||
'databases': {},
|
||||
}
|
||||
|
||||
for name, source_path in MANAGED_DATABASES.items():
|
||||
backup_path = backup_dir / source_path.name
|
||||
backup_database(source_path, backup_path)
|
||||
manifest['databases'][name] = {
|
||||
'source': str(source_path),
|
||||
'backup': str(backup_path),
|
||||
'size_bytes': backup_path.stat().st_size,
|
||||
'quick_check': quick_check(backup_path),
|
||||
}
|
||||
|
||||
with (backup_dir / 'manifest.json').open('w', encoding='utf-8') as file:
|
||||
json.dump(manifest, file, ensure_ascii=True, indent=2)
|
||||
return backup_dir
|
||||
|
||||
|
||||
def restore_backup_set(backup_dir: Path):
|
||||
for name, target_path in MANAGED_DATABASES.items():
|
||||
backup_path = backup_dir / target_path.name
|
||||
restore_database(backup_path, target_path)
|
||||
|
||||
|
||||
def prune_backup_sets(keep: int = 3):
|
||||
BACKUP_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
backup_dirs = sorted(
|
||||
[path for path in BACKUP_ROOT.iterdir() if path.is_dir()],
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
removed = []
|
||||
for path in backup_dirs[max(int(keep), 0):]:
|
||||
shutil.rmtree(path)
|
||||
removed.append(str(path))
|
||||
return removed
|
||||
|
||||
|
||||
def backup_storage_status():
|
||||
BACKUP_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
backup_dirs = [path for path in BACKUP_ROOT.iterdir() if path.is_dir()]
|
||||
total_bytes = sum(
|
||||
file.stat().st_size
|
||||
for directory in backup_dirs
|
||||
for file in directory.rglob('*')
|
||||
if file.is_file()
|
||||
)
|
||||
return {
|
||||
'sets': len(backup_dirs),
|
||||
'total_bytes': total_bytes,
|
||||
}
|
||||
|
||||
|
||||
def check_managed_databases():
|
||||
return {
|
||||
name: {
|
||||
'path': str(path),
|
||||
'exists': path.exists(),
|
||||
'size_bytes': path.stat().st_size if path.exists() else 0,
|
||||
'quick_check': quick_check(path),
|
||||
}
|
||||
for name, path in MANAGED_DATABASES.items()
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
DATABASE_ROOT = PROJECT_ROOT / 'database'
|
||||
|
||||
L1_DIR = DATABASE_ROOT / 'L1'
|
||||
L2_DIR = DATABASE_ROOT / 'L2'
|
||||
L3_DIR = DATABASE_ROOT / 'L3'
|
||||
WEB_DIR = DATABASE_ROOT / 'Web'
|
||||
|
||||
L1_DB = L1_DIR / 'L1.db'
|
||||
L2_DB = L2_DIR / 'L2.db'
|
||||
L3_DB = L3_DIR / 'L3.db'
|
||||
WEB_DB = WEB_DIR / 'Web_App.sqlite'
|
||||
|
||||
L2_SCHEMA = L2_DIR / 'schema.sql'
|
||||
L3_SCHEMA = L3_DIR / 'schema.sql'
|
||||
WEB_SCHEMA = WEB_DIR / 'schema.sql'
|
||||
|
||||
OUTPUT_ARENA = PROJECT_ROOT / 'output_arena'
|
||||
BACKUP_ROOT = DATABASE_ROOT / 'backups'
|
||||
PIPELINE_LOCK = DATABASE_ROOT / '.pipeline.lock'
|
||||
|
||||
|
||||
def ensure_runtime_directories():
|
||||
for path in (
|
||||
L1_DIR,
|
||||
L2_DIR,
|
||||
L3_DIR,
|
||||
WEB_DIR,
|
||||
OUTPUT_ARENA,
|
||||
BACKUP_ROOT,
|
||||
):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
import argparse
|
||||
import fcntl
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from database.job_store import JobStore
|
||||
from database.maintenance import (
|
||||
check_managed_databases,
|
||||
create_backup_set,
|
||||
prune_backup_sets,
|
||||
restore_backup_set,
|
||||
)
|
||||
from database.paths import L1_DB, L2_DB, PIPELINE_LOCK
|
||||
|
||||
|
||||
STAGES = (
|
||||
('l1', 15, Path('database/L1/L1_Builder.py')),
|
||||
('l2', 55, Path('database/L2/L2_Builder.py')),
|
||||
('l3', 85, Path('database/L3/L3_Builder.py')),
|
||||
)
|
||||
|
||||
|
||||
class PipelineError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _run_stage(store, job_id, stage, progress, script_path, replace=False):
|
||||
store.update_progress(
|
||||
job_id,
|
||||
stage,
|
||||
progress,
|
||||
f'Running {stage.upper()} builder',
|
||||
)
|
||||
command = [sys.executable, str(PROJECT_ROOT / script_path)]
|
||||
if stage == 'l1' and replace:
|
||||
command.append('--force')
|
||||
if stage == 'l3':
|
||||
command.append('--no-backup')
|
||||
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=str(PROJECT_ROOT),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=1200,
|
||||
)
|
||||
duration = time.monotonic() - started
|
||||
store.append_log(
|
||||
job_id,
|
||||
(
|
||||
f'\n===== {stage.upper()} ({duration:.2f}s) =====\n'
|
||||
f'{result.stdout}\n{result.stderr}'
|
||||
),
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise PipelineError(
|
||||
f'{stage.upper()} builder exited with code {result.returncode}'
|
||||
)
|
||||
|
||||
|
||||
def _validate_pipeline_output(match_id=None):
|
||||
database_status = check_managed_databases()
|
||||
failures = [
|
||||
f"{name}: {status['quick_check']}"
|
||||
for name, status in database_status.items()
|
||||
if status['quick_check'] != 'ok'
|
||||
]
|
||||
if failures:
|
||||
raise PipelineError(
|
||||
'Database quick_check failed: ' + ', '.join(failures)
|
||||
)
|
||||
|
||||
if not match_id:
|
||||
return
|
||||
|
||||
l1 = sqlite3.connect(str(L1_DB))
|
||||
l2 = sqlite3.connect(str(L2_DB))
|
||||
try:
|
||||
raw_count = l1.execute(
|
||||
'SELECT COUNT(*) FROM raw_iframe_network WHERE match_id = ?',
|
||||
[match_id],
|
||||
).fetchone()[0]
|
||||
match_count = l2.execute(
|
||||
'SELECT COUNT(*) FROM fact_matches WHERE match_id = ?',
|
||||
[match_id],
|
||||
).fetchone()[0]
|
||||
player_count = l2.execute(
|
||||
'SELECT COUNT(*) FROM fact_match_players WHERE match_id = ?',
|
||||
[match_id],
|
||||
).fetchone()[0]
|
||||
round_count = l2.execute(
|
||||
'SELECT COUNT(*) FROM fact_rounds WHERE match_id = ?',
|
||||
[match_id],
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
l1.close()
|
||||
l2.close()
|
||||
|
||||
if raw_count != 1:
|
||||
raise PipelineError(f'L1 does not contain imported match {match_id}')
|
||||
if match_count != 1:
|
||||
raise PipelineError(f'L2 does not contain imported match {match_id}')
|
||||
if player_count != 10:
|
||||
raise PipelineError(
|
||||
f'Imported match has {player_count} players; expected 10'
|
||||
)
|
||||
if round_count <= 0:
|
||||
raise PipelineError('Imported match has no round facts')
|
||||
|
||||
|
||||
def run_pipeline(job_id, match_id=None, replace=False):
|
||||
store = JobStore()
|
||||
job = store.get_job(job_id)
|
||||
if not job:
|
||||
raise PipelineError(f'Unknown ETL job: {job_id}')
|
||||
|
||||
started = time.monotonic()
|
||||
backup_dir = None
|
||||
PIPELINE_LOCK.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock_file = PIPELINE_LOCK.open('w')
|
||||
try:
|
||||
try:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError as exc:
|
||||
raise PipelineError('Another database pipeline is already running') from exc
|
||||
|
||||
store.start_job(job_id, 'backup', 'Creating rollback snapshot')
|
||||
backup_dir = create_backup_set(f'job-{job_id}')
|
||||
store.append_log(job_id, f'Backup created: {backup_dir}\n')
|
||||
|
||||
for stage, progress, script_path in STAGES:
|
||||
_run_stage(
|
||||
store,
|
||||
job_id,
|
||||
stage,
|
||||
progress,
|
||||
script_path,
|
||||
replace=replace,
|
||||
)
|
||||
|
||||
store.update_progress(
|
||||
job_id,
|
||||
'validation',
|
||||
95,
|
||||
'Validating imported data',
|
||||
)
|
||||
_validate_pipeline_output(match_id)
|
||||
removed_backups = prune_backup_sets(keep=3)
|
||||
if removed_backups:
|
||||
store.append_log(
|
||||
job_id,
|
||||
f"Pruned old backups: {', '.join(removed_backups)}\n",
|
||||
)
|
||||
duration = time.monotonic() - started
|
||||
store.finish_job(
|
||||
job_id,
|
||||
True,
|
||||
'Pipeline completed and validated',
|
||||
duration,
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
store.append_log(job_id, f'\nPIPELINE FAILED: {exc}\n')
|
||||
if backup_dir:
|
||||
try:
|
||||
restore_backup_set(backup_dir)
|
||||
store.append_log(job_id, 'Rollback snapshot restored successfully\n')
|
||||
except Exception as restore_exc:
|
||||
store.append_log(
|
||||
job_id,
|
||||
f'ROLLBACK FAILED: {restore_exc}\n',
|
||||
)
|
||||
exc = PipelineError(f'{exc}; rollback also failed: {restore_exc}')
|
||||
store.finish_job(
|
||||
job_id,
|
||||
False,
|
||||
str(exc),
|
||||
time.monotonic() - started,
|
||||
)
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
||||
finally:
|
||||
lock_file.close()
|
||||
|
||||
|
||||
def _parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--job-id', type=int, required=True)
|
||||
parser.add_argument('--match-id')
|
||||
parser.add_argument('--replace', action='store_true')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = _parse_args()
|
||||
succeeded = run_pipeline(
|
||||
args.job_id,
|
||||
match_id=args.match_id,
|
||||
replace=args.replace,
|
||||
)
|
||||
raise SystemExit(0 if succeeded else 1)
|
||||
Reference in New Issue
Block a user