Jacky/JKTV-online
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2.0.0 Alpha: Data Refinery

Browse Source
This commit is contained in:
Jacky
2026-08-08 21:31:56 +08:00
parent fa75081d4d
commit 562775e5db
48 changed files with 4172 additions and 661 deletions
Download Patch File Download Diff File Expand all files Collapse all files
database/L3/L3_Builder.py
+490 -12
View File
@@ -6,6 +6,8 @@ import sqlite3
import json
import argparse
import concurrent.futures
from collections import defaultdict, deque
from typing import Optional
# Setup logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -15,10 +17,14 @@ logger = logging.getLogger(__name__)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Points to database/ directory
PROJECT_ROOT = os.path.dirname(BASE_DIR) # Points to project root
sys.path.insert(0, PROJECT_ROOT) # Add project root to Python path
L2_DB_PATH = os.path.join(BASE_DIR, 'L2', 'L2.db')
L3_DB_PATH = os.path.join(BASE_DIR, 'L3', 'L3.db')
WEB_DB_PATH = os.path.join(BASE_DIR, 'Web', 'Web_App.sqlite')
SCHEMA_PATH = os.path.join(BASE_DIR, 'L3', 'schema.sql')
from database.paths import L2_DB, L3_DB, L3_SCHEMA, WEB_DB
L2_DB_PATH = str(L2_DB)
L3_DB_PATH = str(L3_DB)
L3_BACKUP_PATH = f"{L3_DB_PATH}.bak"
WEB_DB_PATH = str(WEB_DB)
SCHEMA_PATH = str(L3_SCHEMA)
def _get_existing_columns(conn, table_name):
cur = conn.execute(f"PRAGMA table_info({table_name})")
@@ -76,7 +82,28 @@ def _get_team_players():
try:
conn = sqlite3.connect(WEB_DB_PATH)
cursor = conn.cursor()
cursor.execute("SELECT player_ids_json FROM team_lineups")
columns = {
row[1] for row in cursor.execute("PRAGMA table_info(team_lineups)")
}
if 'is_active' in columns:
cursor.execute(
"""
SELECT player_ids_json
FROM team_lineups
WHERE is_active = 1
ORDER BY created_at DESC, id DESC
LIMIT 1
"""
)
else:
cursor.execute(
"""
SELECT player_ids_json
FROM team_lineups
ORDER BY created_at DESC, id DESC
LIMIT 1
"""
)
rows = cursor.fetchall()
steam_ids = set()
@@ -150,7 +177,25 @@ def _build_player_record(steam_id: str):
"error": str(e),
}
def main(force_all: bool = False, workers: int = 1):
def _backup_l3_database(source_path=L3_DB_PATH, backup_path=L3_BACKUP_PATH):
if not os.path.exists(source_path):
return None
source = sqlite3.connect(source_path)
backup = sqlite3.connect(backup_path)
try:
source.backup(backup)
result = backup.execute("PRAGMA quick_check").fetchone()[0]
if result != 'ok':
raise RuntimeError(f"L3 backup quick_check failed: {result}")
finally:
source.close()
backup.close()
logger.info("L3 backup created at %s", backup_path)
return backup_path
def main(force_all: bool = False, workers: int = 1, create_backup: bool = True):
"""
Main L3 feature building pipeline using modular processors
"""
@@ -158,6 +203,9 @@ def main(force_all: bool = False, workers: int = 1):
logger.info("Starting L3 Builder with 5-Tier Architecture")
logger.info("========================================")
if create_backup:
_backup_l3_database()
# 1. Ensure Schema is up to date
init_db()
@@ -181,6 +229,7 @@ def main(force_all: bool = False, workers: int = 1):
conn_l3 = sqlite3.connect(L3_DB_PATH)
try:
conn_l3.execute("BEGIN IMMEDIATE")
cursor_l2 = conn_l2.cursor()
if force_all:
logger.info("Force mode enabled: building L3 for all players in L2.")
@@ -240,7 +289,6 @@ def main(force_all: bool = False, workers: int = 1):
)
success_count += 1
if processed_count % 2 == 0:
conn_l3.commit()
logger.info(f"Progress: {processed_count}/{total_players} ({success_count} success, {error_count} errors)")
else:
for idx, row in enumerate(players, 1):
@@ -268,10 +316,20 @@ def main(force_all: bool = False, workers: int = 1):
processed_count = idx
if processed_count % 2 == 0:
conn_l3.commit()
logger.info(f"Progress: {processed_count}/{total_players} ({success_count} success, {error_count} errors)")
# Final commit
if error_count:
raise RuntimeError(
f"L3 feature build failed for {error_count}/{total_players} players"
)
processed_ids = [str(row[0]) for row in players]
_update_percentiles(conn_l3, processed_ids)
_rebuild_auxiliary_marts(conn_l2, conn_l3, processed_ids)
quick_check = conn_l3.execute("PRAGMA quick_check").fetchone()[0]
if quick_check != 'ok':
raise RuntimeError(f"L3 quick_check failed before commit: {quick_check}")
conn_l3.commit()
logger.info("========================================")
@@ -283,9 +341,11 @@ def main(force_all: bool = False, workers: int = 1):
logger.info("========================================")
except Exception as e:
conn_l3.rollback()
logger.error(f"Fatal error during L3 build: {e}")
import traceback
traceback.print_exc()
raise
finally:
conn_l2.close()
@@ -313,7 +373,7 @@ def _get_round_count(steam_id: str, conn_l2: sqlite3.Connection) -> int:
def _upsert_features(conn_l3: sqlite3.Connection, steam_id: str, features: dict,
match_count: int, round_count: int, conn_l2: sqlite3.Connection | None,
match_count: int, round_count: int, conn_l2: Optional[sqlite3.Connection],
first_match_date=None, last_match_date=None):
"""
Insert or update player features in dm_player_features
@@ -353,12 +413,430 @@ def _upsert_features(conn_l3: sqlite3.Connection, steam_id: str, features: dict,
cursor_l3.execute(sql, values)
def _rebuild_auxiliary_marts(conn_l2, conn_l3, steam_ids):
"""Rebuild player-grain marts used by profiles and trend APIs."""
if not steam_ids:
return
logger.info("Rebuilding L3 match, map and weapon marts")
total_history = 0
total_maps = 0
total_weapons = 0
total_periods = 0
total_records = 0
for start in range(0, len(steam_ids), 400):
chunk = steam_ids[start:start + 400]
placeholders = ','.join('?' for _ in chunk)
for table in (
'dm_player_match_history',
'dm_player_map_stats',
'dm_player_weapon_stats',
'dm_player_period_stats',
'dm_player_records',
):
conn_l3.execute(
f"DELETE FROM {table} WHERE steam_id_64 IN ({placeholders})",
chunk,
)
history_rows = conn_l2.execute(
f"""
SELECT
mp.steam_id_64,
mp.match_id,
m.start_time,
mp.rating,
mp.kd_ratio,
mp.adr,
mp.kast,
mp.is_win,
m.map_name,
mp.kills,
mp.deaths,
mp.headshot_count,
(
SELECT AVG(teammate.rating)
FROM fact_match_players teammate
WHERE teammate.match_id = mp.match_id
AND teammate.team_id = mp.team_id
AND teammate.steam_id_64 != mp.steam_id_64
) AS teammate_avg_rating
FROM fact_match_players mp
JOIN fact_matches m ON m.match_id = mp.match_id
WHERE mp.steam_id_64 IN ({placeholders})
ORDER BY mp.steam_id_64, m.start_time, mp.match_id
""",
chunk,
).fetchall()
history_values = []
player_state = defaultdict(lambda: {
'sequence': 0,
'rating_sum': 0.0,
'recent': deque(maxlen=10),
})
for row in history_rows:
steam_id = str(row[0])
state = player_state[steam_id]
rating = float(row[3] or 0.0)
state['sequence'] += 1
state['rating_sum'] += rating
state['recent'].append(rating)
history_values.append((
steam_id,
row[1],
row[2],
state['sequence'],
row[3],
row[4],
row[5],
row[6],
row[7],
row[8],
None,
row[12],
state['rating_sum'] / state['sequence'],
sum(state['recent']) / len(state['recent']),
))
conn_l3.executemany(
"""
INSERT INTO dm_player_match_history (
steam_id_64, match_id, match_date, match_sequence,
rating, kd_ratio, adr, kast, is_win, map_name,
opponent_avg_elo, teammate_avg_rating,
cumulative_rating, rolling_10_rating
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
history_values,
)
total_history += len(history_values)
map_rows = conn_l2.execute(
f"""
SELECT
mp.steam_id_64,
m.map_name,
COUNT(*) AS matches,
SUM(CASE WHEN mp.is_win = 1 THEN 1 ELSE 0 END) AS wins,
AVG(mp.rating) AS avg_rating,
AVG(mp.kd_ratio) AS avg_kd,
AVG(mp.adr) AS avg_adr,
AVG(mp.kast) AS avg_kast,
MAX(mp.rating) AS best_rating,
MIN(mp.rating) AS worst_rating
FROM fact_match_players mp
JOIN fact_matches m ON m.match_id = mp.match_id
WHERE mp.steam_id_64 IN ({placeholders})
AND m.map_name IS NOT NULL
AND m.map_name != ''
GROUP BY mp.steam_id_64, m.map_name
""",
chunk,
).fetchall()
map_values = [
tuple(row[:4]) + (
(row[3] or 0) / row[2] if row[2] else 0.0,
) + tuple(row[4:])
for row in map_rows
]
conn_l3.executemany(
"""
INSERT INTO dm_player_map_stats (
steam_id_64, map_name, matches, wins, win_rate,
avg_rating, avg_kd, avg_adr, avg_kast,
best_rating, worst_rating
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
map_values,
)
total_maps += len(map_values)
round_counts = {
str(row[0]): int(row[1] or 0)
for row in conn_l2.execute(
f"""
SELECT steam_id_64, SUM(round_total)
FROM fact_match_players
WHERE steam_id_64 IN ({placeholders})
GROUP BY steam_id_64
""",
chunk,
)
}
weapon_rows = conn_l2.execute(
f"""
SELECT
attacker_steam_id,
weapon,
COUNT(*) AS total_kills,
SUM(CASE WHEN is_headshot = 1 THEN 1 ELSE 0 END) AS total_headshots,
COUNT(DISTINCT match_id || ':' || round_num) AS usage_rounds
FROM fact_round_events
WHERE event_type = 'kill'
AND attacker_steam_id IN ({placeholders})
AND weapon IS NOT NULL
AND weapon != ''
GROUP BY attacker_steam_id, weapon
""",
chunk,
).fetchall()
weapon_values = []
for row in weapon_rows:
rounds = round_counts.get(str(row[0]), 0)
kills = int(row[2] or 0)
headshots = int(row[3] or 0)
usage_rounds = int(row[4] or 0)
hs_rate = headshots / kills if kills else 0.0
usage_rate = usage_rounds / rounds if rounds else 0.0
kills_per_round = kills / rounds if rounds else 0.0
effectiveness = kills / usage_rounds if usage_rounds else 0.0
weapon_values.append((
str(row[0]),
row[1],
kills,
headshots,
hs_rate,
usage_rounds,
usage_rate,
kills_per_round,
effectiveness,
))
conn_l3.executemany(
"""
INSERT INTO dm_player_weapon_stats (
steam_id_64, weapon_name, total_kills, total_headshots,
hs_rate, usage_rounds, usage_rate,
avg_kills_per_round, effectiveness_score
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
weapon_values,
)
total_weapons += len(weapon_values)
period_values = _calculate_period_rows(history_rows)
conn_l3.executemany(
"""
INSERT INTO dm_player_period_stats (
steam_id_64, period_key, period_label,
period_start, period_end, matches, wins, win_rate,
avg_rating, avg_kd, avg_adr, avg_kast,
total_kills, total_deaths, sample_reliable
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
period_values,
)
total_periods += len(period_values)
record_values = _calculate_record_rows(history_rows)
conn_l3.executemany(
"""
INSERT INTO dm_player_records (
steam_id_64, record_key, record_label, record_value,
match_id, map_name, match_date
) VALUES (?, ?, ?, ?, ?, ?, ?)
""",
record_values,
)
total_records += len(record_values)
logger.info(
"Auxiliary marts rebuilt: %s history, %s map, %s weapon, "
"%s period, %s record rows",
total_history,
total_maps,
total_weapons,
total_periods,
total_records,
)
def _group_player_match_rows(history_rows):
grouped = defaultdict(list)
for row in history_rows:
grouped[str(row['steam_id_64'])].append(row)
for rows in grouped.values():
rows.sort(key=lambda row: (row['start_time'] or 0, row['match_id']))
return grouped
def _safe_average(rows, key):
values = [float(row[key]) for row in rows if row[key] is not None]
return sum(values) / len(values) if values else None
def _calculate_period_rows(history_rows):
result = []
for steam_id, all_rows in _group_player_match_rows(history_rows).items():
latest_time = max(int(row['start_time'] or 0) for row in all_rows)
period_groups = [
('career', '生涯', all_rows),
('last_10', '最近 10 场', all_rows[-10:]),
('last_20', '最近 20 场', all_rows[-20:]),
('last_30', '最近 30 场', all_rows[-30:]),
(
'days_30',
'最近 30 天',
[
row for row in all_rows
if int(row['start_time'] or 0) >= latest_time - 30 * 86400
],
),
(
'days_90',
'最近 90 天',
[
row for row in all_rows
if int(row['start_time'] or 0) >= latest_time - 90 * 86400
],
),
]
for period_key, period_label, rows in period_groups:
if not rows:
continue
matches = len(rows)
wins = sum(1 for row in rows if row['is_win'])
kills = sum(int(row['kills'] or 0) for row in rows)
deaths = sum(int(row['deaths'] or 0) for row in rows)
result.append((
steam_id,
period_key,
period_label,
min(int(row['start_time'] or 0) for row in rows),
max(int(row['start_time'] or 0) for row in rows),
matches,
wins,
wins / matches,
_safe_average(rows, 'rating'),
kills / deaths if deaths else float(kills),
_safe_average(rows, 'adr'),
_safe_average(rows, 'kast'),
kills,
deaths,
1 if matches >= 10 else 0,
))
return result
def _calculate_record_rows(history_rows):
result = []
metric_definitions = (
('highest_rating', '最高 Rating', 'rating'),
('most_kills', '单场最多击杀', 'kills'),
('highest_adr', '单场最高 ADR', 'adr'),
('highest_kd', '单场最高 K/D', 'kd_ratio'),
('most_headshots', '单场最多爆头', 'headshot_count'),
)
for steam_id, rows in _group_player_match_rows(history_rows).items():
for record_key, record_label, field in metric_definitions:
candidates = [row for row in rows if row[field] is not None]
if not candidates:
continue
best = max(
candidates,
key=lambda row: (
float(row[field]),
int(row['start_time'] or 0),
),
)
result.append((
steam_id,
record_key,
record_label,
float(best[field]),
best['match_id'],
best['map_name'],
best['start_time'],
))
longest_streak = 0
current_streak = 0
streak_end = None
for row in rows:
if row['is_win']:
current_streak += 1
if current_streak >= longest_streak:
longest_streak = current_streak
streak_end = row
else:
current_streak = 0
if streak_end is not None:
result.append((
steam_id,
'longest_win_streak',
'最长连胜',
float(longest_streak),
streak_end['match_id'],
streak_end['map_name'],
streak_end['start_time'],
))
return result
def _update_percentiles(conn_l3, steam_ids):
"""Calculate a real percentile among eligible players in this build."""
if not steam_ids:
return
score_rows = []
for start in range(0, len(steam_ids), 400):
chunk = steam_ids[start:start + 400]
placeholders = ','.join('?' for _ in chunk)
conn_l3.execute(
f"""
UPDATE dm_player_features
SET tier_percentile = NULL
WHERE steam_id_64 IN ({placeholders})
""",
chunk,
)
score_rows.extend(conn_l3.execute(
f"""
SELECT steam_id_64, score_overall
FROM dm_player_features
WHERE steam_id_64 IN ({placeholders})
AND score_overall > 0
""",
chunk,
).fetchall())
if not score_rows:
return
scores = [float(row[1]) for row in score_rows]
percentile_values = []
for row in score_rows:
score = float(row[1])
percentile = sum(value <= score for value in scores) / len(scores) * 100
percentile_values.append((round(percentile, 2), str(row[0])))
conn_l3.executemany(
"""
UPDATE dm_player_features
SET tier_percentile = ?
WHERE steam_id_64 = ?
""",
percentile_values,
)
logger.info("Updated percentiles for %s eligible players", len(score_rows))
def _parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--force", action="store_true")
parser.add_argument("--workers", type=int, default=1)
parser.add_argument("--no-backup", action="store_true")
return parser.parse_args()
if __name__ == "__main__":
args = _parse_args()
main(force_all=args.force, workers=args.workers)
main(
force_all=args.force,
workers=args.workers,
create_backup=not args.no_backup,
)