Jacky/JKTV-online
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2.0.0 Alpha: Data Refinery

Browse Source
This commit is contained in:
Jacky
2026-08-08 21:31:56 +08:00
parent fa75081d4d
commit 562775e5db
48 changed files with 4172 additions and 661 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/services/etl_service.py
+41 -3
View File
@@ -4,13 +4,51 @@ import sys
from web.config import Config
class EtlService:
SCRIPT_PATHS = {
'L1A.py': os.path.join('database', 'L1', 'L1_Builder.py'),
'L2_Builder.py': os.path.join('database', 'L2', 'L2_Builder.py'),
'L3_Builder.py': os.path.join('database', 'L3', 'L3_Builder.py'),
}
@staticmethod
def start_pipeline(job_id, match_id=None, replace=False):
script_path = os.path.join(
Config.BASE_DIR,
'database',
'pipeline.py',
)
command = [
sys.executable,
script_path,
'--job-id',
str(int(job_id)),
]
if match_id:
command.extend(['--match-id', str(match_id)])
if replace:
command.append('--replace')
process = subprocess.Popen(
command,
cwd=Config.BASE_DIR,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
)
return process.pid
@staticmethod
def run_script(script_name, args=None):
"""
Executes an ETL script located in the ETL directory.
Executes an allow-listed data builder from its actual repository path.
Returns (success, message)
"""
script_path = os.path.join(Config.BASE_DIR, 'ETL', script_name)
relative_path = EtlService.SCRIPT_PATHS.get(script_name)
if not relative_path:
return False, f"Unsupported data script: {script_name}"
script_path = os.path.join(Config.BASE_DIR, relative_path)
if not os.path.exists(script_path):
return False, f"Script not found: {script_path}"
@@ -28,7 +66,7 @@ class EtlService:
cwd=Config.BASE_DIR,
capture_output=True,
text=True,
timeout=300 # 5 min timeout
timeout=900
)
if result.returncode == 0:
web/services/feature_service.py
+69 -34
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Any, Iterable
from typing import Any
from web.database import query_db
@@ -138,20 +138,47 @@ class FeatureService:
}
order_col = sort_map.get(sort_by, "core_avg_rating")
where = []
args: list[Any] = []
if search:
where.append("steam_id_64 IN (SELECT steam_id_64 FROM dim_players WHERE username LIKE ?)")
args.append(f"%{search}%")
where_sql = f"WHERE {' AND '.join(where)}" if where else ""
rows = query_db(
"l3",
f"SELECT * FROM dm_player_features {where_sql} ORDER BY {order_col} DESC LIMIT ? OFFSET ?",
args + [per_page, offset],
)
total_row = query_db("l3", f"SELECT COUNT(*) as cnt FROM dm_player_features {where_sql}", args, one=True)
total = int(total_row["cnt"]) if total_row else 0
dim_rows = query_db(
"l2",
"""
SELECT steam_id_64
FROM dim_players
WHERE LOWER(username) LIKE LOWER(?) OR steam_id_64 LIKE ?
""",
[f"%{search}%", f"%{search}%"],
)
matching_ids = [str(row["steam_id_64"]) for row in dim_rows]
rows = []
for start in range(0, len(matching_ids), 500):
chunk = matching_ids[start:start + 500]
placeholders = ",".join("?" for _ in chunk)
rows.extend(query_db(
"l3",
f"SELECT * FROM dm_player_features "
f"WHERE steam_id_64 IN ({placeholders})",
chunk,
))
rows = sorted(
rows,
key=lambda row: row[order_col] if row[order_col] is not None else float("-inf"),
reverse=True,
)
total = len(rows)
rows = rows[offset:offset + per_page]
else:
rows = query_db(
"l3",
f"SELECT * FROM dm_player_features "
f"ORDER BY {order_col} DESC LIMIT ? OFFSET ?",
[per_page, offset],
)
total_row = query_db(
"l3",
"SELECT COUNT(*) as cnt FROM dm_player_features",
one=True,
)
total = int(total_row["cnt"]) if total_row else 0
players = [FeatureService._normalize_features(dict(r)) for r in rows] if rows else []
players = [p for p in players if p]
@@ -160,19 +187,11 @@ class FeatureService:
@staticmethod
def get_roster_features_distribution(target_steam_id: str):
from web.services.web_service import WebService
import json
from web.services.team_context_service import TeamContextService
lineups = WebService.get_lineups()
roster_ids: list[str] = []
if lineups:
try:
p_ids = [str(i) for i in json.loads(lineups[0].get("player_ids_json") or "[]")]
if str(target_steam_id) in p_ids:
roster_ids = p_ids
except Exception:
roster_ids = []
roster_ids = TeamContextService.get_active_roster_ids()
if str(target_steam_id) not in roster_ids:
roster_ids = []
if not roster_ids:
return None
@@ -202,7 +221,17 @@ class FeatureService:
sample_keys = list(p.keys())
break
lower_is_better = {"int_timing_first_contact_time", "tac_avg_fd", "core_avg_match_duration"}
lower_is_better = {
"int_timing_first_contact_time",
"int_trade_response_time",
"tac_avg_fd",
"tac_fd_rate",
"core_avg_match_duration",
"core_dpr",
"meta_rating_volatility",
"meta_map_stability",
"meta_elo_tier_stability",
}
result: dict[str, Any] = {}
for m in sample_keys:
@@ -224,16 +253,22 @@ class FeatureService:
values = []
for p in stats_map.values():
v = (p or {}).get(m)
if v is None:
continue
try:
values.append(float(v) if v is not None else 0.0)
values.append(float(v))
except (ValueError, TypeError):
values.append(0.0)
continue
target_val_raw = (stats_map.get(target_steam_id) or {}).get(m)
if target_val_raw is None or not values:
result[m] = None
continue
try:
target_val = float(target_val_raw) if target_val_raw is not None else 0.0
target_val = float(target_val_raw)
except (ValueError, TypeError):
target_val = 0.0
result[m] = None
continue
is_reverse = m not in lower_is_better
# Sort values. For standard metrics, higher is better (reverse=True).
@@ -251,9 +286,9 @@ class FeatureService:
"val": target_val,
"rank": rank,
"total": len(values_sorted),
"min": min(values_sorted) if values_sorted else 0,
"max": max(values_sorted) if values_sorted else 0,
"avg": (sum(values_sorted) / len(values_sorted)) if values_sorted else 0,
"min": min(values_sorted),
"max": max(values_sorted),
"avg": sum(values_sorted) / len(values_sorted),
"inverted": not is_reverse,
}
return result
web/services/import_service.py
+186
View File
@@ -0,0 +1,186 @@
import hashlib
import json
import os
from pathlib import Path
import re
import sqlite3
from typing import Any, Dict
from database.job_store import JobStore
from database.paths import L1_DB, OUTPUT_ARENA
from web.config import Config
MATCH_ID_PATTERN = re.compile(r'\bg161-[0-9]{10,}\b')
class ImportValidationError(ValueError):
pass
class DuplicateMatchError(ImportValidationError):
pass
class MatchImportService:
@staticmethod
def validate_capture(raw_bytes: bytes) -> Dict[str, Any]:
if not raw_bytes:
raise ImportValidationError('Uploaded file is empty')
try:
text = raw_bytes.decode('utf-8-sig')
except UnicodeDecodeError as exc:
raise ImportValidationError('Capture must be UTF-8 JSON') from exc
try:
capture = json.loads(text)
except json.JSONDecodeError as exc:
raise ImportValidationError(
f'Invalid JSON at line {exc.lineno}, column {exc.colno}'
) from exc
if not isinstance(capture, list) or not capture:
raise ImportValidationError(
'Capture root must be a non-empty list of network responses'
)
urls = []
successful_responses = 0
for index, item in enumerate(capture):
if not isinstance(item, dict):
raise ImportValidationError(
f'Capture item {index} must be an object'
)
url = item.get('url')
if not isinstance(url, str) or not url:
raise ImportValidationError(
f'Capture item {index} has no URL'
)
urls.append(url)
if item.get('status') == 200 and item.get('body') is not None:
successful_responses += 1
match_ids = sorted({
match.group(0)
for url in urls
for match in MATCH_ID_PATTERN.finditer(url)
})
if len(match_ids) != 1:
raise ImportValidationError(
f'Capture must reference exactly one match ID; found {match_ids}'
)
if successful_responses < 2:
raise ImportValidationError(
'Capture does not contain enough successful API responses'
)
match_id = match_ids[0]
has_match_data = any(
f'/api/data/match/{match_id}' in url for url in urls
)
has_round_data = any(
f'/api/match/round/{match_id}' in url for url in urls
)
if not has_match_data or not has_round_data:
missing = []
if not has_match_data:
missing.append('match data')
if not has_round_data:
missing.append('round data')
raise ImportValidationError(
f"Capture is missing required endpoint(s): {', '.join(missing)}"
)
return {
'match_id': match_id,
'content_sha256': hashlib.sha256(raw_bytes).hexdigest(),
'response_count': len(capture),
'successful_responses': successful_responses,
'text': text,
}
@staticmethod
def _existing_l1_content(match_id: str):
if not L1_DB.exists():
return None
db = sqlite3.connect(str(L1_DB))
try:
row = db.execute(
"""
SELECT content
FROM raw_iframe_network
WHERE match_id = ?
""",
[match_id],
).fetchone()
return row[0] if row else None
finally:
db.close()
@staticmethod
def prepare_import(
raw_bytes: bytes,
original_filename: str,
created_by: str,
replace: bool = False,
):
validation = MatchImportService.validate_capture(raw_bytes)
match_id = validation['match_id']
content_hash = validation['content_sha256']
existing_content = MatchImportService._existing_l1_content(match_id)
if existing_content is not None:
existing_hash = hashlib.sha256(
existing_content.encode('utf-8')
).hexdigest()
if existing_hash == content_hash:
raise DuplicateMatchError(
f'Match {match_id} is already imported with identical data'
)
if not replace:
raise DuplicateMatchError(
f'Match {match_id} already exists with different data; '
'explicit replacement is required'
)
match_dir = OUTPUT_ARENA / match_id
match_dir.mkdir(parents=True, exist_ok=True)
destination = match_dir / 'iframe_network.json'
if destination.exists() and not replace:
current_hash = hashlib.sha256(destination.read_bytes()).hexdigest()
if current_hash == content_hash:
raise DuplicateMatchError(
f'Match {match_id} is already queued with identical data'
)
raise DuplicateMatchError(
f'Pending capture already exists for {match_id}'
)
temporary = destination.with_suffix('.json.tmp')
temporary.write_bytes(raw_bytes)
os.replace(str(temporary), str(destination))
store = JobStore(Config.DB_WEB_PATH)
job_id = store.create_job(
'match_import',
match_id=match_id,
input_path=str(destination),
created_by=created_by,
)
store.upsert_match_import(
match_id,
content_hash,
str(destination),
'queued',
job_id,
)
return {
'job_id': job_id,
'match_id': match_id,
'content_sha256': content_hash,
'response_count': validation['response_count'],
'source_path': str(destination),
'original_filename': Path(original_filename or '').name,
'replace': bool(replace),
}
web/services/integrity_service.py
+511
View File
@@ -0,0 +1,511 @@
from datetime import datetime, timezone
import json
import os
import sqlite3
from database.maintenance import backup_storage_status
from web.config import Config
from web.services.team_context_service import TeamContextService
class IntegrityService:
DATABASES = {
'L2': Config.DB_L2_PATH,
'L3': Config.DB_L3_PATH,
'Web': Config.DB_WEB_PATH,
}
@staticmethod
def _check(checks, name, status, detail, value=None):
checks.append({
'name': name,
'status': status,
'detail': detail,
'value': value,
})
@staticmethod
def _connect(path):
db = sqlite3.connect(path, timeout=Config.SQLITE_TIMEOUT_SECONDS)
db.row_factory = sqlite3.Row
return db
@staticmethod
def build_report():
checks = []
counts = {}
connections = {}
try:
for name, path in IntegrityService.DATABASES.items():
if not os.path.exists(path):
IntegrityService._check(
checks,
f'{name} database',
'fail',
f'Missing file: {path}',
)
continue
try:
db = IntegrityService._connect(path)
connections[name] = db
result = db.execute('PRAGMA quick_check').fetchone()[0]
IntegrityService._check(
checks,
f'{name} database',
'pass' if result == 'ok' else 'fail',
f'quick_check: {result}',
os.path.getsize(path),
)
except sqlite3.Error as exc:
IntegrityService._check(
checks,
f'{name} database',
'fail',
str(exc),
)
l2 = connections.get('L2')
if l2:
IntegrityService._check_l2(l2, checks, counts)
l3 = connections.get('L3')
roster_ids = TeamContextService.get_active_roster_ids()
counts['active_roster'] = len(roster_ids)
if l3:
IntegrityService._check_l3(l3, roster_ids, checks, counts)
web = connections.get('Web')
if web:
IntegrityService._check_web(web, checks, counts)
backup_status = backup_storage_status()
counts['backup_sets'] = backup_status['sets']
counts['backup_bytes'] = backup_status['total_bytes']
IntegrityService._check(
checks,
'Backup retention',
'warn' if backup_status['sets'] > 3 else 'pass',
(
f"{backup_status['sets']} backup sets, "
f"{backup_status['total_bytes']:,} bytes"
),
backup_status['sets'],
)
finally:
for db in connections.values():
db.close()
status_order = {'pass': 0, 'warn': 1, 'fail': 2}
overall_status = max(
(check['status'] for check in checks),
key=lambda status: status_order[status],
default='fail',
)
return {
'generated_at': datetime.now(timezone.utc).isoformat(),
'overall_status': overall_status,
'counts': counts,
'checks': checks,
'totals': {
status: sum(1 for check in checks if check['status'] == status)
for status in ('pass', 'warn', 'fail')
},
}
@staticmethod
def _table_names(db):
return {
row[0]
for row in db.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
)
}
@staticmethod
def _check_l2(db, checks, counts):
required_tables = {
'dim_players',
'fact_matches',
'fact_match_teams',
'fact_match_players',
'fact_rounds',
'fact_round_events',
'fact_round_player_economy',
}
missing = sorted(required_tables - IntegrityService._table_names(db))
IntegrityService._check(
checks,
'L2 required tables',
'fail' if missing else 'pass',
f"Missing: {', '.join(missing)}" if missing else 'All required tables exist',
)
if missing:
return
table_count_map = {
'matches': 'fact_matches',
'players': 'dim_players',
'player_match_rows': 'fact_match_players',
'rounds': 'fact_rounds',
'events': 'fact_round_events',
'economy_rows': 'fact_round_player_economy',
}
for key, table in table_count_map.items():
counts[key] = db.execute(f'SELECT COUNT(*) FROM {table}').fetchone()[0]
orphan_players = db.execute(
"""
SELECT COUNT(*)
FROM fact_match_players mp
LEFT JOIN fact_matches m ON m.match_id = mp.match_id
WHERE m.match_id IS NULL
"""
).fetchone()[0]
IntegrityService._check(
checks,
'Player-match referential integrity',
'fail' if orphan_players else 'pass',
f'{orphan_players} player rows reference missing matches',
orphan_players,
)
orphan_events = db.execute(
"""
SELECT COUNT(*)
FROM fact_round_events e
LEFT JOIN fact_rounds r
ON r.match_id = e.match_id AND r.round_num = e.round_num
WHERE r.match_id IS NULL
"""
).fetchone()[0]
IntegrityService._check(
checks,
'Round-event referential integrity',
'fail' if orphan_events else 'pass',
f'{orphan_events} events reference missing rounds',
orphan_events,
)
unusual_rosters = db.execute(
"""
SELECT COUNT(*)
FROM (
SELECT match_id, COUNT(*) AS player_count
FROM fact_match_players
GROUP BY match_id
HAVING player_count != 10
)
"""
).fetchone()[0]
IntegrityService._check(
checks,
'Match player cardinality',
'warn' if unusual_rosters else 'pass',
f'{unusual_rosters} matches do not contain exactly 10 players',
unusual_rosters,
)
missing_names = db.execute(
"SELECT COUNT(*) FROM dim_players WHERE username IS NULL OR TRIM(username) = ''"
).fetchone()[0]
IntegrityService._check(
checks,
'Player identity coverage',
'warn' if missing_names else 'pass',
f'{missing_names} players have no username',
missing_names,
)
required_indexes = {
'idx_match_players_player_match',
'idx_match_players_match_team',
'idx_match_players_party',
'idx_round_events_victim',
'idx_economy_player_match',
'idx_matches_map_time',
}
existing_indexes = {
row[0] for row in db.execute(
"SELECT name FROM sqlite_master WHERE type = 'index'"
)
}
missing_indexes = sorted(required_indexes - existing_indexes)
IntegrityService._check(
checks,
'L2 operational indexes',
'fail' if missing_indexes else 'pass',
(
f"Missing: {', '.join(missing_indexes)}"
if missing_indexes else
'All high-frequency query indexes exist'
),
len(required_indexes) - len(missing_indexes),
)
@staticmethod
def _check_l3(db, roster_ids, checks, counts):
required_tables = {
'dm_player_features',
'dm_player_match_history',
'dm_player_map_stats',
'dm_player_period_stats',
'dm_player_records',
'dm_player_weapon_stats',
}
missing = sorted(required_tables - IntegrityService._table_names(db))
IntegrityService._check(
checks,
'L3 required tables',
'fail' if missing else 'pass',
f"Missing: {', '.join(missing)}" if missing else 'All required tables exist',
)
if missing:
return
counts['l3_features'] = db.execute(
'SELECT COUNT(*) FROM dm_player_features'
).fetchone()[0]
counts['l3_history'] = db.execute(
'SELECT COUNT(*) FROM dm_player_match_history'
).fetchone()[0]
counts['l3_maps'] = db.execute(
'SELECT COUNT(*) FROM dm_player_map_stats'
).fetchone()[0]
counts['l3_weapons'] = db.execute(
'SELECT COUNT(*) FROM dm_player_weapon_stats'
).fetchone()[0]
counts['l3_periods'] = db.execute(
'SELECT COUNT(*) FROM dm_player_period_stats'
).fetchone()[0]
counts['l3_records'] = db.execute(
'SELECT COUNT(*) FROM dm_player_records'
).fetchone()[0]
if roster_ids:
placeholders = ','.join('?' for _ in roster_ids)
covered = db.execute(
f"""
SELECT COUNT(DISTINCT steam_id_64)
FROM dm_player_features
WHERE steam_id_64 IN ({placeholders})
""",
roster_ids,
).fetchone()[0]
IntegrityService._check(
checks,
'Active roster feature coverage',
'pass' if covered == len(roster_ids) else 'fail',
f'{covered}/{len(roster_ids)} roster players have L3 features',
covered,
)
expected_history = db.execute(
f"""
SELECT COALESCE(SUM(total_matches), 0)
FROM dm_player_features
WHERE steam_id_64 IN ({placeholders})
""",
roster_ids,
).fetchone()[0]
actual_history = db.execute(
f"""
SELECT COUNT(*)
FROM dm_player_match_history
WHERE steam_id_64 IN ({placeholders})
""",
roster_ids,
).fetchone()[0]
IntegrityService._check(
checks,
'Roster history completeness',
'pass' if actual_history == expected_history else 'fail',
f'{actual_history}/{expected_history} player-match rows materialized',
actual_history,
)
score_rows = db.execute(
f"""
SELECT steam_id_64, score_overall, tier_percentile
FROM dm_player_features
WHERE steam_id_64 IN ({placeholders})
AND score_overall > 0
""",
roster_ids,
).fetchall()
scores = [float(row['score_overall']) for row in score_rows]
invalid_percentiles = 0
for row in score_rows:
expected = (
sum(value <= float(row['score_overall']) for value in scores)
/ len(scores)
* 100
)
actual = row['tier_percentile']
if actual is None or abs(float(actual) - expected) > 0.011:
invalid_percentiles += 1
IntegrityService._check(
checks,
'Roster percentile correctness',
'pass' if invalid_percentiles == 0 else 'warn',
f'{invalid_percentiles} eligible players have stale percentiles',
invalid_percentiles,
)
for table, label in (
('dm_player_period_stats', 'Roster period-stat coverage'),
('dm_player_records', 'Roster record coverage'),
):
covered = db.execute(
f"""
SELECT COUNT(DISTINCT steam_id_64)
FROM {table}
WHERE steam_id_64 IN ({placeholders})
""",
roster_ids,
).fetchone()[0]
IntegrityService._check(
checks,
label,
'pass' if covered == len(roster_ids) else 'fail',
f'{covered}/{len(roster_ids)} roster players covered',
covered,
)
for key, label in (
('l3_history', 'Player match history mart'),
('l3_maps', 'Player map stats mart'),
('l3_weapons', 'Player weapon stats mart'),
('l3_periods', 'Player period stats mart'),
('l3_records', 'Player records mart'),
):
value = counts[key]
IntegrityService._check(
checks,
label,
'pass' if value else 'warn',
f'{value} rows',
value,
)
if roster_ids:
placeholders = ','.join('?' for _ in roster_ids)
scope_sql = f"AND steam_id_64 IN ({placeholders})"
scope_args = roster_ids
else:
scope_sql = ''
scope_args = []
placeholder_rows = db.execute(
f"""
SELECT COUNT(*)
FROM dm_player_features
WHERE int_pos_site_a_control_rate = 0.33
AND int_pos_site_b_control_rate = 0.33
AND int_pos_mid_control_rate = 0.34
{scope_sql}
""",
scope_args,
).fetchone()[0]
IntegrityService._check(
checks,
'Experimental spatial metrics',
'warn' if placeholder_rows else 'pass',
f'{placeholder_rows} active-roster rows contain placeholder site-control values',
placeholder_rows,
)
@staticmethod
def _check_web(db, checks, counts):
required_tables = {
'comments',
'etl_jobs',
'match_imports',
'player_metadata',
'schema_migrations',
'strategy_boards',
'team_lineups',
'wiki_pages',
}
missing = sorted(required_tables - IntegrityService._table_names(db))
IntegrityService._check(
checks,
'Web required tables',
'fail' if missing else 'pass',
f"Missing: {', '.join(missing)}" if missing else 'All required tables exist',
)
if missing:
return
counts['etl_jobs'] = db.execute(
'SELECT COUNT(*) FROM etl_jobs'
).fetchone()[0]
counts['match_imports'] = db.execute(
'SELECT COUNT(*) FROM match_imports'
).fetchone()[0]
schema_version = db.execute(
'SELECT COALESCE(MAX(version), 0) FROM schema_migrations'
).fetchone()[0]
counts['web_schema_version'] = schema_version
IntegrityService._check(
checks,
'Web schema version',
'pass' if schema_version == Config.WEB_SCHEMA_VERSION else 'fail',
f'{schema_version}/{Config.WEB_SCHEMA_VERSION}',
schema_version,
)
foreign_key_errors = db.execute(
'PRAGMA foreign_key_check'
).fetchall()
IntegrityService._check(
checks,
'Web foreign key integrity',
'fail' if foreign_key_errors else 'pass',
f'{len(foreign_key_errors)} foreign key violations',
len(foreign_key_errors),
)
running_jobs = db.execute(
"""
SELECT COUNT(*)
FROM etl_jobs
WHERE status = 'running'
"""
).fetchone()[0]
IntegrityService._check(
checks,
'Pipeline concurrency',
'warn' if running_jobs > 1 else 'pass',
f'{running_jobs} running pipeline jobs',
running_jobs,
)
lineups = db.execute(
'SELECT id, player_ids_json, is_active FROM team_lineups'
).fetchall()
counts['lineups'] = len(lineups)
invalid_lineups = 0
for lineup in lineups:
try:
player_ids = json.loads(lineup['player_ids_json'] or '[]')
if not isinstance(player_ids, list):
invalid_lineups += 1
except (TypeError, json.JSONDecodeError):
invalid_lineups += 1
IntegrityService._check(
checks,
'Lineup JSON validity',
'fail' if invalid_lineups else 'pass',
f'{invalid_lineups} lineups contain invalid player ID JSON',
invalid_lineups,
)
active_count = sum(1 for lineup in lineups if lineup['is_active'] == 1)
IntegrityService._check(
checks,
'Active lineup',
'pass' if active_count == 1 else 'warn',
f'{active_count} active lineups configured',
active_count,
)
web/services/opponent_service.py
+46 -54
View File
@@ -1,19 +1,10 @@
from web.database import query_db
from web.services.web_service import WebService
import json
from web.services.team_context_service import TeamContextService
class OpponentService:
@staticmethod
def _get_active_roster_ids():
lineups = WebService.get_lineups()
active_roster_ids = []
if lineups:
try:
raw_ids = json.loads(lineups[0]['player_ids_json'])
active_roster_ids = [str(uid) for uid in raw_ids]
except:
pass
return active_roster_ids
return TeamContextService.get_active_roster_ids()
@staticmethod
def get_opponent_list(page=1, per_page=20, sort_by='matches', search=None):
@@ -21,30 +12,21 @@ class OpponentService:
if not roster_ids:
return [], 0
# Placeholders
roster_ph = ','.join('?' for _ in roster_ids)
# 1. Identify Matches involving our roster (at least 1 member? usually 2 for 'team' match)
# Let's say at least 1 for broader coverage as requested ("1 match sample")
# But "Our Team" usually implies the entity. Let's stick to matches where we can identify "Us".
# If we use >=1, we catch solo Q matches of roster members. The user said "Non-team members or 1 match sample",
# but implied "facing different our team lineups".
# Let's use the standard "candidate matches" logic (>=2 roster members) to represent "The Team".
# OR, if user wants "Opponent Analysis" for even 1 match, maybe they mean ANY match in DB?
# "Left Top add Opponent Analysis... (non-team member or 1 sample)"
# This implies we analyze PLAYERS who are NOT us.
# Let's stick to matches where >= 1 roster member played, to define "Us" vs "Them".
# Actually, let's look at ALL matches in DB, and any player NOT in active roster is an "Opponent".
# This covers "1 sample".
# Query:
# Select all players who are NOT in active roster.
# Group by steam_id.
# Aggregate stats.
where_clauses = [f"CAST(mp.steam_id_64 AS TEXT) NOT IN ({roster_ph})"]
args = list(roster_ids)
where_clauses = [
f"CAST(mp.steam_id_64 AS TEXT) NOT IN ({roster_ph})",
f"""
EXISTS (
SELECT 1
FROM fact_match_players roster_mp
WHERE roster_mp.match_id = mp.match_id
AND CAST(roster_mp.steam_id_64 AS TEXT) IN ({roster_ph})
AND roster_mp.team_id != mp.team_id
)
""",
]
args = list(roster_ids) + list(roster_ids)
if search:
where_clauses.append("(LOWER(p.username) LIKE LOWER(?) OR mp.steam_id_64 LIKE ?)")
@@ -61,16 +43,6 @@ class OpponentService:
elif sort_by == 'win_rate':
sort_sql = "win_rate DESC"
# Main Aggregation Query
# We need to join fact_matches to get match info (win/loss, elo) if needed,
# but fact_match_players has is_win (boolean) usually? No, it has team_id.
# We need to determine if THEY won.
# fact_match_players doesn't store is_win directly in schema (I should check schema, but stats_service calculates it).
# Wait, stats_service.get_player_trend uses `mp.is_win`?
# Let's check schema. `fact_match_players` usually has `match_id`, `team_id`.
# `fact_matches` has `winner_team`.
# So we join.
offset = (page - 1) * per_page
sql = f"""
@@ -151,10 +123,17 @@ class OpponentService:
LEFT JOIN fact_match_teams fmt_gid ON mp.match_id = fmt_gid.match_id AND fmt_gid.group_id = mp.team_id
LEFT JOIN fact_match_teams fmt_tid ON mp.match_id = fmt_tid.match_id AND fmt_tid.group_tid = mp.match_team_id
WHERE CAST(mp.steam_id_64 AS TEXT) NOT IN ({roster_ph})
AND EXISTS (
SELECT 1
FROM fact_match_players roster_mp
WHERE roster_mp.match_id = mp.match_id
AND CAST(roster_mp.steam_id_64 AS TEXT) IN ({roster_ph})
AND roster_mp.team_id != mp.team_id
)
GROUP BY mp.steam_id_64
"""
rows = query_db('l2', sql, roster_ids)
rows = query_db('l2', sql, roster_ids + roster_ids)
# Initialize Buckets
elo_buckets = {'<1000': 0, '1000-1200': 0, '1200-1400': 0, '1400-1600': 0, '1600-1800': 0, '1800-2000': 0, '>2000': 0}
@@ -216,13 +195,12 @@ class OpponentService:
player = dict(info)
player['avatar_url'] = StatsService.resolve_avatar_url(steam_id, player.get('avatar_url'))
# 2. Match History vs Us (All matches this player played)
# We define "Us" as matches where this player is an opponent.
# But actually, we just show ALL their matches in our DB, assuming our DB only contains matches relevant to us?
# Usually yes, but if we have a huge DB, we might want to filter by "Contains Roster Member".
# For now, show all matches in DB for this player.
sql_history = """
roster_ids = OpponentService._get_active_roster_ids()
if not roster_ids:
return None
roster_ph = ','.join('?' for _ in roster_ids)
sql_history = f"""
SELECT
m.match_id, m.start_time, m.map_name, m.score_team1, m.score_team2, m.winner_team,
mp.team_id, mp.match_team_id, mp.rating, mp.kd_ratio, mp.adr, mp.kills, mp.deaths,
@@ -236,9 +214,16 @@ class OpponentService:
LEFT JOIN fact_match_teams fmt_gid ON mp.match_id = fmt_gid.match_id AND fmt_gid.group_id = mp.team_id
LEFT JOIN fact_match_teams fmt_tid ON mp.match_id = fmt_tid.match_id AND fmt_tid.group_tid = mp.match_team_id
WHERE mp.steam_id_64 = ?
AND EXISTS (
SELECT 1
FROM fact_match_players roster_mp
WHERE roster_mp.match_id = mp.match_id
AND CAST(roster_mp.steam_id_64 AS TEXT) IN ({roster_ph})
AND roster_mp.team_id != mp.team_id
)
ORDER BY m.start_time DESC
"""
history = query_db('l2', sql_history, [steam_id])
history = query_db('l2', sql_history, [steam_id] + roster_ids)
# 3. Aggregation by ELO
elo_buckets = {
@@ -389,11 +374,18 @@ class OpponentService:
LEFT JOIN fact_match_teams fmt_gid ON mp.match_id = fmt_gid.match_id AND fmt_gid.group_id = mp.team_id
LEFT JOIN fact_match_teams fmt_tid ON mp.match_id = fmt_tid.match_id AND fmt_tid.group_tid = mp.match_team_id
WHERE CAST(mp.steam_id_64 AS TEXT) NOT IN ({roster_ph})
AND EXISTS (
SELECT 1
FROM fact_match_players roster_mp
WHERE roster_mp.match_id = mp.match_id
AND CAST(roster_mp.steam_id_64 AS TEXT) IN ({roster_ph})
AND roster_mp.team_id != mp.team_id
)
AND m.map_name IS NOT NULL AND m.map_name <> ''
GROUP BY m.map_name
ORDER BY matches DESC
"""
rows = query_db('l2', sql, roster_ids)
rows = query_db('l2', sql, roster_ids + roster_ids)
results = []
for r in rows:
d = dict(r)
web/services/player_profile_service.py
+123
View File
@@ -0,0 +1,123 @@
from web.database import query_db
class PlayerProfileService:
PERIOD_KEYS = (
'career',
'last_10',
'last_20',
'last_30',
'days_30',
'days_90',
)
@staticmethod
def get_period_stats(steam_id):
rows = query_db(
'l3',
"""
SELECT *
FROM dm_player_period_stats
WHERE steam_id_64 = ?
ORDER BY CASE period_key
WHEN 'career' THEN 1
WHEN 'last_10' THEN 2
WHEN 'last_20' THEN 3
WHEN 'last_30' THEN 4
WHEN 'days_30' THEN 5
WHEN 'days_90' THEN 6
ELSE 99
END
""",
[steam_id],
)
return [dict(row) for row in rows]
@staticmethod
def get_period(steam_id, period_key):
if period_key not in PlayerProfileService.PERIOD_KEYS:
return None
row = query_db(
'l3',
"""
SELECT *
FROM dm_player_period_stats
WHERE steam_id_64 = ? AND period_key = ?
""",
[steam_id, period_key],
one=True,
)
return dict(row) if row else None
@staticmethod
def get_records(steam_id):
rows = query_db(
'l3',
"""
SELECT *
FROM dm_player_records
WHERE steam_id_64 = ?
ORDER BY CASE record_key
WHEN 'highest_rating' THEN 1
WHEN 'most_kills' THEN 2
WHEN 'highest_adr' THEN 3
WHEN 'highest_kd' THEN 4
WHEN 'most_headshots' THEN 5
WHEN 'longest_win_streak' THEN 6
ELSE 99
END
""",
[steam_id],
)
return [dict(row) for row in rows]
@staticmethod
def get_period_history(steam_id, period_key):
period = PlayerProfileService.get_period(steam_id, period_key)
if not period:
return []
rows = query_db(
'l3',
"""
SELECT
match_date AS start_time,
rating,
kd_ratio,
adr,
kast,
match_id,
map_name,
is_win,
match_sequence AS match_index
FROM dm_player_match_history
WHERE steam_id_64 = ?
AND match_date BETWEEN ? AND ?
ORDER BY match_date, match_id
""",
[steam_id, period['period_start'], period['period_end']],
)
return [dict(row) for row in rows]
@staticmethod
def get_map_stats(steam_id):
rows = query_db(
'l3',
"""
SELECT
map_name,
matches,
wins,
win_rate,
avg_rating AS rating,
avg_kd AS kd,
avg_adr AS adr,
avg_kast AS kast,
best_rating,
worst_rating
FROM dm_player_map_stats
WHERE steam_id_64 = ?
ORDER BY matches DESC, map_name
""",
[steam_id],
)
return [dict(row) for row in rows]
web/services/stats_service.py
+102 -163
View File
@@ -1,4 +1,4 @@
from web.database import query_db, execute_db
from web.database import query_db
from flask import current_app, url_for
import os
@@ -13,7 +13,7 @@ class StatsService:
try:
# Check local file first (User Request: "directly associate if exists")
base = os.path.join(current_app.root_path, 'static', 'avatars')
for ext in ('.jpg', '.png', '.jpeg'):
for ext in ('.jpg', '.png', '.jpeg', '.webp'):
fname = f"{steam_id}{ext}"
fpath = os.path.join(base, fname)
if os.path.exists(fpath):
@@ -38,18 +38,9 @@ class StatsService:
'round_stats': [{'type', 'count', 'wins', 'win_rate'}]
}
"""
# 1. Get Active Roster
from web.services.web_service import WebService
import json
lineups = WebService.get_lineups()
active_roster_ids = []
if lineups:
try:
raw_ids = json.loads(lineups[0]['player_ids_json'])
active_roster_ids = [str(uid) for uid in raw_ids]
except:
pass
from web.services.team_context_service import TeamContextService
active_roster_ids = TeamContextService.get_active_roster_ids()
if not active_roster_ids:
return {}
@@ -60,21 +51,23 @@ class StatsService:
placeholders = ','.join('?' for _ in active_roster_ids)
# Step A: Get Candidate Match IDs (matches with >= 2 roster players)
# Also get the team_id of our players in that match to determine win
candidate_sql = f"""
SELECT mp.match_id, MAX(mp.team_id) as our_team_id
SELECT mp.match_id, mp.team_id as our_team_id,
COUNT(DISTINCT mp.steam_id_64) as roster_count
FROM fact_match_players mp
WHERE CAST(mp.steam_id_64 AS TEXT) IN ({placeholders})
GROUP BY mp.match_id
GROUP BY mp.match_id, mp.team_id
HAVING COUNT(DISTINCT mp.steam_id_64) >= 2
ORDER BY mp.match_id, roster_count DESC, mp.team_id
"""
candidate_rows = query_db('l2', candidate_sql, active_roster_ids)
if not candidate_rows:
return {}
candidate_map = {row['match_id']: row['our_team_id'] for row in candidate_rows}
candidate_map = {}
for row in candidate_rows:
candidate_map.setdefault(row['match_id'], row['our_team_id'])
match_ids = list(candidate_map.keys())
match_placeholders = ','.join('?' for _ in match_ids)
@@ -221,11 +214,15 @@ class StatsService:
args.append(map_name)
if date_from:
where_clauses.append("start_time >= ?")
where_clauses.append(
"start_time >= CAST(strftime('%s', ?) AS INTEGER)"
)
args.append(date_from)
if date_to:
where_clauses.append("start_time <= ?")
where_clauses.append(
"start_time < CAST(strftime('%s', date(?, '+1 day')) AS INTEGER)"
)
args.append(date_to)
where_str = " AND ".join(where_clauses)
@@ -270,109 +267,51 @@ class StatsService:
party_rows = query_db('l2', party_sql, match_ids)
party_map = {row['match_id']: row['max_party'] for row in party_rows}
# --- New: Determine "Our Team" Result ---
# Logic: Check if any player from `active_roster` played in these matches.
# Use WebService to get the active roster
from web.services.web_service import WebService
import json
lineups = WebService.get_lineups()
active_roster_ids = []
if lineups:
try:
# Load IDs and ensure they are all strings for DB comparison consistency
raw_ids = json.loads(lineups[0]['player_ids_json'])
active_roster_ids = [str(uid) for uid in raw_ids]
except:
pass
from web.services.team_context_service import TeamContextService
active_roster_ids = TeamContextService.get_active_roster_ids()
# If no roster, we can't determine "Our Result"
if not active_roster_ids:
result_map = {}
else:
# 1. Get UIDs for Roster Members involved in these matches
# We query fact_match_players to ensure we get the UIDs actually used in these matches
roster_placeholders = ','.join('?' for _ in active_roster_ids)
uid_sql = f"""
SELECT DISTINCT steam_id_64, uid
roster_team_sql = f"""
SELECT match_id, team_id,
COUNT(DISTINCT steam_id_64) as roster_count
FROM fact_match_players
WHERE match_id IN ({placeholders})
AND CAST(steam_id_64 AS TEXT) IN ({roster_placeholders})
GROUP BY match_id, team_id
"""
combined_args_uid = match_ids + active_roster_ids
uid_rows = query_db('l2', uid_sql, combined_args_uid)
# Set of "Our UIDs" (as strings)
our_uids = set()
for r in uid_rows:
if r['uid']:
our_uids.add(str(r['uid']))
# 2. Get Group UIDs and Winner info from fact_match_teams
# We need to know which group contains our UIDs
teams_sql = f"""
SELECT fmt.match_id, fmt.group_id, fmt.group_uids, m.winner_team
FROM fact_match_teams fmt
JOIN fact_matches m ON fmt.match_id = m.match_id
WHERE fmt.match_id IN ({placeholders})
"""
teams_rows = query_db('l2', teams_sql, match_ids)
# 3. Determine Result per Match
roster_team_rows = query_db(
'l2',
roster_team_sql,
match_ids + active_roster_ids,
)
winner_by_match = {
str(match['match_id']): match['winner_team']
for match in matches
}
teams_by_match = {}
for row in roster_team_rows:
teams_by_match.setdefault(str(row['match_id']), []).append(
row['team_id']
)
result_map = {}
# Group data by match
match_groups = {} # match_id -> {group_id: [uids...], winner: int}
for r in teams_rows:
mid = r['match_id']
gid = r['group_id']
uids_str = r['group_uids'] or ""
# Split and clean UIDs
uids = set(str(u).strip() for u in uids_str.split(',') if u.strip())
if mid not in match_groups:
match_groups[mid] = {'groups': {}, 'winner': r['winner_team']}
match_groups[mid]['groups'][gid] = uids
# Analyze
for mid, data in match_groups.items():
winner_gid = data['winner']
groups = data['groups']
our_in_winner = False
our_in_loser = False
# Check each group
for gid, uids in groups.items():
# Intersection of Our UIDs and Group UIDs
common = our_uids.intersection(uids)
if common:
if gid == winner_gid:
our_in_winner = True
else:
our_in_loser = True
if our_in_winner and not our_in_loser:
result_map[mid] = 'win'
elif our_in_loser and not our_in_winner:
result_map[mid] = 'loss'
elif our_in_winner and our_in_loser:
result_map[mid] = 'mixed'
else:
# Fallback: If UID matching failed (maybe missing UIDs), try old team_id method?
# Or just leave it as None (safe)
pass
for match_id, team_ids in teams_by_match.items():
unique_team_ids = set(team_ids)
if len(unique_team_ids) > 1:
result_map[match_id] = 'mixed'
continue
our_team_id = next(iter(unique_team_ids))
result_map[match_id] = (
'win'
if str(our_team_id) == str(winner_by_match.get(match_id))
else 'loss'
)
# Convert to dict to modify
matches = [dict(m) for m in matches]
for m in matches:
m['avg_elo'] = elo_map.get(m['match_id'], 0)
m['max_party'] = party_map.get(m['match_id'], 1)
m['our_result'] = result_map.get(m['match_id'])
# Convert to dict to modify
matches = [dict(m) for m in matches]
for m in matches:
m['avg_elo'] = elo_map.get(m['match_id'], 0)
@@ -542,33 +481,20 @@ class StatsService:
@staticmethod
def get_shared_matches(steam_ids):
# Find matches where ALL steam_ids were present
if not steam_ids or len(steam_ids) < 1:
return []
steam_ids = list(dict.fromkeys(str(steam_id) for steam_id in steam_ids))
placeholders = ','.join('?' for _ in steam_ids)
count = len(steam_ids)
# We need to know which team the players were on to determine win/loss
# Assuming they were on the SAME team for "shared experience"
# If count=1, it's just match history
# Query: Get matches where all steam_ids are present
# Also join to get team_id to check if they were on the same team (optional but better)
# For simplicity in v1: Just check presence in the match.
# AND check if the player won.
# We need to return: match_id, map_name, score, result (Win/Loss)
# "Result" is relative to the lineup.
# If they were on the winning team, it's a Win.
sql = f"""
SELECT m.match_id, m.start_time, m.map_name, m.score_team1, m.score_team2, m.winner_team,
MAX(mp.team_id) as player_team_id -- Just take one team_id (assuming same)
mp.team_id as player_team_id
FROM fact_matches m
JOIN fact_match_players mp ON m.match_id = mp.match_id
WHERE mp.steam_id_64 IN ({placeholders})
GROUP BY m.match_id
GROUP BY m.match_id, mp.team_id
HAVING COUNT(DISTINCT mp.steam_id_64) = ?
ORDER BY m.start_time DESC
"""
@@ -580,14 +506,7 @@ class StatsService:
results = []
for r in rows:
# Determine if Win
# winner_team in DB is 'Team 1' or 'Team 2' usually, or the team name.
# fact_matches.winner_team stores the NAME of the winner? Or 'team1'/'team2'?
# Let's check how L2_Builder stores it. Usually it stores the name.
# But fact_match_players.team_id stores the name too.
# Logic: If m.winner_team == mp.team_id, then Win.
is_win = (r['winner_team'] == r['player_team_id'])
is_win = str(r['winner_team']) == str(r['player_team_id'])
# If winner_team is NULL or empty, it's a draw?
if not r['winner_team']:
@@ -628,7 +547,31 @@ class StatsService:
"""
l3_rows = query_db("l3", l3_sql, [steam_id, limit])
if l3_rows:
return l3_rows
history = [dict(row) for row in l3_rows]
match_ids = [row['match_id'] for row in history]
placeholders = ','.join('?' for _ in match_ids)
party_rows = query_db(
"l2",
f"""
SELECT me.match_id, COUNT(p.steam_id_64) AS party_size
FROM fact_match_players me
LEFT JOIN fact_match_players p
ON p.match_id = me.match_id
AND p.match_team_id = me.match_team_id
AND me.match_team_id > 0
WHERE me.steam_id_64 = ?
AND me.match_id IN ({placeholders})
GROUP BY me.match_id
""",
[steam_id] + match_ids,
)
party_map = {
row['match_id']: max(int(row['party_size'] or 0), 1)
for row in party_rows
}
for row in history:
row['party_size'] = party_map.get(row['match_id'], 1)
return history
sql = """
SELECT * FROM (
@@ -729,19 +672,10 @@ class StatsService:
Calculates rank and distribution of the target player within the active roster.
Now covers all L3 Basic Features for Detailed Panel.
"""
from web.services.web_service import WebService
from web.services.feature_service import FeatureService
import json
# 1. Get Active Roster IDs
lineups = WebService.get_lineups()
active_roster_ids = []
if lineups:
try:
raw_ids = json.loads(lineups[0]['player_ids_json'])
active_roster_ids = [str(uid) for uid in raw_ids]
except:
pass
from web.services.team_context_service import TeamContextService
active_roster_ids = TeamContextService.get_active_roster_ids()
if not active_roster_ids:
return None
@@ -851,33 +785,38 @@ class StatsService:
"basic_avg_rating", "basic_avg_kd", "basic_avg_adr", "basic_avg_kast", "basic_avg_rws",
]
lower_is_better = []
lower_is_better = {
"int_timing_first_contact_time",
"int_trade_response_time",
"tac_avg_fd",
"tac_fd_rate",
"core_avg_match_duration",
"core_dpr",
"meta_rating_volatility",
"meta_map_stability",
"meta_elo_tier_stability",
}
result = {}
for m in metrics:
values = []
non_numeric = False
for p in stats_map.values():
raw = (p or {}).get(m)
if raw is None:
raw = 0
continue
try:
values.append(float(raw))
except Exception:
non_numeric = True
break
except (TypeError, ValueError):
continue
raw_target = (stats_map.get(target_steam_id) or {}).get(m)
if raw_target is None:
raw_target = 0
result[m] = None
continue
try:
target_val = float(raw_target)
except Exception:
non_numeric = True
target_val = 0
if non_numeric:
except (TypeError, ValueError):
result[m] = None
continue
web/services/team_context_service.py
+36
View File
@@ -0,0 +1,36 @@
import json
from web.services.web_service import WebService
class TeamContextService:
"""Single source of truth for the private team's active roster."""
@staticmethod
def get_active_lineup():
lineup = WebService.get_active_lineup()
return dict(lineup) if lineup else None
@staticmethod
def get_active_roster_ids():
lineup = TeamContextService.get_active_lineup()
if not lineup:
return []
try:
raw_ids = json.loads(lineup.get('player_ids_json') or '[]')
except (TypeError, json.JSONDecodeError):
return []
if not isinstance(raw_ids, list):
return []
seen = set()
roster_ids = []
for raw_id in raw_ids:
steam_id = str(raw_id).strip()
if steam_id and steam_id not in seen:
seen.add(steam_id)
roster_ids.append(steam_id)
return roster_ids
web/services/web_service.py
+25 -3
View File
@@ -53,17 +53,39 @@ class WebService:
sql = "UPDATE team_lineups SET name=?, description=?, player_ids_json=? WHERE id=?"
return execute_db('web', sql, [name, description, ids_json, lineup_id])
else:
sql = "INSERT INTO team_lineups (name, description, player_ids_json) VALUES (?, ?, ?)"
return execute_db('web', sql, [name, description, ids_json])
active = 0 if WebService.get_active_lineup() else 1
sql = """
INSERT INTO team_lineups
(name, description, player_ids_json, is_active)
VALUES (?, ?, ?, ?)
"""
return execute_db('web', sql, [name, description, ids_json, active])
@staticmethod
def get_lineups():
return query_db('web', "SELECT * FROM team_lineups ORDER BY created_at DESC")
return query_db(
'web',
"SELECT * FROM team_lineups ORDER BY is_active DESC, created_at DESC, id DESC",
)
@staticmethod
def get_lineup(lineup_id):
return query_db('web', "SELECT * FROM team_lineups WHERE id = ?", [lineup_id], one=True)
@staticmethod
def get_active_lineup():
lineup = query_db(
'web',
"SELECT * FROM team_lineups WHERE is_active = 1 ORDER BY id LIMIT 1",
one=True,
)
if lineup:
return lineup
return query_db(
'web',
"SELECT * FROM team_lineups ORDER BY created_at DESC, id DESC LIMIT 1",
one=True,
)
# --- Users / Auth ---
@staticmethod