import subprocess import os import sys from web.config import Config class EtlService: SCRIPT_PATHS = { 'L1A.py': os.path.join('database', 'L1', 'L1_Builder.py'), 'L2_Builder.py': os.path.join('database', 'L2', 'L2_Builder.py'), 'L3_Builder.py': os.path.join('database', 'L3', 'L3_Builder.py'), } @staticmethod def start_pipeline(job_id, match_id=None, replace=False): script_path = os.path.join( Config.BASE_DIR, 'database', 'pipeline.py', ) command = [ sys.executable, script_path, '--job-id', str(int(job_id)), ] if match_id: command.extend(['--match-id', str(match_id)]) if replace: command.append('--replace') process = subprocess.Popen( command, cwd=Config.BASE_DIR, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, ) return process.pid @staticmethod def run_script(script_name, args=None): """ Executes an allow-listed data builder from its actual repository path. Returns (success, message) """ relative_path = EtlService.SCRIPT_PATHS.get(script_name) if not relative_path: return False, f"Unsupported data script: {script_name}" script_path = os.path.join(Config.BASE_DIR, relative_path) if not os.path.exists(script_path): return False, f"Script not found: {script_path}" try: # Use the same python interpreter python_exe = sys.executable cmd = [python_exe, script_path] if args: cmd.extend(args) result = subprocess.run( cmd, cwd=Config.BASE_DIR, capture_output=True, text=True, timeout=900 ) if result.returncode == 0: return True, f"Success:\n{result.stdout}" else: return False, f"Failed (Code {result.returncode}):\n{result.stderr}\n{result.stdout}" except Exception as e: return False, str(e)