2.0.0 Alpha: Data Refinery

web/services/etl_service.py

@@ -4,13 +4,51 @@ import sys
 from web.config import Config
 
 class EtlService:
+    SCRIPT_PATHS = {
+        'L1A.py': os.path.join('database', 'L1', 'L1_Builder.py'),
+        'L2_Builder.py': os.path.join('database', 'L2', 'L2_Builder.py'),
+        'L3_Builder.py': os.path.join('database', 'L3', 'L3_Builder.py'),
+    }
+
+    @staticmethod
+    def start_pipeline(job_id, match_id=None, replace=False):
+        script_path = os.path.join(
+            Config.BASE_DIR,
+            'database',
+            'pipeline.py',
+        )
+        command = [
+            sys.executable,
+            script_path,
+            '--job-id',
+            str(int(job_id)),
+        ]
+        if match_id:
+            command.extend(['--match-id', str(match_id)])
+        if replace:
+            command.append('--replace')
+
+        process = subprocess.Popen(
+            command,
+            cwd=Config.BASE_DIR,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            start_new_session=True,
+        )
+        return process.pid
+
     @staticmethod
     def run_script(script_name, args=None):
         """
-        Executes an ETL script located in the ETL directory.
+        Executes an allow-listed data builder from its actual repository path.
         Returns (success, message)
         """
-        script_path = os.path.join(Config.BASE_DIR, 'ETL', script_name)
+        relative_path = EtlService.SCRIPT_PATHS.get(script_name)
+        if not relative_path:
+            return False, f"Unsupported data script: {script_name}"
+
+        script_path = os.path.join(Config.BASE_DIR, relative_path)
         
         if not os.path.exists(script_path):
             return False, f"Script not found: {script_path}"
@@ -28,7 +66,7 @@ class EtlService:
                 cwd=Config.BASE_DIR,
                 capture_output=True,
                 text=True,
-                timeout=300 # 5 min timeout
+                timeout=900
             )
             
             if result.returncode == 0: