2.0.0 Alpha: Data Refinery

database/L1/L1_Builder.py

@@ -14,13 +14,18 @@ import os
 import json
 import sqlite3
 import glob
-import argparse  # Added
+import argparse
+import sys
 
-# Paths
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-OUTPUT_ARENA_DIR = os.path.join(BASE_DIR, 'output_arena')
-DB_DIR = os.path.join(BASE_DIR, 'database', 'L1')
-DB_PATH = os.path.join(DB_DIR, 'L1.db')
+if BASE_DIR not in sys.path:
+    sys.path.insert(0, BASE_DIR)
+
+from database.paths import L1_DB, L1_DIR, OUTPUT_ARENA
+
+OUTPUT_ARENA_DIR = str(OUTPUT_ARENA)
+DB_DIR = str(L1_DIR)
+DB_PATH = str(L1_DB)
 
 def init_db():
     if not os.path.exists(DB_DIR):
@@ -65,6 +70,7 @@ def process_files():
     
     count = 0
     skipped = 0
+    errors = 0
     
     for file_path in files:
         try:
@@ -92,11 +98,14 @@ def process_files():
                 conn.commit()
                 
         except Exception as e:
+            errors += 1
             print(f"Error processing {file_path}: {e}")
             
     conn.commit()
     conn.close()
-    print(f"Finished. Processed: {count}, Skipped: {skipped}.")
+    print(f"Finished. Processed: {count}, Skipped: {skipped}, Errors: {errors}.")
+    if errors:
+        raise RuntimeError(f"L1 ingestion failed for {errors} file(s)")
 
 if __name__ == '__main__':
-    process_files()
+    process_files()