Notes

import os
import logging
from fastapi import FastAPI, UploadFile, File, Form, HTTPException
from fastapi.responses import JSONResponse

# --- 1. Import all refactored functions ---
from fetch_raw_ddl import run_fetch_raw_ddl
from table_column_description import run_generate_tcd
from ddl_generator import run_generate_final_ddl
from ddl_formatter import run_format_final_ddl
from generator import run_generate_documentation
from validation import run_validation

# --- 2. Configuration and Setup ---
OUTPUT_DIR = 'output'
INPUT_DIR = 'input'
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs(INPUT_DIR, exist_ok=True)
LOG_FILE = os.path.join(OUTPUT_DIR, 'app.log')

logging.basicConfig(filename=LOG_FILE, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filemode='w')

app = FastAPI(
title="Data Pipeline Automation",
description="Single endpoint to run the full DDL and documentation generation flow."
)

@app.post("/run-pipeline/")
async def run_data_pipeline_endpoint(
include_file: UploadFile = File(..., description="File containing table names (one per line)."),
config_section: str = Form("r360", description="The configuration section from config.ini to use (e.g., 'r360', 'sales').")
):
"""
Runs the entire data generation process end-to-end, dynamically naming output files.
"""
logging.info(f"Pipeline START: Section '{config_section}'. File: '{include_file.filename}'.")

# 1. Define Dynamic File Naming (e.g., 'r360', 'sales')
file_suffix = config_section.lower()

# Define all file paths dynamically based on the suffix
RAW_DDL_PATH = os.path.join(OUTPUT_DIR, f'raw_ddl_{file_suffix}.yaml')
TCD_PATH = os.path.join(OUTPUT_DIR, f'table_column_description_{file_suffix}.yaml')
FINAL_DDL_PATH = os.path.join(OUTPUT_DIR, f'{file_suffix.upper()}_ddl.yaml')
FORMATTED_DDL_PATH = os.path.join(OUTPUT_DIR, f'reformatted_ddls_{file_suffix}.yaml')
DOC_PATH = os.path.join(OUTPUT_DIR, f'documentation_{file_suffix}.yaml')

# 2. Save the dynamic input file
input_file_path = os.path.join(INPUT_DIR, include_file.filename)
try:
with open(input_file_path, "wb") as f:
f.write(await include_file.read())
logging.info(f"Input file saved to: {input_file_path}")
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to save uploaded file: {e}")

# --- 3. PIPELINE ORCHESTRATION ---
try:
# Step 1: Generate Raw DDLs (Output: RAW_DDL_PATH)
run_fetch_raw_ddl(input_file_path, config_section, RAW_DDL_PATH)

# Step 2: Generate TCD (Input: RAW_DDL_PATH, Output: TCD_PATH)
run_generate_tcd(RAW_DDL_PATH, TCD_PATH)

# Step 3: Generate Final DDL (Input: TCD_PATH, Output: FINAL_DDL_PATH)
run_generate_final_ddl(TCD_PATH, config_section, FINAL_DDL_PATH)

# Step 4: Format DDL (Input: FINAL_DDL_PATH, Output: FORMATTED_DDL_PATH)
run_format_final_ddl(FINAL_DDL_PATH, FORMATTED_DDL_PATH)

# Step 5: Generate Documentation (Input: TCD_PATH, Output: DOC_PATH)
run_generate_documentation(TCD_PATH, config_section, DOC_PATH)

# Step 6: RUN VALIDATION (Task 4)
# Validation needs the suffix to find the dynamically named files
validation_results = run_validation(OUTPUT_DIR, file_suffix)

logging.info("Pipeline SUCCESS.")

except Exception as e:
# Report error to user
raise HTTPException(
status_code=500,
detail={"message": "Pipeline execution failed.", "error": str(e)}
)
finally:
# Clean up the temporary input file
if os.path.exists(input_file_path):
os.remove(input_file_path)

# 4. Return Success Response
return JSONResponse(
status_code=200,
content={
"message": "Data generation pipeline completed successfully.",
"config_used": config_section,
"output_folder": f"Files are in the '{OUTPUT_DIR}' folder, named with suffix '_{file_suffix}'.",
"validation_status": validation_results
}
)