Notes

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Flask Application Factory - LLM Ozet ve Dogrulama Sistemi
UTF-8 Uyumlu + Redis Endpoint + Gelismis SRT
"""

import os
import json
import uuid
import threading
import time
import logging
from flask import Flask, render_template, request, jsonify, send_file, session
from flask_session import Session
from werkzeug.utils import secure_filename
from queue import PriorityQueue
import jinja2

logger = logging.getLogger(__name__)
if not logger.hasHandlers():
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

jobs_store = {}
user_jobs = {}
priority_queue = PriorityQueue()
worker_lock = threading.Lock()
active_workers = 0
model_cache = {}


def _export_files(text, filename, formats, config, suffix='', segments=None):
output_files = {}
base_name = os.path.splitext(filename)[0] + suffix

# GÜNCELLENDİ: Yeni LLM son eklerini kontrol et
if suffix in ['_LLM_Ozet', '_LLM_Dogrulama', '_TR']:
text_to_write = text
current_sentences = text.split('n')
else:
split_sentences_for_export = []
for sent in text.split('.'):
sent = sent.strip()
if sent: split_sentences_for_export.append(sent + '.')
text_to_write = 'n'.join(split_sentences_for_export)
current_sentences = split_sentences_for_export

try:
if 'txt' in formats:
txt_path = os.path.join(config.OUTPUT_FOLDER, f'{base_name}.txt')
with open(txt_path, 'w', encoding='utf-8') as f: f.write(text_to_write)
output_files[f'txt{suffix}'] = f'{base_name}.txt'
logger.info(f" OK TXT: {txt_path}")

if 'docx' in formats:
try:
from docx import Document
docx_path = os.path.join(config.OUTPUT_FOLDER, f'{base_name}.docx')
doc = Document()
for line in current_sentences:
if line.strip(): doc.add_paragraph(line.strip())
doc.save(docx_path)
if os.path.exists(docx_path):
output_files[f'docx{suffix}'] = f'{base_name}.docx'
logger.info(f" OK DOCX: {docx_path}")
except ImportError:
logger.error(" ERR DOCX: python-docx kutuphanesi kurulu degil. `pip install python-docx` komutu ile kurun.")
except Exception as e:
logger.error(f" ERR DOCX: {e}")

if 'json' in formats:
json_path = os.path.join(config.OUTPUT_FOLDER, f'{base_name}.json')
data_to_save = {'text': text, 'filename': filename, 'word_count': len(text.split()), 'char_count': len(text)}
# GÜNCELLENDİ: Yeni LLM son eklerini kontrol et
if suffix not in ['_LLM_Ozet', '_LLM_Dogrulama', '_TR']:
data_to_save['sentences'] = current_sentences
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(data_to_save, f, ensure_ascii=False, indent=2)
output_files[f'json{suffix}'] = f'{base_name}.json'
logger.info(f" OK JSON: {json_path}")

if 'srt' in formats:
srt_path = os.path.join(config.OUTPUT_FOLDER, f'{base_name}.srt')
with open(srt_path, 'w', encoding='utf-8') as f:
# GÜNCELLENDİ: Yeni LLM son eklerini kontrol et
if segments and suffix not in ['_LLM_Ozet', '_LLM_Dogrulama', '_TR']:
logger.info(f" SRT: Whisper zaman damgalari kullaniliyor ({len(segments)} segment)")
for i, segment in enumerate(segments, 1):
start, end, text_segment = segment['start'], segment['end'], segment['text'].strip()
start_h, start_m, start_s, start_ms = int(start // 3600), int((start % 3600) // 60), int(start % 60), int((start % 1) * 1000)
end_h, end_m, end_s, end_ms = int(end // 3600), int((end % 3600) // 60), int(end % 60), int((end % 1) * 1000)
f.write(f"{i}n{start_h:02d}:{start_m:02d}:{start_s:02d},{start_ms:03d} --> {end_h:02d}:{end_m:02d}:{end_s:02d},{end_ms:03d}n{text_segment}nn")
else:
logger.info(f" SRT: Tahmini zaman damgalari kullaniliyor ({len(current_sentences)} cumle/satir)")
duration = 3
for i, sentence in enumerate(current_sentences, 1):
sentence = sentence.strip()
if not sentence: continue
start_s, end_s = (i - 1) * duration, i * duration
start_h, start_m, start_secs = start_s // 3600, (start_s % 3600) // 60, start_s % 60
end_h, end_m, end_secs = end_s // 3600, (end_s % 3600) // 60, end_s % 60
f.write(f"{i}n{start_h:02d}:{start_m:02d}:{start_secs:02d},000 --> {end_h:02d}:{end_m:02d}:{end_secs:02d},000n{sentence}nn")
output_files[f'srt{suffix}'] = f'{base_name}.srt'
logger.info(f" OK SRT: {srt_path}")

if 'xml' in formats:
xml_path = os.path.join(config.OUTPUT_FOLDER, f'{base_name}.xml')
with open(xml_path, 'w', encoding='utf-8') as f:
f.write('<?xml version="1.0" encoding="UTF-8"?>n<transcription>n')
f.write(f' <filename>{filename}</filename>n<word_count>{len(text.split())}</word_count>n<sentence_count>{len(current_sentences)}</sentence_count>n <content>n')
from xml.sax.saxutils import escape
for i, sentence in enumerate(current_sentences, 1):
sentence = sentence.strip()
if not sentence: continue
f.write(f' <sentence id="{i}">{escape(sentence)}</sentence>n')
f.write(' </content>n</transcription>n')
output_files[f'xml{suffix}'] = f'{base_name}.xml'
logger.info(f" OK XML: {xml_path}")

except Exception as e:
logger.error(f" ERR Export: {e}", exc_info=True)

return output_files


def create_app(config):
app = Flask(__name__, template_folder='templates')
app.config.from_object(config)
app.config['SESSION_TYPE'] = 'filesystem'
app.config['SESSION_PERMANENT'] = False
app.config['SESSION_USE_SIGNER'] = True
app.config['SECRET_KEY'] = config.SECRET_KEY
Session(app)
logger.info("OK Flask creating...")
_create_folders(config)
register_routes(app, config)
_start_workers(config)
logger.info("OK Flask started!")
return app


def _create_folders(config):
folders = [config.UPLOAD_FOLDER, config.OUTPUT_FOLDER, config.SHARED_FOLDER, 'logs']
for folder in folders:
try: os.makedirs(folder, exist_ok=True)
except Exception as e: logger.error(f"ERR Creating Folder {folder}: {e}")


def _start_workers(config):
logger.info(f"nWorker starting...")

def worker_loop():
redis_service = None
try:
from app.services.redis_service import get_redis_service
redis_service = get_redis_service()
logger.info(f"Redis connection {'successful' if redis_service.is_enabled() else 'failed/disabled'}.")
except Exception as e:
logger.error(f"Redis initialization error: {e}")

llm_service_available = False
if getattr(config, 'LLM_ENABLED', False):
try:
from app.services.llm_service import get_llm_service
llm_service_instance = get_llm_service()
if llm_service_instance.enabled:
llm_service_available = True
logger.info("LLM service check successful during startup.")
else:
logger.warning("LLM service enabled in config but connection/check failed.")
except Exception as e:
logger.error(f"LLM service check error during startup: {e}")

while True:
try:
if priority_queue.empty():
time.sleep(0.5)
continue

priority, job_id = priority_queue.get(block=False)
if job_id not in jobs_store:
logger.warning(f"Job {job_id} in queue but not in store. Discarding.")
continue

job = jobs_store[job_id]
start_time = time.time()
logger.info(f"n{'='*60}nJOB STARTED: {job_id} - File: {job['filename']}n Info: Lang={job.get('language') or 'Auto'}, LLM={job.get('use_llm')}, Translate={job.get('translate_to_turkish')}n{'='*60}")
job['status'], job['progress'] = 'processing', 10

try:
# Model Yükleme
job['progress'], job['status'] = 20, 'loading_model'
model_name = config.WHISPER_MODEL
model = model_cache.get(model_name)

if not model:
try:
import whisper; import torch
device = "cuda" if torch.cuda.is_available() and getattr(config, 'USE_GPU', False) else "cpu"
logger.info(f"Job {job_id}: Loading {model_name} model onto {device}...")
model = whisper.load_model(model_name, device=device)
model_cache[model_name] = model
logger.info(f"Job {job_id}: Model {model_name} loaded.")
except Exception as load_err:
logger.error(f"Job {job_id}: Failed to load Whisper model '{model_name}': {load_err}", exc_info=True); raise
else:
logger.info(f"Job {job_id}: Using cached {model_name} model.")

# Transkripsiyon
job['progress'], job['status'] = 50, 'transcribing'
logger.info(f"Job {job_id}: Starting transcription...")
result = model.transcribe(job['filepath'], language=job.get('language') or None, verbose=False)
original_text = result.get("text", "").strip() or "(Transkripsiyon bos metin uretti)"
segments = result.get("segments")
job['text'] = original_text
logger.info(f"Job {job_id}: Transcription finished (Length: {len(original_text)} chars).")

# Export (İlk)
job['progress'], job['status'] = 70, 'exporting'
logger.info(f"Job {job_id}: Exporting original files...")
job['files'] = _export_files(original_text, job['filename'], job['formats'], config, segments=segments)
logger.info(f"Job {job_id}: Original export finished ({list(job['files'].keys())}).")

# Redis Kaydet (İlk)
if redis_service and redis_service.is_enabled():
if redis_service.save_job(job): logger.info(f"Job {job_id}: Initial state saved to Redis.")
else: logger.warning(f"Job {job_id}: Failed to save initial state to Redis.")

# Çeviri
if job.get('translate_to_turkish') and job.get('language') != 'tr':
job['status'], job['progress'] = 'translating', 80
logger.info(f"Job {job_id}: Translating ({job.get('language', 'auto')} -> tr)...")
try:
from app.services.translate_service import get_translate_service
ts = get_translate_service()
if ts.is_enabled():
translated = ts.translate_to_turkish(original_text, job.get('language'))
if translated and translated.strip():
job['translated_text'] = translated.strip()
logger.info(f"Job {job_id}: Translation successful (Length: {len(job['translated_text'])} chars). Exporting...")
tf = _export_files(job['translated_text'], job['filename'], job['formats'], config, suffix='_TR')
job['files'].update(tf); logger.info(f"Job {job_id}: Translated export finished ({list(tf.keys())}).")
if redis_service and redis_service.is_enabled(): redis_service.save_job(job)
else: logger.warning(f"Job {job_id}: Translation returned empty."); job['translation_error'] = "Ceviri servisi bos sonuc dondu."
else: logger.warning(f"Job {job_id}: Translation requested but service unavailable."); job['translation_error'] = "Ceviri servisi aktif degil."
except Exception as te: logger.error(f"Job {job_id}: Translation error: {te}", exc_info=True); job['translation_error'] = str(te)

# --- LLM ADIMI GÜNCELLENDİ (validate_and_summarize çağrısı) ---
if job.get('use_llm') and getattr(config, 'LLM_ENABLED', False):
job['status'], job['progress'] = 'llm_processing', 90
logger.info(f"Job {job_id}: LLM processing (Ozet + Dogrulama)...")
try:
from app.services.llm_service import get_llm_service
llm_service = get_llm_service()

if llm_service.enabled:
text_llm = job.get('translated_text', original_text)
lang_llm = 'tr' if job.get('translated_text') else job.get('language', 'tr')
logger.info(f"Job {job_id}: Sending text ({len(text_llm)} chars, lang: {lang_llm}) to LLM...")

# YENİ FONKSİYONU ÇAĞIR
llm_res = llm_service.validate_and_summarize(text_llm, lang_llm)

if llm_res.get('enabled') and llm_res.get('validated') is not None and llm_res.get('summary') is not None:
# YENİ ANAHTARLARI KULLAN
job['llm_validated'] = llm_res.get('validated', '').strip() # Düzeltilmiş/Doğrulanmış tam metin
job['llm_summary'] = llm_res.get('summary', '').strip() # Özet metin
logger.info(f"Job {job_id}: LLM successful (Validated: {len(job['llm_validated'])}, Summary: {len(job['llm_summary'])})")

base = os.path.splitext(job['filename'])[0]
logger.info(f"Job {job_id}: Exporting LLM files...")

llm_files_generated = {}
try: # Doğrulama Export (Düzeltilmiş tam metin)
validated_filename = f'{base}_LLM_Dogrulama.txt'
validated_path = os.path.join(config.OUTPUT_FOLDER, validated_filename)
with open(validated_path, 'w', encoding='utf-8') as f: f.write(job['llm_validated'])
llm_files_generated['llm_validated'] = validated_filename # Anahtar: llm_validated
logger.info(f" OK LLM Validation: {validated_path}")
except Exception as file_err: logger.error(f"Job {job_id}: Error writing LLM validation file: {file_err}")

try: # Özet Export
summary_filename = f'{base}_LLM_Ozet.txt'
summary_path = os.path.join(config.OUTPUT_FOLDER, summary_filename)
with open(summary_path, 'w', encoding='utf-8') as f: f.write(job['llm_summary'])
llm_files_generated['llm_summary'] = summary_filename # Anahtar: llm_summary
logger.info(f" OK LLM Summary: {summary_path}")
except Exception as file_err: logger.error(f"Job {job_id}: Error writing LLM summary file: {file_err}")

job['files'].update(llm_files_generated)
logger.info(f"Job {job_id}: LLM export finished. Updated job files dict: {list(job['files'].keys())}")

if redis_service and redis_service.is_enabled():
if redis_service.save_job(job): logger.info(f"Job {job_id}: LLM state saved to Redis.")
else: logger.warning(f"Job {job_id}: Failed to save LLM state to Redis.")
else:
err_msg = llm_res.get('summary', "LLM islemi basarisiz oldu veya eksik sonuc dondu.")
logger.warning(f"Job {job_id}: LLM processing failed. Reason: {err_msg}")
job['llm_error'] = err_msg
else:
logger.warning(f"Job {job_id}: LLM requested but service unavailable.");
job['llm_error'] = "LLM servisi aktif degil veya baslatilamadi."
except Exception as le: logger.error(f"Job {job_id}: LLM error: {le}", exc_info=True); job['llm_error'] = str(le)
# --- LLM ADIMI SONU ---

job['progress'], job['status'], job['error'] = 100, 'completed', None
job['duration'] = round(time.time() - start_time, 2)
if redis_service and redis_service.is_enabled():
redis_service.save_job(job);
redis_service.increment_stats('completed')
logger.info(f"Job {job_id}: Final completed state saved to Redis.")
logger.info(f"nJOB COMPLETED: {job_id} in {job['duration']}s")
logger.info(f" Original Text Sample: {original_text[:100]}...")
if job.get('translated_text'): logger.info(f" Translated Text Sample: {job['translated_text'][:100]}...")
if job.get('llm_validated'): logger.info(f" LLM Validation Sample: {job['llm_validated'][:100]}...")
if job.get('llm_summary'): logger.info(f" LLM Summary Sample: {job['llm_summary'][:100]}...")
logger.info(f" Final Output Files: {list(job['files'].keys())}")
logger.info(f"{'='*60}n")
except Exception as e:
et = time.time(); job['status'], job['error'], job['progress'] = 'error', str(e), job.get('progress', 10)
job['duration'] = round(et - start_time, 2)
logger.error(f"nJOB ERROR: {job_id} failed at status '{job['status']}' after {job['duration']}sn Error: {str(e)}n{'='*60}n", exc_info=True)
if redis_service and redis_service.is_enabled(): redis_service.save_job(job); redis_service.increment_stats('failed')
except Exception as e:
logger.error(f"Critical error in worker loop: {e}", exc_info=True); time.sleep(5)
worker_thread = threading.Thread(target=worker_loop, daemon=True, name="Worker")
worker_thread.start()
logger.info(f"Worker thread started successfully.")

def register_routes(app, config):

@app.route('/')
def index():
try: return render_template('index.html')
except jinja2.exceptions.TemplateNotFound: logger.error(f"Template 'index.html' not found in folder: {app.template_folder}"); return "Template not found.", 404
except Exception as e:
logger.error(f"Error rendering index.html: {e}", exc_info=True)
if isinstance(e, UnicodeDecodeError): logger.error(">>> HINT: Ensure 'app/templates/index.html' is saved as UTF-8 without BOM."); return "Internal server error related to template encoding. Check logs.", 500
return "Internal server error.", 500

@app.route('/api/system-info')
def system_info():
try:
if 'user_id' not in session: session['user_id'] = str(uuid.uuid4())
user_id = session['user_id']
user_job_ids = user_jobs.get(user_id, [])
user_job_list = [job for jid in user_job_ids if (job := jobs_store.get(jid))]
import torch; import psutil
gpu_available, device_name, gpu_memory = False, 'CPU', "N/A"
try:
use_gpu_config = getattr(config, 'USE_GPU', False)
if torch.cuda.is_available() and use_gpu_config:
gpu_available=True; device_name=str(torch.cuda.get_device_name(0)); gpu_memory=f"{torch.cuda.get_device_properties(0).total_memory/1e9:.2f}GB"
except Exception as gpu_err: logger.warning(f"Could not get GPU info: {gpu_err}")
info = { 'gpu_available': gpu_available, 'device': device_name, 'gpu_memory': gpu_memory,
'cpu_percent': psutil.cpu_percent(interval=0.1), 'ram_percent': psutil.virtual_memory().percent,
'active_jobs': len([j for j in user_job_list if j.get('status') == 'processing']),
'queued_jobs': len([j for j in user_job_list if j.get('status') == 'queued']),
'completed_jobs': len([j for j in user_job_list if j.get('status') == 'completed']) }
return jsonify(info)
except Exception as e: logger.error(f"Error getting system info: {e}", exc_info=True); return jsonify({'error': "Sistem bilgisi alinirken bir hata olustu."}), 500

@app.route('/api/redis-status')
def redis_status():
try:
from app.services.redis_service import get_redis_service
redis_service = get_redis_service()
if not redis_service.is_enabled(): return jsonify({'enabled': False, 'message': 'Redis aktif degil'})
all_keys = redis_service.get_all_keys("transcription:*")
job_keys = [k for k in all_keys if ':job:' in k]; user_keys = [k for k in all_keys if ':user:' in k]; result_keys = [k for k in all_keys if ':result:' in k]
stats = redis_service.get_stats(7); key_types = {}
for key in all_keys: parts = key.split(':');
if len(parts) > 1: key_type=parts[1]; key_types[key_type] = key_types.get(key_type, 0) + 1
return jsonify({'enabled': True, 'total_keys': len(all_keys), 'job_count': len(job_keys),
'user_count': len(user_keys), 'result_count': len(result_keys),
'key_types': key_types, 'stats': stats})
except Exception as e: logger.error(f"Error getting redis status: {e}", exc_info=True); return jsonify({'error': "Redis durumu alinirken bir hata olustu.", 'enabled': False}), 500

@app.route('/api/jobs')
def get_jobs():
try:
if 'user_id' not in session: session['user_id'] = str(uuid.uuid4())
user_id = session['user_id']
user_job_ids = user_jobs.get(user_id, [])
jobs_list = [job for jid in user_job_ids if (job := jobs_store.get(jid))]
jobs_list.sort(key=lambda x: x.get('id', ''), reverse=True)
return jsonify(jobs_list)
except Exception as e: logger.error(f"Error getting jobs for user {session.get('user_id', 'unknown')}: {e}", exc_info=True); return jsonify([])

@app.route('/api/job/<job_id>')
def get_job(job_id):
try:
job = jobs_store.get(job_id)
if not job: return jsonify({'error': 'Is bulunamadi'}), 404
return jsonify(job)
except Exception as e: logger.error(f"Error getting job details for {job_id}: {e}", exc_info=True); return jsonify({'error': "Is detaylari alinirken bir hata olustu."}), 500

@app.route('/upload', methods=['POST'])
def upload_file():
try:
if 'user_id' not in session: session['user_id'] = str(uuid.uuid4())
user_id = session['user_id']
file = request.files.get('audioFile')
if not file or file.filename == '': return jsonify({'error': 'Dosya secilmedi.'}), 400
language = request.form.get('language');
if language == 'None' or language == '': language = None
formats_str = request.form.get('formats', '["txt"]')
email = request.form.get('email', '')
priority = int(request.form.get('priority', 5))
use_llm = request.form.get('useLLM', 'false').lower() == 'true'
translate_to_turkish = request.form.get('translateToTurkish', 'false').lower() == 'true'
try:
formats = json.loads(formats_str)
if not isinstance(formats, list) or not formats: formats = ['txt']
except json.JSONDecodeError: logger.warning(f"Invalid formats string: {formats_str}"); formats = ['txt', 'docx']
filename = secure_filename(file.filename); filename_base, file_ext = os.path.splitext(filename)
unique_suffix = uuid.uuid4().hex[:6]; unique_filename = f"{filename_base}_{unique_suffix}{file_ext}"
filepath = os.path.join(config.UPLOAD_FOLDER, unique_filename)
file.save(filepath); logger.info(f"User {user_id} uploaded '{filename}', saved as '{unique_filename}'")
job_id = str(uuid.uuid4())
job = {'id': job_id, 'user_id': user_id, 'filename': filename, 'filepath': filepath, 'language': language,
'formats': formats, 'email': email, 'priority': priority, 'use_llm': use_llm,
'translate_to_turkish': translate_to_turkish, 'status': 'queued', 'progress': 0, 'error': None,
'text': None, 'translated_text': None,
'llm_validated': None, 'llm_summary': None, # GÜNCELLENDİ: Anahtarlar
'files': {}, 'duration': 0}
jobs_store[job_id] = job
if user_id not in user_jobs: user_jobs[user_id] = []
user_jobs[user_id].append(job_id)
priority_queue.put((priority, job_id))
logger.info(f"Job {job_id} created and queued for user {user_id}.")
return jsonify({'job_id': job_id, 'filename': filename})
except Exception as e: logger.error(f"Upload error: {e}", exc_info=True); return jsonify({'error': "Dosya yuklenirken bir sunucu hatasi olustu."}), 500

@app.route('/download/<filename>')
def download_file(filename):
try:
safe_filename = secure_filename(filename)
if safe_filename != filename: logger.warning(f"Invalid download filename: '{filename}' -> '{safe_filename}'"); return jsonify({'error': 'Gecersiz dosya adi.'}), 400
safe_path = os.path.abspath(os.path.join(config.OUTPUT_FOLDER, safe_filename))
if not safe_path.startswith(os.path.abspath(config.OUTPUT_FOLDER)): logger.error(f"Security Alert: Path traversal attempt: {filename}"); return jsonify({'error': 'Yetkisiz erisim.'}), 403
if not os.path.exists(safe_path) or not os.path.isfile(safe_path): logger.warning(f"Download request non-existent file: {filename}"); return jsonify({'error': 'Dosya bulunamadi.'}), 404
logger.info(f"Serving download for: {safe_filename}")
return send_file(safe_path, as_attachment=True)
except Exception as e: logger.error(f"Error serving download for {filename}: {e}", exc_info=True); return jsonify({'error': "Dosya indirilirken bir hata olustu."}), 500

__all__ = ['create_app', 'jobs_store', 'priority_queue', 'worker_lock', 'model_cache']