"""
Question-Answer (Q&A) related routes
Similar structure to quiz_routes.py but for open-ended Q&A system
"""
import json
import uuid
import logging
import mysql.connector
import asyncio
import time
import sys
import os
from datetime import datetime
from flask import Blueprint, request, jsonify
from utils import (require_api_key, encrypt_response, format_api_response, 
                   create_error_response, get_db_connection)

# Add parent directory to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Import AI generation capabilities for Q&A evaluation
from services.ai_provider import ai_generate_async
from services.ai_config import is_groq_enabled

try:
    import g4f
    g4f_available = True
except ImportError:
    g4f_available = False

ai_available = g4f_available or is_groq_enabled()

# Import dynamic AI model configuration
from aimodel_selector.models_config import get_default_model, get_model_config, get_all_models

logger = logging.getLogger(__name__)
qa_bp = Blueprint('qa', __name__)

def safe_log_unicode(text, max_length=800):
    """Safely log Unicode text without encoding errors"""
    try:
        if isinstance(text, str):
            return text[:max_length].encode('utf-8', errors='replace').decode('utf-8')
        return str(text)[:max_length]
    except Exception:
        return f"[Logging Error: Could not safely encode text of length {len(str(text)) if text else 0}]"

import re

def extract_content_from_response(response):
    """
    Extract the actual text content from AI response.
    g4f returns different formats depending on provider:
    - String: direct text content
    - Dict: {'choices': [{'message': {'content': '...'}}], ...}
    - String repr of dict: "{'choices': [...]}"
    """
    if not response:
        return ""
    
    # Log response type for debugging
    logger.debug(f"Response type: {type(response).__name__}")
    
    # If response is a dict (common g4f format) - CHECK THIS FIRST
    if isinstance(response, dict):
        logger.debug(f"Response is dict, keys: {list(response.keys())[:5]}")
        # Format: {'choices': [{'message': {'content': '...'}}]}
        if 'choices' in response:
            choices = response.get('choices', [])
            logger.debug(f"Choices type: {type(choices).__name__}, length: {len(choices) if choices else 0}")
            if choices and len(choices) > 0:
                first_choice = choices[0]
                logger.debug(f"First choice type: {type(first_choice).__name__}")
                if isinstance(first_choice, dict):
                    message = first_choice.get('message', {})
                    logger.debug(f"Message type: {type(message).__name__}")
                    if isinstance(message, dict):
                        content = message.get('content', '')
                        logger.debug(f"Content extracted, length: {len(content) if content else 0}")
                        if content:
                            return content
        # Fallback format: {'content': '...'} or {'message': {'content': '...'}}
        if 'content' in response:
            return response['content']
        if 'message' in response and isinstance(response['message'], dict):
            return response['message'].get('content', '')
        # If dict but no content found, log and fall through
        logger.warning(f"Dict response but no content found. Keys: {list(response.keys())}")
    
    # If it's a string
    if isinstance(response, str):
        stripped = response.strip()
        
        # Check if it's a string representation of a dict (starts with { and contains 'choices')
        if stripped.startswith('{') and 'choices' in stripped:
            try:
                import ast
                # Try ast.literal_eval for Python dict format
                parsed = ast.literal_eval(stripped)
                return extract_content_from_response(parsed)  # Recursively extract
            except:
                try:
                    # Try json.loads for JSON format
                    parsed = json.loads(stripped)
                    return extract_content_from_response(parsed)
                except:
                    pass
        
        # It's a regular string content, return as is
        return stripped
    
    # If response is a list (some providers)
    if isinstance(response, list) and len(response) > 0:
        first_item = response[0]
        if isinstance(first_item, dict):
            return first_item.get('message', {}).get('content', '')
    
    # Last resort: convert to string
    logger.warning(f"Falling back to str(response) for type: {type(response).__name__}")
    return str(response)

def _fix_invalid_json_escapes(text):
    """
    Fix invalid JSON escape sequences such as LaTeX \\m, \\a, \\( etc.
    Valid JSON escapes after backslash: " \\ / b f n r t u
    """
    VALID = {'"', '\\', '/', 'b', 'f', 'n', 'r', 't', 'u'}
    def _fix(m):
        s = m.group(0)
        return s if s[1] in VALID else '\\\\' + s[1]
    return re.sub(r'\\(.)', _fix, text, flags=re.DOTALL)


def parse_ai_json(text):
    """
    Parse AI JSON response with multi-stage error recovery.
    1. Try direct parse (handles well-formed JSON from Groq etc.)
    2. Fix invalid JSON escape sequences (LaTeX math like \\mr^2)
    3. Fallback: convert literal \\n sequences (legacy g4f models)
    """
    if not text:
        raise ValueError("Empty text")

    text = text.strip()

    # Remove markdown code blocks if present
    if text.startswith('```'):
        text = re.sub(r'^```(?:json)?\s*', '', text)
        text = re.sub(r'\s*```$', '', text)
        text = text.strip()

    # Extract JSON array if surrounded by extra text
    start_idx = text.find('[')
    end_idx = text.rfind(']') + 1
    if start_idx != -1 and end_idx > 0:
        text = text[start_idx:end_idx]

    # Stage 1: direct parse
    try:
        return json.loads(text)
    except json.JSONDecodeError:
        pass

    # Stage 2: fix invalid escape sequences (e.g. LaTeX \m inside JSON strings)
    try:
        return json.loads(_fix_invalid_json_escapes(text))
    except json.JSONDecodeError:
        pass

    # Stage 3: legacy fallback — some older models output literal \n as 2 chars
    text = text.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r')
    return json.loads(text)

def get_available_ai_models():
    """Get list of available AI models for frontend - filtered by active backend"""
    try:
        models = get_all_models()
        try:
            from services.ai_config import is_groq_enabled, is_g4f_enabled
            if is_groq_enabled():
                models = [m for m in models if m.get('backend_type') == 'groq']
            elif is_g4f_enabled():
                models = [m for m in models if m.get('backend_type') == 'g4f']
        except ImportError:
            pass
        return [{'key': model['name'], 'name': model['description']} for model in models]
    except Exception as e:
        logger.error(f"Error fetching models: {e}")
        # Return empty list - let frontend handle no models gracefully
        return []

def get_dynamic_model_config(model_name=None, user_id=None):
    """Get model configuration dynamically from database"""
    try:
        if not model_name:
            model_name = get_default_model(user_id=user_id)
        config = get_model_config(model_name)
        return {
            'model': config.get('model', 'geminisearch'),
            'name': config.get('description', 'AI Model'),
            'provider': config.get('provider', 'PollinationsAI'),
            'timeout': 60
        }
    except Exception as e:
        logger.error(f"Error getting model config: {e}")
        # Fallback
        return {
            'model': 'geminisearch',
            'name': 'Gemini Search',
            'provider': 'PollinationsAI',
            'timeout': 60
        }

async def generate_ai_content_with_retry(model_key, messages, max_retries=3, user_id=None):
    """Generate AI content with retry mechanism for Q&A evaluation - now with dynamic models"""
    try:
        model_config = get_dynamic_model_config(model_key, user_id=user_id)
    except Exception:
        # Use default if specific model not found
        model_config = get_dynamic_model_config(user_id=user_id)
    
    for attempt in range(max_retries):
        try:
            logger.info(f"AI evaluation attempt {attempt + 1}/{max_retries} with model {model_key}")
            
            create_kwargs = {
                'model': model_config['model'],
                'messages': messages,
            }

            result = await ai_generate_async(
                messages=messages,
                model=model_config['model'],
            )
            response = result.content
            
            if not response:
                raise Exception("Empty response from AI")
            
            # Extract actual content from response (handles dict/string formats)
            content = response
            
            if content and content.strip():
                logger.info(f"Successfully generated AI content with {model_key} on attempt {attempt + 1}")
                return content
            else:
                raise Exception("Empty content extracted from AI response")
                
        except Exception as e:
            logger.warning(f"AI evaluation attempt {attempt + 1} failed with {model_key}: {e}")
            
            if attempt < max_retries - 1:
                wait_time = 2 ** attempt
                logger.info(f"Waiting {wait_time} seconds before retry...")
                await asyncio.sleep(wait_time)
            else:
                # Try to get default model as fallback
                try:
                    default_model = get_default_model()
                    if model_key != default_model:
                        logger.info(f"Falling back to default model '{default_model}' after {model_key} failed")
                        return await generate_ai_content_with_retry(default_model, messages, max_retries=2)
                except Exception:
                    pass
                raise Exception(f"All attempts failed for model {model_key}: {str(e)}")
    
    raise Exception(f"Failed to generate AI evaluation after {max_retries} attempts")

async def generate_and_parse_ai_qa_with_retry(model_key, messages, max_retries=5, user_id=None):
    """Generate AI Q&A content and parse JSON with retry mechanism and robust cleaning"""
    try:
        model_config = get_dynamic_model_config(model_key, user_id=user_id)
    except Exception:
        model_config = get_dynamic_model_config(user_id=user_id)
    
    for attempt in range(max_retries):
        try:
            logger.info(f"AI Q&A generation attempt {attempt + 1}/{max_retries} with model {model_key}")
            
            create_kwargs = {
                'model': model_config['model'],
                'messages': messages,
            }
                
            result = await ai_generate_async(
                messages=messages,
                model=model_config['model'],
            )
            response = result.content
            
            if not response:
                raise Exception("Empty response from AI")
            
            # 1. Extract actual content (handles raw API response objects)
            content = response
            if not content:
                raise Exception("Could not extract content from AI response")
            
            logger.debug(f"Extracted content length: {len(content)} chars")
            
            # 2. Parse directly - AI should output valid JSON
            try:
                ai_questions = parse_ai_json(content)
            except json.JSONDecodeError as e:
                logger.error(f"JSON Parse Error: {e.msg} at position {e.pos}")
                logger.debug(f"Content start: {content[:300]}")
                raise e

            # Validation
            if not isinstance(ai_questions, list) or len(ai_questions) == 0:
                raise ValueError("Parsed JSON is not a list or is empty")
            
            if not ai_questions[0].get('question_text'):
                raise ValueError("JSON structure incorrect: missing 'question_text'")

            logger.info(f"Successfully generated and parsed {len(ai_questions)} questions with {model_key}")
            return ai_questions

        except (json.JSONDecodeError, ValueError) as e:
            logger.warning(f"JSON parsing failed on attempt {attempt + 1}: {str(e)[:100]}")
            
            if attempt < max_retries - 1:
                wait_time = 2 ** attempt
                await asyncio.sleep(wait_time)
            else:
                # Fallback logic
                try:
                    default_model = get_default_model()
                    if model_key != default_model:
                        logger.info(f"Falling back to default model '{default_model}'")
                        return await generate_and_parse_ai_qa_with_retry(default_model, messages, max_retries=2, user_id=user_id)
                except Exception as e:
                    logger.warning(f"Error validating AI model '{default_model}': {e}, using default")
                    return await generate_and_parse_ai_qa_with_retry(get_default_model(user_id=user_id), messages, user_id=user_id, max_retries=2)
                raise Exception(f"Final JSON parsing failure: {str(e)}")
        
        except Exception as e:
            # Handle network/timeout errors
            logger.warning(f"AI generation error on attempt {attempt + 1}: {e}")
            if attempt < max_retries - 1:
                await asyncio.sleep(2 ** attempt)
            else:
                 raise Exception(f"All attempts failed: {str(e)}")
                 
    raise Exception("Max retries reached")

@qa_bp.route('/api/ai-models', methods=['GET'])
@require_api_key
@encrypt_response
def get_ai_models():
    """Get available AI models for Q&A generation and evaluation"""
    try:
        if not ai_available:
            return create_error_response("AI service is not available", 503)
        
        models = get_available_ai_models()
        return format_api_response({'models': models}, include_legacy_fields={'models': models})
    except Exception as e:
        logger.error(f"Error getting AI models: {str(e)}")
        return create_error_response(f"Failed to get AI models: {str(e)}")

@qa_bp.route('/api/random-qa', methods=['POST'])
@require_api_key
@encrypt_response
def generate_random_qa():
    """Generate a random Q&A session from selected topics"""
    conn = None
    cursor = None
    try:
        data = request.get_json() or {}
        user_id = data.get('user_id')
        topic_ids = data.get('topic_ids', [])
        question_count = min(int(data.get('question_count', 5)), 20)  # Max 20 Q&A questions
        difficulty_level = data.get('difficulty_level', 'all')
        use_user_questions = data.get('use_user_questions', False)
        ai_only = data.get('ai_only', False)
        
        if not user_id:
            return create_error_response("User ID is required", 400)
        
        if not topic_ids:
            return create_error_response("At least one topic must be selected", 400)
        
        from database import get_cached_connection
        conn = get_cached_connection()
        if not conn:
            return create_error_response("Database connection failed", 500)
        
        cursor = conn.cursor(dictionary=True, buffered=True)
        cursor.execute("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci")
        
        # Verify user has access to these topics
        placeholders = ','.join(['%s'] * len(topic_ids))
        cursor.execute(f"""
            SELECT t.id, t.title, t.full_path, c.title as course_title
            FROM topics t
            JOIN courses c ON t.course_id = c.id
            JOIN enrollments e ON c.id = e.course_id AND e.user_id = %s AND e.payment_status IN ('free', 'paid')
            WHERE t.id IN ({placeholders})
        """, [user_id] + topic_ids)
        
        accessible_topics = cursor.fetchall()
        accessible_topic_ids = [t['id'] for t in accessible_topics]
        
        if not accessible_topics:
            return create_error_response("No accessible topics found", 403)
        
        # Build filters
        difficulty_filter = ""
        difficulty_params = []
        if difficulty_level != 'all':
            difficulty_filter = "AND q.difficulty = %s"
            difficulty_params = [difficulty_level]
        
        user_filter = ""
        user_params = []
        if ai_only:
            user_filter = "AND q.user_id = %s AND q.is_ai_generated = 1"
            user_params = [user_id]
        elif use_user_questions:
            user_filter = "AND (q.user_id = %s OR q.user_id IS NULL)"
            user_params = [user_id]
        else:
            user_filter = "AND q.user_id IS NULL"
        
        # Get random Q&A questions
        placeholders = ','.join(['%s'] * len(accessible_topic_ids))
        cursor.execute(f"""
            SELECT 
                q.id as question_id,
                q.question_text,
                q.expected_answer,
                q.answer_keywords,
                q.difficulty,
                q.explanation,
                q.marks,
                q.time_limit,
                q.is_ai_generated,
                t.title as topic_title,
                t.id as topic_id,
                t.full_path,
                c.title as course_title
            FROM qa_questions q
            JOIN topics t ON q.topic_id = t.id
            JOIN courses c ON t.course_id = c.id
            WHERE q.topic_id IN ({placeholders}) {difficulty_filter} {user_filter}
            ORDER BY RAND()
            LIMIT %s
        """, accessible_topic_ids + difficulty_params + user_params + [question_count])
        
        questions = cursor.fetchall()
        
        if not questions:
            return create_error_response("No Q&A questions found for selected topics", 404)
        
        # Format questions for frontend
        formatted_questions = []
        for q in questions:
            # Parse keywords
            keywords = []
            if q['answer_keywords']:
                try:
                    keywords = json.loads(q['answer_keywords'])
                except (json.JSONDecodeError, TypeError):
                    keywords = []
            
            formatted_questions.append({
                'question_id': str(q['question_id']),
                'question_text': q['question_text'],
                'expected_answer': q.get('expected_answer', ''),
                'answer_keywords': keywords,
                'difficulty': q['difficulty'],
                'explanation': q.get('explanation', ''),
                'topic_title': q['topic_title'],
                'topic_id': q['topic_id'],
                'topic_full_path': q.get('full_path'),
                'course_title': q['course_title'],
                'marks': q.get('marks', 1),
                'time_limit': q.get('time_limit', 300)
            })
        
        # Generate Q&A session metadata
        qa_data = {
            'qa_id': str(uuid.uuid4()),
            'title': f"Random Q&A - {len(accessible_topics)} Topics",
            'description': f"Practice Q&A with {len(formatted_questions)} questions from selected topics",
            'questions': formatted_questions,
            'total_questions': len(formatted_questions),
            'total_time_limit': sum(q.get('time_limit', 300) for q in questions),
            'difficulty_level': difficulty_level,
            'topics_covered': accessible_topic_ids,
            'created_at': datetime.now().isoformat()
        }
        
        return format_api_response(qa_data, include_legacy_fields=qa_data)
            
    except Exception as e:
        logger.error(f"Error in generate_random_qa: {str(e)}")
        return create_error_response(f"Failed to generate Q&A session: {str(e)}")
    finally:
        if cursor:
            cursor.close()
        if conn:
            try:
                conn.close()
            except Exception:
                pass

@qa_bp.route('/api/random-qa/submit', methods=['POST'])
@require_api_key
@encrypt_response
def submit_random_qa():
    """Submit Q&A answers and get AI evaluation"""
    conn = None
    cursor = None
    try:
        data = request.get_json() or {}
        user_id = data.get('user_id')
        qa_id = data.get('qa_id')
        answers = data.get('answers', {})  # {question_id: user_answer}
        topic_ids = data.get('topic_ids', [])
        total_questions = data.get('total_questions', len(answers))
        qa_question_ids = data.get('qa_question_ids', [])
        auto_evaluate = data.get('auto_evaluate', True)
        ai_model = data.get('ai_model', get_default_model(user_id=user_id))  # Use user's preferred model or default
        
        if not user_id or not qa_id:
            return create_error_response("Missing required fields (user_id and qa_id)", 400)
        
        # answers can be empty if user didn't answer any questions
        if answers is None:
            answers = {}
        
        logger.info(f"Q&A submission - User: {user_id}, Q&A: {qa_id}, Answers: {len(answers)}, Model: {ai_model}")
        
        from database import get_cached_connection
        conn = get_cached_connection()
        if not conn:
            return create_error_response("Database connection failed", 500)
        
        cursor = conn.cursor(dictionary=True, buffered=True)
        cursor.execute("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci")
        
        # Get questions for verification
        if qa_question_ids:
            placeholders = ','.join(['%s'] * len(qa_question_ids))
            cursor.execute(f"""
                SELECT 
                    q.id as question_id,
                    q.expected_answer,
                    q.answer_keywords,
                    q.difficulty,
                    q.marks,
                    t.id as topic_id
                FROM qa_questions q
                JOIN topics t ON q.topic_id = t.id
                WHERE q.id IN ({placeholders})
            """, qa_question_ids)
        else:
            question_ids = list(answers.keys())
            if question_ids:
                placeholders = ','.join(['%s'] * len(question_ids))
                cursor.execute(f"""
                    SELECT 
                        q.id as question_id,
                        q.expected_answer,
                        q.answer_keywords,
                        q.difficulty,
                        q.marks,
                        t.id as topic_id
                    FROM qa_questions q
                    JOIN topics t ON q.topic_id = t.id
                    WHERE q.id IN ({placeholders})
                """, question_ids)
            else:
                return create_error_response("No questions to evaluate", 400)
        
        questions = cursor.fetchall()
        questions_dict = {str(q['question_id']): q for q in questions}
        
        # Create Q&A session
        cursor.execute("""
            INSERT INTO qa_sessions (
                user_id, qa_type, qa_id, topic_ids, total_questions, 
                answered_questions, total_marks_possible, completed_at, 
                evaluation_status
            ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
        """, (
            user_id, 'random', qa_id, json.dumps(topic_ids),
            total_questions, len(answers),
            sum(q.get('marks', 1) for q in questions),
            datetime.now(), 'pending'
        ))
        
        session_id = cursor.lastrowid
        
        # Store answers and prepare for AI evaluation
        total_ai_score = 0
        total_possible_score = 0
        responses_data = []
        
        for question_id, user_answer in answers.items():
            if question_id in questions_dict:
                question = questions_dict[question_id]
                marks = question.get('marks', 1)
                total_possible_score += marks
                
                # Store the response
                cursor.execute("""
                    INSERT INTO qa_responses (
                        session_id, question_id, user_answer, topic_id, time_taken
                    ) VALUES (%s, %s, %s, %s, %s)
                """, (
                    session_id, int(question_id), user_answer,
                    question['topic_id'], 0  # Time tracking can be added later
                ))
                
                response_id = cursor.lastrowid
                responses_data.append({
                    'response_id': response_id,
                    'question_id': question_id,
                    'user_answer': user_answer,
                    'expected_answer': question.get('expected_answer', ''),
                    'answer_keywords': question.get('answer_keywords'),
                    'marks': marks
                })
        
        # AI Evaluation (if enabled and available)
        if auto_evaluate and ai_available:
            try:
                logger.info(f"Starting AI evaluation with model: {ai_model}")
                for response_data in responses_data:
                    ai_score = evaluate_answer_with_ai(
                        response_data['user_answer'],
                        response_data['expected_answer'],
                        response_data['answer_keywords'],
                        response_data['marks'],
                        model_key=ai_model,  # Pass user's preferred model
                        user_id=user_id
                    )
                    
                    total_ai_score += ai_score
                    
                    # Update response with AI evaluation
                    cursor.execute("""
                        UPDATE qa_responses 
                        SET ai_score = %s, final_score = %s 
                        WHERE id = %s
                    """, (ai_score, ai_score, response_data['response_id']))
                
                # Update session with scores
                score_percentage = (total_ai_score / total_possible_score * 100) if total_possible_score > 0 else 0
                cursor.execute("""
                    UPDATE qa_sessions 
                    SET marks_obtained = %s, score_percentage = %s, evaluation_status = %s
                    WHERE id = %s
                """, (total_ai_score, score_percentage, 'ai_evaluated', session_id))
                
            except Exception as ai_error:
                logger.error(f"AI evaluation failed: {str(ai_error)}")
                # Mark as pending for manual evaluation
                cursor.execute("""
                    UPDATE qa_sessions SET evaluation_status = %s WHERE id = %s
                """, ('pending', session_id))
        
        conn.commit()
        
        result = {
            'qa_id': qa_id,
            'session_id': session_id,
            'total_questions': total_questions,
            'answered_questions': len(answers),
            'total_possible_score': total_possible_score,
            'ai_evaluated': auto_evaluate and ai_available,
            'evaluation_status': 'ai_evaluated' if (auto_evaluate and ai_available) else 'pending',
            'completed_at': datetime.now().isoformat()
        }
        
        if auto_evaluate and ai_available:
            result.update({
                'ai_score': total_ai_score,
                'score_percentage': (total_ai_score / total_possible_score * 100) if total_possible_score > 0 else 0
            })
        
        return format_api_response(result, include_legacy_fields=result)
        
    except Exception as e:
        logger.error(f"Error in submit_random_qa: {str(e)}")
        return create_error_response(f"Failed to submit Q&A: {str(e)}")
    finally:
        if cursor:
            cursor.close()
        if conn:
            try:
                conn.close()
            except Exception:
                pass

def evaluate_answer_with_ai(user_answer, expected_answer, answer_keywords, max_marks, model_key=None, user_id=None):
    """Evaluate user answer using AI and return score
    
    Args:
        user_answer: The student's answer to evaluate
        expected_answer: The expected/correct answer
        answer_keywords: List of key concepts/terms expected
        max_marks: Maximum marks for this question
        model_key: Optional AI model key to use (defaults to get_default_model())
    """
    try:
        keywords_text = ", ".join(answer_keywords) if answer_keywords else "No specific keywords"
        
        evaluation_prompt = f"""
You are "Sathi AI", the friendly AI learning companion of Ashlya Academy, acting as an expert educational evaluator. Evaluate this student's answer and provide a fair, constructive score.

**Question Context:**
Expected Answer: {expected_answer}
Key Concepts/Keywords: {keywords_text}
Maximum Marks: {max_marks}

**Student's Answer:**
{user_answer}

**Evaluation Criteria:**
1. Conceptual Understanding (40%)
2. Use of Key Terms/Keywords (30%)
3. Clarity and Structure (20%)
4. Completeness (10%)

Please provide:
1. A score out of {max_marks} (use decimals like 2.5 if needed)
2. Brief feedback explaining the score

Return your evaluation in this format:
SCORE: [number between 0 and {max_marks}]
FEEDBACK: [brief explanation]
"""

        messages = [
            {"role": "system", "content": "You are 'Sathi AI', the friendly AI learning companion of Ashlya Academy, acting as an expert educational evaluator. Provide fair and constructive evaluation of student answers with encouragement."},
            {"role": "user", "content": evaluation_prompt}
        ]
        
        # Use AI to evaluate with user's preferred model or default
        from utils.async_runner import run_async
        evaluation_model = model_key if model_key else get_default_model(user_id=user_id)
        logger.info(f"Using model '{evaluation_model}' for Q&A evaluation")
        ai_response = run_async(
            generate_ai_content_with_retry(evaluation_model, messages, user_id=user_id, max_retries=3)
        )
        
        # Parse AI response to extract score
        score = 0
        if "SCORE:" in ai_response:
            try:
                score_line = ai_response.split("SCORE:")[1].split("\n")[0].strip()
                score = float(score_line)
                score = min(max(score, 0), max_marks)  # Ensure score is within bounds
            except (ValueError, IndexError):
                logger.warning(f"Could not parse AI score from: {ai_response[:100]}")
                score = max_marks * 0.5  # Default to 50% if parsing fails
        
        return score
        
    except Exception as e:
        logger.error(f"AI evaluation error: {str(e)}")
        return max_marks * 0.5  # Return 50% as fallback

@qa_bp.route('/api/ai-qa/generate', methods=['POST'])
@require_api_key
@encrypt_response
def generate_ai_qa():
    """Generate Q&A questions using AI based on selected topics"""
    conn = None
    cursor = None
    try:
        if not ai_available:
            return create_error_response("AI service is not available", 503)
        
        data = request.get_json() or {}
        user_id = data.get('user_id')
        topic_ids = data.get('topic_ids', [])
        question_count = min(int(data.get('question_count', 5)), 10)  # Max 10 AI Q&A questions
        difficulty_level = data.get('difficulty_level', 'medium')
        custom_instructions = data.get('custom_instructions', '')
        ai_model = data.get('ai_model') or get_default_model(user_id=user_id)
        
        if not user_id:
            return create_error_response("User ID is required", 400)
        
        if not topic_ids:
            return create_error_response("At least one topic must be selected", 400)
        
        from database import get_cached_connection
        conn = get_cached_connection()
        if not conn:
            return create_error_response("Database connection failed", 500)
        
        cursor = conn.cursor(dictionary=True, buffered=True)
        cursor.execute("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci")
        
        # Get topic details
        placeholders = ','.join(['%s'] * len(topic_ids))
        cursor.execute(f"""
            SELECT t.id, t.title, c.title as course_title
            FROM topics t
            JOIN courses c ON t.course_id = c.id
            JOIN enrollments e ON c.id = e.course_id AND e.user_id = %s AND e.payment_status IN ('free', 'paid')
            WHERE t.id IN ({placeholders})
        """, [user_id] + topic_ids)
        
        accessible_topics = cursor.fetchall()
        
        if not accessible_topics:
            return create_error_response("No accessible topics found", 403)
        
        # Prepare topic information for AI
        topic_info = []
        for topic in accessible_topics:
            topic_info.append(f"- {topic['title']} (from {topic['course_title']})")
        
        topics_text = "\n".join(topic_info)
        
        ai_prompt = f"""
You are "Sathi AI", the friendly AI learning companion of Ashlya Academy, creating open-ended questions for student practice.

Generate exactly {question_count} open-ended questions based on these topics:

{topics_text}

Requirements:
- Difficulty: {difficulty_level}
- Questions should require detailed explanations/answers (not just yes/no)
- Include expected answer for each question
- Provide keywords that should be present in a good answer
- Each question should test understanding, not just memorization

{f'Additional instructions: {custom_instructions}' if custom_instructions else ''}

Return ONLY valid JSON in this exact format (NO markdown blocks):
[
  {{
    "question_text": "**Question:** Explain the concept of... [question here]",
    "expected_answer": "A detailed model answer showing what a good response should include...",
    "answer_keywords": ["keyword1", "keyword2", "keyword3", "keyword4"],
    "difficulty": "{difficulty_level}",
    "explanation": "This question tests understanding of... [explanation of what the question assesses]",
    "marks": 2,
    "time_limit": 300
  }}
]
"""
        
        logger.info(f"Generating AI Q&A for user {user_id} with topics: {topic_ids}")
        
        # AI generation using the robust parsing function
        try:
            # Validate model exists in database
            try:
                model_config = get_model_config(ai_model)
                if not model_config:
                    logger.warning(f"Invalid AI model '{ai_model}', using default model")
                    current_model = get_default_model(user_id=user_id)
                else:
                    current_model = ai_model
            except Exception as e:
                logger.warning(f"Error validating AI model '{ai_model}': {e}, using default")
                current_model = get_default_model(user_id=user_id)
            
            messages = [
                {"role": "system", "content": "You are 'Sathi AI', the friendly AI learning companion of Ashlya Academy. Generate open-ended questions that require detailed explanations. Return only valid JSON array without any markdown wrappers, extra text, or formatting."},
                {"role": "user", "content": ai_prompt}
            ]
            
            # Use async loop to call the robust parsing function
            from utils.async_runner import run_async
            ai_questions = run_async(
                generate_and_parse_ai_qa_with_retry(current_model, messages, user_id=user_id, max_retries=5)
            )
            
            logger.info(f"Successfully generated {len(ai_questions)} AI Q&A questions")
            

            # Store AI-generated Q&A questions
            stored_questions = []
            qa_id = str(uuid.uuid4())
            
            for i, ai_question in enumerate(ai_questions):
                try:
                    # Validation already done in generate_and_validate_qa()
                    # Proceeding with storage
                    
                    cursor.execute("""
                        INSERT INTO qa_questions (
                            topic_id, question_text, expected_answer, answer_keywords,
                            difficulty, explanation, marks, time_limit, is_ai_generated,
                            ai_metadata, user_id, ai_model
                        ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                    """, (
                        accessible_topics[i % len(accessible_topics)]['id'],
                        ai_question['question_text'],
                        ai_question['expected_answer'],
                        json.dumps(ai_question.get('answer_keywords', [])),
                        ai_question.get('difficulty', difficulty_level),
                        ai_question.get('explanation', ''),
                        ai_question.get('marks', 2),
                        ai_question.get('time_limit', 300),
                        True,
                        json.dumps({
                            'qa_id': qa_id,
                            'generated_at': datetime.now().isoformat(),
                            'model_used': ai_model,
                            'custom_instructions': custom_instructions
                        }),
                        user_id,
                        ai_model
                    ))
                    
                    question_id = cursor.lastrowid
                    
                    stored_questions.append({
                        'question_id': str(question_id),
                        'question_text': ai_question['question_text'],
                        'expected_answer': ai_question['expected_answer'],
                        'answer_keywords': ai_question.get('answer_keywords', []),
                        'difficulty': ai_question.get('difficulty', difficulty_level),
                        'explanation': ai_question.get('explanation', ''),
                        'topic_title': accessible_topics[i % len(accessible_topics)]['title'],
                        'topic_id': accessible_topics[i % len(accessible_topics)]['id'],
                        'course_title': accessible_topics[i % len(accessible_topics)]['course_title'],
                        'marks': ai_question.get('marks', 2),
                        'time_limit': ai_question.get('time_limit', 300)
                    })
                    
                except Exception as e:
                    logger.error(f"Error storing AI Q&A question {i}: {e}")
                    continue
            
            conn.commit()
            
            if not stored_questions:
                return create_error_response("Failed to generate valid Q&A questions", 500)
            
            qa_data = {
                'qa_id': qa_id,
                'title': f"AI-Generated Q&A - {len(accessible_topics)} Topics",
                'description': f"AI-generated Q&A with {len(stored_questions)} questions covering selected topics",
                'questions': stored_questions,
                'total_questions': len(stored_questions),
                'total_time_limit': sum(q.get('time_limit', 300) for q in stored_questions),
                'difficulty_level': difficulty_level,
                'topics_covered': [t['id'] for t in accessible_topics],
                'created_at': datetime.now().isoformat(),
                'is_ai_generated': True
            }
            
            logger.info(f"Successfully generated AI Q&A with {len(stored_questions)} questions")
            return format_api_response(qa_data, include_legacy_fields=qa_data)
        
        except Exception as ai_error:
            logger.error(f"AI Q&A generation/storage error: {str(ai_error)}")
            return create_error_response(f"Failed to generate AI Q&A: {str(ai_error)}", 500)
            
    except Exception as e:
        logger.error(f"Error in generate_ai_qa: {str(e)}")
        return create_error_response(f"Failed to generate AI Q&A: {str(e)}")
    finally:
        if cursor:
            cursor.close()
        if conn:
            try:
                conn.close()
            except Exception:
                pass

@qa_bp.route('/api/qa-library', methods=['GET'])
@require_api_key
@encrypt_response
def get_qa_library():
    """Get Q&A questions for browsing and learning with search and filters"""
    conn = None
    cursor = None
    try:
        user_id = request.args.get('user_id')
        course_id = request.args.get('course_id')  # Filter by specific course
        search_query = request.args.get('search', '').strip()
        topic_ids = request.args.getlist('topic_ids')
        difficulty = request.args.get('difficulty', 'all')
        source_type = request.args.get('source_type', 'all')  # all, ai_generated, manual
        page = int(request.args.get('page', 1))
        per_page = min(int(request.args.get('per_page', 10)), 50)  # Default 10, max 50 per page
        
        if not user_id:
            return create_error_response("User ID is required", 400)
        
        from database import get_cached_connection
        conn = get_cached_connection()
        if not conn:
            return create_error_response("Database connection failed", 500)
        
        cursor = conn.cursor(dictionary=True, buffered=True)
        cursor.execute("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci")
        
        # Build WHERE conditions
        conditions = []
        params = [user_id]
        
        # Only show questions from enrolled courses
        conditions.append("""
            EXISTS (
                SELECT 1 FROM enrollments e 
                JOIN courses c ON e.course_id = c.id 
                JOIN topics t ON c.id = t.course_id 
                WHERE e.user_id = %s AND t.id = q.topic_id
            )
        """)
        
        # Filter by course ID if provided
        if course_id:
            conditions.append("c.id = %s")
            params.append(int(course_id))
        
        # Search in question text and explanation
        if search_query:
            conditions.append("(q.question_text LIKE %s OR q.explanation LIKE %s OR top.title LIKE %s)")
            search_param = f"%{search_query}%"
            params.extend([search_param, search_param, search_param])
        
        # Filter by topic IDs
        if topic_ids:
            topic_placeholders = ','.join(['%s'] * len(topic_ids))
            conditions.append(f"q.topic_id IN ({topic_placeholders})")
            params.extend([int(tid) for tid in topic_ids])
        
        # Filter by difficulty
        if difficulty != 'all':
            conditions.append("q.difficulty = %s")
            params.append(difficulty)
        
        # Filter by source type
        if source_type == 'ai_generated':
            conditions.append("q.is_ai_generated = 1")
        elif source_type == 'manual':
            conditions.append("(q.is_ai_generated = 0 OR q.is_ai_generated IS NULL)")
        
        where_clause = "WHERE " + " AND ".join(conditions) if conditions else ""
        
        # Get total count for pagination
        count_query = f"""
            SELECT COUNT(*) as total
            FROM qa_questions q
            JOIN topics top ON q.topic_id = top.id
            JOIN courses c ON top.course_id = c.id
            {where_clause}
        """
        
        cursor.execute(count_query, params)
        total_questions = cursor.fetchone()['total']
        
        # Calculate pagination
        offset = (page - 1) * per_page
        total_pages = (total_questions + per_page - 1) // per_page
        
        # Get questions with pagination
        main_query = f"""
            SELECT 
                q.id as question_id,
                q.question_text,
                q.expected_answer,
                q.answer_keywords,
                q.difficulty,
                q.explanation,
                q.marks,
                q.time_limit,
                q.is_ai_generated,
                q.ai_model,
                q.created_at,
                top.id as topic_id,
                top.title as topic_title,
                top.full_path as topic_path,
                c.id as course_id,
                c.title as course_title,
                c.image_url as course_thumbnail
            FROM qa_questions q
            JOIN topics top ON q.topic_id = top.id
            JOIN courses c ON top.course_id = c.id
            {where_clause}
            ORDER BY q.created_at DESC, q.difficulty, top.title
            LIMIT %s OFFSET %s
        """
        
        cursor.execute(main_query, params + [per_page, offset])
        questions = cursor.fetchall()
        
        # Format questions for frontend
        formatted_questions = []
        for q in questions:
            # Parse keywords
            keywords = []
            if q['answer_keywords']:
                try:
                    keywords = json.loads(q['answer_keywords'])
                except (json.JSONDecodeError, TypeError):
                    keywords = []
            
            formatted_questions.append({
                'question_id': str(q['question_id']),
                'question_text': q['question_text'],
                'expected_answer': q['expected_answer'],
                'answer_keywords': keywords,
                'difficulty': q['difficulty'],
                'explanation': q['explanation'],
                'marks': q['marks'] or 1,
                'time_limit': q['time_limit'] or 300,
                'is_ai_generated': bool(q['is_ai_generated']),
                'ai_model': q['ai_model'] if q['is_ai_generated'] else None,
                'created_at': q['created_at'].isoformat() if q['created_at'] else None,
                'topic': {
                    'topic_id': q['topic_id'],
                    'topic_title': q['topic_title'],
                    'topic_path': q['topic_path']
                },
                'course': {
                    'course_id': q['course_id'],
                    'course_title': q['course_title'],
                    'course_thumbnail': q['course_thumbnail']
                }
            })
        
        # Get available filters for the sidebar
        filters_query = """
            SELECT 
                DISTINCT q.difficulty,
                COUNT(*) as count
            FROM qa_questions q
            JOIN topics top ON q.topic_id = top.id
            JOIN courses c ON top.course_id = c.id
            WHERE EXISTS (
                SELECT 1 FROM enrollments e 
                WHERE e.course_id = c.id AND e.user_id = %s AND e.payment_status IN ('free', 'paid')
            )
            GROUP BY q.difficulty
            ORDER BY q.difficulty
        """
        
        cursor.execute(filters_query, [user_id])
        difficulty_filters = cursor.fetchall()
        
        # Get topics with question counts
        topics_query = """
            SELECT 
                top.id as topic_id,
                top.title as topic_title,
                top.full_path as topic_path,
                c.title as course_title,
                COUNT(q.id) as question_count
            FROM topics top
            JOIN courses c ON top.course_id = c.id
            JOIN enrollments e ON c.id = e.course_id AND e.user_id = %s AND e.payment_status IN ('free', 'paid')
            LEFT JOIN qa_questions q ON top.id = q.topic_id
            GROUP BY top.id, top.title, c.title
            HAVING question_count > 0
            ORDER BY c.title, top.title
        """
        
        cursor.execute(topics_query, [user_id])
        topics_with_questions = cursor.fetchall()
        
        response_data = {
            'questions': formatted_questions,
            'pagination': {
                'current_page': page,
                'per_page': per_page,
                'total_questions': total_questions,
                'total_pages': total_pages,
                'has_next': page < total_pages,
                'has_prev': page > 1
            },
            'filters': {
                'difficulties': [{'difficulty': df['difficulty'], 'count': df['count']} for df in difficulty_filters],
                'topics': [{'topic_id': t['topic_id'], 'topic_title': t['topic_title'], 
                          'topic_path': t['topic_path'], 'course_title': t['course_title'], 
                          'question_count': t['question_count']} for t in topics_with_questions]
            },
            'search_query': search_query,
            'applied_filters': {
                'topic_ids': topic_ids,
                'difficulty': difficulty,
                'source_type': source_type
            }
        }
        
        logger.info(f"Q&A Library: User {user_id} retrieved {len(formatted_questions)} questions (page {page})")
        return format_api_response(response_data, include_legacy_fields=response_data)
        
    except Exception as e:
        logger.error(f"Error in get_qa_library: {str(e)}")
        return create_error_response(f"Failed to fetch Q&A library: {str(e)}")
    finally:
        if cursor:
            cursor.close()
        if conn:
            try:
                conn.close()
            except Exception:
                pass

@qa_bp.route('/api/qa-library/question', methods=['GET'])
@require_api_key
@encrypt_response
def get_qa_question():
    """Get a single Q&A question by ID with navigation info"""
    conn = None
    cursor = None
    try:
        user_id = request.args.get('user_id')
        question_id = request.args.get('question_id')
        
        if not user_id:
            return create_error_response("User ID is required", 400)
        if not question_id:
            return create_error_response("Question ID is required", 400)
        
        from database import get_cached_connection
        conn = get_cached_connection()
        if not conn:
            return create_error_response("Database connection failed", 500)
        
        cursor = conn.cursor(dictionary=True, buffered=True)
        cursor.execute("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci")
        
        # Get the question
        question_query = """
            SELECT 
                q.id as question_id,
                q.question_text,
                q.expected_answer,
                q.answer_keywords,
                q.difficulty,
                q.explanation,
                q.marks,
                q.time_limit,
                q.is_ai_generated,
                q.ai_model,
                q.created_at,
                top.id as topic_id,
                top.title as topic_title,
                top.full_path as topic_path,
                c.id as course_id,
                c.title as course_title,
                c.image_url as course_thumbnail
            FROM qa_questions q
            JOIN topics top ON q.topic_id = top.id
            JOIN courses c ON top.course_id = c.id
            JOIN enrollments e ON c.id = e.course_id AND e.user_id = %s AND e.payment_status IN ('free', 'paid')
            WHERE q.id = %s
        """
        
        cursor.execute(question_query, [user_id, question_id])
        question_row = cursor.fetchone()
        
        if not question_row:
            return create_error_response("Question not found or you don't have access", 404)
        
        # Parse answer keywords
        keywords = []
        if question_row['answer_keywords']:
            try:
                if isinstance(question_row['answer_keywords'], list):
                    keywords = question_row['answer_keywords']
                elif isinstance(question_row['answer_keywords'], str):
                    import json
                    keywords = json.loads(question_row['answer_keywords'])
            except:
                keywords = []
        
        question_data = {
            'question_id': str(question_row['question_id']),
            'question_text': question_row['question_text'],
            'expected_answer': question_row['expected_answer'] or '',
            'answer_keywords': keywords,
            'difficulty': question_row['difficulty'],
            'explanation': question_row['explanation'] or '',
            'marks': question_row['marks'],
            'time_limit': question_row['time_limit'],
            'is_ai_generated': bool(question_row['is_ai_generated']),
            'ai_model': question_row['ai_model'],
            'created_at': question_row['created_at'].isoformat() if question_row['created_at'] else None,
            'topic': {
                'topic_id': question_row['topic_id'],
                'topic_title': question_row['topic_title'],
                'topic_path': question_row['topic_path']
            },
            'course': {
                'course_id': question_row['course_id'],
                'course_title': question_row['course_title'],
                'course_thumbnail': question_row['course_thumbnail']
            }
        }
        
        # Get navigation (previous and next questions in the same course)
        course_id = question_row['course_id']
        
        # Get all question IDs for this course ordered by creation
        nav_query = """
            SELECT q.id as question_id, q.question_text
            FROM qa_questions q
            JOIN topics top ON q.topic_id = top.id
            WHERE top.course_id = %s
            ORDER BY q.created_at DESC, q.id
        """
        cursor.execute(nav_query, [course_id])
        all_questions = cursor.fetchall()
        
        # Find current question index
        current_index = 0
        previous_question = None
        next_question = None
        
        for i, q in enumerate(all_questions):
            if str(q['question_id']) == str(question_id):
                current_index = i + 1  # 1-indexed for display
                if i > 0:
                    previous_question = {
                        'question_id': str(all_questions[i-1]['question_id']),
                        'question_text': all_questions[i-1]['question_text'][:100] + '...' if len(all_questions[i-1]['question_text']) > 100 else all_questions[i-1]['question_text']
                    }
                if i < len(all_questions) - 1:
                    next_question = {
                        'question_id': str(all_questions[i+1]['question_id']),
                        'question_text': all_questions[i+1]['question_text'][:100] + '...' if len(all_questions[i+1]['question_text']) > 100 else all_questions[i+1]['question_text']
                    }
                break
        
        response_data = {
            'question': question_data,
            'navigation': {
                'previous': previous_question,
                'next': next_question,
                'current_index': current_index,
                'total_questions': len(all_questions)
            }
        }
        
        logger.info(f"Q&A Question: User {user_id} retrieved question {question_id}")
        return format_api_response(response_data, include_legacy_fields=response_data)
        
    except Exception as e:
        logger.error(f"Error in get_qa_question: {str(e)}")
        return create_error_response(f"Failed to fetch question: {str(e)}")
    finally:
        if cursor:
            cursor.close()
        if conn:
            try:
                conn.close()
            except Exception:
                pass

@qa_bp.route('/api/qa-progress/store', methods=['POST'])
@require_api_key
@encrypt_response
def store_qa_progress():
    """Store Q&A progress and results for analytics"""
    try:
        data = request.get_json()
        if not data:
            logger.error("No data provided in Q&A progress request")
            return create_error_response("No data provided", 400)

        required_fields = ['user_id', 'qa_type', 'total_questions', 'answered_questions']
        for field in required_fields:
            if field not in data:
                logger.error(f"Missing required field in Q&A progress: {field}")
                return create_error_response(f"Missing required field: {field}", 400)

        logger.info(f"Storing Q&A progress for user: {data.get('user_id')}, type: {data.get('qa_type')}")

        def parse_datetime(datetime_str):
            if not datetime_str:
                return datetime.now()
            try:
                if isinstance(datetime_str, str):
                    if datetime_str.endswith('Z'):
                        datetime_str = datetime_str[:-1] + '+00:00'
                    from datetime import datetime as dt
                    import dateutil.parser
                    parsed_dt = dateutil.parser.parse(datetime_str)
                    return parsed_dt.replace(tzinfo=None)
                return datetime_str
            except Exception as e:
                logger.warning(f"Failed to parse datetime '{datetime_str}': {e}")
                return datetime.now()

        with get_db_connection() as conn:
            cursor = conn.cursor()
            completed_at = parse_datetime(data.get('completed_at'))

            # Store Q&A session
            qa_session_query = """
            INSERT INTO qa_sessions (
                user_id, qa_type, qa_id, course_id, topic_ids,
                total_questions, answered_questions, total_marks_possible,
                marks_obtained, human_marks_obtained, score_percentage,
                human_score_percentage, time_taken, completed_at,
                is_ai_generated, ai_qa_metadata, evaluation_status
            ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            """
            
            cursor.execute(qa_session_query, (
                data['user_id'],
                data['qa_type'],
                data.get('qa_id'),
                data.get('course_id'),
                json.dumps(data.get('topic_ids', [])),
                data['total_questions'],
                data['answered_questions'],
                data.get('total_marks_possible', 0),
                data.get('marks_obtained', 0),
                data.get('human_marks_obtained'),
                data.get('score_percentage', 0),
                data.get('human_score_percentage'),
                data.get('time_taken', 0),
                completed_at,
                data.get('is_ai_generated', False),
                json.dumps(data.get('ai_qa_metadata', {})),
                data.get('evaluation_status', 'pending')
            ))
            
            session_id = cursor.lastrowid
            logger.info(f"Created Q&A session with ID: {session_id}")

            # Store individual responses if provided
            responses_stored = 0
            if 'responses_data' in data and data['responses_data']:
                response_query = """
                INSERT INTO qa_responses (
                    session_id, question_id, user_answer, ai_evaluation,
                    human_evaluation, ai_score, human_score, final_score,
                    time_taken, topic_id, answer_quality, feedback
                ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                """
                
                for response in data['responses_data']:
                    try:
                        cursor.execute(response_query, (
                            session_id,
                            response.get('question_id'),
                            response.get('user_answer', ''),
                            json.dumps(response.get('ai_evaluation', {})),
                            json.dumps(response.get('human_evaluation', {})) if response.get('human_evaluation') else None,
                            response.get('ai_score'),
                            response.get('human_score'),
                            response.get('final_score'),
                            response.get('time_taken', 0),
                            response.get('topic_id'),
                            response.get('answer_quality'),
                            response.get('feedback')
                        ))
                        responses_stored += 1
                    except mysql.connector.Error as e:
                        logger.error(f"Error storing Q&A response: {e}")

            conn.commit()
            
            response_data = {
                'session_id': session_id,
                'responses_stored': responses_stored,
                'message': f'Q&A progress stored successfully. Session ID: {session_id}'
            }
            
            logger.info(f"Successfully stored Q&A progress: {response_data}")
            return format_api_response(response_data, include_legacy_fields=response_data)

    except mysql.connector.Error as db_error:
        logger.error(f"Database error in store_qa_progress: {str(db_error)}")
        return create_error_response(f"Database error: {str(db_error)}", 500)
    except Exception as e:
        logger.error(f"Unexpected error in store_qa_progress: {str(e)}")
        return create_error_response(f"Failed to store Q&A progress: {str(e)}", 500)

@qa_bp.route('/api/qa-sessions', methods=['GET'])
@require_api_key
@encrypt_response
def get_qa_sessions():
    """Get Q&A session history for a user"""
    conn = None
    cursor = None
    try:
        user_id = request.args.get('user_id')
        page = int(request.args.get('page', 1))
        limit = min(int(request.args.get('limit', 10)), 50)  # Max 50 per page
        
        if not user_id:
            return create_error_response("User ID is required", 400)
        
        from database import get_cached_connection
        conn = get_cached_connection()
        if not conn:
            return create_error_response("Database connection failed", 500)
        
        cursor = conn.cursor(dictionary=True, buffered=True)
        cursor.execute("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci")
        
        # Calculate offset
        offset = (page - 1) * limit
        
        # Get total count
        cursor.execute("""
            SELECT COUNT(*) as total
            FROM qa_sessions
            WHERE user_id = %s
        """, (user_id,))
        
        total_sessions = cursor.fetchone()['total']
        
        # Get sessions with pagination
        cursor.execute("""
            SELECT 
                id as session_id,
                qa_type,
                qa_id,
                course_id,
                topic_ids,
                total_questions,
                answered_questions,
                total_marks_possible,
                marks_obtained,
                human_marks_obtained,
                score_percentage,
                human_score_percentage,
                time_taken,
                completed_at,
                created_at,
                is_ai_generated,
                evaluation_status
            FROM qa_sessions
            WHERE user_id = %s
            ORDER BY completed_at DESC, created_at DESC
            LIMIT %s OFFSET %s
        """, (user_id, limit, offset))
        
        sessions = cursor.fetchall()
        
        # Format sessions for Flutter
        formatted_sessions = []
        for session in sessions:
            try:
                topic_ids = json.loads(session['topic_ids']) if session['topic_ids'] else []
            except (json.JSONDecodeError, TypeError):
                topic_ids = []
            
            formatted_sessions.append({
                'id': session['session_id'],
                'qaType': session['qa_type'],
                'qaId': session['qa_id'],
                'courseId': session['course_id'],
                'topicIds': topic_ids,
                'questionsCount': session['total_questions'],
                'answeredQuestions': session['answered_questions'],
                'totalMarks': session['total_marks_possible'],
                'marksObtained': session['marks_obtained'],
                'humanMarks': session['human_marks_obtained'],
                'scorePercentage': session['score_percentage'],
                'humanScorePercentage': session['human_score_percentage'],
                'timeLimit': session['time_taken'],
                'completedAt': session['completed_at'].isoformat() if session['completed_at'] else None,
                'createdAt': session['created_at'].isoformat() if session['created_at'] else None,
                'isAIGenerated': bool(session['is_ai_generated']),
                'evaluationStatus': session['evaluation_status'],
                'completed': session['completed_at'] is not None,
                # Add derived fields that Flutter expects
                'lastAttempted': session['completed_at'] if session['completed_at'] else session['created_at'],
                'title': f"{session['qa_type'].title()} Q&A - {session['total_questions']} Questions",
                'difficulty': 'mixed',  # Default since we don't store this at session level
                'tags': []  # Default empty
            })
        
        response_data = {
            'sessions': formatted_sessions,
            'pagination': {
                'current_page': page,
                'per_page': limit,
                'total': total_sessions,
                'total_pages': (total_sessions + limit - 1) // limit,
                'has_next': (page * limit) < total_sessions,
                'has_prev': page > 1
            }
        }
        
        logger.info(f"Retrieved {len(formatted_sessions)} QA sessions for user {user_id}")
        return format_api_response(response_data, include_legacy_fields=response_data)
        
    except Exception as e:
        logger.error(f"Error in get_qa_sessions: {str(e)}")
        return create_error_response(f"Failed to fetch QA sessions: {str(e)}")
    finally:
        if cursor:
            cursor.close()
        if conn:
            try:
                conn.close()
            except Exception:
                pass

@qa_bp.route('/api/qa-sessions/<int:session_id>', methods=['GET'])
@require_api_key
@encrypt_response
def get_qa_session_details(session_id):
    """Get detailed information about a specific QA session"""
    conn = None
    cursor = None
    try:
        user_id = request.args.get('user_id')
        
        if not user_id:
            return create_error_response("User ID is required", 400)
        
        from database import get_cached_connection
        conn = get_cached_connection()
        if not conn:
            return create_error_response("Database connection failed", 500)
        
        cursor = conn.cursor(dictionary=True, buffered=True)
        cursor.execute("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci")
        
        # Get session details
        cursor.execute("""
            SELECT 
                id as session_id,
                qa_type,
                qa_id,
                course_id,
                topic_ids,
                total_questions,
                answered_questions,
                total_marks_possible,
                marks_obtained,
                human_marks_obtained,
                score_percentage,
                human_score_percentage,
                time_taken,
                completed_at,
                created_at,
                is_ai_generated,
                evaluation_status,
                ai_qa_metadata
            FROM qa_sessions
            WHERE id = %s AND user_id = %s
        """, (session_id, user_id))
        
        session = cursor.fetchone()
        
        if not session:
            return create_error_response("Session not found", 404)
        
        # Get session responses
        cursor.execute("""
            SELECT 
                r.id as response_id,
                r.question_id,
                r.user_answer,
                r.ai_evaluation,
                r.human_evaluation,
                r.ai_score,
                r.human_score,
                r.final_score,
                r.time_taken,
                r.topic_id,
                r.answer_quality,
                r.feedback,
                q.question_text,
                q.expected_answer,
                q.answer_keywords,
                q.explanation,
                q.difficulty,
                q.marks as question_marks,
                t.title as topic_title
            FROM qa_responses r
            LEFT JOIN qa_questions q ON r.question_id = q.id
            LEFT JOIN topics t ON r.topic_id = t.id
            WHERE r.session_id = %s
            ORDER BY r.id
        """, (session_id,))
        
        responses = cursor.fetchall()
        
        # Format session data
        try:
            topic_ids = json.loads(session['topic_ids']) if session['topic_ids'] else []
            ai_metadata = json.loads(session['ai_qa_metadata']) if session['ai_qa_metadata'] else {}
        except (json.JSONDecodeError, TypeError):
            topic_ids = []
            ai_metadata = {}
        
        # Format responses
        formatted_responses = []
        for response in responses:
            try:
                ai_eval = json.loads(response['ai_evaluation']) if response['ai_evaluation'] else {}
                human_eval = json.loads(response['human_evaluation']) if response['human_evaluation'] else {}
                keywords = json.loads(response['answer_keywords']) if response['answer_keywords'] else []
            except (json.JSONDecodeError, TypeError):
                ai_eval = {}
                human_eval = {}
                keywords = []
            
            formatted_responses.append({
                'responseId': response['response_id'],
                'questionId': str(response['question_id']) if response['question_id'] else None,
                'userAnswer': response['user_answer'],
                'aiEvaluation': ai_eval,
                'humanEvaluation': human_eval,
                'aiScore': response['ai_score'],
                'humanScore': response['human_score'],
                'finalScore': response['final_score'],
                'timeTaken': response['time_taken'],
                'topicId': response['topic_id'],
                'answerQuality': response['answer_quality'],
                'feedback': response['feedback'],
                'question': {
                    'questionText': response['question_text'],
                    'expectedAnswer': response['expected_answer'],
                    'answerKeywords': keywords,
                    'explanation': response['explanation'],
                    'difficulty': response['difficulty'],
                    'marks': response['question_marks'],
                    'topicTitle': response['topic_title']
                } if response['question_text'] else None
            })
        
        session_details = {
            'id': session['session_id'],
            'qaType': session['qa_type'],
            'qaId': session['qa_id'],
            'courseId': session['course_id'],
            'topicIds': topic_ids,
            'questionsCount': session['total_questions'],
            'answeredQuestions': session['answered_questions'],
            'totalMarks': session['total_marks_possible'],
            'marksObtained': session['marks_obtained'],
            'humanMarks': session['human_marks_obtained'],
            'scorePercentage': session['score_percentage'],
            'humanScorePercentage': session['human_score_percentage'],
            'timeLimit': session['time_taken'],
            'completedAt': session['completed_at'].isoformat() if session['completed_at'] else None,
            'createdAt': session['created_at'].isoformat() if session['created_at'] else None,
            'isAIGenerated': bool(session['is_ai_generated']),
            'evaluationStatus': session['evaluation_status'],
            'completed': session['completed_at'] is not None,
            'aiMetadata': ai_metadata,
            'responses': formatted_responses,
            # Add derived fields that Flutter expects
            'lastAttempted': session['completed_at'] if session['completed_at'] else session['created_at'],
            'title': f"{session['qa_type'].title()} Q&A - {session['total_questions']} Questions",
            'difficulty': 'mixed',
            'tags': []
        }
        
        logger.info(f"Retrieved QA session details for session {session_id}")
        return format_api_response(session_details, include_legacy_fields=session_details)
        
    except Exception as e:
        logger.error(f"Error in get_qa_session_details: {str(e)}")
        return create_error_response(f"Failed to fetch session details: {str(e)}")
    finally:
        if cursor:
            cursor.close()
        if conn:
            try:
                conn.close()
            except Exception:
                pass

@qa_bp.route('/api/course-topics-qa', methods=['GET'])
@require_api_key
@encrypt_response
def get_course_topics_for_qa():
    """Get courses with topics and QA question counts for the user"""
    conn = None
    cursor = None
    try:
        user_id = request.args.get('user_id')
        
        if not user_id:
            return create_error_response("User ID is required", 400)
        
        from database import get_cached_connection
        conn = get_cached_connection()
        if not conn:
            return create_error_response("Database connection failed", 500)
        
        cursor = conn.cursor(dictionary=True, buffered=True)
        cursor.execute("SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci")
        
        # Get courses with QA question counts
        cursor.execute("""
            SELECT 
                c.id as course_id,
                c.title as course_title,
                c.description as course_description,
                c.image_url as course_thumbnail,
                COUNT(DISTINCT t.id) as total_topics,
                COUNT(DISTINCT q.id) as total_qa_questions,
                COUNT(DISTINCT v.id) as total_videos,
                COALESCE(SUM(v.duration), 0) as total_duration
            FROM courses c
            JOIN enrollments e ON c.id = e.course_id AND e.user_id = %s AND e.payment_status IN ('free', 'paid')
            LEFT JOIN topics t ON c.id = t.course_id
            LEFT JOIN qa_questions q ON t.id = q.topic_id
            LEFT JOIN videos v ON c.id = v.course_id
            GROUP BY c.id, c.title, c.description, c.image_url
            HAVING total_qa_questions > 0
            ORDER BY c.title
        """, (user_id,))
        
        courses_data = cursor.fetchall()
        
        # Get topics with QA question counts for each course
        courses_with_topics = []
        for course in courses_data:
            course_id = course['course_id']
            
            # Get hierarchical topics with QA question counts
            cursor.execute("""
                SELECT 
                    t.id as topic_id,
                    t.title as topic_title,
                    t.parent_topic_id,
                    t.level,
                    t.full_path,
                    t.order_sequence,
                    COUNT(q.id) as question_count
                FROM topics t
                LEFT JOIN qa_questions q ON t.id = q.topic_id
                WHERE t.course_id = %s
                GROUP BY t.id, t.title, t.parent_topic_id, t.level, t.full_path, t.order_sequence
                ORDER BY t.level, t.order_sequence, t.title
            """, (course_id,))
            
            topics_data = cursor.fetchall()
            
            # Build hierarchical structure
            def build_topic_hierarchy(topics_list, parent_id=None):
                result = []
                for topic in topics_list:
                    if topic['parent_topic_id'] == parent_id:
                        children = build_topic_hierarchy(topics_list, topic['topic_id'])
                        topic_dict = {
                            'topic_id': topic['topic_id'],
                            'topic_title': topic['topic_title'],
                            'parent_topic_id': topic['parent_topic_id'],
                            'level': topic['level'],
                            'full_path': topic['full_path'],
                            'order_sequence': topic['order_sequence'],
                            'question_count': topic['question_count']
                        }
                        if children:
                            topic_dict['children'] = children
                        result.append(topic_dict)
                return result
            
            hierarchical_topics = build_topic_hierarchy(topics_data)
            
            course_with_topics = {
                'course_id': course['course_id'],
                'course_title': course['course_title'],
                'course_description': course['course_description'],
                'course_thumbnail': course['course_thumbnail'],
                'total_topics': course['total_topics'],
                'total_qa_questions': course['total_qa_questions'],
                'total_videos': course['total_videos'],
                'total_duration': course['total_duration'],
                'topics': hierarchical_topics
            }
            
            courses_with_topics.append(course_with_topics)
        
        response_data = {
            'courses': courses_with_topics
        }
        
        logger.info(f"Retrieved {len(courses_with_topics)} courses with QA topics for user {user_id}")
        return format_api_response(response_data, include_legacy_fields=response_data)
        
    except Exception as e:
        logger.error(f"Error in get_course_topics_for_qa: {str(e)}")
        return create_error_response(f"Failed to fetch course topics for QA: {str(e)}")
    finally:
        if cursor:
            cursor.close()
        if conn:
            try:
                conn.close()
            except Exception:
                pass

@qa_bp.route('/api/qa-submit', methods=['POST'])
@require_api_key
@encrypt_response  
def submit_qa_session():
    """Submit QA session - alias for random-qa/submit to match Flutter expectations"""
    # This is just a redirect to the existing submit_random_qa function
    return submit_random_qa()