adding beifongai project

README.md

@@ -77,6 +77,7 @@ A curated collection of **Awesome LLM apps built with RAG, AI Agents, Multi-agen
 *   [🧠 AI Mental Wellbeing Agent](advanced_ai_agents/multi_agent_apps/ai_mental_wellbeing_agent/)
 *   [📑 AI Meeting Agent](advanced_ai_agents/single_agent_apps/ai_meeting_agent/)
 *   [🧬 AI Self-Evolving Agent](advanced_ai_agents/multi_agent_apps/ai_Self-Evolving_agent/)
+*   [🎧 AI Social Media News and Podcast Agent](advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agent/)
 
 ### 🎮 Autonomous Game Playing Agents
 

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/agents/audio_generate_agent.py

@@ -0,0 +1,359 @@
+from agno.agent import Agent
+import os
+from datetime import datetime
+import tempfile
+import numpy as np
+import soundfile as sf
+from typing import Any, Dict, List, Optional, Tuple
+from utils.load_api_keys import load_api_key
+from openai import OpenAI
+from scipy import signal
+
+
+PODCASTS_FOLDER = "podcasts"
+PODCAST_AUDIO_FOLDER = os.path.join(PODCASTS_FOLDER, "audio")
+PODCAST_MUSIC_FOLDER = os.path.join('static', "musics")
+OPENAI_VOICES = {1: "alloy", 2: "echo", 3: "fable", 4: "onyx", 5: "nova", 6: "shimmer"}
+DEFAULT_VOICE_MAP = {1: "alloy", 2: "nova"}
+TTS_MODEL = "gpt-4o-mini-tts"
+INTRO_MUSIC_FILE = os.path.join(PODCAST_MUSIC_FOLDER, "intro_audio.mp3")
+OUTRO_MUSIC_FILE = os.path.join(PODCAST_MUSIC_FOLDER, "intro_audio.mp3")
+
+
+def resample_audio_scipy(audio, original_sr, target_sr):
+    if original_sr == target_sr:
+        return audio
+    resampling_ratio = target_sr / original_sr
+    num_samples = int(len(audio) * resampling_ratio)
+    resampled = signal.resample(audio, num_samples)
+    return resampled
+
+
+def create_silence_audio(silence_duration: float, sampling_rate: int) -> np.ndarray:
+    if sampling_rate <= 0:
+        print(f"Invalid sampling rate ({sampling_rate}) for silence generation")
+        return np.zeros(0, dtype=np.float32)
+    return np.zeros(int(sampling_rate * silence_duration), dtype=np.float32)
+
+
+def combine_audio_segments(audio_segments: List[np.ndarray], silence_duration: float, sampling_rate: int) -> np.ndarray:
+    if not audio_segments:
+        return np.zeros(0, dtype=np.float32)
+    silence = create_silence_audio(silence_duration, sampling_rate)
+    combined_segments = []
+    for i, segment in enumerate(audio_segments):
+        combined_segments.append(segment)
+        if i < len(audio_segments) - 1:
+            combined_segments.append(silence)
+    combined = np.concatenate(combined_segments)
+    max_amp = np.max(np.abs(combined))
+    if max_amp > 0:
+        combined = combined / max_amp * 0.95
+    return combined
+
+
+def process_audio_file(temp_path: str) -> Optional[Tuple[np.ndarray, int]]:
+    try:
+        from pydub import AudioSegment
+
+        audio_segment = AudioSegment.from_mp3(temp_path)
+        channels = audio_segment.channels
+        sample_width = audio_segment.sample_width
+        frame_rate = audio_segment.frame_rate
+        samples = np.array(audio_segment.get_array_of_samples())
+        if channels == 2:
+            samples = samples.reshape(-1, 2).mean(axis=1)
+        max_possible_value = float(2 ** (8 * sample_width - 1))
+        samples = samples.astype(np.float32) / max_possible_value
+        return samples, frame_rate
+    except ImportError:
+        print("Pydub not available, falling back to soundfile")
+    except Exception as e:
+        print(f"Pydub processing failed: {e}")
+    try:
+        audio_np, samplerate = sf.read(temp_path)
+        return audio_np, samplerate
+    except Exception as e:
+        print(f"Failed to process audio with soundfile: {e}")
+        try:
+            from pydub import AudioSegment
+
+            sound = AudioSegment.from_mp3(temp_path)
+            wav_path = temp_path.replace(".mp3", ".wav")
+            sound.export(wav_path, format="wav")
+            audio_np, samplerate = sf.read(wav_path)
+            os.unlink(wav_path)
+            return audio_np, samplerate
+        except Exception as e:
+            print(f"All audio processing methods failed: {e}")
+    return None
+
+
+def resample_audio(audio, orig_sr, target_sr):
+    try:
+        import librosa
+
+        return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
+    except ImportError:
+        print("Librosa not available for resampling")
+        return audio
+    except Exception as e:
+        print(f"Resampling failed: {e}")
+        return audio
+
+
+def text_to_speech_openai(
+    client: OpenAI,
+    text: str,
+    speaker_id: int,
+    voice_map: Dict[int, str] = None,
+    model: str = TTS_MODEL,
+) -> Optional[Tuple[np.ndarray, int]]:
+    if not text.strip():
+        print("Empty text provided, skipping TTS generation")
+        return None
+    voice_map = voice_map or DEFAULT_VOICE_MAP
+    voice = voice_map.get(speaker_id)
+    if not voice:
+        if speaker_id in OPENAI_VOICES:
+            voice = OPENAI_VOICES[speaker_id]
+        else:
+            voice = next(iter(voice_map.values()), "alloy")
+        print(f"No voice mapping for speaker {speaker_id}, using {voice}")
+    try:
+        print(f"Generating TTS for speaker {speaker_id} using voice '{voice}'")
+        response = client.audio.speech.create(
+            model=model,
+            voice=voice,
+            input=text,
+            response_format="mp3",
+        )
+        audio_data = response.content
+        if not audio_data:
+            print("OpenAI TTS returned empty response")
+            return None
+        print(f"Received {len(audio_data)} bytes from OpenAI TTS")
+        temp_file = tempfile.NamedTemporaryFile(suffix=".mp3", delete=False)
+        temp_path = temp_file.name
+        temp_file.close()
+        with open(temp_path, "wb") as f:
+            f.write(audio_data)
+        try:
+            return process_audio_file(temp_path)
+        finally:
+            if os.path.exists(temp_path):
+                os.unlink(temp_path)
+    except Exception as e:
+        print(f"OpenAI TTS API error: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return None
+
+
+def create_podcast(
+    script: Any,
+    output_path: str,
+    tts_engine: str = "openai",
+    language_code: str = "en",
+    silence_duration: float = 0.7,
+    voice_map: Dict[int, str] = None,
+    model: str = TTS_MODEL,
+) -> Optional[str]:
+    if tts_engine.lower() != "openai":
+        print(f"Only OpenAI TTS engine is available in this standalone version. Requested: {tts_engine}")
+        return None
+    try:
+        api_key = load_api_key("OPENAI_API_KEY")
+        if not api_key:
+            print("No OpenAI API key provided")
+            return None
+        client = OpenAI(api_key=api_key)
+        print("OpenAI client initialized")
+    except Exception as e:
+        print(f"Failed to initialize OpenAI client: {e}")
+        return None
+    output_path = os.path.abspath(output_path)
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    if voice_map is None:
+        voice_map = DEFAULT_VOICE_MAP.copy()
+    model_to_use = model
+    if model == "tts-1" and language_code == "en":
+        model_to_use = "tts-1-hd"
+        print(f"Using high-definition TTS model for English: {model_to_use}")
+    generated_segments = []
+    sampling_rate_detected = None
+
+    if hasattr(script, "entries"):
+        entries = script.entries
+    else:
+        entries = script
+
+    print(f"Processing {len(entries)} script entries")
+    for i, entry in enumerate(entries):
+        if hasattr(entry, "speaker"):
+            speaker_id = entry.speaker
+            entry_text = entry.text
+        else:
+            speaker_id = entry["speaker"]
+            entry_text = entry["text"]
+        print(f"Processing entry {i + 1}/{len(entries)}: Speaker {speaker_id}")
+        result = text_to_speech_openai(
+            client=client,
+            text=entry_text,
+            speaker_id=speaker_id,
+            voice_map=voice_map,
+            model=model_to_use,
+        )
+        if result:
+            segment_audio, segment_rate = result
+            if sampling_rate_detected is None:
+                sampling_rate_detected = segment_rate
+                print(f"Using sample rate: {sampling_rate_detected} Hz")
+            elif sampling_rate_detected != segment_rate:
+                print(f"Sample rate mismatch: {sampling_rate_detected} vs {segment_rate}")
+                try:
+                    segment_audio = resample_audio(segment_audio, segment_rate, sampling_rate_detected)
+                    print(f"Resampled to {sampling_rate_detected} Hz")
+                except Exception as e:
+                    sampling_rate_detected = segment_rate
+                    print(f"Resampling failed: {e}")
+            generated_segments.append(segment_audio)
+        else:
+            print(f"Failed to generate audio for entry {i + 1}")
+    if not generated_segments:
+        print("No audio segments were generated")
+        return None
+    if sampling_rate_detected is None:
+        print("Could not determine sample rate")
+        return None
+    print(f"Combining {len(generated_segments)} audio segments")
+    full_audio = combine_audio_segments(generated_segments, silence_duration, sampling_rate_detected)
+    if full_audio.size == 0:
+        print("Combined audio is empty")
+        return None
+
+    try:
+        if os.path.exists(INTRO_MUSIC_FILE):
+            intro_music, intro_sr = sf.read(INTRO_MUSIC_FILE)
+            print(f"Loaded intro music: {len(intro_music) / intro_sr:.1f} seconds")
+
+            if intro_music.ndim == 2:
+                intro_music = np.mean(intro_music, axis=1)
+
+            if intro_sr != sampling_rate_detected:
+                intro_music = resample_audio_scipy(intro_music, intro_sr, sampling_rate_detected)
+
+            full_audio = np.concatenate([intro_music, full_audio])
+            print("Added intro music")
+
+        if os.path.exists(OUTRO_MUSIC_FILE):
+            outro_music, outro_sr = sf.read(OUTRO_MUSIC_FILE)
+            print(f"Loaded outro music: {len(outro_music) / outro_sr:.1f} seconds")
+
+            if outro_music.ndim == 2:
+                outro_music = np.mean(outro_music, axis=1)
+
+            if outro_sr != sampling_rate_detected:
+                outro_music = resample_audio_scipy(outro_music, outro_sr, sampling_rate_detected)
+
+            full_audio = np.concatenate([full_audio, outro_music])
+            print("Added outro music")
+
+    except Exception as e:
+        print(f"Could not add intro/outro music: {e}")
+        print("Continuing without background music")
+
+    print(f"Writing audio to {output_path}")
+    try:
+        sf.write(output_path, full_audio, sampling_rate_detected)
+    except Exception as e:
+        print(f"Failed to write audio file: {e}")
+        return None
+    if os.path.exists(output_path):
+        file_size = os.path.getsize(output_path)
+        print(f"Audio file created: {output_path} ({file_size / 1024:.1f} KB)")
+        return output_path
+    else:
+        print(f"Failed to create audio file at {output_path}")
+        return None
+
+
+def audio_generate_agent_run(agent: Agent) -> str:
+    """
+    Generate an audio file for the podcast using the selected TTS engine.
+
+    Args:
+        agent: The agent instance
+
+    Returns:
+        A message with the result of audio generation
+    """
+    from services.internal_session_service import SessionService
+
+    session_id = agent.session_id
+    session = SessionService.get_session(session_id)
+    session_state = session["state"]
+    script_data = session_state.get("generated_script", {})
+    if not script_data or (isinstance(script_data, dict) and not script_data.get("sections")):
+        error_msg = "Cannot generate audio: No podcast script data found. Please generate a script first."
+        print(error_msg)
+        return error_msg
+    if isinstance(script_data, dict):
+        podcast_title = script_data.get("title", "Your Podcast")
+    else:
+        podcast_title = "Your Podcast"
+    session_state["stage"] = "audio"
+    audio_dir = PODCAST_AUDIO_FOLDER
+    audio_filename = f"podcast_{datetime.now().strftime('%Y%m%d_%H%M%S')}.wav"
+    audio_path = os.path.join(audio_dir, audio_filename)
+    try:
+        if isinstance(script_data, dict) and "sections" in script_data:
+            speaker_map = {"ALEX": 1, "MORGAN": 2}
+            script_entries = []
+            for section in script_data.get("sections", []):
+                for dialog in section.get("dialog", []):
+                    speaker = dialog.get("speaker", "ALEX")
+                    text = dialog.get("text", "")
+
+                    if text and speaker in speaker_map:
+                        script_entries.append({"text": text, "speaker": speaker_map[speaker]})
+            if not script_entries:
+                error_msg = "Cannot generate audio: No dialog found in the script."
+                print(error_msg)
+                return error_msg
+
+            selected_language = session_state.get("selected_language", {"code": "en", "name": "English"})
+            language_code = selected_language.get("code", "en")
+            language_name = selected_language.get("name", "English")
+            tts_engine = "openai"
+            if tts_engine == "openai" and not load_api_key("OPENAI_API_KEY"):
+                error_msg = "Cannot generate audio: OpenAI API key not found."
+                print(error_msg)
+                return error_msg
+            print(f"Generating podcast audio using {tts_engine} TTS engine in {language_name} language")
+            full_audio_path = create_podcast(
+                script=script_entries,
+                output_path=audio_path,
+                tts_engine=tts_engine,
+                language_code=language_code,
+            )
+            if not full_audio_path:
+                error_msg = f"Failed to generate podcast audio with {tts_engine} TTS engine."
+                print(error_msg)
+                return error_msg
+
+            audio_url = f"{os.path.basename(full_audio_path)}"
+            session_state["audio_url"] = audio_url
+            session_state["show_audio_for_confirmation"] = True
+            SessionService.save_session(session_id, session_state)
+            print(f"Successfully generated podcast audio: {full_audio_path}")
+            return f"I've generated the audio for your '{podcast_title}' podcast using {tts_engine.capitalize()} voices in {language_name}. You can listen to it in the player below. What do you think? If it sounds good, click 'Sounds Great!' to complete your podcast."
+        else:
+            error_msg = "Cannot generate audio: Script is not in the expected format."
+            print(error_msg)
+            return error_msg
+    except Exception as e:
+        error_msg = f"Error generating podcast audio: {str(e)}"
+        print(error_msg)
+        return f"I encountered an error while generating the podcast audio: {str(e)}. Please try again or let me know if you'd like to proceed without audio."

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/agents/image_generate_agent.py

@@ -0,0 +1,100 @@
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from agno.tools.dalle import DalleTools
+from textwrap import dedent
+import json
+from dotenv import load_dotenv
+import uuid
+from db.agent_config_v2 import PODCAST_IMG_DIR
+import os
+import requests
+
+
+load_dotenv()
+
+IMAGE_GENERATION_AGENT_DESCRIPTION = "You are an AI agent that can generate images using DALL-E."
+IMAGE_GENERATION_AGENT_INSTRUCTIONS = dedent("""
+                                             When the user asks you to create an image, use the `create_image` tool to create the image.
+                                             Create a modern, eye-catching podcast cover images that represents a podcast given podcast topic.
+                                             Create 3 images for the given podcast topic.
+
+                                            IMPORTANT INSTRUCTIONS:
+                                            - DO NOT include ANY text in the image
+                                            - DO NOT include any words, titles, or lettering
+                                            - Create a purely visual and symbolic representation
+                                            - Use imagery that represents the specific topics mentioned
+                                            - I like Studio Ghibli flavor if possible
+                                            - The image should work well as a podcast cover thumbnail
+                                            - Create a clean, professional design suitable for a podcast
+                                            - AGAIN, DO NOT INCLUDE ANY TEXT
+                                        """)
+
+
+def download_images(image_urls):
+    local_image_filenames = []
+    try:
+        if image_urls:
+            for image_url in image_urls:
+                unique_id = str(uuid.uuid4())
+                filename = f"podcast_banner_{unique_id}.png"
+                os.makedirs(PODCAST_IMG_DIR, exist_ok=True)
+                print(f"Downloading image: {filename}")
+                response = requests.get(image_url, timeout=30)
+                response.raise_for_status()
+                image_path = os.path.join(PODCAST_IMG_DIR, filename)
+                with open(image_path, "wb") as f:
+                    f.write(response.content)
+                local_image_filenames.append(filename)
+                print(f"Successfully downloaded: {filename}")
+    except requests.exceptions.RequestException as e:
+        print(f"Error downloading images (network): {e}")
+    except Exception as e:
+        print(f"Error downloading images: {e}")
+
+    return local_image_filenames
+
+
+def image_generation_agent_run(agent: Agent, query: str) -> str:
+    """
+    Image Generation Agent that takes the generated_script (internally from session_state) and creates a images for the given podcast script.
+
+    Args:
+        agent: The agent instance
+        query: any custom preferences for the image generation
+    Returns:
+        Response status
+    """
+    from services.internal_session_service import SessionService
+
+    session_id = agent.session_id
+    session = SessionService.get_session(session_id)
+    session_state = session["state"]
+    print("Image Generation Agent input: ", query)
+
+    try:
+        image_agent = Agent(
+            model=OpenAIChat(id="gpt-4o"),
+            tools=[DalleTools()],
+            description=IMAGE_GENERATION_AGENT_DESCRIPTION,
+            instructions=IMAGE_GENERATION_AGENT_INSTRUCTIONS,
+            markdown=True,
+            show_tool_calls=True,
+            session_id=agent.session_id,
+        )
+        image_agent.run(f"query: {query},\n podcast script: {json.dumps(session_state['generated_script'])}", session_id=agent.session_id)
+        images = image_agent.get_images()
+        image_urls = []
+        if images and isinstance(images, list):
+            for image_response in images:
+                image_url = image_response.url
+                image_urls.append(image_url)
+        local_image_filenames = download_images(image_urls)
+        session_state["banner_images"] = local_image_filenames
+        if local_image_filenames:
+            session_state["banner_url"] = local_image_filenames[0]
+    except Exception as e:
+        print(f"Error in Image Generation Agent: {e}")
+        return "Error in Image Generation Agent"
+    session_state["stage"] = "image"
+    SessionService.save_session(session_id, session_state)
+    return "Required banner images for the podcast are generated successfully."

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/agents/scrape_agent.py

@@ -0,0 +1,148 @@
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from pydantic import BaseModel, Field
+from dotenv import load_dotenv
+from tools.browser_crawler import create_browser_crawler
+from textwrap import dedent
+
+
+load_dotenv()
+
+
+class ScrapedContent(BaseModel):
+    url: str = Field(..., description="The URL of the search result")
+    description: str = Field(description="The description of the search result")
+    full_text: str = Field(
+        ...,
+        description="The full text of the given source URL, if not available or not applicable keep it empty",
+    )
+    published_date: str = Field(
+        ...,
+        description="The published date of the content in ISO format, if not available keep it empty",
+    )
+
+
+SCRAPE_AGENT_DESCRIPTION = "You are a helpful assistant that can scrape the URL for full content."
+SCRAPE_AGENT_INSTRUCTIONS = dedent("""
+    You are a content verification and formatting assistant.
+    
+    You will receive a batch of pre-scraped content from various URLs along with a search query.
+    Your job is to:
+    
+    1. VERIFY RELEVANCE: Ensure each piece of content is relevant to the given query
+    2. QUALITY CONTROL: Filter out low-quality, duplicate, or irrelevant content
+    3. FORMAT CONSISTENCY: Ensure all content follows a consistent format
+    4. LENGTH OPTIMIZATION: Keep content at reasonable length - not too long, not too short
+    5. CLEAN TEXT: Remove any formatting artifacts, ads, or navigation elements from scraped content
+    
+    For each piece of content, return:
+    - full_text: The cleaned, relevant text content (or empty if not relevant/low quality)
+    - published_date: The publication date in ISO format (or empty if not available)
+    
+    Note: Some content may be fallback descriptions (when scraping failed) - treat these appropriately and don't penalize them for being shorter.
+    
+    IMPORTANT: Focus on quality over quantity. It's better to return fewer high-quality, relevant pieces than many low-quality ones.
+    """)
+
+
+def crawl_urls_batch(search_results):
+    url_to_search_results = {}
+    unique_urls = []
+    for search_result in search_results:
+        if not search_result.get("url", False):
+            continue
+        if not search_result.get("is_scrapping_required", True):
+            continue
+        if not search_result.get('original_url'):
+            search_result['original_url'] = search_result['url']
+        url = search_result["url"]
+        if url not in url_to_search_results:
+            url_to_search_results[url] = []
+            unique_urls.append(url)
+        url_to_search_results[url].append(search_result)
+    browser_crawler = create_browser_crawler()
+    scraped_results = browser_crawler.scrape_urls(unique_urls)
+    url_to_scraped = {result["original_url"]: result for result in scraped_results}
+    updated_search_results = []
+    successful_scrapes = 0
+    failed_scrapes = 0
+    for search_result in search_results:
+        original_url = search_result["url"]
+        scraped = url_to_scraped.get(original_url, {})
+        updated_result = search_result.copy()
+        updated_result["original_url"] = original_url
+        if scraped.get("success", False):
+            updated_result["url"] = scraped.get("final_url", original_url)
+            updated_result["full_text"] = scraped.get("full_text", "")
+            updated_result["published_date"] = scraped.get("published_date", "")
+            successful_scrapes += 1
+        else:
+            updated_result["url"] = original_url
+            updated_result["full_text"] = search_result.get("description", "")
+            updated_result["published_date"] = ""
+            failed_scrapes += 1
+        updated_search_results.append(updated_result)
+    return updated_search_results, successful_scrapes, failed_scrapes
+
+
+def verify_content_with_agent(agent, query, search_results, use_agent=True):
+    if not use_agent:
+        return search_results
+    verified_search_results = []
+    for _, search_result in enumerate(search_results):
+        content_for_verification = {
+            "url": search_result["url"],
+            "description": search_result.get("description", ""),
+            "full_text": search_result["full_text"],
+            "published_date": search_result["published_date"],
+        }
+        search_result["agent_verified"] = False
+        try:
+            scrape_agent = Agent(
+                model=OpenAIChat(id="gpt-4o-mini"),
+                instructions=SCRAPE_AGENT_INSTRUCTIONS,
+                description=SCRAPE_AGENT_DESCRIPTION,
+                use_json_mode=True,
+                session_id=agent.session_id,
+                response_model=ScrapedContent,
+            )
+            response = scrape_agent.run(
+                f"Query: {query}\n"
+                f"Verify and format this scraped content. "
+                f"Keep content relevant to the query and ensure quality: {content_for_verification}",
+                session_id=agent.session_id,
+            )
+            verified_item = response.to_dict()["content"]
+            search_result["full_text"] = verified_item.get("full_text", search_result["full_text"])
+            search_result["published_date"] = verified_item.get("published_date", search_result["published_date"])
+            search_result["agent_verified"] = True
+        except Exception as _:
+            pass
+        verified_search_results.append(search_result)
+    return verified_search_results
+
+
+def scrape_agent_run(
+    agent: Agent,
+    query: str,
+) -> str:
+    """
+    Scrape Agent that takes the search_results (internaly from search_results) and scrapes each URL for full content, making sure those contents are of high quality and relevant to the topic.
+    Args:
+        agent: The agent instance
+        query: The search query
+    Returns:
+        Response status
+    """
+    print("Scrape Agent Input:", query)
+    session_id = agent.session_id
+    from services.internal_session_service import SessionService
+
+    session = SessionService.get_session(session_id)
+    current_state = session["state"]
+    updated_results, _, _ = crawl_urls_batch(current_state["search_results"])
+    verified_results = verify_content_with_agent(agent, query, updated_results, use_agent=False)
+    current_state["search_results"] = verified_results
+    SessionService.save_session(session_id, current_state)
+    has_results = "search_results" in current_state and current_state["search_results"]
+    return f"Scraped {len(current_state['search_results'])} sources with full content relevant to '{query}'{' and updated the full text and published date in the search_results items' if has_results else ''}."

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/agents/script_agent.py

@@ -0,0 +1,135 @@
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from pydantic import BaseModel, Field
+from typing import List, Optional
+from dotenv import load_dotenv
+from textwrap import dedent
+from datetime import datetime
+
+load_dotenv()
+
+
+class Dialog(BaseModel):
+    speaker: str = Field(..., description="The speaker name (SHOULD BE 'ALEX' OR 'MORGAN')")
+    text: str = Field(
+        ...,
+        description="The spoken text content for this speaker based on the requested langauge, default is English",
+    )
+
+
+class Section(BaseModel):
+    type: str = Field(..., description="The section type (intro, headlines, article, outro)")
+    title: Optional[str] = Field(None, description="Optional title for the section (required for article type)")
+    dialog: List[Dialog] = Field(..., description="List of dialog exchanges between speakers")
+
+
+class PodcastScript(BaseModel):
+    title: str = Field(..., description="The podcast episode title with date")
+    sections: List[Section] = Field(..., description="List of podcast sections (intro, headlines, articles, outro)")
+
+
+PODCAST_AGENT_DESCRIPTION = "You are a helpful assistant that can generate engaging podcast scripts for the given sources."
+PODCAST_AGENT_INSTRUCTIONS = dedent("""
+    You are a helpful assistant that can generate engaging podcast scripts for the given source content and query.
+    For given content, create an engaging podcast script that should be at least 15 minutes worth of content and your allowed enhance the script beyond given sources if you know something additional info will be interesting to the discussion or not enough conents available.
+    You use the provided sources to ground your podcast script generation process. Keep it engaging and interesting.
+    
+    IMPORTANT: Generate the entire script in the provided language. basically only text field needs to be in requested language,
+    
+    CONTENT GUIDELINES [THIS IS EXAMPLE YOU CAN CHANGE THE GUIDELINES ANYWAY BASED ON THE QUERY OR TOPIC DISCUSSED]:
+    - Provide insightful analysis that helps the audience understand the significance
+    - Include discussions on potential implications and broader context of each story
+    - Explain complex concepts in an accessible but thorough manner
+    - Make connections between current and relevant historical developments when applicable
+    - Provide comparisons and contrasts with similar stories or trends when relevant
+    
+    PERSONALITY NOTES [THIS IS EXAMPLE YOU CAN CHANGE THE PERSONALITY OF ALEX AND MORGAN ANYWAY BASED ON THE QUERY OR TOPIC DISCUSSED]:
+    - Alex is more analytical and fact-focused
+    * Should reference specific details and data points
+    * Should explain complex topics clearly
+    * Should identify key implications of stories
+    - Morgan is more focused on human impact, social context, and practical applications
+    * Should analyze broader implications
+    * Should consider ethical implications and real-world applications
+    - Include natural, conversational banter and smooth transitions between topics
+    - Each article discussion should go beyond the basic summary to provide valuable insights
+    - Maintain a conversational but informed tone that would appeal to a general audience
+    
+    IMPORTNAT:
+        - MAKE SURE PODCAST SCRIPS ARE AT LEAST 15 MINUTES LONG WHICH MEANS YOU NEED TO HAVE DETAILED DISCUSSIONS OFFCOURSE KEEP IT INTERESTING AND ENGAGING.
+    """)
+
+
+def format_search_results_for_podcast(
+    search_results: List[dict],
+) -> tuple[str, List[str]]:
+    created_at = datetime.now().strftime("%B %d, %Y at %I:%M %p")
+    structured_content = []
+    structured_content.append(f"PODCAST CREATION: {created_at}\n")
+    sources = []
+    for idx, search_result in enumerate(search_results):
+        try:
+            if search_result.get("confirmed", False):
+                sources.append(search_result["url"])
+                structured_content.append(
+                    f"""
+                                        SOURCE {idx + 1}:
+                                        Title: {search_result['title']}
+                                        URL: {search_result['url']}
+                                        Content: {search_result.get('full_text') or search_result.get('description', '')}
+                                        ---END OF SOURCE {idx + 1}---
+                                        """.strip()
+                )
+        except Exception as e:
+            print(f"Error processing search result: {e}")
+    content_texts = "\n\n".join(structured_content)
+    return content_texts, sources
+
+
+def podcast_script_agent_run(
+    agent: Agent,
+    query: str,
+    language_name: str,
+) -> str:
+    """
+    Podcast Script Agent that takes the search_results (internally from search_results) and creates a podcast script for the given query and language.
+
+    Args:
+        agent: The agent instance
+        query: The search query
+        language_name: The language the podcast script should be.
+    Returns:
+        Response status
+    """
+    from services.internal_session_service import SessionService
+    session_id = agent.session_id
+    session = SessionService.get_session(session_id)
+    session_state = session["state"]
+    
+    print("Podcast Script Agent Input:", query)
+    content_texts, sources = format_search_results_for_podcast(session_state.get("search_results", []))
+    if not content_texts:
+        return "No confirmed sources found to generate podcast script."
+
+    podcast_script_agent = Agent(
+        model=OpenAIChat(id="gpt-4o-mini"),
+        instructions=PODCAST_AGENT_INSTRUCTIONS,
+        description=PODCAST_AGENT_DESCRIPTION,
+        use_json_mode=True,
+        response_model=PodcastScript,
+        session_id=agent.session_id,
+    )
+    response = podcast_script_agent.run(
+        f"query: {query}\n language_name: {language_name}\n content_texts: {content_texts}\n, IMPORTANT: texts should be in {language_name} language.",
+        session_id=agent.session_id,
+    )
+    response_dict = response.to_dict()
+    response_dict = response_dict["content"]
+    response_dict["sources"] = sources
+    session_state["generated_script"] = response_dict
+    session_state['stage'] = 'script'
+    SessionService.save_session(session_id, session_state)
+
+    if not session_state["generated_script"] and not session_state["generated_script"].get("sections"):
+        return "Failed to generate podcast script."
+    return f"Generated podcast script for '{query}' with {len(sources)} confirmed sources."

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/agents/search_agent.py

@@ -0,0 +1,105 @@
+from typing import List
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from pydantic import BaseModel, Field
+from dotenv import load_dotenv
+from agno.tools.duckduckgo import DuckDuckGoTools
+from textwrap import dedent
+from tools.wikipedia_search import wikipedia_search
+from tools.google_news_discovery import google_news_discovery_run
+from tools.jikan_search import jikan_search
+from tools.embedding_search import embedding_search
+from tools.social_media_search import social_media_search, social_media_trending_search
+from tools.search_articles import search_articles
+from tools.web_search import run_browser_search
+
+
+load_dotenv()
+
+
+class ReturnItem(BaseModel):
+    url: str = Field(..., description="The URL of the search result")
+    title: str = Field(..., description="The title of the search result")
+    description: str = Field(..., description="A brief description or summary of the search result content")
+    source_name: str = Field(
+        ...,
+        description="The name/type of the source (e.g., 'wikipedia', 'general', or any reputable source tag)",
+    )
+    tool_used: str = Field(
+        ...,
+        description="The tools used to generate the search results, unknown if not used or not applicable",
+    )
+    published_date: str = Field(
+        ...,
+        description="The published date of the content in ISO format, if not available keep it empty",
+    )
+    is_scrapping_required: bool = Field(
+        ...,
+        description="Set to True if the content need scraping, False otherwise, default keep it True if not sure",
+    )
+
+
+class SearchResults(BaseModel):
+    items: List[ReturnItem] = Field(..., description="A list of search result items")
+
+
+SEARCH_AGENT_DESCRIPTION = "You are a helpful assistant that can search the web for information."
+SEARCH_AGENT_INSTRUCTIONS = dedent("""
+    You are a helpful assistant that can search the web or any other sources for information.
+    You should create topic for the search from the given query instead of blindly apply the query to the search tools.
+    For a given topic, your job is to search the web or any other sources and return the top 5 to 10 sources about the topic.
+    Keep the search sources of high quality and reputable, and sources should be relevant to the asked topic.
+    Sources should be from diverse platforms with no duplicates.
+    IMPORTANT: User queries might be fuzzy or misspelled. Understand the user's intent and act accordingly.
+    IMPORTANT: The output source_name field can be one of ["wikipedia", "general", or any source tag used"].
+    IMPORTANT: You have access to different search tools use them when appropriate which one is best for the given search query. Don't use particular tool if not required.
+    IMPORTANT: Make sure you are able to detect what tool to use and use it available tool tags = ["google_news_discovery", "duckduckgo", "wikipedia_search", "jikan_search", "social_media_search", "social_media_trending_search", "browser_search", "unknown"].
+    IMPORTANT: If query is news related please prefere google news over other news tools.
+    IMPORTANT: If returned sources are not of high quality or not relevant to the asked topic, don't include them in the returned sources.
+    IMPORTANT: Never include dates to the search query unless user explicitly asks for it.
+    IMPORTANT: You are allowed to use appropriate tools to get the best results even the single tool return enough results diverse check is better.
+    IMPORTANT: You have access to browser agent for searching as well use it when other source can't suitable for the given tasks but input should detailed instruction to the run_browser_search agent to get the best results and also use it conservatively because it's expensive process.
+    """)
+
+
+def search_agent_run(agent: Agent, query: str) -> str:
+    """
+    Search Agent which searches the web and other sources for relevant sources about the given topic or query.
+    Args:
+        agent: The agent instance
+        query: The search query
+    Returns:
+        A formatted string response with the search results (link and gist only)
+    """
+    print("Search Agent Input:", query)
+    session_id = agent.session_id
+    from services.internal_session_service import SessionService
+
+    session = SessionService.get_session(session_id)
+    current_state = session["state"]
+    search_agent = Agent(
+        model=OpenAIChat(id="gpt-4o-mini"),
+        instructions=SEARCH_AGENT_INSTRUCTIONS,
+        description=SEARCH_AGENT_DESCRIPTION,
+        use_json_mode=True,
+        response_model=SearchResults,
+        tools=[
+            google_news_discovery_run,
+            DuckDuckGoTools(),
+            wikipedia_search,
+            jikan_search,
+            embedding_search,
+            social_media_search,
+            social_media_trending_search,
+            search_articles,
+            run_browser_search,
+        ],
+        session_id=session_id,
+    )
+    response = search_agent.run(query, session_id=session_id)
+    response_dict = response.to_dict()
+    current_state["stage"] = "search"
+    current_state["search_results"] = response_dict["content"]["items"]
+    SessionService.save_session(session_id, current_state)
+    has_results = "search_results" in current_state and current_state["search_results"]
+    return f"Found {len(response_dict['content']['items'])} sources about {query} {'and added to the search_results' if has_results else ''}"

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/bootstrap_demo.py

@@ -0,0 +1,75 @@
+import os
+import sys
+import tempfile
+import requests
+import zipfile
+from tqdm import tqdm
+
+DEMO_URL = "https://github.com/arun477/beifong/releases/download/v1.2.0/demo_content.zip"
+TARGET_DIRS = ["databases", "podcasts"]
+
+
+def ensure_empty(dir_path):
+    """check if directory is empty (or create it). exit if not empty."""
+    if os.path.exists(dir_path):
+        if os.listdir(dir_path):
+            print(f"✗ '{dir_path}' is not empty. aborting.")
+            sys.exit(1)
+    else:
+        os.makedirs(dir_path, exist_ok=True)
+
+
+def download_file(url, dest_path):
+    """stream-download a file from url to dest_path with progress bar."""
+    print("↓ downloading demo content...")
+    
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+    
+    
+    total_size = int(response.headers.get('content-length', 0))
+    block_size = 8192
+    
+    with open(dest_path, "wb") as f:
+        with tqdm(total=total_size, unit='B', unit_scale=True, desc="Download Progress") as pbar:
+            for chunk in response.iter_content(chunk_size=block_size):
+                if chunk:
+                    f.write(chunk)
+                    pbar.update(len(chunk))
+
+
+def extract_zip(zip_path, extract_to):
+    """extract zip file into extract_to (project root) with progress bar."""
+    print("✂ extracting demo content...")
+    
+    with zipfile.ZipFile(zip_path, "r") as zip_ref:
+        total_files = len(zip_ref.infolist())
+        
+        with tqdm(total=total_files, desc="Extraction Progress") as pbar:
+            for file in zip_ref.infolist():
+                zip_ref.extract(file, extract_to)
+                pbar.update(1)
+
+
+def main():
+    print("populating demo folders…")
+    for d in TARGET_DIRS:
+        ensure_empty(d)
+    
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_zip = os.path.join(tmp, "demo_content.zip")
+        download_file(DEMO_URL, tmp_zip)
+        extract_zip(tmp_zip, os.getcwd())
+    
+    print("✓ demo folders populated successfully.")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("\n✗ Download cancelled by user.")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n✗ Error: {e}")
+        sys.exit(1)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/celery_worker.py

@@ -0,0 +1,13 @@
+from services.celery_tasks import app
+
+worker_options = [
+    "worker",
+    "--loglevel=INFO",
+    "--concurrency=4",
+    "--hostname=beifong_worker@%h",
+    "--pool=threads",
+]
+
+if __name__ == "__main__":
+    print("Starting Beifong podcast agent workers...")
+    app.worker_main(worker_options)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/db/agent_config_v2.py

@@ -0,0 +1,169 @@
+from agno.storage.sqlite import SqliteStorage
+from db.config import get_agent_session_db_path
+import json
+
+AGENT_MODEL = "gpt-4o"
+AVAILABLE_LANGS = [
+    {"code": "en", "name": "English"},
+    {"code": "zh", "name": "Chinese"},
+    {"code": "de", "name": "German"},
+    {"code": "es", "name": "Spanish"},
+    {"code": "ru", "name": "Russian"},
+    {"code": "ko", "name": "Korean"},
+    {"code": "fr", "name": "French"},
+    {"code": "ja", "name": "Japanese"},
+    {"code": "pt", "name": "Portuguese"},
+    {"code": "tr", "name": "Turkish"},
+    {"code": "pl", "name": "Polish"},
+    {"code": "ca", "name": "Catalan"},
+    {"code": "nl", "name": "Dutch"},
+    {"code": "ar", "name": "Arabic"},
+    {"code": "sv", "name": "Swedish"},
+    {"code": "it", "name": "Italian"},
+    {"code": "id", "name": "Indonesian"},
+    {"code": "hi", "name": "Hindi"},
+    {"code": "fi", "name": "Finnish"},
+    {"code": "vi", "name": "Vietnamese"},
+    {"code": "he", "name": "Hebrew"},
+    {"code": "uk", "name": "Ukrainian"},
+    {"code": "el", "name": "Greek"},
+    {"code": "ms", "name": "Malay"},
+    {"code": "cs", "name": "Czech"},
+    {"code": "ro", "name": "Romanian"},
+    {"code": "da", "name": "Danish"},
+    {"code": "hu", "name": "Hungarian"},
+    {"code": "ta", "name": "Tamil"},
+    {"code": "no", "name": "Norwegian"},
+    {"code": "th", "name": "Thai"},
+    {"code": "ur", "name": "Urdu"},
+    {"code": "hr", "name": "Croatian"},
+    {"code": "bg", "name": "Bulgarian"},
+    {"code": "lt", "name": "Lithuanian"},
+    {"code": "la", "name": "Latin"},
+    {"code": "mi", "name": "Maori"},
+    {"code": "ml", "name": "Malayalam"},
+    {"code": "cy", "name": "Welsh"},
+    {"code": "sk", "name": "Slovak"},
+    {"code": "te", "name": "Telugu"},
+    {"code": "fa", "name": "Persian"},
+    {"code": "lv", "name": "Latvian"},
+    {"code": "bn", "name": "Bengali"},
+    {"code": "sr", "name": "Serbian"},
+    {"code": "az", "name": "Azerbaijani"},
+    {"code": "sl", "name": "Slovenian"},
+    {"code": "kn", "name": "Kannada"},
+    {"code": "et", "name": "Estonian"},
+    {"code": "mk", "name": "Macedonian"},
+    {"code": "br", "name": "Breton"},
+    {"code": "eu", "name": "Basque"},
+    {"code": "is", "name": "Icelandic"},
+    {"code": "hy", "name": "Armenian"},
+    {"code": "ne", "name": "Nepali"},
+    {"code": "mn", "name": "Mongolian"},
+    {"code": "bs", "name": "Bosnian"},
+    {"code": "kk", "name": "Kazakh"},
+    {"code": "sq", "name": "Albanian"},
+    {"code": "sw", "name": "Swahili"},
+    {"code": "gl", "name": "Galician"},
+    {"code": "mr", "name": "Marathi"},
+    {"code": "pa", "name": "Punjabi"},
+    {"code": "si", "name": "Sinhala"},
+    {"code": "km", "name": "Khmer"},
+    {"code": "sn", "name": "Shona"},
+    {"code": "yo", "name": "Yoruba"},
+    {"code": "so", "name": "Somali"},
+    {"code": "af", "name": "Afrikaans"},
+    {"code": "oc", "name": "Occitan"},
+    {"code": "ka", "name": "Georgian"},
+    {"code": "be", "name": "Belarusian"},
+    {"code": "tg", "name": "Tajik"},
+    {"code": "sd", "name": "Sindhi"},
+    {"code": "gu", "name": "Gujarati"},
+    {"code": "am", "name": "Amharic"},
+    {"code": "yi", "name": "Yiddish"},
+    {"code": "lo", "name": "Lao"},
+    {"code": "uz", "name": "Uzbek"},
+    {"code": "fo", "name": "Faroese"},
+    {"code": "ht", "name": "Haitian creole"},
+    {"code": "ps", "name": "Pashto"},
+    {"code": "tk", "name": "Turkmen"},
+    {"code": "nn", "name": "Nynorsk"},
+    {"code": "mt", "name": "Maltese"},
+    {"code": "sa", "name": "Sanskrit"},
+    {"code": "lb", "name": "Luxembourgish"},
+    {"code": "my", "name": "Myanmar"},
+    {"code": "bo", "name": "Tibetan"},
+    {"code": "tl", "name": "Tagalog"},
+    {"code": "mg", "name": "Malagasy"},
+    {"code": "as", "name": "Assamese"},
+    {"code": "tt", "name": "Tatar"},
+    {"code": "haw", "name": "Hawaiian"},
+    {"code": "ln", "name": "Lingala"},
+    {"code": "ha", "name": "Hausa"},
+    {"code": "ba", "name": "Bashkir"},
+    {"code": "jw", "name": "Javanese"},
+    {"code": "su", "name": "Sundanese"},
+    {"code": "yue", "name": "Cantonese"},
+]
+
+TOGGLE_UI_STATES = [
+    "show_sources_for_selection",
+    "show_script_for_confirmation",
+    "show_banner_for_confirmation",
+    "show_audio_for_confirmation",
+]
+
+AGENT_DESCRIPTION = "You are name is Beifong, a helpful assistant that guides users to choose best sources for the podcast and allow them to generate the podcast script."
+
+# sacred commandments, touch these with devotion.
+AGENT_INSTRUCTIONS = [
+    "Guide users to choose the best sources for the podcast and allow them to generate the podcast script and images for the podcast and audio for the podcast.",
+    "1. Make sure you get the intent of the topic from the user. It can be fuzzy and contain spelling mistakes from the user, so act as intent detection and get clarification only if needed.",
+    "1a. Keep this phase as quick as possible. Try to avoid too many back and forth conversations. Try to infer the intent if you're confident you can go right to the next search phase without confirming with the user. The less back and forth, the better for the user experience.",
+    "2. Once you understand the intent of the topic, use the available search tools (we have search agent where you can pass the query and along with appropriate prompt search agent has lot of search tools and api access which you don't have so you can instruct if needed) to get diverse and high-quality sources for the topic.  and also make sure give appropriate short title for the chat and update the chat title using update_chat_title tool",
+    "2a. Once we receive the search results. do the full scraping using appropriate scraping tool to get the full text of the each source.",
+    "2b. Don't do back and forth during this source collection process with the user. Either you have the results or not, then inform the user and ask the user if they want to try again or give more details about the topic.",
+    "3. Once you have the results, ask the user to pick which sources they want to use for the podcast. You don't have to list out the found sources; just tell them to pick from the list of sources that will be visible in the UI.",
+    "4. User do the selection by selecting the index of sources from the list of sources that will be visible in the UI. so response will be a list of indices of sources selected by the user. sometime user will also send prefered language for the podcast along with the selection."
+    "4a. You have to use user_source_selection tool to update the user selection. and after this point immediately switch off any UI states.",
+    "4b. If user sends prefered language for the podcast along with the selection, you have to use update_language tool to update the user language if not leave it default is english. and after this point immediately switch off any UI states.",
+    "5. Once you we have the confimed selection from the user let's immediatly call the podcast script agent to generate the podcast script for the given sources and query and perfered language (pass full lanage name).",
+    "5a. Once podcast script is ready switch on the podcast_script UI state and so user will see if the generated podcast script is fine or not, you dont' have to show the script active podasshow_script_for_confirmation will take care of it.",
+    "6. Once you got the confirmation from the user (throug UI) let's immediatly call the image generation agent to generate the image for the given podcast script.",
+    "6a. Once image generation successfully generated switch on the image UI state and so user will see if the generated image is fine or not, you just ask user to confirm the image through UI. show_banner_for_confirmation will take care of it."
+    "7. Once you got the confirmation from the user (throug UI) let's immediatly call the audio generation agent to generate the audio for the given podcast script.",
+    "7a. Once audio generation successfully generated switch on the audio UI state and so user will see if the generated audio is fine or not, you just ask user to confirm the audio through UI. show_audio_for_confirmation will take care of it."
+    "8. Once you got the confirmation from the user for audio (throug UI) let's immediatly call the mark_session_finished tool to mark the session as finished and if finish is successful then no further conversation are allowed and only new session can be started.",
+    "8a. It's important mark_session_finished should be called only when we have all the stages search->selection->script->image->audio are completed."
+    "APPENDIX:",
+    "1. You can enable appropriate UI states using the ui_manager tool, which takes [state_type, active] as input, and it takes care of the appropriate UI state for activating appropriate UI state.",
+    f"1a. Available UI state types: {TOGGLE_UI_STATES}",
+    "1b. During the conversation, at any place you feel a UI state is not necessary, you can disable it using the ui_manager tool by setting active to False. For switching off all states, pass all to False.",
+    f"2. Supported Languges: {json.dumps(AVAILABLE_LANGS)}",
+    "3. Search Agent has a lot off tools, so you can instruct the search query as prompt to get the best results as because search agent has lot of tools you can instruct instead of directly passing the query to search agent when required.",
+    "4. You are not allowed to include year or date in your seach query construction for the search agent unless that request explicilty with yeear or date  come for the users.",
+]
+DB_PATH = "databases"
+PODCAST_DIR = "podcasts"
+PODCAST_IMG_DIR = PODCAST_DIR + "/images"
+PODCAST_AUIDO_DIR = PODCAST_DIR + "/audio"
+PODCAST_RECORDINGS_DIR = PODCAST_DIR + "/recordings"
+
+INITIAL_SESSION_STATE = {
+    "search_results": [],
+    "show_sources_for_selection": False,
+    "show_script_for_confirmation": False,
+    "generated_script": {},
+    "selected_language": {"code": "en", "name": "English"},
+    "available_languages": AVAILABLE_LANGS,
+    "banner_images": [],
+    "banner_url": "",
+    "audio_url": "",
+    "title": "Untitled",
+    "created_at": "",
+    "finished": False,
+    "show_banner_for_confirmation": False,
+    "show_audio_for_confirmation": False,
+}
+
+STORAGE = SqliteStorage(table_name="podcast_sessions", db_file=get_agent_session_db_path())

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/db/articles.py

@@ -0,0 +1,276 @@
+import json
+from datetime import datetime
+from .connection import db_connection, execute_query
+
+
+def store_crawled_article(tracking_db_path, entry, raw_content, metadata):
+    metadata_json = json.dumps(metadata)
+    query = """
+    INSERT INTO crawled_articles 
+    (entry_id, source_id, feed_id, title, url, published_date, raw_content, metadata)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    """
+    try:
+        params = (
+            entry["id"],
+            entry.get("source_id"),
+            entry.get("feed_id"),
+            entry.get("title", ""),
+            entry.get("link", ""),
+            entry.get("published_date", datetime.now().isoformat()),
+            raw_content,
+            metadata_json,
+        )
+        execute_query(tracking_db_path, query, params)
+        return True
+    except Exception:
+        return False
+
+
+def update_entry_status(tracking_db_path, entry_id, status):
+    query = """
+    UPDATE feed_entries
+    SET crawl_attempts = crawl_attempts + 1, crawl_status = ?
+    WHERE id = ?
+    """
+    return execute_query(tracking_db_path, query, (status, entry_id))
+
+
+def get_unprocessed_articles(tracking_db_path, limit=5, max_attempts=1):
+    reset_stuck_articles(tracking_db_path)
+    query = """
+    SELECT id, entry_id, source_id, feed_id, title, url, published_date, raw_content, metadata, ai_attempts
+    FROM crawled_articles
+    WHERE (ai_status = 'pending' OR ai_status = 'error')
+          AND ai_attempts < ?
+          AND processed = 0
+    ORDER BY published_date DESC
+    LIMIT ?
+    """
+    articles = execute_query(tracking_db_path, query, (max_attempts, limit), fetch=True)
+    for article in articles:
+        if article.get("metadata"):
+            try:
+                article["metadata"] = json.loads(article["metadata"])
+            except json.JSONDecodeError:
+                article["metadata"] = {}
+    if articles:
+        article_ids = [a["id"] for a in articles]
+        mark_articles_as_processing(tracking_db_path, article_ids)
+    return articles
+
+
+def reset_stuck_articles(tracking_db_path):
+    query = """
+    UPDATE crawled_articles 
+    SET ai_status = 'pending' 
+    WHERE ai_status = 'processing'
+    """
+    return execute_query(tracking_db_path, query)
+
+
+def mark_articles_as_processing(tracking_db_path, article_ids):
+    if not article_ids:
+        return 0
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        placeholders = ",".join(["?"] * len(article_ids))
+        query = f"""
+        UPDATE crawled_articles 
+        SET ai_status = 'processing' 
+        WHERE id IN ({placeholders})
+        """
+        cursor.execute(query, article_ids)
+        conn.commit()
+        return cursor.rowcount
+
+
+def save_article_categories(tracking_db_path, article_id, categories):
+    if not categories:
+        return 0
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute(
+            """
+        DELETE FROM article_categories
+        WHERE article_id = ?
+        """,
+            (article_id,),
+        )
+        count = 0
+        for category in categories:
+            try:
+                cursor.execute(
+                    """
+                INSERT INTO article_categories (article_id, category_name)
+                VALUES (?, ?)
+                """,
+                    (article_id, category.lower().strip()),
+                )
+                count += 1
+            except Exception as _:
+                pass
+        conn.commit()
+        return count
+
+
+def get_article_categories(tracking_db_path, article_id):
+    query = """
+    SELECT category_name
+    FROM article_categories
+    WHERE article_id = ?
+    """
+    results = execute_query(tracking_db_path, query, (article_id,), fetch=True)
+    return [row["category_name"] for row in results]
+
+
+def update_article_status(tracking_db_path, article_id, results=None, success=False, error_message=None):
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute(
+            """
+        UPDATE crawled_articles
+        SET ai_attempts = ai_attempts + 1
+        WHERE id = ?
+        """,
+            (article_id,),
+        )
+        if success and results:
+            categories = []
+            if "categories" in results:
+                if isinstance(results["categories"], str):
+                    try:
+                        categories = json.loads(results["categories"])
+                    except json.JSONDecodeError:
+                        categories = [c.strip() for c in results["categories"].split(",") if c.strip()]
+                elif isinstance(results["categories"], list):
+                    categories = results["categories"]
+            cursor.execute(
+                """
+            UPDATE crawled_articles
+            SET summary = ?, content = ?, processed = 1, ai_status = 'success'
+            WHERE id = ?
+            """,
+                (results.get("summary", ""), results.get("content", ""), article_id),
+            )
+            conn.commit()
+            if categories:
+                save_article_categories(tracking_db_path, article_id, categories)
+        else:
+            cursor.execute(
+                """
+            UPDATE crawled_articles
+            SET ai_status = 'error', ai_error = ?
+            WHERE id = ?
+            """,
+                (error_message, article_id),
+            )
+            cursor.execute(
+                """
+            UPDATE crawled_articles
+            SET ai_status = 'failed'
+            WHERE id = ? AND ai_attempts >= 3
+            """,
+                (article_id,),
+            )
+            conn.commit()
+        return cursor.rowcount
+
+
+def get_articles_by_date_range(tracking_db_path, start_date=None, end_date=None, limit=None, offset=0):
+    query_parts = [
+        "SELECT ca.id, ca.feed_id, ca.source_id, ca.title, ca.url, ca.published_date,",
+        "ca.summary, ca.content",
+        "FROM crawled_articles ca",
+        "WHERE ca.processed = 1",
+        "AND ca.ai_status = 'success'",
+    ]
+    query_params = []
+    if start_date:
+        query_parts.append("AND ca.published_date >= ?")
+        query_params.append(start_date)
+    if end_date:
+        query_parts.append("AND ca.published_date <= ?")
+        query_params.append(end_date)
+    query_parts.append("ORDER BY ca.published_date DESC")
+    if limit is not None:
+        query_parts.append("LIMIT ? OFFSET ?")
+        query_params.append(limit)
+        query_params.append(offset)
+    query = " ".join(query_parts)
+    return execute_query(tracking_db_path, query, tuple(query_params), fetch=True)
+
+
+def get_article_by_id(tracking_db_path, article_id):
+    query = """
+    SELECT id, entry_id, source_id, feed_id, title, url, published_date, 
+           raw_content, content, summary, metadata, ai_status, ai_error, 
+           ai_attempts, crawled_date, processed
+    FROM crawled_articles
+    WHERE id = ?
+    """
+    article = execute_query(tracking_db_path, query, (article_id,), fetch=True, fetch_one=True)
+    if article:
+        if article.get("metadata"):
+            try:
+                article["metadata"] = json.loads(article["metadata"])
+            except json.JSONDecodeError:
+                article["metadata"] = {}
+        article["categories"] = get_article_categories(tracking_db_path, article_id)
+    return article
+
+
+def get_articles_by_category(tracking_db_path, category, limit=20, offset=0):
+    query = """
+    SELECT ca.id, ca.title, ca.url, ca.published_date, ca.summary
+    FROM crawled_articles ca
+    JOIN article_categories ac ON ca.id = ac.article_id
+    WHERE ac.category_name = ?
+    AND ca.processed = 1
+    AND ca.ai_status = 'success'
+    ORDER BY ca.published_date DESC
+    LIMIT ? OFFSET ?
+    """
+    return execute_query(tracking_db_path, query, (category, limit, offset), fetch=True)
+
+
+def get_article_stats(tracking_db_path):
+    query = """
+    SELECT 
+        COUNT(*) as total_articles,
+        SUM(CASE WHEN processed = 1 THEN 1 ELSE 0 END) as processed_articles,
+        SUM(CASE WHEN ai_status = 'pending' THEN 1 ELSE 0 END) as pending_articles,
+        SUM(CASE WHEN ai_status = 'processing' THEN 1 ELSE 0 END) as processing_articles,
+        SUM(CASE WHEN ai_status = 'success' THEN 1 ELSE 0 END) as success_articles,
+        SUM(CASE WHEN ai_status = 'error' THEN 1 ELSE 0 END) as error_articles,
+        SUM(CASE WHEN ai_status = 'failed' THEN 1 ELSE 0 END) as failed_articles
+    FROM crawled_articles
+    """
+    return execute_query(tracking_db_path, query, fetch=True, fetch_one=True)
+
+
+def get_categories_with_counts(tracking_db_path, limit=20):
+    query = """
+    SELECT category_name, COUNT(*) as article_count
+    FROM article_categories
+    GROUP BY category_name
+    ORDER BY article_count DESC
+    LIMIT ?
+    """
+    return execute_query(tracking_db_path, query, (limit,), fetch=True)
+
+
+def get_articles_with_source_info(tracking_db_path, limit=20, offset=0):
+    query = """
+    SELECT ca.id, ca.title, ca.url, ca.published_date, ca.summary, 
+           ft.feed_url, s.name as source_name
+    FROM crawled_articles ca
+    LEFT JOIN feed_tracking ft ON ca.feed_id = ft.feed_id
+    LEFT JOIN source_feeds sf ON ca.feed_id = sf.id
+    LEFT JOIN sources s ON sf.source_id = s.id
+    WHERE ca.processed = 1 
+    AND ca.ai_status = 'success'
+    ORDER BY ca.published_date DESC
+    LIMIT ? OFFSET ?
+    """
+    return execute_query(tracking_db_path, query, (limit, offset), fetch=True)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/db/config.py

@@ -0,0 +1,71 @@
+import os
+from pathlib import Path
+from dotenv import load_dotenv
+
+env_path = Path(".") / ".env"
+load_dotenv(dotenv_path=env_path)
+DEFAULT_DB_PATHS = {
+    "sources_db": "databases/sources.db",
+    "tracking_db": "databases/feed_tracking.db",
+    "podcasts_db": "databases/podcasts.db",
+    "tasks_db": "databases/tasks.db",
+    "agent_session_db": "databases/agent_sessions.db",
+    "faiss_index_db": "databases/faiss/article_index.faiss",
+    "faiss_mapping_file": "databases/faiss/article_id_map.npy",
+    "internal_sessions_db": "databases/internal_sessions.db",
+    "social_media_db": "databases/social_media.db",
+    "slack_sessions_db": "databases/slack_sessions.db",
+}
+
+
+def get_db_path(db_name):
+    env_var = f"{db_name.upper()}_PATH"
+    path = os.environ.get(env_var, DEFAULT_DB_PATHS.get(db_name))
+    db_dir = os.path.dirname(path)
+    os.makedirs(db_dir, exist_ok=True)
+    return path
+
+
+def get_sources_db_path():
+    return get_db_path("sources_db")
+
+
+def get_tracking_db_path():
+    return get_db_path("tracking_db")
+
+
+def get_podcasts_db_path():
+    return get_db_path("podcasts_db")
+
+
+def get_tasks_db_path():
+    return get_db_path("tasks_db")
+
+
+def get_agent_session_db_path():
+    return get_db_path("agent_session_db")
+
+
+def get_faiss_db_path():
+    return get_db_path("faiss_index_db"), get_db_path("faiss_mapping_file")
+
+
+def get_internal_sessions_db_path():
+    return get_db_path("internal_sessions_db")
+
+
+def get_social_media_db_path():
+    return get_db_path("social_media_db")
+
+
+def get_browser_session_path():
+    return "browsers/playwright_persistent_profile"
+
+def get_slack_sessions_db_path():
+    return get_db_path("slack_sessions_db")
+
+DB_PATH = "databases"
+PODCAST_DIR = "podcasts"
+PODCAST_IMG_DIR = PODCAST_DIR + "/images"
+PODCAST_AUIDO_DIR = PODCAST_DIR + "/audio"
+PODCAST_RECORDINGS_DIR = PODCAST_DIR + "/recordings"

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/db/connection.py

@@ -0,0 +1,27 @@
+import sqlite3
+from contextlib import contextmanager
+
+
+@contextmanager
+def db_connection(db_path):
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+
+def execute_query(db_path, query, params=(), fetch=False, fetch_one=False):
+    with db_connection(db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute(query, params)
+
+        if fetch_one:
+            result = cursor.fetchone()
+            return dict(result) if result else None
+        elif fetch:
+            return [dict(row) for row in cursor.fetchall()]
+        else:
+            conn.commit()
+            return cursor.lastrowid

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/db/feeds.py

@@ -0,0 +1,179 @@
+from datetime import datetime
+import sqlite3
+from .connection import db_connection, execute_query
+
+
+def get_active_feeds(sources_db_path, limit=None, offset=0):
+    if limit:
+        query = """
+        SELECT sf.id, sf.source_id, sf.feed_url, sf.feed_type, sf.last_crawled, 
+               s.name as source_name
+        FROM source_feeds sf
+        JOIN sources s ON sf.source_id = s.id
+        WHERE sf.is_active = 1 AND s.is_active = 1
+        LIMIT ? OFFSET ?
+        """
+        return execute_query(sources_db_path, query, (limit, offset), fetch=True)
+    else:
+        query = """
+        SELECT sf.id, sf.source_id, sf.feed_url, sf.feed_type, sf.last_crawled, 
+               s.name as source_name
+        FROM source_feeds sf
+        JOIN sources s ON sf.source_id = s.id
+        WHERE sf.is_active = 1 AND s.is_active = 1
+        """
+        return execute_query(sources_db_path, query, fetch=True)
+
+
+def count_active_feeds(sources_db_path):
+    query = """
+    SELECT COUNT(*) as count
+    FROM source_feeds sf
+    JOIN sources s ON sf.source_id = s.id
+    WHERE sf.is_active = 1 AND s.is_active = 1
+    """
+    result = execute_query(sources_db_path, query, fetch=True, fetch_one=True)
+    return result["count"] if result else 0
+
+
+def get_feed_tracking_info(tracking_db_path, feed_id):
+    query = "SELECT * FROM feed_tracking WHERE feed_id = ?"
+    return execute_query(tracking_db_path, query, (feed_id,), fetch=True, fetch_one=True)
+
+
+def update_feed_tracking(tracking_db_path, feed_id, etag, modified, entry_hash):
+    query = """
+    UPDATE feed_tracking 
+    SET last_processed = ?, last_etag = ?, last_modified = ?, entry_hash = ?
+    WHERE feed_id = ?
+    """
+    params = (datetime.now().isoformat(), etag, modified, entry_hash, feed_id)
+    return execute_query(tracking_db_path, query, params)
+
+
+def store_feed_entries(tracking_db_path, feed_id, source_id, entries):
+    count = 0
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        for entry in entries:
+            try:
+                cursor.execute(
+                    """
+                INSERT INTO feed_entries 
+                (feed_id, source_id, entry_id, title, link, published_date, content, summary)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                    (
+                        feed_id,
+                        source_id,
+                        entry.get("entry_id", ""),
+                        entry.get("title", ""),
+                        entry.get("link", ""),
+                        entry.get("published_date", datetime.now().isoformat()),
+                        entry.get("content", ""),
+                        entry.get("summary", ""),
+                    ),
+                )
+                count += 1
+            except sqlite3.IntegrityError:
+                pass
+        conn.commit()
+    return count
+
+
+def update_tracking_info(tracking_db_path, feeds):
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        for feed in feeds:
+            cursor.execute(
+                """
+            INSERT OR IGNORE INTO feed_tracking 
+            (feed_id, source_id, feed_url, last_processed)
+            VALUES (?, ?, ?, NULL)
+            """,
+                (feed["id"], feed["source_id"], feed["feed_url"]),
+            )
+        conn.commit()
+
+
+def get_uncrawled_entries(tracking_db_path, limit=20, max_attempts=3):
+    reset_stuck_entries(tracking_db_path)
+    query = """
+    SELECT e.id, e.feed_id, e.source_id, e.title, e.link, e.published_date,
+           e.crawl_attempts, e.entry_id as original_entry_id
+    FROM feed_entries e
+    WHERE (e.crawl_status = 'pending' OR e.crawl_status = 'failed') 
+          AND e.crawl_attempts < ?
+          AND e.link IS NOT NULL
+          AND e.link != ''
+          AND NOT EXISTS (
+              SELECT 1 FROM crawled_articles ca WHERE ca.url = e.link
+          )
+    ORDER BY e.published_date DESC
+    LIMIT ?
+    """
+    entries = execute_query(tracking_db_path, query, (max_attempts, limit), fetch=True)
+    if entries:
+        entry_ids = [e["id"] for e in entries]
+        mark_entries_as_processing(tracking_db_path, entry_ids)
+    return entries
+
+
+def reset_stuck_entries(tracking_db_path):
+    query = """
+    UPDATE feed_entries 
+    SET crawl_status = 'pending' 
+    WHERE crawl_status = 'processing'
+    """
+    return execute_query(tracking_db_path, query)
+
+
+def mark_entries_as_processing(tracking_db_path, entry_ids):
+    if not entry_ids:
+        return 0
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        placeholders = ",".join(["?"] * len(entry_ids))
+        query = f"""
+        UPDATE feed_entries 
+        SET crawl_status = 'processing' 
+        WHERE id IN ({placeholders})
+        """
+        cursor.execute(query, entry_ids)
+        conn.commit()
+        return cursor.rowcount
+
+
+def ensure_feed_tracking_exists(tracking_db_path, feed_id, source_id, feed_url):
+    query = """
+    INSERT OR IGNORE INTO feed_tracking 
+    (feed_id, source_id, feed_url, last_processed)
+    VALUES (?, ?, ?, NULL)
+    """
+    return execute_query(tracking_db_path, query, (feed_id, source_id, feed_url))
+
+
+def get_feed_sources_with_categories(sources_db_path):
+    query = """
+    SELECT sf.id, sf.source_id, sf.feed_url, sf.feed_type, sf.last_crawled, 
+           s.name as source_name, c.name as category_name
+    FROM source_feeds sf
+    JOIN sources s ON sf.source_id = s.id
+    LEFT JOIN source_categories sc ON s.id = sc.source_id
+    LEFT JOIN categories c ON sc.category_id = c.id
+    WHERE sf.is_active = 1 AND s.is_active = 1
+    """
+    return execute_query(sources_db_path, query, fetch=True)
+
+
+def get_feed_stats(tracking_db_path):
+    query = """
+    SELECT 
+        COUNT(*) as total_entries,
+        SUM(CASE WHEN crawl_status = 'pending' THEN 1 ELSE 0 END) as pending_entries,
+        SUM(CASE WHEN crawl_status = 'processing' THEN 1 ELSE 0 END) as processing_entries,
+        SUM(CASE WHEN crawl_status = 'success' THEN 1 ELSE 0 END) as success_entries,
+        SUM(CASE WHEN crawl_status = 'failed' THEN 1 ELSE 0 END) as failed_entries
+    FROM feed_entries
+    """
+    return execute_query(tracking_db_path, query, fetch=True, fetch_one=True)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/db/podcast_configs.py

@@ -0,0 +1,205 @@
+import sqlite3
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+
+def get_podcast_config(db_path: str, config_id: int) -> Optional[Dict[str, Any]]:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        cursor = conn.cursor()
+        cursor.execute(
+            """
+            SELECT id, name, description, prompt, time_range_hours, limit_articles, 
+                   is_active, tts_engine, language_code, podcast_script_prompt, 
+                   image_prompt, created_at, updated_at
+            FROM podcast_configs
+            WHERE id = ?
+            """,
+            (config_id,),
+        )
+        row = cursor.fetchone()
+        if not row:
+            return None
+        config = dict(row)
+        config["is_active"] = bool(config.get("is_active", 0))
+        return config
+    except Exception as e:
+        print(f"Error fetching podcast config: {e}")
+        return None
+    finally:
+        conn.close()
+
+
+def get_all_podcast_configs(db_path: str, active_only: bool = False) -> List[Dict[str, Any]]:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        cursor = conn.cursor()
+        if active_only:
+            query = """
+            SELECT id, name, description, prompt, time_range_hours, limit_articles, 
+                   is_active, tts_engine, language_code, podcast_script_prompt, 
+                   image_prompt, created_at, updated_at
+            FROM podcast_configs
+            WHERE is_active = 1
+            ORDER BY name
+            """
+            cursor.execute(query)
+        else:
+            query = """
+            SELECT id, name, description, prompt, time_range_hours, limit_articles, 
+                   is_active, tts_engine, language_code, podcast_script_prompt, 
+                   image_prompt, created_at, updated_at
+            FROM podcast_configs
+            ORDER BY name
+            """
+            cursor.execute(query)
+        configs = []
+        for row in cursor.fetchall():
+            config = dict(row)
+            config["is_active"] = bool(config.get("is_active", 0))
+            configs.append(config)
+        return configs
+    except Exception as e:
+        print(f"Error fetching podcast configs: {e}")
+        return []
+    finally:
+        conn.close()
+
+
+def create_podcast_config(
+    db_path: str,
+    name: str,
+    prompt: str,
+    description: Optional[str] = None,
+    time_range_hours: int = 24,
+    limit_articles: int = 20,
+    is_active: bool = True,
+    tts_engine: str = "kokoro",
+    language_code: str = "en",
+    podcast_script_prompt: Optional[str] = None,
+    image_prompt: Optional[str] = None,
+) -> Optional[int]:
+    conn = sqlite3.connect(db_path)
+    try:
+        cursor = conn.cursor()
+        now = datetime.now().isoformat()
+        cursor.execute(
+            """
+            INSERT INTO podcast_configs
+            (name, description, prompt, time_range_hours, limit_articles, 
+             is_active, tts_engine, language_code, podcast_script_prompt, 
+             image_prompt, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                name,
+                description,
+                prompt,
+                time_range_hours,
+                limit_articles,
+                1 if is_active else 0,
+                tts_engine,
+                language_code,
+                podcast_script_prompt,
+                image_prompt,
+                now,
+                now,
+            ),
+        )
+        conn.commit()
+        return cursor.lastrowid
+    except Exception as e:
+        conn.rollback()
+        print(f"Error creating podcast config: {e}")
+        return None
+    finally:
+        conn.close()
+
+
+def update_podcast_config(db_path: str, config_id: int, updates: Dict[str, Any]) -> bool:
+    conn = sqlite3.connect(db_path)
+    try:
+        cursor = conn.cursor()
+        cursor.execute("SELECT 1 FROM podcast_configs WHERE id = ?", (config_id,))
+        if not cursor.fetchone():
+            return False
+        if not updates:
+            return True
+        set_clauses = []
+        params = []
+        set_clauses.append("updated_at = ?")
+        params.append(datetime.now().isoformat())
+        allowed_fields = [
+            "name",
+            "description",
+            "prompt",
+            "time_range_hours",
+            "limit_articles",
+            "is_active",
+            "tts_engine",
+            "language_code",
+            "podcast_script_prompt",
+            "image_prompt",
+        ]
+        for field, value in updates.items():
+            if field in allowed_fields:
+                if field == "is_active":
+                    value = 1 if value else 0
+                set_clauses.append(f"{field} = ?")
+                params.append(value)
+        params.append(config_id)
+        query = f"""
+        UPDATE podcast_configs
+        SET {", ".join(set_clauses)}
+        WHERE id = ?
+        """
+        cursor.execute(query, tuple(params))
+        conn.commit()
+        return True
+    except Exception as e:
+        conn.rollback()
+        print(f"Error updating podcast config: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def delete_podcast_config(db_path: str, config_id: int) -> bool:
+    conn = sqlite3.connect(db_path)
+    try:
+        cursor = conn.cursor()
+        cursor.execute("DELETE FROM podcast_configs WHERE id = ?", (config_id,))
+        conn.commit()
+
+        return cursor.rowcount > 0
+    except Exception as e:
+        conn.rollback()
+        print(f"Error deleting podcast config: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def toggle_podcast_config(db_path: str, config_id: int, is_active: bool) -> bool:
+    conn = sqlite3.connect(db_path)
+    try:
+        cursor = conn.cursor()
+        now = datetime.now().isoformat()
+        cursor.execute(
+            """
+            UPDATE podcast_configs
+            SET is_active = ?, updated_at = ?
+            WHERE id = ?
+            """,
+            (1 if is_active else 0, now, config_id),
+        )
+        conn.commit()
+        return cursor.rowcount > 0
+    except Exception as e:
+        conn.rollback()
+        print(f"Error toggling podcast config: {e}")
+        return False
+    finally:
+        conn.close()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/db/podcasts.py

@@ -0,0 +1,124 @@
+import json
+from datetime import datetime
+from typing import Dict, Any, Optional
+from .connection import execute_query
+
+
+def store_podcast(
+    podcasts_db_path: str,
+    podcast_data: Dict[str, Any],
+    audio_path: Optional[str],
+    banner_path: Optional[str],
+    tts_engine: str = "kokoro",
+    language_code: str = "en",
+) -> int:
+    today = datetime.now().strftime("%Y-%m-%d")
+    podcast_json = json.dumps(podcast_data)
+    audio_generated = 1 if audio_path else 0
+    sources_json = json.dumps(podcast_data.get("sources", []))
+    query = """
+    INSERT INTO podcasts
+    (title, date, content_json, audio_generated, audio_path, banner_img_path, 
+     tts_engine, language_code, sources_json, created_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    """
+    params = (
+        podcast_data.get("title", f"Podcast {today}"),
+        today,
+        podcast_json,
+        audio_generated,
+        audio_path,
+        banner_path,
+        tts_engine,
+        language_code,
+        sources_json,
+        datetime.now().isoformat(),
+    )
+    return execute_query(podcasts_db_path, query, params)
+
+
+def get_podcast(podcasts_db_path: str, podcast_id: int) -> Optional[Dict[str, Any]]:
+    query = """
+    SELECT id, title, date, content_json, audio_generated, audio_path, banner_img_path, 
+           tts_engine, language_code, sources_json, created_at
+    FROM podcasts
+    WHERE id = ?
+    """
+    podcast = execute_query(podcasts_db_path, query, (podcast_id,), fetch=True, fetch_one=True)
+    if podcast:
+        if podcast.get("content_json"):
+            try:
+                podcast["content"] = json.loads(podcast["content_json"])
+            except json.JSONDecodeError:
+                podcast["content"] = {}
+
+        if podcast.get("sources_json"):
+            try:
+                podcast["sources"] = json.loads(podcast["sources_json"])
+            except json.JSONDecodeError:
+                podcast["sources"] = []
+    return podcast
+
+
+def get_recent_podcasts(podcasts_db_path: str, limit: int = 10) -> list:
+    query = """
+    SELECT id, title, date, audio_generated, audio_path, banner_img_path, 
+           tts_engine, language_code, sources_json, created_at
+    FROM podcasts
+    ORDER BY date DESC, created_at DESC
+    LIMIT ?
+    """
+    podcasts = execute_query(podcasts_db_path, query, (limit,), fetch=True)
+    for podcast in podcasts:
+        if podcast.get("sources_json"):
+            try:
+                podcast["sources"] = json.loads(podcast["sources_json"])
+            except json.JSONDecodeError:
+                podcast["sources"] = []
+    return podcasts
+
+
+def update_podcast_audio(podcasts_db_path: str, podcast_id: int, audio_path: str) -> bool:
+    query = """
+    UPDATE podcasts
+    SET audio_path = ?, audio_generated = 1
+    WHERE id = ?
+    """
+    rows_affected = execute_query(podcasts_db_path, query, (audio_path, podcast_id))
+    return rows_affected > 0
+
+
+def update_podcast_banner(podcasts_db_path: str, podcast_id: int, banner_path: str) -> bool:
+    query = """
+    UPDATE podcasts
+    SET banner_img_path = ?
+    WHERE id = ?
+    """
+    rows_affected = execute_query(podcasts_db_path, query, (banner_path, podcast_id))
+    return rows_affected > 0
+
+
+def update_podcast_metadata(
+    podcasts_db_path: str, podcast_id: int, tts_engine: Optional[str] = None, language_code: Optional[str] = None, sources_json: Optional[str] = None
+) -> bool:
+    update_parts = []
+    params = []
+    if tts_engine is not None:
+        update_parts.append("tts_engine = ?")
+        params.append(tts_engine)
+    if language_code is not None:
+        update_parts.append("language_code = ?")
+        params.append(language_code)
+    if sources_json is not None:
+        update_parts.append("sources_json = ?")
+        params.append(sources_json)
+    if not update_parts:
+        return False
+    query = f"""
+    UPDATE podcasts
+    SET {", ".join(update_parts)}
+    WHERE id = ?
+    """
+    params.append(podcast_id)
+    rows_affected = execute_query(podcasts_db_path, query, tuple(params))
+    return rows_affected > 0

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/db/tasks.py

@@ -0,0 +1,247 @@
+from datetime import datetime, timedelta
+from .connection import execute_query, db_connection
+
+
+def create_task(
+    tasks_db_path,
+    name,
+    command,
+    frequency,
+    frequency_unit,
+    description=None,
+    enabled=True,
+):
+    query = """
+    INSERT INTO tasks 
+    (name, description, command, frequency, frequency_unit, enabled, created_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+    """
+    params = (
+        name,
+        description,
+        command,
+        frequency,
+        frequency_unit,
+        1 if enabled else 0,
+        datetime.now().isoformat(),
+    )
+    return execute_query(tasks_db_path, query, params)
+
+
+def is_task_running(tasks_db_path, task_id):
+    query = """
+    SELECT 1 
+    FROM task_executions 
+    WHERE task_id = ? AND status = 'running'
+    LIMIT 1
+    """
+    try:
+        result = execute_query(tasks_db_path, query, (task_id,), fetch=True, fetch_one=True)
+        return result is not None  # True if a running entry exists
+    except Exception as e:
+        print(f"Database error checking running status for task {task_id}: {e}")
+        return True
+
+
+def get_task(tasks_db_path, task_id):
+    query = """
+    SELECT id, name, description, command, frequency, frequency_unit, enabled, last_run, created_at
+    FROM tasks
+    WHERE id = ?
+    """
+    return execute_query(tasks_db_path, query, (task_id,), fetch=True, fetch_one=True)
+
+
+def get_all_tasks(tasks_db_path, include_disabled=False):
+    if include_disabled:
+        query = """
+        SELECT id, name, description, command, frequency, frequency_unit, enabled, last_run, created_at
+        FROM tasks
+        ORDER BY name
+        """
+        return execute_query(tasks_db_path, query, fetch=True)
+    else:
+        query = """
+        SELECT id, name, description, command, frequency, frequency_unit, enabled, last_run, created_at
+        FROM tasks
+        WHERE enabled = 1
+        ORDER BY name
+        """
+        return execute_query(tasks_db_path, query, fetch=True)
+
+
+def update_task(tasks_db_path, task_id, updates):
+    allowed_fields = [
+        "name",
+        "description",
+        "command",
+        "frequency",
+        "frequency_unit",
+        "enabled",
+    ]
+
+    set_clauses = []
+    params = []
+    for field, value in updates.items():
+        if field in allowed_fields:
+            if field == "enabled":
+                value = 1 if value else 0
+            set_clauses.append(f"{field} = ?")
+            params.append(value)
+    if not set_clauses:
+        return 0
+    query = f"""
+    UPDATE tasks
+    SET {", ".join(set_clauses)}
+    WHERE id = ?
+    """
+    params.append(task_id)
+    return execute_query(tasks_db_path, query, tuple(params))
+
+
+def delete_task(tasks_db_path, task_id):
+    query = """
+    DELETE FROM tasks
+    WHERE id = ?
+    """
+    return execute_query(tasks_db_path, query, (task_id,))
+
+
+def update_task_last_run(tasks_db_path, task_id, timestamp=None):
+    if timestamp is None:
+        timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+    query = """
+    UPDATE tasks
+    SET last_run = ?
+    WHERE id = ?
+    """
+    return execute_query(tasks_db_path, query, (timestamp, task_id))
+
+
+def create_task_execution(tasks_db_path, task_id, status, error_message=None, output=None):
+    start_time = datetime.now().isoformat()
+    query = """
+    INSERT INTO task_executions 
+    (task_id, start_time, status, error_message, output)
+    VALUES (?, ?, ?, ?, ?)
+    """
+    params = (task_id, start_time, status, error_message, output)
+    execute_query(tasks_db_path, query, params)
+    try:
+        with db_connection(tasks_db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(query, params)
+            conn.commit()
+            return cursor.lastrowid  # Return the ID of the inserted row
+    except Exception as e:
+        print(f"Database error in create_task_execution: {e}")
+        return None  #
+
+
+def update_task_execution(tasks_db_path, execution_id, status, error_message=None, output=None):
+    end_time = datetime.now().isoformat()
+    query = """
+    UPDATE task_executions
+    SET end_time = ?, status = ?, error_message = ?, output = ?
+    WHERE id = ?
+    """
+    params = (end_time, status, error_message, output, execution_id)
+    return execute_query(tasks_db_path, query, params)
+
+
+def get_recent_task_executions(tasks_db_path, task_id=None, limit=10):
+    if task_id:
+        query = """
+        SELECT id, task_id, start_time, end_time, status, error_message, output
+        FROM task_executions
+        WHERE task_id = ?
+        ORDER BY start_time DESC
+        LIMIT ?
+        """
+        params = (task_id, limit)
+    else:
+        query = """
+        SELECT id, task_id, start_time, end_time, status, error_message, output
+        FROM task_executions
+        ORDER BY start_time DESC
+        LIMIT ?
+        """
+        params = (limit,)
+    return execute_query(tasks_db_path, query, params, fetch=True)
+
+
+def get_task_execution(tasks_db_path, execution_id):
+    query = """
+    SELECT id, task_id, start_time, end_time, status, error_message, output
+    FROM task_executions
+    WHERE id = ?
+    """
+    return execute_query(tasks_db_path, query, (execution_id,), fetch=True, fetch_one=True)
+
+
+def mark_task_disabled(tasks_db_path, task_id):
+    query = """
+    UPDATE tasks
+    SET enabled = 0
+    WHERE id = ?
+    """
+    return execute_query(tasks_db_path, query, (task_id,))
+
+
+def mark_task_enabled(tasks_db_path, task_id):
+    query = """
+    UPDATE tasks
+    SET enabled = 1
+    WHERE id = ?
+    """
+    return execute_query(tasks_db_path, query, (task_id,))
+
+
+def get_task_stats(tasks_db_path):
+    query = """
+    SELECT 
+        COUNT(*) as total_tasks,
+        SUM(CASE WHEN enabled = 1 THEN 1 ELSE 0 END) as active_tasks,
+        SUM(CASE WHEN enabled = 0 THEN 1 ELSE 0 END) as disabled_tasks,
+        SUM(CASE WHEN last_run IS NULL THEN 1 ELSE 0 END) as never_run_tasks
+    FROM tasks
+    """
+    return execute_query(tasks_db_path, query, fetch=True, fetch_one=True)
+
+
+def get_execution_stats(tasks_db_path, days=7):
+    cutoff_date = (datetime.now() - timedelta(days=days)).isoformat()
+    query = """
+    SELECT 
+        COUNT(*) as total_executions,
+        COALESCE(SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END), 0) as successful_executions,
+        COALESCE(SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END), 0) as failed_executions,
+        COALESCE(SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END), 0) as running_executions,
+        COALESCE(AVG(CASE WHEN end_time IS NOT NULL 
+            THEN (julianday(end_time) - julianday(start_time)) * 86400.0 
+            ELSE NULL END), 0) as avg_execution_time_seconds
+    FROM task_executions
+    WHERE start_time >= ?
+    """
+    return execute_query(tasks_db_path, query, (cutoff_date,), fetch=True, fetch_one=True)
+
+
+def get_pending_tasks(tasks_db_path):
+    query = """
+    SELECT id, name, description, command, frequency, frequency_unit, enabled, last_run
+    FROM tasks
+    WHERE enabled = 1
+    AND (
+        last_run IS NULL 
+        OR 
+        CASE frequency_unit
+            WHEN 'minutes' THEN datetime(last_run, '+' || frequency || ' minutes') <= datetime('now', 'localtime')
+            WHEN 'hours' THEN datetime(last_run, '+' || frequency || ' hours') <= datetime('now', 'localtime')
+            WHEN 'days' THEN datetime(last_run, '+' || frequency || ' days') <= datetime('now', 'localtime')
+            ELSE datetime(last_run, '+' || frequency || ' seconds') <= datetime('now', 'localtime')
+        END
+    )
+    ORDER BY last_run
+    """
+    tasks = execute_query(tasks_db_path, query, fetch=True)
+    return tasks

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/main.py

@@ -0,0 +1,187 @@
+from fastapi import FastAPI, Request, Response
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse, StreamingResponse
+import uvicorn
+import os
+import aiofiles
+from contextlib import asynccontextmanager
+from routers import article_router, podcast_router, source_router, task_router, podcast_config_router, async_podcast_agent_router, social_media_router
+from services.db_init import init_databases
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+CLIENT_BUILD_PATH = os.environ.get(
+    "CLIENT_BUILD_PATH",
+    "../web/build",
+)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    print("Starting up application...")
+    os.makedirs("databases", exist_ok=True)
+    os.makedirs("browsers", exist_ok=True)
+    os.makedirs("podcasts/audio", exist_ok=True)
+    os.makedirs("podcasts/images", exist_ok=True)
+    os.makedirs("podcasts/recordings", exist_ok=True)
+    await init_databases()
+    if not os.path.exists(CLIENT_BUILD_PATH):
+        print(f"WARNING: React client build path not found: {CLIENT_BUILD_PATH}")
+    print("Application startup complete!")
+    yield
+    print("Shutting down application...")
+    print("Shutdown complete")
+
+
+app = FastAPI(title="Beifong API", description="Beifong API", version="1.0.0", lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+app.include_router(article_router.router, prefix="/api/articles", tags=["articles"])
+app.include_router(source_router.router, prefix="/api/sources", tags=["sources"])
+app.include_router(podcast_router.router, prefix="/api/podcasts", tags=["podcasts"])
+app.include_router(task_router.router, prefix="/api/tasks", tags=["tasks"])
+app.include_router(podcast_config_router.router, prefix="/api/podcast-configs", tags=["podcast-configs"])
+app.include_router(async_podcast_agent_router.router, prefix="/api/podcast-agent", tags=["podcast-agent"])
+app.include_router(social_media_router.router, prefix="/api/social-media", tags=["social-media"])
+
+
+@app.get("/stream-audio/{filename}")
+async def stream_audio(filename: str, request: Request):
+    audio_path = os.path.join("podcasts/audio", filename)
+    if not os.path.exists(audio_path):
+        return Response(status_code=404, content="Audio file not found")
+    file_size = os.path.getsize(audio_path)
+    range_header = request.headers.get("Range", "").strip()
+    start = 0
+    end = file_size - 1
+    if range_header:
+        try:
+            range_data = range_header.replace("bytes=", "").split("-")
+            start = int(range_data[0]) if range_data[0] else 0
+            end = int(range_data[1]) if len(range_data) > 1 and range_data[1] else file_size - 1
+        except ValueError:
+            return Response(status_code=400, content="Invalid range header")
+    end = min(end, file_size - 1)
+    content_length = end - start + 1
+    headers = {
+        "Accept-Ranges": "bytes",
+        "Content-Range": f"bytes {start}-{end}/{file_size}",
+        "Content-Length": str(content_length),
+        "Content-Disposition": f"inline; filename={filename}",
+        "Content-Type": "audio/wav",
+    }
+
+    async def file_streamer():
+        async with aiofiles.open(audio_path, "rb") as f:
+            await f.seek(start)
+            remaining = content_length
+            chunk_size = 64 * 1024
+            while remaining > 0:
+                chunk = await f.read(min(chunk_size, remaining))
+                if not chunk:
+                    break
+                remaining -= len(chunk)
+                yield chunk
+
+    status_code = 206 if range_header else 200
+    return StreamingResponse(file_streamer(), status_code=status_code, headers=headers)
+
+
+@app.get("/stream-recording/{session_id}/{filename}")
+async def stream_recording(session_id: str, filename: str, request: Request):
+    recording_path = os.path.join("podcasts/recordings", session_id, filename)
+    if not os.path.exists(recording_path):
+        return Response(status_code=404, content="Recording video not found")
+    file_size = os.path.getsize(recording_path)
+    range_header = request.headers.get("Range", "").strip()
+    start = 0
+    end = file_size - 1
+    if range_header:
+        try:
+            range_data = range_header.replace("bytes=", "").split("-")
+            start = int(range_data[0]) if range_data[0] else 0
+            end = int(range_data[1]) if len(range_data) > 1 and range_data[1] else file_size - 1
+        except ValueError:
+            return Response(status_code=400, content="Invalid range header")
+    end = min(end, file_size - 1)
+    content_length = end - start + 1
+    headers = {
+        "Accept-Ranges": "bytes",
+        "Content-Range": f"bytes {start}-{end}/{file_size}",
+        "Content-Length": str(content_length),
+        "Content-Disposition": f"inline; filename={filename}",
+        "Content-Type": "video/webm",
+    }
+
+    async def file_streamer():
+        async with aiofiles.open(recording_path, "rb") as f:
+            await f.seek(start)
+            remaining = content_length
+            chunk_size = 64 * 1024
+            while remaining > 0:
+                chunk = await f.read(min(chunk_size, remaining))
+                if not chunk:
+                    break
+                remaining -= len(chunk)
+                yield chunk
+
+    status_code = 206 if range_header else 200
+    return StreamingResponse(file_streamer(), status_code=status_code, headers=headers)
+
+
+app.mount("/audio", StaticFiles(directory="podcasts/audio"), name="audio")
+app.mount("/server_static", StaticFiles(directory="static"), name="server_static")
+app.mount("/podcast_img", StaticFiles(directory="podcasts/images"), name="podcast_img")
+if os.path.exists(os.path.join(CLIENT_BUILD_PATH, "static")):
+    app.mount("/static", StaticFiles(directory=os.path.join(CLIENT_BUILD_PATH, "static")), name="react_static")
+
+
+@app.get("/favicon.ico")
+async def favicon():
+    favicon_path = os.path.join(CLIENT_BUILD_PATH, "favicon.ico")
+    if os.path.exists(favicon_path):
+        return FileResponse(favicon_path)
+    return {"detail": "Favicon not found"}
+
+
+@app.get("/manifest.json")
+async def manifest():
+    manifest_path = os.path.join(CLIENT_BUILD_PATH, "manifest.json")
+    if os.path.exists(manifest_path):
+        return FileResponse(manifest_path)
+    return {"detail": "Manifest not found"}
+
+
+@app.get("/logo{rest_of_path:path}")
+async def logo(rest_of_path: str):
+    logo_path = os.path.join(CLIENT_BUILD_PATH, f"logo{rest_of_path}")
+    if os.path.exists(logo_path):
+        return FileResponse(logo_path)
+    return {"detail": "Logo not found"}
+
+
+@app.get("/{full_path:path}")
+async def serve_react(full_path: str, request: Request):
+    if full_path.startswith("api/") or request.url.path.startswith("/api/"):
+        return {"detail": "Not Found"}
+    index_path = os.path.join(CLIENT_BUILD_PATH, "index.html")
+    if os.path.exists(index_path):
+        return FileResponse(index_path)
+    else:
+        return {"detail": "React client not found. Build the client or set the correct CLIENT_BUILD_PATH."}
+
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 7000))
+    uvicorn.run("main:app", host="0.0.0.0", port=port, reload=False, timeout_keep_alive=120, timeout_graceful_shutdown=120)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/models/article_schemas.py

@@ -0,0 +1,29 @@
+from pydantic import BaseModel, ConfigDict
+from typing import Optional, List, Dict, Any
+
+
+class ArticleBase(BaseModel):
+    title: str
+    url: Optional[str] = None
+    published_date: str
+    summary: Optional[str] = None
+    content: Optional[str] = None
+    categories: Optional[List[str]] = []
+    source_name: Optional[str] = None
+
+
+class Article(ArticleBase):
+    id: int
+    metadata: Optional[Dict[str, Any]] = {}
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class PaginatedArticles(BaseModel):
+    items: List[Article]
+    total: int
+    page: int
+    per_page: int
+    total_pages: int
+    has_next: bool
+    has_prev: bool

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/models/podcast_config_schemas.py

@@ -0,0 +1,41 @@
+from pydantic import BaseModel, Field
+from typing import Optional
+
+
+class PodcastConfigBase(BaseModel):
+    name: str
+    prompt: str
+    description: Optional[str] = None
+    time_range_hours: int = Field(24, ge=1, le=168)
+    limit_articles: int = Field(20, ge=5, le=50)
+    is_active: bool = True
+    tts_engine: str = "kokoro"
+    language_code: str = "en"
+    podcast_script_prompt: Optional[str] = None
+    image_prompt: Optional[str] = None
+
+
+class PodcastConfig(PodcastConfigBase):
+    id: int
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+    class Config:
+        from_attributes = True
+
+
+class PodcastConfigCreate(PodcastConfigBase):
+    pass
+
+
+class PodcastConfigUpdate(BaseModel):
+    name: Optional[str] = None
+    prompt: Optional[str] = None
+    description: Optional[str] = None
+    time_range_hours: Optional[int] = Field(None, ge=1, le=168)
+    limit_articles: Optional[int] = Field(None, ge=5, le=50)
+    is_active: Optional[bool] = None
+    tts_engine: Optional[str] = None
+    language_code: Optional[str] = None
+    podcast_script_prompt: Optional[str] = None
+    image_prompt: Optional[str] = None

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/models/podcast_schemas.py

@@ -0,0 +1,81 @@
+from pydantic import BaseModel
+from typing import Optional, List, Dict, Any, Union
+
+
+class PodcastBase(BaseModel):
+    title: str
+    date: str
+    audio_generated: bool = False
+    banner_img: Optional[str] = None
+    identifier: str
+    language_code: Optional[str] = "en"
+    tts_engine: Optional[str] = "kokoro"
+
+
+class Podcast(PodcastBase):
+    id: int
+    created_at: Optional[str] = None
+    audio_path: Optional[str] = None
+
+    class Config:
+        from_attributes = True
+
+
+class PodcastContent(BaseModel):
+    title: str
+    sections: List[Dict[str, Any]]
+
+
+class PodcastSource(BaseModel):
+    title: Optional[str] = None
+    url: Optional[str] = None
+    source: Optional[str] = None
+
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, v):
+        if isinstance(v, str):
+            return cls(url=v)
+        if isinstance(v, dict):
+            return cls(**v)
+        raise ValueError("Source must be a string or a dict")
+
+
+class PodcastDetail(BaseModel):
+    podcast: Podcast
+    content: PodcastContent
+    audio_url: Optional[str] = None
+    sources: Optional[List[Union[PodcastSource, str]]] = None
+    banner_images: Optional[List[str]] = None
+
+
+class PodcastCreate(BaseModel):
+    title: str
+    date: Optional[str] = None
+    content: Dict[str, Any]
+    sources: Optional[List[Union[Dict[str, str], str]]] = None
+    language_code: Optional[str] = "en"
+    tts_engine: Optional[str] = "kokoro"
+
+
+class PodcastUpdate(BaseModel):
+    title: Optional[str] = None
+    date: Optional[str] = None
+    content: Optional[Dict[str, Any]] = None
+    audio_generated: Optional[bool] = None
+    sources: Optional[List[Union[Dict[str, str], str]]] = None
+    language_code: Optional[str] = None
+    tts_engine: Optional[str] = None
+
+
+class PaginatedPodcasts(BaseModel):
+    items: List[Podcast]
+    total: int
+    page: int
+    per_page: int
+    total_pages: int
+    has_next: bool
+    has_prev: bool

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/models/social_media_schemas.py

@@ -0,0 +1,56 @@
+from typing import List, Optional
+from pydantic import BaseModel
+from datetime import datetime
+
+class PostBase(BaseModel):
+    post_id: str
+    platform: str
+    user_display_name: Optional[str] = None
+    user_handle: Optional[str] = None
+    user_profile_pic_url: Optional[str] = None
+    post_timestamp: Optional[str] = None
+    post_display_time: Optional[str] = None
+    post_url: Optional[str] = None
+    post_text: Optional[str] = None
+    post_mentions: Optional[str] = None
+
+class PostEngagement(BaseModel):
+    replies: Optional[int] = None
+    retweets: Optional[int] = None
+    likes: Optional[int] = None
+    bookmarks: Optional[int] = None
+    views: Optional[int] = None
+
+class MediaItem(BaseModel):
+    type: str 
+    url: str
+
+class Post(PostBase):
+    engagement: Optional[PostEngagement] = None
+    media: Optional[List[MediaItem]] = None
+    media_count: Optional[int] = 0
+    is_ad: Optional[bool] = False
+    sentiment: Optional[str] = None
+    categories: Optional[List[str]] = None
+    tags: Optional[List[str]] = None
+    analysis_reasoning: Optional[str] = None
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+
+class PaginatedPosts(BaseModel):
+    items: List[Post]
+    total: int
+    page: int
+    per_page: int
+    total_pages: int
+    has_next: bool
+    has_prev: bool
+
+class PostFilterParams(BaseModel):
+    platform: Optional[str] = None
+    user_handle: Optional[str] = None
+    sentiment: Optional[str] = None
+    category: Optional[str] = None
+    date_from: Optional[str] = None
+    date_to: Optional[str] = None
+    search: Optional[str] = None

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/models/source_schemas.py

@@ -0,0 +1,68 @@
+from pydantic import BaseModel
+from typing import Optional, List
+
+
+class SourceFeed(BaseModel):
+    id: int
+    feed_url: str
+    feed_type: str
+    description: Optional[str] = None
+    is_active: bool
+    created_at: str
+    last_crawled: Optional[str] = None
+
+
+class SourceFeedCreate(BaseModel):
+    feed_url: str
+    feed_type: str = "main"
+    description: Optional[str] = None
+    is_active: bool = True
+
+
+class SourceBase(BaseModel):
+    name: str
+    url: Optional[str] = None
+    categories: Optional[List[str]] = []
+    description: Optional[str] = None
+    is_active: bool = True
+
+
+class Source(SourceBase):
+    id: int
+    created_at: Optional[str] = None
+    last_crawled: Optional[str] = None
+
+    class Config:
+        from_attributes = True
+
+
+class SourceCreate(SourceBase):
+    feeds: Optional[List[SourceFeedCreate]] = []
+
+
+class SourceUpdate(BaseModel):
+    name: Optional[str] = None
+    url: Optional[str] = None
+    categories: Optional[List[str]] = None
+    description: Optional[str] = None
+    is_active: Optional[bool] = None
+
+
+class SourceWithFeeds(Source):
+    feeds: List[SourceFeed] = []
+
+
+class PaginatedSources(BaseModel):
+    items: List[Source]
+    total: int
+    page: int
+    per_page: int
+    total_pages: int
+    has_next: bool
+    has_prev: bool
+
+
+class Category(BaseModel):
+    id: int
+    name: str
+    description: Optional[str] = None

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/models/tasks_schemas.py

@@ -0,0 +1,118 @@
+from pydantic import BaseModel, validator
+from typing import Optional, List, Dict, Any
+from enum import Enum
+
+
+class TaskType(str, Enum):
+    feed_processor = "feed_processor"
+    url_crawler = "url_crawler"
+    ai_analyzer = "ai_analyzer"
+    podcast_generator = "podcast_generator"
+    embedding_processor = "embedding_processor"
+    faiss_indexer = "faiss_indexer"
+    social_x_scraper = "social_x_scraper"
+    social_fb_scraper = "social_fb_scraper"
+
+
+TASK_TYPES = {
+    "feed_processor": {
+        "name": "Feed Processor",
+        "command": "python -m processors.feed_processor",
+        "description": "Processes RSS feeds and stores new entries",
+    },
+    "url_crawler": {"name": "URL Crawler", "command": "python -m processors.url_processor", "description": "Crawls URLs and extracts content"},
+    "ai_analyzer": {
+        "name": "AI Analyzer",
+        "command": "python -m processors.ai_analysis_processor",
+        "description": "Analyzes article content using AI",
+    },
+    "podcast_generator": {
+        "name": "Podcast Generator",
+        "command": "python -m processors.podcast_generator_processor",
+        "description": "Generates podcasts from articles",
+    },
+    "embedding_processor": {
+        "name": "Embedding Processor",
+        "command": "python -m processors.embedding_processor",
+        "description": "Generates embeddings for processed articles using OpenAI",
+    },
+    "faiss_indexer": {
+        "name": "FAISS Indexer",
+        "command": "python -m processors.faiss_indexing_processor",
+        "description": "Updates FAISS vector index with new article embeddings",
+    },
+    "social_x_scraper": {
+        "name": "X.com Scraper",
+        "command": "python -m processors.x_scraper_processor",
+        "description": "Scrapes X.com profiles and analyzes sentiment",
+    },
+    "social_fb_scraper": {
+        "name": "Facebook.com Scraper",
+        "command": "python -m processors.fb_scraper_processor",
+        "description": "Scrapes Facebook.com profiles and analyzes sentiment",
+    },
+}
+
+
+class TaskBase(BaseModel):
+    name: str
+    task_type: TaskType
+    frequency: int
+    frequency_unit: str
+    description: Optional[str] = None
+    enabled: bool = True
+
+    @validator("task_type")
+    def set_command_from_type(cls, v):
+        if v not in TASK_TYPES:
+            raise ValueError(f"Invalid task type: {v}")
+        return v
+
+
+class Task(TaskBase):
+    id: int
+    command: str
+    last_run: Optional[str] = None
+    created_at: Optional[str] = None
+
+    class Config:
+        from_attributes = True
+
+
+class TaskCreate(TaskBase):
+    pass
+
+
+class TaskUpdate(BaseModel):
+    name: Optional[str] = None
+    task_type: Optional[TaskType] = None
+    frequency: Optional[int] = None
+    frequency_unit: Optional[str] = None
+    description: Optional[str] = None
+    enabled: Optional[bool] = None
+
+
+class TaskExecution(BaseModel):
+    id: int
+    task_id: int
+    task_name: Optional[str] = None
+    start_time: str
+    end_time: Optional[str] = None
+    status: str
+    error_message: Optional[str] = None
+    output: Optional[str] = None
+
+
+class PaginatedTaskExecutions(BaseModel):
+    items: List[TaskExecution]
+    total: int
+    page: int
+    per_page: int
+    total_pages: int
+    has_next: bool
+    has_prev: bool
+
+
+class TaskStats(BaseModel):
+    tasks: Dict[str, int]
+    executions: Dict[str, Any]

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/pack_demo.py

@@ -0,0 +1,25 @@
+import os
+import zipfile
+
+SOURCE_DIRS = ["databases", "podcasts"]
+OUTPUT_ZIP = "demo_content.zip"
+
+
+def create_zip(source_dirs, output_zip):
+    print("packing.....")
+    """zip up each source directory into a single archive."""
+    with zipfile.ZipFile(output_zip, "w", zipfile.ZIP_DEFLATED) as z:
+        for src in source_dirs:
+            if not os.path.isdir(src):
+                print(f"✗ source '{src}' not found, skipping.")
+                continue
+            for root, _, files in os.walk(src):
+                for file in files:
+                    full_path = os.path.join(root, file)
+                    arcname = os.path.relpath(full_path, os.getcwd())
+                    z.write(full_path, arcname)
+    print(f"✓ created '{output_zip}' containing: {', '.join(source_dirs)}")
+
+
+if __name__ == "__main__":
+    create_zip(SOURCE_DIRS, OUTPUT_ZIP)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/processors/ai_analysis_processor.py

@@ -0,0 +1,188 @@
+import json
+import time
+import random
+import argparse
+from bs4 import BeautifulSoup
+from openai import OpenAI
+from db.config import get_tracking_db_path
+from db.articles import get_unprocessed_articles, update_article_status
+from utils.load_api_keys import load_api_key
+
+WEB_PAGE_ANALYSE_MODEL = "gpt-4o"
+MODEL_INSTRUCTION = "You are a helpful assistant that analyzes articles and extracts structured information."
+
+
+def extract_clean_text(raw_html, max_tokens=8000):
+    soup = BeautifulSoup(raw_html, "html.parser")
+    for element in soup(["script", "style", "nav", "header", "footer", "aside"]):
+        element.decompose()
+    text = soup.get_text(separator="\n", strip=True)
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    text = "\n".join(lines)
+    approx_tokens = len(text) / 4
+    if approx_tokens > max_tokens:
+        text = text[: max_tokens * 4]
+    return text
+
+
+def process_article_with_ai(client, article, max_tokens=8000):
+    clean_text = extract_clean_text(article["raw_content"], max_tokens)
+    metadata = article.get("metadata", {})
+    title = article["title"]
+    url = article["url"]
+    description = ""
+    if metadata and isinstance(metadata, dict):
+        if "description" in metadata:
+            description = metadata["description"]
+        elif "og" in metadata and "description" in metadata["og"]:
+            description = metadata["og"]["description"]
+    try:
+        response = client.chat.completions.create(
+            model=WEB_PAGE_ANALYSE_MODEL,
+            response_format={"type": "json_object"},
+            messages=[
+                {
+                    "role": "system",
+                    "content": MODEL_INSTRUCTION,
+                },
+                {
+                    "role": "user",
+                    "content": f"""
+                                Analyze this article and provide a structured output with three components:
+
+                                1. A list of 3-5 relevant categories for this article
+                                2. A concise 2-3 sentence summary of the article
+                                3. The extracted main article content, removing any navigation, ads, or irrelevant elements
+
+                                Article Title: {title}
+                                Article URL: {url}
+                                Description: {description}
+
+                                Article Text:
+                                {clean_text}
+
+                                Provide your response as a JSON object with these keys:
+                                - categories: an array of 3-5 relevant categories (as strings)
+                                - summary: a 2-3 sentence summary of the article
+                                - content: the cleaned main article content
+                                """,
+                },
+            ],
+            temperature=0.3,
+            max_tokens=1500,
+        )
+        response_json = json.loads(response.choices[0].message.content)
+        categories = response_json.get("categories", [])
+        if isinstance(categories, str):
+            categories = [cat.strip() for cat in categories.split(",") if cat.strip()]
+        results = {
+            "categories": categories,
+            "summary": response_json.get("summary", ""),
+            "content": response_json.get("content", ""),
+        }
+        return results, True, None
+    except Exception as e:
+        error_message = str(e)
+        print(f"Error processing article with AI: {error_message}")
+        return None, False, error_message
+
+
+def analyze_articles(tracking_db_path=None, openai_api_key=None, batch_size=5, delay_range=(1, 3)):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    if openai_api_key is None:
+        raise ValueError("OpenAI API key is required")
+    client = OpenAI(api_key=openai_api_key)
+    articles = get_unprocessed_articles(tracking_db_path, limit=batch_size)
+    stats = {"total_articles": len(articles), "success_count": 0, "failed_count": 0}
+    for i, article in enumerate(articles):
+        article_id = article["id"]
+        title = article["title"]
+        attempt = article.get("ai_attempts", 0) + 1
+        print(f"[{i + 1}/{len(articles)}] Processing article: {title} (Attempt {attempt})")
+        results, success, error_message = process_article_with_ai(client, article)
+        update_article_status(tracking_db_path, article_id, results, success, error_message)
+        if success:
+            categories_display = ", ".join(results["categories"])
+            print(f"Successfully processed article ID {article_id}")
+            print(f"Categories: {categories_display}")
+            print(f"Summary: {results['summary'][:100]}..." if len(results["summary"]) > 100 else f"Summary: {results['summary']}")
+            stats["success_count"] += 1
+        else:
+            print(f"Failed to process article ID {article_id}: {error_message}")
+            stats["failed_count"] += 1
+
+        if i < len(articles) - 1:
+            delay = random.uniform(delay_range[0], delay_range[1])
+            time.sleep(delay)
+    return stats
+
+
+def print_stats(stats):
+    print("\nAI Analysis Statistics:")
+    print(f"Total articles processed: {stats['total_articles']}")
+    print(f"Successfully analyzed: {stats['success_count']}")
+    print(f"Failed: {stats['failed_count']}")
+
+
+def analyze_in_batches(
+    tracking_db_path=None,
+    openai_api_key=None,
+    batch_size=20,
+    total_batches=1,
+    delay_between_batches=10,
+):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    if openai_api_key is None:
+        raise ValueError("OpenAI API key is required")
+    total_stats = {"total_articles": 0, "success_count": 0, "failed_count": 0}
+    for i in range(total_batches):
+        print(f"\nProcessing batch {i + 1}/{total_batches}")
+        batch_stats = analyze_articles(
+            tracking_db_path=tracking_db_path,
+            openai_api_key=openai_api_key,
+            batch_size=batch_size,
+        )
+        total_stats["total_articles"] += batch_stats["total_articles"]
+        total_stats["success_count"] += batch_stats["success_count"]
+        total_stats["failed_count"] += batch_stats["failed_count"]
+        if batch_stats["total_articles"] == 0:
+            print("No more articles to process")
+            break
+        if i < total_batches - 1:
+            print(f"Waiting {delay_between_batches} seconds before next batch...")
+            time.sleep(delay_between_batches)
+    return total_stats
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Process articles with AI analysis")
+    parser.add_argument("--api_key", help="OpenAI API Key (overrides environment variables)")
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=10,
+        help="Number of articles to process in each batch",
+    )
+    parser.add_argument(
+        "--total_batches",
+        type=int,
+        default=1,
+        help="Total number of batches to process",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    api_key = args.api_key or load_api_key()
+    if not api_key:
+        print("Error: No OpenAI API key provided. Please provide via --api_key or set OPENAI_API_KEY in .env file")
+        exit(1)
+    stats = analyze_in_batches(
+        openai_api_key=api_key,
+        batch_size=args.batch_size,
+        total_batches=args.total_batches,
+    )
+    print_stats(stats)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/processors/embedding_processor.py

@@ -0,0 +1,229 @@
+import time
+import argparse
+import random
+import numpy as np
+from openai import OpenAI
+from db.config import get_tracking_db_path
+from db.connection import db_connection, execute_query
+from utils.load_api_keys import load_api_key
+
+EMBEDDING_MODEL = "text-embedding-3-small"
+
+def create_embedding_table(tracking_db_path):
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute("""
+        SELECT name FROM sqlite_master 
+        WHERE type='table' AND name='article_embeddings'
+        """)
+        table_exists = cursor.fetchone() is not None
+        if not table_exists:
+            cursor.execute("""
+            CREATE TABLE IF NOT EXISTS article_embeddings (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                article_id INTEGER NOT NULL,
+                embedding BLOB NOT NULL,
+                embedding_model TEXT NOT NULL,
+                created_at TEXT NOT NULL,
+                in_faiss_index INTEGER DEFAULT 0,
+                FOREIGN KEY (article_id) REFERENCES crawled_articles(id)
+            )
+            """)
+            cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_article_embeddings_article_id 
+            ON article_embeddings(article_id)
+            """)
+            cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_article_embeddings_in_faiss 
+            ON article_embeddings(in_faiss_index)
+            """)
+            conn.commit()
+            print("Article embeddings table created successfully.")
+        else:
+            print("Article embeddings table already exists.")
+
+
+def get_articles_without_embeddings(tracking_db_path, limit=20):
+    query = """
+    SELECT ca.id, ca.title, ca.summary, ca.content
+    FROM crawled_articles ca
+    WHERE ca.processed = 1 
+    AND ca.ai_status = 'success'
+    AND NOT EXISTS (
+        SELECT 1 FROM article_embeddings ae 
+        WHERE ae.article_id = ca.id
+    )
+    ORDER BY ca.published_date DESC
+    LIMIT ?
+    """
+    return execute_query(tracking_db_path, query, (limit,), fetch=True)
+
+
+def mark_articles_as_processing(tracking_db_path, article_ids):
+    if not article_ids:
+        return 0
+    try:
+        with db_connection(tracking_db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("PRAGMA table_info(crawled_articles)")
+            columns = [col[1] for col in cursor.fetchall()]
+            if "embedding_status" in columns:
+                placeholders = ",".join(["?"] * len(article_ids))
+                query = f"""
+                UPDATE crawled_articles 
+                SET embedding_status = 'processing' 
+                WHERE id IN ({placeholders})
+                """
+                cursor.execute(query, article_ids)
+                conn.commit()
+                return cursor.rowcount
+            else:
+                print("Note: embedding_status column doesn't exist in crawled_articles table.")
+                print("Skipping article marking step (this is non-critical).")
+                return len(article_ids)
+    except Exception as e:
+        print(f"Error marking articles as processing: {str(e)}")
+        print("Continuing without marking articles (this is non-critical).")
+        return 0
+
+
+def generate_embedding(client, text, model=EMBEDDING_MODEL):
+    try:
+        response = client.embeddings.create(input=text, model=model)
+        return response.data[0].embedding, model
+    except Exception as e:
+        print(f"Error generating embedding: {str(e)}")
+        return None, None
+
+
+def prepare_article_text(article):
+    title = article.get("title", "")
+    summary = article.get("summary", "")
+    content = article.get("content", "")
+    full_text = f"Title: {title}\n\nSummary: {summary}\n\nContent: {content}"
+    return full_text
+
+
+def store_embedding(tracking_db_path, article_id, embedding, model):
+    from datetime import datetime
+    import sqlite3
+    embedding_blob = np.array(embedding, dtype=np.float32).tobytes()
+    query = """
+    INSERT INTO article_embeddings 
+    (article_id, embedding, embedding_model, created_at, in_faiss_index)
+    VALUES (?, ?, ?, ?, 0)
+    """
+    params = (article_id, embedding_blob, model, datetime.now().isoformat())
+    try:
+        execute_query(tracking_db_path, query, params)
+        return True
+    except sqlite3.IntegrityError:
+        print(f"Warning: Embedding already exists for article {article_id}")
+        return False
+    except Exception as e:
+        print(f"Error storing embedding: {str(e)}")
+        return False
+
+
+def process_articles_for_embedding(tracking_db_path=None, openai_api_key=None, batch_size=20, delay_range=(1, 3)):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    if openai_api_key is None:
+        raise ValueError("OpenAI API key is required")
+    create_embedding_table(tracking_db_path)
+    client = OpenAI(api_key=openai_api_key)
+    articles = get_articles_without_embeddings(tracking_db_path, limit=batch_size)
+    if not articles:
+        print("No articles found that need embeddings")
+        return {"total_articles": 0, "success_count": 0, "failed_count": 0}
+    article_ids = [article["id"] for article in articles]
+    mark_articles_as_processing(tracking_db_path, article_ids)
+    stats = {"total_articles": len(articles), "success_count": 0, "failed_count": 0}
+    for i, article in enumerate(articles):
+        article_id = article["id"]
+        try:
+            print(f"[{i + 1}/{len(articles)}] Generating embedding for article {article_id}: {article['title']}")
+            text = prepare_article_text(article)
+            embedding, model = generate_embedding(client, text)
+            if embedding:
+                success = store_embedding(tracking_db_path, article_id, embedding, model)
+                if success:
+                    print(f"Successfully stored embedding for article {article_id}")
+                    stats["success_count"] += 1
+                else:
+                    print(f"Failed to store embedding for article {article_id}")
+                    stats["failed_count"] += 1
+            else:
+                print(f"Failed to generate embedding for article {article_id}")
+                stats["failed_count"] += 1
+        except Exception as e:
+            print(f"Error processing article {article_id}: {str(e)}")
+            stats["failed_count"] += 1
+        if i < len(articles) - 1:
+            delay = random.uniform(delay_range[0], delay_range[1])
+            time.sleep(delay)
+    return stats
+
+
+def print_stats(stats):
+    print("\nEmbedding Generation Statistics:")
+    print(f"Total articles processed: {stats['total_articles']}")
+    print(f"Successfully embedded: {stats['success_count']}")
+    print(f"Failed: {stats['failed_count']}")
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Generate embeddings for processed articles")
+    parser.add_argument("--api_key", help="OpenAI API Key (overrides environment variables)")
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=20,
+        help="Number of articles to process in each batch",
+    )
+    return parser.parse_args()
+
+
+def process_in_batches(
+    tracking_db_path=None,
+    openai_api_key=None,
+    batch_size=20,
+    total_batches=1,
+    delay_between_batches=10,
+):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    if openai_api_key is None:
+        raise ValueError("OpenAI API key is required")
+    total_stats = {"total_articles": 0, "success_count": 0, "failed_count": 0}
+    for i in range(total_batches):
+        print(f"\nProcessing batch {i + 1}/{total_batches}")
+        batch_stats = process_articles_for_embedding(
+            tracking_db_path=tracking_db_path,
+            openai_api_key=openai_api_key,
+            batch_size=batch_size,
+        )
+        total_stats["total_articles"] += batch_stats["total_articles"]
+        total_stats["success_count"] += batch_stats["success_count"]
+        total_stats["failed_count"] += batch_stats["failed_count"]
+        if batch_stats["total_articles"] == 0:
+            print("No more articles to process")
+            break
+        if i < total_batches - 1:
+            print(f"Waiting {delay_between_batches} seconds before next batch...")
+            time.sleep(delay_between_batches)
+    return total_stats
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    api_key = args.api_key or load_api_key()
+    if not api_key:
+        print("Error: No OpenAI API key provided. Please provide via --api_key or set OPENAI_API_KEY in .env file")
+        exit(1)
+    stats = process_in_batches(
+        openai_api_key=api_key,
+        batch_size=args.batch_size,
+        total_batches=3,
+    )
+    print_stats(stats)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/processors/faiss_indexing_processor.py

@@ -0,0 +1,339 @@
+import os
+import time
+import argparse
+import numpy as np
+import faiss
+from db.config import get_tracking_db_path, get_faiss_db_path
+from db.connection import db_connection, execute_query
+
+
+def initialize_faiss_index(dimension=1536, index_path=None, index_type="hnsw", n_list=100):
+    if index_path and os.path.exists(index_path):
+        print(f"Loading existing FAISS index from {index_path}")
+        try:
+            index = faiss.read_index(index_path)
+            print(f"Loaded index with {index.ntotal} vectors")
+            return index
+        except Exception as e:
+            print(f"Error loading FAISS index: {str(e)}")
+            print("Creating a new index instead")
+    print(f"Creating new FAISS index with dimension {dimension}, type: {index_type}")
+    if index_type == "flat":
+        return faiss.IndexFlatL2(dimension)
+
+    elif index_type == "ivfflat":
+        quantizer = faiss.IndexFlatL2(dimension)
+        index = faiss.IndexIVFFlat(quantizer, dimension, n_list)
+        print("Training IVF index with random vectors...")
+        train_size = max(10000, n_list * 10)
+        train_vectors = np.random.random((train_size, dimension)).astype(np.float32)
+        index.train(train_vectors)
+        index.nprobe = min(10, n_list // 10)
+        return index
+    elif index_type == "ivfpq":
+        quantizer = faiss.IndexFlatL2(dimension)
+        m = 16
+        bits = 8
+        index = faiss.IndexIVFPQ(quantizer, dimension, n_list, m, bits)
+        print("Training IVF-PQ index with random vectors...")
+        train_size = max(10000, n_list * 10)
+        train_vectors = np.random.random((train_size, dimension)).astype(np.float32)
+        index.train(train_vectors)
+        index.nprobe = min(10, n_list // 10)
+        return index
+    elif index_type == "hnsw":
+        m = 32
+        ef_construction = 100
+        index = faiss.IndexHNSWFlat(dimension, m)
+        index.hnsw.efConstruction = ef_construction
+        index.hnsw.efSearch = 64
+        return index
+    else:
+        print(f"Unknown index type '{index_type}', falling back to IVF Flat")
+        quantizer = faiss.IndexFlatL2(dimension)
+        index = faiss.IndexIVFFlat(quantizer, dimension, n_list)
+        print("Training IVF index with random vectors...")
+        train_size = max(10000, n_list * 10)
+        train_vectors = np.random.random((train_size, dimension)).astype(np.float32)
+        index.train(train_vectors)
+        index.nprobe = min(10, n_list // 10)
+        return index
+
+
+def save_faiss_index(index, index_path):
+    try:
+        index_dir = os.path.dirname(index_path)
+        os.makedirs(index_dir, exist_ok=True)
+        temp_path = f"{index_path}.tmp"
+        faiss.write_index(index, temp_path)
+        os.replace(temp_path, index_path)
+        print(f"FAISS index saved to {index_path}")
+        return True
+    except Exception as e:
+        print(f"Error saving FAISS index: {str(e)}")
+        return False
+
+
+def save_id_mapping(id_map, mapping_path):
+    try:
+        mapping_dir = os.path.dirname(mapping_path)
+        os.makedirs(mapping_dir, exist_ok=True)
+        np.save(mapping_path, np.array(id_map))
+        print(f"ID mapping saved to {mapping_path}")
+        return True
+    except Exception as e:
+        print(f"Error saving ID mapping: {str(e)}")
+        try:
+            print("Trying alternative save method...")
+            simple_path = os.path.join(mapping_dir, "article_id_map.npy")
+            np.save(simple_path, np.array(id_map))
+            print(f"ID mapping saved to {simple_path} (alternative path)")
+            return True
+        except Exception as e:
+            print(f"Alternative save method also failed: {str(e)}")
+            return False
+
+
+def load_id_mapping(mapping_path):
+    if os.path.exists(mapping_path):
+        try:
+            return np.load(mapping_path).tolist()
+        except Exception as e:
+            print(f"Error loading ID mapping: {str(e)}")
+    return []
+
+
+def get_embeddings_not_in_index(tracking_db_path, limit=100):
+    query = """
+    SELECT ae.id, ae.article_id, ae.embedding, ae.embedding_model
+    FROM article_embeddings ae
+    WHERE ae.in_faiss_index = 0
+    LIMIT ?
+    """
+    return execute_query(tracking_db_path, query, (limit,), fetch=True)
+
+
+def mark_embeddings_as_indexed(tracking_db_path, embedding_ids):
+    if not embedding_ids:
+        return 0
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        placeholders = ",".join(["?"] * len(embedding_ids))
+        query = f"""
+        UPDATE article_embeddings 
+        SET in_faiss_index = 1 
+        WHERE id IN ({placeholders})
+        """
+        cursor.execute(query, embedding_ids)
+        conn.commit()
+        return cursor.rowcount
+
+
+def add_embeddings_to_index(embeddings_data, faiss_index, id_map):
+    if not embeddings_data:
+        return 0, []
+    embeddings = []
+    article_ids = []
+    embedding_ids = []
+    for data in embeddings_data:
+        try:
+            embedding_blob = data["embedding"]
+            embedding = np.frombuffer(embedding_blob, dtype=np.float32)
+
+            if embedding.shape[0] != faiss_index.d:
+                print(f"Embedding dimension mismatch: expected {faiss_index.d}, got {embedding.shape[0]}")
+                continue
+            embeddings.append(embedding)
+            article_ids.append(data["article_id"])
+            embedding_ids.append(data["id"])
+        except Exception as e:
+            print(f"Error processing embedding {data['id']}: {str(e)}")
+    if not embeddings:
+        return 0, []
+    try:
+        embeddings_array = np.vstack(embeddings).astype(np.float32)
+        faiss_index.add(embeddings_array)
+        for article_id in article_ids:
+            id_map.append(article_id)
+        print(f"Added {len(embeddings)} embeddings to FAISS index")
+        return len(embeddings), embedding_ids
+    except Exception as e:
+        print(f"Error adding embeddings to FAISS index: {str(e)}")
+        return 0, []
+
+
+def process_embeddings_for_indexing(
+    tracking_db_path=None,
+    index_path=None,
+    mapping_path=None,
+    batch_size=100,
+    index_type="ivfflat",
+    n_list=100,
+):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    index_dir = os.path.dirname(index_path)
+    os.makedirs(index_dir, exist_ok=True)
+    id_map = load_id_mapping(mapping_path)
+    with db_connection(tracking_db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute("""
+        SELECT name FROM sqlite_master 
+        WHERE type='table' AND name='article_embeddings'
+        """)
+        table_exists = cursor.fetchone() is not None
+
+        if not table_exists:
+            print("article_embeddings table does not exist. Please run embedding_processor first.")
+            return {"processed": 0, "added": 0, "errors": 0, "total_vectors": 0, "status": "table_missing"}
+    sample_query = """
+    SELECT embedding FROM article_embeddings LIMIT 1
+    """
+    sample = execute_query(tracking_db_path, sample_query, fetch=True, fetch_one=True)
+    if not sample:
+        print("No embeddings found in the database")
+        default_dimension = 1536
+        print(f"Using default dimension: {default_dimension}")
+
+        faiss_index = initialize_faiss_index(
+            dimension=default_dimension, index_path=index_path if os.path.exists(index_path) else None, index_type=index_type, n_list=n_list
+        )
+        return {
+            "processed": 0,
+            "added": 0,
+            "errors": 0,
+            "total_vectors": faiss_index.ntotal if hasattr(faiss_index, "ntotal") else 0,
+            "status": "no_embeddings",
+        }
+    embedding_dimension = len(np.frombuffer(sample["embedding"], dtype=np.float32))
+    print(f"Detected embedding dimension: {embedding_dimension}")
+    faiss_index = initialize_faiss_index(dimension=embedding_dimension, index_path=index_path, index_type=index_type, n_list=n_list)
+    embeddings_data = get_embeddings_not_in_index(tracking_db_path, limit=batch_size)
+    if not embeddings_data:
+        print("No new embeddings to add to the index")
+        return {"processed": 0, "added": 0, "errors": 0, "total_vectors": faiss_index.ntotal, "status": "no_new_embeddings"}
+    added_count, embedding_ids = add_embeddings_to_index(embeddings_data, faiss_index, id_map)
+    if added_count > 0:
+        save_faiss_index(faiss_index, index_path)
+        save_id_mapping(id_map, mapping_path)
+        marked_count = mark_embeddings_as_indexed(tracking_db_path, embedding_ids)
+        print(f"Marked {marked_count} embeddings as indexed in the database")
+    stats = {
+        "processed": len(embeddings_data),
+        "added": added_count,
+        "errors": len(embeddings_data) - added_count,
+        "total_vectors": faiss_index.ntotal,
+        "index_type": index_type,
+        "status": "success",
+    }
+    return stats
+
+
+def process_in_batches(
+    tracking_db_path=None,
+    index_path=None,
+    mapping_path=None,
+    batch_size=100,
+    total_batches=5,
+    delay_between_batches=2,
+    index_type="ivfflat",
+    n_list=100,
+):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    total_stats = {"processed": 0, "added": 0, "errors": 0, "index_type": index_type}
+    for i in range(total_batches):
+        print(f"\nProcessing batch {i + 1}/{total_batches}")
+        batch_stats = process_embeddings_for_indexing(
+            tracking_db_path=tracking_db_path,
+            index_path=index_path,
+            mapping_path=mapping_path,
+            batch_size=batch_size,
+            index_type=index_type,
+            n_list=n_list,
+        )
+        total_stats["processed"] += batch_stats["processed"]
+        total_stats["added"] += batch_stats["added"]
+        total_stats["errors"] += batch_stats["errors"]
+        if "total_vectors" in batch_stats:
+            total_stats["total_vectors"] = batch_stats["total_vectors"]
+        if batch_stats["processed"] == 0:
+            print("No more embeddings to process")
+            break
+        if i < total_batches - 1:
+            print(f"Waiting {delay_between_batches} seconds before next batch...")
+            time.sleep(delay_between_batches)
+    return total_stats
+
+
+def print_stats(stats):
+    print("\nFAISS Indexing Statistics:")
+    print(f"Total embeddings processed: {stats['processed']}")
+    print(f"Successfully added to index: {stats['added']}")
+    print(f"Errors: {stats['errors']}")
+    if "total_vectors" in stats:
+        print(f"Total vectors in index: {stats['total_vectors']}")
+    if "index_type" in stats:
+        print(f"Index type: {stats['index_type']}")
+        if stats["index_type"] == "flat":
+            print("Index performance: Most accurate but slowest for large datasets")
+        elif stats["index_type"] == "ivfflat":
+            print("Index performance: Good balance of accuracy and speed")
+        elif stats["index_type"] == "ivfpq":
+            print("Index performance: Memory efficient, good for very large datasets")
+        elif stats["index_type"] == "hnsw":
+            print("Index performance: Excellent search speed with good accuracy")
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Process embeddings and add to FAISS index")
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=100,
+        help="Number of embeddings to process in each batch",
+    )
+    parser.add_argument(
+        "--index_path",
+        default="databases/faiss/article_index.faiss",
+        help="Path to save the FAISS index",
+    )
+    parser.add_argument(
+        "--mapping_path",
+        default="databases/faiss/article_id_map.npy",
+        help="Path to save the ID mapping file",
+    )
+    parser.add_argument(
+        "--index_type",
+        choices=["flat", "ivfflat", "ivfpq", "hnsw"],
+        default="hnsw",
+        help="Type of FAISS index to create",
+    )
+    parser.add_argument(
+        "--n_list",
+        type=int,
+        default=100,
+        help="Number of clusters for IVF-based indexes",
+    )
+    parser.add_argument(
+        "--total_batches",
+        type=int,
+        default=5,
+        help="Total number of batches to process",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    index_path, mapping_path = get_faiss_db_path()
+    index_path = args.index_path or index_path
+    mapping_path = args.mapping_path or mapping_path
+    stats = process_in_batches(
+        batch_size=args.batch_size,
+        index_path=index_path,
+        mapping_path=mapping_path,
+        total_batches=args.total_batches,
+        index_type=args.index_type,
+        n_list=args.n_list,
+    )

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/processors/fb_scraper_processor.py

@@ -0,0 +1,24 @@
+
+import sys
+from datetime import datetime
+from db.config import get_social_media_db_path
+from tools.social.fb_scraper import crawl_facebook_feed
+
+
+def main():
+    print(f"Starting facebook.com feed scraping at {datetime.now().isoformat()}")
+    db_path = get_social_media_db_path()
+
+    try:
+        print("Running facebook.com feed scraper")
+        posts = crawl_facebook_feed("https://facebook.com", db_file=db_path)
+        print(f"facebook.com scraping completed at {datetime.now().isoformat()}")
+        print(f"Collected {posts} posts from feed")
+
+    except Exception as e:
+        print(f"Error executing facebook.com feed scraper: {str(e)}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/processors/feed_processor.py

@@ -0,0 +1,89 @@
+import time
+import random
+from utils.rss_feed_parser import get_feed_data
+from db.config import get_sources_db_path, get_tracking_db_path
+from db.feeds import (
+    get_active_feeds,
+    count_active_feeds,
+    get_feed_tracking_info,
+    update_feed_tracking,
+    store_feed_entries,
+    update_tracking_info,
+)
+
+
+def fetch_and_process_feeds(sources_db_path=None, tracking_db_path=None, delay_between_feeds=2, batch_size=100):
+    if sources_db_path is None:
+        sources_db_path = get_sources_db_path()
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    total_feeds = count_active_feeds(sources_db_path)
+    stats = {
+        "total_feeds": total_feeds,
+        "processed_feeds": 0,
+        "new_entries": 0,
+        "unchanged_feeds": 0,
+        "failed_feeds": 0,
+    }
+    offset = 0
+    while offset < total_feeds:
+        feeds = get_active_feeds(sources_db_path, limit=batch_size, offset=offset)
+        if not feeds:
+            break
+        update_tracking_info(tracking_db_path, feeds)
+        for feed in feeds:
+            feed_id = feed["id"]
+            source_id = feed["source_id"]
+            feed_url = feed["feed_url"]
+            tracking_info = get_feed_tracking_info(tracking_db_path, feed_id)
+            etag = tracking_info.get("last_etag") if tracking_info else None
+            modified = tracking_info.get("last_modified") if tracking_info else None
+            last_hash = tracking_info.get("entry_hash") if tracking_info else None
+            try:
+                feed_data = get_feed_data(feed_url, etag=etag, modified=modified)
+                if not feed_data["is_rss_feed"]:
+                    print(f"Feed {feed_url} is not a valid RSS feed")
+                    stats["failed_feeds"] += 1
+                    continue
+                if feed_data["status"] == 304:
+                    print(f"Feed {feed_url} not modified since last check")
+                    stats["unchanged_feeds"] += 1
+                    continue
+                current_hash = feed_data["current_hash"]
+                if last_hash and current_hash == last_hash:
+                    print(f"Feed {feed_url} content unchanged based on hash")
+                    stats["unchanged_feeds"] += 1
+                    continue
+                parsed_entries = feed_data["parsed_entries"]
+                if parsed_entries:
+                    new_entries = store_feed_entries(tracking_db_path, feed_id, source_id, parsed_entries)
+                    stats["new_entries"] += new_entries
+                    print(f"Stored {new_entries} new entries from {feed_url}")
+                update_feed_tracking(
+                    tracking_db_path,
+                    feed_id,
+                    feed_data["etag"],
+                    feed_data["modified"],
+                    current_hash,
+                )
+                stats["processed_feeds"] += 1
+            except Exception as e:
+                print(f"Error processing feed {feed_url}: {str(e)}")
+                stats["failed_feeds"] += 1
+            time.sleep(random.uniform(1, delay_between_feeds))
+        offset += batch_size
+    return stats
+
+
+def print_stats(stats):
+    print("\nFeed Processing Statistics:")
+    print(f"Total feeds: {stats['total_feeds']}")
+    print(f"Processed feeds: {stats['processed_feeds']}")
+    print(f"Unchanged feeds: {stats['unchanged_feeds']}")
+    print(f"Failed feeds: {stats['failed_feeds']}")
+    print(f"New entries: {stats['new_entries']}")
+
+
+if __name__ == "__main__":
+    stats = fetch_and_process_feeds()
+    print_stats(stats)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/processors/podcast_generator_processor.py

@@ -0,0 +1,333 @@
+import os
+import json
+from datetime import datetime
+from typing import List, Dict, Any, Optional
+from tools.pipeline.search_agent import search_agent_run
+from tools.pipeline.scrape_agent import scrape_agent_run
+from tools.pipeline.script_agent import script_agent_run
+from tools.pipeline.image_generate_agent import image_generation_agent_run
+from db.config import get_tracking_db_path, get_podcasts_db_path, get_tasks_db_path
+from db.podcast_configs import get_podcast_config, get_all_podcast_configs
+from db.agent_config_v2 import AVAILABLE_LANGS
+from utils.tts_engine_selector import generate_podcast_audio
+from utils.load_api_keys import load_api_key
+from tools.session_state_manager import _save_podcast_to_database_sync
+
+PODCAST_ASSETS_DIR = "podcasts"
+
+
+def get_language_name(language_code: str) -> str:
+    language_map = {lang["code"]: lang["name"] for lang in AVAILABLE_LANGS}
+    return language_map.get(language_code, "English")
+
+
+def convert_script_to_audio_format(podcast_data: Dict[str, Any]) -> Dict[str, List[Dict[str, Any]]]:
+    speaker_map = {"ALEX": 1, "MORGAN": 2}
+    dict_entries = []
+    for section in podcast_data.get("sections", []):
+        for dialog in section.get("dialog", []):
+            speaker = dialog.get("speaker", "ALEX")
+            text = dialog.get("text", "")
+            if text and speaker in speaker_map:
+                dict_entries.append({"text": text, "speaker": speaker_map[speaker]})
+    return {"entries": dict_entries}
+
+
+def generate_podcast_from_prompt_v2(
+    prompt: str,
+    openai_api_key: str,
+    tracking_db_path: Optional[str] = None,
+    podcasts_db_path: Optional[str] = None,
+    output_dir: str = PODCAST_ASSETS_DIR,
+    tts_engine: str = "kokoro",
+    language_code: str = "en",
+    podcast_script_prompt: Optional[str] = None,
+    image_prompt: Optional[str] = None,
+    debug: bool = False,
+) -> Dict[str, Any]:
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    if podcasts_db_path is None:
+        podcasts_db_path = get_podcasts_db_path()
+    os.makedirs(output_dir, exist_ok=True)
+    images_dir = os.path.join(output_dir, "images")
+    os.makedirs(images_dir, exist_ok=True)
+    print(f"Starting enhanced podcast generation for prompt: {prompt}")
+    try:
+        search_results = search_agent_run(prompt)
+        if not search_results:
+            print(f"WARNING: No search results found for prompt: {prompt}")
+            return {"error": "No search results found"}
+        print(f"Found {len(search_results)} search results")
+        if debug:
+            print("Search results:", json.dumps(search_results[:2], indent=2))
+    except Exception as e:
+        print(f"ERROR: Search agent failed: {e}")
+        return {"error": f"Search agent failed: {str(e)}"}
+    try:
+        scraped_results = scrape_agent_run(prompt, search_results)
+        if not scraped_results:
+            print("WARNING: No content could be scraped")
+            return {"error": "No content could be scraped"}
+        confirmed_results = []
+        for result in scraped_results:
+            if result.get("full_text") and len(result["full_text"].strip()) > 100:
+                result["confirmed"] = True
+                confirmed_results.append(result)
+        if not confirmed_results:
+            print("WARNING: No high-quality content available after scraping")
+            return {"error": "No high-quality content available"}
+        print(f"Successfully scraped {len(confirmed_results)} high-quality articles")
+        if debug:
+            print("Sample scraped content:", confirmed_results[0].get("full_text", "")[:200])
+    except Exception as e:
+        print(f"ERROR: Scrape agent failed: {e}")
+        return {"error": f"Scrape agent failed: {str(e)}"}
+    try:
+        language_name = get_language_name(language_code)
+        podcast_data = script_agent_run(query=prompt, search_results=confirmed_results, language_name=language_name)
+        if not podcast_data or not isinstance(podcast_data, dict):
+            print("ERROR: Failed to generate podcast script")
+            return {"error": "Failed to generate podcast script"}
+        if not podcast_data.get("sections"):
+            print("ERROR: Generated podcast script is missing required sections")
+            return {"error": "Invalid podcast script structure"}
+        print(f"Generated script with {len(podcast_data['sections'])} sections")
+        if debug:
+            print("Script title:", podcast_data.get("title", "No title"))
+    except Exception as e:
+        print(f"ERROR: Script agent failed: {e}")
+        return {"error": f"Script agent failed: {str(e)}"}
+    banner_filenames = []
+    banner_url = None
+    try:
+        image_query = image_prompt if image_prompt else prompt
+        image_result = image_generation_agent_run(image_query, podcast_data)
+        if image_result and image_result.get("banner_images"):
+            banner_filenames = image_result["banner_images"]
+            banner_url = image_result.get("banner_url")
+            print(f"Generated {len(banner_filenames)} banner images")
+        else:
+            print("WARNING: No images were generated")
+    except Exception as e:
+        print(f"ERROR: Image generation failed: {e}")
+    audio_filename = None
+    full_audio_path = None
+    try:
+        audio_format = convert_script_to_audio_format(podcast_data)
+        audio_filename = f"podcast_{datetime.now().strftime('%Y%m%d_%H%M%S')}.wav"
+        audio_path = os.path.join(output_dir, "audio", audio_filename)
+
+        class DictPodcastScript:
+            def __init__(self, entries):
+                self.entries = entries
+
+            def __iter__(self):
+                return iter(self.entries)
+
+        script_obj = DictPodcastScript(audio_format["entries"])
+        full_audio_path = generate_podcast_audio(
+            script=script_obj,
+            output_path=audio_path,
+            tts_engine=tts_engine,
+            language_code=language_code,
+        )
+        if full_audio_path:
+            print(f"Generated podcast audio: {full_audio_path}")
+        else:
+            print("ERROR: Failed to generate audio")
+            audio_filename = None
+    except Exception as e:
+        print(f"ERROR: Error generating audio: {e}")
+        import traceback
+
+        traceback.print_exc()
+        audio_filename = None
+    try:
+        session_state = {
+            "generated_script": podcast_data,
+            "banner_url": banner_url,
+            "banner_images": banner_filenames,
+            "audio_url": full_audio_path,
+            "tts_engine": tts_engine,
+            "selected_language": {"code": language_code, "name": get_language_name(language_code)},
+            "podcast_info": {"topic": prompt},
+        }
+        success, message, podcast_id = _save_podcast_to_database_sync(session_state)
+        if success:
+            print(f"Stored podcast data with ID: {podcast_id}")
+        else:
+            print(f"ERROR: Failed to save to database: {message}")
+            podcast_id = 0
+    except Exception as e:
+        print(f"ERROR: Error storing podcast data: {e}")
+        podcast_id = 0
+    if audio_filename:
+        frontend_audio_path = os.path.join(output_dir, audio_filename).replace("\\", "/")
+    else:
+        frontend_audio_path = None
+    if banner_url:
+        frontend_banner_path = banner_url.replace("\\", "/")
+    else:
+        frontend_banner_path = None
+    return {
+        "podcast_id": podcast_id,
+        "title": podcast_data.get("title", "Podcast"),
+        "audio_path": frontend_audio_path,
+        "banner_path": frontend_banner_path,
+        "banner_images": banner_filenames,
+        "script": podcast_data,
+        "tts_engine": tts_engine,
+        "language": language_code,
+        "sources_count": len(confirmed_results),
+        "processing_stats": {
+            "search_results": len(search_results),
+            "scraped_results": len(scraped_results),
+            "confirmed_results": len(confirmed_results),
+            "images_generated": len(banner_filenames),
+            "audio_generated": bool(audio_filename),
+        },
+    }
+
+
+def generate_podcast_from_config_v2(
+    config_id: int,
+    openai_api_key: str,
+    tracking_db_path: Optional[str] = None,
+    podcasts_db_path: Optional[str] = None,
+    tasks_db_path: Optional[str] = None,
+    output_dir: str = PODCAST_ASSETS_DIR,
+    debug: bool = False,
+) -> Dict[str, Any]:
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    if podcasts_db_path is None:
+        podcasts_db_path = get_podcasts_db_path()
+    if tasks_db_path is None:
+        tasks_db_path = get_tasks_db_path()
+    config = get_podcast_config(tasks_db_path, config_id)
+    if not config:
+        print(f"ERROR: Podcast configuration not found: {config_id}")
+        return {"error": f"Podcast configuration not found: {config_id}"}
+    prompt = config.get("prompt", "")
+    time_range_hours = config.get("time_range_hours", 24)
+    limit_articles = config.get("limit_articles", 20)
+    tts_engine = config.get("tts_engine", "elevenlabs")
+    language_code = config.get("language_code", "en")
+    podcast_script_prompt = config.get("podcast_script_prompt")
+    image_prompt = config.get("image_prompt")
+    print(f"Generating podcast with enhanced config: {config.get('name', 'Unnamed')}")
+    print(f"Prompt: {prompt}")
+    print(f"Time range: {time_range_hours} hours")
+    print(f"Limit: {limit_articles} articles")
+    print(f"TTS Engine: {tts_engine}")
+    print(f"Language: {language_code}")
+    return generate_podcast_from_prompt_v2(
+        prompt=prompt,
+        openai_api_key=openai_api_key,
+        tracking_db_path=tracking_db_path,
+        podcasts_db_path=podcasts_db_path,
+        output_dir=output_dir,
+        tts_engine=tts_engine,
+        language_code=language_code,
+        podcast_script_prompt=podcast_script_prompt,
+        image_prompt=image_prompt,
+        debug=debug,
+    )
+
+
+def process_all_active_configs_v2(
+    openai_api_key: str,
+    tracking_db_path: Optional[str] = None,
+    podcasts_db_path: Optional[str] = None,
+    tasks_db_path: Optional[str] = None,
+    output_dir: str = PODCAST_ASSETS_DIR,
+    debug: bool = False,
+) -> List[Dict[str, Any]]:
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    if podcasts_db_path is None:
+        podcasts_db_path = get_podcasts_db_path()
+    if tasks_db_path is None:
+        tasks_db_path = get_tasks_db_path()
+    configs = get_all_podcast_configs(tasks_db_path, active_only=True)
+    if not configs:
+        print("WARNING: No active podcast configurations found")
+        return [{"error": "No active podcast configurations found"}]
+    results = []
+    total_configs = len(configs)
+    print(f"Processing {total_configs} active podcast configurations with enhanced pipeline...")
+    for i, config in enumerate(configs, 1):
+        config_id = config["id"]
+        config_name = config["name"]
+        print(f"\n[{i}/{total_configs}] Processing podcast configuration {config_id}: {config_name}")
+        try:
+            result = generate_podcast_from_config_v2(
+                config_id=config_id,
+                openai_api_key=openai_api_key,
+                tracking_db_path=tracking_db_path,
+                podcasts_db_path=podcasts_db_path,
+                tasks_db_path=tasks_db_path,
+                output_dir=output_dir,
+                debug=debug,
+            )
+            result["config_id"] = config_id
+            result["config_name"] = config_name
+            results.append(result)
+            if "error" not in result:
+                stats = result.get("processing_stats", {})
+                print(f"Success - Podcast ID: {result.get('podcast_id', 'Unknown')}")
+                print(f"Sources: {stats.get('confirmed_results', 0)} articles processed")
+                print(f"Images: {stats.get('images_generated', 0)} generated")
+                print(f"Audio: {'Yes' if stats.get('audio_generated') else 'No'}")
+            else:
+                print(f"Failed: {result['error']}")
+        except Exception as e:
+            print(f"ERROR: Error generating podcast for config {config_id}: {e}")
+            results.append({"config_id": config_id, "config_name": config_name, "error": str(e)})
+    return results
+
+
+def main():
+    openai_api_key = load_api_key()
+    tasks_db_path = get_tasks_db_path()
+    if not openai_api_key:
+        print("ERROR: No OpenAI API key provided. Please set OPENAI_API_KEY environment variable.")
+        return 1
+    output_dir = PODCAST_ASSETS_DIR
+    debug = False
+    print("Starting Enhanced Agent-Based Podcast Generation System")
+    print("=" * 60)
+    results = process_all_active_configs_v2(
+        openai_api_key=openai_api_key,
+        tasks_db_path=tasks_db_path,
+        output_dir=output_dir,
+        debug=debug,
+    )
+    print("\n" + "=" * 60)
+    print("PODCAST GENERATION RESULTS SUMMARY")
+    print("=" * 60)
+    successful = 0
+    failed = 0
+    for result in results:
+        config_id = result.get("config_id", "Unknown")
+        config_name = result.get("config_name", "Unknown")
+        if "error" in result:
+            print(f"Config {config_id} ({config_name}): {result['error']}")
+            failed += 1
+        else:
+            podcast_id = result.get("podcast_id", "Unknown")
+            stats = result.get("processing_stats", {})
+            print(f"Config {config_id} ({config_name}): Success")
+            print(f"Podcast ID: {podcast_id}")
+            print(f"Sources: {stats.get('confirmed_results', 0)} articles")
+            print(f"Images: {stats.get('images_generated', 0)} generated")
+            successful += 1
+    print("=" * 60)
+    print(f"FINAL STATS: {successful} successful, {failed} failed out of {len(results)} total")
+    print("=" * 60)
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/processors/url_processor.py

@@ -0,0 +1,82 @@
+from db.config import get_tracking_db_path
+from db.feeds import get_uncrawled_entries
+from db.articles import store_crawled_article, update_entry_status
+from utils.crawl_url import get_web_data
+
+
+def crawl_pending_entries(tracking_db_path=None, batch_size=20, delay_range=(1, 3), max_attempts=3):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    entries = get_uncrawled_entries(tracking_db_path, limit=batch_size, max_attempts=max_attempts)
+    stats = {
+        "total_entries": len(entries),
+        "success_count": 0,
+        "failed_count": 0,
+        "skipped_count": 0,
+    }
+    for entry in entries:
+        entry_id = entry["id"]
+        url = entry["link"]
+        if not url or url.strip() == "":
+            update_entry_status(tracking_db_path, entry_id, "skipped")
+            stats["skipped_count"] += 1
+            continue
+        print(f"Crawling URL: {url}")
+        try:
+            web_data = get_web_data(url)
+            if not web_data or not web_data["raw_html"]:
+                print(f"No content retrieved for {url}")
+                update_entry_status(tracking_db_path, entry_id, "failed")
+                stats["failed_count"] += 1
+                continue
+            success = store_crawled_article(tracking_db_path, entry, web_data["raw_html"], web_data["metadata"])
+            if success:
+                update_entry_status(tracking_db_path, entry_id, "success")
+                stats["success_count"] += 1
+                print(f"Successfully crawled: {url}")
+            else:
+                update_entry_status(tracking_db_path, entry_id, "failed")
+                stats["failed_count"] += 1
+                print(f"Failed to store: {url} (likely duplicate)")
+        except Exception as e:
+            print(f"Error crawling {url}: {str(e)}")
+            update_entry_status(tracking_db_path, entry_id, "failed")
+            stats["failed_count"] += 1
+    return stats
+
+
+def print_stats(stats):
+    print("\nCrawl Statistics:")
+    print(f"Total entries processed: {stats['total_entries']}")
+    print(f"Successfully crawled: {stats['success_count']}")
+    print(f"Failed: {stats['failed_count']}")
+    print(f"Skipped (no URL): {stats['skipped_count']}")
+
+
+def crawl_in_batches(tracking_db_path=None, batch_size=20, total_batches=5, delay_between_batches=10):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    total_stats = {
+        "total_entries": 0,
+        "success_count": 0,
+        "failed_count": 0,
+        "skipped_count": 0,
+    }
+    for i in range(total_batches):
+        print(f"\nProcessing batch {i + 1}/{total_batches}")
+        batch_stats = crawl_pending_entries(tracking_db_path=tracking_db_path, batch_size=batch_size)
+        total_stats["total_entries"] += batch_stats["total_entries"]
+        total_stats["success_count"] += batch_stats["success_count"]
+        total_stats["failed_count"] += batch_stats["failed_count"]
+        total_stats["skipped_count"] += batch_stats["skipped_count"]
+        if batch_stats["total_entries"] == 0:
+            print("No more entries to process")
+            break
+        if i < total_batches - 1:
+            print(f"Waiting {delay_between_batches} seconds before next batch...")
+    return total_stats
+
+
+if __name__ == "__main__":
+    stats = crawl_in_batches(batch_size=20, total_batches=50)
+    print_stats(stats)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/processors/x_scraper_processor.py

@@ -0,0 +1,24 @@
+
+import sys
+from datetime import datetime
+from db.config import get_social_media_db_path
+from tools.social.x_scraper import crawl_x_profile
+
+
+def main():
+    print(f"Starting X.com feed scraping at {datetime.now().isoformat()}")
+    db_path = get_social_media_db_path()
+
+    try:
+        print("Running X.com feed scraper")
+        posts = crawl_x_profile("home", db_file=db_path)
+        print(f"X.com feed scraping completed at {datetime.now().isoformat()}")
+        print(f"Collected {posts} posts from feed")
+
+    except Exception as e:
+        print(f"Error executing X.com feed scraper: {str(e)}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/requirements.txt

@@ -0,0 +1,236 @@
+addict==2.4.0
+agno==1.4.2
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.18
+aioredis==2.0.1
+aiosignal==1.3.2
+aiosqlite==0.21.0
+amqp==5.3.1
+annotated-types==0.7.0
+anthropic==0.49.0
+anyio==4.9.0
+APScheduler==3.11.0
+asgi-csrf==0.11
+asgiref==3.8.1
+async-timeout==5.0.1
+attrs==25.3.0
+audioread==3.0.1
+babel==2.17.0
+backoff==2.2.1
+beautifulsoup4==4.13.4
+billiard==4.2.1
+blis==1.2.1
+boto3==1.38.3
+botocore==1.38.3
+browser-use==0.2.4
+cachetools==5.5.2
+catalogue==2.0.10
+celery==5.5.2
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+click==8.1.8
+click-default-group==1.2.4
+click-didyoumean==0.3.1
+click-plugins==1.1.1
+click-repl==0.3.0
+cloudpathlib==0.21.0
+colorama==0.4.6
+confection==0.1.5
+csvw==3.5.1
+curated-tokenizers==0.0.9
+curated-transformers==0.1.1
+cymem==2.0.11
+Cython==3.0.12
+datasette==0.65.1
+defusedxml==0.7.1
+distro==1.9.0
+dlinfo==2.0.0
+dnspython==2.7.0
+docopt==0.6.2
+docstring_parser==0.16
+duckduckgo_search==8.0.1
+elevenlabs==1.57.0
+espeakng-loader==0.2.4
+faiss-cpu==1.9.0.post1
+fastapi==0.115.12
+feedparser==6.0.11
+filelock==3.18.0
+filetype==1.2.0
+flexcache==0.3
+flexparser==0.4
+flower==2.0.1
+frozenlist==1.6.0
+fsspec==2025.3.2
+gitdb==4.0.12
+GitPython==3.1.44
+gnews==0.4.1
+google-ai-generativelanguage==0.6.17
+google-api-core==2.24.2
+google-auth==2.39.0
+googleapis-common-protos==1.70.0
+greenlet==3.2.0
+grpcio==1.72.0rc1
+grpcio-status==1.72.0rc1
+h11==0.14.0
+h2==4.2.0
+hpack==4.1.0
+httpcore==1.0.8
+httpx==0.28.1
+huggingface-hub==0.30.2
+humanize==4.12.3
+hupper==1.12.1
+hyperframe==6.1.0
+idna==3.10
+isodate==0.7.2
+itsdangerous==2.2.0
+janus==2.0.0
+Jinja2==3.1.6
+jiter==0.9.0
+jmespath==1.0.1
+joblib==1.4.2
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+kokoro==0.9.4
+kombu==5.5.3
+langchain==0.3.22
+langchain-anthropic==0.3.3
+langchain-aws==0.2.19
+langchain-core==0.3.49
+langchain-deepseek==0.1.3
+langchain-google-genai==2.1.2
+langchain-ollama==0.3.0
+langchain-openai==0.3.11
+langchain-text-splitters==0.3.7
+langcodes==3.5.0
+langsmith==0.3.31
+language-tags==1.2.0
+language_data==1.3.0
+lazy_loader==0.4
+librosa==0.11.0
+linkify-it-py==2.0.3
+llvmlite==0.44.0
+loguru==0.7.3
+lxml==5.4.0
+lxml_html_clean==0.4.2
+marisa-trie==1.2.1
+markdown-it-py==3.0.0
+markdownify==1.1.0
+MarkupSafe==3.0.2
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+mem0ai==0.1.93
+mergedeep==1.3.4
+misaki==0.9.4
+monotonic==1.6
+mpmath==1.3.0
+msgpack==1.1.0
+multidict==6.4.3
+murmurhash==1.0.12
+networkx==3.4.2
+newspaper4k==0.9.3.1
+nltk==3.9.1
+num2words==0.5.14
+numba==0.61.2
+numpy==1.26.4
+ollama==0.4.7
+openai==1.74.0
+orjson==3.10.16
+packaging==24.2
+pandas==2.2.3
+phonemizer-fork==3.3.2
+pillow==11.2.1
+pip-chill==1.0.3
+playwright==1.52.0
+pluggy==1.6.0
+pooch==1.8.2
+portalocker==2.10.1
+posthog==3.25.0
+preshed==3.0.9
+primp==0.15.0
+prometheus_client==0.22.0
+propcache==0.3.1
+proto-plus==1.26.1
+protobuf==6.30.2
+psycopg2-binary==2.9.10
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.22
+pydantic==2.10.6
+pydantic-settings==2.9.1
+pydantic_core==2.27.2
+pydub==0.25.1
+pyee==13.0.0
+PyJWT==2.9.0
+pyparsing==3.2.3
+pyperclip==1.9.0
+python-dotenv==1.1.0
+python-multipart==0.0.20
+pytz==2024.2
+PyYAML==6.0.2
+qdrant-client==1.14.2
+rdflib==7.1.4
+redis==5.3.0
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+requests-file==2.1.0
+requests-toolbelt==1.0.0
+rfc3986==1.5.0
+rich==14.0.0
+rpds-py==0.24.0
+rsa==4.9.1
+s3transfer==0.12.0
+safetensors==0.5.3
+scikit-learn==1.6.1
+scipy==1.15.2
+screeninfo==0.8.1
+segments==2.3.0
+sentence-transformers==4.1.0
+sgmllib3k==1.0.0
+shellingham==1.5.4
+smart-open==7.1.0
+smmap==5.0.2
+sniffio==1.3.1
+soundfile==0.13.1
+soupsieve==2.6
+soxr==0.5.0.post1
+spacy==3.8.5
+spacy-curated-transformers==0.3.0
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+SQLAlchemy==2.0.40
+srsly==2.5.1
+starlette==0.46.2
+sympy==1.14.0
+tabulate==0.9.0
+tenacity==9.1.2
+textual==3.2.0
+thinc==8.3.4
+threadpoolctl==3.6.0
+tiktoken==0.9.0
+tldextract==5.3.0
+tokenizers==0.21.1
+tomli==2.2.1
+torch==2.2.2
+tqdm==4.67.1
+transformers==4.51.3
+typer==0.15.2
+typing-inspection==0.4.0
+tzdata==2025.2
+tzlocal==5.3.1
+uc-micro-py==1.0.3
+uritemplate==4.1.1
+urllib3==2.4.0
+uvicorn==0.34.2
+vine==5.1.0
+wasabi==1.1.3
+weasel==0.4.1
+websockets==15.0.1
+widgetsnbextension==4.0.14
+wrapt==1.17.2
+yarl==1.20.0
+zstandard==0.23.0

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/routers/article_router.py

@@ -0,0 +1,54 @@
+from fastapi import APIRouter, Query
+from typing import List, Optional, Dict, Any
+from models.article_schemas import Article, PaginatedArticles
+from services.article_service import article_service
+
+router = APIRouter()
+
+
+@router.get("/", response_model=PaginatedArticles)
+async def read_articles(
+    page: int = Query(1, ge=1, description="Page number"),
+    per_page: int = Query(10, ge=1, le=100, description="Items per page"),
+    source: Optional[str] = Query(None, description="Filter by source name"),
+    category: Optional[str] = Query(None, description="Filter by category"),
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+    search: Optional[str] = Query(None, description="Search in title and summary"),
+):
+    """
+    Get all articles with pagination and filtering.
+
+    - **page**: Page number (starting from 1)
+    - **per_page**: Number of items per page (max 100)
+    - **source**: Filter by source name
+    - **category**: Filter by category
+    - **date_from**: Filter by start date (format: YYYY-MM-DD)
+    - **date_to**: Filter by end date (format: YYYY-MM-DD)
+    - **search**: Search in title and summary
+    """
+    return await article_service.get_articles(
+        page=page, per_page=per_page, source=source, category=category, date_from=date_from, date_to=date_to, search=search
+    )
+
+
+@router.get("/{article_id}", response_model=Article)
+async def read_article(article_id: int):
+    """
+    Get a specific article by ID.
+
+    - **article_id**: ID of the article to retrieve
+    """
+    return await article_service.get_article(article_id=article_id)
+
+
+@router.get("/sources/list", response_model=List[str])
+async def read_sources():
+    """Get all available sources."""
+    return await article_service.get_sources()
+
+
+@router.get("/categories/list", response_model=List[Dict[str, Any]])
+async def read_categories():
+    """Get all available categories with article counts."""
+    return await article_service.get_categories()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/routers/async_podcast_agent_router.py

@@ -0,0 +1,72 @@
+from fastapi import APIRouter
+from typing import Optional
+from pydantic import BaseModel
+from services.async_podcast_agent_service import podcast_agent_service
+
+router = APIRouter()
+
+
+class SessionRequest(BaseModel):
+    session_id: Optional[str] = None
+
+
+class ChatRequest(BaseModel):
+    session_id: str
+    message: str
+
+
+class ChatResponse(BaseModel):
+    session_id: str
+    response: str
+    stage: str
+    session_state: str
+    is_processing: bool = False
+    process_type: Optional[str] = None
+    task_id: Optional[str] = None
+    browser_recording_path: Optional[str] = None
+
+
+class StatusRequest(BaseModel):
+    session_id: str
+    task_id: Optional[str] = None  
+
+
+@router.post("/session")
+async def create_session(request: SessionRequest = None):
+    """Create or reuse a session with the podcast agent"""
+    return await podcast_agent_service.create_session(request)
+
+
+@router.post("/chat", response_model=ChatResponse)
+async def chat(request: ChatRequest):
+    """Send a message to the podcast agent and get a response"""
+    return await podcast_agent_service.chat(request)
+
+
+@router.post("/status", response_model=ChatResponse)
+async def check_status(request: StatusRequest):
+    """Check if a result is available for the session"""
+    return await podcast_agent_service.check_result_status(request)
+
+
+@router.get("/sessions")
+async def list_sessions(page: int = 1, per_page: int = 10):
+    """List all saved podcast sessions with pagination"""
+    return await podcast_agent_service.list_sessions(page, per_page)
+
+
+@router.get("/session_history")
+async def get_session_history(session_id: str):
+    """Get the complete message history for a session"""
+    return await podcast_agent_service.get_session_history(session_id)
+
+
+@router.delete("/session/{session_id}")
+async def delete_session(session_id: str):
+    """Delete a podcast session and all its data"""
+    return await podcast_agent_service.delete_session(session_id)
+
+@router.get("/languages")
+async def get_supported_languages():
+    """Get the list of supported languages"""
+    return await podcast_agent_service.get_supported_languages()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/routers/podcast_config_router.py

@@ -0,0 +1,103 @@
+from fastapi import APIRouter, Query, Path, Body, status
+from typing import List
+from models.podcast_config_schemas import PodcastConfig, PodcastConfigCreate, PodcastConfigUpdate
+from services.podcast_config_service import podcast_config_service
+
+
+router = APIRouter()
+
+
+@router.get("/", response_model=List[PodcastConfig])
+async def get_podcast_configs(
+    active_only: bool = Query(False, description="Include only active podcast configs"),
+):
+    """
+    Get all podcast configurations with optional filtering.
+
+    - **active_only**: If true, includes only active podcast configurations
+    """
+    return await podcast_config_service.get_all_configs(active_only=active_only)
+
+
+@router.get("/{config_id}", response_model=PodcastConfig)
+async def get_podcast_config(
+    config_id: int = Path(..., description="The ID of the podcast config to retrieve"),
+):
+    """
+    Get a specific podcast configuration by ID.
+
+    - **config_id**: The ID of the podcast configuration to retrieve
+    """
+    return await podcast_config_service.get_config(config_id=config_id)
+
+
+@router.post("/", response_model=PodcastConfig, status_code=status.HTTP_201_CREATED)
+async def create_podcast_config(config_data: PodcastConfigCreate = Body(...)):
+    """
+    Create a new podcast configuration.
+
+    - **config_data**: Data for the new podcast configuration
+    """
+    return await podcast_config_service.create_config(
+        name=config_data.name,
+        prompt=config_data.prompt,
+        description=config_data.description,
+        time_range_hours=config_data.time_range_hours,
+        limit_articles=config_data.limit_articles,
+        is_active=config_data.is_active,
+        tts_engine=config_data.tts_engine,
+        language_code=config_data.language_code,
+        podcast_script_prompt=config_data.podcast_script_prompt,
+        image_prompt=config_data.image_prompt,
+    )
+
+
+@router.put("/{config_id}", response_model=PodcastConfig)
+async def update_podcast_config(
+    config_id: int = Path(..., description="The ID of the podcast config to update"),
+    config_data: PodcastConfigUpdate = Body(...),
+):
+    """
+    Update an existing podcast configuration.
+
+    - **config_id**: The ID of the podcast configuration to update
+    - **config_data**: Updated data for the podcast configuration
+    """
+    updates = {k: v for k, v in config_data.dict().items() if v is not None}
+    return await podcast_config_service.update_config(config_id=config_id, updates=updates)
+
+
+@router.delete("/{config_id}")
+async def delete_podcast_config(
+    config_id: int = Path(..., description="The ID of the podcast config to delete"),
+):
+    """
+    Delete a podcast configuration.
+
+    - **config_id**: The ID of the podcast configuration to delete
+    """
+    return await podcast_config_service.delete_config(config_id=config_id)
+
+
+@router.post("/{config_id}/enable", response_model=PodcastConfig)
+async def enable_podcast_config(
+    config_id: int = Path(..., description="The ID of the podcast config to enable"),
+):
+    """
+    Enable a podcast configuration.
+
+    - **config_id**: The ID of the podcast configuration to enable
+    """
+    return await podcast_config_service.toggle_config(config_id=config_id, enable=True)
+
+
+@router.post("/{config_id}/disable", response_model=PodcastConfig)
+async def disable_podcast_config(
+    config_id: int = Path(..., description="The ID of the podcast config to disable"),
+):
+    """
+    Disable a podcast configuration.
+
+    - **config_id**: The ID of the podcast configuration to disable
+    """
+    return await podcast_config_service.toggle_config(config_id=config_id, enable=False)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/routers/podcast_router.py

@@ -0,0 +1,200 @@
+from fastapi import APIRouter, HTTPException, File, UploadFile, Body, Query, Path
+from fastapi.responses import FileResponse
+from typing import List, Optional
+import os
+from datetime import datetime
+from models.podcast_schemas import Podcast, PodcastDetail, PodcastCreate, PodcastUpdate, PaginatedPodcasts
+from services.podcast_service import podcast_service
+
+router = APIRouter()
+
+
+@router.get("/", response_model=PaginatedPodcasts)
+async def get_podcasts(
+    page: int = Query(1, ge=1, description="Page number"),
+    per_page: int = Query(10, ge=1, le=100, description="Items per page"),
+    search: Optional[str] = Query(None, description="Search in title"),
+    date_from: Optional[str] = Query(None, description="Filter by date from (YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by date to (YYYY-MM-DD)"),
+    language_code: Optional[str] = Query(None, description="Filter by language code"),
+    tts_engine: Optional[str] = Query(None, description="Filter by TTS engine"),
+    has_audio: Optional[bool] = Query(None, description="Filter by audio availability"),
+):
+    """
+    Get a paginated list of podcasts with optional filtering.
+    """
+    return await podcast_service.get_podcasts(
+        page=page,
+        per_page=per_page,
+        search=search,
+        date_from=date_from,
+        date_to=date_to,
+        language_code=language_code,
+        tts_engine=tts_engine,
+        has_audio=has_audio,
+    )
+
+
+@router.get("/formats", response_model=List[str])
+async def get_podcast_formats():
+    """
+    Get a list of available podcast formats for filtering.
+    """
+    return await podcast_service.get_podcast_formats()
+
+
+@router.get("/language-codes", response_model=List[str])
+async def get_language_codes():
+    """
+    Get a list of available language codes for filtering.
+    """
+    return await podcast_service.get_language_codes()
+
+
+@router.get("/tts-engines", response_model=List[str])
+async def get_tts_engines():
+    """
+    Get a list of available TTS engines for filtering.
+    """
+    return await podcast_service.get_tts_engines()
+
+
+@router.get("/{podcast_id}", response_model=PodcastDetail)
+async def get_podcast(podcast_id: int = Path(..., description="The ID of the podcast to retrieve")):
+    """
+    Get detailed information about a specific podcast.
+
+    Parameters:
+    - **podcast_id**: The ID of the podcast to retrieve
+
+    Returns the podcast metadata and content.
+    """
+    podcast = await podcast_service.get_podcast(podcast_id)
+    content = await podcast_service.get_podcast_content(podcast_id)
+    audio_url = await podcast_service.get_podcast_audio_url(podcast)
+    sources = podcast.get("sources", [])
+    if "sources" in podcast:
+        del podcast["sources"]
+    banner_images = podcast.get("banner_images", [])
+    if "banner_images" in podcast:
+        del podcast["banner_images"]
+    return {"podcast": podcast, "content": content, "audio_url": audio_url, "sources": sources, "banner_images": banner_images}
+
+
+@router.get("/by-identifier/{identifier}", response_model=PodcastDetail)
+async def get_podcast_by_identifier(identifier: str = Path(..., description="The unique identifier of the podcast")):
+    """
+    Get detailed information about a specific podcast using string identifier.
+
+    Parameters:
+    - **identifier**: The unique identifier of the podcast to retrieve
+
+    Returns the podcast metadata and content.
+    """
+    podcast = await podcast_service.get_podcast_by_identifier(identifier)
+    podcast_id = int(podcast["id"])
+    content = await podcast_service.get_podcast_content(podcast_id)
+    audio_url = await podcast_service.get_podcast_audio_url(podcast)
+    sources = podcast.get("sources", [])
+    if "sources" in podcast:
+        del podcast["sources"]
+    banner_images = podcast.get("banner_images", [])
+    if "banner_images" in podcast:
+        del podcast["banner_images"]
+    return {"podcast": podcast, "content": content, "audio_url": audio_url, "sources": sources, "banner_images": banner_images}
+
+
+@router.post("/", response_model=Podcast)
+async def create_podcast(podcast_data: PodcastCreate = Body(...)):
+    """
+    Create a new podcast.
+
+    Parameters:
+    - **podcast_data**: Podcast data including title, date, content, sources, language_code, and tts_engine
+
+    Returns the created podcast metadata.
+    """
+    date = podcast_data.date or datetime.now().strftime("%Y-%m-%d")
+    return await podcast_service.create_podcast(
+        title=podcast_data.title,
+        date=date,
+        content=podcast_data.content,
+        sources=podcast_data.sources,
+        language_code=podcast_data.language_code,
+        tts_engine=podcast_data.tts_engine,
+    )
+
+
+@router.put("/{podcast_id}", response_model=Podcast)
+async def update_podcast(podcast_id: int = Path(..., description="The ID of the podcast to update"), podcast_data: PodcastUpdate = Body(...)):
+    """
+    Update an existing podcast's metadata and/or content.
+
+    Parameters:
+    - **podcast_id**: The ID of the podcast to update
+    - **podcast_data**: Updated data for the podcast
+
+    Returns the updated podcast metadata.
+    """
+    update_data = {k: v for k, v in podcast_data.dict().items() if v is not None}
+    return await podcast_service.update_podcast(podcast_id, update_data)
+
+
+@router.delete("/{podcast_id}")
+async def delete_podcast(podcast_id: int = Path(..., description="The ID of the podcast to delete")):
+    """
+    Delete a podcast.
+
+    Parameters:
+    - **podcast_id**: The ID of the podcast to delete
+
+    Returns a success message.
+    """
+    success = await podcast_service.delete_podcast(podcast_id)
+    if success:
+        return {"message": f"Podcast {podcast_id} deleted successfully"}
+    return {"message": "No podcast was deleted"}
+
+
+@router.post("/{podcast_id}/audio", response_model=Podcast)
+async def upload_audio(podcast_id: int = Path(..., description="The ID of the podcast"), file: UploadFile = File(...)):
+    """
+    Upload an audio file for a podcast.
+
+    Parameters:
+    - **podcast_id**: The ID of the podcast to attach the audio to
+    - **file**: The audio file to upload
+
+    Returns the updated podcast.
+    """
+    return await podcast_service.upload_podcast_audio(podcast_id, file)
+
+
+@router.post("/{podcast_id}/banner", response_model=Podcast)
+async def upload_banner(podcast_id: int = Path(..., description="The ID of the podcast"), file: UploadFile = File(...)):
+    """
+    Upload a banner image for a podcast.
+
+    Parameters:
+    - **podcast_id**: The ID of the podcast to attach the banner to
+    - **file**: The image file to upload
+
+    Returns the updated podcast.
+    """
+    return await podcast_service.upload_podcast_banner(podcast_id, file)
+
+
+@router.get("/audio/{filename}")
+async def get_audio_file(filename: str = Path(..., description="The filename of the audio file")):
+    """
+    Get the audio file for a podcast.
+
+    Parameters:
+    - **filename**: The filename of the audio file to retrieve
+
+    Returns the audio file as a download.
+    """
+    audio_path = os.path.join("podcasts", "audio", filename)
+    if not os.path.exists(audio_path):
+        raise HTTPException(status_code=404, detail="Audio file not found")
+    return FileResponse(audio_path)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/routers/social_media_router.py

@@ -0,0 +1,173 @@
+from fastapi import APIRouter, Query
+from typing import List, Optional, Dict, Any
+from services.social_media_service import social_media_service
+from models.social_media_schemas import PaginatedPosts, Post
+import threading
+from tools.social.browser import setup_session_multi
+
+router = APIRouter()
+
+
+@router.get("/", response_model=PaginatedPosts)
+async def read_posts(
+    page: int = Query(1, ge=1, description="Page number"),
+    per_page: int = Query(10, ge=1, le=100, description="Items per page"),
+    platform: Optional[str] = Query(None, description="Filter by platform (e.g., x.com, instagram)"),
+    user_handle: Optional[str] = Query(None, description="Filter by user handle"),
+    sentiment: Optional[str] = Query(None, description="Filter by sentiment"),
+    category: Optional[str] = Query(None, description="Filter by category"),
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+    search: Optional[str] = Query(None, description="Search in post text, user display name, or handle"),
+):
+    """
+    Get all social media posts with pagination and filtering.
+    """
+    return await social_media_service.get_posts(
+        page=page,
+        per_page=per_page,
+        platform=platform,
+        user_handle=user_handle,
+        sentiment=sentiment,
+        category=category,
+        date_from=date_from,
+        date_to=date_to,
+        search=search,
+    )
+
+
+@router.get("/{post_id}", response_model=Post)
+async def read_post(post_id: str):
+    """
+    Get a specific social media post by ID.
+    """
+    return await social_media_service.get_post(post_id=post_id)
+
+
+@router.get("/platforms/list", response_model=List[str])
+async def read_platforms():
+    """Get all available platforms."""
+    return await social_media_service.get_platforms()
+
+
+@router.get("/sentiments/list", response_model=List[Dict[str, Any]])
+async def read_sentiments(
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get sentiment distribution with post counts."""
+    return await social_media_service.get_sentiments(date_from=date_from, date_to=date_to)
+
+
+@router.get("/users/top", response_model=List[Dict[str, Any]])
+async def read_top_users(
+    platform: Optional[str] = Query(None, description="Filter by platform"),
+    limit: int = Query(10, ge=1, le=50, description="Number of top users to return"),
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get top users by post count."""
+    return await social_media_service.get_top_users(platform=platform, limit=limit, date_from=date_from, date_to=date_to)
+
+
+@router.get("/categories/list", response_model=List[Dict[str, Any]])
+async def read_categories(
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get all categories with post counts."""
+    return await social_media_service.get_categories(date_from=date_from, date_to=date_to)
+
+
+@router.get("/users/sentiment", response_model=List[Dict[str, Any]])
+async def read_user_sentiment(
+    limit: int = Query(10, ge=1, le=50, description="Number of users to return"),
+    platform: Optional[str] = Query(None, description="Filter by platform"),
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get users with their sentiment breakdown."""
+    return await social_media_service.get_user_sentiment(limit=limit, platform=platform, date_from=date_from, date_to=date_to)
+
+
+@router.get("/categories/sentiment", response_model=List[Dict[str, Any]])
+async def read_category_sentiment(
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get sentiment distribution by category."""
+    return await social_media_service.get_category_sentiment(date_from=date_from, date_to=date_to)
+
+
+@router.get("/topic/trends", response_model=List[Dict[str, Any]])
+async def read_trending_topics(
+    limit: int = Query(10, ge=1, le=50, description="Number of topics to return"),
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get trending topics with sentiment breakdown."""
+    return await social_media_service.get_trending_topics(date_from=date_from, date_to=date_to, limit=limit)
+
+
+@router.get("/trends/time", response_model=List[Dict[str, Any]])
+async def read_sentiment_over_time(
+    platform: Optional[str] = Query(None, description="Filter by platform"),
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get sentiment trends over time."""
+    return await social_media_service.get_sentiment_over_time(date_from=date_from, date_to=date_to, platform=platform)
+
+
+@router.get("/posts/influential", response_model=List[Dict[str, Any]])
+async def read_influential_posts(
+    sentiment: Optional[str] = Query(None, description="Filter by sentiment"),
+    limit: int = Query(5, ge=1, le=20, description="Number of posts to return"),
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get most influential posts by engagement, optionally filtered by sentiment."""
+    return await social_media_service.get_influential_posts(sentiment=sentiment, limit=limit, date_from=date_from, date_to=date_to)
+
+
+@router.get("/engagement/stats", response_model=Dict[str, Any])
+async def read_engagement_stats(
+    date_from: Optional[str] = Query(None, description="Filter by start date (format: YYYY-MM-DD)"),
+    date_to: Optional[str] = Query(None, description="Filter by end date (format: YYYY-MM-DD)"),
+):
+    """Get overall engagement statistics."""
+    return await social_media_service.get_engagement_stats(date_from=date_from, date_to=date_to)
+
+
+def _run_browser_setup_background(sites: Optional[List[str]] = None):
+    """Background task to run browser session setup in separate thread."""
+    try:
+        if sites and len(sites) > 1:
+            setup_session_multi(sites)
+        elif sites and len(sites) > 0:
+            setup_session_multi(sites)
+        else:
+            default_sites = ["https://x.com", "https://facebook.com"]
+            setup_session_multi(default_sites)
+    except Exception as e:
+        print(f"Browser setup error: {e}")
+
+
+@router.post("/session/setup")
+async def setup_browser_session(sites: Optional[List[str]] = Query(None, description="List of sites to setup sessions for")):
+    """
+    Trigger browser session setup in a completely separate thread.
+    This will open a browser window for manual login to social media platforms.
+    The API immediately returns while the browser setup runs independently.
+    """
+    thread = threading.Thread(
+        target=_run_browser_setup_background,
+        args=(sites,),
+        daemon=True,
+    )
+    thread.start()
+    return {
+        "status": "ok",
+        "message": "Browser session setup triggered successfully",
+        "note": "Browser window will open shortly for manual authentication",
+    }

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/routers/source_router.py

@@ -0,0 +1,136 @@
+from fastapi import APIRouter, Body, Path, status
+from typing import List, Optional, Dict, Any
+from models.source_schemas import Source, SourceWithFeeds, Category, PaginatedSources, SourceCreate, SourceUpdate, SourceFeedCreate
+from services.source_service import source_service
+
+router = APIRouter()
+
+
+@router.get("/", response_model=PaginatedSources)
+async def read_sources(
+    page: int = 1, per_page: int = 10, category: Optional[str] = None, search: Optional[str] = None, include_inactive: bool = False
+):
+    """
+    Get sources with pagination and filtering.
+
+    - **page**: Page number (starting from 1)
+    - **per_page**: Number of items per page
+    - **category**: Filter by source category
+    - **search**: Search in name and description
+    - **include_inactive**: Include inactive sources
+    """
+    return await source_service.get_sources(page=page, per_page=per_page, category=category, search=search, include_inactive=include_inactive)
+
+
+@router.get("/categories", response_model=List[Category])
+async def read_categories():
+    """Get all available source categories."""
+    return await source_service.get_categories()
+
+
+@router.get("/by-category/{category_name}", response_model=List[Source])
+async def read_sources_by_category(category_name: str):
+    """
+    Get sources by category name.
+
+    - **category_name**: Name of the category to filter by
+    """
+    return await source_service.get_source_by_category(category_name=category_name)
+
+
+@router.get("/{source_id}", response_model=SourceWithFeeds)
+async def read_source(source_id: int = Path(..., gt=0)):
+    """
+    Get detailed information about a specific source.
+
+    - **source_id**: ID of the source to retrieve
+    """
+    source = await source_service.get_source(source_id=source_id)
+    feeds = await source_service.get_source_feeds(source_id=source_id)
+    source_with_feeds = {**source, "feeds": feeds}
+    return source_with_feeds
+
+
+@router.get("/by-name/{name}", response_model=SourceWithFeeds)
+async def read_source_by_name(name: str):
+    """
+    Get detailed information about a specific source by name.
+
+    - **name**: Name of the source to retrieve
+    """
+    source = await source_service.get_source_by_name(name=name)
+    feeds = await source_service.get_source_feeds(source_id=source["id"])
+    source_with_feeds = {**source, "feeds": feeds}
+    return source_with_feeds
+
+
+@router.post("/", response_model=SourceWithFeeds, status_code=status.HTTP_201_CREATED)
+async def create_source(source_data: SourceCreate):
+    """
+    Create a new source.
+
+    - **source_data**: Data for the new source
+    """
+    source = await source_service.create_source(source_data)
+    feeds = await source_service.get_source_feeds(source_id=source["id"])
+    source_with_feeds = {**source, "feeds": feeds}
+    return source_with_feeds
+
+
+@router.put("/{source_id}", response_model=SourceWithFeeds)
+async def update_source(source_data: SourceUpdate, source_id: int = Path(..., gt=0)):
+    """
+    Update an existing source.
+
+    - **source_id**: ID of the source to update
+    - **source_data**: Updated data for the source
+    """
+    source = await source_service.update_source(source_id, source_data)
+    feeds = await source_service.get_source_feeds(source_id=source["id"])
+    source_with_feeds = {**source, "feeds": feeds}
+    return source_with_feeds
+
+
+@router.delete("/{source_id}", response_model=Dict[str, Any])
+async def delete_source(source_id: int = Path(..., gt=0), permanent: bool = False):
+    """
+    Delete a source.
+
+    - **source_id**: ID of the source to delete
+    - **permanent**: If true, permanently deletes the source; otherwise, performs a soft delete
+    """
+    if permanent:
+        return await source_service.hard_delete_source(source_id)
+    return await source_service.delete_source(source_id)
+
+
+@router.post("/{source_id}/feeds", response_model=List[Dict[str, Any]])
+async def add_feed(feed_data: SourceFeedCreate, source_id: int = Path(..., gt=0)):
+    """
+    Add a new feed to a source.
+
+    - **source_id**: ID of the source to add the feed to
+    - **feed_data**: Data for the new feed
+    """
+    return await source_service.add_feed_to_source(source_id, feed_data)
+
+
+@router.put("/feeds/{feed_id}", response_model=Dict[str, Any])
+async def update_feed(feed_data: Dict[str, Any] = Body(...), feed_id: int = Path(..., gt=0)):
+    """
+    Update an existing feed.
+
+    - **feed_id**: ID of the feed to update
+    - **feed_data**: Updated data for the feed
+    """
+    return await source_service.update_feed(feed_id, feed_data)
+
+
+@router.delete("/feeds/{feed_id}", response_model=Dict[str, str])
+async def delete_feed(feed_id: int = Path(..., gt=0)):
+    """
+    Delete a feed.
+
+    - **feed_id**: ID of the feed to delete
+    """
+    return await source_service.delete_feed(feed_id)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/routers/task_router.py

@@ -0,0 +1,138 @@
+from fastapi import APIRouter, Query, Path, Body, status
+from typing import List, Optional, Dict
+from models.tasks_schemas import Task, TaskCreate, TaskUpdate, TaskStats, TASK_TYPES
+from services.task_service import task_service
+
+router = APIRouter()
+
+
+@router.get("/", response_model=List[Task])
+async def get_tasks(
+    include_disabled: bool = Query(False, description="Include disabled tasks"),
+):
+    """
+    Get all tasks with optional filtering.
+
+    - **include_disabled**: If true, includes disabled tasks
+    """
+    return await task_service.get_tasks(include_disabled=include_disabled)
+
+
+@router.get("/pending", response_model=List[Task])
+async def get_pending_tasks():
+    """
+    Get tasks that are due to run.
+    """
+    return await task_service.get_pending_tasks()
+
+
+@router.get("/stats", response_model=TaskStats)
+async def get_task_stats():
+    """
+    Get task statistics.
+    """
+    return await task_service.get_stats()
+
+
+@router.get("/executions")
+async def get_task_executions(
+    task_id: Optional[int] = Query(None, description="Filter by task ID"),
+    page: int = Query(1, ge=1, description="Page number"),
+    per_page: int = Query(10, ge=1, le=100, description="Items per page"),
+):
+    """
+    Get paginated task executions.
+
+    - **task_id**: Filter by task ID
+    - **page**: Page number (starting from 1)
+    - **per_page**: Number of items per page (max 100)
+    """
+    return await task_service.get_task_executions(task_id=task_id, page=page, per_page=per_page)
+
+
+@router.get("/types", response_model=Dict[str, Dict[str, str]])
+async def get_task_types():
+    """
+    Get all available task types.
+    """
+    return TASK_TYPES
+
+
+@router.get("/{task_id}", response_model=Task)
+async def get_task(
+    task_id: int = Path(..., description="The ID of the task to retrieve"),
+):
+    """
+    Get a specific task by ID.
+
+    - **task_id**: The ID of the task to retrieve
+    """
+    return await task_service.get_task(task_id=task_id)
+
+
+@router.post("/", response_model=Task, status_code=status.HTTP_201_CREATED)
+async def create_task(task_data: TaskCreate = Body(...)):
+    """
+    Create a new task.
+
+    - **task_data**: Data for the new task
+    """
+    return await task_service.create_task(
+        name=task_data.name,
+        task_type=task_data.task_type,
+        frequency=task_data.frequency,
+        frequency_unit=task_data.frequency_unit,
+        description=task_data.description,
+        enabled=task_data.enabled,
+    )
+
+
+@router.put("/{task_id}", response_model=Task)
+async def update_task(
+    task_id: int = Path(..., description="The ID of the task to update"),
+    task_data: TaskUpdate = Body(...),
+):
+    """
+    Update an existing task.
+
+    - **task_id**: The ID of the task to update
+    - **task_data**: Updated data for the task
+    """
+    updates = {k: v for k, v in task_data.dict().items() if v is not None}
+    return await task_service.update_task(task_id=task_id, updates=updates)
+
+
+@router.delete("/{task_id}")
+async def delete_task(
+    task_id: int = Path(..., description="The ID of the task to delete"),
+):
+    """
+    Delete a task.
+
+    - **task_id**: The ID of the task to delete
+    """
+    return await task_service.delete_task(task_id=task_id)
+
+
+@router.post("/{task_id}/enable", response_model=Task)
+async def enable_task(
+    task_id: int = Path(..., description="The ID of the task to enable"),
+):
+    """
+    Enable a task.
+
+    - **task_id**: The ID of the task to enable
+    """
+    return await task_service.toggle_task(task_id=task_id, enable=True)
+
+
+@router.post("/{task_id}/disable", response_model=Task)
+async def disable_task(
+    task_id: int = Path(..., description="The ID of the task to disable"),
+):
+    """
+    Disable a task.
+
+    - **task_id**: The ID of the task to disable
+    """
+    return await task_service.toggle_task(task_id=task_id, enable=False)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/ruff.toml

@@ -0,0 +1 @@
+line-length = 150

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/scheduler.py

@@ -0,0 +1,205 @@
+import os
+import time
+import signal
+import subprocess
+from datetime import datetime
+import traceback
+from concurrent.futures import ThreadPoolExecutor
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.interval import IntervalTrigger
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from db.config import get_tasks_db_path
+from db.connection import db_connection
+from db.tasks import (
+    get_pending_tasks,
+    update_task_last_run,
+    update_task_execution,
+)
+
+running = True
+MAX_WORKERS = 5
+DEFAULT_TASK_TIMEOUT = 3600
+
+
+def cleanup_stuck_tasks():
+    tasks_db_path = get_tasks_db_path()
+    try:
+        with db_connection(tasks_db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT id FROM task_executions 
+                WHERE status = 'running'
+                LIMIT 100
+                """
+            )
+            running_executions = [dict(row) for row in cursor.fetchall()]
+            if running_executions:
+                print(f"WARNING: Found {len(running_executions)} tasks stuck in 'running' state. Marking as failed.")
+                for execution in running_executions:
+                    execution_id = execution["id"]
+                    error_message = "Task was interrupted by system shutdown or crash"
+                    cursor.execute(
+                        """
+                        UPDATE task_executions
+                        SET end_time = ?, status = ?, error_message = ?
+                        WHERE id = ?
+                        """,
+                        (datetime.now().isoformat(), "failed", error_message, execution_id),
+                    )
+                    print(f"INFO: Marked execution {execution_id} as failed")
+                conn.commit()
+            else:
+                print("INFO: No stuck tasks found")
+    except Exception as e:
+        print(f"ERROR: Error cleaning up stuck tasks: {str(e)}")
+        print(f"ERROR: {traceback.format_exc()}")
+
+
+def execute_task(task_id, command):
+    tasks_db_path = get_tasks_db_path()
+    with db_connection(tasks_db_path) as conn:
+        conn.execute("BEGIN EXCLUSIVE TRANSACTION")
+        try:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT 1 FROM task_executions 
+                WHERE task_id = ? AND status = 'running'
+                LIMIT 1
+                """,
+                (task_id,),
+            )
+            is_running = cursor.fetchone() is not None
+            if is_running:
+                print(f"WARNING: Task {task_id} is already running, skipping this execution")
+                conn.commit()
+                return
+            cursor.execute(
+                """
+                INSERT INTO task_executions 
+                (task_id, start_time, status)
+                VALUES (?, ?, ?)
+                """,
+                (task_id, datetime.now().isoformat(), "running"),
+            )
+            execution_id = cursor.lastrowid
+            conn.commit()
+            if not execution_id:
+                print(f"ERROR: Failed to create execution record for task {task_id}")
+                return
+        except Exception as e:
+            conn.rollback()
+            print(f"ERROR: Transaction error for task {task_id}: {str(e)}")
+            return
+    print(f"INFO: Starting task {task_id}: {command}")
+    try:
+        process = subprocess.Popen(
+            command,
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        try:
+            stdout, stderr = process.communicate(timeout=DEFAULT_TASK_TIMEOUT)
+            if process.returncode == 0:
+                status = "success"
+                error_message = None
+                print(f"INFO: Task {task_id} completed successfully")
+            else:
+                status = "failed"
+                error_message = stderr if stderr else f"Process exited with code {process.returncode}"
+                print(f"ERROR: Task {task_id} failed: {error_message}")
+        except subprocess.TimeoutExpired:
+            process.kill()
+            stdout, stderr = process.communicate()
+            status = "failed"
+            error_message = f"Task timed out after {DEFAULT_TASK_TIMEOUT} seconds"
+            print(f"ERROR: Task {task_id} timed out")
+        output = f"STDOUT:\n{stdout}\n\nSTDERR:\n{stderr}" if stderr else stdout
+        update_task_execution(tasks_db_path, execution_id, status, error_message, output)
+        timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+        update_task_last_run(tasks_db_path, task_id, timestamp)
+    except Exception as e:
+        print(f"ERROR: Error executing task {task_id}: {str(e)}")
+        error_message = traceback.format_exc()
+        update_task_execution(tasks_db_path, execution_id, "failed", error_message)
+        timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+        update_task_last_run(tasks_db_path, task_id, timestamp)
+
+
+def check_for_tasks():
+    tasks_db_path = get_tasks_db_path()
+    try:
+        print("DEBUG: Checking for pending tasks...")
+        pending_tasks = get_pending_tasks(tasks_db_path)
+        if not pending_tasks:
+            print("DEBUG: No pending tasks found")
+            return
+        print(f"INFO: Found {len(pending_tasks)} pending tasks")
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            for task in pending_tasks:
+                task_id = task["id"]
+                command = task["command"]
+                print(f"INFO: Scheduling task {task_id}: {task['name']} (Last run: {task['last_run']})")
+                executor.submit(execute_task, task_id, command)
+    except Exception as e:
+        print(f"ERROR: Error in check_for_tasks: {str(e)}")
+        print(f"ERROR: {traceback.format_exc()}")
+
+
+def check_missed_tasks():
+    tasks_db_path = get_tasks_db_path()
+    try:
+        print("INFO: Checking for missed tasks during downtime...")
+        pending_tasks = get_pending_tasks(tasks_db_path)
+        if pending_tasks:
+            print(f"INFO: Found {len(pending_tasks)} tasks to run (including any missed during downtime)")
+        else:
+            print("INFO: No missed tasks found")
+    except Exception as e:
+        print(f"ERROR: Error checking for missed tasks: {str(e)}")
+
+
+def signal_handler(sig, frame):
+    global running
+    print("INFO: Shutdown signal received, stopping scheduler...")
+    running = False
+
+
+def main():
+    global running
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+    print("INFO: Starting task scheduler")
+    tasks_db_path = get_tasks_db_path()
+    cleanup_stuck_tasks()
+    scheduler_dir = os.path.dirname(tasks_db_path)
+    os.makedirs(scheduler_dir, exist_ok=True)
+    scheduler_db_path = os.path.join(scheduler_dir, "scheduler.sqlite")
+    jobstores = {"default": SQLAlchemyJobStore(url=f"sqlite:///{scheduler_db_path}")}
+    scheduler = BackgroundScheduler(jobstores=jobstores, daemon=True)
+    scheduler.add_job(
+        check_for_tasks,
+        IntervalTrigger(minutes=1),
+        id="check_tasks",
+        name="Check for pending tasks",
+        replace_existing=True,
+    )
+    scheduler.start()
+    print("INFO: Scheduler started, checking for tasks every minute")
+    check_for_tasks()
+    check_missed_tasks()
+    try:
+        while running:
+            time.sleep(1)
+    except (KeyboardInterrupt, SystemExit):
+        print("INFO: Scheduler interrupted")
+    finally:
+        scheduler.shutdown()
+        print("INFO: Scheduler shutdown complete")
+
+
+if __name__ == "__main__":
+    main()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/article_service.py

@@ -0,0 +1,174 @@
+from typing import List, Optional, Dict, Any
+from fastapi import HTTPException
+import json
+from services.db_service import tracking_db, sources_db
+from models.article_schemas import Article, PaginatedArticles
+
+
+class ArticleService:
+    """Service for managing article operations with the new database structure."""
+
+    async def get_articles(
+        self,
+        page: int = 1,
+        per_page: int = 10,
+        source: Optional[str] = None,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        search: Optional[str] = None,
+        category: Optional[str] = None,
+    ) -> PaginatedArticles:
+        """Get articles with pagination and filtering."""
+        try:
+            offset = (page - 1) * per_page
+            query_parts = [
+                "SELECT ca.id, ca.title, ca.url, ca.published_date, ca.summary, ca.feed_id",
+                "FROM crawled_articles ca",
+                "WHERE ca.processed = 1 AND ca.ai_status = 'success'",
+            ]
+            query_params = []
+            if source:
+                source_id_query = "SELECT id FROM sources WHERE name = ?"
+                source_id_result = await sources_db.execute_query(source_id_query, (source,), fetch=True, fetch_one=True)
+                if source_id_result and source_id_result.get("id"):
+                    feed_ids_query = "SELECT id FROM source_feeds WHERE source_id = ?"
+                    feed_ids_result = await sources_db.execute_query(feed_ids_query, (source_id_result["id"],), fetch=True)
+                    if feed_ids_result:
+                        feed_ids = [item["id"] for item in feed_ids_result]
+                        placeholders = ",".join(["?" for _ in feed_ids])
+                        query_parts.append(f"AND ca.feed_id IN ({placeholders})")
+                        query_params.extend(feed_ids)
+            if category:
+                query_parts.append("""
+                    AND EXISTS (
+                        SELECT 1 FROM article_categories ac 
+                        WHERE ac.article_id = ca.id AND ac.category_name = ?
+                    )
+                """)
+                query_params.append(category.lower())
+            if date_from:
+                query_parts.append("AND datetime(ca.published_date) >= datetime(?)")
+                query_params.append(date_from)
+            if date_to:
+                query_parts.append("AND datetime(ca.published_date) <= datetime(?)")
+                query_params.append(date_to)
+            if search:
+                query_parts.append("AND (ca.title LIKE ? OR ca.summary LIKE ?)")
+                search_param = f"%{search}%"
+                query_params.extend([search_param, search_param])
+            count_query = " ".join(query_parts).replace(
+                "SELECT ca.id, ca.title, ca.url, ca.published_date, ca.summary, ca.feed_id",
+                "SELECT COUNT(*)",
+            )
+            total_articles = await tracking_db.execute_query(count_query, tuple(query_params), fetch=True, fetch_one=True)
+            total_count = total_articles.get("COUNT(*)", 0) if total_articles else 0
+            query_parts.append("ORDER BY datetime(ca.published_date) DESC, ca.id DESC")
+            query_parts.append("LIMIT ? OFFSET ?")
+            query_params.extend([per_page, offset])
+            articles_query = " ".join(query_parts)
+            articles = await tracking_db.execute_query(articles_query, tuple(query_params), fetch=True)
+            feed_ids = [article["feed_id"] for article in articles if article.get("feed_id")]
+            source_names = {}
+            if feed_ids:
+                feed_ids_str = ",".join("?" for _ in feed_ids)
+                source_query = f"""
+                SELECT sf.id as feed_id, s.name as source_name
+                FROM source_feeds sf
+                JOIN sources s ON sf.source_id = s.id
+                WHERE sf.id IN ({feed_ids_str})
+                """
+                sources_result = await sources_db.execute_query(source_query, tuple(feed_ids), fetch=True)
+                source_names = {item["feed_id"]: item["source_name"] for item in sources_result}
+            for article in articles:
+                feed_id = article.get("feed_id")
+                article["source_name"] = source_names.get(feed_id, "Unknown Source")
+                article.pop("feed_id", None)
+                article["categories"] = await self.get_article_categories(article["id"])
+            total_pages = (total_count + per_page - 1) // per_page if total_count > 0 else 0
+            has_next = page < total_pages
+            has_prev = page > 1
+            return PaginatedArticles(
+                items=articles,
+                total=total_count,
+                page=page,
+                per_page=per_page,
+                total_pages=total_pages,
+                has_next=has_next,
+                has_prev=has_prev,
+            )
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching articles: {str(e)}")
+
+    async def get_article(self, article_id: int) -> Article:
+        """Get a specific article by ID."""
+        try:
+            article_query = """
+            SELECT id, title, url, published_date, content, summary, feed_id,
+                   metadata, ai_status
+            FROM crawled_articles
+            WHERE id = ? AND processed = 1
+            """
+            article = await tracking_db.execute_query(article_query, (article_id,), fetch=True, fetch_one=True)
+            if not article:
+                raise HTTPException(status_code=404, detail="Article not found")
+            if article.get("feed_id"):
+                source_query = """
+                SELECT s.name as source_name
+                FROM source_feeds sf
+                JOIN sources s ON sf.source_id = s.id
+                WHERE sf.id = ?
+                """
+                source_result = await sources_db.execute_query(source_query, (article["feed_id"],), fetch=True, fetch_one=True)
+                if source_result:
+                    article["source_name"] = source_result["source_name"]
+                else:
+                    article["source_name"] = "Unknown Source"
+            else:
+                article["source_name"] = "Unknown Source"
+            article.pop("feed_id", None)
+            if article.get("metadata"):
+                try:
+                    article["metadata"] = json.loads(article["metadata"])
+                except json.JSONDecodeError:
+                    article["metadata"] = {}
+            article["categories"] = await self.get_article_categories(article_id)
+            return article
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching article: {str(e)}")
+
+    async def get_article_categories(self, article_id: int) -> List[str]:
+        """Get categories for a specific article."""
+        query = """
+        SELECT category_name
+        FROM article_categories
+        WHERE article_id = ?
+        """
+        categories = await tracking_db.execute_query(query, (article_id,), fetch=True)
+        return [category.get("category_name", "") for category in categories]
+
+    async def get_sources(self) -> List[str]:
+        """Get all available active sources."""
+        query = """
+        SELECT DISTINCT name FROM sources 
+        WHERE is_active = 1
+        ORDER BY name
+        """
+        result = await sources_db.execute_query(query, fetch=True)
+        return [row.get("name", "") for row in result if row.get("name")]
+
+    async def get_categories(self) -> List[Dict[str, Any]]:
+        """Get all categories with article counts."""
+        query = """
+        SELECT category_name, COUNT(DISTINCT article_id) as article_count
+        FROM article_categories
+        GROUP BY category_name
+        ORDER BY article_count DESC
+        """
+        return await tracking_db.execute_query(query, fetch=True)
+
+
+article_service = ArticleService()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/async_podcast_agent_service.py

@@ -0,0 +1,392 @@
+import os
+import json
+import uuid
+from fastapi import status
+from fastapi.responses import JSONResponse
+import aiosqlite
+import glob
+from redis.asyncio import ConnectionPool, Redis
+from db.config import get_agent_session_db_path
+from db.agent_config_v2 import PODCAST_DIR, PODCAST_AUIDO_DIR, PODCAST_IMG_DIR, PODCAST_RECORDINGS_DIR, AVAILABLE_LANGS
+from services.celery_tasks import agent_chat
+from dotenv import load_dotenv
+from services.internal_session_service import SessionService
+
+load_dotenv()
+
+
+class PodcastAgentService:
+    def __init__(self):
+        os.makedirs(PODCAST_DIR, exist_ok=True)
+        os.makedirs(PODCAST_AUIDO_DIR, exist_ok=True)
+        os.makedirs(PODCAST_IMG_DIR, exist_ok=True)
+        os.makedirs(PODCAST_RECORDINGS_DIR, exist_ok=True)
+
+        self.redis_host = os.environ.get("REDIS_HOST", "localhost")
+        self.redis_port = int(os.environ.get("REDIS_PORT", 6379))
+        self.redis_db = int(os.environ.get("REDIS_DB", 0))
+        self.redis_pool = ConnectionPool.from_url(f"redis://{self.redis_host}:{self.redis_port}/{self.redis_db + 1}", max_connections=10)
+        self.redis = Redis(connection_pool=self.redis_pool)
+
+    async def get_active_task(self, session_id):
+        try:
+            lock_info = await self.redis.get(f"lock_info:{session_id}")
+            if lock_info:
+                try:
+                    lock_data = json.loads(lock_info.decode("utf-8"))
+                    task_id = lock_data.get("task_id")
+                    if task_id:
+                        return task_id
+                except (json.JSONDecodeError, AttributeError) as e:
+                    print(f"Error parsing lock info: {e}")
+            return None
+        except Exception as e:
+            print(f"Error checking active task: {e}")
+            return None
+
+    async def create_session(self, request=None):
+        if request and request.session_id:
+            session_id = request.session_id
+            try:
+                db_path = get_agent_session_db_path()
+                async with aiosqlite.connect(db_path) as conn:
+                    async with conn.execute("SELECT 1 FROM podcast_sessions WHERE session_id = ?", (session_id,)) as cursor:
+                        row = await cursor.fetchone()
+                        exists = row is not None
+                if exists:
+                    return {"session_id": session_id}
+            except Exception as e:
+                print(f"Error checking session existence: {e}")
+        new_session_id = str(uuid.uuid4())
+        return {"session_id": new_session_id}
+
+    async def chat(self, request):
+        try:
+            session_id = request.session_id
+            task_id = await self.get_active_task(session_id)
+            if task_id:
+                return {
+                    "session_id": session_id,
+                    "response": "Your request is already being processed.",
+                    "stage": "processing",
+                    "session_state": "{}",
+                    "is_processing": True,
+                    "process_type": "chat",
+                    "task_id": task_id,
+                }
+            task = agent_chat.delay(request.session_id, request.message)
+            return {
+                "session_id": request.session_id,
+                "response": "Your request is being processed.",
+                "stage": "processing",
+                "session_state": "{}",
+                "is_processing": True,
+                "process_type": "chat",
+                "task_id": task.id,
+            }
+        except Exception as e:
+            return JSONResponse(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                content={
+                    "session_id": request.session_id,
+                    "response": f"I encountered an error while processing your request: {str(e)}. Please try again.",
+                    "stage": "error",
+                    "session_state": "{}",
+                    "error": str(e),
+                    "is_processing": False,
+                },
+            )
+
+    def _browser_recording(self, session_id):
+        try:
+            recordings_dir = os.path.join("podcasts/recordings", session_id)
+            webm_files = glob.glob(os.path.join(recordings_dir, "*.webm"))
+            if webm_files:
+                browser_recording_path = webm_files[0]
+                if (os.path.exists(browser_recording_path) and 
+                    os.path.getsize(browser_recording_path) > 8192 and 
+                    os.access(browser_recording_path, os.R_OK)):
+                    return browser_recording_path
+            return None
+        except Exception as _:
+            return None
+
+    async def check_result_status(self, request):
+        try:
+            if not request.session_id:
+                return JSONResponse(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    content={"error": "Session ID is required"},
+                )
+
+            browser_recording_path = self._browser_recording(request.session_id)
+
+            task_id = getattr(request, "task_id", None)
+            if task_id:
+                task = agent_chat.AsyncResult(task_id)
+                if task.state == "PENDING" or task.state == "STARTED":
+                    return {
+                        "session_id": request.session_id,
+                        "response": "Your request is still being processed.",
+                        "stage": "processing",
+                        "session_state": "{}",
+                        "is_processing": True,
+                        "process_type": "chat",
+                        "task_id": task_id,
+                        "browser_recording_path": browser_recording_path,
+                    }
+                elif task.state == "SUCCESS":
+                    result = task.result
+                    if result and isinstance(result, dict):
+                        if result.get("session_id") != request.session_id:
+                            return {
+                                "session_id": request.session_id,
+                                "response": "Error: Received result for wrong session.",
+                                "stage": "error",
+                                "session_state": "{}",
+                                "is_processing": False,
+                                "browser_recording_path": browser_recording_path,
+                            }
+                        return result
+                else:
+                    error_info = str(task.result) if task.result else f"Task failed with state: {task.state}"
+                    return {
+                        "session_id": request.session_id,
+                        "response": f"Error processing request: {error_info}",
+                        "stage": "error",
+                        "session_state": "{}",
+                        "is_processing": False,
+                        "browser_recording_path": browser_recording_path,
+                    }
+            return await self.get_session_state(request.session_id)
+        except Exception as e:
+            return JSONResponse(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                content={
+                    "error": f"Error checking result status: {str(e)}",
+                    "session_id": request.session_id,
+                    "response": f"Error checking result status: {str(e)}",
+                    "stage": "error",
+                    "session_state": "{}",
+                    "is_processing": False,
+                    "browser_recording_path": browser_recording_path,
+                },
+            )
+
+    async def get_session_state(self, session_id):
+        try:
+            db_path = get_agent_session_db_path()
+            async with aiosqlite.connect(db_path) as conn:
+                conn.row_factory = lambda cursor, row: {col[0]: row[idx] for idx, col in enumerate(cursor.description)}
+                async with conn.execute("SELECT session_data FROM podcast_sessions WHERE session_id = ?", (session_id,)) as cursor:
+                    row = await cursor.fetchone()
+            if not row:
+                return {
+                    "session_id": session_id,
+                    "response": "No session data found.",
+                    "stage": "idle",
+                    "session_state": "{}",
+                    "is_processing": False,
+                }
+
+            session = SessionService.get_session(session_id)
+            session_state = session.get("state", {})
+            return {
+                "session_id": session_id,
+                "response": "",
+                "stage": session_state.get("stage", "idle"),
+                "session_state": json.dumps(session_state),
+                "is_processing": False,
+            }
+        except Exception as e:
+            return {
+                "session_id": session_id,
+                "response": f"Error retrieving session state: {str(e)}",
+                "stage": "error",
+                "session_state": "{}",
+                "is_processing": False,
+            }
+
+    async def list_sessions(self, page=1, per_page=10):
+        try:
+            db_path = get_agent_session_db_path()
+            async with aiosqlite.connect(db_path) as conn:
+                conn.row_factory = lambda cursor, row: {col[0]: row[idx] for idx, col in enumerate(cursor.description)}
+                async with conn.execute(
+                    """
+                    select name from sqlite_master
+                    where type='table' and name='podcast_sessions'
+                    """
+                ) as cursor:
+                    table = await cursor.fetchone()
+                if not table:
+                    return {
+                        "sessions": [],
+                        "pagination": {
+                            "total": 0,
+                            "page": page,
+                            "per_page": per_page,
+                            "total_pages": 0,
+                        },
+                    }
+                async with conn.execute("SELECT COUNT(*) as count FROM podcast_sessions") as cursor:
+                    row = await cursor.fetchone()
+                    total_sessions = row["count"] if row else 0
+                offset = (page - 1) * per_page
+                async with conn.execute(
+                    "SELECT session_id, session_data, updated_at FROM podcast_sessions ORDER BY updated_at DESC LIMIT ? OFFSET ?", (per_page, offset)
+                ) as cursor:
+                    rows = await cursor.fetchall()
+                    sessions = []
+                    for row in rows:
+                        try:
+                            session = SessionService.get_session(row["session_id"])
+                            session_state = session.get("state", {})
+                            title = session_state.get("title", "Untitled Podcast")
+                            stage = session_state.get("stage", "welcome")
+                            updated_at = row["updated_at"]
+                            sessions.append({"session_id": row["session_id"], "topic": title, "stage": stage, "updated_at": updated_at})
+                        except Exception as e:
+                            print(f"Error parsing session data: {e}")
+            return {
+                "sessions": sessions,
+                "pagination": {
+                    "total": total_sessions,
+                    "page": page,
+                    "per_page": per_page,
+                    "total_pages": (total_sessions + per_page - 1) // per_page,
+                },
+            }
+        except Exception as e:
+            print(f"Error listing sessions: {e}")
+            return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"error": f"Failed to list sessions: {str(e)}"})
+
+    async def delete_session(self, session_id: str):
+        try:
+            db_path = get_agent_session_db_path()
+            async with aiosqlite.connect(db_path) as conn:
+                conn.row_factory = lambda cursor, row: {col[0]: row[idx] for idx, col in enumerate(cursor.description)}
+                async with conn.execute("SELECT session_data FROM podcast_sessions WHERE session_id = ?", (session_id,)) as cursor:
+                    row = await cursor.fetchone()
+                if not row:
+                    return JSONResponse(status_code=status.HTTP_404_NOT_FOUND, content={"error": f"Session with ID {session_id} not found"})
+                try:
+                    session = SessionService.get_session(session_id)
+                    session_state = session.get("state", {})
+                    stage = session_state.get("stage")
+                    is_completed = stage == "complete" or session_state.get("podcast_generated", False)
+                    banner_url = session_state.get("banner_url")
+                    audio_url = session_state.get("audio_url")
+                    web_search_recording = session_state.get("web_search_recording")
+                    await conn.execute("DELETE FROM podcast_sessions WHERE session_id = ?", (session_id,))
+                    await conn.commit()
+                    if is_completed:
+                        print(f"Session {session_id} is in 'complete' stage, keeping assets but removing session record")
+                    else:
+                        if banner_url:
+                            banner_path = os.path.join(PODCAST_IMG_DIR, banner_url)
+                            if os.path.exists(banner_path):
+                                try:
+                                    os.remove(banner_path)
+                                    print(f"Deleted banner image: {banner_path}")
+                                except Exception as e:
+                                    print(f"Error deleting banner image: {e}")
+                        if audio_url:
+                            audio_path = os.path.join(PODCAST_AUIDO_DIR, audio_url)
+                            if os.path.exists(audio_path):
+                                try:
+                                    os.remove(audio_path)
+                                    print(f"Deleted audio file: {audio_path}")
+                                except Exception as e:
+                                    print(f"Error deleting audio file: {e}")
+                        if web_search_recording:
+                            recording_dir = os.path.join(PODCAST_RECORDINGS_DIR, session_id)
+                            if os.path.exists(recording_dir):
+                                try:
+                                    import shutil
+
+                                    shutil.rmtree(recording_dir)
+                                    print(f"Deleted recordings directory: {recording_dir}")
+                                except Exception as e:
+                                    print(f"Error deleting recordings directory: {e}")
+                    if is_completed:
+                        return {"success": True, "message": f"Session {session_id} deleted, but assets preserved"}
+                    else:
+                        return {"success": True, "message": f"Session {session_id} and its associated data deleted successfully"}
+                except Exception as e:
+                    print(f"Error parsing session data for deletion: {e}")
+                    return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"error": f"Error deleting session: {str(e)}"})
+        except Exception as e:
+            print(f"Error deleting session: {e}")
+            return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"error": f"Failed to delete session: {str(e)}"})
+
+    async def _get_chat_messages(self, row, session_id):
+        formatted_messages = []
+        session_state = {}
+        if row["session_data"]:
+            try:
+                session = SessionService.get_session(session_id)
+                session_state = session.get("state", {})
+            except Exception as e:
+                print(f"Error parsing session_data: {e}")
+
+        if row["memory"]:
+            try:
+                memory_data = json.loads(row["memory"]) if isinstance(row["memory"], str) else row["memory"]
+                if "runs" in memory_data and isinstance(memory_data["runs"], list):
+                    for run in memory_data["runs"]:
+                        if "messages" in run and isinstance(run["messages"], list):
+                            for msg in run["messages"]:
+                                if msg.get("role") in ["user", "assistant"] and "content" in msg:
+                                    if msg.get("role") == "assistant" and "tool_calls" in msg:
+                                        if not msg.get("content"):
+                                            continue
+                                    if msg.get("content"):
+                                        formatted_messages.append({"role": msg["role"], "content": msg["content"]})
+            except json.JSONDecodeError as e:
+                print(f"Error parsing memory data: {e}")
+
+        return formatted_messages, session_state
+
+    async def get_session_history(self, session_id: str):
+        try:
+            db_path = get_agent_session_db_path()
+            async with aiosqlite.connect(db_path) as conn:
+                conn.row_factory = lambda cursor, row: {col[0]: row[idx] for idx, col in enumerate(cursor.description)}
+                async with conn.execute(
+                    """
+                    select name from sqlite_master
+                    where type='table' and name='podcast_sessions'
+                    """
+                ) as cursor:
+                    table = await cursor.fetchone()
+                    if not table:
+                        return {"session_id": session_id, "messages": [], "state": "{}", "is_processing": False, "process_type": None}
+                async with conn.execute("SELECT memory, session_data FROM podcast_sessions WHERE session_id = ?", (session_id,)) as cursor:
+                    row = await cursor.fetchone()
+                if not row:
+                    return {"session_id": session_id, "messages": [], "state": "{}", "is_processing": False, "process_type": None}
+                formatted_messages, session_state = await self._get_chat_messages(row, session_id)
+
+            task_id = await self.get_active_task(session_id)
+            is_processing = bool(task_id)
+            process_type = "chat" if is_processing else None
+            browser_recording_path = self._browser_recording(session_id)
+
+            return {
+                "session_id": session_id,
+                "messages": formatted_messages,
+                "state": json.dumps(session_state),
+                "is_processing": is_processing,
+                "process_type": process_type,
+                "task_id": task_id if task_id and is_processing else None,
+                "browser_recording_path": browser_recording_path,
+            }
+        except Exception as e:
+            return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"error": f"Error retrieving session history: {str(e)}"})
+
+    async def get_supported_languages(self):
+        return {"languages": AVAILABLE_LANGS}
+
+
+podcast_agent_service = PodcastAgentService()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/celery_app.py

@@ -0,0 +1,70 @@
+from celery import Celery, Task
+import redis
+import os
+import time
+import json
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
+REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
+REDIS_DB = int(os.environ.get("REDIS_DB", 0))
+REDIS_LOCK_EXP_TIME_SEC = 60 * 10
+REDIS_LOCK_INFO_EXP_TIME_SEC = 60 * 15
+STALE_LOCK_THRESHOLD_SEC = 60 * 15
+
+redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB + 1)
+
+app = Celery("beifong_tasks", broker=f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}", backend=f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}")
+
+app.conf.update(
+    result_expires=60 * 2,
+    task_track_started=True,
+    worker_concurrency=2,
+    task_acks_late=True,
+    task_time_limit=600,
+    task_soft_time_limit=540,
+)
+
+
+class SessionLockedTask(Task):
+    def __call__(self, *args, **kwargs):
+        session_id = args[0] if args else kwargs.get("session_id")
+        if not session_id:
+            return super().__call__(*args, **kwargs)
+
+        lock_key = f"lock:{session_id}"
+        lock_info = redis_client.get(f"lock_info:{session_id}")
+        if lock_info:
+            try:
+                lock_data = json.loads(lock_info.decode("utf-8"))
+                lock_time = lock_data.get("timestamp", 0)
+                if time.time() - lock_time > STALE_LOCK_THRESHOLD_SEC:
+                    redis_client.delete(lock_key)
+                    redis_client.delete(f"lock_info:{session_id}")
+            except (ValueError, TypeError) as e:
+                print(f"Error checking lock time: {e}")
+
+        acquired = redis_client.set(lock_key, "1", nx=True, ex=REDIS_LOCK_EXP_TIME_SEC)
+        if acquired:
+            lock_data = {"timestamp": time.time(), "task_id": self.request.id if hasattr(self, "request") else None}
+            redis_client.set(f"lock_info:{session_id}", json.dumps(lock_data), ex=REDIS_LOCK_INFO_EXP_TIME_SEC)
+
+        if not acquired:
+            return {
+                "error": "Session busy",
+                "response": "This session is already processing a message. Please wait.",
+                "session_id": session_id,
+                "stage": "busy",
+                "session_state": "{}",
+                "is_processing": True,
+                "process_type": "chat",
+            }
+
+        try:
+            return super().__call__(*args, **kwargs)
+        finally:
+            redis_client.delete(lock_key)
+            redis_client.delete(f"lock_info:{session_id}")

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/celery_tasks.py

@@ -0,0 +1,85 @@
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from agno.storage.sqlite import SqliteStorage
+import os
+from dotenv import load_dotenv
+from services.celery_app import app, SessionLockedTask
+from db.config import get_agent_session_db_path
+from db.agent_config_v2 import (
+    AGENT_DESCRIPTION,
+    AGENT_INSTRUCTIONS,
+    AGENT_MODEL,
+    INITIAL_SESSION_STATE,
+)
+from agents.search_agent import search_agent_run
+from agents.scrape_agent import scrape_agent_run
+from agents.script_agent import podcast_script_agent_run
+from tools.ui_manager import ui_manager_run
+from tools.user_source_selection import user_source_selection_run
+from tools.session_state_manager import update_language, update_chat_title, mark_session_finished
+from agents.image_generate_agent import image_generation_agent_run
+from agents.audio_generate_agent import audio_generate_agent_run
+import json
+
+load_dotenv()
+
+db_file = get_agent_session_db_path()
+
+
+@app.task(bind=True, max_retries=0, base=SessionLockedTask)
+def agent_chat(self, session_id, message):
+    try:
+        print(f"Processing message for session {session_id}: {message[:50]}...")
+        db_file = get_agent_session_db_path()
+        os.makedirs(os.path.dirname(db_file), exist_ok=True)
+        from services.internal_session_service import SessionService
+
+        session_state = SessionService.get_session(session_id).get("state", INITIAL_SESSION_STATE)
+
+        _agent = Agent(
+            model=OpenAIChat(id=AGENT_MODEL, api_key=os.getenv("OPENAI_API_KEY")),
+            storage=SqliteStorage(table_name="podcast_sessions", db_file=db_file),
+            add_history_to_messages=True,
+            read_chat_history=True,
+            add_state_in_messages=True,
+            num_history_runs=30,
+            instructions=AGENT_INSTRUCTIONS,
+            description=AGENT_DESCRIPTION,
+            session_state=session_state,
+            session_id=session_id,
+            tools=[
+                search_agent_run,
+                scrape_agent_run,
+                ui_manager_run,
+                user_source_selection_run,
+                update_language,
+                podcast_script_agent_run,
+                image_generation_agent_run,
+                audio_generate_agent_run,
+                update_chat_title,
+                mark_session_finished,
+            ],
+            markdown=True,
+        )
+        response = _agent.run(message, session_id=session_id)
+        print(f"Response generated for session {session_id}")
+        _agent.write_to_storage(session_id=session_id)
+        session_state = SessionService.get_session(session_id).get("state", INITIAL_SESSION_STATE)
+        return {
+            "session_id": session_id,
+            "response": response.content,
+            "stage": _agent.session_state.get("stage", "unknown"),
+            "session_state": json.dumps(session_state),
+            "is_processing": False,
+            "process_type": None,
+        }
+    except Exception as e:
+        print(f"Error in agent_chat for session {session_id}: {str(e)}")
+        return {
+            "session_id": session_id,
+            "response": f"I'm sorry, I encountered an error: {str(e)}. Please try again.",
+            "stage": "error",
+            "session_state": "{}",
+            "is_processing": False,
+            "process_type": None,
+        }

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/db_init.py

@@ -0,0 +1,369 @@
+import os
+import sqlite3
+import asyncio
+import time
+from contextlib import contextmanager
+from concurrent.futures import ThreadPoolExecutor
+from services.db_service import get_db_path
+
+
+@contextmanager
+def db_connection(db_path):
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute("PRAGMA cache_size=10000")
+    conn.execute("PRAGMA temp_store=MEMORY")
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+
+def init_sources_db():
+    start_time = time.time()
+    db_path = get_db_path("sources_db")
+    with db_connection(db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute("BEGIN TRANSACTION")
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS sources (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            description TEXT,
+            url TEXT,
+            is_active BOOLEAN DEFAULT 1,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS categories (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT UNIQUE NOT NULL,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS source_categories (
+            source_id INTEGER,
+            category_id INTEGER,
+            PRIMARY KEY (source_id, category_id),
+            FOREIGN KEY (source_id) REFERENCES sources(id),
+            FOREIGN KEY (category_id) REFERENCES categories(id)
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS source_feeds (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            source_id INTEGER,
+            feed_url TEXT UNIQUE,
+            feed_type TEXT,
+            is_active BOOLEAN DEFAULT 1,
+            last_crawled TIMESTAMP,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (source_id) REFERENCES sources(id)
+        )
+        """)
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_source_feeds_source_id ON source_feeds(source_id)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_sources_is_active ON sources(is_active)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_source_feeds_is_active ON source_feeds(is_active)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_source_categories_source_id ON source_categories(source_id)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_source_categories_category_id ON source_categories(category_id)")
+        conn.commit()
+
+    elapsed = time.time() - start_time
+    print(f"Sources database initialized in {elapsed:.3f}s")
+
+
+def init_tracking_db():
+    start_time = time.time()
+    db_path = get_db_path("tracking_db")
+    with db_connection(db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute("BEGIN TRANSACTION")
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS feed_tracking (
+            feed_id INTEGER PRIMARY KEY,
+            source_id INTEGER,
+            feed_url TEXT,
+            last_processed TIMESTAMP,
+            last_etag TEXT,
+            last_modified TEXT,
+            entry_hash TEXT
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS feed_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            feed_id INTEGER,
+            source_id INTEGER,
+            entry_id TEXT,
+            title TEXT,
+            link TEXT UNIQUE,
+            published_date TIMESTAMP,
+            content TEXT,
+            summary TEXT,
+            crawl_status TEXT DEFAULT 'pending',
+            crawl_attempts INTEGER DEFAULT 0,
+            processed_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            UNIQUE(feed_id, entry_id)
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS crawled_articles (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            entry_id INTEGER,
+            source_id INTEGER,
+            feed_id INTEGER,
+            title TEXT,
+            url TEXT UNIQUE,
+            published_date TIMESTAMP,
+            raw_content TEXT,
+            content TEXT,
+            summary TEXT,
+            metadata TEXT,
+            ai_status TEXT DEFAULT 'pending',
+            ai_error TEXT DEFAULT NULL,
+            ai_attempts INTEGER DEFAULT 0,
+            crawled_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            processed BOOLEAN DEFAULT 0,
+            embedding_status TEXT DEFAULT NULL,
+            FOREIGN KEY (entry_id) REFERENCES feed_entries(id)
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS article_categories (
+            article_id INTEGER,
+            category_name TEXT NOT NULL,
+            PRIMARY KEY (article_id, category_name),
+            FOREIGN KEY (article_id) REFERENCES crawled_articles(id)
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS article_embeddings (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            article_id INTEGER NOT NULL,
+            embedding BLOB NOT NULL,
+            embedding_model TEXT NOT NULL,
+            created_at TEXT NOT NULL,
+            in_faiss_index INTEGER DEFAULT 0,
+            FOREIGN KEY (article_id) REFERENCES crawled_articles(id)
+        )
+        """)
+        indexes = [
+            "CREATE INDEX IF NOT EXISTS idx_feed_entries_feed_id ON feed_entries(feed_id)",
+            "CREATE INDEX IF NOT EXISTS idx_feed_entries_link ON feed_entries(link)",
+            "CREATE INDEX IF NOT EXISTS idx_crawled_articles_url ON crawled_articles(url)",
+            "CREATE INDEX IF NOT EXISTS idx_crawled_articles_entry_id ON crawled_articles(entry_id)",
+            "CREATE INDEX IF NOT EXISTS idx_feed_entries_crawl_status ON feed_entries(crawl_status)",
+            "CREATE INDEX IF NOT EXISTS idx_crawled_articles_processed ON crawled_articles(processed)",
+            "CREATE INDEX IF NOT EXISTS idx_crawled_articles_ai_status ON crawled_articles(ai_status)",
+            "CREATE INDEX IF NOT EXISTS idx_article_categories_article_id ON article_categories(article_id)",
+            "CREATE INDEX IF NOT EXISTS idx_article_categories_category_name ON article_categories(category_name)",
+            "CREATE INDEX IF NOT EXISTS idx_article_embeddings_article_id ON article_embeddings(article_id)",
+            "CREATE INDEX IF NOT EXISTS idx_article_embeddings_in_faiss ON article_embeddings(in_faiss_index)",
+            "CREATE INDEX IF NOT EXISTS idx_crawled_articles_embedding_status ON crawled_articles(embedding_status)",
+        ]
+        for index_sql in indexes:
+            cursor.execute(index_sql)
+        conn.commit()
+    elapsed = time.time() - start_time
+    print(f"Tracking database initialized in {elapsed:.3f}s")
+
+
+def init_podcasts_db():
+    start_time = time.time()
+    db_path = get_db_path("podcasts_db")
+    with db_connection(db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute("BEGIN TRANSACTION")
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS podcasts (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            title TEXT,
+            date TEXT,
+            content_json TEXT,
+            audio_generated BOOLEAN DEFAULT 0,
+            audio_path TEXT,
+            banner_img_path TEXT,
+            tts_engine TEXT DEFAULT 'elevenlabs',
+            language_code TEXT DEFAULT 'en',
+            sources_json TEXT,
+            banner_images TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """)
+        indexes = [
+            "CREATE INDEX IF NOT EXISTS idx_podcasts_date ON podcasts(date)",
+            "CREATE INDEX IF NOT EXISTS idx_podcasts_audio_generated ON podcasts(audio_generated)",
+            "CREATE INDEX IF NOT EXISTS idx_podcasts_tts_engine ON podcasts(tts_engine)",
+            "CREATE INDEX IF NOT EXISTS idx_podcasts_language_code ON podcasts(language_code)",
+        ]
+        for index_sql in indexes:
+            cursor.execute(index_sql)
+        conn.commit()
+    elapsed = time.time() - start_time
+    print(f"Podcasts database initialized in {elapsed:.3f}s")
+
+
+def init_tasks_db():
+    start_time = time.time()
+    db_path = get_db_path("tasks_db")
+    with db_connection(db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute("BEGIN TRANSACTION")
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS tasks (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            task_type TEXT,
+            description TEXT,
+            command TEXT NOT NULL,
+            frequency INTEGER NOT NULL,
+            frequency_unit TEXT NOT NULL,
+            enabled BOOLEAN DEFAULT 1,
+            last_run TIMESTAMP,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS task_executions (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            task_id INTEGER NOT NULL,
+            start_time TIMESTAMP NOT NULL,
+            end_time TIMESTAMP,
+            status TEXT NOT NULL,
+            error_message TEXT,
+            output TEXT,
+            FOREIGN KEY (task_id) REFERENCES tasks(id)
+        )
+        """)
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS podcast_configs (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            description TEXT,
+            prompt TEXT NOT NULL,
+            time_range_hours INTEGER DEFAULT 24,
+            limit_articles INTEGER DEFAULT 20,
+            is_active BOOLEAN DEFAULT 1,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            tts_engine TEXT DEFAULT 'elevenlabs',
+            language_code TEXT DEFAULT 'en',
+            podcast_script_prompt TEXT,
+            image_prompt TEXT
+        )
+        """)
+        indexes = [
+            "CREATE INDEX IF NOT EXISTS idx_tasks_enabled ON tasks(enabled)",
+            "CREATE INDEX IF NOT EXISTS idx_tasks_frequency ON tasks(frequency, frequency_unit)",
+            "CREATE INDEX IF NOT EXISTS idx_tasks_last_run ON tasks(last_run)",
+            "CREATE INDEX IF NOT EXISTS idx_task_executions_task_id ON task_executions(task_id)",
+            "CREATE INDEX IF NOT EXISTS idx_task_executions_status ON task_executions(status)",
+            "CREATE INDEX IF NOT EXISTS idx_task_executions_start_time ON task_executions(start_time)",
+            "CREATE INDEX IF NOT EXISTS idx_podcast_configs_is_active ON podcast_configs(is_active)",
+            "CREATE INDEX IF NOT EXISTS idx_podcast_configs_name ON podcast_configs(name)",
+        ]
+        for index_sql in indexes:
+            cursor.execute(index_sql)
+        conn.commit()
+    elapsed = time.time() - start_time
+    print(f"Tasks database initialized in {elapsed:.3f}s")
+
+
+async def init_agent_session_db():
+    """Initialize the agent session database. auto generated"""
+    pass
+
+
+def init_internal_sessions_db():
+    start_time = time.time()
+    db_path = get_db_path("internal_sessions_db")
+    with db_connection(db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute("BEGIN TRANSACTION")
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS session_state (
+            session_id TEXT PRIMARY KEY,
+            state JSON,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """)
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_session_state_session_id ON session_state(session_id)")
+        conn.commit()
+    elapsed = time.time() - start_time
+    print(f"Internal sessions database initialized in {elapsed:.3f}s")
+
+
+def init_social_media_db():
+    start_time = time.time()
+    db_path = get_db_path("social_media_db")
+    with db_connection(db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute("BEGIN TRANSACTION")
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS posts (
+            post_id TEXT PRIMARY KEY,
+            platform TEXT,
+            user_display_name TEXT,
+            user_handle TEXT,
+            user_profile_pic_url TEXT,
+            post_timestamp TEXT,
+            post_display_time TEXT,
+            post_url TEXT,
+            post_text TEXT,
+            post_mentions TEXT,
+            engagement_reply_count INTEGER,
+            engagement_retweet_count INTEGER,
+            engagement_like_count INTEGER,
+            engagement_bookmark_count INTEGER,
+            engagement_view_count INTEGER,
+            media TEXT,
+            media_count INTEGER,
+            is_ad BOOLEAN,
+            sentiment TEXT,
+            categories TEXT,
+            tags TEXT,
+            analysis_reasoning TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        """)
+        indexes = [
+            "CREATE INDEX IF NOT EXISTS idx_posts_platform ON posts(platform)",
+            "CREATE INDEX IF NOT EXISTS idx_posts_user_handle ON posts(user_handle)",
+            "CREATE INDEX IF NOT EXISTS idx_posts_post_timestamp ON posts(post_timestamp)",
+            "CREATE INDEX IF NOT EXISTS idx_posts_sentiment ON posts(sentiment)",
+        ]
+        for index_sql in indexes:
+            cursor.execute(index_sql)
+        conn.commit()
+    elapsed = time.time() - start_time
+    print(f"Social media database initialized in {elapsed:.3f}s")
+
+
+async def init_databases():
+    total_start = time.time()
+    print("Initializing all databases...")
+    for db_name in ["sources_db", "tracking_db", "podcasts_db", "tasks_db"]:
+        db_path = get_db_path(db_name)
+        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+    loop = asyncio.get_event_loop()
+    with ThreadPoolExecutor(max_workers=4) as executor:
+        tasks = [
+            loop.run_in_executor(executor, init_sources_db),
+            loop.run_in_executor(executor, init_tracking_db),
+            loop.run_in_executor(executor, init_podcasts_db),
+            loop.run_in_executor(executor, init_tasks_db),
+            loop.run_in_executor(executor, init_internal_sessions_db),
+            loop.run_in_executor(executor, init_social_media_db),
+        ]
+        await asyncio.gather(*tasks)
+    total_elapsed = time.time() - total_start
+    print(f"All databases initialized in {total_elapsed:.3f}s")
+
+
+def init_all_databases():
+    asyncio.run(init_databases())

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/db_service.py

@@ -0,0 +1,75 @@
+import os
+from typing import Dict, List, Any, Tuple, Union
+from fastapi import HTTPException
+from contextlib import contextmanager
+import sqlite3
+from db.config import get_db_path
+
+
+@contextmanager
+def db_connection(db_path: str):
+    """Context manager for database connections."""
+    if not os.path.exists(db_path):
+        raise HTTPException(status_code=404, detail=f"Database {db_path} not found. Initialize the database first.")
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+
+class DatabaseService:
+    """Service for managing database connections and operations."""
+
+    def __init__(self, db_name: str):
+        """
+        Initialize the database service.
+
+        Args:
+            db_name: Name of the database (sources_db, tracking_db, etc.)
+        """
+        self.db_path = get_db_path(db_name)
+
+    async def execute_query(
+        self, query: str, params: Tuple = (), fetch: bool = False, fetch_one: bool = False
+    ) -> Union[List[Dict[str, Any]], Dict[str, Any], int]:
+        """Execute a query with error handling for FastAPI."""
+        try:
+            with db_connection(self.db_path) as conn:
+                cursor = conn.cursor()
+                cursor.execute(query, params)
+
+                if fetch_one:
+                    result = cursor.fetchone()
+                    return dict(result) if result else None
+                elif fetch:
+                    return [dict(row) for row in cursor.fetchall()]
+                else:
+                    conn.commit()
+                    return cursor.lastrowid
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
+
+    async def execute_write_many(self, query: str, params_list: List[Tuple]) -> int:
+        """Execute multiple write operations in a single transaction."""
+        try:
+            with db_connection(self.db_path) as conn:
+                cursor = conn.cursor()
+                cursor.executemany(query, params_list)
+                conn.commit()
+                return cursor.rowcount
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
+
+
+
+sources_db = DatabaseService(db_name="sources_db")
+tracking_db = DatabaseService(db_name="tracking_db")
+podcasts_db = DatabaseService(db_name="podcasts_db")
+tasks_db = DatabaseService(db_name="tasks_db")
+social_media_db = DatabaseService(db_name="social_media_db")

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/internal_session_service.py

@@ -0,0 +1,155 @@
+from typing import Optional, Dict, Any
+from fastapi import HTTPException
+import json
+from datetime import datetime
+from db.config import get_db_path
+from db.agent_config_v2 import INITIAL_SESSION_STATE
+import sqlite3
+from contextlib import contextmanager
+
+
+@contextmanager
+def get_db_connection(db_name: str):
+    """Get a fresh database connection each time."""
+    db_path = get_db_path(db_name)
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+
+class SessionService:
+    """Service for managing internal session operations."""
+
+    @staticmethod
+    def get_session(session_id: str) -> Dict[str, Any]:
+        try:
+            with get_db_connection("internal_sessions_db") as conn:
+                cursor = conn.cursor()
+                query = """
+                SELECT session_id, state, created_at
+                FROM session_state
+                WHERE session_id = ?
+                """
+                cursor.execute(query, (session_id,))
+                session = cursor.fetchone()
+                if not session:
+                    return SessionService._initialize_session(session_id)
+                session_dict = dict(session)
+                if session_dict.get("state"):
+                    try:
+                        session_dict["state"] = json.loads(session_dict["state"])
+                    except json.JSONDecodeError:
+                        session_dict["state"] = {}
+                return session_dict
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching session: {str(e)}")
+
+    @staticmethod
+    def _initialize_session(session_id: str) -> Dict[str, Any]:
+        try:
+            with get_db_connection("internal_sessions_db") as conn:
+                cursor = conn.cursor()
+                state_json = json.dumps(INITIAL_SESSION_STATE)
+                insert_query = """
+                INSERT INTO session_state (session_id, state, created_at)
+                VALUES (?, ?, ?)
+                """
+                current_time = datetime.now().isoformat()
+                cursor.execute(insert_query, (session_id, state_json, current_time))
+                conn.commit()
+                return {"session_id": session_id, "state": INITIAL_SESSION_STATE, "created_at": current_time}
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Error initializing session: {str(e)}")
+
+    @staticmethod
+    def save_session(session_id: str, state: Dict[str, Any]) -> Dict[str, Any]:
+        try:
+            state_json = json.dumps(state)
+            with get_db_connection("internal_sessions_db") as conn:
+                cursor = conn.cursor()
+                conn.execute("BEGIN IMMEDIATE")
+                existing_query = "SELECT session_id FROM session_state WHERE session_id = ?"
+                cursor.execute(existing_query, (session_id,))
+                existing_session = cursor.fetchone()
+                if existing_session:
+                    update_query = "UPDATE session_state SET state = ? WHERE session_id = ?"
+                    cursor.execute(update_query, (state_json, session_id))
+                else:
+                    insert_query = "INSERT INTO session_state (session_id, state, created_at) VALUES (?, ?, ?)"
+                    current_time = datetime.now().isoformat()
+                    cursor.execute(insert_query, (session_id, state_json, current_time))
+                conn.commit()
+            return SessionService.get_session(session_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error saving session: {str(e)}")
+
+    @staticmethod
+    def delete_session(session_id: str) -> Dict[str, str]:
+        try:
+            with get_db_connection("internal_sessions_db") as conn:
+                cursor = conn.cursor()
+                existing_query = "SELECT session_id FROM session_state WHERE session_id = ?"
+                cursor.execute(existing_query, (session_id,))
+                existing_session = cursor.fetchone()
+                if not existing_session:
+                    raise HTTPException(status_code=404, detail="Session not found")
+                delete_query = "DELETE FROM session_state WHERE session_id = ?"
+                cursor.execute(delete_query, (session_id,))
+                conn.commit()
+                return {"message": f"Session with ID {session_id} successfully deleted"}
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error deleting session: {str(e)}")
+
+    @staticmethod
+    def list_sessions(page: int = 1, per_page: int = 10, search: Optional[str] = None) -> Dict[str, Any]:
+        try:
+            with get_db_connection("internal_sessions_db") as conn:
+                cursor = conn.cursor()
+                offset = (page - 1) * per_page
+                query_parts = [
+                    "SELECT session_id, created_at",
+                    "FROM session_state",
+                ]
+                query_params = []
+                if search:
+                    query_parts.append("WHERE session_id LIKE ?")
+                    search_param = f"%{search}%"
+                    query_params.append(search_param)
+
+                count_query = " ".join(query_parts).replace(
+                    "SELECT session_id, created_at",
+                    "SELECT COUNT(*)",
+                )
+                cursor.execute(count_query, tuple(query_params))
+                total_count = cursor.fetchone()[0]
+                query_parts.append("ORDER BY created_at DESC")
+                query_parts.append("LIMIT ? OFFSET ?")
+                query_params.extend([per_page, offset])
+                sessions_query = " ".join(query_parts)
+                cursor.execute(sessions_query, tuple(query_params))
+                sessions = [dict(row) for row in cursor.fetchall()]
+                total_pages = (total_count + per_page - 1) // per_page if total_count > 0 else 0
+                has_next = page < total_pages
+                has_prev = page > 1
+                return {
+                    "items": sessions,  
+                    "total": total_count,
+                    "page": page,
+                    "per_page": per_page,
+                    "total_pages": total_pages,
+                    "has_next": has_next,
+                    "has_prev": has_prev,
+                }
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error listing sessions: {str(e)}")

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/podcast_config_service.py

@@ -0,0 +1,177 @@
+from typing import List, Optional, Dict, Any
+from datetime import datetime
+from fastapi import HTTPException
+from services.db_service import tasks_db
+
+
+class PodcastConfigService:
+    """Service for managing podcast configurations."""
+
+    async def get_all_configs(self, active_only: bool = False) -> List[Dict[str, Any]]:
+        """Get all podcast configurations with optional filtering."""
+        try:
+            if active_only:
+                query = """
+                SELECT id, name, description, prompt, time_range_hours, limit_articles, 
+                       is_active, tts_engine, language_code, podcast_script_prompt, 
+                       image_prompt, created_at, updated_at
+                FROM podcast_configs
+                WHERE is_active = 1
+                ORDER BY name
+                """
+                params = ()
+            else:
+                query = """
+                SELECT id, name, description, prompt, time_range_hours, limit_articles, 
+                       is_active, tts_engine, language_code, podcast_script_prompt, 
+                       image_prompt, created_at, updated_at
+                FROM podcast_configs
+                ORDER BY name
+                """
+                params = ()
+            configs = await tasks_db.execute_query(query, params, fetch=True)
+            for config in configs:
+                config["is_active"] = bool(config.get("is_active", 0))
+            return configs
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching podcast configurations: {str(e)}")
+
+    async def get_config(self, config_id: int) -> Dict[str, Any]:
+        """Get a specific podcast configuration by ID."""
+        try:
+            query = """
+            SELECT id, name, description, prompt, time_range_hours, limit_articles, 
+                   is_active, tts_engine, language_code, podcast_script_prompt, 
+                   image_prompt, created_at, updated_at
+            FROM podcast_configs
+            WHERE id = ?
+            """
+            config = await tasks_db.execute_query(query, (config_id,), fetch=True, fetch_one=True)
+            if not config:
+                raise HTTPException(status_code=404, detail="Podcast configuration not found")
+            config["is_active"] = bool(config.get("is_active", 0))
+            return config
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching podcast configuration: {str(e)}")
+
+    async def create_config(
+        self,
+        name: str,
+        prompt: str,
+        description: Optional[str] = None,
+        time_range_hours: int = 24,
+        limit_articles: int = 20,
+        is_active: bool = True,
+        tts_engine: str = "kokoro",
+        language_code: str = "en",
+        podcast_script_prompt: Optional[str] = None,
+        image_prompt: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Create a new podcast configuration."""
+        try:
+            current_time = datetime.now().isoformat()
+            query = """
+            INSERT INTO podcast_configs 
+            (name, description, prompt, time_range_hours, limit_articles, 
+             is_active, tts_engine, language_code, podcast_script_prompt, 
+             image_prompt, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """
+            params = (
+                name,
+                description,
+                prompt,
+                time_range_hours,
+                limit_articles,
+                1 if is_active else 0,
+                tts_engine,
+                language_code,
+                podcast_script_prompt,
+                image_prompt,
+                current_time,
+                current_time,
+            )
+            config_id = await tasks_db.execute_query(query, params)
+            return await self.get_config(config_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error creating podcast configuration: {str(e)}")
+
+    async def update_config(self, config_id: int, updates: Dict[str, Any]) -> Dict[str, Any]:
+        """Update an existing podcast configuration."""
+        try:
+            allowed_fields = [
+                "name",
+                "description",
+                "prompt",
+                "time_range_hours",
+                "limit_articles",
+                "is_active",
+                "tts_engine",
+                "language_code",
+                "podcast_script_prompt",
+                "image_prompt",
+            ]
+            set_clauses = []
+            params = []
+            set_clauses.append("updated_at = ?")
+            params.append(datetime.now().isoformat())
+            for field, value in updates.items():
+                if field in allowed_fields:
+                    if field == "is_active":
+                        value = 1 if value else 0
+                    set_clauses.append(f"{field} = ?")
+                    params.append(value)
+            if not set_clauses:
+                return await self.get_config(config_id)
+            params.append(config_id)
+            update_query = f"""
+            UPDATE podcast_configs
+            SET {", ".join(set_clauses)}
+            WHERE id = ?
+            """
+            await tasks_db.execute_query(update_query, tuple(params))
+            return await self.get_config(config_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error updating podcast configuration: {str(e)}")
+
+    async def delete_config(self, config_id: int) -> Dict[str, str]:
+        """Delete a podcast configuration."""
+        try:
+            config = await self.get_config(config_id)
+            query = """
+            DELETE FROM podcast_configs
+            WHERE id = ?
+            """
+            await tasks_db.execute_query(query, (config_id,))
+            return {"message": f"Podcast configuration '{config['name']}' has been deleted"}
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error deleting podcast configuration: {str(e)}")
+
+    async def toggle_config(self, config_id: int, enable: bool) -> Dict[str, Any]:
+        """Enable or disable a podcast configuration."""
+        try:
+            query = """
+            UPDATE podcast_configs
+            SET is_active = ?, updated_at = ?
+            WHERE id = ?
+            """
+            current_time = datetime.now().isoformat()
+            await tasks_db.execute_query(query, (1 if enable else 0, current_time, config_id))
+            return await self.get_config(config_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error updating podcast configuration: {str(e)}")
+
+
+podcast_config_service = PodcastConfigService()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/podcast_service.py

@@ -0,0 +1,349 @@
+import os
+import json
+from typing import List, Dict, Optional, Any
+from datetime import datetime
+from fastapi import HTTPException, UploadFile
+from services.db_service import podcasts_db
+import math
+
+AUDIO_DIR = "podcasts/audio"
+IMAGE_DIR = "podcasts/images"
+
+
+class PodcastService:
+    """Service for managing podcast operations with the new database structure."""
+
+    def __init__(self):
+        """Initialize the podcast service with directories."""
+        os.makedirs(AUDIO_DIR, exist_ok=True)
+        os.makedirs(IMAGE_DIR, exist_ok=True)
+
+    async def get_podcasts(
+        self,
+        page: int = 1,
+        per_page: int = 10,
+        search: str = None,
+        date_from: str = None,
+        date_to: str = None,
+        language_code: str = None,
+        tts_engine: str = None,
+        has_audio: bool = None,
+    ) -> Dict[str, Any]:
+        """
+        Get a paginated list of podcasts with optional filtering.
+        """
+        try:
+            offset = (page - 1) * per_page
+            count_query = "SELECT COUNT(*) as count FROM podcasts"
+            query = """
+            SELECT id, title, date, audio_generated, audio_path, banner_img_path,
+                   language_code, tts_engine, created_at
+            FROM podcasts
+            """
+            where_conditions = []
+            params = []
+            if search:
+                where_conditions.append("(title LIKE ?)")
+                search_param = f"%{search}%"
+                params.append(search_param)
+            if date_from:
+                where_conditions.append("date >= ?")
+                params.append(date_from)
+            if date_to:
+                where_conditions.append("date <= ?")
+                params.append(date_to)
+            if language_code:
+                where_conditions.append("language_code = ?")
+                params.append(language_code)
+            if tts_engine:
+                where_conditions.append("tts_engine = ?")
+                params.append(tts_engine)
+            if has_audio is not None:
+                where_conditions.append("audio_generated = ?")
+                params.append(1 if has_audio else 0)
+            if where_conditions:
+                where_clause = " WHERE " + " AND ".join(where_conditions)
+                query += where_clause
+                count_query += where_clause
+            query += " ORDER BY date DESC, created_at DESC"
+            query += " LIMIT ? OFFSET ?"
+            params.extend([per_page, offset])
+            total_result = await podcasts_db.execute_query(count_query, tuple(params[:-2] if params else ()), fetch=True, fetch_one=True)
+            total_items = total_result.get("count", 0) if total_result else 0
+            total_pages = math.ceil(total_items / per_page) if total_items > 0 else 0
+            podcasts = await podcasts_db.execute_query(query, tuple(params), fetch=True)
+            for podcast in podcasts:
+                podcast["audio_generated"] = bool(podcast.get("audio_generated", 0))
+                if podcast.get("banner_img_path"):
+                    podcast["banner_img"] = podcast.get("banner_img_path")
+                else:
+                    podcast["banner_img"] = None
+                podcast.pop("banner_img_path", None)
+                podcast["identifier"] = str(podcast.get("id", ""))
+            has_next = page < total_pages
+            has_prev = page > 1
+            return {
+                "items": podcasts,
+                "total": total_items,
+                "page": page,
+                "per_page": per_page,
+                "total_pages": total_pages,
+                "has_next": has_next,
+                "has_prev": has_prev,
+            }
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Error loading podcasts: {str(e)}")
+
+    async def get_podcast(self, podcast_id: int) -> Optional[Dict[str, Any]]:
+        """Get a specific podcast by ID without content."""
+        try:
+            query = """
+            SELECT id, title, date, audio_generated, audio_path, banner_img_path,
+                language_code, tts_engine, created_at, banner_images
+            FROM podcasts
+            WHERE id = ?
+            """
+            podcast = await podcasts_db.execute_query(query, (podcast_id,), fetch=True, fetch_one=True)
+            if not podcast:
+                raise HTTPException(status_code=404, detail="Podcast not found")
+            podcast["audio_generated"] = bool(podcast.get("audio_generated", 0))
+            if podcast.get("banner_img_path"):
+                podcast["banner_img"] = podcast.get("banner_img_path")
+            else:
+                podcast["banner_img"] = None
+            podcast.pop("banner_img_path", None)
+            podcast["identifier"] = str(podcast.get("id", ""))
+            sources_query = "SELECT sources_json FROM podcasts WHERE id = ?"
+            sources_result = await podcasts_db.execute_query(sources_query, (podcast_id,), fetch=True, fetch_one=True)
+            sources = []
+            if sources_result and sources_result.get("sources_json"):
+                try:
+                    parsed_sources = json.loads(sources_result["sources_json"])
+                    if isinstance(parsed_sources, list):
+                        sources = parsed_sources
+                    else:
+                        sources = [parsed_sources]
+                except json.JSONDecodeError:
+                    sources = []
+            podcast["sources"] = sources
+            
+            try:
+                banner_images = json.loads(podcast.get("banner_images", "[]"))
+            except json.JSONDecodeError:
+                banner_images = []
+            podcast["banner_images"] = banner_images
+            
+            return podcast
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error loading podcast: {str(e)}")
+
+    async def get_podcast_by_identifier(self, identifier: str) -> Optional[Dict[str, Any]]:
+        """Get a specific podcast by string identifier (which is actually the ID)."""
+        try:
+            try:
+                podcast_id = int(identifier)
+            except ValueError:
+                raise HTTPException(status_code=404, detail="Invalid podcast identifier")
+            return await self.get_podcast(podcast_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error loading podcast: {str(e)}")
+
+    async def get_podcast_content(self, podcast_id: int) -> Dict[str, Any]:
+        """Get the content of a specific podcast."""
+        try:
+            query = """
+            SELECT content_json FROM podcasts WHERE id = ?
+            """
+            result = await podcasts_db.execute_query(query, (podcast_id,), fetch=True, fetch_one=True)
+            if not result or not result.get("content_json"):
+                raise HTTPException(status_code=404, detail="Podcast content not found")
+            try:
+                content = json.loads(result["content_json"])
+                return content
+            except json.JSONDecodeError:
+                raise HTTPException(status_code=500, detail="Invalid podcast content format")
+
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error loading podcast content: {str(e)}")
+
+    async def get_podcast_audio_url(self, podcast: Dict[str, Any]) -> Optional[str]:
+        """Get the URL for the podcast audio file if available."""
+        if podcast.get("audio_generated") and podcast.get("audio_path"):
+            return f"/audio/{podcast.get('audio_path')}"
+        return None
+
+    async def get_podcast_formats(self) -> List[str]:
+        """Get list of available podcast formats for filtering."""
+        # Note: This may need to be adapted if format field is added
+        return ["daily", "weekly", "tech", "news"]
+
+    async def get_language_codes(self) -> List[str]:
+        try:
+            query = """
+            SELECT DISTINCT language_code FROM podcasts WHERE language_code IS NOT NULL
+            """
+            results = await podcasts_db.execute_query(query, (), fetch=True)
+            language_codes = [result["language_code"] for result in results if result["language_code"]]
+            if "en" not in language_codes:
+                language_codes.append("en")
+            return sorted(language_codes)
+        except Exception as _:
+            return ["en"]
+
+    async def get_tts_engines(self) -> List[str]:
+        """Get list of available TTS engines for filtering."""
+        try:
+            query = """
+            SELECT DISTINCT tts_engine FROM podcasts WHERE tts_engine IS NOT NULL
+            """
+            results = await podcasts_db.execute_query(query, (), fetch=True)
+            tts_engines = [result["tts_engine"] for result in results if result["tts_engine"]]
+            default_engines = ["elevenlabs", "openai", "kokoro"]
+            for engine in default_engines:
+                if engine not in tts_engines:
+                    tts_engines.append(engine)
+            return sorted(tts_engines)
+        except Exception as e:
+            return ["elevenlabs", "openai", "kokoro"]
+
+    async def create_podcast(
+        self, title: str, date: str, content: Dict[str, Any], sources: List[str] = None, language_code: str = "en", tts_engine: str = "kokoro"
+    ) -> Dict[str, Any]:
+        """Create a new podcast in the database."""
+        try:
+            content_json = json.dumps(content)
+            sources_json = json.dumps(sources) if sources else None
+            current_time = datetime.now().isoformat()
+            query = """
+            INSERT INTO podcasts 
+            (title, date, content_json, audio_generated, sources_json, language_code, tts_engine, created_at)
+            VALUES (?, ?, ?, 0, ?, ?, ?, ?)
+            """
+            params = (title, date, content_json, sources_json, language_code, tts_engine, current_time)
+            podcast_id = await podcasts_db.execute_query(query, params)
+            return await self.get_podcast(podcast_id)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Error creating podcast: {str(e)}")
+
+    async def update_podcast(self, podcast_id: int, podcast_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Update podcast metadata and content."""
+        try:
+            existing = await self.get_podcast(podcast_id)
+            if not existing:
+                raise HTTPException(status_code=404, detail="Podcast not found")
+            fields = []
+            params = []
+            if "title" in podcast_data:
+                fields.append("title = ?")
+                params.append(podcast_data["title"])
+            if "date" in podcast_data:
+                fields.append("date = ?")
+                params.append(podcast_data["date"])
+            if "content" in podcast_data and isinstance(podcast_data["content"], dict):
+                fields.append("content_json = ?")
+                params.append(json.dumps(podcast_data["content"]))
+            if "audio_generated" in podcast_data:
+                fields.append("audio_generated = ?")
+                params.append(1 if podcast_data["audio_generated"] else 0)
+            if "audio_path" in podcast_data:
+                fields.append("audio_path = ?")
+                params.append(podcast_data["audio_path"])
+            if "banner_img_path" in podcast_data:
+                fields.append("banner_img_path = ?")
+                params.append(podcast_data["banner_img_path"])
+            if "sources" in podcast_data:
+                fields.append("sources_json = ?")
+                params.append(json.dumps(podcast_data["sources"]))
+            if "language_code" in podcast_data:
+                fields.append("language_code = ?")
+                params.append(podcast_data["language_code"])
+            if "tts_engine" in podcast_data:
+                fields.append("tts_engine = ?")
+                params.append(podcast_data["tts_engine"])
+            if not fields:
+                return existing
+            params.append(podcast_id)
+            query = f"""
+            UPDATE podcasts SET {", ".join(fields)}
+            WHERE id = ?
+            """
+            await podcasts_db.execute_query(query, tuple(params))
+            return await self.get_podcast(podcast_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error updating podcast: {str(e)}")
+
+    async def delete_podcast(self, podcast_id: int, delete_assets: bool = False) -> bool:
+        """Delete a podcast from the database."""
+        try:
+            existing = await self.get_podcast(podcast_id)
+            if not existing:
+                raise HTTPException(status_code=404, detail="Podcast not found")
+            query = "DELETE FROM podcasts WHERE id = ?"
+            result = await podcasts_db.execute_query(query, (podcast_id,))
+            if delete_assets:
+                if existing.get("audio_path"):
+                    audio_path = os.path.join(AUDIO_DIR, existing["audio_path"])
+                    if os.path.exists(audio_path):
+                        os.remove(audio_path)
+                if existing.get("banner_img_path"):
+                    img_path = os.path.join(IMAGE_DIR, existing["banner_img_path"])
+                    if os.path.exists(img_path):
+                        os.remove(img_path)
+            return result > 0
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error deleting podcast: {str(e)}")
+
+    async def upload_podcast_audio(self, podcast_id: int, file: UploadFile) -> Dict[str, Any]:
+        """Upload an audio file for a podcast."""
+        try:
+            await self.get_podcast(podcast_id)
+            filename = f"podcast_{podcast_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}"
+            if file.filename:
+                ext = os.path.splitext(file.filename)[1]
+                filename = f"{filename}{ext}"
+            else:
+                filename = f"{filename}.mp3"
+            file_path = os.path.join(AUDIO_DIR, filename)
+            contents = await file.read()
+            with open(file_path, "wb") as f:
+                f.write(contents)
+            update_data = {"audio_generated": True, "audio_path": filename}
+            return await self.update_podcast(podcast_id, update_data)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error uploading audio: {str(e)}")
+
+    async def upload_podcast_banner(self, podcast_id: int, file: UploadFile) -> Dict[str, Any]:
+        """Upload a banner image for a podcast."""
+        try:
+            await self.get_podcast(podcast_id)
+            filename = f"banner_{podcast_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}"
+            if file.filename:
+                ext = os.path.splitext(file.filename)[1]
+                filename = f"{filename}{ext}"
+            else:
+                filename = f"{filename}.jpg"
+            file_path = os.path.join(IMAGE_DIR, filename)
+            contents = await file.read()
+            with open(file_path, "wb") as f:
+                f.write(contents)
+            update_data = {"banner_img_path": filename}
+            return await self.update_podcast(podcast_id, update_data)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error uploading banner: {str(e)}")
+
+
+podcast_service = PodcastService()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/social_media_service.py

@@ -0,0 +1,603 @@
+import json
+from typing import List, Optional, Dict, Any
+from fastapi import HTTPException
+from services.db_service import social_media_db
+from models.social_media_schemas import PaginatedPosts, Post
+from datetime import datetime, timedelta
+
+
+class SocialMediaService:
+    """Service for managing social media posts."""
+
+    async def get_posts(
+        self,
+        page: int = 1,
+        per_page: int = 10,
+        platform: Optional[str] = None,
+        user_handle: Optional[str] = None,
+        sentiment: Optional[str] = None,
+        category: Optional[str] = None,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        search: Optional[str] = None,
+    ) -> PaginatedPosts:
+        """Get social media posts with pagination and filtering."""
+        try:
+            offset = (page - 1) * per_page
+            query_parts = [
+                "SELECT * FROM posts",
+                "WHERE 1=1",
+            ]
+            query_params = []
+            if platform:
+                query_parts.append("AND platform = ?")
+                query_params.append(platform)
+            if user_handle:
+                query_parts.append("AND user_handle = ?")
+                query_params.append(user_handle)
+            if sentiment:
+                query_parts.append("AND sentiment = ?")
+                query_params.append(sentiment)
+            if category:
+                query_parts.append("AND categories LIKE ?")
+                query_params.append(f'%"{category}"%')
+            if date_from:
+                query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+                query_params.append(date_from)
+            if date_to:
+                query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+                query_params.append(date_to)
+            if search:
+                query_parts.append("AND (post_text LIKE ? OR user_display_name LIKE ? OR user_handle LIKE ?)")
+                search_param = f"%{search}%"
+                query_params.extend([search_param, search_param, search_param])
+            count_query = " ".join(query_parts).replace("SELECT *", "SELECT COUNT(*)")
+            total_posts = await social_media_db.execute_query(count_query, tuple(query_params), fetch=True, fetch_one=True)
+            total_count = total_posts.get("COUNT(*)", 0) if total_posts else 0
+            query_parts.append("ORDER BY datetime(post_timestamp) DESC, post_id DESC")
+            query_parts.append("LIMIT ? OFFSET ?")
+            query_params.extend([per_page, offset])
+            posts_query = " ".join(query_parts)
+            posts_data = await social_media_db.execute_query(posts_query, tuple(query_params), fetch=True)
+            posts = []
+            for post in posts_data:
+                post_dict = dict(post)
+                if post_dict.get("media"):
+                    try:
+                        post_dict["media"] = json.loads(post_dict["media"])
+                    except json.JSONDecodeError:
+                        post_dict["media"] = []
+                if post_dict.get("categories"):
+                    try:
+                        post_dict["categories"] = json.loads(post_dict["categories"])
+                    except json.JSONDecodeError:
+                        post_dict["categories"] = []
+                if post_dict.get("tags"):
+                    try:
+                        post_dict["tags"] = json.loads(post_dict["tags"])
+                    except json.JSONDecodeError:
+                        post_dict["tags"] = []
+                post_dict["engagement"] = {
+                    "replies": post_dict.pop("engagement_reply_count", 0),
+                    "retweets": post_dict.pop("engagement_retweet_count", 0),
+                    "likes": post_dict.pop("engagement_like_count", 0),
+                    "bookmarks": post_dict.pop("engagement_bookmark_count", 0),
+                    "views": post_dict.pop("engagement_view_count", 0),
+                }
+                posts.append(post_dict)
+            total_pages = (total_count + per_page - 1) // per_page if total_count > 0 else 0
+            has_next = page < total_pages
+            has_prev = page > 1
+            return PaginatedPosts(
+                items=posts,
+                total=total_count,
+                page=page,
+                per_page=per_page,
+                total_pages=total_pages,
+                has_next=has_next,
+                has_prev=has_prev,
+            )
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching social media posts: {str(e)}")
+
+    async def get_post(self, post_id: str) -> Dict[str, Any]:
+        """Get a specific post by ID."""
+        try:
+            query = "SELECT * FROM posts WHERE post_id = ?"
+            post = await social_media_db.execute_query(query, (post_id,), fetch=True, fetch_one=True)
+            if not post:
+                raise HTTPException(status_code=404, detail="Post not found")
+            post_dict = dict(post)
+            if post_dict.get("media"):
+                try:
+                    post_dict["media"] = json.loads(post_dict["media"])
+                except json.JSONDecodeError:
+                    post_dict["media"] = []
+            if post_dict.get("categories"):
+                try:
+                    post_dict["categories"] = json.loads(post_dict["categories"])
+                except json.JSONDecodeError:
+                    post_dict["categories"] = []
+            if post_dict.get("tags"):
+                try:
+                    post_dict["tags"] = json.loads(post_dict["tags"])
+                except json.JSONDecodeError:
+                    post_dict["tags"] = []
+            post_dict["engagement"] = {
+                "replies": post_dict.pop("engagement_reply_count", 0),
+                "retweets": post_dict.pop("engagement_retweet_count", 0),
+                "likes": post_dict.pop("engagement_like_count", 0),
+                "bookmarks": post_dict.pop("engagement_bookmark_count", 0),
+                "views": post_dict.pop("engagement_view_count", 0),
+            }
+            return post_dict
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching social media post: {str(e)}")
+
+    async def get_platforms(self) -> List[str]:
+        """Get all platforms that have posts."""
+        query = "SELECT DISTINCT platform FROM posts ORDER BY platform"
+        result = await social_media_db.execute_query(query, fetch=True)
+        return [row.get("platform", "") for row in result if row.get("platform")]
+
+    async def get_sentiments(self, date_from: Optional[str] = None, date_to: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Get sentiment distribution with post counts."""
+        try:
+            query_parts = [
+                """
+                SELECT 
+                    sentiment, COUNT(*) as post_count 
+                FROM posts 
+                WHERE sentiment IS NOT NULL
+                """
+            ]
+            params = []
+            if date_from:
+                query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+                params.append(date_from)
+            if date_to:
+                query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+                params.append(date_to)
+            query_parts.append("GROUP BY sentiment ORDER BY post_count DESC")
+            query = " ".join(query_parts)
+            return await social_media_db.execute_query(query, tuple(params), fetch=True)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching sentiments: {str(e)}")
+
+    async def get_top_users(
+        self, 
+        platform: Optional[str] = None,
+        limit: int = 10,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """Get top users by post count."""
+        query_parts = ["SELECT user_handle, user_display_name, COUNT(*) as post_count", "FROM posts", "WHERE user_handle IS NOT NULL"]
+        params = []
+        if platform:
+            query_parts.append("AND platform = ?")
+            params.append(platform)
+        if date_from:
+            query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+            params.append(date_from)
+        if date_to:
+            query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+            params.append(date_to)
+        query_parts.extend(["GROUP BY user_handle", "ORDER BY post_count DESC", "LIMIT ?"])
+        params.append(limit)
+        query = " ".join(query_parts)
+        return await social_media_db.execute_query(query, tuple(params), fetch=True)
+
+    async def get_categories(self, date_from: Optional[str] = None, date_to: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Get all categories with post counts."""
+        try:
+            query_parts = ["SELECT categories FROM posts WHERE categories IS NOT NULL"]
+            params = []
+            if date_from:
+                query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+                params.append(date_from)
+            if date_to:
+                query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+                params.append(date_to) 
+            query = " ".join(query_parts)
+            result = await social_media_db.execute_query(query, tuple(params), fetch=True)
+            category_counts = {}
+            for row in result:
+                if row.get("categories"):
+                    try:
+                        categories = json.loads(row["categories"])
+                        for category in categories:
+                            if category in category_counts:
+                                category_counts[category] += 1
+                            else:
+                                category_counts[category] = 1
+                    except json.JSONDecodeError:
+                        pass
+            return [{"category": category, "post_count": count} for category, count in sorted(category_counts.items(), key=lambda x: x[1], reverse=True)]
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching categories: {str(e)}")
+
+    async def get_user_sentiment(
+        self,
+        limit: int = 10,
+        platform: Optional[str] = None,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """Get users with their sentiment breakdown."""
+        try:
+            query_parts = [
+                """
+                SELECT 
+                    user_handle, 
+                    user_display_name,
+                    COUNT(*) as total_posts,
+                    SUM(CASE WHEN sentiment = 'positive' THEN 1 ELSE 0 END) as positive_count,
+                    SUM(CASE WHEN sentiment = 'negative' THEN 1 ELSE 0 END) as negative_count,
+                    SUM(CASE WHEN sentiment = 'neutral' THEN 1 ELSE 0 END) as neutral_count,
+                    SUM(CASE WHEN sentiment = 'critical' THEN 1 ELSE 0 END) as critical_count
+                FROM posts
+                WHERE user_handle IS NOT NULL
+                """
+            ]
+            params = []
+            if platform:
+                query_parts.append("AND platform = ?")
+                params.append(platform)
+            if date_from:
+                query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+                params.append(date_from)
+            if date_to:
+                query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+                params.append(date_to)
+            query_parts.extend(["GROUP BY user_handle, user_display_name", "ORDER BY total_posts DESC", "LIMIT ?"])
+            params.append(limit)
+            query = " ".join(query_parts)
+            result = await social_media_db.execute_query(query, tuple(params), fetch=True)
+            for user in result:
+                total = user["total_posts"]
+                user["positive_percent"] = (user["positive_count"] / total) * 100 if total > 0 else 0
+                user["negative_percent"] = (user["negative_count"] / total) * 100 if total > 0 else 0
+                user["neutral_percent"] = (user["neutral_count"] / total) * 100 if total > 0 else 0
+                user["critical_percent"] = (user["critical_count"] / total) * 100 if total > 0 else 0
+            return result
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching user sentiment: {str(e)}")
+
+    async def get_category_sentiment(self, date_from: Optional[str] = None, date_to: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Get sentiment distribution by category."""
+        try:
+            date_filter = ""
+            params = []
+            if date_from or date_to:
+                date_filter = "WHERE "
+                if date_from:
+                    date_filter += "datetime(p.post_timestamp) >= datetime(?)"
+                    params.append(date_from)
+                    if date_to:
+                        date_filter += " AND "
+                if date_to:
+                    date_filter += "datetime(p.post_timestamp) <= datetime(?)"
+                    params.append(date_to)
+            query = f"""
+            WITH category_data AS (
+                SELECT 
+                    json_each.value as category,
+                    sentiment,
+                    COUNT(*) as count
+                FROM 
+                    posts p,
+                    json_each(p.categories)
+                {date_filter}
+                GROUP BY 
+                    json_each.value, sentiment
+            )
+            SELECT 
+                category,
+                SUM(count) as total_count,
+                SUM(CASE WHEN sentiment = 'positive' THEN count ELSE 0 END) as positive_count,
+                SUM(CASE WHEN sentiment = 'negative' THEN count ELSE 0 END) as negative_count,
+                SUM(CASE WHEN sentiment = 'neutral' THEN count ELSE 0 END) as neutral_count,
+                SUM(CASE WHEN sentiment = 'critical' THEN count ELSE 0 END) as critical_count
+            FROM 
+                category_data
+            GROUP BY 
+                category
+            ORDER BY 
+                total_count DESC
+            """
+            result = await social_media_db.execute_query(query, tuple(params), fetch=True)
+            for category in result:
+                total = category["total_count"]
+                category["positive_percent"] = (category["positive_count"] / total) * 100 if total > 0 else 0
+                category["negative_percent"] = (category["negative_count"] / total) * 100 if total > 0 else 0
+                category["neutral_percent"] = (category["neutral_count"] / total) * 100 if total > 0 else 0
+                category["critical_percent"] = (category["critical_count"] / total) * 100 if total > 0 else 0
+            return result
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching category sentiment: {str(e)}")
+
+    async def get_trending_topics(
+        self, 
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """Get trending topics with sentiment breakdown."""
+        try:
+            query_parts = [
+                """
+                WITH topic_data AS (
+                    SELECT 
+                        json_each.value as topic,
+                        sentiment,
+                        COUNT(*) as count
+                    FROM 
+                        posts,
+                        json_each(posts.tags)
+                    WHERE tags IS NOT NULL
+                """
+            ]
+            params = []
+            if date_from:
+                query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+                params.append(date_from)
+            if date_to:
+                query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+                params.append(date_to)  
+            query_parts.append(
+                """
+                GROUP BY 
+                    json_each.value, sentiment
+                )
+                SELECT 
+                    topic,
+                    SUM(count) as total_count,
+                    SUM(CASE WHEN sentiment = 'positive' THEN count ELSE 0 END) as positive_count,
+                    SUM(CASE WHEN sentiment = 'negative' THEN count ELSE 0 END) as negative_count,
+                    SUM(CASE WHEN sentiment = 'neutral' THEN count ELSE 0 END) as neutral_count,
+                    SUM(CASE WHEN sentiment = 'critical' THEN count ELSE 0 END) as critical_count
+                FROM 
+                    topic_data
+                GROUP BY 
+                    topic
+                ORDER BY 
+                    total_count DESC
+                LIMIT ?
+                """
+            )
+            params.append(limit)
+            query = " ".join(query_parts)
+            result = await social_media_db.execute_query(query, tuple(params), fetch=True)
+            for topic in result:
+                total = topic["total_count"]
+                topic["positive_percent"] = (topic["positive_count"] / total) * 100 if total > 0 else 0
+                topic["negative_percent"] = (topic["negative_count"] / total) * 100 if total > 0 else 0
+                topic["neutral_percent"] = (topic["neutral_count"] / total) * 100 if total > 0 else 0
+                topic["critical_percent"] = (topic["critical_count"] / total) * 100 if total > 0 else 0
+            return result
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching trending topics: {str(e)}")
+
+    async def get_sentiment_over_time(
+        self, 
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        platform: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """Get sentiment trends over time."""
+        try:
+            date_range_query = ""
+            if date_from and date_to:
+                date_range_query = f"""
+                WITH RECURSIVE date_range(date) AS (
+                    SELECT date('{date_from}')
+                    UNION ALL
+                    SELECT date(date, '+1 day')
+                    FROM date_range
+                    WHERE date < date('{date_to}')
+                )
+                SELECT date as post_date FROM date_range
+                """
+            else:
+                days_ago = (datetime.now() - timedelta(days=30)).isoformat()
+                date_range_query = f"""
+                WITH RECURSIVE date_range(date) AS (
+                    SELECT date('{days_ago}')
+                    UNION ALL
+                    SELECT date(date, '+1 day')
+                    FROM date_range
+                    WHERE date < date('now')
+                )
+                SELECT date as post_date FROM date_range
+                """
+            query_parts = [
+                f"""
+                WITH dates AS (
+                    {date_range_query}
+                )
+                SELECT 
+                    dates.post_date,
+                    COALESCE(SUM(CASE WHEN sentiment = 'positive' THEN 1 ELSE 0 END), 0) as positive_count,
+                    COALESCE(SUM(CASE WHEN sentiment = 'negative' THEN 1 ELSE 0 END), 0) as negative_count,
+                    COALESCE(SUM(CASE WHEN sentiment = 'neutral' THEN 1 ELSE 0 END), 0) as neutral_count,
+                    COALESCE(SUM(CASE WHEN sentiment = 'critical' THEN 1 ELSE 0 END), 0) as critical_count,
+                    COUNT(posts.post_id) as total_count
+                FROM 
+                    dates
+                LEFT JOIN 
+                    posts ON date(posts.post_timestamp) = dates.post_date
+                """
+            ]
+            params = []
+            if platform:
+                query_parts.append("AND posts.platform = ?")
+                params.append(platform)
+            query_parts.append("GROUP BY dates.post_date ORDER BY dates.post_date")
+            query = " ".join(query_parts)
+            result = await social_media_db.execute_query(query, tuple(params), fetch=True)
+            for day in result:
+                total = day["total_count"]
+                day["positive_percent"] = (day["positive_count"] / total) * 100 if total > 0 else 0
+                day["negative_percent"] = (day["negative_count"] / total) * 100 if total > 0 else 0
+                day["neutral_percent"] = (day["neutral_count"] / total) * 100 if total > 0 else 0
+                day["critical_percent"] = (day["critical_count"] / total) * 100 if total > 0 else 0
+            return result
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching sentiment over time: {str(e)}")
+
+    async def get_influential_posts(
+        self, 
+        sentiment: Optional[str] = None,
+        limit: int = 5,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """Get most influential posts by engagement, optionally filtered by sentiment."""
+        try:
+            query_parts = [
+                """
+                SELECT *,
+                    (COALESCE(engagement_reply_count, 0) + 
+                    COALESCE(engagement_retweet_count, 0) + 
+                    COALESCE(engagement_like_count, 0) + 
+                    COALESCE(engagement_bookmark_count, 0)) as total_engagement
+                FROM posts
+                WHERE 1=1
+                """
+            ]
+            params = []
+            if sentiment:
+                query_parts.append("AND sentiment = ?")
+                params.append(sentiment)  
+            if date_from:
+                query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+                params.append(date_from)
+            if date_to:
+                query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+                params.append(date_to)
+            query_parts.extend(["ORDER BY total_engagement DESC", "LIMIT ?"])
+            params.append(limit)
+            query = " ".join(query_parts)
+            result = await social_media_db.execute_query(query, tuple(params), fetch=True)
+            processed_posts = []
+            for post in result:
+                post_dict = dict(post)
+                if post_dict.get("media"):
+                    try:
+                        post_dict["media"] = json.loads(post_dict["media"])
+                    except json.JSONDecodeError:
+                        post_dict["media"] = []
+                if post_dict.get("categories"):
+                    try:
+                        post_dict["categories"] = json.loads(post_dict["categories"])
+                    except json.JSONDecodeError:
+                        post_dict["categories"] = []
+                if post_dict.get("tags"):
+                    try:
+                        post_dict["tags"] = json.loads(post_dict["tags"])
+                    except json.JSONDecodeError:
+                        post_dict["tags"] = []
+                post_dict["engagement"] = {
+                    "replies": post_dict.pop("engagement_reply_count", 0),
+                    "retweets": post_dict.pop("engagement_retweet_count", 0),
+                    "likes": post_dict.pop("engagement_like_count", 0),
+                    "bookmarks": post_dict.pop("engagement_bookmark_count", 0),
+                    "views": post_dict.pop("engagement_view_count", 0),
+                }
+                processed_posts.append(post_dict)
+            return processed_posts
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching influential posts: {str(e)}")
+
+    async def get_engagement_stats(
+        self,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Get overall engagement statistics."""
+        try:
+            query_parts = [
+                """
+                SELECT 
+                    AVG(COALESCE(engagement_reply_count, 0)) as avg_replies,
+                    AVG(COALESCE(engagement_retweet_count, 0)) as avg_retweets,
+                    AVG(COALESCE(engagement_like_count, 0)) as avg_likes,
+                    AVG(COALESCE(engagement_bookmark_count, 0)) as avg_bookmarks,
+                    AVG(COALESCE(engagement_view_count, 0)) as avg_views,
+                    MAX(COALESCE(engagement_reply_count, 0)) as max_replies,
+                    MAX(COALESCE(engagement_retweet_count, 0)) as max_retweets,
+                    MAX(COALESCE(engagement_like_count, 0)) as max_likes,
+                    MAX(COALESCE(engagement_bookmark_count, 0)) as max_bookmarks,
+                    MAX(COALESCE(engagement_view_count, 0)) as max_views,
+                    COUNT(*) as total_posts,
+                    COUNT(DISTINCT user_handle) as unique_authors
+                FROM posts
+                WHERE 1=1
+                """
+            ]
+            params = []
+            if date_from:
+                query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+                params.append(date_from)
+            if date_to:
+                query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+                params.append(date_to)
+            query = " ".join(query_parts)
+            result = await social_media_db.execute_query(query, tuple(params), fetch=True, fetch_one=True)
+            if not result:
+                return {"avg_engagement": 0, "total_posts": 0, "unique_authors": 0}
+            result_dict = dict(result)
+            result_dict["avg_engagement"] = (
+                result_dict["avg_replies"] + result_dict["avg_retweets"] + result_dict["avg_likes"] + result_dict["avg_bookmarks"]
+            )
+            platform_query_parts = [
+                """
+                SELECT 
+                    platform, 
+                    COUNT(*) as post_count
+                FROM posts
+                WHERE 1=1
+                """
+            ]
+            if date_from:
+                platform_query_parts.append("AND datetime(post_timestamp) >= datetime(?)")
+            if date_to:
+                platform_query_parts.append("AND datetime(post_timestamp) <= datetime(?)")
+            platform_query_parts.extend([
+                "GROUP BY platform",
+                "ORDER BY post_count DESC",
+                "LIMIT 10"
+            ])
+            platforms = await social_media_db.execute_query(
+                " ".join(platform_query_parts), 
+                tuple(params), 
+                fetch=True
+            )
+            result_dict["platforms"] = platforms
+            return result_dict
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching engagement stats: {str(e)}")
+
+
+social_media_service = SocialMediaService()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/source_service.py

@@ -0,0 +1,375 @@
+from typing import List, Optional, Dict, Any
+from fastapi import HTTPException
+from datetime import datetime
+from services.db_service import sources_db, tracking_db
+from models.source_schemas import SourceCreate, SourceUpdate, SourceFeedCreate, PaginatedSources
+
+
+class SourceService:
+    """Service for managing source operations with the new database structure."""
+
+    async def get_sources(
+        self, page: int = 1, per_page: int = 10, category: Optional[str] = None, search: Optional[str] = None, include_inactive: bool = False
+    ) -> PaginatedSources:
+        """Get sources with pagination and filtering."""
+        try:
+            query_parts = ["SELECT s.id, s.name, s.url, s.description, s.is_active, s.created_at", "FROM sources s", "WHERE 1=1"]
+            query_params = []
+            if not include_inactive:
+                query_parts.append("AND s.is_active = 1")
+            if category:
+                query_parts.append("""
+                    AND EXISTS (
+                        SELECT 1 FROM source_categories sc 
+                        JOIN categories c ON sc.category_id = c.id
+                        WHERE sc.source_id = s.id AND c.name = ?
+                    )
+                """)
+                query_params.append(category)
+            if search:
+                query_parts.append("AND (s.name LIKE ? OR s.description LIKE ?)")
+                search_param = f"%{search}%"
+                query_params.extend([search_param, search_param])
+            count_query = " ".join(query_parts).replace("SELECT s.id, s.name, s.url, s.description, s.is_active, s.created_at", "SELECT COUNT(*)")
+            total_sources = await sources_db.execute_query(count_query, tuple(query_params), fetch=True, fetch_one=True)
+            total_count = total_sources.get("COUNT(*)", 0) if total_sources else 0
+            query_parts.append("ORDER BY s.name")
+            offset = (page - 1) * per_page
+            query_parts.append("LIMIT ? OFFSET ?")
+            query_params.extend([per_page, offset])
+            final_query = " ".join(query_parts)
+            sources = await sources_db.execute_query(final_query, tuple(query_params), fetch=True)
+            for source in sources:
+                source["categories"] = await self.get_source_categories(source["id"])
+                source["last_crawled"] = await self.get_source_last_crawled(source["id"])
+                source["website"] = source["url"]
+                if source["categories"] and isinstance(source["categories"], list):
+                    source["category"] = source["categories"][0] if source["categories"] else ""
+            total_pages = (total_count + per_page - 1) // per_page if total_count > 0 else 0
+            has_next = page < total_pages
+            has_prev = page > 1
+            return PaginatedSources(
+                items=sources, total=total_count, page=page, per_page=per_page, total_pages=total_pages, has_next=has_next, has_prev=has_prev
+            )
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching sources: {str(e)}")
+
+    async def get_source(self, source_id: int) -> Dict[str, Any]:
+        """Get a specific source by ID."""
+        query = """
+        SELECT id, name, url, description, is_active, created_at
+        FROM sources
+        WHERE id = ?
+        """
+        source = await sources_db.execute_query(query, (source_id,), fetch=True, fetch_one=True)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+        source["categories"] = await self.get_source_categories(source["id"])
+        source["last_crawled"] = await self.get_source_last_crawled(source["id"])
+        source["website"] = source["url"]
+        if source["categories"] and isinstance(source["categories"], list):
+            source["category"] = source["categories"][0] if source["categories"] else ""
+        return source
+
+    async def get_source_categories(self, source_id: int) -> List[str]:
+        """Get all categories for a specific source."""
+        query = """
+        SELECT c.name
+        FROM source_categories sc
+        JOIN categories c ON sc.category_id = c.id
+        WHERE sc.source_id = ?
+        ORDER BY c.name
+        """
+        categories = await sources_db.execute_query(query, (source_id,), fetch=True)
+        return [category.get("name", "") for category in categories if category.get("name")]
+
+    async def get_source_last_crawled(self, source_id: int) -> Optional[str]:
+        """Get the last crawl time for a source's feeds."""
+        query = """
+        SELECT MAX(ft.last_processed) as last_crawled
+        FROM feed_tracking ft
+        WHERE ft.source_id = ?
+        """
+        result = await tracking_db.execute_query(query, (source_id,), fetch=True, fetch_one=True)
+        return result.get("last_crawled") if result else None
+
+    async def get_source_by_name(self, name: str) -> Dict[str, Any]:
+        """Get a specific source by name."""
+        query = """
+        SELECT id, name, url, description, is_active, created_at
+        FROM sources
+        WHERE name = ?
+        """
+        source = await sources_db.execute_query(query, (name,), fetch=True, fetch_one=True)
+        if not source:
+            raise HTTPException(status_code=404, detail="Source not found")
+        source["categories"] = await self.get_source_categories(source["id"])
+        source["last_crawled"] = await self.get_source_last_crawled(source["id"])
+        source["website"] = source["url"]
+        if source["categories"] and isinstance(source["categories"], list):
+            source["category"] = source["categories"][0] if source["categories"] else ""
+        return source
+
+    async def get_source_feeds(self, source_id: int) -> List[Dict[str, Any]]:
+        """Get all feeds for a specific source."""
+        query = """
+        SELECT id, feed_url, feed_type, is_active, created_at, last_crawled
+        FROM source_feeds
+        WHERE source_id = ?
+        ORDER BY feed_type
+        """
+        feeds = await sources_db.execute_query(query, (source_id,), fetch=True)
+        for feed in feeds:
+            feed["description"] = feed.get("feed_type", "Main feed").capitalize()
+        return feeds
+
+    async def get_categories(self) -> List[Dict[str, Any]]:
+        """Get all source categories."""
+        query = """
+        SELECT id, name
+        FROM categories
+        ORDER BY name
+        """
+        categories = await sources_db.execute_query(query, fetch=True)
+        for category in categories:
+            category["description"] = f"Articles about {category.get('name', '')}"
+        return categories
+
+    async def get_source_by_category(self, category_name: str) -> List[Dict[str, Any]]:
+        """Get sources by category using the junction table."""
+        query = """
+        SELECT s.id, s.name, s.url, s.description, s.is_active
+        FROM sources s
+        JOIN source_categories sc ON s.id = sc.source_id
+        JOIN categories c ON sc.category_id = c.id
+        WHERE c.name = ? AND s.is_active = 1
+        ORDER BY s.name
+        """
+        sources = await sources_db.execute_query(query, (category_name,), fetch=True)
+        for source in sources:
+            source["categories"] = await self.get_source_categories(source["id"])
+            source["website"] = source["url"]
+            if source["categories"] and isinstance(source["categories"], list):
+                source["category"] = source["categories"][0] if source["categories"] else ""
+        return sources
+
+    async def create_source(self, source_data: SourceCreate) -> Dict[str, Any]:
+        """Create a new source."""
+        try:
+            source_query = """
+            INSERT INTO sources (name, url, description, is_active, created_at)
+            VALUES (?, ?, ?, ?, ?)
+            """
+            source_params = (source_data.name, source_data.url, source_data.description, source_data.is_active, datetime.now().isoformat())
+            source_id = await sources_db.execute_query(source_query, source_params)
+            if source_data.categories:
+                for category_name in source_data.categories:
+                    await self.add_source_category(source_id, category_name)
+            elif hasattr(source_data, "category") and source_data.category:
+                await self.add_source_category(source_id, source_data.category)
+            if source_data.feeds:
+                for feed in source_data.feeds:
+                    await self.add_feed_to_source(source_id, feed)
+            return await self.get_source(source_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            if "UNIQUE constraint failed" in str(e) and "name" in str(e):
+                raise HTTPException(status_code=409, detail="Source with this name already exists")
+            raise HTTPException(status_code=500, detail=f"Error creating source: {str(e)}")
+
+    async def add_source_category(self, source_id: int, category_name: str) -> None:
+        """Add a category to a source, creating the category if it doesn't exist."""
+        category_query = """
+        INSERT OR IGNORE INTO categories (name, created_at)
+        VALUES (?, ?)
+        """
+        await sources_db.execute_query(category_query, (category_name, datetime.now().isoformat()))
+        get_category_id_query = "SELECT id FROM categories WHERE name = ?"
+        category = await sources_db.execute_query(get_category_id_query, (category_name,), fetch=True, fetch_one=True)
+        if not category:
+            raise HTTPException(status_code=500, detail=f"Failed to find or create category: {category_name}")
+        link_query = """
+        INSERT OR IGNORE INTO source_categories (source_id, category_id)
+        VALUES (?, ?)
+        """
+        await sources_db.execute_query(link_query, (source_id, category["id"]))
+
+    async def update_source(self, source_id: int, source_data: SourceUpdate) -> Dict[str, Any]:
+        """Update an existing source."""
+        try:
+            await self.get_source(source_id)
+            update_fields = []
+            update_params = []
+            if source_data.name is not None:
+                update_fields.append("name = ?")
+                update_params.append(source_data.name)
+            if source_data.url is not None:
+                update_fields.append("url = ?")
+                update_params.append(source_data.url)
+            if source_data.description is not None:
+                update_fields.append("description = ?")
+                update_params.append(source_data.description)
+            if source_data.is_active is not None:
+                update_fields.append("is_active = ?")
+                update_params.append(source_data.is_active)
+            if update_fields:
+                update_params.append(source_id)
+                update_query = f"""
+                UPDATE sources
+                SET {", ".join(update_fields)}
+                WHERE id = ?
+                """
+                await sources_db.execute_query(update_query, tuple(update_params))
+            if source_data.categories is not None:
+                delete_categories_query = "DELETE FROM source_categories WHERE source_id = ?"
+                await sources_db.execute_query(delete_categories_query, (source_id,))
+                if source_data.categories:
+                    for category_name in source_data.categories:
+                        await self.add_source_category(source_id, category_name)
+            elif hasattr(source_data, "category") and source_data.category is not None:
+                delete_categories_query = "DELETE FROM source_categories WHERE source_id = ?"
+                await sources_db.execute_query(delete_categories_query, (source_id,))
+                if source_data.category:
+                    await self.add_source_category(source_id, source_data.category)
+            return await self.get_source(source_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            if "UNIQUE constraint failed" in str(e) and "name" in str(e):
+                raise HTTPException(status_code=409, detail="Source with this name already exists")
+            raise HTTPException(status_code=500, detail=f"Error updating source: {str(e)}")
+
+    async def delete_source(self, source_id: int) -> Dict[str, Any]:
+        """Delete a source (soft delete by setting is_active to false)."""
+        try:
+            source = await self.get_source(source_id)
+            update_query = """
+            UPDATE sources
+            SET is_active = 0
+            WHERE id = ?
+            """
+            await sources_db.execute_query(update_query, (source_id,))
+            feeds_query = """
+            UPDATE source_feeds
+            SET is_active = 0
+            WHERE source_id = ?
+            """
+            await sources_db.execute_query(feeds_query, (source_id,))
+            return {**source, "is_active": False}
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error deleting source: {str(e)}")
+
+    async def hard_delete_source(self, source_id: int) -> Dict[str, str]:
+        """Permanently delete a source and its feeds."""
+        try:
+            source = await self.get_source(source_id)
+            delete_feeds_query = """
+            DELETE FROM source_feeds
+            WHERE source_id = ?
+            """
+            await sources_db.execute_query(delete_feeds_query, (source_id,))
+            delete_categories_query = """
+            DELETE FROM source_categories
+            WHERE source_id = ?
+            """
+            await sources_db.execute_query(delete_categories_query, (source_id,))
+            delete_source_query = """
+            DELETE FROM sources
+            WHERE id = ?
+            """
+            await sources_db.execute_query(delete_source_query, (source_id,))
+            return {"message": f"Source '{source['name']}' has been permanently deleted"}
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error deleting source: {str(e)}")
+
+    async def add_feed_to_source(self, source_id: int, feed_data: SourceFeedCreate) -> Dict[str, Any]:
+        """Add a new feed to an existing source."""
+        try:
+            await self.get_source(source_id)
+            check_query = """
+            SELECT id, source_id FROM source_feeds WHERE feed_url = ?
+            """
+            existing_feed = await sources_db.execute_query(check_query, (feed_data.feed_url,), fetch=True, fetch_one=True)
+            if existing_feed:
+                source_query = """
+                SELECT name FROM sources WHERE id = ?
+                """
+                source = await sources_db.execute_query(source_query, (existing_feed["source_id"],), fetch=True, fetch_one=True)
+                source_name = source["name"] if source else "another source"
+                raise HTTPException(
+                    status_code=409,
+                    detail=f"A feed with this URL already exists for {source_name}. Please edit the existing feed (ID: {existing_feed['id']}) instead.",
+                )
+            feed_query = """
+            INSERT INTO source_feeds (source_id, feed_url, feed_type, is_active, created_at)
+            VALUES (?, ?, ?, ?, ?)
+            """
+            feed_params = (source_id, feed_data.feed_url, feed_data.feed_type, feed_data.is_active, datetime.now().isoformat())
+            await sources_db.execute_query(feed_query, feed_params)
+            return await self.get_source_feeds(source_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            if "UNIQUE constraint failed" in str(e) and "feed_url" in str(e):
+                raise HTTPException(status_code=409, detail="This feed URL already exists. Please check your existing feeds or try a different URL.")
+            raise HTTPException(status_code=500, detail=f"Error adding feed: {str(e)}")
+
+    async def update_feed(self, feed_id: int, feed_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Update an existing feed."""
+        try:
+            feed_query = "SELECT id, source_id FROM source_feeds WHERE id = ?"
+            feed = await sources_db.execute_query(feed_query, (feed_id,), fetch=True, fetch_one=True)
+            if not feed:
+                raise HTTPException(status_code=404, detail="Feed not found")
+            update_fields = []
+            update_params = []
+            if "feed_url" in feed_data:
+                update_fields.append("feed_url = ?")
+                update_params.append(feed_data["feed_url"])
+            if "feed_type" in feed_data:
+                update_fields.append("feed_type = ?")
+                update_params.append(feed_data["feed_type"])
+            if "is_active" in feed_data:
+                update_fields.append("is_active = ?")
+                update_params.append(feed_data["is_active"])
+            if not update_fields:
+                return await self.get_source_feeds(feed["source_id"])
+            update_params.append(feed_id)
+            update_query = f"""
+            UPDATE source_feeds
+            SET {", ".join(update_fields)}
+            WHERE id = ?
+            """
+            await sources_db.execute_query(update_query, tuple(update_params))
+            return await self.get_source_feeds(feed["source_id"])
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            if "UNIQUE constraint failed" in str(e) and "feed_url" in str(e):
+                raise HTTPException(status_code=409, detail="Feed URL already exists")
+            raise HTTPException(status_code=500, detail=f"Error updating feed: {str(e)}")
+
+    async def delete_feed(self, feed_id: int) -> Dict[str, str]:
+        """Delete a feed."""
+        try:
+            feed_query = "SELECT * FROM source_feeds WHERE id = ?"
+            feed = await sources_db.execute_query(feed_query, (feed_id,), fetch=True, fetch_one=True)
+            if not feed:
+                raise HTTPException(status_code=404, detail="Feed not found")
+            delete_query = "DELETE FROM source_feeds WHERE id = ?"
+            await sources_db.execute_query(delete_query, (feed_id,))
+            return {"message": "Feed has been deleted"}
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error deleting feed: {str(e)}")
+
+
+source_service = SourceService()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/services/task_service.py

@@ -0,0 +1,315 @@
+from typing import List, Optional, Dict, Any
+from datetime import datetime, timedelta
+from fastapi import HTTPException
+from services.db_service import tasks_db
+from models.tasks_schemas import TASK_TYPES
+
+
+class TaskService:
+    """Service for managing scheduled tasks."""
+
+    async def get_tasks(self, include_disabled: bool = False) -> List[Dict[str, Any]]:
+        """Get all tasks with optional filtering."""
+        try:
+            if include_disabled:
+                query = """
+                SELECT id, name, description, command, task_type, frequency, frequency_unit, 
+                       enabled, last_run, created_at
+                FROM tasks
+                ORDER BY name
+                """
+                params = ()
+            else:
+                query = """
+                SELECT id, name, description, command, task_type, frequency, frequency_unit, 
+                       enabled, last_run, created_at
+                FROM tasks
+                WHERE enabled = 1
+                ORDER BY name
+                """
+                params = ()
+            tasks = await tasks_db.execute_query(query, params, fetch=True)
+            for task in tasks:
+                task["enabled"] = bool(task.get("enabled", 0))
+            return tasks
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching tasks: {str(e)}")
+
+    async def get_task(self, task_id: int) -> Dict[str, Any]:
+        """Get a specific task by ID."""
+        try:
+            query = """
+            SELECT id, name, description, command, task_type, frequency, frequency_unit, 
+                   enabled, last_run, created_at
+            FROM tasks
+            WHERE id = ?
+            """
+            task = await tasks_db.execute_query(query, (task_id,), fetch=True, fetch_one=True)
+            if not task:
+                raise HTTPException(status_code=404, detail="Task not found")
+            task["enabled"] = bool(task.get("enabled", 0))
+            return task
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching task: {str(e)}")
+
+    async def check_task_exists(self, task_type: str) -> Optional[Dict[str, Any]]:
+        """Check if a task with the given type already exists."""
+        try:
+            query = """
+            SELECT id, name, task_type
+            FROM tasks
+            WHERE task_type = ?
+            LIMIT 1
+            """
+            task = await tasks_db.execute_query(query, (task_type,), fetch=True, fetch_one=True)
+            return task
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error checking task existence: {str(e)}")
+
+    async def create_task(
+        self,
+        name: str,
+        task_type: str,
+        frequency: int,
+        frequency_unit: str,
+        description: Optional[str] = None,
+        enabled: bool = True,
+    ) -> Dict[str, Any]:
+        """Create a new task."""
+        try:
+            existing_task = await self.check_task_exists(task_type)
+            if existing_task:
+                raise HTTPException(
+                    status_code=409,
+                    detail=f"A task with type '{task_type}' already exists (Task: '{existing_task['name']}', ID: {existing_task['id']}). Please edit the existing task instead of creating a duplicate.",
+                )
+            if task_type not in TASK_TYPES:
+                raise HTTPException(
+                    status_code=400, detail=f"Invalid task type: '{task_type}'. Please select a valid task type from the available options."
+                )
+            command = TASK_TYPES[task_type]["command"]
+            current_time = datetime.now().isoformat()
+            query = """
+            INSERT INTO tasks 
+            (name, description, command, task_type, frequency, frequency_unit, enabled, created_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """
+            params = (
+                name,
+                description,
+                command,
+                task_type,
+                frequency,
+                frequency_unit,
+                1 if enabled else 0,
+                current_time,
+            )
+            task_id = await tasks_db.execute_query(query, params)
+            return await self.get_task(task_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error creating task: {str(e)}")
+
+    async def update_task(self, task_id: int, updates: Dict[str, Any]) -> Dict[str, Any]:
+        """Update an existing task."""
+        try:
+            current_task = await self.get_task(task_id)
+            if "task_type" in updates and updates["task_type"] != current_task["task_type"]:
+                existing_task = await self.check_task_exists(updates["task_type"])
+                if existing_task and existing_task["id"] != task_id:
+                    raise HTTPException(
+                        status_code=409,
+                        detail=f"A task with type '{updates['task_type']}' already exists (Task: '{existing_task['name']}', ID: {existing_task['id']}). You cannot have duplicate task types in the system.",
+                    )
+                if updates["task_type"] in TASK_TYPES:
+                    updates["command"] = TASK_TYPES[updates["task_type"]]["command"]
+            allowed_fields = [
+                "name",
+                "description",
+                "command",
+                "task_type",
+                "frequency",
+                "frequency_unit",
+                "enabled",
+            ]
+            set_clauses = []
+            params = []
+            for field, value in updates.items():
+                if field in allowed_fields:
+                    if field == "enabled":
+                        value = 1 if value else 0
+                    set_clauses.append(f"{field} = ?")
+                    params.append(value)
+            if not set_clauses:
+                return await self.get_task(task_id)
+            params.append(task_id)
+            update_query = f"""
+            UPDATE tasks
+            SET {", ".join(set_clauses)}
+            WHERE id = ?
+            """
+            await tasks_db.execute_query(update_query, tuple(params))
+            return await self.get_task(task_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error updating task: {str(e)}")
+
+    async def delete_task(self, task_id: int) -> Dict[str, str]:
+        """Delete a task."""
+        try:
+            task = await self.get_task(task_id)
+            query = """
+            DELETE FROM tasks
+            WHERE id = ?
+            """
+            await tasks_db.execute_query(query, (task_id,))
+            return {"message": f"Task '{task['name']}' has been deleted"}
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error deleting task: {str(e)}")
+
+    async def toggle_task(self, task_id: int, enable: bool) -> Dict[str, Any]:
+        """Enable or disable a task."""
+        try:
+            query = """
+            UPDATE tasks
+            SET enabled = ?
+            WHERE id = ?
+            """
+            await tasks_db.execute_query(query, (1 if enable else 0, task_id))
+            return await self.get_task(task_id)
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error updating task: {str(e)}")
+
+    async def get_task_executions(self, task_id: Optional[int] = None, page: int = 1, per_page: int = 10) -> Dict[str, Any]:
+        """Get paginated task executions."""
+        try:
+            offset = (page - 1) * per_page
+            if task_id:
+                count_query = """
+                SELECT COUNT(*) as count
+                FROM task_executions
+                WHERE task_id = ?
+                """
+                count_params = (task_id,)
+                query = """
+                SELECT id, task_id, start_time, end_time, status, error_message, output
+                FROM task_executions
+                WHERE task_id = ?
+                ORDER BY start_time DESC
+                LIMIT ? OFFSET ?
+                """
+                params = (task_id, per_page, offset)
+            else:
+                count_query = """
+                SELECT COUNT(*) as count
+                FROM task_executions
+                """
+                count_params = ()
+                query = """
+                SELECT id, task_id, start_time, end_time, status, error_message, output
+                FROM task_executions
+                ORDER BY start_time DESC
+                LIMIT ? OFFSET ?
+                """
+                params = (per_page, offset)
+            count_result = await tasks_db.execute_query(count_query, count_params, fetch=True, fetch_one=True)
+            total_items = count_result.get("count", 0) if count_result else 0
+            executions = await tasks_db.execute_query(query, params, fetch=True)
+            for execution in executions:
+                if execution.get("task_id"):
+                    try:
+                        task = await self.get_task(execution["task_id"])
+                        execution["task_name"] = task.get("name", "Unknown Task")
+                    except Exception as _:
+                        execution["task_name"] = "Unknown Task"
+            total_pages = (total_items + per_page - 1) // per_page if total_items > 0 else 0
+            has_next = page < total_pages
+            has_prev = page > 1
+            return {
+                "items": executions,
+                "total": total_items,
+                "page": page,
+                "per_page": per_page,
+                "total_pages": total_pages,
+                "has_next": has_next,
+                "has_prev": has_prev,
+            }
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching task executions: {str(e)}")
+
+    async def get_pending_tasks(self) -> List[Dict[str, Any]]:
+        """Get tasks that are due to run."""
+        try:
+            query = """
+            SELECT id, name, description, command, task_type, frequency, frequency_unit, enabled, last_run
+            FROM tasks
+            WHERE enabled = 1
+            AND (
+                last_run IS NULL 
+                OR 
+                CASE frequency_unit
+                    WHEN 'minutes' THEN datetime(last_run, '+' || frequency || ' minutes') <= datetime('now', 'localtime')
+                    WHEN 'hours' THEN datetime(last_run, '+' || frequency || ' hours') <= datetime('now', 'localtime')
+                    WHEN 'days' THEN datetime(last_run, '+' || frequency || ' days') <= datetime('now', 'localtime')
+                    ELSE datetime(last_run, '+' || frequency || ' seconds') <= datetime('now', 'localtime')
+                END
+            )
+            ORDER BY last_run
+            """
+            tasks = await tasks_db.execute_query(query, fetch=True)
+            for task in tasks:
+                task["enabled"] = bool(task.get("enabled", 0))
+            return tasks
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching pending tasks: {str(e)}")
+
+    async def get_stats(self) -> Dict[str, Any]:
+        """Get task statistics."""
+        try:
+            task_query = """
+            SELECT 
+                COUNT(*) as total_tasks,
+                SUM(CASE WHEN enabled = 1 THEN 1 ELSE 0 END) as active_tasks,
+                SUM(CASE WHEN enabled = 0 THEN 1 ELSE 0 END) as disabled_tasks,
+                SUM(CASE WHEN last_run IS NULL THEN 1 ELSE 0 END) as never_run_tasks
+            FROM tasks
+            """
+            task_stats = await tasks_db.execute_query(task_query, fetch=True, fetch_one=True)
+            cutoff_date = (datetime.now() - timedelta(days=7)).isoformat()
+            exec_query = """
+            SELECT 
+                COUNT(*) as total_executions,
+                COALESCE(SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END), 0) as successful_executions,
+                COALESCE(SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END), 0) as failed_executions,
+                COALESCE(SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END), 0) as running_executions,
+                COALESCE(AVG(CASE WHEN end_time IS NOT NULL 
+                    THEN (julianday(end_time) - julianday(start_time)) * 86400.0 
+                    ELSE NULL END), 0) as avg_execution_time_seconds
+            FROM task_executions
+            WHERE start_time >= ?
+            """
+            exec_stats = await tasks_db.execute_query(exec_query, (cutoff_date,), fetch=True, fetch_one=True)
+            return {"tasks": task_stats or {}, "executions": exec_stats or {}}
+        except Exception as e:
+            if isinstance(e, HTTPException):
+                raise e
+            raise HTTPException(status_code=500, detail=f"Error fetching task statistics: {str(e)}")
+
+
+task_service = TaskService()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tests/agent_agno_test.py

@@ -0,0 +1,12 @@
+from typing import Iterator
+from agno.agent import Agent, RunResponse
+from agno.models.openai import OpenAIChat
+from agno.utils.pprint import pprint_run_response
+from dotenv import load_dotenv
+
+load_dotenv()
+agent = Agent(model=OpenAIChat(id="gpt-4o-mini"))
+response: RunResponse = agent.run("Tell me a 5 second short story about a robot")
+response_stream: Iterator[RunResponse] = agent.run("Tell me a 5 second short story about a lion", stream=True)
+pprint_run_response(response, markdown=True)
+pprint_run_response(response_stream, markdown=True)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tests/embedding_search_test.py

@@ -0,0 +1,194 @@
+import argparse
+import os
+import numpy as np
+import faiss
+from openai import OpenAI
+from db.config import get_tracking_db_path
+from db.connection import execute_query
+from utils.load_api_keys import load_api_key
+from db.config import get_faiss_db_path
+
+EMBEDDING_MODEL = "text-embedding-3-small"
+FAISS_INDEX_PATH, FAIS_MAPPING_PATH = get_faiss_db_path()
+
+
+def generate_query_embedding(client, query_text, model=EMBEDDING_MODEL):
+    try:
+        response = client.embeddings.create(input=query_text, model=model)
+        return response.data[0].embedding, model
+    except Exception as e:
+        print(f"Error generating query embedding: {str(e)}")
+        return None, None
+
+
+def load_faiss_index(index_path=FAISS_INDEX_PATH):
+    if not os.path.exists(index_path):
+        raise FileNotFoundError(f"FAISS index not found at {index_path}")
+    return faiss.read_index(index_path)
+
+
+def load_id_mapping(mapping_path=FAIS_MAPPING_PATH):
+    if not os.path.exists(mapping_path):
+        raise FileNotFoundError(f"ID mapping not found at {mapping_path}")
+    return np.load(mapping_path).tolist()
+
+
+def get_article_details(tracking_db_path, article_ids):
+    if not article_ids:
+        return []
+    placeholders = ",".join(["?"] * len(article_ids))
+    query = f"""
+    SELECT id, title, url, published_date, summary 
+    FROM crawled_articles
+    WHERE id IN ({placeholders})
+    """
+    return execute_query(tracking_db_path, query, article_ids, fetch=True)
+
+
+def search_articles(
+    query_text,
+    tracking_db_path=None,
+    openai_api_key=None,
+    index_path="databases/faiss/article_index.faiss",
+    mapping_path="databases/faiss/article_id_map.npy",
+    top_k=5,
+    search_params=None,
+):
+    if tracking_db_path is None:
+        tracking_db_path = get_tracking_db_path()
+    if openai_api_key is None:
+        openai_api_key = load_api_key()
+        if not openai_api_key:
+            raise ValueError("OpenAI API key is required")
+    client = OpenAI(api_key=openai_api_key)
+    query_embedding, _ = generate_query_embedding(client, query_text)
+    if not query_embedding:
+        raise ValueError("Failed to generate query embedding")
+    query_vector = np.array([query_embedding]).astype(np.float32)
+    try:
+        faiss_index = load_faiss_index(index_path)
+        id_map = load_id_mapping(mapping_path)
+        if search_params:
+            if isinstance(faiss_index, faiss.IndexIVF) and "nprobe" in search_params:
+                faiss_index.nprobe = search_params["nprobe"]
+                print(f"Set nprobe to {faiss_index.nprobe}")
+            if hasattr(faiss_index, "hnsw") and "ef" in search_params:
+                faiss_index.hnsw.efSearch = search_params["ef"]
+                print(f"Set efSearch to {faiss_index.hnsw.efSearch}")
+        index_type = "unknown"
+        if isinstance(faiss_index, faiss.IndexFlatL2):
+            index_type = "flat"
+        elif isinstance(faiss_index, faiss.IndexIVFFlat):
+            index_type = "ivfflat"
+            print(f"Using IVF index with nprobe = {faiss_index.nprobe}")
+        elif isinstance(faiss_index, faiss.IndexIVFPQ):
+            index_type = "ivfpq"
+            print(f"Using IVF-PQ index with nprobe = {faiss_index.nprobe}")
+        elif hasattr(faiss_index, "hnsw"):
+            index_type = "hnsw"
+            print(f"Using HNSW index with efSearch = {faiss_index.hnsw.efSearch}")
+        print(f"Searching {index_type} FAISS index with {len(id_map)} articles...")
+        distances, indices = faiss_index.search(query_vector, top_k)
+        result_article_ids = [id_map[idx] for idx in indices[0] if idx < len(id_map)]
+        results = get_article_details(tracking_db_path, result_article_ids)
+        for i, result in enumerate(results):
+            distance = float(distances[0][i])
+            similarity = float(np.exp(-distance))
+            result["distance"] = distance
+            result["similarity"] = similarity
+            result["score"] = similarity
+        return results
+    except Exception as e:
+        print(f"Error during search: {str(e)}")
+        import traceback
+
+        traceback.print_exc()
+        return []
+
+
+def print_search_results(results):
+    if not results:
+        print("No results found.")
+        return
+    print(f"\nFound {len(results)} results:\n")
+    for i, result in enumerate(results):
+        similarity_pct = result.get("similarity", 0) * 100
+        print(f"{i + 1}. {result['title']}")
+        print(f"   Relevance: {similarity_pct:.1f}%")
+        if "distance" in result:
+            print(f"   Vector distance: {result['distance']:.4f}")
+        print(f"   Published: {result['published_date']}")
+        print(f"   URL: {result['url']}")
+        if len(result["summary"]) > 150:
+            print(f"   Summary: {result['summary'][:150]}...")
+        else:
+            print(f"   Summary: {result['summary']}")
+        print()
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Search for articles using FAISS")
+    parser.add_argument(
+        "query",
+        help="Search query text",
+    )
+    parser.add_argument(
+        "--api_key",
+        help="OpenAI API Key (overrides environment variables)",
+    )
+    parser.add_argument(
+        "--top_k",
+        type=int,
+        default=5,
+        help="Number of results to return",
+    )
+    parser.add_argument(
+        "--nprobe",
+        type=int,
+        help="Number of clusters to search (for IVF indexes)",
+    )
+    parser.add_argument(
+        "--ef",
+        type=int,
+        help="Search depth (for HNSW indexes)",
+    )
+    parser.add_argument(
+        "--index_path",
+        default=FAISS_INDEX_PATH,
+        help="Path to the FAISS index file",
+    )
+    parser.add_argument(
+        "--mapping_path",
+        default=FAIS_MAPPING_PATH,
+        help="Path to the ID mapping file",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    api_key = args.api_key or load_api_key()
+    if not api_key:
+        print("Error: No OpenAI API key provided. Please provide via --api_key or set OPENAI_API_KEY in .env file")
+        exit(1)
+    search_params = {}
+    if args.nprobe:
+        search_params["nprobe"] = args.nprobe
+    if args.ef:
+        search_params["ef"] = args.ef
+    try:
+        results = search_articles(
+            query_text=args.query,
+            openai_api_key=api_key,
+            top_k=args.top_k,
+            index_path=args.index_path,
+            mapping_path=args.mapping_path,
+            search_params=search_params,
+        )
+        print_search_results(results)
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        import traceback
+
+        traceback.print_exc()
+        exit(1)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tests/index_faiss_test.py

@@ -0,0 +1,25 @@
+import numpy as np
+import faiss
+import time
+
+
+dimension = 128
+nb_vectors = 10000
+np.random.seed(42)
+database = np.random.random((nb_vectors, dimension)).astype("float32")
+query = np.random.random((1, dimension)).astype("float32")
+
+start_time = time.time()
+index = faiss.IndexFlatL2(dimension)
+index.add(database)
+print(f"Index built in {time.time() - start_time:.4f} seconds")
+print(f"Index contains {index.ntotal} vectors")
+
+k = 5
+start_time = time.time()
+distances, indices = index.search(query, k)
+print(f"Search completed in {time.time() - start_time:.4f} seconds")
+print("\nSearch Results:")
+print("Query vector finds these nearest neighbors (indices):", indices[0])
+print("With these distances:", distances[0])
+print("\nFAISS is working correctly!")

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tests/tool_browseruse_test.py

@@ -0,0 +1,20 @@
+from langchain_openai import ChatOpenAI
+from browser_use import Agent
+from dotenv import load_dotenv
+import asyncio
+
+
+load_dotenv()
+llm = ChatOpenAI(model="gpt-4o")
+
+
+async def main():
+    agent = Agent(
+        task="Compare the price of gpt-4o and DeepSeek-V3",
+        llm=llm,
+    )
+    result = await agent.run()
+    print(result)
+
+
+asyncio.run(main())

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tests/tts_kokoro_test.py

@@ -0,0 +1,60 @@
+import os
+import soundfile as sf
+import platform
+import time
+import warnings
+
+
+os.environ["PYTHONWARNINGS"] = "ignore"
+os.environ["TORCH_CPP_LOG_LEVEL"] = "ERROR"
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+warnings.filterwarnings("ignore")
+
+from kokoro import KPipeline
+
+
+def play_audio(file_path):
+    system = platform.system()
+    try:
+        if system == "Darwin":
+            os.system(f"afplay {file_path}")
+        elif system == "Linux":
+            os.system(f"aplay {file_path}")
+        elif system == "Windows":
+            os.system(f'start "" "{file_path}"')
+        else:
+            print(f"Audio saved to {file_path} (auto-play not supported on this system)")
+    except Exception as e:
+        print(f"Failed to auto-play audio: {e}")
+        print(f"Audio saved to {file_path}")
+
+
+print("Testing Kokoro TTS in English and Hindi...")
+
+print("\n=== Testing English ===")
+pipeline_en = KPipeline(lang_code="a")
+english_text = "This is a test of the Kokoro text-to-speech system in English."
+generator_en = pipeline_en(english_text, voice="af_heart")
+
+for i, (gs, ps, audio) in enumerate(generator_en):
+    output_file = f"english_sample_{i}.wav"
+    sf.write(output_file, audio, 24000)
+    print(f"Generated English audio: {output_file}")
+    print(f"Text: {gs}")
+    play_audio(output_file)
+    time.sleep(2)
+
+print("\n=== Testing Hindi ===")
+pipeline_hi = KPipeline(lang_code="h")
+hindi_text = "यह हिंदी में कोकोरो टेक्स्ट-टू-स्पीच सिस्टम का एक परीक्षण है।"
+generator_hi = pipeline_hi(hindi_text, voice="af_heart")
+
+for i, (gs, ps, audio) in enumerate(generator_hi):
+    output_file = f"hindi_sample_{i}.wav"
+    sf.write(output_file, audio, 24000)
+    print(f"Generated Hindi audio: {output_file}")
+    print(f"Text: {gs}")
+    play_audio(output_file)
+    time.sleep(2)
+
+print("\nTest completed. Audio files have been generated and should have auto-played.")

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/browser_crawler.py

@@ -0,0 +1,146 @@
+from playwright.sync_api import sync_playwright
+import newspaper
+import time
+from typing import Dict, List
+from datetime import datetime
+
+
+class PlaywrightScraper:
+    def __init__(
+        self,
+        headless: bool = True,
+        timeout: int = 20000,
+        fresh_context_per_url: bool = False,
+    ):
+        self.headless = headless
+        self.timeout = timeout
+        self.fresh_context_per_url = fresh_context_per_url
+
+    def scrape_urls(self, urls: List[str]) -> List[Dict]:
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(
+                headless=self.headless,
+                args=["--no-sandbox", "--disable-setuid-sandbox"],
+            )
+            if self.fresh_context_per_url:
+                results = []
+                for url in urls:
+                    result = self._scrape_single_with_new_context(browser, url)
+                    results.append(result)
+            else:
+                results = self._scrape_with_reused_page(browser, urls)
+            browser.close()
+            return results
+
+    def _scrape_with_reused_page(self, browser, urls: List[str]) -> List[Dict]:
+        context = browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            viewport={"width": 1920, "height": 1080},
+        )
+        page = context.new_page()
+        page.set_extra_http_headers(
+            {
+                "Accept-Language": "en-US,en;q=0.9",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            }
+        )
+        results = []
+        try:
+            for i, url in enumerate(urls):
+                print(f"Scraping {i+1}/{len(urls)}")
+                result = self._scrape_single_url(page, url)
+                results.append(result)
+                if i < len(urls) - 1:
+                    time.sleep(1)
+        finally:
+            context.close()
+        return results
+
+    def _scrape_single_url(self, page, url: str) -> Dict:
+        max_retries = 0
+        for attempt in range(max_retries + 1):
+            try:
+                page.goto(url, wait_until="load", timeout=self.timeout)
+                page.wait_for_selector("body", timeout=5000)
+                page.wait_for_timeout(2000)
+                final_url = page.url
+                return self._parse_with_newspaper(url, final_url)
+            except Exception as e:
+                if attempt < max_retries:
+                    print(f"Retry {attempt + 1} for {url}")
+                    time.sleep(2**attempt)
+                    continue
+                else:
+                    return {
+                        "originalUrl": url,
+                        "error": str(e),
+                        "success": False,
+                        "timestamp": datetime.now().isoformat(),
+                    }
+
+    def _scrape_single_with_new_context(self, browser, url: str) -> Dict:
+        max_retries = 0
+        for attempt in range(max_retries + 1):
+            context = None
+            try:
+                context = browser.new_context(
+                    user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                    viewport={"width": 1920, "height": 1080},
+                )
+                page = context.new_page()
+                page.set_extra_http_headers(
+                    {
+                        "Accept-Language": "en-US,en;q=0.9",
+                        "Accept-Encoding": "gzip, deflate, br",
+                        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+                    }
+                )
+                page.goto(url, wait_until="load", timeout=self.timeout)
+                page.wait_for_selector("body", timeout=5000)
+                page.wait_for_timeout(2000)
+                final_url = page.url
+                return self._parse_with_newspaper(url, final_url)
+            except Exception as e:
+                if attempt < max_retries:
+                    time.sleep(2**attempt)
+                    continue
+                else:
+                    return {
+                        "originalUrl": url,
+                        "error": str(e),
+                        "success": False,
+                        "timestamp": datetime.now().isoformat(),
+                    }
+            finally:
+                if context:
+                    context.close()
+
+    def _parse_with_newspaper(self, original_url: str, final_url: str) -> Dict:
+        try:
+            article = newspaper.article(final_url)
+            return {
+                "original_url": original_url,
+                "final_url": final_url,
+                "title": article.title or "",
+                "authors": article.authors or [],
+                "published_date": article.publish_date.isoformat() if article.publish_date else None,
+                "full_text": article.text or "",
+                "success": True,
+            }
+        except Exception as e:
+            return {
+                "original_url": original_url,
+                "final_url": final_url,
+                "error": f"Newspaper4k parsing failed: {str(e)}",
+                "success": False,
+            }
+
+
+def create_browser_crawler(headless=True, timeout=20000, fresh_context_per_url=False):
+    """Factory function to create a new PlaywrightScraper instance."""
+    return PlaywrightScraper(
+        headless=headless,
+        timeout=timeout,
+        fresh_context_per_url=fresh_context_per_url
+    )

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/embedding_search.py

@@ -0,0 +1,153 @@
+from agno.agent import Agent
+import os
+import numpy as np
+import faiss
+from openai import OpenAI
+from db.config import get_tracking_db_path, get_faiss_db_path, get_sources_db_path
+from db.connection import execute_query
+from utils.load_api_keys import load_api_key
+import traceback
+import json
+
+EMBEDDING_MODEL = "text-embedding-3-small"
+
+
+def generate_query_embedding(query_text, model=EMBEDDING_MODEL):
+    try:
+        api_key = load_api_key("OPENAI_API_KEY")
+        if not api_key:
+            return None, "OpenAI API key not found"
+        client = OpenAI(api_key=api_key)
+        response = client.embeddings.create(input=query_text, model=model)
+        return response.data[0].embedding, None
+    except Exception as e:
+        return None, str(e)
+
+
+def load_faiss_index(index_path):
+    if not os.path.exists(index_path):
+        return None, f"FAISS index not found at {index_path}"
+    try:
+        return faiss.read_index(index_path), None
+    except Exception as e:
+        return None, f"Error loading FAISS index: {str(e)}"
+
+
+def load_id_mapping(mapping_path):
+    if not os.path.exists(mapping_path):
+        return None, f"ID mapping not found at {mapping_path}"
+    try:
+        return np.load(mapping_path).tolist(), None
+    except Exception as e:
+        return None, f"Error loading ID mapping: {str(e)}"
+
+
+def get_article_details(tracking_db_path, article_ids):
+    if not article_ids:
+        return []
+    placeholders = ",".join(["?"] * len(article_ids))
+    query = f"""
+    SELECT id, title, url, published_date, summary, source_id, feed_id, content
+    FROM crawled_articles
+    WHERE id IN ({placeholders})
+    """
+    return execute_query(tracking_db_path, query, article_ids, fetch=True)
+
+
+def get_source_names(source_ids):
+    if not source_ids:
+        return {}
+    unique_ids = list(set([src_id for src_id in source_ids if src_id]))
+    if not unique_ids:
+        return {}
+    try:
+        sources_db_path = get_sources_db_path()
+        check_query = """
+        SELECT name FROM sqlite_master 
+        WHERE type='table' AND name='sources'
+        """
+        table_exists = execute_query(sources_db_path, check_query, fetch=True)
+        if not table_exists:
+            return {}
+        placeholders = ",".join(["?"] * len(unique_ids))
+        query = f"""
+        SELECT id, name FROM sources
+        WHERE id IN ({placeholders})
+        """
+        results = execute_query(sources_db_path, query, unique_ids, fetch=True)
+        return {str(row["id"]): row["name"] for row in results} if results else {}
+    except Exception as _:
+        return {}
+
+
+def embedding_search(agent: Agent, prompt: str) -> str:
+    """
+    Perform a semantic search using embeddings to find articles related to the query on internal articles databse which are crawled from preselected user rss feeds.
+    This search uses vector representations to find semantically similar content,
+    filtering for only high-quality matches (similarity score ≥ 85%).
+
+    Args:
+        agent: The Agno agent instance
+        prompt: The search query
+
+    Returns:
+        Search results
+    """
+    print("Embedding Search Input:", prompt)
+    tracking_db_path = get_tracking_db_path()
+    index_path, mapping_path = get_faiss_db_path()
+    top_k = 20
+    similarity_threshold = 0.85
+    if not os.path.exists(index_path) or not os.path.exists(mapping_path):
+        return "Embedding search not available: index files not found. Continuing with other search methods."
+    query_embedding, error = generate_query_embedding(prompt)
+    if not query_embedding:
+        return f"Semantic search unavailable: {error}. Continuing with other search methods."
+    query_vector = np.array([query_embedding]).astype(np.float32)
+    try:
+        faiss_index, error = load_faiss_index(index_path)
+        if error:
+            return f"Semantic search unavailable: {error}. Continuing with other search methods."
+        id_map, error = load_id_mapping(mapping_path)
+        if error:
+            return f"Semantic search unavailable: {error}. Continuing with other search methods."
+        distances, indices = faiss_index.search(query_vector, top_k)
+        results_with_metrics = []
+        for i, idx in enumerate(indices[0]):
+            if idx >= 0 and idx < len(id_map):
+                distance = float(distances[0][i])
+                similarity = float(np.exp(-distance)) if distance > 0 else 0
+                if similarity >= similarity_threshold:
+                    article_id = id_map[idx]
+                    results_with_metrics.append((idx, distance, similarity, article_id))
+        results_with_metrics.sort(key=lambda x: x[2], reverse=True)
+        result_article_ids = [item[3] for item in results_with_metrics]
+        if not result_article_ids:
+            return "No high-quality semantic matches found (threshold: 85%). Continuing with other search methods."
+        results = get_article_details(tracking_db_path, result_article_ids)
+        source_ids = [result.get("source_id") for result in results if result.get("source_id")]
+        source_names = get_source_names(source_ids)
+        formatted_results = []
+        for i, result in enumerate(results):
+            article_id = result.get("id")
+            similarity = next((item[2] for item in results_with_metrics if item[3] == article_id), 0)
+            similarity_percent = int(similarity * 100)
+            source_id = str(result.get("source_id", "unknown"))
+            source_name = source_names.get(source_id, source_id)
+            formatted_result = {
+                "id": article_id,
+                "title": f"{result.get('title', 'Untitled')} (Relevance: {similarity_percent}%)",
+                "url": result.get("url", "#"),
+                "published_date": result.get("published_date"),
+                "description": result.get("summary", result.get("content", "")),
+                "source_id": source_id,
+                "source_name": source_name,
+                "similarity": similarity,
+                "categories": ["semantic"],
+                "is_scrapping_required": False,
+            }
+            formatted_results.append(formatted_result)
+        return f"Found {len(formatted_results)}, results: {json.dumps(formatted_results, indent=2)}"
+    except Exception as e:
+        traceback.print_exc()
+        return f"Error in semantic search: {str(e)}. Continuing with other search methods."

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/google_news_discovery.py

@@ -0,0 +1,53 @@
+
+def search_news(google_news, keyword):
+    resutls = google_news.get_news(keyword)
+    return resutls
+
+
+def get_top_news(google_news):
+    resutls = google_news.get_top_news()
+    return resutls
+
+
+def get_news_by_topic(google_news, topic):
+    resutls = google_news.get_news_by_topic(topic)
+    return resutls
+
+
+def google_news_discovery_run(
+    keyword: str = None,
+    max_results: int = 5,
+    top_news: bool = False,
+) -> str:
+    from gnews import GNews
+    import json
+    
+    """
+    This is a wrapper function for the google news.
+
+    Args:
+        keyword: The search query for specific news
+        top_news: Whether to get top news instead of keyword search (default: False)
+        max_results: The maximum number of results to return (default: 20)
+
+    Returns:
+        List of news results
+
+    Note:
+        Either set top_news=True for top headlines or provide a keyword for search.
+        If both are provided, top_news takes precedence.
+    """
+    print("Google News Discovery:", keyword)
+    google_news = GNews(
+        language=None,
+        country=None,
+        period=None,
+        max_results=max_results,
+        exclude_websites=[],
+    )
+    if top_news:
+        results = get_top_news(google_news)
+    if keyword:
+        results = search_news(google_news, keyword)
+    print('google news search found:', len(results))
+    return f"for all results is_scrapping_required: True, results: {json.dumps(results)}"

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/jikan_search.py

@@ -0,0 +1,199 @@
+from agno.agent import Agent
+import requests
+import time
+from typing import List, Dict, Any, Optional
+import html
+import json
+
+
+def jikan_search(agent: Agent, query: str) -> str:
+    """
+    Search for anime information using the Jikan API (MyAnimeList API).
+    This provides anime data, reviews, and recommendations to enhance podcast content.
+    
+    Jikan scrapes public MyAnimeList pages.
+    The service consists of two core parts
+    
+    Args:
+        agent: The agent instance
+        query: The search query
+
+    Returns:
+        Search results
+    """
+    print("Jikan Search:", query)
+
+    try:
+        formatted_query = query.replace(" ", "%20")
+        anime_results = _search_anime(formatted_query)
+        if not anime_results:
+            return "No relevant anime found for this topic. Continuing with other search methods."
+        results = []
+        for anime in anime_results[:5]:
+            anime_id = anime.get("mal_id")
+            if not anime_id:
+                continue
+            anime_details = _get_anime_details(anime_id)
+            if anime_details:
+                results.append(anime_details)
+            time.sleep(0.5)
+        if not results:
+            return "No detailed anime information could be retrieved. Continuing with other search methods."
+        return f"Found {len(results)} anime titles related to your topic. results {json.dumps(results, indent=2)}."
+    except Exception as e:
+        return f"Error in anime search: {str(e)}. Continuing with other search methods."
+
+
+def _search_anime(query: str) -> List[Dict[str, Any]]:
+    try:
+        search_url = f"https://api.jikan.moe/v4/anime?q={query}&sfw=true&order_by=popularity&sort=asc&limit=10"
+        response = requests.get(search_url)
+        if response.status_code == 429:
+            time.sleep(0.5)
+            response = requests.get(search_url)
+        if response.status_code != 200:
+            return []
+        data = response.json()
+        if "data" not in data:
+            return []
+        return data["data"]
+    except Exception as _:
+        return []
+
+
+def _get_anime_details(anime_id: int) -> Optional[Dict[str, Any]]:
+    try:
+        details_url = f"https://api.jikan.moe/v4/anime/{anime_id}/full"
+        details_response = requests.get(details_url)
+        if details_response.status_code == 429:
+            time.sleep(0.5)
+            details_response = requests.get(details_url)
+        if details_response.status_code != 200:
+            return None
+        details_data = details_response.json()
+        if "data" not in details_data:
+            return None
+        anime = details_data["data"]
+        return _format_anime_info(anime)
+    except Exception as _:
+        return None
+
+
+def _get_anime_recommendations(anime_id: int) -> List[Dict[str, Any]]:
+    try:
+        recs_url = f"https://api.jikan.moe/v4/anime/{anime_id}/recommendations"
+        recs_response = requests.get(recs_url)
+        if recs_response.status_code == 429:
+            time.sleep(0.5)
+            recs_response = requests.get(recs_url)
+        if recs_response.status_code != 200:
+            return []
+        recs_data = recs_response.json()
+        if "data" not in recs_data:
+            return []
+
+        recommendations = []
+        for rec in recs_data["data"][:5]:
+            if "entry" in rec:
+                title = rec["entry"].get("title", "")
+                if title:
+                    recommendations.append(title)
+        return recommendations
+    except Exception as _:
+        return []
+
+
+def _format_anime_info(anime: Dict[str, Any]) -> Dict[str, Any]:
+    try:
+        mal_id = anime.get("mal_id")
+        title = anime.get("title", "Unknown Anime")
+        title_english = anime.get("title_english")
+        if title_english and title_english != title:
+            title_display = f"{title} ({title_english})"
+        else:
+            title_display = title
+
+        url = anime.get("url", f"https://myanimelist.net/anime/{mal_id}")
+        synopsis = anime.get("synopsis", "No synopsis available.")
+        synopsis = html.unescape(synopsis)
+        episodes = anime.get("episodes", "Unknown")
+        status = anime.get("status", "Unknown")
+        aired_string = anime.get("aired", {}).get("string", "Unknown")
+        score = anime.get("score", "N/A")
+        scored_by = anime.get("scored_by", 0)
+        rank = anime.get("rank", "N/A")
+        popularity = anime.get("popularity", "N/A")
+        studios = []
+        for studio in anime.get("studios", []):
+            if "name" in studio:
+                studios.append(studio["name"])
+        studio_text = ", ".join(studios) if studios else "Unknown"
+        genres = []
+        for genre in anime.get("genres", []):
+            if "name" in genre:
+                genres.append(genre["name"])
+        genre_text = ", ".join(genres) if genres else "Unknown"
+        themes = []
+        for theme in anime.get("themes", []):
+            if "name" in theme:
+                themes.append(theme["name"])
+        demographics = []
+        for demo in anime.get("demographics", []):
+            if "name" in demo:
+                demographics.append(demo["name"])
+        content = f"Title: {title_display}\n"
+        content += f"Score: {score} (rated by {scored_by:,} users)\n"
+        content += f"Rank: {rank}, Popularity: {popularity}\n"
+        content += f"Episodes: {episodes}\n"
+        content += f"Status: {status}\n"
+        content += f"Aired: {aired_string}\n"
+        content += f"Studio: {studio_text}\n"
+        content += f"Genres: {genre_text}\n"
+        if themes:
+            content += f"Themes: {', '.join(themes)}\n"
+        if demographics:
+            content += f"Demographics: {', '.join(demographics)}\n"
+        content += f"\nSynopsis:\n{synopsis}\n"
+        if mal_id:
+            recommendations = _get_anime_recommendations(mal_id)
+            if recommendations:
+                content += f"\nSimilar Anime: {', '.join(recommendations)}\n"
+        summary = f"{title_display} - {genre_text} anime with {episodes} episodes. "
+        summary += f"Rating: {score}/10. "
+        if synopsis:
+            short_synopsis = synopsis[:150] + "..." if len(synopsis) > 150 else synopsis
+            summary += short_synopsis
+        categories = ["anime", "japanese animation", "entertainment"]
+        if genres:
+            categories.extend(genres[:5])
+        if themes:
+            categories.extend(themes[:2])
+        return {
+            "id": f"jikan_{mal_id}",
+            "title": f"{title_display} (Anime)",
+            "url": url,
+            "published_date": aired_string.split(" to ")[0] if " to " in aired_string else aired_string,
+            "description": content,
+            "source_id": "jikan",
+            "source_name": "MyAnimeList",
+            "categories": categories,
+            "is_scrapping_required": False,
+
+        }
+    except Exception as _:
+        return {
+            "id": f"jikan_{anime.get('mal_id', 'unknown')}",
+            "title": f"{anime.get('title', 'Unknown Anime')} (Anime)",
+            "url": anime.get("url", "https://myanimelist.net"),
+            "published_date": None,
+            "description": anime.get("synopsis", "No information available."),
+            "source_id": "jikan",
+            "source_name": "MyAnimeList",
+            "categories": ["anime", "japanese animation", "entertainment"],
+            "is_scrapping_required": False,
+        }
+        
+        
+if __name__ == "__main__":
+    print(jikan_search({}, "One Piece anime overview and details"))
+    

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/pipeline/image_generate_agent.py

@@ -0,0 +1,81 @@
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from agno.tools.dalle import DalleTools
+from textwrap import dedent
+import json
+from dotenv import load_dotenv
+import uuid
+from db.agent_config_v2 import PODCAST_IMG_DIR
+import os
+import requests
+from typing import Any
+
+
+load_dotenv()
+
+IMAGE_GENERATION_AGENT_DESCRIPTION = "You are an AI agent that can generate images using DALL-E."
+IMAGE_GENERATION_AGENT_INSTRUCTIONS = dedent("""
+                                             When the user asks you to create an image, use the `create_image` tool to create the image.
+                                             Create a modern, eye-catching podcast cover images that represents a podcast given podcast topic.
+                                             Create 3 images for the given podcast topic.
+
+                                            IMPORTANT INSTRUCTIONS:
+                                            - DO NOT include ANY text in the image
+                                            - DO NOT include any words, titles, or lettering
+                                            - Create a purely visual and symbolic representation
+                                            - Use imagery that represents the specific topics mentioned
+                                            - I like Studio Ghibli flavor if possible
+                                            - The image should work well as a podcast cover thumbnail
+                                            - Create a clean, professional design suitable for a podcast
+                                            - AGAIN, DO NOT INCLUDE ANY TEXT
+                                        """)
+
+
+def download_images(image_urls):
+    local_image_filenames = []
+    try:
+        if image_urls:
+            for image_url in image_urls:
+                unique_id = str(uuid.uuid4())
+                filename = f"podcast_banner_{unique_id}.png"
+                os.makedirs(PODCAST_IMG_DIR, exist_ok=True)
+                print(f"Downloading image: {filename}")
+                response = requests.get(image_url, timeout=30)
+                response.raise_for_status()
+                image_path = os.path.join(PODCAST_IMG_DIR, filename)
+                with open(image_path, "wb") as f:
+                    f.write(response.content)
+                local_image_filenames.append(filename)
+                print(f"Successfully downloaded: {filename}")
+    except requests.exceptions.RequestException as e:
+        print(f"Error downloading images (network): {e}")
+    except Exception as e:
+        print(f"Error downloading images: {e}")
+
+    return local_image_filenames
+
+
+def image_generation_agent_run(query: str, generated_script: Any) -> str:
+    session_id = str(uuid.uuid4())
+    try:
+        image_agent = Agent(
+            model=OpenAIChat(id="gpt-4o"),
+            tools=[DalleTools()],
+            description=IMAGE_GENERATION_AGENT_DESCRIPTION,
+            instructions=IMAGE_GENERATION_AGENT_INSTRUCTIONS,
+            markdown=True,
+            show_tool_calls=True,
+            session_id=session_id,
+        )
+        image_agent.run(f"query: {query},\n podcast script: {json.dumps(generated_script)}", session_id=session_id)
+        images = image_agent.get_images()
+        image_urls = []
+        if images and isinstance(images, list):
+            for image_response in images:
+                image_url = image_response.url
+                image_urls.append(image_url)
+        local_image_filenames = download_images(image_urls)
+        return {"banner_images": local_image_filenames, "banner_url": local_image_filenames[0] if local_image_filenames else None}
+    except Exception as e:
+        print(f"Error in Image Generation Agent: {e}")
+        return {}

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/pipeline/scrape_agent.py

@@ -0,0 +1,136 @@
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from pydantic import BaseModel, Field
+from dotenv import load_dotenv
+from tools.browser_crawler import create_browser_crawler
+from textwrap import dedent
+import uuid
+
+
+load_dotenv()
+
+
+class ScrapedContent(BaseModel):
+    url: str = Field(..., description="The URL of the search result")
+    description: str = Field(description="The description of the search result")
+    full_text: str = Field(
+        ...,
+        description="The full text of the given source URL, if not available or not applicable keep it empty",
+    )
+    published_date: str = Field(
+        ...,
+        description="The published date of the content in ISO format, if not available keep it empty",
+    )
+
+
+SCRAPE_AGENT_DESCRIPTION = "You are a helpful assistant that can scrape the URL for full content."
+SCRAPE_AGENT_INSTRUCTIONS = dedent("""
+    You are a content verification and formatting assistant.
+    
+    You will receive a batch of pre-scraped content from various URLs along with a search query.
+    Your job is to:
+    
+    1. VERIFY RELEVANCE: Ensure each piece of content is relevant to the given query
+    2. QUALITY CONTROL: Filter out low-quality, duplicate, or irrelevant content
+    3. FORMAT CONSISTENCY: Ensure all content follows a consistent format
+    4. LENGTH OPTIMIZATION: Keep content at reasonable length - not too long, not too short
+    5. CLEAN TEXT: Remove any formatting artifacts, ads, or navigation elements from scraped content
+    
+    For each piece of content, return:
+    - full_text: The cleaned, relevant text content (or empty if not relevant/low quality)
+    - published_date: The publication date in ISO format (or empty if not available)
+    
+    Note: Some content may be fallback descriptions (when scraping failed) - treat these appropriately and don't penalize them for being shorter.
+    
+    IMPORTANT: Focus on quality over quantity. It's better to return fewer high-quality, relevant pieces than many low-quality ones.
+    """)
+
+
+def crawl_urls_batch(search_results):
+    url_to_search_results = {}
+    unique_urls = []
+    for search_result in search_results:
+        if not search_result.get("url", False):
+            continue
+        if not search_result.get("is_scrapping_required", True):
+            continue
+        if not search_result.get('original_url'):
+            search_result['original_url'] = search_result['url']
+        url = search_result["url"]
+        if url not in url_to_search_results:
+            url_to_search_results[url] = []
+            unique_urls.append(url)
+        url_to_search_results[url].append(search_result)
+    browser_crawler = create_browser_crawler()
+    scraped_results = browser_crawler.scrape_urls(unique_urls)
+    url_to_scraped = {result["original_url"]: result for result in scraped_results}
+    updated_search_results = []
+    successful_scrapes = 0
+    failed_scrapes = 0
+    for search_result in search_results:
+        original_url = search_result["url"]
+        scraped = url_to_scraped.get(original_url, {})
+        updated_result = search_result.copy()
+        updated_result["original_url"] = original_url
+        if scraped.get("success", False):
+            updated_result["url"] = scraped.get("final_url", original_url)
+            updated_result["full_text"] = scraped.get("full_text", "")
+            updated_result["published_date"] = scraped.get("published_date", "")
+            successful_scrapes += 1
+        else:
+            updated_result["url"] = original_url
+            updated_result["full_text"] = search_result.get("description", "")
+            updated_result["published_date"] = ""
+            failed_scrapes += 1
+        updated_search_results.append(updated_result)
+    return updated_search_results, successful_scrapes, failed_scrapes
+
+
+def verify_content_with_agent(query, search_results, use_agent=True):
+    if not use_agent:
+        return search_results
+    verified_search_results = []
+    for _, search_result in enumerate(search_results):
+        content_for_verification = {
+            "url": search_result["url"],
+            "description": search_result.get("description", ""),
+            "full_text": search_result["full_text"],
+            "published_date": search_result["published_date"],
+        }
+        search_result["agent_verified"] = False
+        try:
+            session_id = str(uuid.uuid4())
+            scrape_agent = Agent(
+                model=OpenAIChat(id="gpt-4o-mini"),
+                instructions=SCRAPE_AGENT_INSTRUCTIONS,
+                description=SCRAPE_AGENT_DESCRIPTION,
+                use_json_mode=True,
+                session_id=session_id,
+                response_model=ScrapedContent,
+            )
+            response = scrape_agent.run(
+                f"Query: {query}\n"
+                f"Verify and format this scraped content. "
+                f"Keep content relevant to the query and ensure quality: {content_for_verification}",
+                session_id=session_id,
+            )
+            verified_item = response.to_dict()["content"]
+            search_result["full_text"] = verified_item.get("full_text", search_result["full_text"])
+            search_result["published_date"] = verified_item.get("published_date", search_result["published_date"])
+            search_result["agent_verified"] = True
+        except Exception as _:
+            pass
+        verified_search_results.append(search_result)
+    return verified_search_results
+
+
+def scrape_agent_run(query: str, search_results) -> str:
+    try:
+        updated_results, _, _ = crawl_urls_batch(search_results)
+        verified_results = verify_content_with_agent(query, updated_results, use_agent=False)
+        return verified_results
+    except Exception as _:
+        import traceback
+
+        traceback.print_exc()
+        return []

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/pipeline/script_agent.py

@@ -0,0 +1,120 @@
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from pydantic import BaseModel, Field
+from typing import List, Optional
+from dotenv import load_dotenv
+from textwrap import dedent
+from datetime import datetime
+import uuid
+
+load_dotenv()
+
+
+class Dialog(BaseModel):
+    speaker: str = Field(..., description="The speaker name (SHOULD BE 'ALEX' OR 'MORGAN')")
+    text: str = Field(
+        ...,
+        description="The spoken text content for this speaker based on the requested langauge, default is English",
+    )
+
+
+class Section(BaseModel):
+    type: str = Field(..., description="The section type (intro, headlines, article, outro)")
+    title: Optional[str] = Field(None, description="Optional title for the section (required for article type)")
+    dialog: List[Dialog] = Field(..., description="List of dialog exchanges between speakers")
+
+
+class PodcastScript(BaseModel):
+    title: str = Field(..., description="The podcast episode title with date")
+    sections: List[Section] = Field(..., description="List of podcast sections (intro, headlines, articles, outro)")
+
+
+PODCAST_AGENT_DESCRIPTION = "You are a helpful assistant that can generate engaging podcast scripts for the given sources."
+PODCAST_AGENT_INSTRUCTIONS = dedent("""
+    You are a helpful assistant that can generate engaging podcast scripts for the given source content and query.
+    For given content, create an engaging podcast script that should be at least 15 minutes worth of content and your allowed enhance the script beyond given sources if you know something additional info will be interesting to the discussion or not enough conents available.
+    You use the provided sources to ground your podcast script generation process. Keep it engaging and interesting.
+    
+    IMPORTANT: Generate the entire script in the provided language. basically only text field needs to be in requested language,
+    
+    CONTENT GUIDELINES [THIS IS EXAMPLE YOU CAN CHANGE THE GUIDELINES ANYWAY BASED ON THE QUERY OR TOPIC DISCUSSED]:
+    - Provide insightful analysis that helps the audience understand the significance
+    - Include discussions on potential implications and broader context of each story
+    - Explain complex concepts in an accessible but thorough manner
+    - Make connections between current and relevant historical developments when applicable
+    - Provide comparisons and contrasts with similar stories or trends when relevant
+    
+    PERSONALITY NOTES [THIS IS EXAMPLE YOU CAN CHANGE THE PERSONALITY OF ALEX AND MORGAN ANYWAY BASED ON THE QUERY OR TOPIC DISCUSSED]:
+    - Alex is more analytical and fact-focused
+    * Should reference specific details and data points
+    * Should explain complex topics clearly
+    * Should identify key implications of stories
+    - Morgan is more focused on human impact, social context, and practical applications
+    * Should analyze broader implications
+    * Should consider ethical implications and real-world applications
+    - Include natural, conversational banter and smooth transitions between topics
+    - Each article discussion should go beyond the basic summary to provide valuable insights
+    - Maintain a conversational but informed tone that would appeal to a general audience
+    
+    IMPORTNAT:
+        - MAKE SURE PODCAST SCRIPS ARE AT LEAST 15 MINUTES LONG WHICH MEANS YOU NEED TO HAVE DETAILED DISCUSSIONS OFFCOURSE KEEP IT INTERESTING AND ENGAGING.
+    """)
+
+
+def format_search_results_for_podcast(
+    search_results: List[dict],
+) -> tuple[str, List[str]]:
+    created_at = datetime.now().strftime("%B %d, %Y at %I:%M %p")
+    structured_content = []
+    structured_content.append(f"PODCAST CREATION: {created_at}\n")
+    sources = []
+    for idx, search_result in enumerate(search_results):
+        try:
+            if search_result.get("confirmed", False):
+                sources.append(search_result["url"])
+                structured_content.append(
+                    f"""
+                                        SOURCE {idx + 1}:
+                                        Title: {search_result['title']}
+                                        URL: {search_result['url']}
+                                        Content: {search_result.get('full_text') or search_result.get('description', '')}
+                                        ---END OF SOURCE {idx + 1}---
+                                        """.strip()
+                )
+        except Exception as e:
+            print(f"Error processing search result: {e}")
+    content_texts = "\n\n".join(structured_content)
+    return content_texts, sources
+
+
+def script_agent_run(
+    query: str,
+    search_results: List[dict],
+    language_name: str,
+) -> str:
+    try:
+        session_id = str(uuid.uuid4())
+        content_texts, sources = format_search_results_for_podcast(search_results)
+        if not content_texts:
+            return {}
+        podcast_script_agent = Agent(
+            model=OpenAIChat(id="gpt-4o-mini"),
+            instructions=PODCAST_AGENT_INSTRUCTIONS,
+            description=PODCAST_AGENT_DESCRIPTION,
+            use_json_mode=True,
+            response_model=PodcastScript,
+            session_id=session_id,
+        )
+        response = podcast_script_agent.run(
+            f"query: {query}\n language_name: {language_name}\n content_texts: {content_texts}\n, IMPORTANT: texts should be in {language_name} language.",
+            session_id=session_id,
+        )
+        response_dict = response.to_dict()
+        response_dict = response_dict["content"]
+        response_dict["sources"] = sources
+        return response_dict
+    except Exception as _:
+        import traceback
+
+        traceback.print_exc()
+        return {}

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/pipeline/search_agent.py

@@ -0,0 +1,90 @@
+from typing import List
+import uuid
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from pydantic import BaseModel, Field
+from dotenv import load_dotenv
+from agno.tools.duckduckgo import DuckDuckGoTools
+from textwrap import dedent
+from tools.wikipedia_search import wikipedia_search
+from tools.google_news_discovery import google_news_discovery_run
+from tools.jikan_search import jikan_search
+from tools.embedding_search import embedding_search
+from tools.social_media_search import social_media_search, social_media_trending_search
+
+
+load_dotenv()
+
+
+class ReturnItem(BaseModel):
+    url: str = Field(..., description="The URL of the search result")
+    title: str = Field(..., description="The title of the search result")
+    description: str = Field(..., description="A brief description or summary of the search result content")
+    source_name: str = Field(
+        ...,
+        description="The name/type of the source (e.g., 'wikipedia', 'general', or any reputable source tag)",
+    )
+    tool_used: str = Field(
+        ...,
+        description="The tools used to generate the search results, unknown if not used or not applicable",
+    )
+    published_date: str = Field(
+        ...,
+        description="The published date of the content in ISO format, if not available keep it empty",
+    )
+    is_scrapping_required: bool = Field(
+        ...,
+        description="Set to True if the content need scraping, False otherwise, default keep it True if not sure",
+    )
+
+
+class SearchResults(BaseModel):
+    items: List[ReturnItem] = Field(..., description="A list of search result items")
+
+
+SEARCH_AGENT_DESCRIPTION = "You are a helpful assistant that can search the web for information."
+SEARCH_AGENT_INSTRUCTIONS = dedent("""
+    You are a helpful assistant that can search the web or any other sources for information.
+    You should create topic for the search from the given query instead of blindly apply the query to the search tools.
+    For a given topic, your job is to search the web or any other sources and return the top 5 to 10 sources about the topic.
+    Keep the search sources of high quality and reputable, and sources should be relevant to the asked topic.
+    Sources should be from diverse platforms with no duplicates.
+    IMPORTANT: User queries might be fuzzy or misspelled. Understand the user's intent and act accordingly.
+    IMPORTANT: The output source_name field can be one of ["wikipedia", "general", or any source tag used"].
+    IMPORTANT: You have access to different search tools use them when appropriate which one is best for the given search query. Don't use particular tool if not required.
+    IMPORTANT: Make sure you are able to detect what tool to use and use it available tool tags = ["google_news_discovery", "duckduckgo", "wikipedia_search", "jikan_search", "social_media_search", "social_media_trending_search", "unknown"].
+    IMPORTANT: If query is news related please prefere google news over other news tools.
+    IMPORTANT: If returned sources are not of high quality or not relevant to the asked topic, don't include them in the returned sources.
+    IMPORTANT: Never include dates to the search query unless user explicitly asks for it.
+    IMPORTANT: You are allowed to use appropriate tools to get the best results even the single tool return enough results diverse check is better.
+    """)
+
+
+def search_agent_run(query: str) -> str:
+    try:
+        session_id = str(uuid.uuid4())
+        search_agent = Agent(
+            model=OpenAIChat(id="gpt-4o-mini"),
+            instructions=SEARCH_AGENT_INSTRUCTIONS,
+            description=SEARCH_AGENT_DESCRIPTION,
+            use_json_mode=True,
+            response_model=SearchResults,
+            tools=[
+                google_news_discovery_run,
+                DuckDuckGoTools(),
+                wikipedia_search,
+                jikan_search,
+                embedding_search,
+                social_media_search,
+                social_media_trending_search,
+            ],
+            session_id=session_id,
+        )
+        response = search_agent.run(query, session_id=session_id)
+        response_dict = response.to_dict()
+        return response_dict["content"]["items"]
+    except Exception as _:
+        import traceback
+
+        traceback.print_exc()
+        return []

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/search_articles.py

@@ -0,0 +1,67 @@
+import sqlite3
+from typing import List, Union
+from agno.agent import Agent
+from db.config import get_tracking_db_path
+import json
+
+
+def search_articles(agent: Agent, terms: Union[str, List[str]]) -> str:
+    """
+    Search for articles related to a podcast topic using direct SQL queries.
+    The agent can pass either a string topic or a list of search terms.
+
+    Args:
+        agent: The agent instance
+        terms: Either a single topic string or a list of search terms
+
+    Returns:
+        A formatted string response with the search results
+    """
+    print(f"Search Internal Articles terms: {terms}")
+    search_terms = terms if isinstance(terms, list) else [terms]
+    limit = 3
+    db_path = get_tracking_db_path()
+    try:
+        with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:
+            conn.row_factory = lambda cursor, row: {col[0]: row[idx] for idx, col in enumerate(cursor.description)}
+            results = execute_simple_search(conn, search_terms, limit)
+            if not results:
+                return "No relevant articles found in our database. Would you like to try a different topic or provide specific URLs?"
+            for article in results:
+                article["categories"] = get_article_categories(conn, article["id"])
+                article["source_name"] = article.get("source_id", "Unknown Source")
+            return f"is_scrapping_required: False, Found {len(results)}, {json.dumps(results, indent=2)} potential sources that might be relevant to your topic careful my search is text bassed do quality check and ignore invalid resutls."
+    except Exception as e:
+        print(f"Error searching articles: {e}")
+        return "I encountered a database error while searching. Would you like to try a different approach?"
+
+
+def execute_simple_search(conn, terms, limit):
+    base_query = """
+        SELECT DISTINCT ca.id, ca.title, ca.url, ca.published_date, 
+               COALESCE(ca.summary, ca.content) as content,
+               ca.source_id, ca.feed_id
+        FROM crawled_articles ca
+        WHERE ca.processed = 1 
+          AND (
+    """
+    clauses = []
+    params = []
+    for term in terms:
+        like_term = f"%{term}%"
+        clauses.append("(ca.title LIKE ? OR ca.content LIKE ? OR ca.summary LIKE ?)")
+        params.extend([like_term, like_term, like_term])
+
+    query = base_query + " OR ".join(clauses) + ") ORDER BY ca.published_date DESC LIMIT ?"
+    params.append(limit)
+    cursor = conn.execute(query, params)
+    return [dict(row) for row in cursor.fetchall()]
+
+
+def get_article_categories(conn, article_id):
+    try:
+        cursor = conn.execute("SELECT category_name FROM article_categories WHERE article_id = ?", (article_id,))
+        return [row["category_name"] for row in cursor.fetchall()]
+    except Exception as e:
+        print(f"Error fetching article categories: {e}")
+        return []

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/session_state_manager.py

@@ -0,0 +1,204 @@
+from agno.agent import Agent
+from datetime import datetime
+from db.config import get_podcasts_db_path, DB_PATH
+import os
+import sqlite3
+import json
+
+
+def _save_podcast_to_database_sync(session_state: dict) -> tuple[bool, str, int]:
+    try:
+        if session_state.get("podcast_id"):
+            return (
+                True,
+                f"Podcast already saved with ID: {session_state['podcast_id']}",
+                session_state["podcast_id"],
+            )
+        tts_engine = session_state.get("tts_engine", "openai")
+        podcast_info = session_state.get("podcast_info", {})
+        generated_script = session_state.get("generated_script", {})
+        banner_url = session_state.get("banner_url")
+        banner_images = json.dumps(session_state.get("banner_images", []))
+        audio_url = session_state.get("audio_url")
+        selected_language = session_state.get("selected_language", {"code": "en", "name": "English"})
+        language_code = selected_language.get("code", "en")
+        if not generated_script or not isinstance(generated_script, dict):
+            return (
+                False,
+                "Cannot complete podcast: Generated script is missing or invalid.",
+                None,
+            )
+        if "title" not in generated_script:
+            generated_script["title"] = podcast_info.get("topic", "Untitled Podcast")
+        if "sections" not in generated_script or not isinstance(generated_script["sections"], list):
+            return (
+                False,
+                "Cannot complete podcast: Generated script is missing required 'sections' array.",
+                None,
+            )
+        sources = []
+        if "sources" in generated_script and generated_script["sources"]:
+            for source in generated_script["sources"]:
+                if isinstance(source, str):
+                    sources.append(source)
+                elif isinstance(source, dict) and "url" in source:
+                    sources.append(source["url"])
+                elif isinstance(source, dict) and "link" in source:
+                    sources.append(source["link"])
+        generated_script["sources"] = sources
+        db_path = get_podcasts_db_path()
+        db_directory = DB_PATH
+        os.makedirs(db_directory, exist_ok=True)
+
+        conn = sqlite3.connect(db_path)
+        content_json = json.dumps(generated_script)
+        sources_json = json.dumps(sources) if sources else None
+        current_time = datetime.now().isoformat()
+        query = """
+            INSERT INTO podcasts (
+                title, 
+                date, 
+                content_json, 
+                audio_generated, 
+                audio_path,
+                banner_img_path, 
+                tts_engine, 
+                language_code, 
+                sources_json,
+                created_at,
+                banner_images
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """
+        conn.execute(
+            query,
+            (
+                generated_script.get("title", "Untitled Podcast"),
+                datetime.now().strftime("%Y-%m-%d"),
+                content_json,
+                1 if audio_url else 0,
+                audio_url,
+                banner_url,
+                tts_engine,
+                language_code,
+                sources_json,
+                current_time,
+                banner_images,
+            ),
+        )
+        conn.commit()
+
+        cursor = conn.execute("SELECT last_insert_rowid()")
+        podcast_id = cursor.fetchone()
+        podcast_id = podcast_id[0] if podcast_id else None
+        cursor.close()
+        conn.close()
+
+        session_state["podcast_id"] = podcast_id
+        return True, f"Podcast successfully saved with ID: {podcast_id}", podcast_id
+    except Exception as e:
+        print(f"Error saving podcast to database: {e}")
+        return False, f"Error saving podcast to database: {str(e)}", None
+
+
+def update_language(agent: Agent, language_code: str) -> str:
+    """
+    Update the podcast language with the specified language code.
+    This ensures the language is properly tracked for generating content and audio.
+    Args:
+        agent: The agent instance
+        language_code: The language code (e.g., 'en', 'es', 'fr', etc..)
+
+    Returns:
+        Confirmation message
+    """
+    from services.internal_session_service import SessionService
+
+    session_id = agent.session_id
+    session = SessionService.get_session(session_id)
+    session_state = session["state"]
+    language_name = "English"
+    for lang in session_state.get("available_languages", []):
+        if lang.get("code") == language_code:
+            language_name = lang.get("name")
+            break
+    session_state["selected_language"] = {
+        "code": language_code,
+        "name": language_name,
+    }
+    SessionService.save_session(session_id, session_state)
+    return f"Podcast language set to: {language_name} ({language_code})"
+
+
+def update_chat_title(agent: Agent, title: str) -> str:
+    """
+    Update the chat title with the specified short title.
+    Args:
+        agent: The agent instance
+        title: The short title to set for the chat
+
+    Returns:
+        Confirmation message
+    """
+    from services.internal_session_service import SessionService
+
+    session_id = agent.session_id
+    session = SessionService.get_session(session_id)
+    current_state = session["state"]
+    current_state["title"] = title
+    current_state["created_at"] = datetime.now().isoformat()
+    SessionService.save_session(session_id, current_state)
+    return f"Chat title updated to: {title}"
+
+
+def toggle_podcast_generated(session_state: dict, status: bool = False) -> str:
+    """
+    Toggle the podcast_generated flag.
+    When set to true, this indicates the podcast creation process is complete and
+    the UI should show the final presentation view with all components.
+    If status is True, also saves the podcast to the podcasts database.
+    """
+    if status:
+        session_state["podcast_generated"] = status
+        session_state["stage"] = "complete" if status else session_state.get("stage")
+        if status:
+            try:
+                success, message, podcast_id = _save_podcast_to_database_sync(session_state)
+                if success and podcast_id:
+                    session_state["podcast_id"] = podcast_id
+                    return f"Podcast generated and saved to database with ID: {podcast_id}. You can now access it from the Podcasts section."
+                else:
+                    return f"Podcast generated, but there was an issue with saving: {message}"
+            except Exception as e:
+                print(f"Error saving podcast to database: {e}")
+                return f"Podcast generated, but there was an error saving it to the database: {str(e)}"
+    else:
+        session_state["podcast_generated"] = status
+        session_state["stage"] = "complete" if status else session_state.get("stage")
+    return f"Podcast generated status changed to: {status}"
+
+
+def mark_session_finished(agent: Agent) -> str:
+    """
+    Mark the session as finished.
+    Args:
+        agent: The agent instance
+
+    Returns:
+        Confirmation message
+    """
+    from services.internal_session_service import SessionService
+
+    session_id = agent.session_id
+    session = SessionService.get_session(session_id)
+    session_state = session["state"]
+    if not session_state.get("generated_script"):
+        return "Podcast Script is not generated yet."
+    if not session_state.get("banner_url"):
+        return "Banner is not generated yet."
+    if not session_state.get("audio_url"):
+        return "Audio is not generated yet."
+    session_state["finished"] = True
+    session_state["stage"] = "complete"
+    toggle_podcast_generated(session_state, True)
+    SessionService.save_session(session_id, session_state)
+    return "Session marked as finished and generated podcast stored into podcasts database and No further conversation are allowed and only new session can be started."

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/browser.py

@@ -0,0 +1,90 @@
+import os
+from playwright.sync_api import sync_playwright
+from tools.social.config import USER_DATA_DIR
+from contextlib import contextmanager
+
+
+def setup_session(TARGET_SITE):
+    print("🔐 SESSION SETUP MODE")
+    print(f"Opening browser with persistent profile at: {USER_DATA_DIR}")
+    print(f"Please log in {TARGET_SITE} manually to establish your session")
+    with sync_playwright() as playwright:
+        os.makedirs(USER_DATA_DIR, exist_ok=True)
+        browser_context = playwright.chromium.launch_persistent_context(
+            user_data_dir=USER_DATA_DIR,
+            headless=False,
+            viewport={"width": 1280, "height": 800},
+        )
+        try:
+            if browser_context.pages:
+                page = browser_context.pages[0]
+            else:
+                page = browser_context.new_page()
+            page.goto(TARGET_SITE)
+            print("\n✅ Browser is now open. Please:")
+            print("1. Log in to your account manually")
+            print("2. Navigate through the site as needed")
+            print("3. Press Ctrl+C in this terminal when you're done\n")
+            try:
+                while True:
+                    page.wait_for_timeout(1000)
+            except KeyboardInterrupt:
+                print("\n🔑 Session saved! You can now run monitoring tasks with this session.")
+        finally:
+            browser_context.close()
+
+
+def setup_session_multi(TARGET_SITES):
+    print("🔐 MULTI-SITE SESSION SETUP MODE")
+    print(f"Opening browser with persistent profile at: {USER_DATA_DIR}")
+    print(f"Setting up sessions for {len(TARGET_SITES)} sites:")
+    for i, site in enumerate(TARGET_SITES, 1):
+        print(f"  {i}. {site}")
+    with sync_playwright() as playwright:
+        os.makedirs(USER_DATA_DIR, exist_ok=True)
+        browser_context = playwright.chromium.launch_persistent_context(
+            user_data_dir=USER_DATA_DIR,
+            headless=False,
+            viewport={"width": 1280, "height": 800},
+        )
+        try:
+            pages = []
+            for i, site in enumerate(TARGET_SITES):
+                if i == 0 and browser_context.pages:
+                    page = browser_context.pages[0]
+                else:
+                    page = browser_context.new_page()
+                print(f"📂 Opening: {site}")
+                page.goto(site)
+                pages.append(page)
+
+            print("\n✅ All sites opened in separate tabs. Please:")
+            print("1. Log in to your accounts manually in each tab")
+            print("2. Navigate through the sites as needed")
+            print("3. Press Ctrl+C in this terminal when you're done\n")
+            print("💡 Tip: Use Ctrl+Tab to switch between tabs")
+            try:
+                while True:
+                    pages[0].wait_for_timeout(1000)
+            except KeyboardInterrupt:
+                print(f"\n🔑 Sessions saved for all {len(TARGET_SITES)} sites! " "You can now run monitoring tasks with these sessions.")
+        finally:
+            browser_context.close()
+
+
+@contextmanager
+def create_browser_context():
+    with sync_playwright() as playwright:
+        browser_context = playwright.chromium.launch_persistent_context(
+            user_data_dir=USER_DATA_DIR,
+            headless=False,
+            viewport={"width": 1280, "height": 800},
+        )
+        try:
+            if browser_context.pages:
+                page = browser_context.pages[0]
+            else:
+                page = browser_context.new_page()
+            yield browser_context, page
+        finally:
+            browser_context.close()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/config.py

@@ -0,0 +1,3 @@
+from db.config import get_browser_session_path
+
+USER_DATA_DIR = get_browser_session_path()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/db.py

@@ -0,0 +1,161 @@
+import sqlite3
+import json
+
+
+def create_connection(db_file="x_posts.db"):
+    conn = sqlite3.connect(db_file)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def setup_database(conn):
+    create_posts_table = """
+    CREATE TABLE IF NOT EXISTS posts (
+        post_id TEXT PRIMARY KEY,
+        platform TEXT,
+        user_display_name TEXT,
+        user_handle TEXT,
+        user_profile_pic_url TEXT,
+        post_timestamp TEXT,
+        post_display_time TEXT,
+        post_url TEXT,
+        post_text TEXT,
+        post_mentions TEXT,
+        engagement_reply_count INTEGER,
+        engagement_retweet_count INTEGER,
+        engagement_like_count INTEGER,
+        engagement_bookmark_count INTEGER,
+        engagement_view_count INTEGER,
+        media TEXT,  -- Stored as JSON
+        media_count INTEGER,
+        is_ad BOOLEAN,
+        sentiment TEXT,
+        categories TEXT,
+        tags TEXT,
+        analysis_reasoning TEXT,
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    )
+    """
+
+    conn.execute(create_posts_table)
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_posts_platform ON posts(platform)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_posts_user_handle ON posts(user_handle)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_posts_post_timestamp ON posts(post_timestamp)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_posts_sentiment ON posts(sentiment)")
+    conn.commit()
+
+
+def parse_engagement_count(count_str):
+    if not count_str:
+        return 0
+    count_str = str(count_str).strip()
+    if not count_str:
+        return 0
+    multiplier = 1
+    if "K" in count_str:
+        multiplier = 1000
+        count_str = count_str.replace("K", "")
+    elif "M" in count_str:
+        multiplier = 1000000
+        count_str = count_str.replace("M", "")
+    elif "B" in count_str:
+        multiplier = 1000000000
+        count_str = count_str.replace("B", "")
+    try:
+        return int(float(count_str) * multiplier)
+    except (ValueError, TypeError):
+        return 0
+
+
+def process_post_data(post_data):
+    data = post_data.copy()
+    metrics = ["engagement_reply_count", "engagement_retweet_count", "engagement_like_count", "engagement_bookmark_count", "engagement_view_count"]
+    for metric in metrics:
+        if metric in data:
+            data[metric] = parse_engagement_count(data[metric])
+    if "media" in data and isinstance(data["media"], list):
+        data["media"] = json.dumps(data["media"])
+    if "categories" in data and isinstance(data["categories"], list):
+        data["categories"] = json.dumps(data["categories"])
+    if "tags" in data and isinstance(data["tags"], list):
+        data["tags"] = json.dumps(data["tags"])
+    if "is_ad" in data:
+        data["is_ad"] = 1 if data["is_ad"] else 0
+    return data
+
+
+def get_post(conn, post_id):
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM posts WHERE post_id = ?", (post_id,))
+    return cursor.fetchone()
+
+
+def insert_post(conn, post_data):
+    data = process_post_data(post_data)
+    columns = ", ".join(data.keys())
+    placeholders = ", ".join(["?"] * len(data))
+    values = list(data.values())
+    sql = f"INSERT INTO posts ({columns}) VALUES ({placeholders})"
+    conn.execute(sql, values)
+    conn.commit()
+
+
+def check_and_store_post(conn, post_data):
+    post_id = post_data.get("post_id")
+    if not post_id:
+        return False
+    if post_data.get("is_ad", False):
+        return False
+    data = process_post_data(post_data)
+    existing_post = get_post(conn, post_id)
+    if not existing_post:
+        needs_analysis = bool(data.get("post_text"))
+        insert_post(conn, data)
+        return needs_analysis
+    else:
+        update_changed_metrics(conn, existing_post, data)
+        return False
+
+
+def update_changed_metrics(conn, existing_post, new_data):
+    metrics = ["engagement_reply_count", "engagement_retweet_count", "engagement_like_count", "engagement_bookmark_count", "engagement_view_count"]
+    changes = {}
+    for metric in metrics:
+        if metric in new_data and metric in existing_post:
+            if new_data[metric] != existing_post[metric]:
+                changes[metric] = new_data[metric]
+    if changes:
+        set_clauses = [f"{metric} = ?" for metric in changes]
+        set_sql = ", ".join(set_clauses)
+        set_sql += ", updated_at = CURRENT_TIMESTAMP"
+        sql = f"UPDATE posts SET {set_sql} WHERE post_id = ?"
+        params = list(changes.values()) + [existing_post["post_id"]]
+        conn.execute(sql, params)
+        conn.commit()
+
+
+def update_posts_with_analysis(conn, post_ids, analysis_results):
+    if not analysis_results:
+        return
+    analysis_by_id = {}
+    for analysis in analysis_results:
+        post_id = analysis.get("post_id")
+        if post_id:
+            analysis_by_id[post_id] = analysis
+    for post_id in post_ids:
+        if post_id in analysis_by_id:
+            analysis = analysis_by_id[post_id]
+            categories = json.dumps(analysis.get("categories", []))
+            tags = json.dumps(analysis.get("tags", []))
+            conn.execute(
+                """UPDATE posts SET 
+                   sentiment = ?, 
+                   categories = ?, 
+                   tags = ?, 
+                   analysis_reasoning = ?,
+                   updated_at = CURRENT_TIMESTAMP 
+                   WHERE post_id = ?""",
+                (analysis.get("sentiment"), categories, tags, analysis.get("reasoning"), post_id),
+            )
+    conn.commit()

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/fb_post_extractor.py

@@ -0,0 +1,415 @@
+import json
+from datetime import datetime
+from typing import Dict, Any
+import re
+
+
+def parse_facebook_posts(data):
+    posts = []
+    try:
+        for post in data["data"]["viewer"]["news_feed"]["edges"]:
+            posts.append(parse_facebook_post(post["node"]))
+    except Exception as e:
+        print(f"Error parsing post data: {e}")
+    return posts
+
+
+def parse_facebook_post(story_node: Dict[str, Any]) -> Dict[str, Any]:
+    try:
+        post_info = {
+            "post_id": story_node.get("post_id"),
+            "story_id": story_node.get("id"),
+            "creation_time": None,
+            "creation_time_formatted": None,
+            "url": None,
+        }
+        creation_time_sources = [
+            story_node.get("creation_time"),
+            (story_node.get("comet_sections") or {}).get("context_layout", {}).get("story", {}).get("creation_time"),
+            (story_node.get("comet_sections") or {}).get("timestamp", {}).get("story", {}).get("creation_time"),
+        ]
+        for source in creation_time_sources:
+            if source:
+                post_info["creation_time"] = source
+                post_info["creation_time_formatted"] = datetime.fromtimestamp(source).strftime("%Y-%m-%d %H:%M:%S")
+                break
+        url_sources = [
+            (story_node.get("comet_sections") or {}).get("content", {}).get("story", {}).get("wwwURL"),
+            (story_node.get("comet_sections") or {}).get("feedback", {}).get("story", {}).get("story_ufi_container", {}).get("story", {}).get("url"),
+            (story_node.get("comet_sections") or {})
+            .get("feedback", {})
+            .get("story", {})
+            .get("story_ufi_container", {})
+            .get("story", {})
+            .get("shareable_from_perspective_of_feed_ufi", {})
+            .get("url"),
+            story_node.get("wwwURL"),
+            story_node.get("url"),
+        ]
+        for source in url_sources:
+            if source:
+                post_info["url"] = source
+                break
+        message_info = extract_message_content(story_node)
+        post_info.update(message_info)
+        actors_info = extract_actors_info(story_node)
+        post_info.update(actors_info)
+        attachments_info = extract_attachments(story_node)
+        post_info.update(attachments_info)
+        engagement_info = extract_engagement_data(story_node)
+        post_info.update(engagement_info)
+        privacy_info = extract_privacy_info(story_node)
+        post_info.update(privacy_info)
+        return post_info
+    except (KeyError, IndexError, TypeError) as e:
+        print(f"Error parsing post data: {e}")
+        return {}
+
+
+def extract_message_content(story_node: Dict[str, Any]) -> Dict[str, Any]:
+    message_info = {"message_text": "", "hashtags": [], "mentions": [], "links": []}
+    try:
+        message_sources = [
+            (story_node.get("message") or {}).get("text", ""),
+            ((((story_node.get("comet_sections") or {}).get("content") or {}).get("story") or {}).get("message") or {}).get("text", ""),
+        ]
+        for source in message_sources:
+            if source:
+                message_info["message_text"] = source
+                break
+        message_data = story_node.get("message", {})
+        if "ranges" in message_data:
+            for range_item in message_data["ranges"]:
+                entity = range_item.get("entity", {})
+                entity_type = entity.get("__typename")
+
+                if entity_type == "Hashtag":
+                    hashtag_text = message_info["message_text"][range_item["offset"] : range_item["offset"] + range_item["length"]]
+                    message_info["hashtags"].append(
+                        {
+                            "text": hashtag_text,
+                            "url": entity.get("url"),
+                            "id": entity.get("id"),
+                        }
+                    )
+                elif entity_type == "User":
+                    mention_text = message_info["message_text"][range_item["offset"] : range_item["offset"] + range_item["length"]]
+                    message_info["mentions"].append(
+                        {
+                            "text": mention_text,
+                            "url": entity.get("url"),
+                            "id": entity.get("id"),
+                        }
+                    )
+    except (KeyError, TypeError):
+        pass
+    return message_info
+
+
+def extract_actors_info(story_node: Dict[str, Any]) -> Dict[str, Any]:
+    actors_info = {
+        "author_name": "",
+        "author_id": "",
+        "author_url": "",
+        "author_profile_picture": "",
+        "is_verified": False,
+        "page_info": {},
+    }
+    try:
+        actors = story_node.get("actors", [])
+        if actors:
+            main_actor = actors[0]
+            actors_info.update(
+                {
+                    "author_name": main_actor.get("name", ""),
+                    "author_id": main_actor.get("id", ""),
+                    "author_url": main_actor.get("url", ""),
+                }
+            )
+        context_sections = (story_node.get("comet_sections") or {}).get("context_layout", {})
+        if context_sections:
+            actor_photo = (context_sections.get("story") or {}).get("comet_sections", {}).get("actor_photo", {})
+            if actor_photo:
+                story_actors = (actor_photo.get("story") or {}).get("actors", [])
+                if story_actors:
+                    profile_pic = story_actors[0].get("profile_picture", {})
+                    actors_info["author_profile_picture"] = profile_pic.get("uri", "")
+    except (KeyError, TypeError, IndexError):
+        pass
+    return actors_info
+
+
+def extract_attachments(story_node: Dict[str, Any]) -> Dict[str, Any]:
+    attachments_info = {"attachments": [], "photos": [], "videos": [], "links": []}
+    try:
+        attachments = story_node.get("attachments", [])
+        for attachment in attachments:
+            attachment_data = {
+                "type": attachment.get("__typename", ""),
+                "style_list": attachment.get("style_list", []),
+            }
+            if "photo" in attachment.get("style_list", []):
+                target = attachment.get("target", {})
+                if target.get("__typename") == "Photo":
+                    styles = attachment.get("styles", {})
+                    if styles:
+                        media = (styles.get("attachment") or {}).get("media", {})
+                        photo_info = {
+                            "id": target.get("id"),
+                            "url": media.get("url", ""),
+                            "width": (media.get("viewer_image") or {}).get("width"),
+                            "height": (media.get("viewer_image") or {}).get("height"),
+                            "image_uri": "",
+                            "accessibility_caption": media.get("accessibility_caption", ""),
+                        }
+                        resolution_renderer = media.get("comet_photo_attachment_resolution_renderer", {})
+                        if resolution_renderer:
+                            image = resolution_renderer.get("image", {})
+                            photo_info["image_uri"] = image.get("uri", "")
+
+                        attachments_info["photos"].append(photo_info)
+            attachments_info["attachments"].append(attachment_data)
+    except (KeyError, TypeError):
+        pass
+    return attachments_info
+
+
+def extract_engagement_data(story_node: Dict[str, Any]) -> Dict[str, Any]:
+    """Extract likes, comments, shares, and other engagement metrics"""
+    engagement_info = {
+        "reaction_count": 0,
+        "comment_count": 0,
+        "share_count": 0,
+        "reactions_breakdown": {},
+        "top_reactions": [],
+    }
+    try:
+        feedback_story = (story_node.get("comet_sections") or {}).get("feedback", {}).get("story", {})
+        if feedback_story:
+            ufi_container = (feedback_story.get("story_ufi_container") or {}).get("story", {})
+            if ufi_container:
+                feedback_context = ufi_container.get("feedback_context", {})
+                feedback_target = feedback_context.get("feedback_target_with_context", {})
+
+                if feedback_target:
+                    summary_renderer = feedback_target.get("comet_ufi_summary_and_actions_renderer", {})
+                    if summary_renderer:
+                        feedback_data = summary_renderer.get("feedback", {})
+                        if "i18n_reaction_count" in feedback_data:
+                            engagement_info["reaction_count"] = int(feedback_data["i18n_reaction_count"])
+                        elif "reaction_count" in feedback_data and isinstance(feedback_data["reaction_count"], dict):
+                            engagement_info["reaction_count"] = feedback_data["reaction_count"].get("count", 0)
+                        if "i18n_share_count" in feedback_data:
+                            engagement_info["share_count"] = int(feedback_data["i18n_share_count"])
+                        elif "share_count" in feedback_data and isinstance(feedback_data["share_count"], dict):
+                            engagement_info["share_count"] = feedback_data["share_count"].get("count", 0)
+                        top_reactions = feedback_data.get("top_reactions", {})
+                        if "edges" in top_reactions:
+                            for edge in top_reactions["edges"]:
+                                reaction_node = edge.get("node", {})
+                                engagement_info["top_reactions"].append(
+                                    {
+                                        "reaction_id": reaction_node.get("id"),
+                                        "name": reaction_node.get("localized_name"),
+                                        "count": edge.get("reaction_count", 0),
+                                    }
+                                )
+                    comment_rendering = feedback_target.get("comment_rendering_instance", {})
+                    if comment_rendering:
+                        comments = comment_rendering.get("comments", {})
+                        engagement_info["comment_count"] = comments.get("total_count", 0)
+
+    except (KeyError, TypeError, ValueError):
+        pass
+    return engagement_info
+
+
+def extract_privacy_info(story_node: Dict[str, Any]) -> Dict[str, Any]:
+    privacy_info = {"privacy_scope": "", "audience": ""}
+    try:
+        privacy_sources = [
+            (story_node.get("comet_sections") or {}).get("context_layout", {}).get("story", {}).get("privacy_scope", {}),
+            story_node.get("privacy_scope", {}),
+            next(
+                (
+                    (meta.get("story") or {}).get("privacy_scope", {})
+                    for meta in (story_node.get("comet_sections") or {})
+                    .get("context_layout", {})
+                    .get("story", {})
+                    .get("comet_sections", {})
+                    .get("metadata", [])
+                    if isinstance(meta, dict) and meta.get("__typename") == "CometFeedStoryAudienceStrategy"
+                ),
+                {},
+            ),
+        ]
+        for privacy_scope in privacy_sources:
+            if privacy_scope and "description" in privacy_scope:
+                privacy_info["privacy_scope"] = privacy_scope["description"]
+                break
+        if not privacy_info["privacy_scope"]:
+            context_layout = (story_node.get("comet_sections") or {}).get("context_layout", {})
+            if context_layout:
+                story = context_layout.get("story", {})
+                comet_sections = story.get("comet_sections", {})
+                metadata = comet_sections.get("metadata", [])
+                for meta_item in metadata:
+                    if isinstance(meta_item, dict) and meta_item.get("__typename") == "CometFeedStoryAudienceStrategy":
+                        story_data = meta_item.get("story", {})
+                        privacy_scope = story_data.get("privacy_scope", {})
+                        if privacy_scope:
+                            privacy_info["privacy_scope"] = privacy_scope.get("description", "")
+                            break
+    except (KeyError, TypeError):
+        pass
+    return privacy_info
+
+
+def normalize_facebook_post(fb_post_data):
+    normalized = {
+        "platform": "facebook.com",
+        "post_id": fb_post_data.get("post_id", ""),
+        "user_display_name": fb_post_data.get("author_name", ""),
+        "user_handle": extract_handle_from_url(fb_post_data.get("author_url", "")),
+        "user_profile_pic_url": fb_post_data.get("author_profile_picture", ""),
+        "post_timestamp": format_timestamp(fb_post_data.get("creation_time")),
+        "post_display_time": fb_post_data.get("creation_time_formatted", ""),
+        "post_url": fb_post_data.get("url", ""),
+        "post_text": fb_post_data.get("message_text", ""),
+        "post_mentions": format_mentions(fb_post_data.get("mentions", [])),
+        "engagement_reply_count": fb_post_data.get("comment_count", 0),
+        "engagement_retweet_count": fb_post_data.get("share_count", 0),
+        "engagement_like_count": fb_post_data.get("reaction_count", 0),
+        "engagement_bookmark_count": 0,
+        "engagement_view_count": 0,
+        "media": format_media(fb_post_data),
+        "media_count": calculate_media_count(fb_post_data),
+        "is_ad": False,
+        "sentiment": None,
+        "categories": None,
+        "tags": None,
+        "analysis_reasoning": None,
+    }
+    return normalized
+
+
+def extract_handle_from_url(author_url: str) -> str:
+    if not author_url:
+        return ""
+    patterns = [
+        r"facebook\.com/([^/?]+)",
+        r"facebook\.com/profile\.php\?id=(\d+)",
+        r"facebook\.com/pages/[^/]+/(\d+)",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, author_url)
+        if match:
+            return match.group(1)
+    return ""
+
+
+def format_timestamp(creation_time) -> str:
+    if not creation_time:
+        return ""
+    try:
+        if isinstance(creation_time, (int, float)):
+            dt = datetime.fromtimestamp(creation_time)
+            return dt.isoformat()
+        elif isinstance(creation_time, str):
+            return creation_time
+    except (ValueError, TypeError):
+        pass
+    return ""
+
+
+def format_mentions(mentions):
+    if not mentions:
+        return ""
+    mention_texts = []
+    for mention in mentions:
+        if isinstance(mention, dict):
+            text = mention.get("text", "")
+            if text and not text.startswith("@"):
+                text = f"@{text}"
+            mention_texts.append(text)
+        elif isinstance(mention, str):
+            if not mention.startswith("@"):
+                mention = f"@{mention}"
+            mention_texts.append(mention)
+    return ",".join(mention_texts)
+
+
+def format_media(fb_post_data: Dict[str, Any]) -> str:
+    media_items = []
+    photos = fb_post_data.get("photos", [])
+    for photo in photos:
+        if isinstance(photo, dict):
+            media_item = {
+                "type": "image",
+                "url": photo.get("image_uri") or photo.get("url", ""),
+                "width": photo.get("width"),
+                "height": photo.get("height"),
+                "id": photo.get("id"),
+                "accessibility_caption": photo.get("accessibility_caption", ""),
+            }
+            media_item = {k: v for k, v in media_item.items() if v is not None}
+            media_items.append(media_item)
+
+    videos = fb_post_data.get("videos", [])
+    for video in videos:
+        if isinstance(video, dict):
+            media_item = {
+                "type": "video",
+                "url": video.get("url", ""),
+                "id": video.get("id"),
+            }
+            media_item = {k: v for k, v in media_item.items() if v is not None}
+            media_items.append(media_item)
+
+    attachments = fb_post_data.get("attachments", [])
+    for attachment in attachments:
+        if isinstance(attachment, dict):
+            attachment_type = attachment.get("type", "")
+            style_list = attachment.get("style_list", [])
+            if "photo" in style_list:
+                media_item = {"type": "image", "attachment_type": attachment_type}
+                media_items.append(media_item)
+            elif "video" in style_list:
+                media_item = {"type": "video", "attachment_type": attachment_type}
+                media_items.append(media_item)
+    return media_items or []
+
+
+def calculate_media_count(fb_post_data):
+    count = 0
+    count += len(fb_post_data.get("photos", []))
+    count += len(fb_post_data.get("videos", []))
+    attachments = fb_post_data.get("attachments", [])
+    for attachment in attachments:
+        if isinstance(attachment, dict):
+            style_list = attachment.get("style_list", [])
+            if any(style in style_list for style in ["photo", "video"]):
+                count += 1
+    return count
+
+
+def normalize_facebook_posts_batch(fb_posts_data):
+    normalized_posts = []
+    for fb_post in fb_posts_data:
+        try:
+            normalized_post = normalize_facebook_post(fb_post)
+            if normalized_post.get("post_id"):
+                normalized_posts.append(normalized_post)
+        except Exception as e:
+            print(f"Error normalizing Facebook post: {e}")
+            continue
+    return normalized_posts
+
+
+if __name__ == "__main__":
+    with open("fb_post_input.json", "r", encoding="utf-8") as f:
+        facebook_data = json.load(f)
+
+    parsed_post = parse_facebook_posts(facebook_data)
+    normalized_post = normalize_facebook_posts_batch(parsed_post)
+    print(normalized_post)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/fb_scraper.py

@@ -0,0 +1,127 @@
+import time
+import json
+from tools.social.browser import create_browser_context
+from tools.social.fb_post_extractor import parse_facebook_posts, normalize_facebook_posts_batch
+from tools.social.x_agent import analyze_posts_sentiment
+from tools.social.db import create_connection, setup_database, check_and_store_post, update_posts_with_analysis
+
+
+def contains_facebook_posts(json_obj):
+    if not isinstance(json_obj, dict):
+        return False
+    try:
+        data = json_obj.get("data", {})
+        viewer = data.get("viewer", {})
+        news_feed = viewer.get("news_feed", {})
+        edges = news_feed.get("edges", [])
+
+        if isinstance(edges, list) and len(edges) > 0:
+            for edge in edges:
+                if isinstance(edge, dict) and "node" in edge:
+                    node = edge["node"]
+                    if isinstance(node, dict) and node.get("__typename") == "Story":
+                        return True
+        return False
+    except (KeyError, TypeError, AttributeError):
+        return False
+
+
+def process_facebook_graphql_response(response_text, seen_post_ids, analysis_queue, queue_post_ids, conn):
+    posts_processed = 0
+    if not response_text:
+        return posts_processed
+    lines = response_text.split("\n")
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            json_obj = json.loads(line)
+            if contains_facebook_posts(json_obj):
+                parsed_posts = parse_facebook_posts(json_obj)
+                if not parsed_posts:
+                    continue
+                normalized_posts = normalize_facebook_posts_batch(parsed_posts)
+                for post_data in normalized_posts:
+                    post_id = post_data.get("post_id")
+                    if not post_id or post_id in seen_post_ids:
+                        continue
+                    seen_post_ids.add(post_id)
+                    posts_processed += 1
+                    needs_analysis = check_and_store_post(conn, post_data)
+                    if needs_analysis and post_data.get("post_text"):
+                        analysis_queue.append(post_data)
+                        queue_post_ids.append(post_id)
+        except json.JSONDecodeError:
+            continue
+        except Exception as e:
+            print(f"Error processing Facebook post: {e}")
+            continue
+
+    return posts_processed
+
+
+def crawl_facebook_feed(target_url="https://facebook.com", db_file="fb_posts.db"):
+    conn = create_connection(db_file)
+    setup_database(conn)
+    seen_post_ids = set()
+    analysis_queue = []
+    queue_post_ids = []
+    post_count = 0
+    batch_size = 5
+    scroll_count = 0
+
+    with create_browser_context() as (browser_context, page):
+
+        def handle_response(response):
+            nonlocal post_count, analysis_queue, queue_post_ids
+            url = response.url
+            if "/api/graphql/" not in url:
+                return
+            try:
+                content_type = response.headers.get("content-type", "")
+                if 'text/html; charset="utf-8"' not in content_type:
+                    return
+                response_text = response.text()
+                posts_found = process_facebook_graphql_response(response_text, seen_post_ids, analysis_queue, queue_post_ids, conn)
+                if posts_found > 0:
+                    post_count += posts_found
+                if len(analysis_queue) >= batch_size:
+                    analysis_batch = analysis_queue[:batch_size]
+                    batch_post_ids = queue_post_ids[:batch_size]
+                    analysis_queue = analysis_queue[batch_size:]
+                    queue_post_ids = queue_post_ids[batch_size:]
+                    try:
+                        analysis_results = analyze_posts_sentiment(analysis_batch)
+                        update_posts_with_analysis(conn, batch_post_ids, analysis_results)
+                    except Exception:
+                        pass
+            except Exception:
+                pass
+
+        page.on("response", handle_response)
+        page.goto(target_url)
+        time.sleep(5)
+
+        def scroll_page():
+            page.evaluate("window.scrollBy(0, window.innerHeight)")
+
+        try:
+            while True:
+                scroll_page()
+                time.sleep(3)
+                scroll_count += 1
+                if scroll_count >= 50:
+                    break
+        except KeyboardInterrupt:
+            pass
+
+        if analysis_queue:
+            try:
+                analysis_results = analyze_posts_sentiment(analysis_queue)
+                update_posts_with_analysis(conn, queue_post_ids, analysis_results)
+            except Exception:
+                pass
+
+        conn.close()
+        return post_count

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/session_setup.py

@@ -0,0 +1,9 @@
+from tools.social.browser import setup_session_multi
+
+target_sites = [
+    "https://x.com",
+    "https://facebook.com"
+]
+
+if __name__ == "__main__":
+    setup_session_multi(target_sites)

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/x_agent.py

@@ -0,0 +1,103 @@
+from typing import List
+from pydantic import BaseModel, Field
+from enum import Enum
+from textwrap import dedent
+from agno.agent import Agent
+from agno.models.openai import OpenAIChat
+from dotenv import load_dotenv
+import uuid
+
+
+load_dotenv()
+
+
+class SentimentType(str, Enum):
+    POSITIVE = "positive"
+    NEGATIVE = "negative"
+    NEUTRAL = "neutral"
+    CRITICAL = "critical"
+
+
+class Category(str, Enum):
+    POLITICS = "politics"
+    TECHNOLOGY = "technology"
+    ENTERTAINMENT = "entertainment"
+    SPORTS = "sports"
+    BUSINESS = "business"
+    HEALTH = "health"
+    SCIENCE = "science"
+    EDUCATION = "education"
+    ENVIRONMENT = "environment"
+    SOCIAL_ISSUES = "social_issues"
+    PERSONAL = "personal"
+    NEWS = "news"
+    OTHER = "other"
+
+
+class AnalyzedPost(BaseModel):
+    post_id: str = Field(..., description="The unique identifier for the post")
+    sentiment: SentimentType = Field(..., description="The sentiment of the post")
+    categories: List[Category] = Field(..., description="List of categories that best describe this post")
+    tags: List[str] = Field(..., description="List of relevant tags or keywords extracted from the post")
+    reasoning: str = Field(..., description="Brief explanation of why these sentiments and categories were assigned")
+
+
+class AnalysisResponse(BaseModel):
+    analyzed_posts: List[AnalyzedPost] = Field(..., description="List of analyzed posts with sentiment and categorization")
+
+
+SENTIMENT_AGENT_DESCRIPTION = "Expert sentiment analyzer and content categorizer for social media posts."
+SENTIMENT_AGENT_INSTRUCTIONS = dedent("""
+    You are an expert sentiment analyzer and content categorizer for social media posts.
+    
+    You will receive a batch of social media posts, each with a unique post_id. Your task is to analyze each post and return:
+    
+    1. The EXACT same post_id that was provided (this is critical for matching)
+    2. The sentiment (positive, negative, neutral, or critical)
+    3. Relevant categories from the predefined list
+    4. Generated tags or keywords
+    5. Brief reasoning for your analysis
+    
+    Guidelines for sentiment classification:
+    - POSITIVE: Expresses joy, gratitude, excitement, optimism, or other positive emotions
+    - NEGATIVE: Expresses sadness, anger, disappointment, fear, or other negative emotions
+    - NEUTRAL: Factual, objective, or balanced without strong emotional tone
+    - CRITICAL: Contains criticism, skepticism, or questioning, but in a constructive or analytical way
+    
+    IMPORTANT: You MUST maintain the exact post_id provided for each post in your analysis.
+    IMPORTANT: Categories must be chosen ONLY from the predefined list.
+    IMPORTANT: Return analysis for ALL posts provided in the input.
+    IMPORTANT: You can add news tag if you think that could you be used to write news articles.
+""")
+
+
+def analyze_posts_sentiment(posts_data):
+    session_id = str(uuid.uuid4())
+    analysis_agent = Agent(
+        model=OpenAIChat(id="gpt-4o"),
+        instructions=SENTIMENT_AGENT_INSTRUCTIONS,
+        description=SENTIMENT_AGENT_DESCRIPTION,
+        use_json_mode=True,
+        response_model=AnalysisResponse,
+        session_id=session_id,
+    )
+    posts_prompt = "Analyze the sentiment and categorize the following social media posts:\n\n"
+    valid_posts = []
+    for post in posts_data:
+        post_text = post.get("post_text", "")
+        post_id = post.get("post_id", "")
+        if post_text and post_id:
+            valid_posts.append(post)
+            posts_prompt += f"POST (ID: {post_id}):\n{post_text}\n\n"
+    if not valid_posts:
+        return []
+    response = analysis_agent.run(posts_prompt, session_id=session_id)
+    analysis_results = response.to_dict()["content"]["analyzed_posts"]
+    validated_results = []
+    valid_post_ids = {post.get("post_id") for post in valid_posts}
+    for analysis in analysis_results:
+        if analysis.get("post_id") in valid_post_ids:
+            validated_results.append(analysis)
+        else:
+            print(f"Warning: Analysis returned with invalid post_id: {analysis.get('post_id')}")
+    return validated_results

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/x_post_extractor.py

@@ -0,0 +1,112 @@
+import os
+from bs4 import BeautifulSoup
+import re
+import json
+
+
+def check_ad(soup):
+    is_ad = False
+    ad_label = soup.find(
+        lambda tag: tag.name and tag.text and tag.text.strip() == "Ad" and "r-bcqeeo" in tag.get("class", []) if hasattr(tag, "get") else False
+    )
+    if ad_label:
+        is_ad = True
+    username_element = soup.select_one("div[data-testid='User-Name'] a[role='link'] span")
+    if username_element and username_element.text.strip() in [
+        "Premium",
+        "Twitter",
+        "X",
+    ]:
+        is_ad = True
+    handle_element = soup.select_one("div[data-testid='User-Name'] div[dir='ltr'] span")
+    if handle_element and "@premium" in handle_element.text:
+        is_ad = True
+    ad_tracking_links = soup.select("a[href*='referring_page=ad_'], a[href*='twclid=']")
+    if ad_tracking_links:
+        is_ad = True
+    return is_ad
+
+
+def x_post_extractor(html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    data = {"platform": "x.com"}
+    data["media"] = []
+    data["is_ad"] = check_ad(soup)
+    user_element = soup.find("div", {"data-testid": "User-Name"})
+    if user_element:
+        display_name_element = user_element.find(lambda tag: tag.name and (tag.name == "span" or tag.name == "div") and "Henrik" in tag.text)
+        if display_name_element:
+            data["user_display_name"] = display_name_element.text.strip()
+        username_element = user_element.find(lambda tag: tag.name == "a" and "@" in tag.text)
+        if username_element:
+            data["user_handle"] = username_element.text.strip()
+        profile_pic = soup.select_one("[data-testid='UserAvatar-Container-HenrikTaro'] img")
+        if profile_pic and profile_pic.has_attr("src"):
+            data["user_profile_pic_url"] = profile_pic["src"]
+
+    time_element = soup.find("time")
+    if time_element:
+        if time_element.has_attr("datetime"):
+            data["post_timestamp"] = time_element["datetime"]
+        data["post_display_time"] = time_element.text.strip()
+
+    post_link = soup.select_one("a[href*='/status/']")
+    if post_link and post_link.has_attr("href"):
+        status_url = post_link["href"]
+        post_id_match = re.search(r"/status/(\d+)", status_url)
+        if post_id_match:
+            data["post_id"] = post_id_match.group(1)
+            data["post_url"] = f"https://twitter.com{status_url}"
+
+    text_element = soup.find("div", {"data-testid": "tweetText"})
+    if text_element:
+        data["post_text"] = text_element.get_text(strip=True)
+        mentions = []
+        mention_elements = text_element.select("a[href^='/'][role='link']")
+        for mention in mention_elements:
+            mention_text = mention.text.strip()
+            if mention_text.startswith("@"):
+                mentions.append(mention_text)
+
+        if mentions:
+            data["post_mentions"] = ",".join(mentions)
+
+    reply_button = soup.find("button", {"data-testid": "reply"})
+    if reply_button:
+        count_span = reply_button.select_one("span[data-testid='app-text-transition-container'] span span")
+        if count_span:
+            data["engagement_reply_count"] = count_span.text.strip()
+
+    retweet_button = soup.find("button", {"data-testid": "retweet"})
+    if retweet_button:
+        count_span = retweet_button.select_one("span[data-testid='app-text-transition-container'] span span")
+        if count_span:
+            data["engagement_retweet_count"] = count_span.text.strip()
+
+    like_button = soup.find("button", {"data-testid": "like"})
+    if like_button:
+        count_span = like_button.select_one("span[data-testid='app-text-transition-container'] span span")
+        if count_span:
+            data["engagement_like_count"] = count_span.text.strip()
+
+    bookmark_button = soup.find("button", {"data-testid": "bookmark"})
+    if bookmark_button:
+        count_span = bookmark_button.select_one("span[data-testid='app-text-transition-container'] span span")
+        if count_span:
+            data["engagement_bookmark_count"] = count_span.text.strip()
+
+    views_element = soup.select_one("a[href*='/analytics']")
+    if views_element:
+        views_span = views_element.select_one("span[data-testid='app-text-transition-container'] span span")
+        if views_span:
+            data["engagement_view_count"] = views_span.text.strip()
+
+    media_elements = soup.find_all("div", {"data-testid": "tweetPhoto"})
+    for media in media_elements:
+        img = media.find("img")
+        if img and img.has_attr("src"):
+            data["media"].append({"type": "image", "url": img["src"]})
+
+    data["media_count"] = len(data["media"])
+
+    return data

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social/x_scraper.py

@@ -0,0 +1,88 @@
+import time
+from tools.social.browser import create_browser_context
+from tools.social.x_post_extractor import x_post_extractor
+from tools.social.x_agent import analyze_posts_sentiment
+from tools.social.db import create_connection, setup_database, check_and_store_post, update_posts_with_analysis
+
+
+def crawl_x_profile(profile_url, db_file="x_posts.db"):
+    if not profile_url.startswith("http"):
+        profile_url = f"https://x.com/{profile_url}"
+
+    conn = create_connection(db_file)
+    setup_database(conn)
+    seen_post_ids = set()
+    analysis_queue = []
+    queue_post_ids = []
+    post_count = 0
+    batch_size = 5
+    scroll_count = 0
+
+    with create_browser_context() as (browser_context, page):
+        page.goto(profile_url)
+        time.sleep(5)
+
+        try:
+            while True:
+                tweet_articles = page.query_selector_all('article[role="article"]')
+                for article in tweet_articles:
+                    article_id = article.evaluate('(element) => element.getAttribute("id")')
+                    if article_id in seen_post_ids:
+                        continue
+
+                    show_more = article.query_selector('button[data-testid="tweet-text-show-more-link"]')
+                    if show_more:
+                        try:
+                            show_more.click()
+                            time.sleep(1)
+                        except Exception as e:
+                            pass
+
+                    tweet_html = article.evaluate("(element) => element.outerHTML")
+                    post_data = x_post_extractor(tweet_html)
+
+                    post_id = post_data.get("post_id")
+                    if not post_id or post_data.get("is_ad", False):
+                        continue
+
+                    seen_post_ids.add(post_id)
+                    post_count += 1
+
+                    needs_analysis = check_and_store_post(conn, post_data)
+
+                    if needs_analysis and post_data.get("post_text"):
+                        analysis_queue.append(post_data)
+                        queue_post_ids.append(post_id)
+
+                if len(analysis_queue) >= batch_size:
+                    analysis_batch = analysis_queue[:batch_size]
+                    batch_post_ids = queue_post_ids[:batch_size]
+
+                    analysis_queue = analysis_queue[batch_size:]
+                    queue_post_ids = queue_post_ids[batch_size:]
+
+                    try:
+                        analysis_results = analyze_posts_sentiment(analysis_batch)
+
+                        update_posts_with_analysis(conn, batch_post_ids, analysis_results)
+                    except Exception:
+                        pass
+
+                page.evaluate("window.scrollBy(0, 800)")
+                time.sleep(3)
+
+                scroll_count += 1
+                if scroll_count >= 30:
+                    break
+
+        except KeyboardInterrupt:
+            if analysis_queue:
+                try:
+                    analysis_results = analyze_posts_sentiment(analysis_queue)
+                    update_posts_with_analysis(conn, queue_post_ids, analysis_results)
+                except Exception:
+                    pass
+
+            conn.close()
+            return post_count
+        return post_count

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/social_media_search.py

@@ -0,0 +1,142 @@
+import sqlite3
+import json
+from datetime import datetime, timedelta
+from contextlib import contextmanager
+from agno.agent import Agent
+from db.config import get_db_path
+
+
+@contextmanager
+def get_social_media_db():
+    db_path = get_db_path("social_media_db")
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+
+def social_media_search(agent: Agent, topic: str, limit: int = 10) -> str:
+    """
+    Search social media database for the given topic on social media post texts, so topic needs to be extact keyword or phrase.
+    Returns minimal fields required for source selection.
+
+    Args:
+        agent: The agent instance
+        topic: The search topic
+        limit: Maximum number of results to return (default: 10)
+
+    Returns:
+        Search results matching agent's expected format
+    """
+    print(f"Social Media News Search: {topic}")
+    try:
+        days_back: int = 7
+        date_from = (datetime.now() - timedelta(days=days_back)).isoformat()
+        with get_social_media_db() as conn:
+            cursor = conn.cursor()
+            sql_query = """
+            SELECT 
+                post_id,
+                user_display_name,
+                post_timestamp,
+                post_url,
+                post_text,
+                platform
+            FROM posts 
+            WHERE 
+                categories LIKE '%"news"%' 
+                AND sentiment = 'positive'
+                AND datetime(post_timestamp) >= datetime(?)
+                AND (post_text LIKE ? OR user_display_name LIKE ?)
+            ORDER BY datetime(post_timestamp) DESC
+            LIMIT ?
+            """
+            search_term = f"%{topic}%"
+            cursor.execute(sql_query, (date_from, search_term, search_term, limit))
+            rows = cursor.fetchall()
+            if not rows:
+                return f"No positive news posts found for '{topic}' in the last {days_back} days."
+            results = []
+            for row in rows:
+                result = {
+                    "id": f"social_{row['post_id']}",
+                    "url": row["post_url"] or f"https://{row['platform']}/post/{row['post_id']}",
+                    "published_date": row["post_timestamp"],
+                    "description": row["post_text"][:200] + "..." if len(row["post_text"]) > 200 else row["post_text"],
+                    "source_id": "social_media_db",
+                    "source_name": f"{row['platform'].title()}",
+                    "categories": ["news"],
+                    "is_scrapping_required": False,
+                }
+                results.append(result)
+            return f"Found {len(results)} positive news posts. {json.dumps({'results': results}, indent=2)}"
+    except Exception as e:
+        return f"Error searching social media database: {str(e)}"
+
+
+def social_media_trending_search(agent: Agent, limit: int = 10) -> str:
+    """
+    Get trending positive news posts from social media.
+    Returns trending news posts in standard results format.
+
+
+    Args:
+        agent: The agent instance
+        limit: Maximum number of trending results (default: 10)
+
+    Returns:
+        Trending positive news posts
+    """
+    print(f"Social Media Trending Search: {limit}")
+    days_back = 3
+    try:
+        date_from = (datetime.now() - timedelta(days=days_back)).isoformat()
+        with get_social_media_db() as conn:
+            cursor = conn.cursor()
+            trending_sql = """
+            SELECT 
+                post_id,
+                user_display_name,
+                post_timestamp,
+                post_url,
+                post_text,
+                platform
+            FROM posts
+            WHERE 
+                categories LIKE '%"news"%'
+                AND sentiment = 'positive'
+                AND datetime(post_timestamp) >= datetime(?)
+            ORDER BY 
+                (COALESCE(engagement_like_count, 0) + 
+                 COALESCE(engagement_retweet_count, 0) + 
+                 COALESCE(engagement_reply_count, 0)) DESC,
+                datetime(post_timestamp) DESC
+            LIMIT ?
+            """
+            cursor.execute(trending_sql, (date_from, limit))
+            rows = cursor.fetchall()
+            if not rows:
+                return f"No trending positive news found in the last {days_back} days."
+            results = []
+            for row in rows:
+                result = {
+                    "id": f"social_{row['post_id']}",
+                    "url": row["post_url"] or f"https://{row['platform']}/post/{row['post_id']}",
+                    "published_date": row["post_timestamp"],
+                    "description": row["post_text"],
+                    "source_id": "social_media_trending",
+                    "source_name": f"{row['platform'].title()} Trending",
+                    "categories": ["news"],
+                    "is_scrapping_required": False,
+                }
+                results.append(result)
+            return f"Found {len(results)} trending positive news posts. {json.dumps({'results': results}, indent=2)}"
+    except Exception as e:
+        return f"Error getting trending news: {str(e)}"
+
+
+if __name__ == "__main__":
+    print("here...")
+    print(social_media_trending_search(None, 5))

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/ui_manager.py

@@ -0,0 +1,33 @@
+from agno.agent import Agent
+from dotenv import load_dotenv
+from db.agent_config_v2 import TOGGLE_UI_STATES
+
+load_dotenv()
+
+
+def ui_manager_run(
+    agent: Agent,
+    state_type: str,
+    active: bool,
+) -> str:
+    """
+    UI Manager that takes the state_type and active as input and updates the sessions UI state.
+    Args:
+        agent: The agent instance
+        state_type: The state type to update
+        active: The active state
+    Returns:
+        Response status
+    """
+    from services.internal_session_service import SessionService
+
+    session_id = agent.session_id
+    session = SessionService.get_session(session_id)
+    current_state = session["state"]
+    current_state[state_type] = active
+    all_ui_states = TOGGLE_UI_STATES
+    for ui_state in all_ui_states:
+        if ui_state != state_type:
+            current_state[ui_state] = False
+    SessionService.save_session(session_id, current_state)
+    return f"Updated {state_type} to {active}{' and all other UI states to False' if all_ui_states else ''}."

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/user_source_selection.py

@@ -0,0 +1,30 @@
+from agno.agent import Agent
+from dotenv import load_dotenv
+from typing import List
+
+load_dotenv()
+
+
+def user_source_selection_run(
+    agent: Agent,
+    selected_sources: List[int],
+) -> str:
+    """
+    User Source Selection that takes the selected sources indices as input and updates the final confirmed sources.
+    Args:
+        agent: The agent instance
+        selected_sources: The selected sources indices
+    Returns:
+        Response status
+    """
+    from services.internal_session_service import SessionService
+    session_id = agent.session_id
+    session = SessionService.get_session(session_id)
+    session_state = session["state"]
+    for i, src in enumerate(session_state["search_results"]):
+        if (i+1) in selected_sources:
+            src["confirmed"] = True
+        else:
+            src["confirmed"] = False
+    SessionService.save_session(session_id, session_state)
+    return f"Updated selected sources to {selected_sources}."

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/web_search.py

@@ -0,0 +1,99 @@
+import os
+import asyncio
+from typing import List
+from pydantic import BaseModel, Field
+from browser_use import Agent as BrowserAgent, Controller, BrowserSession, BrowserProfile
+from langchain_openai import ChatOpenAI
+from dotenv import load_dotenv
+from agno.agent import Agent
+
+
+import json
+
+load_dotenv()
+
+BROWSER_AGENT_MODEL = "gpt-4o"
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+MAX_STEPS = 15
+MAX_ACTIONS_PER_STEP = 5
+USER_DATA_DIR = "browsers/playwright_persistent_profile_web"
+
+
+class WebSearchResult(BaseModel):
+    title: str = Field(..., description="The title of the search result")
+    url: str = Field(..., description="The URL of the search result")
+    content: str = Field(..., description="Full content of the source, be elaborate at the same time be concise")
+
+
+class WebSearchResults(BaseModel):
+    results: List[WebSearchResult] = Field(..., description="List of search results")
+
+
+def run_browser_search(agent: Agent, instruction: str) -> str:
+    """
+    Run browser search to get the results.
+    Args:
+        agent: The agent instance
+        instruction: The instruction to run the browser search, give detailed step by step prompt on how to collect the information.
+    Returns:
+        The results of the browser search
+    """
+    print("Browser Search Input:", instruction)
+    try:
+        controller = Controller(output_model=WebSearchResults)
+        session_id = agent.session_id
+        recordings_dir = os.path.join("podcasts/recordings", session_id)
+        os.makedirs(recordings_dir, exist_ok=True)
+
+        headless = True
+        browser_profile = BrowserProfile(
+            user_data_dir=USER_DATA_DIR, headless=headless, viewport={"width": 1280, "height": 800}, record_video_dir=recordings_dir,
+            downloads_path="podcasts/browseruse_downloads",
+        )
+
+        browser_session = BrowserSession(
+            browser_profile=browser_profile,
+            headless=headless,
+            disable_security=False,
+            record_video=True,
+            record_video_dir=recordings_dir,
+        )
+
+        browser_agent = BrowserAgent(
+            browser_session=browser_session,
+            task=instruction,
+            llm=ChatOpenAI(model=BROWSER_AGENT_MODEL, api_key=os.getenv("OPENAI_API_KEY")),
+            use_vision=False,
+            controller=controller,
+            max_actions_per_step=MAX_ACTIONS_PER_STEP,
+        )
+        try:
+            try:
+                loop = asyncio.get_running_loop()
+            except RuntimeError:
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+            history = loop.run_until_complete(browser_agent.run(max_steps=MAX_STEPS))
+        except RuntimeError:
+            history = asyncio.run(browser_agent.run(max_steps=MAX_STEPS))
+        result = history.final_result()
+        if result:
+            parsed: WebSearchResults = WebSearchResults.model_validate_json(result)
+            results_list = [
+                {"title": post.title, "url": post.url, "description": post.content, "is_scrapping_required": False} for post in parsed.results
+            ]
+            return f"is_scrapping_required: False, results: {json.dumps(results_list)}"
+        else:
+            return "No results found, something went wrong with browser based search."
+    except Exception as e:
+        return f"Error running browser search: {e}"
+    finally:
+        pass
+
+
+def main():
+    return run_browser_search(agent={"session_id": "123"}, instruction="gene therapy")
+
+
+if __name__ == "__main__":
+    print(main())

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/tools/wikipedia_search.py

@@ -0,0 +1,62 @@
+from agno.agent import Agent
+
+
+def wikipedia_search(agent: Agent, query: str, srlimit: int = 5) -> str:
+    """
+    Search Wikipedia for articles using Wikipedia API.
+    Returns only links and short summaries without fetching full content.
+
+    Args:
+        agent: The agent instance
+        query: The search query
+        srlimit: The number of results to return
+
+    Returns:
+        A formatted string response with the search results (link and gist only)
+    """
+    import requests
+    import html
+    import json
+    
+    print("Wikipedia Search Input:", query)
+    try:
+        search_url = "https://en.wikipedia.org/w/api.php"
+        search_params = {
+            "action": "query",
+            "format": "json",
+            "list": "search",
+            "srsearch": query,
+            "srlimit": srlimit,
+            "utf8": 1,
+        }
+        search_response = requests.get(search_url, params=search_params)
+        search_data = search_response.json()
+        if (
+            "query" not in search_data
+            or "search" not in search_data["query"]
+            or not search_data["query"]["search"]
+        ):
+            print(f"No Wikipedia results found for query: {query}")
+            return "No relevant Wikipedia articles found for this topic."
+        results = []
+        for item in search_data["query"]["search"]:
+            title = item["title"]
+            snippet = html.unescape(
+                item["snippet"]
+                .replace('<span class="searchmatch">', "")
+                .replace("</span>", "")
+            )
+            url = f"https://en.wikipedia.org/wiki/{title.replace(' ', '_')}"
+            result = {
+                "title": title,
+                "url": url,
+                "gist": snippet,
+                "source": "Wikipedia",
+            }
+            results.append(result)
+        if not results:
+            return "No Wikipedia search results found."
+        return f"for all results is_scrapping_required: True, results: {json.dumps(results, ensure_ascii=False, indent=2)}"
+    except Exception as e:
+        print(f"Error during Wikipedia search: {str(e)}")
+        return f"Error in Wikipedia search: {str(e)}"

advanced_ai_agents/multi_agent_apps/ai_news_and_podcast_agents/beifong/utils/crawl_url.py

@@ -0,0 +1,70 @@
+import requests
+from bs4 import BeautifulSoup
+import random
+from typing import Dict, List, TypedDict
+
+
+class MetadataDict(TypedDict):
+    title: str
+    description: str
+    og: Dict[str, str]
+    twitter: Dict[str, str]
+    other_meta: Dict[str, str]
+
+
+class WebData(TypedDict):
+    raw_html: str
+    metadata: MetadataDict
+
+
+USER_AGENTS: List[str] = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
+]
+HEADERS: Dict[str, str] = {
+    "User-Agent": random.choice(USER_AGENTS),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+def extract_meta_tags(soup: BeautifulSoup) -> MetadataDict:
+    metadata: MetadataDict = {
+        "title": "",
+        "description": "",
+        "og": {},
+        "twitter": {},
+        "other_meta": {},
+    }
+    title_tag = soup.find("title")
+    if title_tag:
+        metadata["title"] = title_tag.text.strip()
+    for meta in soup.find_all("meta"):
+        name = meta.get("name", "").lower()
+        prop = meta.get("property", "").lower()
+        content = meta.get("content", "")
+        if prop.startswith("ogg:"):
+            og_key = prop[3:]
+            metadata["og"][og_key] = content
+        elif prop.startswith("twitter:") or name.startswith("twitter:"):
+            twitter_key = prop[8:] if prop.startswith("twitter:") else name[8:]
+            metadata["twitter"][twitter_key] = content
+        elif name in ["description", "keywords", "author", "robots", "viewport"]:
+            metadata["other_meta"][name] = content
+            if name == "description":
+                metadata["description"] = content
+    return metadata
+
+
+def get_web_data(url: str) -> WebData:
+    HEADERS["User-Agent"] = random.choice(USER_AGENTS)
+    response = requests.get(url, headers=HEADERS, timeout=10)
+    soup = BeautifulSoup(response.text, "html.parser")
+    metadata = extract_meta_tags(soup)
+    body = str(soup.find("body"))
+    return {"raw_html": body, "metadata": metadata}