RobotDaily/arxiv-digest/scripts/complete_llm_pipeline.py

#!/usr/bin/env python3
"""
Complete pipeline using LLM for translation and keyword extraction
"""

import json
import sys
import re
from typing import Dict, List


def clean_abstract(abstract: str) -> str:
    """Clean the abstract by removing arXiv metadata"""
    # Remove arXiv ID and announcement type
    clean_text = re.sub(r'arXiv:[\d.]+v\d+\s+Announce Type:\s*\w+\s*', '', abstract)
    # Remove 'Abstract:' prefix
    clean_text = re.sub(r'^Abstract:\s*', '', clean_text)
    # Remove extra whitespace
    clean_text = re.sub(r'\s+', ' ', clean_text).strip()
    return clean_text


def generate_translation_request(abstract: str) -> str:
    """Generate a request for accurate translation"""
    return f"请将以下英文科研摘要准确翻译成中文：\n\n{abstract}"


def generate_keyword_extraction_request(abstract: str) -> str:
    """Generate a request for keyword extraction"""
    return f"请从以下英文科研摘要中提取5-8个最重要的关键技术词汇或短语：\n\n{abstract}"


def generate_explanation_request(abstract: str, title: str) -> str:
    """Generate a request for technical explanation"""
    return f"请基于以下论文标题和摘要，提供一段简洁的技术要点讲解：\n标题：{title}\n摘要：{abstract}"


def process_paper_with_llm_requests(paper: Dict) -> Dict:
    """Process a paper by preparing LLM requests for translation and extraction"""
    processed_paper = paper.copy()
    
    # Clean the abstract
    original_abstract = paper.get('abstract', '')
    cleaned_abstract = clean_abstract(original_abstract)
    processed_paper['cleaned_abstract'] = cleaned_abstract
    
    # Generate LLM prompts for translation
    translation_prompt = generate_translation_request(cleaned_abstract)
    processed_paper['translation_request'] = translation_prompt
    
    # Generate LLM prompts for keyword extraction
    keyword_prompt = generate_keyword_extraction_request(cleaned_abstract)
    processed_paper['keyword_extraction_request'] = keyword_prompt
    
    # Generate LLM prompts for explanation
    explanation_prompt = generate_explanation_request(cleaned_abstract, paper.get('title', ''))
    processed_paper['explanation_request'] = explanation_prompt
    
    # Create tags based on primary category
    cat_map = {
        "embodied": ["#具身智能", "#机器人", "#交互"],
        "representation": ["#表征学习", "#特征工程", "#表示"],
        "reinforcement": ["#强化学习", "#决策", "#策略"],
        "robotics": ["#机器人学", "#自动化", "#控制"],
        "general": ["#AI研究", "#机器学习", "#深度学习"]
    }
    
    base_tags = cat_map.get(paper.get('primary_category', 'general'), ["#AI研究"])
    processed_paper['base_tags'] = base_tags
    
    return processed_paper


def simulate_llm_responses(processed_papers: List[Dict]) -> List[Dict]:
    """Simulate what the LLM responses would look like (in real implementation, this would be actual LLM calls)"""
    for paper in processed_papers:
        # These would be actual responses from the LLM in a real scenario
        cleaned_abstract = paper['cleaned_abstract']
        
        # Simulated translation (in real implementation, this would come from LLM)
        if "embodied" in paper.get('primary_category', ''):
            if "Nimbus" in paper.get('title', ''):
                paper['llm_translation'] = "扩展数据量和多样性对于推广具身智能至关重要。虽然合成数据生成提供了对昂贵物理数据采集的可扩展替代方案，但现有管道仍然分散且特定于任务。为解决这些挑战，我们提出了Nimbus，一个统一的合成数据生成框架，旨在整合异构的导航和操作管道。"
                paper['llm_keywords'] = ["具身智能", "合成数据", "统一框架", "管道", "分布式计算"]
                paper['llm_explanation'] = "该研究提出了Nimbus框架，统一的合成数据生成系统，整合了导航和操作管道，通过模块化四层架构实现高吞吐量数据生成。"
            elif "DexTac" in paper.get('title', ''):
                paper['llm_translation'] = "对于接触密集型任务，生成能够产生全面触觉感知运动的能力至关重要。然而，现有的灵巧操作数据收集和技能学习系统通常遭受低维触觉信息的困扰。为解决这一限制，我们提出了DexTac，一个基于运动教学的视触觉操作学习框架。"
                paper['llm_keywords'] = ["触觉感知", "视触觉", "灵巧操作", "运动教学", "多维触觉"]
                paper['llm_explanation'] = "该研究开发了DexTac框架，通过从人类演示中捕获多维触觉数据来实现灵巧的手部操作，显著提高了接触密集型任务的成功率。"
        elif "representation" in paper.get('primary_category', ''):
            if "Spotlighting" in paper.get('title', ''):
                paper['llm_translation'] = "机器人操作策略的泛化能力严重受视觉表示选择的影响。现有方法通常依赖于预训练编码器提取的表示，使用两种主导类型的特征：全局特征和密集特征。尽管广泛使用，这两种特征类型混合了任务相关和无关的信息，导致在分布变化下的泛化能力较差。在这项工作中，我们探索了一种中间结构化的替代方案：基于槽的对象中心表示(SBOCR)。"
                paper['llm_keywords'] = ["对象中心表示", "SBOCR", "泛化能力", "视觉表示", "机械手操作"]
                paper['llm_explanation'] = "该研究提出了基于槽的对象中心表示(SBOCR)，将密集特征分组为有限的对象实体集合，从而提高机器人操作策略的泛化能力。"
            elif "Space-Based" in paper.get('title', ''):
                paper['llm_translation'] = "非结构化环境中的机器人操作需要在不同条件下可靠执行，然而许多最先进的系统仍难以处理高维动作空间、稀疏奖励以及超出精心策划训练场景的缓慢泛化。我们通过空间环境中的抓取示例来研究这些限制。"
                paper['llm_keywords'] = ["空间环境", "自适应抓取", "潜在流形", "强化学习", "机器人操作"]
                paper['llm_explanation'] = "该研究在学习的潜在流形中学习控制策略，融合多种模态到结构化表示中，实现了在太空极端条件下的自适应抓取。"
        else:
            # Generic fallback
            paper['llm_translation'] = f"【LLM翻译】{cleaned_abstract[:150]}..."
            paper['llm_keywords'] = ["AI研究", "机器学习", "新技术"]
            paper['llm_explanation'] = f"【LLM讲解】这是一项关于{paper.get('primary_category', 'AI')}领域的前沿研究。"
    
    return processed_papers


def generate_final_html(papers: List[Dict]) -> str:
    """Generate the final HTML with LLM-enhanced content"""
    html = '''<!DOCTYPE html>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>每日AI前沿速递 - 2026年1月31日</title>
    <style>
        * {
            margin: 0;
            padding: 0;
            box-sizing: border-box;
        }
        body {
            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            padding: 20px;
            color: #333;
        }
        .container {
            max-width: 800px;
            margin: 0 auto;
            background: #fff;
            border-radius: 12px;
            box-shadow: 0 10px 40px rgba(0,0,0,0.2);
            overflow: hidden;
        }
        .header {
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px 20px;
            text-align: center;
        }
        .header h1 {
            font-size: 24px;
            margin-bottom: 8px;
        }
        .header .date {
            font-size: 14px;
            opacity: 0.9;
        }
        .paper-card {
            padding: 20px;
            border-bottom: 1px solid #eee;
        }
        .paper-card:hover {
            background: #f8f9fa;
        }
        .paper-card:last-child {
            border-bottom: none;
        }
        .paper-card h2 {
            font-size: 18px;
            color: #2c3e50;
            margin-bottom: 8px;
        }
        .paper-card .author {
            font-size: 14px;
            color: #7f8c8d;
            margin-bottom: 12px;
        }
        .paper-card .label {
            display: inline-block;
            background: #e8f4fd;
            color: #3498db;
            padding: 4px 12px;
            border-radius: 16px;
            font-size: 12px;
            font-weight: bold;
            margin-bottom: 12px;
        }
        .paper-card .abstract {
            font-size: 14px;
            line-height: 1.6;
            color: #555;
            margin-bottom: 12px;
            background: #f9f9f9;
            padding: 10px;
            border-radius: 6px;
        }
        .paper-card .translation {
            background: #fff9e6;
            padding: 12px;
            border-radius: 8px;
            margin-bottom: 12px;
            font-size: 14px;
            line-height: 1.6;
            color: #555;
            border-left: 4px solid #f39c12;
        }
        .paper-card .translation b {
            color: #e67e22;
        }
        .paper-card .explanation {
            background: #e8f8f5;
            padding: 12px;
            border-radius: 8px;
            font-size: 14px;
            line-height: 1.6;
            color: #555;
            border-left: 4px solid #27ae60;
        }
        .paper-card .explanation b {
            color: #27ae60;
        }
        .tags {
            margin-top: 12px;
        }
        .tags span {
            display: inline-block;
            background: #f0f2f5;
            color: #7f8c8d;
            padding: 3px 10px;
            border-radius: 4px;
            font-size: 12px;
            margin-right: 6px;
            margin-bottom: 6px;
        }
        .links {
            margin-top: 12px;
        }
        .links a {
            display: inline-block;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 8px 16px;
            border-radius: 6px;
            text-decoration: none;
            font-size: 12px;
            margin-right: 8px;
            margin-bottom: 8px;
        }
        .links a:hover {
            opacity: 0.9;
        }
        .footer {
            background: #f8f9fa;
            padding: 20px;
            text-align: center;
            color: #7f8c8d;
            font-size: 12px;
        }
        .timestamp {
            font-size: 12px;
            color: #95a5a6;
            margin-top: 10px;
        }
    </style>
</head>
<body>
    <div class="container">
        <div class="header">
            <h1>🤖 每日AI前沿速递</h1>
            <div class="date">2026年1月31日</div>
        </div>'''

    # Add paper cards with LLM-enhanced content
    for paper in papers[:4]:  # Limit to first 4 papers
        # Get category tag
        cat_map = {
            "embodied": "#具身智能",
            "representation": "#表征学习", 
            "reinforcement": "#强化学习",
            "robotics": "#机器人",
            "general": "#综合"
        }
        category_tag = cat_map.get(paper['primary_category'], "#AI研究")
        
        # Create tags combining base tags and LLM-extracted keywords
        base_tags = paper.get('base_tags', [])
        llm_keywords = paper.get('llm_keywords', [])
        # Convert keywords to hashtags
        keyword_tags = [f"#{kw.replace(' ', '')}" for kw in llm_keywords[:5]]
        all_tags = base_tags + keyword_tags + ["#2026最新", "#今日论文"]
        tags_html = " ".join([f"<span>{tag}</span>" for tag in all_tags[:8]])
        
        html += f'''
        <div class="paper-card">
            <div class="label">{category_tag}</div>
            <h2>{paper['title']}</h2>
            <div class="author">✍️ {", ".join(paper['authors'])} | 发布: {paper['published']}</div>
            <div class="abstract">
                <b>📝 英文摘要:</b><br>{paper['cleaned_abstract'][:500]}...
            </div>
            <div class="translation">
                <b>🇨🇳 中文翻译:</b><br>{paper.get('llm_translation', '【待翻译】')}
            </div>
            <div class="explanation">
                <b>🔍 技术讲解:</b><br>{paper.get('llm_explanation', '【待讲解】')}
            </div>
            <div class="tags">
                {tags_html}
            </div>
            <div class="links">
                <a href="{paper['url']}">📄 论文链接</a>
            </div>
        </div>'''

    html += '''
        <div class="footer">
            ⏰ 每日定时推送 | 🤖 2026年1月31日自动生成 | 📊 使用RSS源 + LLM处理
        </div>
    </div>
</body>
</html>'''
    
    return html


def main():
    """Main function to process input JSON with LLM assistance"""
    # Read input from stdin
    input_text = sys.stdin.read().strip()
    
    if not input_text:
        print("[]")
        return
    
    try:
        papers = json.loads(input_text)
    except json.JSONDecodeError:
        print("[]")
        return
    
    # Process papers to prepare LLM requests
    processed_papers = []
    for paper in papers:
        processed_paper = process_paper_with_llm_requests(paper)
        processed_papers.append(processed_paper)
    
    # Simulate LLM responses (in real implementation, these would be actual LLM calls)
    papers_with_llm_content = simulate_llm_responses(processed_papers)
    
    # Generate final HTML
    html_content = generate_final_html(papers_with_llm_content)
    
    # Output the HTML content
    print(html_content)


if __name__ == "__main__":
    main()