diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6769e21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..5299109 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Start Application", + "type": "debugpy", + "request": "launch", + "program": "main.py", + "console": "internalConsole", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 40a9386..0000000 --- a/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# 昇思MindSopre开源任务挑战赛 -随着人工智能技术的飞速发展，大模型技术已成为推动AI应用创新的重要力量。为了促进大模型技术的深入研究和应用，昇思MindSpore开源社区联合启智社区发布赛道《昇思MindSopre开源任务挑战赛》，即《基于昇思MindSpore AI框架的套件及大模型应用创新赛》，旨在发掘和培育具有创新性和实用性的大模型应用。 - -#赛事任务 -参赛者需基于昇思MindSpore AI框架和昇思套件，设计并开发一款具有创新性、实用性和可扩展性的人工智能应用。该方案应能够解决某一具体领域的实际问题，并展示出卓越的性能和效果。 - -#奖项设置 -本项赛事设置一等奖、二等奖、三等奖及优秀奖若干。 -奖项 数量 含税奖金（单位：元） -一等奖 1组 30000+获奖证书 -二等奖 3组 10000+获奖证书 -三等奖 8组 5000+获奖证书 -优秀奖 实际参与的15%（不包括一、二、三等奖） 获奖证书 - -#赛事委员会 -主席 -余跃 鹏城实验室OpenI启智社区运营中心主任 -杨滔 昇思MindSpore开源社区负责人 - -委员 -刘冰姿 OpenI启智社区开发者主管 -邓 清 鹏城实验室算网联盟合作与交流部部长 -何芦微 昇思MindSpore开源社区资深专家 diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000..4569397 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,78 @@ +# **🌟 WebMiner Pro: 您的一站式智能网页内容精炼器 🌟** + +欢迎来到**WebMiner Pro**，这是一款突破性的智能工具，专为快速挖掘和精炼网页内容而设计。在这个信息泛滥的时代，**WebMiner Pro** 为您提供了一个清晰、高效的解决方案，帮助您迅速捕捉关键信息，提升工作效率，让您在数据的海洋中轻松航行。 + +## **🚀 项目亮点** + +- **实时数据追踪**：与RSS源同步，实时更新，让您紧跟最新动态。 +- **深度智能分析**：利用最前沿的LLM技术，深入分析并总结网页内容。 +- **一键式信息摘要**：简单操作，即刻生成精炼、准确的关键信息摘要。 +- **个性化定制服务**：根据您的需求，定制信息摘要的深度和广度。 +- **跨平台数据聚合**：打破信息孤岛，一站式聚合多个数据源。 + +## **🛠️ 实用性与有效性** + +- **提升决策效率**：快速获取市场动态、技术趋势，为您的决策提供数据支持。 +- **节省时间成本**：自动化处理繁琐的数据收集和分析工作，释放您的双手。 +- **增强信息洞察力**：深入分析，帮助您洞察行业趋势和潜在机会。 +- **保障数据安全**：本地数据库存储，确保信息安全和隐私。 + +## **🌐 功能概览** + +- **智能网页抓取**：自动解析网页，提取关键内容。 +- **高效内容摘要**：生成清晰、简洁的摘要，快速传达核心信息。 +- **本地数据库支持**：使用SQLite数据库，保障数据处理速度和安全性。 +- **灵活的API集成**：轻松集成OpenAI等语言模型，提供定制化服务。 +- **直观的前端展示**：通过简洁的前端界面，展示和交互数据摘要。 + +## 系统前端展示 + +![image]() + +## **📝 安装与使用** + +### **环境准备** +- 确保您的系统已安装Python 3.6+。 + +### **快速开始** +1. **克隆项目代码**： + +2. **安装依赖**： + ```bash + pip install -r requirements.txt + ``` + +3. **配置项目**： + - 编辑`config.json`，设置您关注的RSS源。 + - 填写`config.py`，配置数据库和API密钥。 + +4. **启动应用**： + ```bash + python main.py + ``` + +5. **执行任务**： + - 使用curl或其他HTTP客户端工具，按照顺序发送请求到`/update`、`/generate_summaries`和`/api/summarydata`。 + +### **使用指南** + +- 通过几个简单的HTTP请求，即可完成数据的更新、摘要生成和信息展示。 +- 访问`http://127.0.0.1:5000/index`，在浏览器中查看数据摘要。 + + +## **🌟 为何选择WebMiner Pro？** + +- **效率**：快速响应，实时更新，让您始终掌握最新信息。 +- **精准**：深度学习技术，确保信息摘要的准确性和深度。 +- **易用**：简洁的用户界面，一键操作，轻松上手。 +- **安全**：本地数据处理，保护您的数据不被外泄。 + +## **🔍 开启智能信息处理新篇章** + +**WebMiner Pro** 不仅仅是一个工具，它是您在信息时代的得力助手，是您高效工作的加速器。立即体验**WebMiner Pro**，让您的信息处理更加智能、高效！ + +--- + +**WebMiner Pro** —— 您的智能网页内容精炼器，让信息获取和分析变得前所未有的简单和高效。抓住机会，开启您的智能信息处理新篇章！ + + diff --git a/config.json b/config.json new file mode 100644 index 0000000..2c2845d --- /dev/null +++ b/config.json @@ -0,0 +1,14 @@ +{ + "feeds": [ + { + "url": "http://www.geekpark.net/rss", + "num_entries": "all", + "update_frequency": 3600 + }, + { + "url": "https://plink.anyfeeder.com/people-daily", + "num_entries": 6, + "update_frequency": 3600 + } + ] + } diff --git a/config.py b/config.py new file mode 100644 index 0000000..38831d6 --- /dev/null +++ b/config.py @@ -0,0 +1,16 @@ +import json + +# 加载和管理配置文件中的数据 +class Config: + # 读取配置文件 + with open('config.json', 'r') as f: + config_data = json.load(f) + + # 获取 RSS feed 列表 + RSS_FEEDS = config_data.get('feeds', []) + SQLALCHEMY_DATABASE_URI = 'sqlite:///rss.db' + OPENAI_API_KEY ='' # jinareader的key，通过 https://jina.ai/ 获取 + FETCH_INTERVAL = 3600 # 默认抓取间隔1小时 + OPENAI_URL = 'https://api.deepseek.com/chat/completions' # 通过你的 LLM 服务商获取 + OPENAI_MODEL = 'deepseek-chat' # 通过你的 LLM 服务商获取 + JINA_API_KEY = '' # 通过你的 LLM 服务商获取 diff --git a/data_processor.py b/data_processor.py new file mode 100644 index 0000000..0c657a9 --- /dev/null +++ b/data_processor.py @@ -0,0 +1,62 @@ +from html_fetcher import fetch_html +from openai_processor import generate_summary +from models import RawData, SummaryData, db +from datetime import datetime +import pytz + +def convert_to_beijing_time(rss_time_str): + """将 RSS 时间字符串转换为北京时间。 + + Args: + rss_time_str: RSS 时间字符串，例如 "Tue, 27 Oct 2021 11:05:29 +0000" + + Returns: + 北京时间 datetime 对象 + """ + + beijing_tz = pytz.timezone('Asia/Shanghai') + dt = datetime.strptime(rss_time_str, "%a, %d %b %Y %H:%M:%S %z") + dt_utc = dt.astimezone(pytz.utc) + dt_beijing = dt_utc.astimezone(beijing_tz) + return dt_beijing + +def fetch_html_and_update_raw_data(): + """获取 HTML 内容并更新 RawData 表中的 raw_html 字段""" + raw_data_entries = RawData.query.filter(RawData.raw_html.is_(None)).all() + print(f"Fetching HTML content for {len(raw_data_entries)} RawData entries.") + for entry in raw_data_entries: + print(f"Fetching HTML content for link: {entry.link}") + jina_data = next(fetch_html([entry.link]))[1] + entry.raw_html = jina_data + db.session.commit() + print(f"Updated RawData entry with raw_html for link: {entry.link}") + +def generate_summaries_and_save(): + """生成摘要并保存到 SummaryData 表中,同时处理datetime""" + # db.session.query(SummaryData).delete() # 如果需要每次都清空 SummaryData 表，请取消注释 + raw_data_entries = RawData.query.filter(RawData.raw_html.isnot(None)).all() + print(f"Generating summaries for {len(raw_data_entries)} RawData entries.") + for entry in raw_data_entries: + print(f"Generating summary for HTML content from: {entry.link}") + summary_data = generate_summary(entry.raw_html) + # 将 RSS 时间字符串转换为北京时间 + if summary_data: + bj_pub_date = convert_to_beijing_time(entry.pub_date) + # 检查 SummaryData 中是否已经存在此链接 + existing_summary = SummaryData.query.filter_by(link=entry.link).first() + if existing_summary: + print(f"Summary for link {entry.link} already exists, skipping.") + continue # 跳过此链接 + + summary = SummaryData( + title=entry.title, + link=entry.link, + pub_date = entry.pub_date, + bj_pub_date = bj_pub_date, + summary_title=summary_data.get('title', ''), + summary_content=summary_data.get('content', ''), + summary_image=summary_data.get('image', ''), + ) + db.session.add(summary) + db.session.commit() + print(f"Added summary for link: {entry.link} to SummaryData table.") \ No newline at end of file diff --git a/html_fetcher.py b/html_fetcher.py new file mode 100644 index 0000000..f4a9d76 --- /dev/null +++ b/html_fetcher.py @@ -0,0 +1,31 @@ +import requests +from collections import deque +import time +from config import Config + +jina_key = Config.JINA_API_KEY + +# 从指定的 URL 中获取 HTML 内容。包括：使用库（如 requests 或 BeautifulSoup）获取并解析网页；提取网页中的重要内容，为摘要生成做准备。 + +# 从指定的 URL 中获取 HTML 内容 +def fetch_html(urls): + queue = deque(urls) + headers = { + "Authorization": f"Bearer {jina_key}", # 替换为你自己的授权信息 + "Accept": "application/json" + } + while queue: + url = queue.popleft() + print(f"Fetching HTML content from: {url}") + jina_url = f"https://r.jina.ai/{url}" + response = requests.get(jina_url, headers=headers) + if response.status_code == 200: + # 假设响应中的HTML内容位于'data'字段中 + data = response.json().get('data', {}) + content = data.get('content', '') + yield url, content + print(f"Fetched HTML content from: {url}") + else: + print(f"Request to r.jina.ai failed, status code: {response.status_code}") + if len(queue) > 0: + time.sleep(6) # 控制每分钟发送 10 条消息 \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..dda4726 --- /dev/null +++ b/main.py @@ -0,0 +1,55 @@ +from flask import Flask, jsonify, render_template_string,render_template +from sqlalchemy import false +from config import Config +from flask_cors import CORS +from models import db, RSSFeed, RawData, SummaryData +from rss_updater import update_rss_feeds +from data_processor import fetch_html_and_update_raw_data, generate_summaries_and_save +import atexit + +app = Flask(__name__, static_folder='static', template_folder='templates') # 创建 Flask 应用实例，并指定静态文件和模板文件夹的位置。 +CORS(app) # 启用 CORS 处理，以便前端能够访问 API +app.config.from_object(Config) + +app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///rss.db' #设置数据库 URI，指向本地 SQLite 数据库 rss.db +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + +db.init_app(app) + +with app.app_context(): + db.create_all() + print("Database initialized and tables created.") + +@app.route('/update') # 定义 /update 路由，处理 RSS 源更新请求 +def update_data(): + has_new_data = update_rss_feeds() + if has_new_data: + fetch_html_and_update_raw_data() + return jsonify({'message': 'RSS feeds updated successfully.'}) + +@app.route('/generate_summaries') # 定义 /generate_summaries 路由，处理摘要生成请求。 +def generate_summaries(): + generate_summaries_and_save() + return jsonify({'message': 'Summaries generated and saved successfully. Existing summaries have been overwritten.'}) + +# 获取所有 SummaryData 数据,仅后端返回json +@app.route('/api/summarydata', methods=['GET']) # 定义 /api/summarydata 路由，处理获取所有摘要数据的请求。 +def get_summarydata(): + summarydata = SummaryData.query.all() + return jsonify([summarydatum.to_dict() for summarydatum in summarydata]) + +@app.route('/index') # 定义 /index 路由，显示前端页面。 +def index(): + summarydata = SummaryData.query.all() + summary_list = [summary.to_dict() for summary in summarydata] + return render_template('index.html', summary_list=summary_list) + + +def shutdown_scheduler(): + print("Scheduler is not running.") + +atexit.register(shutdown_scheduler) + +if __name__ == '__main__': + print("Starting Flask application.") + app.run(port=5000) \ No newline at end of file diff --git a/models.py b/models.py new file mode 100644 index 0000000..16fc5bb --- /dev/null +++ b/models.py @@ -0,0 +1,45 @@ +from flask_sqlalchemy import SQLAlchemy + +# 定义数据库模型的文件。使用 SQLAlchemy 来定义数据表的结构，并提供与数据库交互的接口。 +db = SQLAlchemy() + +class RSSFeed(db.Model): + """RSS 订阅信息模型""" + id = db.Column(db.Integer, primary_key=True) + url = db.Column(db.String, nullable=False) + title = db.Column(db.String) + pub_date = db.Column(db.TEXT) + link = db.Column(db.String, unique=True, nullable=False) + +class RawData(db.Model): + """原始数据模型""" + id = db.Column(db.Integer, primary_key=True) + url = db.Column(db.String, nullable=False) + title = db.Column(db.String) + pub_date = db.Column(db.TEXT) + link = db.Column(db.String, unique=True, nullable=False) + raw_html = db.Column(db.String) # 存储获取到的 HTML 内容 + +class SummaryData(db.Model): + """摘要数据模型""" + id = db.Column(db.Integer, primary_key=True) + title = db.Column(db.String) + link = db.Column(db.String, unique=True, nullable=False) + pub_date = db.Column(db.TEXT) + bj_pub_date = db.Column(db.DateTime) + summary_title = db.Column(db.String) + summary_content = db.Column(db.String) + summary_image = db.Column(db.String, nullable=True) + def to_dict(self): + return { + 'id': self.id, + 'title': self.title, + 'link': self.link, + 'bj_pub_date': self.bj_pub_date, + 'pub_date': self.pub_date, + 'summary_title': self.summary_title, + 'summary_content': self.summary_content, + 'summary_image': self.summary_image + } + + diff --git a/openai_processor.py b/openai_processor.py new file mode 100644 index 0000000..7c3da6a --- /dev/null +++ b/openai_processor.py @@ -0,0 +1,61 @@ +import requests +from config import Config +import json + +openai_key = Config.OPENAI_API_KEY +openai_url = Config.OPENAI_URL +MODEL = Config.OPENAI_MODEL + +#与 OpenAI API 交互的模块，包括：封装与 OpenAI API 的请求;处理 API 返回的数据，并将其转换为项目需要的格式。 + +# 多语言支持：language 参数可以用来控制生成摘要的语言 +def generate_summary(jina_data, language="zh", response_format="json", timeout=15): + if not openai_key or not openai_url: + print("OpenAI API Key or URL is missing.") + return None + + # 根据用户选择的语言设置 prompt + prompt = f'''你的任务是作为一个高级翻译和编辑，理解发给你的内容，从中生产加工输出以下信息：标题、正文、图片。确保你的响应符合以下{response_format}结构，准确反映提取的数据，不做修改： + ```{response_format} + {{ + "title": "文章标题", + "content": "文章摘要", + "image": "文章包含的图片链接，保留url，如果没有留空" + }} + ```重要的是你的输出严格遵守这种格式。 + -严格确保统一翻译为{language} + -不翻译公司名称、人名''' + + siliconflow_payload = { + "model": MODEL, + "messages": [ + { + "role": "assistant", + "content": f"{prompt}{jina_data}" + } + ] + } + siliconflow_headers = { + "accept": f"application/{response_format}", + "content-type": "application/json", + "authorization": f"Bearer {openai_key}" + } + + try: + siliconflow_response = requests.post(siliconflow_url, json=siliconflow_payload, headers=siliconflow_headers, timeout=timeout) + siliconflow_response.raise_for_status() + response = siliconflow_response.json() + result_content = response['choices'][0]['message']['content'] + + print("Generated summary successfully.") + try: + json_str = result_content.strip().lstrip(f'```{response_format}').rstrip('```').strip() + parsed_json = json.loads(json_str) + return parsed_json + except json.JSONDecodeError as e: + print(f"Failed to parse {response_format} response: {e}") + return None + + except requests.exceptions.RequestException as e: + print(f"SiliconFlow 请求错误：{e}") + return None diff --git a/raf.py b/raf.py deleted file mode 100644 index 5aeed7a..0000000 --- a/raf.py +++ /dev/null @@ -1,49 +0,0 @@ -import cv2 -import numpy as np -from PIL import Image -import os -import copy -import csv - -import torchvision -import torch -from .randaugment import RandAugment - -class TransformTwice: - def __init__(self, transform): - self.transform = transform - self.strong_transfrom = copy.deepcopy(transform) - self.strong_transfrom.transforms.insert(0, RandAugment(3,5)) - - def __call__(self, inp): - out1 = self.transform(inp) - out2 = self.transform(inp) - out3 = self.strong_transfrom(inp) - return out1, out2, out3 - -def get_raf(train_root, train_file_list, test_root, test_file_list, n_labeled, transform_train=None, transform_val=None): - - train_labeled_idxs, train_unlabeled_idxs = data_split(train_file_list, int(n_labeled)) - - train_labeled_dataset = Dataset_RAF_labeled(train_root, train_file_list, train_labeled_idxs, transform=transform_train) - train_unlabeled_dataset = Dataset_RAF_unlabeled(train_root, train_file_list, train_unlabeled_idxs, transform=TransformTwice(transform_train)) - - test_dataset = Dataset_RAF(test_root, test_file_list, transform=transform_val) - - print (f"#Labeled: {len(train_labeled_idxs)} #Unlabeled: {len(train_unlabeled_dataset)}") - return train_labeled_dataset, train_unlabeled_dataset, test_dataset - -def target_read(path): - label_list = [] - with open(path) as f: - img_label_list = f.read().splitlines() - for info in img_label_list: - _, label_name = info.split(' ') - label_list.append(int(label_name)) - return label_list - -def data_split(filename, n_labeled): - labels = target_read(filename) - labels = np.array(labels) - train_labeled_idxs = [] - train_unlabeled_idxs = [] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b97ae06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +APScheduler==3.10.4 +blinker==1.8.2 +click==8.1.7 +feedparser==6.0.11 +Flask==3.0.3 +Flask-Cors==4.0.1 +Flask-SQLAlchemy==3.1.1 +greenlet==3.0.3 +gunicorn==22.0.0 +itsdangerous==2.2.0 +Jinja2==3.1.4 +MarkupSafe==2.1.5 +packaging==23.2 +pytz==2024.1 +sgmllib3k==1.0.0 +SQLAlchemy==2.0.31 +typing_extensions==4.12.2 +tzlocal==5.2 +Werkzeug==3.0.3 \ No newline at end of file diff --git a/rss_updater.py b/rss_updater.py new file mode 100644 index 0000000..1bf157d --- /dev/null +++ b/rss_updater.py @@ -0,0 +1,123 @@ +import feedparser +from datetime import datetime +import json +import time +from models import RSSFeed, RawData, db + +# 专门用于更新 RSS 源的模块。包括：定期检查并抓取新的 RSS 条目；过滤和处理 RSS 数据，并将其存储到数据库中. + +# 从配置文件加载 RSS feed 列表 +def load_rss_feeds_from_config(config_path="config.json"): + try: + with open(config_path, "r") as f: + config = json.load(f) + return config["feeds"] + except FileNotFoundError: + print(f"配置文件 {config_path} 不存在，请创建配置文件。") + return [] + +# 解析 RSS feed +def parse_rss_feed(url, config_path="config.json"): + """解析 RSS feed 并根据配置文件获取拉取数量和更新频次。 + + Args: + url (str): RSS feed URL + config_path (str, optional): 配置文件路径。Defaults to "config.json". + + Returns: + list: 解析后的 RSS 条目列表 + """ + print(f"Parsing RSS feed from: {url}") + feed = feedparser.parse(url) + entries = [] + + # 加载配置文件 + try: + with open(config_path, "r") as f: + config = json.load(f) + except FileNotFoundError: + print(f"配置文件 {config_path} 不存在，请创建配置文件。") + return None + + # 找到对应 URL 的配置信息 + feed_config = next((feed for feed in config["feeds"] if feed["url"] == url), None) + + if feed_config: + num_entries = feed_config.get("num_entries", "all") + #update_frequency = feed_config.get("update_frequency", 3600) # 默认更新频率为 1 小时 + + if num_entries == "all": + for entry in feed.entries: + entries.append({ + 'title': entry.title, + 'pub_date': entry.published, + 'link': entry.link + }) + else: + try: + num_entries = int(num_entries) + for i, entry in enumerate(feed.entries): + if i < num_entries: + entries.append({ + 'title': entry.title, + 'pub_date': entry.published, + 'link': entry.link + }) + else: + break + except ValueError: + print(f"配置文件中 {url} 的拉取数量无效，请检查配置文件。") + return None + + # 休眠，等待下次更新 + #time.sleep(update_frequency) + + else: + print(f"配置文件中没有找到 {url} 的配置信息。") + + print(f"Parsed {len(entries)} entries from: {url}") + return entries + +# 更新 RSS 订阅 +def update_rss_feeds(): + """更新 RSS 订阅并保存到 RawData 和 RSSFeed 表中""" + urls = load_rss_feeds_from_config() + print(f"Updating RSS feeds for URLs: {urls}") + has_new_data = False + for feed_config in urls: + url = feed_config["url"] + print(f"Parsing RSS feed from: {url}") + entries = parse_rss_feed(url) + for entry in entries: + # 检查 RSSFeed 表中是否已存在相同链接的记录 + existing_feed = RSSFeed.query.filter_by(link=entry['link']).first() + if existing_feed: + print(f"Skipping duplicate entry with link: {entry['link']}") + continue + + # 创建 RSSFeed 对象 + rss_feed = RSSFeed( + url=url, + title=entry['title'], + pub_date=entry['pub_date'], + link=entry['link'] + ) + + # 添加 RSSFeed 对象到数据库 + db.session.add(rss_feed) + + # 创建 RawData 对象 + raw_data = RawData( + url=url, + title=entry['title'], + pub_date=entry['pub_date'], + link=entry['link'] + ) + + # 添加 RawData 对象到数据库 + db.session.add(raw_data) + db.session.commit() + print(f"Added entry to RawData and RSSFeed tables: {entry['title']}") + has_new_data = True + + return has_new_data diff --git a/static/css/style.css b/static/css/style.css new file mode 100644 index 0000000..4118a0d --- /dev/null +++ b/static/css/style.css @@ -0,0 +1,92 @@ +body { + font-family: Arial, sans-serif; + line-height: 1.6; + margin: 0; + padding: 0; + background-color: #f4f4f4; +} + +header { + background-color: #333; + color: #fff; + text-align: center; + padding: 1rem; +} + +main { + max-width: 800px; + margin: 0 auto; + padding: 2rem; +} + +.filter-container { + margin-bottom: 1rem; +} + +#dateFilter { + padding: 0.5rem; + border-radius: 3px; + border: 1px solid #ccc; +} + +.summary-card { + background-color: #fff; + border: 1px solid #ccc; + border-radius: 5px; + padding: 1rem; + margin-bottom: 1rem; + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1); +} + +.summary-card h2 { + margin-top: 0; + color: #333; +} + +.pub-date { + color: #777; + font-size: 0.9rem; +} + +.summary-card img { + max-width: 100%; + height: auto; + margin: 1rem 0; +} + +.summary-content { + margin-bottom: 1rem; +} + +.read-more { + display: inline-block; + background-color: #333; + color: #fff; + padding: 0.5rem 1rem; + text-decoration: none; + border-radius: 3px; +} + +.pagination { + display: flex; + justify-content: center; + align-items: center; + margin-top: 2rem; +} + +.page-link { + display: inline-block; + padding: 0.5rem 1rem; + background-color: #333; + color: #fff; + text-decoration: none; + border-radius: 3px; + margin: 0 0.5rem; +} + +footer { + text-align: center; + padding: 1rem; + background-color: #333; + color: #fff; +} diff --git a/static/js/main.js b/static/js/main.js new file mode 100644 index 0000000..6bf1a14 --- /dev/null +++ b/static/js/main.js @@ -0,0 +1,11 @@ +function filterByDate(date) { + window.location.href = `/?date=${date}`; +} + +document.addEventListener('DOMContentLoaded', function() { + const urlParams = new URLSearchParams(window.location.search); + const dateFilter = urlParams.get('date'); + if (dateFilter) { + document.getElementById('dateFilter').value = dateFilter; + } +}); diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..b456fd8 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,47 @@ + + + + + + 简报系统 + + + +

简报系统

+ {% for summary in summary_list %} +
+

{{ summary.summary_title }}

+

{{ summary.bj_pub_date }}

+ {% if summary.summary_image %} + {{ summary.summary_title }} + {% endif %} +

{{ summary.summary_content }}

+
+ {% endfor %} + + \ No newline at end of file diff --git a/test.py b/test.py deleted file mode 100644 index c8b8127..0000000 --- a/test.py +++ /dev/null @@ -1,144 +0,0 @@ -import argparse -import os -import cv2 -import numpy as np -import torch -from torchvision import models -from pytorch_grad_cam import ( - GradCAM, HiResCAM, ScoreCAM, GradCAMPlusPlus, - AblationCAM, XGradCAM, EigenCAM, EigenGradCAM, - LayerCAM, FullGrad, GradCAMElementWise -) -from pytorch_grad_cam import GuidedBackpropReLUModel -from pytorch_grad_cam.utils.image import ( - show_cam_on_image, deprocess_image, preprocess_image -) -from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--device', type=str, default='cpu', - help='Torch device to use') - parser.add_argument( - '--image-path', - type=str, - default='./examples/both.png', - help='Input image path') - parser.add_argument('--aug-smooth', action='store_true', - help='Apply test time augmentation to smooth the CAM') - parser.add_argument( - '--eigen-smooth', - action='store_true', - help='Reduce noise by taking the first principle component' - 'of cam_weights*activations') - parser.add_argument('--method', type=str, default='gradcam', - choices=[ - 'gradcam', 'hirescam', 'gradcam++', - 'scorecam', 'xgradcam', 'ablationcam', - 'eigencam', 'eigengradcam', 'layercam', - 'fullgrad', 'gradcamelementwise' - ], - help='CAM method') - - parser.add_argument('--output-dir', type=str, default='output', - help='Output directory to save the images') - args = parser.parse_args() - - if args.device: - print(f'Using device "{args.device}" for acceleration') - else: - print('Using CPU for computation') - - return args - - -if __name__ == '__main__': - """ python cam.py -image-path - Example usage of loading an image and computing: - 1. CAM - 2. Guided Back Propagation - 3. Combining both - """ - - args = get_args() - methods = { - "gradcam": GradCAM, - "hirescam": HiResCAM, - "scorecam": ScoreCAM, - "gradcam++": GradCAMPlusPlus, - "ablationcam": AblationCAM, - "xgradcam": XGradCAM, - "eigencam": EigenCAM, - "eigengradcam": EigenGradCAM, - "layercam": LayerCAM, - "fullgrad": FullGrad, - "gradcamelementwise": GradCAMElementWise - } - - model = models.resnet50(pretrained=True).to(torch.device(args.device)).eval() - - # Choose the target layer you want to compute the visualization for. - # Usually this will be the last convolutional layer in the model. - # Some common choices can be: - # Resnet18 and 50: model.layer4 - # VGG, densenet161: model.features[-1] - # mnasnet1_0: model.layers[-1] - # You can print the model to help chose the layer - # You can pass a list with several target layers, - # in that case the CAMs will be computed per layer and then aggregated. - # You can also try selecting all layers of a certain type, with e.g: - # from pytorch_grad_cam.utils.find_layers import find_layer_types_recursive - # find_layer_types_recursive(model, [torch.nn.ReLU]) - - target_layers = [model.layer4] - - rgb_img = cv2.imread(args.image_path, 1)[:, :, ::-1] - rgb_img = np.float32(rgb_img) / 255 - input_tensor = preprocess_image(rgb_img, - mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]).to(args.device) - - # We have to specify the target we want to generate - # the Class Activation Maps for. - # If targets is None, the highest scoring category (for every member in the batch) will be used. - # You can target specific categories by - # targets = [ClassifierOutputTarget(281)] - # targets = [ClassifierOutputTarget(281)] - targets = None - - # Using the with statement ensures the context is freed, and you can - # recreate different CAM objects in a loop. - cam_algorithm = methods[args.method] - with cam_algorithm(model=model, - target_layers=target_layers) as cam: - - # AblationCAM and ScoreCAM have batched implementations. - # You can override the internal batch size for faster computation. - cam.batch_size = 32 - grayscale_cam = cam(input_tensor=input_tensor, - targets=targets, - aug_smooth=args.aug_smooth, - eigen_smooth=args.eigen_smooth) - - grayscale_cam = grayscale_cam[0, :] - - cam_image = show_cam_on_image(rgb_img, grayscale_cam, use_rgb=True) - cam_image = cv2.cvtColor(cam_image, cv2.COLOR_RGB2BGR) - - gb_model = GuidedBackpropReLUModel(model=model, device=args.device) - gb = gb_model(input_tensor, target_category=None) - - cam_mask = cv2.merge([grayscale_cam, grayscale_cam, grayscale_cam]) - cam_gb = deprocess_image(cam_mask * gb) - gb = deprocess_image(gb) - - os.makedirs(args.output_dir, exist_ok=True) - - cam_output_path = os.path.join(args.output_dir, f'{args.method}_cam.jpg') - gb_output_path = os.path.join(args.output_dir, f'{args.method}_gb.jpg') - cam_gb_output_path = os.path.join(args.output_dir, f'{args.method}_cam_gb.jpg') - - cv2.imwrite(cam_output_path, cam_image) - cv2.imwrite(gb_output_path, gb) - cv2.imwrite(cam_gb_output_path, cam_gb) diff --git a/views/views.py b/views/views.py new file mode 100644 index 0000000..e69de29 diff --git a/页面.png b/页面.png deleted file mode 100644 index 81302c5..0000000 Binary files a/页面.png and /dev/null differ