li
1b24994e74
- Contract.php: 返回合约账户余额(balance_contract) - My.php: 地址管理增加BTC/ETH - AppContract.php: 一键平仓(closeall) - AppProxy.php: 代理专属注册链接 + 分级权限(L1/L2) - site.php: 手续费减半(0.018→0.009) - agent_permission_setup.sql: 代理权限SQL - crypto_news_crawler.py: 新闻自动采集脚本
248 lines
7.5 KiB
Python
248 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
Crypto News Crawler for BT Panel Scheduled Task
|
|
数据源: CryptoCompare News API (免费) + CoinDesk RSS (备用)
|
|
目标表: fa_app_zixun
|
|
"""
|
|
|
|
import requests
|
|
import pymysql
|
|
import time
|
|
import hashlib
|
|
import re
|
|
import sys
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
# ============ 数据库配置 - 根据实际服务器修改 ============
|
|
DB_CONFIG = {
|
|
'host': '127.0.0.1',
|
|
'port': 3306,
|
|
'user': 'jys',
|
|
'password': 'YmJKJ6JcRXK7AbzT',
|
|
'database': 'jys',
|
|
'charset': 'utf8mb4',
|
|
}
|
|
|
|
TABLE_PREFIX = 'fa_'
|
|
TABLE_NAME = f'{TABLE_PREFIX}app_zixun'
|
|
ADMIN_USER_ID = 1 # 后台管理员 user_id
|
|
MAX_NEWS_PER_RUN = 20 # 每次最多采集条数
|
|
|
|
# 日志配置
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s [%(levelname)s] %(message)s',
|
|
handlers=[
|
|
logging.StreamHandler(sys.stdout),
|
|
]
|
|
)
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def get_db_connection():
|
|
"""获取数据库连接"""
|
|
return pymysql.connect(**DB_CONFIG, cursorclass=pymysql.cursors.DictCursor)
|
|
|
|
|
|
def news_exists(cursor, title_hash):
|
|
"""检查新闻是否已存在（通过标题哈希去重）"""
|
|
sql = f"SELECT id FROM {TABLE_NAME} WHERE title_hash = %s LIMIT 1"
|
|
cursor.execute(sql, (title_hash,))
|
|
return cursor.fetchone() is not None
|
|
|
|
|
|
def ensure_title_hash_column(cursor):
|
|
"""确保 title_hash 列存在（用于去重）"""
|
|
cursor.execute(f"SHOW COLUMNS FROM {TABLE_NAME} LIKE 'title_hash'")
|
|
if not cursor.fetchone():
|
|
log.info("Adding title_hash column for deduplication...")
|
|
cursor.execute(f"ALTER TABLE {TABLE_NAME} ADD COLUMN title_hash VARCHAR(32) DEFAULT '' AFTER content_en")
|
|
cursor.execute(f"CREATE INDEX idx_title_hash ON {TABLE_NAME}(title_hash)")
|
|
log.info("title_hash column added.")
|
|
|
|
|
|
def clean_html(text):
|
|
"""清除 HTML 标签"""
|
|
if not text:
|
|
return ''
|
|
clean = re.sub(r'<[^>]+>', '', text)
|
|
clean = re.sub(r'&[a-zA-Z]+;', ' ', clean)
|
|
clean = re.sub(r'\s+', ' ', clean).strip()
|
|
return clean
|
|
|
|
|
|
def fetch_cryptocompare_news():
|
|
"""从 CryptoCompare 获取加密货币新闻"""
|
|
url = "https://min-api.cryptocompare.com/data/v2/news/?lang=EN&sortOrder=latest"
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
}
|
|
try:
|
|
resp = requests.get(url, headers=headers, timeout=30)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
if data.get('Type') == 100 and data.get('Data'):
|
|
news_list = []
|
|
for item in data['Data'][:MAX_NEWS_PER_RUN]:
|
|
news_list.append({
|
|
'title_en': item.get('title', ''),
|
|
'content_en': clean_html(item.get('body', '')),
|
|
'cover_image': item.get('imageurl', ''),
|
|
'published_on': item.get('published_on', int(time.time())),
|
|
'source': item.get('source', 'CryptoCompare'),
|
|
'url': item.get('url', ''),
|
|
})
|
|
log.info(f"CryptoCompare: fetched {len(news_list)} articles")
|
|
return news_list
|
|
except Exception as e:
|
|
log.error(f"CryptoCompare fetch failed: {e}")
|
|
return []
|
|
|
|
|
|
def fetch_coindesk_rss():
|
|
"""从 CoinDesk RSS 获取新闻（备用）"""
|
|
url = "https://www.coindesk.com/arc/outboundfeeds/rss/"
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
|
}
|
|
try:
|
|
resp = requests.get(url, headers=headers, timeout=30)
|
|
resp.raise_for_status()
|
|
# 简单 XML 解析 RSS
|
|
import xml.etree.ElementTree as ET
|
|
root = ET.fromstring(resp.content)
|
|
news_list = []
|
|
for item in root.findall('.//item')[:MAX_NEWS_PER_RUN]:
|
|
title = item.findtext('title', '')
|
|
description = clean_html(item.findtext('description', ''))
|
|
pub_date = item.findtext('pubDate', '')
|
|
# 解析 RSS 日期
|
|
try:
|
|
dt = datetime.strptime(pub_date, '%a, %d %b %Y %H:%M:%S %z')
|
|
timestamp = int(dt.timestamp())
|
|
except:
|
|
timestamp = int(time.time())
|
|
|
|
news_list.append({
|
|
'title_en': title,
|
|
'content_en': description,
|
|
'cover_image': '',
|
|
'published_on': timestamp,
|
|
'source': 'CoinDesk',
|
|
'url': item.findtext('link', ''),
|
|
})
|
|
log.info(f"CoinDesk RSS: fetched {len(news_list)} articles")
|
|
return news_list
|
|
except Exception as e:
|
|
log.error(f"CoinDesk RSS fetch failed: {e}")
|
|
return []
|
|
|
|
|
|
def insert_news(news_list):
|
|
"""将新闻插入数据库"""
|
|
if not news_list:
|
|
log.info("No news to insert.")
|
|
return 0
|
|
|
|
conn = get_db_connection()
|
|
cursor = conn.cursor()
|
|
inserted = 0
|
|
|
|
try:
|
|
ensure_title_hash_column(cursor)
|
|
conn.commit()
|
|
|
|
for news in news_list:
|
|
title_en = news['title_en'][:500] if news['title_en'] else ''
|
|
if not title_en:
|
|
continue
|
|
|
|
title_hash = hashlib.md5(title_en.encode('utf-8')).hexdigest()
|
|
|
|
if news_exists(cursor, title_hash):
|
|
continue
|
|
|
|
# title 和 content 使用英文（此项目面向英文用户）
|
|
sql = f"""INSERT INTO {TABLE_NAME}
|
|
(user_id, title, title_en, content, content_en, cover_image,
|
|
addtime, look_num, title_hash)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)"""
|
|
|
|
content_en = news['content_en'][:5000] if news['content_en'] else title_en
|
|
|
|
cursor.execute(sql, (
|
|
ADMIN_USER_ID,
|
|
title_en, # title (中文字段也填英文)
|
|
title_en, # title_en
|
|
content_en, # content
|
|
content_en, # content_en
|
|
news.get('cover_image', ''),
|
|
news['published_on'],
|
|
0, # look_num
|
|
title_hash,
|
|
))
|
|
inserted += 1
|
|
|
|
conn.commit()
|
|
log.info(f"Inserted {inserted} new articles into database.")
|
|
|
|
except Exception as e:
|
|
conn.rollback()
|
|
log.error(f"Database insert failed: {e}")
|
|
finally:
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
return inserted
|
|
|
|
|
|
def cleanup_old_news(days=30):
|
|
"""清理超过 N 天的旧新闻"""
|
|
conn = get_db_connection()
|
|
cursor = conn.cursor()
|
|
try:
|
|
cutoff = int(time.time()) - (days * 86400)
|
|
sql = f"DELETE FROM {TABLE_NAME} WHERE addtime < %s AND addtime > 0"
|
|
cursor.execute(sql, (cutoff,))
|
|
deleted = cursor.rowcount
|
|
conn.commit()
|
|
if deleted > 0:
|
|
log.info(f"Cleaned up {deleted} old articles (older than {days} days).")
|
|
except Exception as e:
|
|
conn.rollback()
|
|
log.error(f"Cleanup failed: {e}")
|
|
finally:
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
|
|
def main():
|
|
log.info("=" * 50)
|
|
log.info("Crypto News Crawler started")
|
|
log.info("=" * 50)
|
|
|
|
# 主数据源: CryptoCompare
|
|
news = fetch_cryptocompare_news()
|
|
|
|
# 备用数据源: CoinDesk RSS
|
|
if not news:
|
|
log.info("Primary source failed, trying CoinDesk RSS...")
|
|
news = fetch_coindesk_rss()
|
|
|
|
if news:
|
|
count = insert_news(news)
|
|
log.info(f"Total new articles: {count}")
|
|
else:
|
|
log.warning("All sources failed. No news fetched.")
|
|
|
|
# 清理30天前的旧新闻
|
|
cleanup_old_news(30)
|
|
|
|
log.info("Crawler finished.")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|