Skip to content

Commit

Permalink
fix 'npm run format' and route '/metred/fuli' (DIYgod#6703) (DIYgod#6720
Browse files Browse the repository at this point in the history
)

* fix 'npm run format'

* fix route '/metred/fuli' (DIYgod#6703)

replace web crawling + cheerio with direct api access
  • Loading branch information
queensferryme authored Jan 18, 2021
1 parent f1eedb0 commit 61fc6af
Show file tree
Hide file tree
Showing 7 changed files with 82 additions and 99 deletions.
3 changes: 2 additions & 1 deletion .eslintrc
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
},
"env": {
"node": true,
"es6": true
"es6": true,
"browser": true
},
"rules": {
"no-console": 2,
Expand Down
8 changes: 3 additions & 5 deletions assets/radar-rules.js
Original file line number Diff line number Diff line change
Expand Up @@ -2615,8 +2615,7 @@
docs: 'https://docs.rsshub.app/university.html#chong-qing-wen-li-xue-yuan',
source: '/:type',
target: (params) => {
if (params.type === 'channel_7721.html')
{
if (params.type === 'channel_7721.html') {
return '/cqwu/news/notify';
}
},
Expand All @@ -2626,12 +2625,11 @@
docs: 'https://docs.rsshub.app/university.html#chong-qing-wen-li-xue-yuan',
source: '/:type',
target: (params) => {
if (params.type === 'channel_7722.html')
{
if (params.type === 'channel_7722.html') {
return '/cqwu/news/academiceve';
}
},
},
]
],
},
});
2 changes: 1 addition & 1 deletion docs/other.md
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ type 为 all 时,category 参数不支持 cost 和 free

### 福利资源 - met.red

<Route author="junfengP" example="/metred/fuli" path="/metred/fuli" />
<Route author="junfengP queensferryme" example="/metred/fuli" path="/metred/fuli" />

## 古诗文网

Expand Down
24 changes: 12 additions & 12 deletions docs/university.md
Original file line number Diff line number Diff line change
Expand Up @@ -2172,18 +2172,6 @@ type 列表:

</Route>

## 重庆文理学院

### 通知公告

<Route author="Fatpandac" example="/cqwu/news/academiceve" path="/cqwu/news/:type?" :paramsDesc="['可选,默认为 academiceve ']" radar="1">

| 通知公告 | 学术活动公告 |
| ------- | ----------- |
| notify | academiceve |

</Route>

## 重庆科技学院

### 教务处公告
Expand Down Expand Up @@ -2216,6 +2204,18 @@ type 列表:

<Route author="Colin-XKL" example="/cqut/libnews" path="/cqut/libnews" radar="1"/>

## 重庆文理学院

### 通知公告

<Route author="Fatpandac" example="/cqwu/news/academiceve" path="/cqwu/news/:type?" :paramsDesc="['可选,默认为 academiceve ']" radar="1">

| 通知公告 | 学术活动公告 |
| -------- | ------------ |
| notify | academiceve |

</Route>

## 中山大学

### 数据科学与计算机学院动态
Expand Down
65 changes: 27 additions & 38 deletions lib/routes/metred/fuli.js
Original file line number Diff line number Diff line change
@@ -1,53 +1,42 @@
const got = require('@/utils/got');
const host = 'https://met.red';
const mainPage = 'https://met.red/h/weal/list#';
const cheerio = require('cheerio');
const url = require('url');

async function load(link, ctx) {
const cache = await ctx.cache.get(link);
async function getItemDetail(itemId, ctx) {
const link = `https://met.red/api/h/weal/getSingleDetail?wealId=${itemId}`;

const cache = ctx.cache.get(link);
if (cache) {
return cache;
}
const response = await got.get(link);
const $ = cheerio.load(response.data);
const images = $('img');
for (let k = 0; k < images.length; k++) {
$(images[k]).replaceWith(`<img src="${url.resolve(host, $(images[k]).attr('src'))}" />`);
}
const couponUrl = $('.layui-btn.layui-btn.layui-btn-lg').attr('href');

const eventHtml = couponUrl ? '<div><p>活动链接:无</p></div>' : `<div><a href="${couponUrl}">点我前往活动</a></div>`;
const { data: response } = await got.get(link);
const image = `<div><img src=${response.data.imgUrl} /></div>`;
const coupon = response.data.url ? `<div><a href='${response.data.url}'>点击前往活动</a></div>` : '';
const content = image + coupon + response.data.content;

ctx.cache.set(link, content);

const description = eventHtml + $('.p-detail-html').html();
ctx.cache.set(link, description);
return { description };
return content;
}

async function getItemList() {
const response = await got('https://met.red/api/h/weal/getListForData');
return response.data.data;
}

module.exports = async (ctx) => {
const response = await got({
method: 'get',
url: mainPage,
});
const data = response.data;
const $ = cheerio.load(data);
const list = $('h4 > a').get();
const process = await Promise.all(
list.map(async (item) => {
const itemUrl = host + $(item).attr('href');
const single = {
title: $(item).text(),
link: itemUrl,
guid: itemUrl,
};
const other = await load(itemUrl, ctx);
return Promise.resolve(Object.assign({}, single, other));
})
);
const itemList = await getItemList();

ctx.state.data = {
title: '福利资源-met.red',
link: mainPage,
url: 'https://met.red/h/weal/list',
description: '福利资源更新提醒',
item: process,
item: await Promise.all(
itemList.map(async (item) => ({
title: item.name,
link: `https://met.red/h/weal/detail/${item.id}`,
description: await getItemDetail(item.id, ctx),
guid: item.id,
}))
),
};
};
7 changes: 3 additions & 4 deletions lib/routes/universities/cqwu/news.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ const url = require('url').resolve;

const host = 'http://www.cqwu.edu.cn';
const map = {
notify:'/channel_7721.html',
notify: '/channel_7721.html',
academiceve: '/channel_7722.html',
};
const titleMap = {
notify:'通知',
academiceve:'学术活动',
notify: '通知',
academiceve: '学术活动',
};

module.exports = async (ctx) => {
Expand Down Expand Up @@ -37,4 +37,3 @@ module.exports = async (ctx) => {
.get(),
};
};

72 changes: 34 additions & 38 deletions lib/routes/universities/uestc/sice.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,62 +6,58 @@ module.exports = async (ctx) => {
const page = await browser.newPage();

// 浏览器伪装
await page.evaluateOnNewDocument(() => { // 在每个新页面打开前执行以下脚本
await page.evaluateOnNewDocument(() => {
// 在每个新页面打开前执行以下脚本
const newProto = navigator.__proto__;
delete newProto.webdriver; // 删除navigator.webdriver字段
delete newProto.webdriver; // 删除navigator.webdriver字段
navigator.__proto__ = newProto;
window.chrome = {}; // 添加window.chrome字段,为增加真实性还需向内部填充一些值
window.chrome.app = {"InstallState":"hehe", "RunningState":"haha", "getDetails":"xixi", "getIsInstalled":"ohno"};
window.chrome.csi = function() {};
window.chrome.loadTimes = function() {};
window.chrome.runtime = function() {};
Object.defineProperty(navigator, 'userAgent', { // userAgent在无头模式下有headless字样,所以需覆写
get: () => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36",
window.chrome = {}; // 添加window.chrome字段,为增加真实性还需向内部填充一些值
window.chrome.app = { InstallState: 'hehe', RunningState: 'haha', getDetails: 'xixi', getIsInstalled: 'ohno' };
window.chrome.csi = function () {};
window.chrome.loadTimes = function () {};
window.chrome.runtime = function () {};
Object.defineProperty(navigator, 'userAgent', {
// userAgent在无头模式下有headless字样,所以需覆写
get: () => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36',
});
Object.defineProperty(navigator, 'plugins', { // 伪装真实的插件信息
get: () => [{"description": "Portable Document Format",
"filename": "internal-pdf-viewer",
"length": 1,
"name": "Chrome PDF Plugin"}]
Object.defineProperty(navigator, 'plugins', {
// 伪装真实的插件信息
get: () => [{ description: 'Portable Document Format', filename: 'internal-pdf-viewer', length: 1, name: 'Chrome PDF Plugin' }],
});
Object.defineProperty(navigator, 'languages', { // 添加语言
get: () => ["zh-CN", "zh", "en"],
Object.defineProperty(navigator, 'languages', {
// 添加语言
get: () => ['zh-CN', 'zh', 'en'],
});
const originalQuery = window.navigator.permissions.query; // notification伪装
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
window.navigator.permissions.query = (parameters) => (parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters));
});


await page.goto(baseIndexUrl, {
waitUntil: "networkidle0"
waitUntil: 'networkidle0',
});
const content = await page.content();
await browser.close();
const $ = cheerio.load(content);

const out = $('.notice p').map((index, item) => {
item = $(item);
let date = new Date(new Date().getFullYear() + '-' + item.find('a.date').text());
if (new Date() < date) {
date = new Date((new Date().getFullYear() - 1) + '-' + item.find('a.date').text());
}
return {
title: item.find('a[href]').text(),
link: baseIndexUrl + item.find('a[href]').attr('href'),
pubDate: date
};
}
).get();

const out = $('.notice p')
.map((index, item) => {
item = $(item);
let date = new Date(new Date().getFullYear() + '-' + item.find('a.date').text());
if (new Date() < date) {
date = new Date(new Date().getFullYear() - 1 + '-' + item.find('a.date').text());
}
return {
title: item.find('a[href]').text(),
link: baseIndexUrl + item.find('a[href]').attr('href'),
pubDate: date,
};
})
.get();

ctx.state.data = {
title: '信通通知公告',
link: 'https://www.sice.uestc.edu.cn/tzgg/yb.htm',
description: '电子科技大学信息与通信工程学院通知公告',
item: out
item: out,
};
};

0 comments on commit 61fc6af

Please sign in to comment.