数据采集：频道采集&新闻采集-经管之家官网！

坛友互助群

扫码加入各岗位、行业、专业交流群

数据采集：频道采集&新闻采集

代码：package controllers;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.List;

import org.h2.mvstore.DataUtils;

import com.avaje.ebean.Ebean;

import com.avaje.ebean.Page;

import com.fasterxml.jackson.databind.JsonNode;

import com.fasterxml.jackson.databind.node.ArrayNode;

import com.yhpl.utils.CollectionUtil;

import com.yhpl.utils.DateUtil;

import com.yhpl.utils.JsonFileUtil;

import com.yhpl.utils.NewsUrlUtil;

import com.yhpl.utils.work.TrimFilter;

import com.yhpl.vo.NewChannalVo;

import com.yhpl.vo.NewChannalsVo;

import com.yhpl.vo.NewsVo;

import models.Channels;

import models.News;

import play.libs.Json;

import play.mvc.Controller;

import play.mvc.Result;

public class CaptureController extends Controller {

public static Result capture() {

return ok("111");

}

public static Result initChannels() {

NewChannalsVo chanals = (NewChannalsVo) JsonFileUtil.getGetUrlContentAsObject(NewsUrlUtil.getChannelUrl(),

NewChannalsVo.class);

if (chanals != null) {

NewChannalVo[] channelArray = chanals.gettList();

List<Channels> beans = new ArrayList<Channels>();

for (int i = 0; i < channelArray.length; i++) {

NewChannalVo jsonObj = channelArray;

Channels bean = new Channels();

bean.cname = jsonObj.getTname();

bean.cid = jsonObj.getTid();

bean.subnum = jsonObj.getSubnum();

beans.add(bean);

}

Ebean.beginTransaction();

for (int i = 0; i < channelArray.length; i++) {

Channels bean = beans.get(i);

Channels target = Channels.getChannelWithCname(bean.cname);

if (target == null) {

Ebean.save(bean);

} else {

System.out.println("exist -- " + target.cname);

}

}

Ebean.commitTransaction();

}

return ok("init Channels success");

}

public static Result initNews() {

Page<Channels> pageChannel = Channels.page(1, 20, "id", "asc");

if (pageChannel.getTotalRowCount() > 0) {

List<Channels> channelBeans = pageChannel.getList();

if (!CollectionUtil.isEmpty(channelBeans)) {

for (int i = 0; i < channelBeans.size(); i++) {

// A.解析数据

Channels channelBean = channelBeans.get(i);

String url = NewsUrlUtil.getChannelNewsUrlWithCidPageCount(channelBean.cid);

JsonNode node = JsonFileUtil.getGetUrlContentAsJsonNode(url);

ArrayNode arrayNodes = (ArrayNode) node.get(channelBean.cid);

Iterator<JsonNode> iter = arrayNodes.iterator();

List<News> mNews = new ArrayList<News>();

while (iter.hasNext()) {

JsonNode childNode = iter.next();

NewsVo childNews = Json.fromJson(childNode, NewsVo.class);

News news = new News();

news.cid = channelBean.cid;

news.cp = childNews.getSource();

news.icon = childNews.getImgsrc();

news.url = childNews.getUrl();

news.title = childNews.getTitle();

news.snapDetail = childNews.getDigest();

news.time = DateUtil.getDateFromString(childNews.getPtime());

mNews.add(news);

System.out.println("child:" + childNews);

}

// B.过滤数据库

CollectionUtil.trimListWithFilter(mNews, new TrimFilter<News>() {

@Override

public boolean isFilter(News t) {

return t != null && News.getNewsWithTitle(t.title) != null;

}

});

// C.POJO -> DB

if (!CollectionUtil.isEmpty(mNews)) {

System.out.println("save --- > :" + channelBean.cid);

Ebean.save(mNews);

}

}

}

}

return ok("init News success");

}

public static Result page(String table) {

if (News.T_NAME.equals(table)) {

Page<News> news = News.page(1, 10, "time", "desc", "netease");

int count = news.getTotalPageCount();

System.out.println("count is " + count);

List<News> mList = news.getList();

return ok(Json.toJson(mList));

} else {

return ok("not found");

}

}

}

1.频道采集：

public static Result initChannels() {

NewChannalsVo chanals = (NewChannalsVo) JsonFileUtil.getGetUrlContentAsObject(NewsUrlUtil.getChannelUrl(),

NewChannalsVo.class);

if (chanals != null) {

NewChannalVo[] channelArray = chanals.gettList();

List<Channels> beans = new ArrayList<Channels>();

for (int i = 0; i < channelArray.length; i++) {

NewChannalVo jsonObj = channelArray;

Channels bean = new Channels();

bean.cname = jsonObj.getTname();

bean.cid = jsonObj.getTid();

bean.subnum = jsonObj.getSubnum();

beans.add(bean);

}

Ebean.beginTransaction();

for (int i = 0; i < channelArray.length; i++) {

Channels bean = beans.get(i);

Channels target = Channels.getChannelWithCname(bean.cname);

if (target == null) {

Ebean.save(bean);

} else {

System.out.println("exist -- " + target.cname);

}

}

Ebean.commitTransaction();

}

return ok("init Channels success");

}

2.新闻采集：

public static Result initNews() {

Page<Channels> pageChannel = Channels.page(1, 20, "id", "asc");

if (pageChannel.getTotalRowCount() > 0) {

List<Channels> channelBeans = pageChannel.getList();

if (!CollectionUtil.isEmpty(channelBeans)) {

for (int i = 0; i < channelBeans.size(); i++) {

// A.解析数据

Channels channelBean = channelBeans.get(i);

String url = NewsUrlUtil.getChannelNewsUrlWithCidPageCount(channelBean.cid);

JsonNode node = JsonFileUtil.getGetUrlContentAsJsonNode(url);

ArrayNode arrayNodes = (ArrayNode) node.get(channelBean.cid);

Iterator<JsonNode> iter = arrayNodes.iterator();

List<News> mNews = new ArrayList<News>();

while (iter.hasNext()) {

JsonNode childNode = iter.next();

NewsVo childNews = Json.fromJson(childNode, NewsVo.class);

News news = new News();

news.cid = channelBean.cid;

news.cp = childNews.getSource();

news.icon = childNews.getImgsrc();

news.url = childNews.getUrl();

news.title = childNews.getTitle();

news.snapDetail = childNews.getDigest();

news.time = DateUtil.getDateFromString(childNews.getPtime());

mNews.add(news);

System.out.println("child:" + childNews);

}

// B.过滤数据库

CollectionUtil.trimListWithFilter(mNews, new TrimFilter<News>() {

@Override

public boolean isFilter(News t) {

return t != null && News.getNewsWithTitle(t.title) != null;

}

});

// C.POJO -> DB

if (!CollectionUtil.isEmpty(mNews)) {

System.out.println("save --- > :" + channelBean.cid);

Ebean.save(mNews);

}

}

}

}

return ok("init News success");

}

3.运行play之后，第一次初始化会要求add scripts，点击add即可。

扫码或添加微信号：坛友素质互助

「经管之家」APP：经管人学习、答疑、交友，就上经管之家！
免流量费下载资料----在经管之家app可以下载论坛上的所有资源，并且不额外收取下载高峰期的论坛币。
涵盖所有经管领域的优秀内容----覆盖经济、管理、金融投资、计量统计、数据分析、国贸、财会等专业的学习宝库，各类资料应有尽有。
来自五湖四海的经管达人----已经有上千万的经管人来到这里，你可以找到任何学科方向、有共同话题的朋友。
经管之家（原人大经济论坛），跨越高校的围墙，带你走进经管知识的新世界。
扫描下方二维码下载并注册APP