1、36壁纸网爬虫实现并保存到磁盘然后持久化到数据库
This commit is contained in:
xjs
parent
491e100071
commit
82a3137f0f
|
|
@ -44,6 +44,10 @@ public class RedisConst {
|
|||
*/
|
||||
public static final String FORECAST_WEATHER = "weather:forecast";
|
||||
|
||||
/**
|
||||
* 爬虫记录循环次数常量信息
|
||||
*/
|
||||
public static final String REPTILE_COUNT= "reptile:count";
|
||||
|
||||
//-------------------有效时间-----------------------
|
||||
public static final Integer TRAN_DICT_EXPIRE = 1; //小时
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ public class ReptileConst {
|
|||
public static final String BOSS_JOB_URL= "https://www.zhipin.com";
|
||||
|
||||
|
||||
public static final String _36_WALLPAPER_URL= "https://www.3gbizhi.com/";
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
package com.xjs._36wallpaper.controller;
|
||||
|
||||
import com.ruoyi.common.core.domain.R;
|
||||
import com.xjs._36wallpaper.task._36wallpaperTask;
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* 36壁纸网爬虫controller
|
||||
* @author xiejs
|
||||
* @since 2022-02-20
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("_36wallpaper")
|
||||
@Api(tags = "爬虫模块-36壁纸网")
|
||||
public class _36wallpaperController {
|
||||
|
||||
@Autowired
|
||||
private _36wallpaperTask wallpaperTask;
|
||||
|
||||
|
||||
//----------------------远程rpc调用---------------------------
|
||||
@GetMapping("taskForPRC")
|
||||
@ApiOperation("供定时任务服务RPC远程调用")
|
||||
public R _36wallpaperControllerTaskForPRC() {
|
||||
Long count = wallpaperTask.reptileWallpaper();
|
||||
return R.ok(count);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package com.xjs._36wallpaper.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.xjs._36wallpaper.pojo._36wallpaper;
|
||||
|
||||
/**
|
||||
* 36壁纸网mapper
|
||||
* @author xiejs
|
||||
* @since 2022-02-20
|
||||
*/
|
||||
public interface _36wallpaperMapper extends BaseMapper<_36wallpaper> {
|
||||
|
||||
/**
|
||||
* 删除重复数据
|
||||
* @return int
|
||||
*/
|
||||
int deleteRepeatData();
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
package com.xjs._36wallpaper.pojo;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.FieldFill;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import com.ruoyi.common.core.annotation.Excel;
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 36壁纸网图片数据实体类
|
||||
* @author xiejs
|
||||
* @since 2022-02-20
|
||||
*/
|
||||
@Data
|
||||
@TableName("webmagic_36wallpaper")
|
||||
public class _36wallpaper implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/** 主键 */
|
||||
@TableId("id")
|
||||
private Long id;
|
||||
|
||||
/** 照片存放地址 */
|
||||
@Excel(name = "照片存放地址")
|
||||
private String pictureUrl;
|
||||
|
||||
/** 照片名称 */
|
||||
@Excel(name = "照片名称")
|
||||
private String pictureName;
|
||||
|
||||
/** 照片类型 */
|
||||
@Excel(name = "照片类型")
|
||||
private String type;
|
||||
|
||||
/** 照片标签(多个用 , 分割) */
|
||||
@Excel(name = "照片标签(多个用 , 分割)")
|
||||
private String label;
|
||||
|
||||
/** 创建时间 */
|
||||
@Excel(name = "创建时间",dateFormat = "yyyy-MM-dd HH:mm:ss")
|
||||
@TableField(fill = FieldFill.INSERT)
|
||||
private Date createTime;
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package com.xjs._36wallpaper.service;
|
||||
|
||||
import com.baomidou.mybatisplus.extension.service.IService;
|
||||
import com.xjs._36wallpaper.pojo._36wallpaper;
|
||||
|
||||
/**
|
||||
* 36壁纸网service接口
|
||||
* @author xiejs
|
||||
* @since 2022-02-20
|
||||
*/
|
||||
public interface _36wallpaperService extends IService<_36wallpaper> {
|
||||
|
||||
/**
|
||||
* 删除重复数据
|
||||
* @return int
|
||||
*/
|
||||
int deleteRepeatData();
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
package com.xjs._36wallpaper.service.impl;
|
||||
|
||||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
||||
import com.xjs._36wallpaper.mapper._36wallpaperMapper;
|
||||
import com.xjs._36wallpaper.pojo._36wallpaper;
|
||||
import com.xjs._36wallpaper.service._36wallpaperService;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
/**
|
||||
* 36壁纸网service实现
|
||||
* @author xiejs
|
||||
* @since 2022-02-20
|
||||
*/
|
||||
@Service
|
||||
public class _36wallpaperServiceImpl extends ServiceImpl<_36wallpaperMapper, _36wallpaper> implements _36wallpaperService {
|
||||
|
||||
|
||||
@Resource
|
||||
private _36wallpaperMapper wallpaperMapper;
|
||||
|
||||
@Override
|
||||
public int deleteRepeatData() {
|
||||
return wallpaperMapper.deleteRepeatData();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package com.xjs._36wallpaper.task;
|
||||
|
||||
import com.xjs._36wallpaper.webmagic._36wallpaperProcessor;
|
||||
import com.xjs.annotation.ReptileLog;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import static com.xjs.consts.ReptileConst._36_WALLPAPER_URL;
|
||||
|
||||
/**
|
||||
* 36壁纸网爬虫任务
|
||||
* @author xiejs
|
||||
* @since 2022-02-20
|
||||
*/
|
||||
@Component
|
||||
@Log4j2
|
||||
public class _36wallpaperTask {
|
||||
|
||||
@Autowired
|
||||
private _36wallpaperProcessor wallpaperProcessor;
|
||||
|
||||
|
||||
/**
|
||||
* 提供定时任务调取
|
||||
* @return 循环次数
|
||||
*/
|
||||
@ReptileLog(name = "36壁纸网", url = _36_WALLPAPER_URL)
|
||||
public Long reptileWallpaper() {
|
||||
return wallpaperProcessor.run();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,9 +1,29 @@
|
|||
package com.xjs._36wallpaper.webmagic;
|
||||
|
||||
import com.ruoyi.common.redis.service.RedisService;
|
||||
import com.xjs._36wallpaper.pojo._36wallpaper;
|
||||
import com.xjs._36wallpaper.service._36wallpaperService;
|
||||
import lombok.extern.log4j.Log4j2;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
|
||||
import us.codecraft.webmagic.scheduler.QueueScheduler;
|
||||
import us.codecraft.webmagic.selector.Selectable;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static com.xjs.consts.RedisConst.REPTILE_COUNT;
|
||||
import static com.xjs.consts.ReptileConst._36_WALLPAPER_URL;
|
||||
|
||||
/**
|
||||
* 36壁纸网站 爬虫处理
|
||||
|
|
@ -11,22 +31,44 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
* @author xiejs
|
||||
* @since 2022-02-19
|
||||
*/
|
||||
@Log4j2
|
||||
@Component
|
||||
public class _36wallpaperProcessor implements PageProcessor {
|
||||
|
||||
private static final String URL = "https://www.3gbizhi.com/";
|
||||
/**
|
||||
* 是否全网爬虫
|
||||
*/
|
||||
private static boolean init = false;
|
||||
|
||||
|
||||
private static RedisService redisService;
|
||||
|
||||
@Autowired
|
||||
public void setRedisService(RedisService redisService) {
|
||||
_36wallpaperProcessor.redisService = redisService;
|
||||
}
|
||||
|
||||
|
||||
private static _36wallpaperService wallpaperService;
|
||||
|
||||
@Autowired
|
||||
public void setWallpaperService(_36wallpaperService wallpaperService) {
|
||||
_36wallpaperProcessor.wallpaperService = wallpaperService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 图片保存到磁盘的路径
|
||||
*/
|
||||
private static final String path = "D:\\Dev\\WebCrawler\\36wallpaper";
|
||||
|
||||
|
||||
private Site site = Site.me()
|
||||
.setCharset("uft8")//设置字符编码
|
||||
.setTimeOut(10000)//设置超时时间
|
||||
.setRetrySleepTime(2000)//设置重试间隔时间
|
||||
.setCycleRetryTimes(3)//设置重试次数
|
||||
.setSleepTime(10)//设置两个页面之间的间隔时间
|
||||
;
|
||||
|
||||
public void setSite(Site site) {
|
||||
this.site = site;
|
||||
}
|
||||
.setCharset("utf8")//设置字符编码
|
||||
.setTimeOut(2000)//设置超时时间
|
||||
.setRetrySleepTime(200)//设置重试间隔时间
|
||||
.setCycleRetryTimes(6)//设置重试次数
|
||||
.setSleepTime(1)//设置两个页面之间的间隔时间
|
||||
;
|
||||
|
||||
//解析页面
|
||||
@Override
|
||||
|
|
@ -44,12 +86,87 @@ public class _36wallpaperProcessor implements PageProcessor {
|
|||
//第三种写法:正则表达式
|
||||
page.putField("url_regex", page.getHtml().css(".tmenu li a").regex(".*图片*.*").all());*/
|
||||
|
||||
// todo 爬取36壁纸图片
|
||||
try {
|
||||
//获取链接
|
||||
page.addTargetRequests(page.getHtml().css(".topmenuc .tmenu li").links().all());
|
||||
|
||||
//爬取所有分页---后期这两段可以注释
|
||||
//当init为true时全网都爬
|
||||
if (init) {
|
||||
page.addTargetRequests(page.getHtml().css("#pageNum a").links().all());
|
||||
}
|
||||
|
||||
page.addTargetRequests(page.getHtml().css(".contlistw > .cl > li > a").links().all());
|
||||
|
||||
|
||||
//获取链接
|
||||
page.addTargetRequests(page.getHtml().css(".tmenu li").links().all());
|
||||
page.putField("url", page.getHtml().css(".imgw").links().all());
|
||||
ArrayList<_36wallpaper> wallpapers = new ArrayList<>();
|
||||
List<Selectable> bodyNote = page.getHtml().css("body").nodes();
|
||||
for (Selectable body : bodyNote) {
|
||||
_36wallpaper wallpaper = new _36wallpaper();
|
||||
|
||||
//爬取图片分类
|
||||
String titleHtml = body.css(".catpos a:nth-child(3)").get();
|
||||
|
||||
String title = "null";
|
||||
if (titleHtml != null) {
|
||||
Document titleDom = Jsoup.parse(titleHtml);
|
||||
title = titleDom.text();
|
||||
wallpaper.setType(title);
|
||||
}
|
||||
|
||||
|
||||
//爬取图片名称
|
||||
String html = body.css(".showtitle h2").get();
|
||||
String pictureName = "null";
|
||||
if (html != null) {
|
||||
pictureName = Jsoup.parse(html).text();
|
||||
wallpaper.setPictureName(pictureName);
|
||||
}
|
||||
|
||||
|
||||
//爬取图片路径
|
||||
String link = body.css(".morew").links().get();
|
||||
wallpaper.setPictureUrl(link);
|
||||
|
||||
|
||||
//保存到磁盘
|
||||
if (link != null) {
|
||||
String thisPath = path + File.separator + title;
|
||||
downloadPicture(link, thisPath, pictureName + ".jpg");
|
||||
}
|
||||
|
||||
//爬取图片标签
|
||||
List<String> tagList = body.css(".showcontw > .showtaglistw a").all();
|
||||
StringBuilder bf = new StringBuilder();
|
||||
|
||||
for (int i = 0; i < tagList.size(); i++) {
|
||||
String text = Jsoup.parse(tagList.get(i)).text();
|
||||
if (i != tagList.size() - 1) {
|
||||
bf.append(text + ",");
|
||||
} else {
|
||||
bf.append(text);
|
||||
}
|
||||
}
|
||||
|
||||
wallpaper.setLabel(bf.toString());
|
||||
|
||||
if (wallpaper.getPictureUrl() != null) {
|
||||
wallpapers.add(wallpaper);
|
||||
}
|
||||
}
|
||||
|
||||
//持久化
|
||||
wallpaperService.saveBatch(wallpapers, 25);
|
||||
|
||||
//循环次数存入redis中
|
||||
Integer count = redisService.getCacheObject(REPTILE_COUNT);
|
||||
redisService.setCacheObject(REPTILE_COUNT, count+1);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
redisService.expire(REPTILE_COUNT, 1, TimeUnit.HOURS);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -61,14 +178,69 @@ public class _36wallpaperProcessor implements PageProcessor {
|
|||
|
||||
/**
|
||||
* 执行爬虫
|
||||
*
|
||||
* @return 返回循环次数
|
||||
*/
|
||||
public void run() {
|
||||
Spider.create(new _36wallpaperProcessor()).addUrl(URL).thread(5).runAsync();
|
||||
public Long run() {
|
||||
Spider.create(new _36wallpaperProcessor()).addUrl(_36_WALLPAPER_URL).thread(20)
|
||||
.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(110000)))
|
||||
.run();
|
||||
|
||||
//删除重复数据
|
||||
int count = wallpaperService.deleteRepeatData();
|
||||
log.info("36壁纸删除重复数据数：" + count);
|
||||
|
||||
//从redis中获取循环次数
|
||||
Integer cache = redisService.getCacheObject(REPTILE_COUNT);
|
||||
redisService.deleteObject(REPTILE_COUNT);
|
||||
|
||||
if (cache != null) {
|
||||
return Long.valueOf(cache);
|
||||
}
|
||||
return 0L;
|
||||
}
|
||||
|
||||
|
||||
//链接url下载图片
|
||||
private void downloadPicture(String urlList, String path, String fileName) {
|
||||
java.net.URL url = null;
|
||||
try {
|
||||
url = new URL(urlList);
|
||||
DataInputStream dataInputStream = new DataInputStream(url.openStream());
|
||||
|
||||
File file = new File(path);
|
||||
if (!file.exists()) {
|
||||
file.mkdir();
|
||||
}
|
||||
|
||||
//替换\ 防止报错
|
||||
if (fileName.contains("/") || fileName.contains("\\")) {
|
||||
fileName = fileName.replace("/", "-");
|
||||
}
|
||||
|
||||
String absolutePath = file.getAbsolutePath() + File.separator + fileName;
|
||||
|
||||
FileOutputStream fileOutputStream = new FileOutputStream(absolutePath);
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
|
||||
byte[] buffer = new byte[1024];
|
||||
int length;
|
||||
|
||||
while ((length = dataInputStream.read(buffer)) > 0) {
|
||||
output.write(buffer, 0, length);
|
||||
}
|
||||
|
||||
fileOutputStream.write(output.toByteArray());
|
||||
dataInputStream.close();
|
||||
fileOutputStream.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
Spider.create(new _36wallpaperProcessor()).addUrl("https://www.3gbizhi.com/").thread(5).run();
|
||||
Spider.create(new _36wallpaperProcessor()).addUrl(_36_WALLPAPER_URL).thread(15).run();
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!DOCTYPE mapper
|
||||
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
|
||||
<mapper namespace="com.xjs._36wallpaper.mapper._36wallpaperMapper">
|
||||
|
||||
|
||||
|
||||
<delete id="deleteRepeatData">
|
||||
delete from webmagic_36wallpaper where id not in (
|
||||
SELECT
|
||||
t.min_id
|
||||
FROM
|
||||
( SELECT min( id ) AS min_id FROM webmagic_36wallpaper GROUP BY picture_url ) AS t
|
||||
)
|
||||
</delete>
|
||||
</mapper>
|
||||
Loading…
Reference in New Issue