From add64ea618479144b0c072f1ded4cc04cfdcf620 Mon Sep 17 00:00:00 2001 From: xjs <1294405880@qq.com> Date: Mon, 21 Feb 2022 17:15:06 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E5=BD=9336=E5=A3=81=E7=BA=B8?= =?UTF-8?q?=E7=88=AC=E8=99=AB=E6=AD=A3=E5=9C=A8=E6=89=A7=E8=A1=8C=E6=97=B6?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E5=8F=AF=E4=BF=AE=E6=94=B9=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=202=E3=80=8136=E5=A3=81=E7=BA=B8=E6=96=B0=E5=A2=9E=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E6=9D=83=E9=99=90=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../webmagic/_36wallpaper/settings/index.vue | 30 +++++--- .../webmagic/_36wallpaper/show/index.vue | 75 ++++++++++++++++++- .../controller/_36wallpaperController.java | 4 + .../service/impl/_36wallpaperServiceImpl.java | 6 ++ .../webmagic/_36wallpaperProcessor.java | 6 ++ 5 files changed, 111 insertions(+), 10 deletions(-) diff --git a/ruoyi-ui/src/views/business/webmagic/_36wallpaper/settings/index.vue b/ruoyi-ui/src/views/business/webmagic/_36wallpaper/settings/index.vue index 8d8a01c1..72c1173d 100644 --- a/ruoyi-ui/src/views/business/webmagic/_36wallpaper/settings/index.vue +++ b/ruoyi-ui/src/views/business/webmagic/_36wallpaper/settings/index.vue @@ -13,13 +13,13 @@ - + - + @@ -30,9 +30,21 @@ - 提交 + 提交 + 重置 - 恢复默认 + 恢复默认 @@ -43,7 +55,7 @@ diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/controller/_36wallpaperController.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/controller/_36wallpaperController.java index 4e1c18ca..c9595325 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/controller/_36wallpaperController.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/controller/_36wallpaperController.java @@ -3,6 +3,7 @@ package com.xjs._36wallpaper.controller; import com.alibaba.fastjson.JSONObject; import com.ruoyi.common.core.domain.R; import com.ruoyi.common.core.web.domain.AjaxResult; +import com.ruoyi.common.security.annotation.RequiresPermissions; import com.xjs._36wallpaper.service._36wallpaperService; import com.xjs._36wallpaper.task._36wallpaperTask; import com.xjs.web.MyBaseController; @@ -33,6 +34,7 @@ public class _36wallpaperController extends MyBaseController { @GetMapping("getSettings") @ApiOperation("获取参数配置") + @RequiresPermissions("webmagic:_36wallpaper:list") public AjaxResult getSettings() { JSONObject jsonObject = wallpaperService.getSettings(); if (Objects.nonNull(jsonObject)) { @@ -44,6 +46,7 @@ public class _36wallpaperController extends MyBaseController { @PutMapping("updateSettings") @ApiOperation("修改参数配置") + @RequiresPermissions("webmagic:_36wallpaper:update") public AjaxResult updateSettings(@RequestParam("json") String json) { boolean b=wallpaperService.updateSettings(json); return toAjax(b); @@ -51,6 +54,7 @@ public class _36wallpaperController extends MyBaseController { @PutMapping("reset") @ApiOperation("重置参数配置") + @RequiresPermissions("webmagic:_36wallpaper:update") public AjaxResult resetSettings() { boolean b=wallpaperService.resetSettings(); return toAjax(b); diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/service/impl/_36wallpaperServiceImpl.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/service/impl/_36wallpaperServiceImpl.java index fe079fb7..fb25b1cf 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/service/impl/_36wallpaperServiceImpl.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/service/impl/_36wallpaperServiceImpl.java @@ -17,6 +17,7 @@ import javax.annotation.Resource; import java.util.regex.Pattern; import static com.xjs._36wallpaper.consts._36wallpaperConst.*; +import static com.xjs.consts.RedisConst.REPTILE_COUNT; import static com.xjs.consts.RegexConst.FILE_PATH_REGEX; /** @@ -74,6 +75,11 @@ public class _36wallpaperServiceImpl extends ServiceImpl<_36wallpaperMapper, _36 @Override public boolean updateSettings(String json) { + //判断爬虫是否正在执行，正在执行不可修改！ + if(redisService.hasKey(REPTILE_COUNT)){ + throw new BusinessException("爬虫正在执行中！暂时无法修改，请稍后再试"); + } + //校验json格式是否正确 try { JSONObject jsonObject = JSONObject.parseObject(json); diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java index 2b898364..8f026d2b 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java @@ -271,6 +271,11 @@ public class _36wallpaperProcessor implements PageProcessor { * @return 返回循环次数 */ public Long run() { + //创建下载器Downloader + //HttpClientDownloader downloader = new HttpClientDownloader(); + //给下载器设置代理服务器 + //downloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("60.191.11.249",3128))); + //执行爬虫 Spider.create(new _36wallpaperProcessor()) .addUrl(_36_WALLPAPER_URL)//设置爬取地址 @@ -278,6 +283,7 @@ public class _36wallpaperProcessor implements PageProcessor { .setScheduler(new QueueScheduler() .setDuplicateRemover(new BloomFilterDuplicateRemover(110000)))//设置url去重过滤器 //.addPipeline(wallpaperPipeline)//设置爬取之后的数据操作 + //.setDownloader(downloader)//设置下载器 .run();//执行 //删除重复数据