clunt
/
RuoYi-Cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1、36壁纸网爬虫定时任务实现

This commit is contained in:
xjs 2022-02-20 18:48:30 +08:00
parent 82a3137f0f
commit 8f36a77d2b
7 changed files with 107 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagic36wallpaperFeign.java Normal file
View File

@ -0,0 +1,24 @@
package com.xjs.business.webmagic;
import com.ruoyi.common.core.constant.ServiceNameConstants;
import com.ruoyi.common.core.domain.R;
import com.xjs.business.webmagic.factory.RemoteWebmagic36wallpaperFactory;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
/**
* 内部 调用 36壁纸网 爬虫定时任务
*
* @author xiejs
* @since 2022-02-20
*/
@FeignClient(contextId = "remoteWebmagic36wallpaperFeign",
value = ServiceNameConstants.BUSINESS_WEBMAGIC_SERVICE,
fallbackFactory = RemoteWebmagic36wallpaperFactory.class)
public interface RemoteWebmagic36wallpaperFeign {
@GetMapping("/_36wallpaper/taskForPRC")
public R _36wallpaperControllerTaskForPRC();
}

27
ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagic36wallpaperFactory.java Normal file
View File

@ -0,0 +1,27 @@
package com.xjs.business.webmagic.factory;
import com.ruoyi.common.core.domain.R;
import com.xjs.business.webmagic.RemoteWebmagic36wallpaperFeign;
import lombok.extern.log4j.Log4j2;
import org.springframework.cloud.openfeign.FallbackFactory;
import org.springframework.stereotype.Component;
/**
* 内部 调用 36壁纸网 爬虫定时任务降级
* @author xiejs
* @since 2022-02-20
*/
@Component
@Log4j2
public class RemoteWebmagic36wallpaperFactory implements FallbackFactory<RemoteWebmagic36wallpaperFeign> {
@Override
public RemoteWebmagic36wallpaperFeign create(Throwable cause) {
return new RemoteWebmagic36wallpaperFeign() {
@Override
public R _36wallpaperControllerTaskForPRC() {
log.error("新浪 爬虫定时任务 降级------服务可能正在运行");
return R.fail("降级处理------服务可能正在运行");
}
};
}
}

4
ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java
View File

@ -19,8 +19,8 @@ public class RemoteWebmagicCopyWritingNetworkFactory implements FallbackFactory<
return new RemoteWebmagicCopyWritingNetworkFeign() {
@Override
public R copyWritingNetworkTaskForPRC() {
log.error("文案网 爬虫定时任务 降级");
return R.fail("降级处理");
log.error("文案网 爬虫定时任务 降级------服务可能正在运行");
return R.fail("降级处理------服务可能正在运行");
}
};
}

4
ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicSinaFactory.java
View File

@ -20,8 +20,8 @@ public class RemoteWebmagicSinaFactory implements FallbackFactory<RemoteWebmagic
return new RemoteWebmagicSinaFeign() {
@Override
public R sinaTaskForPRC() {
log.error("新浪 爬虫定时任务 降级");
return R.fail("降级处理");
log.error("新浪 爬虫定时任务 降级------服务可能正在运行");
return R.fail("降级处理------服务可能正在运行");
}
};
}

42
ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/_36wallpaperTask.java Normal file
View File

@ -0,0 +1,42 @@
package com.xjs.job.task.webmagic;
import cn.hutool.core.date.DateUtil;
import com.ruoyi.common.core.domain.R;
import com.xjs.business.webmagic.RemoteWebmagic36wallpaperFeign;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
/**
* 爬虫 36壁纸网 定时任务
* @author xiejs
* @since 2022-02-20
*/
@Component("Wallpaper_36Task")
@Log4j2
public class _36wallpaperTask {
@Resource
private RemoteWebmagic36wallpaperFeign remoteWebmagic36wallpaperFeign;
/**
* 爬虫 36壁纸网 定时任务执行
*/
public void _36wallpaper() {
log.info("---------------爬虫-36壁纸网定时任务Start-------------------");
LocalDateTime localDateTime1 = DateUtil.date().toLocalDateTime();
R r = remoteWebmagic36wallpaperFeign._36wallpaperControllerTaskForPRC();
log.info("爬虫-36壁纸网定时任务结果:code={},msg={},data={}",r.getCode(),r.getMsg(),r.getData());
LocalDateTime localDateTime2 = DateUtil.date().toLocalDateTime();
long between = ChronoUnit.MILLIS.between(localDateTime1, localDateTime2);
log.info("爬虫-36壁纸网定时任务Job耗费时间:{}ms", between);
log.info("---------------爬虫-36壁纸网定时任务end---------------------");
}
}

11
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java
View File

@ -40,6 +40,9 @@ public class _36wallpaperProcessor implements PageProcessor {
*/
private static boolean init = false;
private static final String headerKey = "User-Agent";
private static final String headerValue = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
private static RedisService redisService;
@ -63,6 +66,7 @@ public class _36wallpaperProcessor implements PageProcessor {
private Site site = Site.me()
.addHeader(headerKey,headerValue)
.setCharset("utf8")//设置字符编码
.setTimeOut(2000)//设置超时时间
.setRetrySleepTime(200)//设置重试间隔时间
@ -160,6 +164,9 @@ public class _36wallpaperProcessor implements PageProcessor {
//循环次数存入redis中
Integer count = redisService.getCacheObject(REPTILE_COUNT);
if (count == null) {
count=0;
}
redisService.setCacheObject(REPTILE_COUNT, count+1);
} catch (Exception e) {
@ -239,8 +246,4 @@ public class _36wallpaperProcessor implements PageProcessor {
}
public static void main(String[] args) {
Spider.create(new _36wallpaperProcessor()).addUrl(_36_WALLPAPER_URL).thread(15).run();
}
}

3
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/aop/reptileLogAspect.java
View File

@ -97,6 +97,9 @@ public class reptileLogAspect {
if (obj instanceof Long) {
webmagicLog.setComplexRate((Long) obj);
}
if (obj instanceof Integer) {
webmagicLog.setComplexRate(Long.parseLong(obj.toString()));
}
this.saveData(webmagicLog);
}
}