diff --git a/ruoyi-ui/src/api/business/log/reptilelog.js b/ruoyi-ui/src/api/business/log/reptilelog.js new file mode 100644 index 00000000..bd6e0c3f --- /dev/null +++ b/ruoyi-ui/src/api/business/log/reptilelog.js @@ -0,0 +1,18 @@ +import request from '@/utils/request' + +// 查询爬虫日志列表 +export function listWebmagicLog(query) { + return request({ + url: '/log/reptileLog/list', + method: 'get', + params: query + }) +} + +// 删除爬虫日志 +export function delWebmagicLog(id) { + return request({ + url: '/log/reptileLog/' + id, + method: 'delete' + }) +} diff --git a/ruoyi-ui/src/views/business/log/apilog/index.vue b/ruoyi-ui/src/views/business/log/apilog/index.vue index c59f2c79..33a768db 100644 --- a/ruoyi-ui/src/views/business/log/apilog/index.vue +++ b/ruoyi-ui/src/views/business/log/apilog/index.vue @@ -131,7 +131,7 @@ @@ -205,7 +205,7 @@ diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/controller/WebmagicLogController.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/controller/WebmagicLogController.java index 76ff5bc0..aa528c8d 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/controller/WebmagicLogController.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/controller/WebmagicLogController.java @@ -1,38 +1,82 @@ package com.xjs.reptileLog.controller; import com.ruoyi.common.core.domain.R; +import com.ruoyi.common.core.utils.poi.ExcelUtil; +import com.ruoyi.common.core.web.domain.AjaxResult; +import com.ruoyi.common.core.web.page.TableDataInfo; +import com.ruoyi.common.log.annotation.Log; +import com.ruoyi.common.log.enums.BusinessType; +import com.ruoyi.common.security.annotation.RequiresPermissions; import com.xjs.reptileLog.domain.WebmagicLog; import com.xjs.reptileLog.service.WebmagicLogService; +import com.xjs.web.MyBaseController; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletResponse; +import java.util.List; /** * 爬虫日志控制器 + * * @author xiejs * @since 2022-02-17 */ @RestController @RequestMapping("reptileLog") @Api(tags = "业务模块-爬虫日志") -public class WebmagicLogController { +public class WebmagicLogController extends MyBaseController { @Autowired private WebmagicLogService webmagicLogService; - //-----------------------内部调用rpc------------------------ @PostMapping("saveForPRC") @ApiOperation("供AOP切面RPC远程调用") public R saveReptileLog(@RequestBody WebmagicLog webmagicLog) { boolean save = webmagicLogService.save(webmagicLog); - return save?R.ok():R.fail(); + return save ? R.ok() : R.fail(); } + + //------------------代码生成---------------------------- + + /** + * 查询爬虫日志列表 + */ + @RequiresPermissions("log:webmagicLog:list") + @GetMapping("/list") + public TableDataInfo list(WebmagicLog webmagicLog) { + startPage(); + List list = webmagicLogService.selectWebmagicLogList(webmagicLog); + return getDataTable(list); + } + + /** + * 导出爬虫日志列表 + */ + @RequiresPermissions("log:webmagicLog:export") + @Log(title = "爬虫日志", businessType = BusinessType.EXPORT) + @PostMapping("/export") + public void export(HttpServletResponse response, WebmagicLog webmagicLog) { + List list = webmagicLogService.selectWebmagicLogList(webmagicLog); + ExcelUtil util = new ExcelUtil<>(WebmagicLog.class); + util.exportExcel(response, list, "爬虫日志数据"); + } + + /** + * 删除爬虫日志 + */ + @RequiresPermissions("log:webmagicLog:remove") + @Log(title = "爬虫日志", businessType = BusinessType.DELETE) + @DeleteMapping("/{ids}") + public AjaxResult remove(@PathVariable Long[] ids) { + return toAjax(webmagicLogService.deleteWebmagicLogByIds(ids)); + } + + } diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/domain/WebmagicLog.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/domain/WebmagicLog.java index 164961aa..fce709e4 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/domain/WebmagicLog.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/domain/WebmagicLog.java @@ -47,4 +47,19 @@ public class WebmagicLog implements Serializable { @TableField(fill = FieldFill.INSERT) private Date createTime; + @TableField(exist = false) + private Date endCreateTime; + + /** + * 查询条件：耗费时间Start + */ + @TableField(exist = false) + private Long beginRequestTime; + + /** + * 查询条件：耗费时间End + */ + @TableField(exist = false) + private Long endRequestTime; + } diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/mapper/WebmagicLogMapper.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/mapper/WebmagicLogMapper.java index a083ca3b..ff980f24 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/mapper/WebmagicLogMapper.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/mapper/WebmagicLogMapper.java @@ -3,6 +3,8 @@ package com.xjs.reptileLog.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.xjs.reptileLog.domain.WebmagicLog; +import java.util.List; + /** * WebmagicLog mapper * @author xiejs @@ -10,4 +12,30 @@ import com.xjs.reptileLog.domain.WebmagicLog; */ public interface WebmagicLogMapper extends BaseMapper { + + //--------------------代码生成----------------------- + + /** + * 查询爬虫日志列表 + * + * @param webmagicLog 爬虫日志 + * @return 爬虫日志集合 + */ + public List selectWebmagicLogList(WebmagicLog webmagicLog); + + /** + * 删除爬虫日志 + * + * @param id 爬虫日志主键 + * @return 结果 + */ + public int deleteWebmagicLogById(Long id); + + /** + * 批量删除爬虫日志 + * + * @param ids 需要删除的数据主键集合 + * @return 结果 + */ + public int deleteWebmagicLogByIds(Long[] ids); } diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/WebmagicLogService.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/WebmagicLogService.java index a262f108..0ca46bc5 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/WebmagicLogService.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/WebmagicLogService.java @@ -3,6 +3,8 @@ package com.xjs.reptileLog.service; import com.baomidou.mybatisplus.extension.service.IService; import com.xjs.reptileLog.domain.WebmagicLog; +import java.util.List; + /** * 爬虫日志 Service接口 * @author xiejs @@ -10,4 +12,30 @@ import com.xjs.reptileLog.domain.WebmagicLog; */ public interface WebmagicLogService extends IService { + //------------------------代码生成------------------------- + + /** + * 查询爬虫日志列表 + * + * @param webmagicLog 爬虫日志 + * @return 爬虫日志集合 + */ + public List selectWebmagicLogList(WebmagicLog webmagicLog); + + /** + * 批量删除爬虫日志 + * + * @param ids 需要删除的爬虫日志主键集合 + * @return 结果 + */ + public int deleteWebmagicLogByIds(Long[] ids); + + /** + * 删除爬虫日志信息 + * + * @param id 爬虫日志主键 + * @return 结果 + */ + public int deleteWebmagicLogById(Long id); + } diff --git a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/impl/WebmagicLogServiceImpl.java b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/impl/WebmagicLogServiceImpl.java index 2515f945..58cce084 100644 --- a/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/impl/WebmagicLogServiceImpl.java +++ b/xjs-business/xjs-business-log/src/main/java/com/xjs/reptileLog/service/impl/WebmagicLogServiceImpl.java @@ -6,6 +6,9 @@ import com.xjs.reptileLog.mapper.WebmagicLogMapper; import com.xjs.reptileLog.service.WebmagicLogService; import org.springframework.stereotype.Service; +import javax.annotation.Resource; +import java.util.List; + /** * @author xiejs * @since 2022-02-17 @@ -13,4 +16,43 @@ import org.springframework.stereotype.Service; @Service public class WebmagicLogServiceImpl extends ServiceImpl implements WebmagicLogService { + @Resource + private WebmagicLogMapper webmagicLogMapper; + + + //------------------------代码生成----------------------------- + + /** + * 查询爬虫日志列表 + * + * @param webmagicLog 爬虫日志 + * @return 爬虫日志 + */ + @Override + public List selectWebmagicLogList(WebmagicLog webmagicLog) { + return webmagicLogMapper.selectWebmagicLogList(webmagicLog); + } + + /** + * 批量删除爬虫日志 + * + * @param ids 需要删除的爬虫日志主键 + * @return 结果 + */ + @Override + public int deleteWebmagicLogByIds(Long[] ids) { + return webmagicLogMapper.deleteWebmagicLogByIds(ids); + } + + /** + * 删除爬虫日志信息 + * + * @param id 爬虫日志主键 + * @return 结果 + */ + @Override + public int deleteWebmagicLogById(Long id) { + return webmagicLogMapper.deleteWebmagicLogById(id); + } + } diff --git a/xjs-business/xjs-business-log/src/main/resources/mapper/log/WebmagicLogMapper.xml b/xjs-business/xjs-business-log/src/main/resources/mapper/log/WebmagicLogMapper.xml new file mode 100644 index 00000000..5f341164 --- /dev/null +++ b/xjs-business/xjs-business-log/src/main/resources/mapper/log/WebmagicLogMapper.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + select id, name, url, request_time, create_time, complex_rate, status + from webmagic_log + + + + + + delete + from webmagic_log + where id = #{id} + + + + delete from webmagic_log where id in + + #{id} + + + + + \ No newline at end of file diff --git a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java index 2b3104f6..3efcfc7d 100644 --- a/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java +++ b/xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; /** * 新浪新闻爬虫任务 + * * @author xiejs * @since 2022-02-15 */ @@ -44,7 +45,7 @@ public class SinaNewsTask { Document document = Jsoup.parse(html); - count = this.parse(document,count); + count = this.parse(document, count); } catch (Exception e) { log.error(e.getMessage()); } @@ -55,36 +56,32 @@ public class SinaNewsTask { * 解析dom * * @param document dom - * @param count 循环次数 + * @param count 循环次数 */ - private Long parse(Document document,Long count) { - try { - //获取子链接 - Elements nav_mod_1 = document.getElementsByClass("nav-mod-1"); - Elements link = nav_mod_1.select("ul > li > a"); - List> hrefList = link.stream().map(a -> { - String href = a.attr("href"); - String text = a.text(); - Map map = new HashMap<>(); - map.put(text, href); - return map; - }).collect(Collectors.toList()); - hrefList.removeIf(s -> s.containsKey("javascript:;")); + private Long parse(Document document, Long count) { + //获取子链接 + Elements nav_mod_1 = document.getElementsByClass("nav-mod-1"); + Elements link = nav_mod_1.select("ul > li > a"); + List> hrefList = link.stream().map(a -> { + String href = a.attr("href"); + String text = a.text(); + Map map = new HashMap<>(); + map.put(text, href); + return map; + }).collect(Collectors.toList()); + hrefList.removeIf(s -> s.containsKey("javascript:;")); - for (Map map : hrefList) { - Set> entrySet = map.entrySet(); - for (Map.Entry entry : entrySet) { - String html = httpUtils.doGetHtml(entry.getValue()); - Document docChild = Jsoup.parse(html); + for (Map map : hrefList) { + Set> entrySet = map.entrySet(); + for (Map.Entry entry : entrySet) { + String html = httpUtils.doGetHtml(entry.getValue()); + Document docChild = Jsoup.parse(html); - //计数 - count++; + Long newCount = this.parseChile(docChild, entry.getKey(), count); - count =this.parseChile(docChild, entry.getKey(),count); - } + count = count + newCount; } - } catch (Exception e) { - log.error(e.getMessage()); + } return count; } @@ -95,7 +92,7 @@ public class SinaNewsTask { * @param docChild 子 * @param key key */ - private Long parseChile(Document docChild, String key,Long count) { + private Long parseChile(Document docChild, String key, Long count) { try { Elements a = docChild.getElementsByTag("a"); ArrayList link = new ArrayList<>();