clunt
/
RuoYi-Cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1、httpUtils类修改异常打印 

2、apiLog切入类修改执行成功失败逻辑
3、文案网前端数据展示实现
This commit is contained in:
xjs 2022-02-17 09:57:14 +08:00
parent cf8b312c88
commit 9f150e483c
20 changed files with 644 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/RemoteWebmagicCopyWritingNetworkFeign.java Normal file
View File

@ -0,0 +1,22 @@
package com.xjs.business.webmagic;
import com.ruoyi.common.core.constant.ServiceNameConstants;
import com.ruoyi.common.core.domain.R;
import com.xjs.business.webmagic.factory.RemoteWebmagicCopyWritingNetworkFactory;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
/**
* 内部 调用 文案网 爬虫定时任务
*
* @author xiejs
* @since 2022-02-17
*/
@FeignClient(contextId = "remoteWebmagicCopyWritingNetworkFeign",
value = ServiceNameConstants.BUSINESS_WEBMAGIC_SERVICE,
fallbackFactory = RemoteWebmagicCopyWritingNetworkFactory.class)
public interface RemoteWebmagicCopyWritingNetworkFeign {
@GetMapping("copyWritingNetwork/taskForPRC")
R copyWritingNetworkTaskForPRC();
}

27
ruoyi-api/ruoyi-api-system/src/main/java/com/xjs/business/webmagic/factory/RemoteWebmagicCopyWritingNetworkFactory.java Normal file
View File

@ -0,0 +1,27 @@
package com.xjs.business.webmagic.factory;
import com.ruoyi.common.core.domain.R;
import com.xjs.business.webmagic.RemoteWebmagicCopyWritingNetworkFeign;
import lombok.extern.log4j.Log4j2;
import org.springframework.cloud.openfeign.FallbackFactory;
import org.springframework.stereotype.Component;
/**
* 内部 调用 文案网 爬虫定时任务 降级
* @author xiejs
* @since 2022-02-17
*/
@Component
@Log4j2
public class RemoteWebmagicCopyWritingNetworkFactory implements FallbackFactory<RemoteWebmagicCopyWritingNetworkFeign> {
@Override
public RemoteWebmagicCopyWritingNetworkFeign create(Throwable cause) {
return new RemoteWebmagicCopyWritingNetworkFeign() {
@Override
public R copyWritingNetworkTaskForPRC() {
log.error("文案网 爬虫定时任务 降级");
return R.fail("降级处理");
}
};
}
}

40
ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/CopyWritingNetworkTask.java Normal file
View File

@ -0,0 +1,40 @@
package com.xjs.job.task.webmagic;
import cn.hutool.core.date.DateUtil;
import com.ruoyi.common.core.domain.R;
import com.xjs.business.webmagic.RemoteWebmagicCopyWritingNetworkFeign;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
/**
* 爬虫 文案网 定时任务
* @author xiejs
* @since 2022-02-17
*/
@Component("CopyWritingNetworkTask")
@Log4j2
public class CopyWritingNetworkTask {
@Resource
private RemoteWebmagicCopyWritingNetworkFeign remoteWebmagicCopyWritingNetworkFeign;
/**
* 爬虫 文案网 定时任务执行
*/
public void copyWritingNetwork() {
log.info("---------------爬虫-文案网定时任务Start-------------------");
LocalDateTime localDateTime1 = DateUtil.date().toLocalDateTime();
R r = remoteWebmagicCopyWritingNetworkFeign.copyWritingNetworkTaskForPRC();
log.info("爬虫-文案网定时任务结果:code={},msg={},data={}",r.getCode(),r.getMsg(),r.getData());
LocalDateTime localDateTime2 = DateUtil.date().toLocalDateTime();
long between = ChronoUnit.MILLIS.between(localDateTime1, localDateTime2);
log.info("爬虫-文案网定时任务Job耗费时间:{}ms", between);
log.info("---------------爬虫-文案网定时任务end---------------------");
}
}

2
ruoyi-modules/ruoyi-job/src/main/java/com/xjs/job/task/webmagic/SinaTask.java
View File

@ -22,7 +22,7 @@ public class SinaTask {
private RemoteWebmagicSinaFeign remoteWebmagicSinaFeign;
/**
* 任务执行
* 爬虫 新浪新闻 定时任务执行
*/
public void sinaNews() {
log.info("---------------爬虫-新浪新闻定时任务Start-------------------");

18
ruoyi-ui/src/api/business/webmagic/copywritingnetwork/copyWritingNetwork.js Normal file
View File

@ -0,0 +1,18 @@
import request from '@/utils/request'
// 查询文案网列表
export function listCopyWritingNetwork(query) {
return request({
url: '/webmagic/copyWritingNetwork/list',
method: 'get',
params: query
})
}
// 删除文案网
export function delCopyWritingNetwork(id) {
return request({
url: '/webmagic/copyWritingNetwork/' + id,
method: 'delete'
})
}

20
ruoyi-ui/src/views/business/log/apilog/index.vue
View File

@ -129,7 +129,11 @@
<el-table-column label="接口名称" align="center" prop="apiName" :show-overflow-tooltip="true"/>
<el-table-column label="请求URL" align="center" prop="url" :show-overflow-tooltip="true"/>
<el-table-column label="请求方法" align="center" prop="method" :show-overflow-tooltip="true"/>
<el-table-column label="请求参数" align="center" prop="request" :show-overflow-tooltip="true"/>
<el-table-column label="请求参数" align="center" prop="request" :show-overflow-tooltip="true">
<template slot-scope="scope">
<span>{{scope.row.request!==""?scope.row.request:"-"}}</span>
</template>
</el-table-column>
<el-table-column label="响应参数" align="center" prop="response" :show-overflow-tooltip="true"/>
<el-table-column label="创建时间" align="center" prop="createTime" :show-overflow-tooltip="true"/>
<el-table-column label="是否请求成功" align="center" prop="isSuccess" :show-overflow-tooltip="true">
@ -201,11 +205,11 @@
</template>
<script>
import {listLog, getLog, delLog,getApiName} from "@/api/business/log/apilog";
import {listLog, getLog, delLog, getApiName} from "@/api/business/log/apilog";
export default {
name: "Apilog",
dicts: ['request_status','request_method'],
dicts: ['request_status', 'request_method'],
data() {
return {
// 遮罩层
@ -246,7 +250,7 @@ export default {
daterangeCreateTime: [],
//api名称
apiName:[],
apiName: [],
};
},
created() {
@ -256,8 +260,8 @@ export default {
methods: {
//获取所有api名称
getApiName() {
getApiName().then(res =>{
this.apiName=res.data
getApiName().then(res => {
this.apiName = res.data
})
},
@ -322,8 +326,8 @@ export default {
/** 重置按钮操作 */
resetQuery() {
this.daterangeCreateTime = [];
this.queryParams.createTime=null
this.queryParams.endCreateTime=null
this.queryParams.createTime = null
this.queryParams.endCreateTime = null
this.resetForm("queryForm");
this.handleQuery();
},

269
ruoyi-ui/src/views/business/webmagic/copywritingnetwork/index.vue Normal file
View File

@ -0,0 +1,269 @@
<template>
<div class="app-container">
<el-form :model="queryParams" ref="queryForm" :inline="true" v-show="showSearch" label-width="68px">
<!-- <el-form-item label="文案标签" prop="type">
<el-select v-model="queryParams.type" placeholder="请选择文案标签" clearable size="small">
<el-option
v-for="dict in dict.type.sys_user_sex"
:key="dict.value"
:label="dict.label"
:value="dict.value"
/>
</el-select>
</el-form-item>-->
<el-form-item label="文案主题" prop="theme">
<el-input
v-model="queryParams.theme"
placeholder="请输入文案主题"
clearable
size="small"
@keyup.enter.native="handleQuery"
/>
</el-form-item>
<el-form-item label="文案内容" prop="content">
<el-input
v-model="queryParams.content"
placeholder="请输入文案内容"
clearable
size="small"
@keyup.enter.native="handleQuery"
/>
</el-form-item>
<el-form-item label="创建时间">
<el-date-picker
v-model="daterangeCreateTime"
size="small"
style="width: 320px"
value-format="yyyy-MM-dd"
format="yyyy 年 MM 月 dd 日"
type="daterange"
range-separator="-"
start-placeholder="开始日期"
end-placeholder="结束日期"
:picker-options="pickerOptions"
@change="handleQuery"
></el-date-picker>
</el-form-item>
<el-form-item>
<el-button type="primary" icon="el-icon-search" size="mini" @click="handleQuery">搜索</el-button>
<el-button icon="el-icon-refresh" size="mini" @click="resetQuery">重置</el-button>
</el-form-item>
</el-form>
<el-row :gutter="10" class="mb8">
<el-col :span="1.5">
<el-button
type="danger"
plain
icon="el-icon-delete"
size="mini"
:disabled="multiple"
@click="handleDelete"
v-hasPermi="['webmagic:copyWritingNetwork:remove']"
>删除
</el-button>
</el-col>
<el-col :span="1.5">
<el-button
type="warning"
plain
icon="el-icon-download"
size="mini"
@click="handleExport"
v-hasPermi="['webmagic:copyWritingNetwork:export']"
>导出
</el-button>
</el-col>
<right-toolbar :showSearch.sync="showSearch" @queryTable="getList"></right-toolbar>
</el-row>
<el-table v-loading="loading" :data="copyWritingNetworkList" @selection-change="handleSelectionChange">
<el-table-column type="selection" width="55" align="center"/>
<el-table-column label="文案标签" align="center" prop="type" :show-overflow-tooltip="true" width="120px" />
<el-table-column label="文案主题" align="center" prop="theme" :show-overflow-tooltip="true" width="200px"/>
<el-table-column label="文案内容" align="center" prop="content" :show-overflow-tooltip="true"/>
<el-table-column label="操作" align="center" class-name="small-padding fixed-width" width="150px">
<template slot-scope="scope">
<el-button
size="mini"
type="text"
icon="el-icon-delete"
@click="handleDelete(scope.row)"
v-hasPermi="['webmagic:copyWritingNetwork:remove']"
>删除
</el-button>
</template>
</el-table-column>
</el-table>
<pagination
v-show="total>0"
:total="total"
:page.sync="queryParams.pageNum"
:limit.sync="queryParams.pageSize"
@pagination="getList"
/>
</div>
</template>
<script>
import {listCopyWritingNetwork, delCopyWritingNetwork} from "@/api/business/webmagic/copywritingnetwork/copyWritingNetwork"
export default {
name: "CopyWritingNetwork",
data() {
return {
// 遮罩层
loading: true,
// 选中数组
ids: [],
// 非单个禁用
single: true,
// 非多个禁用
multiple: true,
// 显示搜索条件
showSearch: true,
// 总条数
total: 0,
// 文案网表格数据
copyWritingNetworkList: [],
// 弹出层标题
title: "",
// 是否显示弹出层
open: false,
// 创建时间时间范围
daterangeCreateTime: [],
// 查询参数
queryParams: {
pageNum: 1,
pageSize: 10,
type: null,
theme: null,
content: null,
createTime: null
},
// 表单参数
form: {},
// 表单校验
rules: {
},
//日期组件
pickerOptions: {
shortcuts: [{
text: '昨天',
onClick(picker) {
const end = new Date();
const start = new Date();
start.setTime(start.getTime() - 3600 * 1000 * 24);
picker.$emit('pick', [start, end]);
}
}, {
text: '最近一周',
onClick(picker) {
const end = new Date();
const start = new Date();
start.setTime(start.getTime() - 3600 * 1000 * 24 * 7);
picker.$emit('pick', [start, end]);
}
}, {
text: '最近一个月',
onClick(picker) {
const end = new Date();
const start = new Date();
start.setTime(start.getTime() - 3600 * 1000 * 24 * 30);
picker.$emit('pick', [start, end]);
}
}, {
text: '最近三个月',
onClick(picker) {
const end = new Date();
const start = new Date();
start.setTime(start.getTime() - 3600 * 1000 * 24 * 90);
picker.$emit('pick', [start, end]);
}
}]
},
};
},
created() {
this.getList();
},
methods: {
/** 查询文案网列表 */
getList() {
this.loading = true;
this.queryParams.params = {};
if (null != this.daterangeCreateTime && '' != this.daterangeCreateTime) {
this.queryParams.createTime = this.daterangeCreateTime[0];
this.queryParams.endCreateTime = this.daterangeCreateTime[1];
}
listCopyWritingNetwork(this.queryParams).then(response => {
this.copyWritingNetworkList = response.rows;
this.total = response.total;
this.loading = false;
});
},
// 取消按钮
cancel() {
this.open = false;
this.reset();
},
// 表单重置
reset() {
this.form = {
id: null,
type: null,
theme: null,
content: null,
createTime: null
};
this.resetForm("form");
},
/** 搜索按钮操作 */
handleQuery() {
this.queryParams.pageNum = 1;
this.getList();
},
/** 重置按钮操作 */
resetQuery() {
this.daterangeCreateTime = [];
this.queryParams.createTime = null
this.queryParams.endCreateTime = null
this.resetForm("queryForm");
this.handleQuery();
},
// 多选框选中数据
handleSelectionChange(selection) {
this.ids = selection.map(item => item.id)
this.single = selection.length!==1
this.multiple = !selection.length
},
/** 删除按钮操作 */
handleDelete(row) {
const ids = row.id || this.ids;
this.$modal.confirm('是否确认删除文案网编号为"' + ids + '"的数据项？').then(function() {
return delCopyWritingNetwork(ids);
}).then(() => {
this.getList();
this.$modal.msgSuccess("删除成功");
}).catch(() => {});
},
/** 导出按钮操作 */
handleExport() {
this.download('webmagic/copyWritingNetwork/export', {
...this.queryParams
}, `copyWritingNetwork_${new Date().getTime()}.xlsx`)
}
}
}
</script>
<style scoped>
</style>

2
ruoyi-ui/src/views/business/webmagic/sina/sinaNews/index.vue
View File

@ -20,7 +20,7 @@
@keyup.enter.native="handleQuery"
/>
</el-form-item>
<el-form-item label="">
<el-form-item label="创建时间">
<el-date-picker
v-model="daterangeCreateTime"
size="small"

5
xjs-business/xjs-business-common/src/main/java/com/xjs/consts/RegexConst.java
View File

@ -27,4 +27,9 @@ public class RegexConst {
* ip地址v4、v6正则
*/
public static final String IP_REGEX ="^((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)$";
/**
* 数字校验正则
*/
public static final String NUMBER_REGEX= "[0-9]*";
}

4
xjs-business/xjs-business-common/src/main/java/com/xjs/consts/TranslationTypeConst.java
View File

@ -2,8 +2,8 @@ package com.xjs.consts;
/**
* @author xiejs
* @desc 翻译类型常量
* @create 2021-12-25
* 翻译类型常量
* @since 2021-12-25
*/
public interface TranslationTypeConst {
Integer BAIDU = 1;

8
xjs-business/xjs-business-openapi/src/main/java/com/xjs/common/aop/ApiLogAspect.java
View File

@ -134,16 +134,18 @@ public class ApiLogAspect {
entity.setRequest(objects.toJSONString());
}
} else {
if(args.length>0){
if (args.length > 0) {
String jsonString = JSON.toJSONString(args[0]);
entity.setRequest(jsonString);
}
}
entity.setMethod(apiLog.method());
String response = null;
if (Objects.nonNull(jsonResult)) {
entity.setResponse(jsonResult.toString());
response = jsonResult.toString();
entity.setResponse(response);
}
if (e != null) {
if (e != null || StringUtils.isEmpty(response)) {
entity.setIsSuccess(ReqConst.ERROR);
} else {
entity.setIsSuccess(ReqConst.SUCCESS);

14
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/common/util/HttpUtils.java
View File

@ -1,6 +1,7 @@
package com.xjs.common.util;
import com.ruoyi.common.core.constant.HttpStatus;
import lombok.extern.log4j.Log4j2;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
@ -22,6 +23,7 @@ import java.util.UUID;
* @since 2022-02-15
*/
@Component
@Log4j2
public class HttpUtils {
private PoolingHttpClientConnectionManager cm;
@ -30,9 +32,9 @@ public class HttpUtils {
this.cm = new PoolingHttpClientConnectionManager();
//设置最大连接数
this.cm.setMaxTotal(100);
this.cm.setMaxTotal(1000);
//设置每个主机最大连接数
this.cm.setDefaultMaxPerRoute(10);
this.cm.setDefaultMaxPerRoute(100);
}
/**
@ -67,14 +69,14 @@ public class HttpUtils {
}
}
} catch (IOException e) {
e.printStackTrace();
log.error(e.getMessage());
} finally {
//关闭response
if (response != null) {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
log.error(e.getMessage());
}
}
}
@ -122,14 +124,14 @@ public class HttpUtils {
}
}
} catch (IOException e) {
e.printStackTrace();
log.error(e.getMessage());
} finally {
//关闭response
if (response != null) {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
log.error(e.getMessage());
}
}
}

83
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/controller/CopyWritingNetworkController.java Normal file
View File

@ -0,0 +1,83 @@
package com.xjs.copywritingNetwork.controller;
import com.ruoyi.common.core.domain.R;
import com.ruoyi.common.core.utils.poi.ExcelUtil;
import com.ruoyi.common.core.web.domain.AjaxResult;
import com.ruoyi.common.core.web.page.TableDataInfo;
import com.ruoyi.common.log.annotation.Log;
import com.ruoyi.common.log.enums.BusinessType;
import com.ruoyi.common.security.annotation.RequiresPermissions;
import com.xjs.copywritingNetwork.pojo.CopyWritingNetwork;
import com.xjs.copywritingNetwork.service.CopyWritingNetworkService;
import com.xjs.copywritingNetwork.task.CopyWritingNetworkTask;
import com.xjs.web.MyBaseController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
/**
* 文案网controller
*
* @author xiejs
* @since 2022-02-17
*/
@RestController
@RequestMapping("copyWritingNetwork")
@Api(tags = "爬虫模块-文案网")
public class CopyWritingNetworkController extends MyBaseController {
@Autowired
private CopyWritingNetworkService copyWritingNetworkService;
@Autowired
private CopyWritingNetworkTask copyWritingNetworkTask;
//----------------------远程rpc调用---------------------------
@GetMapping("taskForPRC")
@ApiOperation("供定时任务服务RPC远程调用")
public R copyWritingNetworkTaskForPRC() {
copyWritingNetworkTask.reptileCopyWriting();
return R.ok();
}
//----------------------------代码生成-----------------------------
/**
* 查询文案网列表
*/
@RequiresPermissions("webmagic:copyWritingNetwork:list")
@GetMapping("/list")
public TableDataInfo list(CopyWritingNetwork copyWritingNetwork) {
startPage();
List<CopyWritingNetwork> list = copyWritingNetworkService.selectCopyWritingNetworkList(copyWritingNetwork);
return getDataTable(list);
}
/**
* 导出文案网列表
*/
@RequiresPermissions("webmagic:copyWritingNetwork:export")
@Log(title = "文案网", businessType = BusinessType.EXPORT)
@PostMapping("/export")
public void export(HttpServletResponse response, CopyWritingNetwork copyWritingNetwork) {
List<CopyWritingNetwork> list = copyWritingNetworkService.selectCopyWritingNetworkList(copyWritingNetwork);
ExcelUtil<CopyWritingNetwork> util = new ExcelUtil<>(CopyWritingNetwork.class);
util.exportExcel(response, list, "文案网数据");
}
/**
* 删除文案网
*/
@RequiresPermissions("webmagic:copyWritingNetwork:remove")
@Log(title = "文案网", businessType = BusinessType.DELETE)
@DeleteMapping("/{ids}")
public AjaxResult remove(@PathVariable Long[] ids) {
return toAjax(copyWritingNetworkService.deleteCopyWritingNetworkByIds(ids));
}
}

30
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/mapper/CopyWritingNetworkMapper.java
View File

@ -3,6 +3,8 @@ package com.xjs.copywritingNetwork.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.xjs.copywritingNetwork.pojo.CopyWritingNetwork;
import java.util.List;
/**
* 文案网mapper
* @author xiejs
@ -15,4 +17,32 @@ public interface CopyWritingNetworkMapper extends BaseMapper<CopyWritingNetwork>
* @return int
*/
int deleteRepeatData();
//------------------------------代码生成-----------------------------------
/**
* 查询文案网列表
*
* @param copyWritingNetwork 文案网
* @return 文案网集合
*/
List<CopyWritingNetwork> selectCopyWritingNetworkList(CopyWritingNetwork copyWritingNetwork);
/**
* 删除文案网
*
* @param id 文案网主键
* @return 结果
*/
public int deleteCopyWritingNetworkById(Long id);
/**
* 批量删除文案网
*
* @param ids 需要删除的数据主键集合
* @return 结果
*/
public int deleteCopyWritingNetworkByIds(Long[] ids);
}

7
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/pojo/CopyWritingNetwork.java
View File

@ -32,4 +32,11 @@ public class CopyWritingNetwork implements Serializable {
/** 创建时间 */
@TableField(fill = FieldFill.INSERT)
private Date createTime;
/**
* 查询条件的结束时间
*/
@TableField(exist = false)
private Date endCreateTime;
}

30
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/CopyWritingNetworkService.java
View File

@ -3,15 +3,45 @@ package com.xjs.copywritingNetwork.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.xjs.copywritingNetwork.pojo.CopyWritingNetwork;
import java.util.List;
/**
* 文案网service接口
*
* @author xiejs
* @since 2022-02-16
*/
public interface CopyWritingNetworkService extends IService<CopyWritingNetwork> {
/**
* 删除重复数据
*
* @return int
*/
int deleteRepeatData();
//---------------------------代码生成--------------------------------
/**
* 查询文案网列表
*
* @param copyWritingNetwork 文案网
* @return 文案网集合
*/
List<CopyWritingNetwork> selectCopyWritingNetworkList(CopyWritingNetwork copyWritingNetwork);
/**
* 批量删除文案网
*
* @param ids 需要删除的文案网主键集合
* @return 结果
*/
int deleteCopyWritingNetworkByIds(Long[] ids);
/**
* 删除文案网信息
*
* @param id 文案网主键
* @return 结果
*/
int deleteCopyWritingNetworkById(Long id);
}

36
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/service/impl/CopyWritingNetworkServiceImpl.java
View File

@ -7,6 +7,7 @@ import com.xjs.copywritingNetwork.service.CopyWritingNetworkService;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
/**
* @author xiejs
@ -22,4 +23,39 @@ public class CopyWritingNetworkServiceImpl extends ServiceImpl<CopyWritingNetwor
public int deleteRepeatData() {
return copyWritingNetworkMapper.deleteRepeatData();
}
//-----------------------------代码生成-----------------------------
/**
* 查询文案网列表
*
* @param copyWritingNetwork 文案网
* @return 文案网
*/
@Override
public List<CopyWritingNetwork> selectCopyWritingNetworkList(CopyWritingNetwork copyWritingNetwork) {
return copyWritingNetworkMapper.selectCopyWritingNetworkList(copyWritingNetwork);
}
/**
* 批量删除文案网
*
* @param ids 需要删除的文案网主键
* @return 结果
*/
@Override
public int deleteCopyWritingNetworkByIds(Long[] ids) {
return copyWritingNetworkMapper.deleteCopyWritingNetworkByIds(ids);
}
/**
* 删除文案网信息
*
* @param id 文案网主键
* @return 结果
*/
@Override
public int deleteCopyWritingNetworkById(Long id) {
return copyWritingNetworkMapper.deleteCopyWritingNetworkById(id);
}
}

15
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/copywritingNetwork/task/CopyWritingNetworkTask.java
View File

@ -10,7 +10,6 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
@ -18,6 +17,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
import static com.xjs.consts.RegexConst.NUMBER_REGEX;
/**
* 文案网爬虫任务 url:https://www.wenanwang.com/
*
@ -36,9 +37,8 @@ public class CopyWritingNetworkTask {
public static final String URL = "https://www.wenanwang.com/";
private static Pattern pattern = Pattern.compile("[0-9]*");
private static final Pattern pattern = Pattern.compile(NUMBER_REGEX);
@Scheduled(fixedDelay = 1000 * 5)
public void reptileCopyWriting() {
try {
String html = httpUtils.doGetHtml(URL);
@ -46,11 +46,9 @@ public class CopyWritingNetworkTask {
Document document = Jsoup.parse(html);
this.parseHtmlGetUrl(document);
} catch (Exception e) {
e.printStackTrace();
}finally {
int i = copyWritingNetworkService.deleteRepeatData();
log.info("删除文案网数据重复数："+i);
log.error(e.getMessage());
}
}
@ -118,6 +116,9 @@ public class CopyWritingNetworkTask {
copyWritingNetworkService.saveBatch(copyWritingNetworks, 20);
int i = copyWritingNetworkService.deleteRepeatData();
log.info("删除文案网数据重复数："+i);
}

7
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/sina/task/SinaNewsTask.java
View File

@ -30,17 +30,18 @@ public class SinaNewsTask {
@Autowired
private SinaNewsService sinaNewsService;
public static final String URL = "https://news.sina.com.cn/";
public void reptileSinaNews() {
try {
String url = "https://news.sina.com.cn/";
String html = httpUtils.doGetHtml(url);
String html = httpUtils.doGetHtml(URL);
Document document = Jsoup.parse(html);
this.parse(document);
} catch (Exception e) {
e.printStackTrace();
log.error(e.getMessage());
}
}

36
xjs-business/xjs-business-webmagic/src/main/resources/mapper/webmagic/CopyWritingNetworkMapper.xml
View File

@ -4,6 +4,31 @@
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.xjs.copywritingNetwork.mapper.CopyWritingNetworkMapper">
<resultMap type="com.xjs.copywritingNetwork.pojo.CopyWritingNetwork" id="CopyWritingNetworkResult">
<result property="id" column="id" />
<result property="type" column="type" />
<result property="theme" column="theme" />
<result property="content" column="content" />
<result property="createTime" column="create_time" />
</resultMap>
<sql id="selectCopyWritingNetworkVo">
select id, type, theme, content, create_time from webmagic_copywriting_network
</sql>
<select id="selectCopyWritingNetworkList" parameterType="com.xjs.copywritingNetwork.pojo.CopyWritingNetwork" resultMap="CopyWritingNetworkResult">
<include refid="selectCopyWritingNetworkVo"/>
<where>
<if test="type != null and type != ''"> and type = #{type}</if>
<if test="theme != null and theme != ''"> and theme like concat('%', #{theme}, '%')</if>
<if test="content != null and content != ''"> and content like concat('%', #{content}, '%')</if>
<if test="createTime != null and endCreateTime != null"> and create_time between #{createTime} and #{endCreateTime}</if>
</where>
order by create_time desc
</select>
<delete id="deleteRepeatData">
delete from webmagic_copywriting_network where id not in (
SELECT
@ -12,4 +37,15 @@
( SELECT min( id ) AS min_id FROM webmagic_copywriting_network GROUP BY content ) AS t
)
</delete>
<delete id="deleteCopyWritingNetworkById" parameterType="Long">
delete from webmagic_copywriting_network where id = #{id}
</delete>
<delete id="deleteCopyWritingNetworkByIds" parameterType="String">
delete from webmagic_copywriting_network where id in
<foreach item="id" collection="array" open="(" separator="," close=")">
#{id}
</foreach>
</delete>
</mapper>