update get data open ipo
This commit is contained in:
vu-tran
@@ -116,182 +116,188 @@ public class Scraper {
|
|||||||
public void getMoneyControllerNewIPOSchedule() {
|
public void getMoneyControllerNewIPOSchedule() {
|
||||||
log.info("定时任务执行获取新股ipo的方法开始执行");
|
log.info("定时任务执行获取新股ipo的方法开始执行");
|
||||||
// 目标 URL
|
// 目标 URL
|
||||||
String url = "https://www.moneycontrol.com/ipo/upcoming-ipos/";
|
List<String> urls = Arrays.asList("https://www.moneycontrol.com/ipo/upcoming-ipos/", "https://www.moneycontrol.com/ipo/open-ipos/");
|
||||||
// 创建 HttpClient 实例
|
// 创建 HttpClient 实例
|
||||||
HttpClient client = HttpClients.createDefault();
|
HttpClient client = HttpClients.createDefault();
|
||||||
// 创建 HttpGet 请求
|
// 创建 HttpGet 请求
|
||||||
HttpGet request = new HttpGet(url);
|
for (String url : urls) {
|
||||||
try {
|
HttpGet request = new HttpGet(url);
|
||||||
// 执行请求
|
try {
|
||||||
HttpResponse response = client.execute(request);
|
// 执行请求
|
||||||
|
HttpResponse response = client.execute(request);
|
||||||
|
|
||||||
// 检查请求是否成功
|
// 检查请求是否成功
|
||||||
if (response.getStatusLine().getStatusCode() == 200) {
|
if (response.getStatusLine().getStatusCode() == 200) {
|
||||||
// 获取响应体
|
// 获取响应体
|
||||||
String responseBody = EntityUtils.toString(response.getEntity());
|
String responseBody = EntityUtils.toString(response.getEntity());
|
||||||
|
|
||||||
// 使用 Jsoup 解析 HTML
|
// 使用 Jsoup 解析 HTML
|
||||||
Document doc = Jsoup.parse(responseBody);
|
Document doc = Jsoup.parse(responseBody);
|
||||||
|
|
||||||
// 找到包含 JSON 数据的 <script> 标签
|
// 找到包含 JSON 数据的 <script> 标签
|
||||||
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
|
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
|
||||||
|
|
||||||
if (scriptTag != null) {
|
if (scriptTag != null) {
|
||||||
// 获取 JSON 数据
|
// 获取 JSON 数据
|
||||||
String jsonDataStr = scriptTag.html();
|
String jsonDataStr = scriptTag.html();
|
||||||
|
|
||||||
// 将 JSON 字符串解析为 Java JSONObject
|
// 将 JSON 字符串解析为 Java JSONObject
|
||||||
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
|
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
|
||||||
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
|
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
|
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
|
||||||
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
|
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// 解析 openData 和 upcomingData
|
// 解析 openData 和 upcomingData
|
||||||
JSONArray openData = ipoTableData.getJSONArray("openData");
|
JSONArray openData = ipoTableData.getJSONArray("openData");
|
||||||
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
|
// if (openData.isEmpty()){
|
||||||
List<StockIpo> listStockIpoList = new ArrayList<>();
|
// openData = ipoTableData.getJSONArray("openUpcomingData");
|
||||||
for (int i = 0; i < openData.size(); i++) {
|
// }
|
||||||
JSONObject entry = openData.getJSONObject(i);
|
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
|
||||||
StockIpo stockIpo = new StockIpo();
|
List<StockIpo> listStockIpoList = new ArrayList<>();
|
||||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
for (int i = 0; i < openData.size(); i++) {
|
||||||
stockIpo.setStockName(entry.getString("company_name"));
|
JSONObject entry = openData.getJSONObject(i);
|
||||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
StockIpo stockIpo = new StockIpo();
|
||||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
stockIpo.setStockName(entry.getString("company_name"));
|
||||||
Integer totalNumber = 0;
|
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||||
Integer issue_size = entry.getInteger("issue_size");
|
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
Integer totalNumber = 0;
|
||||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||||
totalNumber = totalNumberBigDecimal.intValue();
|
Integer issue_size = entry.getInteger("issue_size");
|
||||||
}
|
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||||
stockIpo.setTotalNumber(totalNumber);
|
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||||
stockIpo.setApply(entry.getString("total_subs"));
|
totalNumber = totalNumberBigDecimal.intValue();
|
||||||
stockIpo.setCreateDate(new Date());
|
}
|
||||||
stockIpo.setUpdateDate(new Date());
|
stockIpo.setTotalNumber(totalNumber);
|
||||||
stockIpo.setSourceType("3");
|
stockIpo.setApply(entry.getString("total_subs"));
|
||||||
|
stockIpo.setCreateDate(new Date());
|
||||||
|
stockIpo.setUpdateDate(new Date());
|
||||||
|
stockIpo.setSourceType("3");
|
||||||
|
|
||||||
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
|
String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
|
||||||
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
|
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
|
||||||
|
|
||||||
for (String exchange : exchanges) {
|
for (String exchange : exchanges) {
|
||||||
StockIpo ipo = new StockIpo();
|
StockIpo ipo = new StockIpo();
|
||||||
ipo.setStockCode(stockIpo.getStockCode());
|
ipo.setStockCode(stockIpo.getStockCode());
|
||||||
ipo.setStockName(stockIpo.getStockName());
|
ipo.setStockName(stockIpo.getStockName());
|
||||||
ipo.setStockPrice(stockIpo.getStockPrice());
|
ipo.setStockPrice(stockIpo.getStockPrice());
|
||||||
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
||||||
ipo.setListingDate(stockIpo.getListingDate());
|
ipo.setListingDate(stockIpo.getListingDate());
|
||||||
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
||||||
ipo.setApply(stockIpo.getApply());
|
ipo.setApply(stockIpo.getApply());
|
||||||
ipo.setCreateDate(stockIpo.getCreateDate());
|
ipo.setCreateDate(stockIpo.getCreateDate());
|
||||||
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
||||||
ipo.setExchangeType(exchange);
|
ipo.setExchangeType(exchange);
|
||||||
ipo.setSourceType("3");
|
ipo.setSourceType("3");
|
||||||
listStockIpoList.add(ipo);
|
listStockIpoList.add(ipo);
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < upcomingData.size(); i++) {
|
|
||||||
JSONObject entry = upcomingData.getJSONObject(i);
|
|
||||||
StockIpo stockIpo = new StockIpo();
|
|
||||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
|
||||||
stockIpo.setStockName(entry.getString("company_name"));
|
|
||||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
|
||||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
|
||||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
|
||||||
Integer totalNumber = 0;
|
|
||||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
|
||||||
Integer issue_size = entry.getInteger("issue_size");
|
|
||||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
|
||||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
|
||||||
totalNumber = totalNumberBigDecimal.intValue();
|
|
||||||
}
|
|
||||||
stockIpo.setTotalNumber(totalNumber);
|
|
||||||
stockIpo.setApply(entry.getString("total_subs"));
|
|
||||||
stockIpo.setCreateDate(new Date());
|
|
||||||
stockIpo.setUpdateDate(new Date());
|
|
||||||
stockIpo.setSourceType("3");
|
|
||||||
|
|
||||||
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
|
|
||||||
|
|
||||||
for (String exchange : exchanges) {
|
|
||||||
StockIpo ipo = new StockIpo();
|
|
||||||
ipo.setStockCode(stockIpo.getStockCode());
|
|
||||||
ipo.setStockName(stockIpo.getStockName());
|
|
||||||
ipo.setStockPrice(stockIpo.getStockPrice());
|
|
||||||
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
|
||||||
ipo.setListingDate(stockIpo.getListingDate());
|
|
||||||
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
|
||||||
ipo.setApply(stockIpo.getApply());
|
|
||||||
ipo.setCreateDate(stockIpo.getCreateDate());
|
|
||||||
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
|
||||||
ipo.setExchangeType(exchange);
|
|
||||||
ipo.setSourceType("3");
|
|
||||||
listStockIpoList.add(ipo);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// stockIpoRepository.saveAll(listStockIpoList);
|
|
||||||
|
|
||||||
if (!listStockIpoList.isEmpty())
|
|
||||||
{
|
|
||||||
listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
|
|
||||||
if(CollectionUtil.isNotEmpty(scIdList)){
|
|
||||||
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
|
|
||||||
List<String> existingStockScIds = existStockIpoList.stream()
|
|
||||||
.map(StockIpo::getStockCode)
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
|
|
||||||
for (StockIpo stockIpo : listStockIpoList) {
|
|
||||||
StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().equals(stockIpo.getExchangeType()) )).findFirst().orElse(null);
|
|
||||||
if (existIpo != null) {
|
|
||||||
if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
|
|
||||||
existIpo.setExchangeType(stockIpo.getExchangeType());
|
|
||||||
}
|
|
||||||
stockIpo.setId(existIpo.getId());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < upcomingData.size(); i++) {
|
||||||
|
JSONObject entry = upcomingData.getJSONObject(i);
|
||||||
|
StockIpo stockIpo = new StockIpo();
|
||||||
|
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||||
|
stockIpo.setStockName(entry.getString("company_name"));
|
||||||
|
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||||
|
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||||
|
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||||
|
Integer totalNumber = 0;
|
||||||
|
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||||
|
Integer issue_size = entry.getInteger("issue_size");
|
||||||
|
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||||
|
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||||
|
totalNumber = totalNumberBigDecimal.intValue();
|
||||||
|
}
|
||||||
|
stockIpo.setTotalNumber(totalNumber);
|
||||||
|
stockIpo.setApply(entry.getString("total_subs"));
|
||||||
|
stockIpo.setCreateDate(new Date());
|
||||||
|
stockIpo.setUpdateDate(new Date());
|
||||||
|
stockIpo.setSourceType("3");
|
||||||
|
|
||||||
|
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
|
||||||
|
|
||||||
|
for (String exchange : exchanges) {
|
||||||
|
StockIpo ipo = new StockIpo();
|
||||||
|
ipo.setStockCode(stockIpo.getStockCode());
|
||||||
|
ipo.setStockName(stockIpo.getStockName());
|
||||||
|
ipo.setStockPrice(stockIpo.getStockPrice());
|
||||||
|
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
||||||
|
ipo.setListingDate(stockIpo.getListingDate());
|
||||||
|
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
||||||
|
ipo.setApply(stockIpo.getApply());
|
||||||
|
ipo.setCreateDate(stockIpo.getCreateDate());
|
||||||
|
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
||||||
|
ipo.setExchangeType(exchange);
|
||||||
|
ipo.setSourceType("3");
|
||||||
|
listStockIpoList.add(ipo);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stockIpoRepository.saveAll(listStockIpoList);
|
||||||
|
|
||||||
|
if (!listStockIpoList.isEmpty())
|
||||||
|
{
|
||||||
|
listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
|
||||||
|
if(CollectionUtil.isNotEmpty(scIdList)){
|
||||||
|
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
|
||||||
|
List<String> existingStockScIds = existStockIpoList.stream()
|
||||||
|
.map(StockIpo::getStockCode)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
for (StockIpo stockIpo : listStockIpoList) {
|
||||||
|
StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().equals(stockIpo.getExchangeType()) )).findFirst().orElse(null);
|
||||||
|
if (existIpo != null) {
|
||||||
|
if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
|
||||||
|
existIpo.setExchangeType(stockIpo.getExchangeType());
|
||||||
|
}
|
||||||
|
stockIpo.setId(existIpo.getId());
|
||||||
|
}
|
||||||
|
}
|
||||||
// listStockIpoList = listStockIpoList.stream()
|
// listStockIpoList = listStockIpoList.stream()
|
||||||
// .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
|
// .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
|
||||||
// .collect(Collectors.toList());
|
// .collect(Collectors.toList());
|
||||||
|
|
||||||
//保存全部的新股
|
//保存全部的新股
|
||||||
if(CollectionUtil.isNotEmpty(listStockIpoList)){
|
if(CollectionUtil.isNotEmpty(listStockIpoList)){
|
||||||
stockIpoRepository.saveAll(listStockIpoList);
|
stockIpoRepository.saveAll(listStockIpoList);
|
||||||
|
|
||||||
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
|
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
|
||||||
}else {
|
}else {
|
||||||
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
|
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// 输出整个 JSON 数据
|
||||||
|
} else {
|
||||||
|
log.info("未找到包含 JSON 数据的 <script> 标签");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// 输出整个 JSON 数据
|
|
||||||
} else {
|
} else {
|
||||||
log.info("未找到包含 JSON 数据的 <script> 标签");
|
log.info("HTTP请求失败，状态码：" + response.getStatusLine().getStatusCode());
|
||||||
}
|
}
|
||||||
} else {
|
} catch (IOException e) {
|
||||||
log.info("HTTP请求失败，状态码：" + response.getStatusLine().getStatusCode());
|
log.error("获取新股接口发生异常",e);
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
|
||||||
log.error("获取新股接口发生异常",e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Scheduled(cron = "0 0 19 * * ?")
|
@Scheduled(cron = "0 0 19 * * ?")
|
||||||
|
|||||||
Reference in New Issue
Block a user