nhanvubk/ger-market-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update crawl ipo

Browse Source
This commit is contained in:
vu-tran
2024-12-11 12:39:15 +07:00
parent 0476ad100b
commit d085d8de12
1 changed files with 20 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/main/java/cn/stock/market/infrastructure/job/Scraper.java
View File

@@ -144,9 +144,13 @@ public class Scraper {
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr); JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
log.info("获取到新股的json信息:"+jsonObject.toJSONString()); log.info("获取到新股的json信息:"+jsonObject.toJSONString());
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps"); JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData"); JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
// 解析 openData 和 upcomingData // 解析 openData 和 upcomingData
JSONArray openData = ipoTableData.getJSONArray("openData"); JSONArray openData = ipoTableData.getJSONArray("openData");
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData"); JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
@@ -175,8 +179,8 @@ public class Scraper {
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){ if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
continue; continue;
} }
String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode()); List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
for (String exchange : exchanges) { for (String exchange : exchanges) {
StockIpo ipo = new StockIpo(); StockIpo ipo = new StockIpo();
@@ -190,7 +194,7 @@ public class Scraper {
ipo.setCreateDate(stockIpo.getCreateDate()); ipo.setCreateDate(stockIpo.getCreateDate());
ipo.setUpdateDate(stockIpo.getUpdateDate()); ipo.setUpdateDate(stockIpo.getUpdateDate());
ipo.setExchangeType(exchange); ipo.setExchangeType(exchange);
ipo.setSourceType("3");
listStockIpoList.add(ipo); listStockIpoList.add(ipo);
} }
} }
@@ -234,7 +238,7 @@ public class Scraper {
ipo.setCreateDate(stockIpo.getCreateDate()); ipo.setCreateDate(stockIpo.getCreateDate());
ipo.setUpdateDate(stockIpo.getUpdateDate()); ipo.setUpdateDate(stockIpo.getUpdateDate());
ipo.setExchangeType(exchange); ipo.setExchangeType(exchange);
ipo.setSourceType("3");
listStockIpoList.add(ipo); listStockIpoList.add(ipo);
} }
} }
@@ -253,9 +257,18 @@ public class Scraper {
.map(StockIpo::getStockCode) .map(StockIpo::getStockCode)
.collect(Collectors.toList()); .collect(Collectors.toList());
listStockIpoList = listStockIpoList.stream() for (StockIpo stockIpo : listStockIpoList) {
.filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName())) StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().equals(stockIpo.getExchangeType()) )).findFirst().orElse(null);
.collect(Collectors.toList()); if (existIpo != null) {
if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
existIpo.setExchangeType(stockIpo.getExchangeType());
}
stockIpo.setId(existIpo.getId());
}
}
// listStockIpoList = listStockIpoList.stream()
// .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
// .collect(Collectors.toList());
//保存全部的新股 //保存全部的新股
if(CollectionUtil.isNotEmpty(listStockIpoList)){ if(CollectionUtil.isNotEmpty(listStockIpoList)){