update get data open ipo
This commit is contained in:
vu-tran
@@ -116,182 +116,188 @@ public class Scraper {
|
||||
public void getMoneyControllerNewIPOSchedule() {
|
||||
log.info("定时任务执行获取新股ipo的方法开始执行");
|
||||
// 目标 URL
|
||||
String url = "https://www.moneycontrol.com/ipo/upcoming-ipos/";
|
||||
List<String> urls = Arrays.asList("https://www.moneycontrol.com/ipo/upcoming-ipos/", "https://www.moneycontrol.com/ipo/open-ipos/");
|
||||
// 创建 HttpClient 实例
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
// 创建 HttpGet 请求
|
||||
HttpGet request = new HttpGet(url);
|
||||
try {
|
||||
// 执行请求
|
||||
HttpResponse response = client.execute(request);
|
||||
for (String url : urls) {
|
||||
HttpGet request = new HttpGet(url);
|
||||
try {
|
||||
// 执行请求
|
||||
HttpResponse response = client.execute(request);
|
||||
|
||||
// 检查请求是否成功
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
// 获取响应体
|
||||
String responseBody = EntityUtils.toString(response.getEntity());
|
||||
// 检查请求是否成功
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
// 获取响应体
|
||||
String responseBody = EntityUtils.toString(response.getEntity());
|
||||
|
||||
// 使用 Jsoup 解析 HTML
|
||||
Document doc = Jsoup.parse(responseBody);
|
||||
// 使用 Jsoup 解析 HTML
|
||||
Document doc = Jsoup.parse(responseBody);
|
||||
|
||||
// 找到包含 JSON 数据的 <script> 标签
|
||||
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
|
||||
// 找到包含 JSON 数据的 <script> 标签
|
||||
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
|
||||
|
||||
if (scriptTag != null) {
|
||||
// 获取 JSON 数据
|
||||
String jsonDataStr = scriptTag.html();
|
||||
if (scriptTag != null) {
|
||||
// 获取 JSON 数据
|
||||
String jsonDataStr = scriptTag.html();
|
||||
|
||||
// 将 JSON 字符串解析为 Java JSONObject
|
||||
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
|
||||
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
|
||||
// 将 JSON 字符串解析为 Java JSONObject
|
||||
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
|
||||
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
|
||||
|
||||
|
||||
|
||||
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
|
||||
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
|
||||
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
|
||||
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
|
||||
|
||||
|
||||
|
||||
// 解析 openData 和 upcomingData
|
||||
JSONArray openData = ipoTableData.getJSONArray("openData");
|
||||
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
|
||||
List<StockIpo> listStockIpoList = new ArrayList<>();
|
||||
for (int i = 0; i < openData.size(); i++) {
|
||||
JSONObject entry = openData.getJSONObject(i);
|
||||
StockIpo stockIpo = new StockIpo();
|
||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||
stockIpo.setStockName(entry.getString("company_name"));
|
||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||
Integer totalNumber = 0;
|
||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||
Integer issue_size = entry.getInteger("issue_size");
|
||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||
totalNumber = totalNumberBigDecimal.intValue();
|
||||
}
|
||||
stockIpo.setTotalNumber(totalNumber);
|
||||
stockIpo.setApply(entry.getString("total_subs"));
|
||||
stockIpo.setCreateDate(new Date());
|
||||
stockIpo.setUpdateDate(new Date());
|
||||
stockIpo.setSourceType("3");
|
||||
// 解析 openData 和 upcomingData
|
||||
JSONArray openData = ipoTableData.getJSONArray("openData");
|
||||
// if (openData.isEmpty()){
|
||||
// openData = ipoTableData.getJSONArray("openUpcomingData");
|
||||
// }
|
||||
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
|
||||
List<StockIpo> listStockIpoList = new ArrayList<>();
|
||||
for (int i = 0; i < openData.size(); i++) {
|
||||
JSONObject entry = openData.getJSONObject(i);
|
||||
StockIpo stockIpo = new StockIpo();
|
||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||
stockIpo.setStockName(entry.getString("company_name"));
|
||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||
Integer totalNumber = 0;
|
||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||
Integer issue_size = entry.getInteger("issue_size");
|
||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||
totalNumber = totalNumberBigDecimal.intValue();
|
||||
}
|
||||
stockIpo.setTotalNumber(totalNumber);
|
||||
stockIpo.setApply(entry.getString("total_subs"));
|
||||
stockIpo.setCreateDate(new Date());
|
||||
stockIpo.setUpdateDate(new Date());
|
||||
stockIpo.setSourceType("3");
|
||||
|
||||
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
||||
continue;
|
||||
}
|
||||
String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
|
||||
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
|
||||
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
||||
continue;
|
||||
}
|
||||
String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
|
||||
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
|
||||
|
||||
for (String exchange : exchanges) {
|
||||
StockIpo ipo = new StockIpo();
|
||||
ipo.setStockCode(stockIpo.getStockCode());
|
||||
ipo.setStockName(stockIpo.getStockName());
|
||||
ipo.setStockPrice(stockIpo.getStockPrice());
|
||||
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
||||
ipo.setListingDate(stockIpo.getListingDate());
|
||||
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
||||
ipo.setApply(stockIpo.getApply());
|
||||
ipo.setCreateDate(stockIpo.getCreateDate());
|
||||
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
||||
ipo.setExchangeType(exchange);
|
||||
ipo.setSourceType("3");
|
||||
listStockIpoList.add(ipo);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < upcomingData.size(); i++) {
|
||||
JSONObject entry = upcomingData.getJSONObject(i);
|
||||
StockIpo stockIpo = new StockIpo();
|
||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||
stockIpo.setStockName(entry.getString("company_name"));
|
||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||
Integer totalNumber = 0;
|
||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||
Integer issue_size = entry.getInteger("issue_size");
|
||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||
totalNumber = totalNumberBigDecimal.intValue();
|
||||
}
|
||||
stockIpo.setTotalNumber(totalNumber);
|
||||
stockIpo.setApply(entry.getString("total_subs"));
|
||||
stockIpo.setCreateDate(new Date());
|
||||
stockIpo.setUpdateDate(new Date());
|
||||
stockIpo.setSourceType("3");
|
||||
|
||||
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
||||
continue;
|
||||
}
|
||||
|
||||
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
|
||||
|
||||
for (String exchange : exchanges) {
|
||||
StockIpo ipo = new StockIpo();
|
||||
ipo.setStockCode(stockIpo.getStockCode());
|
||||
ipo.setStockName(stockIpo.getStockName());
|
||||
ipo.setStockPrice(stockIpo.getStockPrice());
|
||||
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
||||
ipo.setListingDate(stockIpo.getListingDate());
|
||||
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
||||
ipo.setApply(stockIpo.getApply());
|
||||
ipo.setCreateDate(stockIpo.getCreateDate());
|
||||
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
||||
ipo.setExchangeType(exchange);
|
||||
ipo.setSourceType("3");
|
||||
listStockIpoList.add(ipo);
|
||||
}
|
||||
}
|
||||
|
||||
// stockIpoRepository.saveAll(listStockIpoList);
|
||||
|
||||
if (!listStockIpoList.isEmpty())
|
||||
{
|
||||
listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
|
||||
if(CollectionUtil.isNotEmpty(scIdList)){
|
||||
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
|
||||
List<String> existingStockScIds = existStockIpoList.stream()
|
||||
.map(StockIpo::getStockCode)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (StockIpo stockIpo : listStockIpoList) {
|
||||
StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().equals(stockIpo.getExchangeType()) )).findFirst().orElse(null);
|
||||
if (existIpo != null) {
|
||||
if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
|
||||
existIpo.setExchangeType(stockIpo.getExchangeType());
|
||||
}
|
||||
stockIpo.setId(existIpo.getId());
|
||||
for (String exchange : exchanges) {
|
||||
StockIpo ipo = new StockIpo();
|
||||
ipo.setStockCode(stockIpo.getStockCode());
|
||||
ipo.setStockName(stockIpo.getStockName());
|
||||
ipo.setStockPrice(stockIpo.getStockPrice());
|
||||
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
||||
ipo.setListingDate(stockIpo.getListingDate());
|
||||
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
||||
ipo.setApply(stockIpo.getApply());
|
||||
ipo.setCreateDate(stockIpo.getCreateDate());
|
||||
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
||||
ipo.setExchangeType(exchange);
|
||||
ipo.setSourceType("3");
|
||||
listStockIpoList.add(ipo);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < upcomingData.size(); i++) {
|
||||
JSONObject entry = upcomingData.getJSONObject(i);
|
||||
StockIpo stockIpo = new StockIpo();
|
||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||
stockIpo.setStockName(entry.getString("company_name"));
|
||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||
Integer totalNumber = 0;
|
||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||
Integer issue_size = entry.getInteger("issue_size");
|
||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||
totalNumber = totalNumberBigDecimal.intValue();
|
||||
}
|
||||
stockIpo.setTotalNumber(totalNumber);
|
||||
stockIpo.setApply(entry.getString("total_subs"));
|
||||
stockIpo.setCreateDate(new Date());
|
||||
stockIpo.setUpdateDate(new Date());
|
||||
stockIpo.setSourceType("3");
|
||||
|
||||
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
||||
continue;
|
||||
}
|
||||
|
||||
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
|
||||
|
||||
for (String exchange : exchanges) {
|
||||
StockIpo ipo = new StockIpo();
|
||||
ipo.setStockCode(stockIpo.getStockCode());
|
||||
ipo.setStockName(stockIpo.getStockName());
|
||||
ipo.setStockPrice(stockIpo.getStockPrice());
|
||||
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
||||
ipo.setListingDate(stockIpo.getListingDate());
|
||||
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
||||
ipo.setApply(stockIpo.getApply());
|
||||
ipo.setCreateDate(stockIpo.getCreateDate());
|
||||
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
||||
ipo.setExchangeType(exchange);
|
||||
ipo.setSourceType("3");
|
||||
listStockIpoList.add(ipo);
|
||||
}
|
||||
}
|
||||
|
||||
// stockIpoRepository.saveAll(listStockIpoList);
|
||||
|
||||
if (!listStockIpoList.isEmpty())
|
||||
{
|
||||
listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
|
||||
if(CollectionUtil.isNotEmpty(scIdList)){
|
||||
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
|
||||
List<String> existingStockScIds = existStockIpoList.stream()
|
||||
.map(StockIpo::getStockCode)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (StockIpo stockIpo : listStockIpoList) {
|
||||
StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().equals(stockIpo.getExchangeType()) )).findFirst().orElse(null);
|
||||
if (existIpo != null) {
|
||||
if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
|
||||
existIpo.setExchangeType(stockIpo.getExchangeType());
|
||||
}
|
||||
stockIpo.setId(existIpo.getId());
|
||||
}
|
||||
}
|
||||
// listStockIpoList = listStockIpoList.stream()
|
||||
// .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
|
||||
// .collect(Collectors.toList());
|
||||
|
||||
//保存全部的新股
|
||||
if(CollectionUtil.isNotEmpty(listStockIpoList)){
|
||||
stockIpoRepository.saveAll(listStockIpoList);
|
||||
//保存全部的新股
|
||||
if(CollectionUtil.isNotEmpty(listStockIpoList)){
|
||||
stockIpoRepository.saveAll(listStockIpoList);
|
||||
|
||||
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
|
||||
}else {
|
||||
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
|
||||
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
|
||||
}else {
|
||||
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 输出整个 JSON 数据
|
||||
} else {
|
||||
log.info("未找到包含 JSON 数据的 <script> 标签");
|
||||
}
|
||||
|
||||
|
||||
// 输出整个 JSON 数据
|
||||
} else {
|
||||
log.info("未找到包含 JSON 数据的 <script> 标签");
|
||||
log.info("HTTP请求失败，状态码：" + response.getStatusLine().getStatusCode());
|
||||
}
|
||||
} else {
|
||||
log.info("HTTP请求失败，状态码：" + response.getStatusLine().getStatusCode());
|
||||
} catch (IOException e) {
|
||||
log.error("获取新股接口发生异常",e);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("获取新股接口发生异常",e);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Scheduled(cron = "0 0 19 * * ?")
|
||||
|
||||
Reference in New Issue
Block a user