nhanvubk/ger-market-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into 'main'

Browse Source 
Develop

See merge request india/india_market_java!57
This commit is contained in:
JinPeng Weng
2025-02-11 08:33:47 +00:00
parent 8f1abca911 f612b5128e
commit 3ecb064f75
2 changed files with 161 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/cn/stock/market/infrastructure/db/po/StockIpoPO.java
View File

@@ -61,11 +61,11 @@ public class StockIpoPO {
/**
* 是否显示【1 显示，2 不显示】 */
Integer isShow;
Integer isShow = 1;
/**
* 是否上市【1 未上市，2 已上市】 */
Integer isList;
Integer isList = 1;
Date createDate;

300
src/main/java/cn/stock/market/infrastructure/job/Scraper.java
View File

@@ -116,169 +116,187 @@ public class Scraper {
public void getMoneyControllerNewIPOSchedule() {
log.info("定时任务执行获取新股ipo的方法开始执行");
// 目标 URL
String url = "https://www.moneycontrol.com/ipo/open-upcoming-ipos";
List<String> urls = Arrays.asList("https://www.moneycontrol.com/ipo/upcoming-ipos/", "https://www.moneycontrol.com/ipo/open-ipos/");
// 创建 HttpClient 实例
HttpClient client = HttpClients.createDefault();
// 创建 HttpGet 请求
HttpGet request = new HttpGet(url);
try {
// 执行请求
HttpResponse response = client.execute(request);
for (String url : urls) {
HttpGet request = new HttpGet(url);
try {
// 执行请求
HttpResponse response = client.execute(request);
// 检查请求是否成功
if (response.getStatusLine().getStatusCode() == 200) {
// 获取响应体
String responseBody = EntityUtils.toString(response.getEntity());
// 检查请求是否成功
if (response.getStatusLine().getStatusCode() == 200) {
// 获取响应体
String responseBody = EntityUtils.toString(response.getEntity());
// 使用 Jsoup 解析 HTML
Document doc = Jsoup.parse(responseBody);
// 使用 Jsoup 解析 HTML
Document doc = Jsoup.parse(responseBody);
// 找到包含 JSON 数据的 <script> 标签
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
// 找到包含 JSON 数据的 <script> 标签
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
if (scriptTag != null) {
// 获取 JSON 数据
String jsonDataStr = scriptTag.html();
if (scriptTag != null) {
// 获取 JSON 数据
String jsonDataStr = scriptTag.html();
// 将 JSON 字符串解析为 Java JSONObject
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
// 将 JSON 字符串解析为 Java JSONObject
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
// 解析 openData 和 upcomingData
JSONArray openData = ipoTableData.getJSONArray("openData");
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
List<StockIpo> listStockIpoList = new ArrayList<>();
for (int i = 0; i < openData.size(); i++) {
JSONObject entry = openData.getJSONObject(i);
StockIpo stockIpo = new StockIpo();
stockIpo.setStockCode(entry.getString("sc_id"));
stockIpo.setStockName(entry.getString("company_name"));
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
Integer totalNumber = 0;
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
Integer issue_size = entry.getInteger("issue_size");
BigDecimal issueSize = new BigDecimal(issue_size);
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
totalNumber = totalNumberBigDecimal.intValue();
}
stockIpo.setTotalNumber(totalNumber);
stockIpo.setApply(entry.getString("total_subs"));
stockIpo.setCreateDate(new Date());
stockIpo.setUpdateDate(new Date());
stockIpo.setSourceType("3");
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
continue;
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
// 解析 openData 和 upcomingData
JSONArray openData = ipoTableData.getJSONArray("openData");
// if (openData.isEmpty()){
// openData = ipoTableData.getJSONArray("openUpcomingData");
// }
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
List<StockIpo> listStockIpoList = new ArrayList<>();
for (int i = 0; i < openData.size(); i++) {
JSONObject entry = openData.getJSONObject(i);
StockIpo stockIpo = new StockIpo();
stockIpo.setStockCode(entry.getString("sc_id"));
stockIpo.setStockName(entry.getString("company_name"));
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
Integer totalNumber = 0;
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
BigDecimal issueSize = new BigDecimal(entry.getLong("issue_size"));
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
totalNumber = totalNumberBigDecimal.intValue();
}
stockIpo.setTotalNumber(totalNumber);
stockIpo.setApply(entry.getString("total_subs"));
stockIpo.setCreateDate(new Date());
stockIpo.setUpdateDate(new Date());
stockIpo.setSourceType("3");
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
continue;
}
String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
for (String exchange : exchanges) {
StockIpo ipo = new StockIpo();
ipo.setStockCode(stockIpo.getStockCode());
ipo.setStockName(stockIpo.getStockName());
ipo.setStockPrice(stockIpo.getStockPrice());
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
ipo.setListingDate(stockIpo.getListingDate());
ipo.setTotalNumber(stockIpo.getTotalNumber());
ipo.setApply(stockIpo.getApply());
ipo.setCreateDate(stockIpo.getCreateDate());
ipo.setUpdateDate(stockIpo.getUpdateDate());
ipo.setExchangeType(exchange);
ipo.setSourceType("3");
listStockIpoList.add(ipo);
}
}
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
for (int i = 0; i < upcomingData.size(); i++) {
JSONObject entry = upcomingData.getJSONObject(i);
StockIpo stockIpo = new StockIpo();
stockIpo.setStockCode(entry.getString("sc_id"));
stockIpo.setStockName(entry.getString("company_name"));
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
Integer totalNumber = 0;
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
Integer issue_size = entry.getInteger("issue_size");
BigDecimal issueSize = new BigDecimal(issue_size);
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
totalNumber = totalNumberBigDecimal.intValue();
}
stockIpo.setTotalNumber(totalNumber);
stockIpo.setApply(entry.getString("total_subs"));
stockIpo.setCreateDate(new Date());
stockIpo.setUpdateDate(new Date());
stockIpo.setSourceType("3");
for (String exchange : exchanges) {
StockIpo ipo = new StockIpo();
ipo.setStockCode(stockIpo.getStockCode());
ipo.setStockName(stockIpo.getStockName());
ipo.setStockPrice(stockIpo.getStockPrice());
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
ipo.setListingDate(stockIpo.getListingDate());
ipo.setTotalNumber(stockIpo.getTotalNumber());
ipo.setApply(stockIpo.getApply());
ipo.setCreateDate(stockIpo.getCreateDate());
ipo.setUpdateDate(stockIpo.getUpdateDate());
ipo.setExchangeType(exchange);
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
continue;
}
listStockIpoList.add(ipo);
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
for (String exchange : exchanges) {
StockIpo ipo = new StockIpo();
ipo.setStockCode(stockIpo.getStockCode());
ipo.setStockName(stockIpo.getStockName());
ipo.setStockPrice(stockIpo.getStockPrice());
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
ipo.setListingDate(stockIpo.getListingDate());
ipo.setTotalNumber(stockIpo.getTotalNumber());
ipo.setApply(stockIpo.getApply());
ipo.setCreateDate(stockIpo.getCreateDate());
ipo.setUpdateDate(stockIpo.getUpdateDate());
ipo.setExchangeType(exchange);
ipo.setSourceType("3");
listStockIpoList.add(ipo);
}
}
// stockIpoRepository.saveAll(listStockIpoList);
if (!listStockIpoList.isEmpty())
{
listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList());
}
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
if(CollectionUtil.isNotEmpty(scIdList)){
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
List<String> existingStockScIds = existStockIpoList.stream()
.map(StockIpo::getStockCode)
.collect(Collectors.toList());
for (StockIpo stockIpo : listStockIpoList) {
StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().equals(stockIpo.getExchangeType()) )).findFirst().orElse(null);
if (existIpo != null) {
if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
existIpo.setExchangeType(stockIpo.getExchangeType());
}
stockIpo.setId(existIpo.getId());
}
}
// listStockIpoList = listStockIpoList.stream()
// .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
// .collect(Collectors.toList());
//保存全部的新股
if(CollectionUtil.isNotEmpty(listStockIpoList)){
stockIpoRepository.saveAll(listStockIpoList);
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
}else {
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
}
}
// 输出整个 JSON 数据
} else {
log.info("未找到包含 JSON 数据的 <script> 标签");
}
for (int i = 0; i < upcomingData.size(); i++) {
JSONObject entry = upcomingData.getJSONObject(i);
StockIpo stockIpo = new StockIpo();
stockIpo.setStockCode(entry.getString("sc_id"));
stockIpo.setStockName(entry.getString("company_name"));
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
Integer totalNumber = 0;
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
Integer issue_size = entry.getInteger("issue_size");
BigDecimal issueSize = new BigDecimal(issue_size);
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
totalNumber = totalNumberBigDecimal.intValue();
}
stockIpo.setTotalNumber(totalNumber);
stockIpo.setApply(entry.getString("total_subs"));
stockIpo.setCreateDate(new Date());
stockIpo.setUpdateDate(new Date());
stockIpo.setSourceType("3");
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
continue;
}
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
for (String exchange : exchanges) {
StockIpo ipo = new StockIpo();
ipo.setStockCode(stockIpo.getStockCode());
ipo.setStockName(stockIpo.getStockName());
ipo.setStockPrice(stockIpo.getStockPrice());
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
ipo.setListingDate(stockIpo.getListingDate());
ipo.setTotalNumber(stockIpo.getTotalNumber());
ipo.setApply(stockIpo.getApply());
ipo.setCreateDate(stockIpo.getCreateDate());
ipo.setUpdateDate(stockIpo.getUpdateDate());
ipo.setExchangeType(exchange);
listStockIpoList.add(ipo);
}
}
// stockIpoRepository.saveAll(listStockIpoList);
if (!listStockIpoList.isEmpty())
{
listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList());
}
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
if(CollectionUtil.isNotEmpty(scIdList)){
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
List<String> existingStockScIds = existStockIpoList.stream()
.map(StockIpo::getStockCode)
.collect(Collectors.toList());
listStockIpoList = listStockIpoList.stream()
.filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
.collect(Collectors.toList());
//保存全部的新股
if(CollectionUtil.isNotEmpty(listStockIpoList)){
stockIpoRepository.saveAll(listStockIpoList);
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
}else {
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
}
}
// 输出整个 JSON 数据
} else {
log.info("未找到包含 JSON 数据的 <script> 标签");
log.info("HTTP请求失败，状态码：" + response.getStatusLine().getStatusCode());
}
} else {
log.info("HTTP请求失败，状态码：" + response.getStatusLine().getStatusCode());
} catch (IOException e) {
log.error("获取新股接口发生异常",e);
}
} catch (IOException e) {
log.error("获取新股接口发生异常",e);
}
}
@Scheduled(cron = "0 0 19 * * ?")