diff --git a/src/main/java/cn/stock/market/infrastructure/db/po/StockIpoPO.java b/src/main/java/cn/stock/market/infrastructure/db/po/StockIpoPO.java index b228275..abedbed 100644 --- a/src/main/java/cn/stock/market/infrastructure/db/po/StockIpoPO.java +++ b/src/main/java/cn/stock/market/infrastructure/db/po/StockIpoPO.java @@ -79,6 +79,8 @@ public class StockIpoPO { * 申请总额 */ String apply; + String exchangeType; + /** * 市盈率 */ BigDecimal peRatio; diff --git a/src/main/java/cn/stock/market/infrastructure/job/Scraper.java b/src/main/java/cn/stock/market/infrastructure/job/Scraper.java index 7a4d8cd..9457301 100644 --- a/src/main/java/cn/stock/market/infrastructure/job/Scraper.java +++ b/src/main/java/cn/stock/market/infrastructure/job/Scraper.java @@ -1,14 +1,18 @@ package cn.stock.market.infrastructure.job; import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DateTime; import cn.stock.market.domain.basic.entity.BtodayStock; +import cn.stock.market.domain.basic.entity.MoneyStock; import cn.stock.market.domain.basic.entity.StockIpo; import cn.stock.market.domain.basic.repository.BtodayStockRepository; +import cn.stock.market.domain.basic.repository.MoneyStockRepository; import cn.stock.market.domain.basic.repository.StockIpoRepository; import cn.stock.market.dto.StockIpoDTO; import cn.stock.market.infrastructure.db.po.QStockIpoPO; import cn.stock.market.infrastructure.db.po.StockIpoPO; import cn.stock.market.infrastructure.db.repo.BtodayStockRepo; +import com.alibaba.druid.sql.visitor.functions.Now; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; @@ -31,14 +35,11 @@ import org.springframework.stereotype.Component; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import javax.annotation.PostConstruct; import java.io.IOException; import java.math.BigDecimal; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -54,6 +55,8 @@ public class Scraper { private StockIpoRepository stockIpoRepository; private final ExecutorService executorService = Executors.newFixedThreadPool(5); + @Autowired + private MoneyStockRepository moneyStockRepository; @Scheduled(cron = "0 0 1 */2 * ?") @@ -103,7 +106,8 @@ public class Scraper { - @Scheduled(cron = "0 0 18 * * ?") +// @Scheduled(cron = "0 0 18 * * ?") + @PostConstruct @RequestMapping("/testScraperGetMoneyControllerNewIPO") public void getMoneyControllerNewIPOSchedule() { log.info("定时任务执行获取新股ipo的方法开始执行"); @@ -163,7 +167,28 @@ public class Scraper { stockIpo.setCreateDate(new Date()); stockIpo.setUpdateDate(new Date()); stockIpo.setSourceType("3"); - listStockIpoList.add(stockIpo); + + if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){ + continue; + } + + List exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode()); + + for (String exchange : exchanges) { + StockIpo ipo = new StockIpo(); + ipo.setStockCode(stockIpo.getStockCode()); + ipo.setStockName(stockIpo.getStockName()); + ipo.setStockPrice(stockIpo.getStockPrice()); + ipo.setSubscriptionDate(stockIpo.getSubscriptionDate()); + ipo.setListingDate(stockIpo.getListingDate()); + ipo.setTotalNumber(stockIpo.getTotalNumber()); + ipo.setApply(stockIpo.getApply()); + ipo.setCreateDate(stockIpo.getCreateDate()); + ipo.setUpdateDate(stockIpo.getUpdateDate()); + ipo.setExchangeType(exchange); + + listStockIpoList.add(ipo); + } } for (int i = 0; i < upcomingData.size(); i++) { @@ -186,11 +211,37 @@ public class Scraper { stockIpo.setCreateDate(new Date()); stockIpo.setUpdateDate(new Date()); stockIpo.setSourceType("3"); - listStockIpoList.add(stockIpo); + + if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){ + continue; + } + + List exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode()); + + for (String exchange : exchanges) { + StockIpo ipo = new StockIpo(); + ipo.setStockCode(stockIpo.getStockCode()); + ipo.setStockName(stockIpo.getStockName()); + ipo.setStockPrice(stockIpo.getStockPrice()); + ipo.setSubscriptionDate(stockIpo.getSubscriptionDate()); + ipo.setListingDate(stockIpo.getListingDate()); + ipo.setTotalNumber(stockIpo.getTotalNumber()); + ipo.setApply(stockIpo.getApply()); + ipo.setCreateDate(stockIpo.getCreateDate()); + ipo.setUpdateDate(stockIpo.getUpdateDate()); + ipo.setExchangeType(exchange); + + listStockIpoList.add(ipo); + } } // stockIpoRepository.saveAll(listStockIpoList); + if (!listStockIpoList.isEmpty()) + { + listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList()); + } + List scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode); if(CollectionUtil.isNotEmpty(scIdList)){ List existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList)); @@ -205,6 +256,10 @@ public class Scraper { //保存全部的新股 if(CollectionUtil.isNotEmpty(listStockIpoList)){ stockIpoRepository.saveAll(listStockIpoList); + + List moneyStockList = convertToMoneyStock(listStockIpoList); + moneyStockRepository.saveAll(moneyStockList); + log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList)); }else { log.info("定时任务执行获取新股ipo的方法开始结束,没有数据"); @@ -378,4 +433,64 @@ public class Scraper { log.info("Getting web detail URL: " + url + "..."); return Jsoup.connect(url).ignoreContentType(true).execute().body(); } + + public List getIpoExchange(String stockName, String stockCode) { + List exchanges = new ArrayList<>(); + String url = "https://www.moneycontrol.com/ipo/{stockName}-{stockCode}-ipodetail"; + url = url.replace("{stockName}", stockName.toLowerCase().replace(" ", "-")).replace("{stockCode}", stockCode.toLowerCase()); + + org.apache.http.client.HttpClient client = HttpClients.createDefault(); + HttpGet request = new HttpGet(url); + try { + org.apache.http.HttpResponse response = client.execute(request); + + if (response.getStatusLine().getStatusCode() == 200) { + String responseBody = EntityUtils.toString(response.getEntity()); + Document doc = Jsoup.parse(responseBody); + + Element scriptTag = doc.selectFirst("script#__NEXT_DATA__"); + + if (scriptTag != null) { + String jsonDataStr = scriptTag.html(); + + com.alibaba.fastjson.JSONObject jsonObject = com.alibaba.fastjson.JSONObject.parseObject(jsonDataStr); + log.info("获取到新股的json信息:" + jsonObject.toJSONString()); + + com.alibaba.fastjson.JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps"); + com.alibaba.fastjson.JSONObject overview = pageProps.getJSONObject("ipoDetails").getJSONObject("ipoDetails").getJSONObject("overview"); + + String exchangeStr = overview.getString("listing_on"); + + if (exchangeStr != null && !exchangeStr.isEmpty() && !exchangeStr.equals("null")) { + exchanges.addAll(new ArrayList<>(Arrays.asList(exchangeStr.split(",")))); + } + } + } + } + catch (Exception e) { + log.info(e.getMessage()); + } + + if (exchanges.isEmpty()) { + exchanges.add(""); + } + + return exchanges; + } + + public List convertToMoneyStock(List stockIpoList) { + return stockIpoList.stream().map(this::convert).collect(Collectors.toList()); + } + + private MoneyStock convert(StockIpo stockIpo) { + MoneyStock moneyStock = new MoneyStock(); + + moneyStock.setStockName(stockIpo.getStockName()); + moneyStock.setMoneyScId(stockIpo.getStockCode()); + moneyStock.setSelfDispId(stockIpo.getStockCode()); + moneyStock.setStockType(stockIpo.getExchangeType()); + moneyStock.setSaveTime(DateTime.now()); + + return moneyStock; + } } diff --git a/src/main/resources/application-base-alpha.yml b/src/main/resources/application-base-alpha.yml index 6572790..590859e 100644 --- a/src/main/resources/application-base-alpha.yml +++ b/src/main/resources/application-base-alpha.yml @@ -3,9 +3,9 @@ spring: show-sql: true # Redis配置 redis: - host: lb-b6sze686-7db6hgvsx05ef8nz.clb.sg-tencentclb.net + host: 43.156.40.39 password: a5v8b86P4mVzFlUqJV - port: 56379 + port: 30031 database: 1 lettuce: pool: @@ -17,7 +17,7 @@ spring: datasource: stock-market: driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://lb-1ju43bvw-p0ecnv4by9ju8six.clb.ap-singapore.tencentclb.com:53306/india_stock?useUnicode=true&characterEncoding=utf-8 + url: jdbc:mysql://43.156.40.39:30030/india_stock?useUnicode=true&characterEncoding=utf-8 username: root password: uNejHIFQGJOUtYTmE maxActive: 500