From 7b693410ae5f06150e5219c364157fdefe0bc9f3 Mon Sep 17 00:00:00 2001 From: Achilles Date: Tue, 26 Dec 2023 18:00:33 +0800 Subject: [PATCH] =?UTF-8?q?bToday=E4=BB=A3=E7=A0=81=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/stock/market/constant/StockData.java | 16 ++ .../infrastructure/db/po/BtodayStockPO.java | 6 + .../market/infrastructure/job/Scraper.java | 217 ++++++++++++++++++ .../market/web/BTodayStockController.java | 106 +++++++++ src/main/resources/rebel.xml | 16 ++ 5 files changed, 361 insertions(+) create mode 100644 src/main/java/cn/stock/market/constant/StockData.java create mode 100644 src/main/java/cn/stock/market/infrastructure/job/Scraper.java create mode 100644 src/main/java/cn/stock/market/web/BTodayStockController.java create mode 100644 src/main/resources/rebel.xml diff --git a/src/main/java/cn/stock/market/constant/StockData.java b/src/main/java/cn/stock/market/constant/StockData.java new file mode 100644 index 0000000..48013f6 --- /dev/null +++ b/src/main/java/cn/stock/market/constant/StockData.java @@ -0,0 +1,16 @@ +package cn.stock.market.constant; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.math.BigDecimal; + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class StockData { + private String upd_date; + private BigDecimal price; + +} diff --git a/src/main/java/cn/stock/market/infrastructure/db/po/BtodayStockPO.java b/src/main/java/cn/stock/market/infrastructure/db/po/BtodayStockPO.java index a949961..fddb048 100644 --- a/src/main/java/cn/stock/market/infrastructure/db/po/BtodayStockPO.java +++ b/src/main/java/cn/stock/market/infrastructure/db/po/BtodayStockPO.java @@ -4,10 +4,13 @@ import java.lang.Integer; import java.lang.String; import java.util.Date; import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.Generated; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; import org.hibernate.annotations.DynamicInsert; @@ -34,6 +37,9 @@ public class BtodayStockPO { /** * 主键 */ @Id + @GeneratedValue( + strategy = GenerationType.IDENTITY + ) Integer id; /** diff --git a/src/main/java/cn/stock/market/infrastructure/job/Scraper.java b/src/main/java/cn/stock/market/infrastructure/job/Scraper.java new file mode 100644 index 0000000..0ce8ba8 --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/job/Scraper.java @@ -0,0 +1,217 @@ +package cn.stock.market.infrastructure.job; + +import cn.stock.market.domain.basic.entity.BtodayStock; +import cn.stock.market.domain.basic.repository.BtodayStockRepository; +import cn.stock.market.infrastructure.db.repo.BtodayStockRepo; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +@RestController +public class Scraper { + + @Autowired + private BtodayStockRepository btodayStockRepo; + + private final ExecutorService executorService = Executors.newFixedThreadPool(5); + + + @Scheduled(cron = "0 0 1 */2 * ?") + @RequestMapping("/testScraperGetBusinessToday") + public void schedule() { + String BASE_URL = "https://akm-img-a-in.tosshub.com/businesstoday/resource/market-widgets/prod/company-master-23-01-2023.json"; + String company_name = "Bhagawati Oxygen Ltd"; + + try { + // 获取 JSON 数据 + String json_data = scrapePage(BASE_URL); + // 解析 JSON 数据 + if (json_data != null) { + List all = btodayStockRepo.findAll(); + Map sefUrlList = getSefUrl(json_data, company_name); + sefUrlList = sefUrlList.entrySet().stream() + .filter(entry -> all.stream().noneMatch(stock -> stock.getStockName().equals(entry.getKey()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // 将 Map 中的数据分成 5 个线程处理 + int batchSize = sefUrlList.size() / 5; // 假设分成 5 个线程 + int threadCount = 5; + CompletableFuture[] futures = new CompletableFuture[threadCount]; + for (int i = 0; i < threadCount; i++) { + int startIndex = i * batchSize; + int endIndex = (i == threadCount - 1) ? sefUrlList.size() : (i + 1) * batchSize; + + Map subMap = sefUrlList.entrySet().stream() + .skip(startIndex) + .limit(endIndex - startIndex) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + futures[i] = CompletableFuture.runAsync(() -> processSubMap(subMap), executorService); + } + // 等待所有异步任务完成 + CompletableFuture.allOf(futures).get(); + } + } catch (Exception e) { + log.error("IOException occurred while processing the JSON data", e); + }finally { + // 关闭线程池 + executorService.shutdown(); + } + } + + private void processSubMap(Map sefUrlList) { + for (Map.Entry entry : sefUrlList.entrySet()) { + String companyName = entry.getKey(); + String url = entry.getValue(); + + // 获取网页 HTML + String webHtml = null; + int maxRetries = 5; + int retryCount = 0; + + while (retryCount < maxRetries) { + try { + webHtml = getWebsiteHtml(url); + break; // If successful, break out of the loop + } catch (java.net.SocketTimeoutException e) { + log.warn("Socket timeout exception occurred. Retrying... (" + (retryCount + 1) + "/" + maxRetries + ")"); + retryCount++; + } catch (IOException e) { + log.warn("IOException occurred. Retrying... (" + (retryCount + 1) + "/" + maxRetries + ")"); + retryCount++; + } + } + if (webHtml != null) { + // 获取公司代码 + String coCode = getCompanyCode(webHtml); + + if (coCode != null) { + // 获取股票市场列表 + String[] stockMarketList = getStockMarket(webHtml); + + for (String stockMarket : stockMarketList) { + // 获取网页详情 + String detailUrl = buildWebDetailUrl(coCode, stockMarket); + // String webInfo = getWebDetail(detailUrl); + log.info("Stock detail coCode:{}, stockMarket:{}: ,detailUrl:{}", coCode, stockMarket,detailUrl); + + BtodayStock btodayStock = new BtodayStock(); + btodayStock.setStockName(companyName); + btodayStock.setCoCode(coCode); + btodayStock.setStockType(stockMarket); + btodayStock.setSelfUrl(url); + btodayStock.setUrl(detailUrl); + btodayStock.setLastUpdateTime(new Date()); + btodayStockRepo.save(btodayStock); + + /* if (webInfo != null) { + log.info("Stock detail for {} in {}: {}", coCode, stockMarket, webInfo); + log.info(webInfo); + + } else { + log.warn("Failed to retrieve web detail information."); + }*/ + } + } else { + log.warn("Failed to retrieve company code."); + } + } else { + log.warn("Failed to retrieve website HTML."); + } + } + } + + private static String buildWebDetailUrl(String coCode, String stockMarket) { + return "https://marketapi.intoday.in/widget/stockdetail/pullview?co_code=" + coCode + "&exchange=" + stockMarket; + } + + private static String scrapePage(String url) throws IOException { + log.info("Scraping " + url + "..."); + return Jsoup.connect(url).ignoreContentType(true).execute().body(); + } + + private static Map getSefUrl(String json_data, String companyName) { + Map sefUrls = new HashMap<>(); + + // 在这里放入你的模糊匹配逻辑 + JSONArray jsonArray = JSONArray.parseArray(json_data); + + for (int i = 0; i < jsonArray.size(); i++) { + JSONObject item = jsonArray.getJSONObject(i); + + // 在这里放入你的模糊匹配逻辑 + String sef_url = item.getString("sef_url"); + String company_name = item.getString("companyname"); + + if (company_name != null && sef_url != null/* && company_name.equals(companyName)*/) { + sefUrls.put(company_name, sef_url); + } + } + + return sefUrls; + } + + private static String getWebsiteHtml(String url) throws IOException { + log.info("Getting website URL: " + url + "..."); + return Jsoup.connect(url).timeout(10000).get().html(); + } + + private static String getCompanyCode(String text) { + Document document = Jsoup.parse(text); + Element companyCodeInput = document.selectFirst("input[id=comapnyCodeId]"); + + if (companyCodeInput != null) { + return companyCodeInput.attr("value"); + } else { + log.warn("No with id=\"companyCodeId\" found on the website."); + return null; + } + } + + private static String[] getStockMarket(String text) { + Document document = Jsoup.parse(text); + Elements ulElements = document.select("ul[class*=wdg_rhs_hdr_ul]"); + + List stockMarketList = new ArrayList<>(); + + for (Element ulElement : ulElements) { + Elements liElements = ulElement.select("li"); + for (Element liElement : liElements) { + Element spanElement = liElement.selectFirst("span[class=wdg_rhs_hdr_lnk]"); + if (spanElement != null) { + stockMarketList.add(spanElement.text()); + } else { + log.warn("Invalid status code while scraping."); + } + } + } + + return stockMarketList.toArray(new String[0]); + } + + private static String getWebDetail(String url) throws IOException { + log.info("Getting web detail URL: " + url + "..."); + return Jsoup.connect(url).ignoreContentType(true).execute().body(); + } +} diff --git a/src/main/java/cn/stock/market/web/BTodayStockController.java b/src/main/java/cn/stock/market/web/BTodayStockController.java new file mode 100644 index 0000000..f99fdb8 --- /dev/null +++ b/src/main/java/cn/stock/market/web/BTodayStockController.java @@ -0,0 +1,106 @@ +package cn.stock.market.web; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestTemplate; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +@RestController +@Api(tags="bToday获取详情接口") +public class BTodayStockController { + + + @Autowired + private RestTemplate restTemplate; + + @GetMapping("/api/bToday/kLine") + @ApiOperation(value = "股票详情K线图",httpMethod = "GET") + @ApiImplicitParams({ + @ApiImplicitParam(name="exchange",value = "BSE或者NSE"), + @ApiImplicitParam(name="co_code",value = "BToday的coCode值"), + @ApiImplicitParam(name="format",value = "S(1D的时候传S),H(5D的时候传H),H(3M的时候传H),H(1Y的时候传H),H(5Y的时候传H),H(10Y的时候传H)"), + @ApiImplicitParam(name="durationtype",value = "D(1D的时候传D),D(5D的时候传D),M(3M的时候传M),Y(1Y的时候传Y),Y(5Y的时候传Y),Y(10Y的时候传Y)"), + @ApiImplicitParam(name="duration",value = "1(1D的时候传1),5(5D的时候传5),3(3M的时候传3),1(1Y的时候传1),5(5Y的时候传5),10(10Y的时候传10)"), + }) + public JSONArray getPriceChartCompanyPullView( + @RequestParam(value = "exchange") String exchange, + @RequestParam(value = "co_code") String coCode, + @RequestParam(value = "format") String format, + @RequestParam(value = "durationtype") String durationType, + @RequestParam(value = "duration") String duration) { + + if (StringUtils.isBlank(exchange) || StringUtils.isBlank(coCode) || StringUtils.isBlank(format) || StringUtils.isBlank(durationType) || StringUtils.isBlank(duration)) { + return new JSONArray(); + } + // 构建请求URL + String apiUrl = buildApiUrl(exchange, coCode, format, durationType, duration); + + // 发起REST请求并获取响应数据 + Map response = restTemplate.getForObject(apiUrl, Map.class); + List> data = (List>) response.get("data"); + + // 转换为StockData列表 + JSONArray jsonArray = new JSONArray(); + data.forEach(item -> { + String updateDate = (String) item.get("upd_date"); + BigDecimal price = new BigDecimal(String.valueOf(item.get("price"))); + + // 创建JSONObject并放入数据 + JSONObject jsonObject = new JSONObject(); + jsonObject.put("updateDate", updateDate); + jsonObject.put("price", price); + + jsonArray.add(jsonObject); + }); + return jsonArray; + } + + + @ApiOperation(value = "股票详情信息",httpMethod = "GET") + @ApiImplicitParams({ + @ApiImplicitParam(name="exchange",value = "BSE或者NSE"), + @ApiImplicitParam(name="co_code",value = "coCode值"), + }) + @GetMapping("/api/bToday/stockDetail") + public com.alibaba.fastjson.JSONObject getPriceChartCompanyPullView( + @RequestParam(value = "exchange") String exchange, + @RequestParam(value = "co_code") String coCode + ) { + + if (StringUtils.isBlank(exchange) || StringUtils.isBlank(coCode) ) { + return new com.alibaba.fastjson.JSONObject(); + } + // 构建请求URL + String apiUrl = buildDetailApiUrl(exchange, coCode); + + String forObject = restTemplate.getForObject(apiUrl, String.class); + + return com.alibaba.fastjson.JSONObject.parseObject(forObject); + } + + private String buildDetailApiUrl(String exchange, String coCode) { + String url = String.format("https://marketapi.intoday.in/widget/stockdetail/pullview?co_code=%s&exchange=%s",coCode,exchange); + return url; + } + + + private String buildApiUrl(String exchange, String coCode, String format, String durationType, String duration) { + // 构建请求URL的逻辑，根据你的实际情况来 + // 示例：return "https://your-api-endpoint/bToday/kLine?exchange=" + exchange + "&co_code=" + coCode + "&format=" + format + "&durationtype=" + durationType + "&duration=" + duration; + String url = String.format("https://marketapi.intoday.in/widget/pricechart_company/pullview?exchange=%s&co_code=%s&format=%s&durationtype=%s&duration=%s", + exchange, coCode, format, durationType, duration); + return url; + } +} \ No newline at end of file diff --git a/src/main/resources/rebel.xml b/src/main/resources/rebel.xml new file mode 100644 index 0000000..7d1c88d --- /dev/null +++ b/src/main/resources/rebel.xml @@ -0,0 +1,16 @@ + + + + + + market + + + + + + +