nhanvubk/ger-market-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bToday代码提交

Browse Source
This commit is contained in:
Achilles
2023-12-26 18:00:33 +08:00
parent d28e97d9ba
commit 7b693410ae
5 changed files with 361 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/cn/stock/market/constant/StockData.java Normal file
View File

@@ -0,0 +1,16 @@
package cn.stock.market.constant;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.math.BigDecimal;
@AllArgsConstructor
@NoArgsConstructor
@Data
public class StockData {
private String upd_date;
private BigDecimal price;
}

6
src/main/java/cn/stock/market/infrastructure/db/po/BtodayStockPO.java
View File

@@ -4,10 +4,13 @@ import java.lang.Integer;
import java.lang.String;
import java.util.Date;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Generated;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import org.hibernate.annotations.DynamicInsert;
@@ -34,6 +37,9 @@ public class BtodayStockPO {
/**
* 主键 */
@Id
@GeneratedValue(
strategy = GenerationType.IDENTITY
)
Integer id;
/**

217
src/main/java/cn/stock/market/infrastructure/job/Scraper.java Normal file
View File

@@ -0,0 +1,217 @@
package cn.stock.market.infrastructure.job;
import cn.stock.market.domain.basic.entity.BtodayStock;
import cn.stock.market.domain.basic.repository.BtodayStockRepository;
import cn.stock.market.infrastructure.db.repo.BtodayStockRepo;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
@Slf4j
@RestController
public class Scraper {
@Autowired
private BtodayStockRepository btodayStockRepo;
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
@Scheduled(cron = "0 0 1 */2 * ?")
@RequestMapping("/testScraperGetBusinessToday")
public void schedule() {
String BASE_URL = "https://akm-img-a-in.tosshub.com/businesstoday/resource/market-widgets/prod/company-master-23-01-2023.json";
String company_name = "Bhagawati Oxygen Ltd";
try {
// 获取 JSON 数据
String json_data = scrapePage(BASE_URL);
// 解析 JSON 数据
if (json_data != null) {
List<BtodayStock> all = btodayStockRepo.findAll();
Map<String, String> sefUrlList = getSefUrl(json_data, company_name);
sefUrlList = sefUrlList.entrySet().stream()
.filter(entry -> all.stream().noneMatch(stock -> stock.getStockName().equals(entry.getKey())))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// 将 Map 中的数据分成 5 个线程处理
int batchSize = sefUrlList.size() / 5; // 假设分成 5 个线程
int threadCount = 5;
CompletableFuture<Void>[] futures = new CompletableFuture[threadCount];
for (int i = 0; i < threadCount; i++) {
int startIndex = i * batchSize;
int endIndex = (i == threadCount - 1) ? sefUrlList.size() : (i + 1) * batchSize;
Map<String, String> subMap = sefUrlList.entrySet().stream()
.skip(startIndex)
.limit(endIndex - startIndex)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
futures[i] = CompletableFuture.runAsync(() -> processSubMap(subMap), executorService);
}
// 等待所有异步任务完成
CompletableFuture.allOf(futures).get();
}
} catch (Exception e) {
log.error("IOException occurred while processing the JSON data", e);
}finally {
// 关闭线程池
executorService.shutdown();
}
}
private void processSubMap(Map<String, String> sefUrlList) {
for (Map.Entry<String, String> entry : sefUrlList.entrySet()) {
String companyName = entry.getKey();
String url = entry.getValue();
// 获取网页 HTML
String webHtml = null;
int maxRetries = 5;
int retryCount = 0;
while (retryCount < maxRetries) {
try {
webHtml = getWebsiteHtml(url);
break; // If successful, break out of the loop
} catch (java.net.SocketTimeoutException e) {
log.warn("Socket timeout exception occurred. Retrying... (" + (retryCount + 1) + "/" + maxRetries + ")");
retryCount++;
} catch (IOException e) {
log.warn("IOException occurred. Retrying... (" + (retryCount + 1) + "/" + maxRetries + ")");
retryCount++;
}
}
if (webHtml != null) {
// 获取公司代码
String coCode = getCompanyCode(webHtml);
if (coCode != null) {
// 获取股票市场列表
String[] stockMarketList = getStockMarket(webHtml);
for (String stockMarket : stockMarketList) {
// 获取网页详情
String detailUrl = buildWebDetailUrl(coCode, stockMarket);
// String webInfo = getWebDetail(detailUrl);
log.info("Stock detail coCode:{}, stockMarket:{}: ,detailUrl:{}", coCode, stockMarket,detailUrl);
BtodayStock btodayStock = new BtodayStock();
btodayStock.setStockName(companyName);
btodayStock.setCoCode(coCode);
btodayStock.setStockType(stockMarket);
btodayStock.setSelfUrl(url);
btodayStock.setUrl(detailUrl);
btodayStock.setLastUpdateTime(new Date());
btodayStockRepo.save(btodayStock);
/* if (webInfo != null) {
log.info("Stock detail for {} in {}: {}", coCode, stockMarket, webInfo);
log.info(webInfo);
} else {
log.warn("Failed to retrieve web detail information.");
}*/
}
} else {
log.warn("Failed to retrieve company code.");
}
} else {
log.warn("Failed to retrieve website HTML.");
}
}
}
private static String buildWebDetailUrl(String coCode, String stockMarket) {
return "https://marketapi.intoday.in/widget/stockdetail/pullview?co_code=" + coCode + "&exchange=" + stockMarket;
}
private static String scrapePage(String url) throws IOException {
log.info("Scraping " + url + "...");
return Jsoup.connect(url).ignoreContentType(true).execute().body();
}
private static Map<String, String> getSefUrl(String json_data, String companyName) {
Map<String, String> sefUrls = new HashMap<>();
// 在这里放入你的模糊匹配逻辑
JSONArray jsonArray = JSONArray.parseArray(json_data);
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject item = jsonArray.getJSONObject(i);
// 在这里放入你的模糊匹配逻辑
String sef_url = item.getString("sef_url");
String company_name = item.getString("companyname");
if (company_name != null && sef_url != null/* && company_name.equals(companyName)*/) {
sefUrls.put(company_name, sef_url);
}
}
return sefUrls;
}
private static String getWebsiteHtml(String url) throws IOException {
log.info("Getting website URL: " + url + "...");
return Jsoup.connect(url).timeout(10000).get().html();
}
private static String getCompanyCode(String text) {
Document document = Jsoup.parse(text);
Element companyCodeInput = document.selectFirst("input[id=comapnyCodeId]");
if (companyCodeInput != null) {
return companyCodeInput.attr("value");
} else {
log.warn("No <input> with id=\"companyCodeId\" found on the website.");
return null;
}
}
private static String[] getStockMarket(String text) {
Document document = Jsoup.parse(text);
Elements ulElements = document.select("ul[class*=wdg_rhs_hdr_ul]");
List<String> stockMarketList = new ArrayList<>();
for (Element ulElement : ulElements) {
Elements liElements = ulElement.select("li");
for (Element liElement : liElements) {
Element spanElement = liElement.selectFirst("span[class=wdg_rhs_hdr_lnk]");
if (spanElement != null) {
stockMarketList.add(spanElement.text());
} else {
log.warn("Invalid status code while scraping.");
}
}
}
return stockMarketList.toArray(new String[0]);
}
private static String getWebDetail(String url) throws IOException {
log.info("Getting web detail URL: " + url + "...");
return Jsoup.connect(url).ignoreContentType(true).execute().body();
}
}

106
src/main/java/cn/stock/market/web/BTodayStockController.java Normal file
View File

@@ -0,0 +1,106 @@
package cn.stock.market.web;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
@RestController
@Api(tags="bToday获取详情接口")
public class BTodayStockController {
@Autowired
private RestTemplate restTemplate;
@GetMapping("/api/bToday/kLine")
@ApiOperation(value = "股票详情K线图",httpMethod = "GET")
@ApiImplicitParams({
@ApiImplicitParam(name="exchange",value = "BSE或者NSE"),
@ApiImplicitParam(name="co_code",value = "BToday的coCode值"),
@ApiImplicitParam(name="format",value = "S(1D的时候传S),H(5D的时候传H),H(3M的时候传H),H(1Y的时候传H),H(5Y的时候传H),H(10Y的时候传H)"),
@ApiImplicitParam(name="durationtype",value = "D(1D的时候传D),D(5D的时候传D),M(3M的时候传M),Y(1Y的时候传Y),Y(5Y的时候传Y),Y(10Y的时候传Y)"),
@ApiImplicitParam(name="duration",value = "1(1D的时候传1),5(5D的时候传5),3(3M的时候传3),1(1Y的时候传1),5(5Y的时候传5),10(10Y的时候传10)"),
})
public JSONArray getPriceChartCompanyPullView(
@RequestParam(value = "exchange") String exchange,
@RequestParam(value = "co_code") String coCode,
@RequestParam(value = "format") String format,
@RequestParam(value = "durationtype") String durationType,
@RequestParam(value = "duration") String duration) {
if (StringUtils.isBlank(exchange) || StringUtils.isBlank(coCode) || StringUtils.isBlank(format) || StringUtils.isBlank(durationType) || StringUtils.isBlank(duration)) {
return new JSONArray();
}
// 构建请求URL
String apiUrl = buildApiUrl(exchange, coCode, format, durationType, duration);
// 发起REST请求并获取响应数据
Map<String, Object> response = restTemplate.getForObject(apiUrl, Map.class);
List<Map<String, Object>> data = (List<Map<String, Object>>) response.get("data");
// 转换为StockData列表
JSONArray jsonArray = new JSONArray();
data.forEach(item -> {
String updateDate = (String) item.get("upd_date");
BigDecimal price = new BigDecimal(String.valueOf(item.get("price")));
// 创建JSONObject并放入数据
JSONObject jsonObject = new JSONObject();
jsonObject.put("updateDate", updateDate);
jsonObject.put("price", price);
jsonArray.add(jsonObject);
});
return jsonArray;
}
@ApiOperation(value = "股票详情信息",httpMethod = "GET")
@ApiImplicitParams({
@ApiImplicitParam(name="exchange",value = "BSE或者NSE"),
@ApiImplicitParam(name="co_code",value = "coCode值"),
})
@GetMapping("/api/bToday/stockDetail")
public com.alibaba.fastjson.JSONObject getPriceChartCompanyPullView(
@RequestParam(value = "exchange") String exchange,
@RequestParam(value = "co_code") String coCode
) {
if (StringUtils.isBlank(exchange) || StringUtils.isBlank(coCode) ) {
return new com.alibaba.fastjson.JSONObject();
}
// 构建请求URL
String apiUrl = buildDetailApiUrl(exchange, coCode);
String forObject = restTemplate.getForObject(apiUrl, String.class);
return com.alibaba.fastjson.JSONObject.parseObject(forObject);
}
private String buildDetailApiUrl(String exchange, String coCode) {
String url = String.format("https://marketapi.intoday.in/widget/stockdetail/pullview?co_code=%s&exchange=%s",coCode,exchange);
return url;
}
private String buildApiUrl(String exchange, String coCode, String format, String durationType, String duration) {
// 构建请求URL的逻辑，根据你的实际情况来
// 示例：return "https://your-api-endpoint/bToday/kLine?exchange=" + exchange + "&co_code=" + coCode + "&format=" + format + "&durationtype=" + durationType + "&duration=" + duration;
String url = String.format("https://marketapi.intoday.in/widget/pricechart_company/pullview?exchange=%s&co_code=%s&format=%s&durationtype=%s&duration=%s",
exchange, coCode, format, durationType, duration);
return url;
}
}

16
src/main/resources/rebel.xml Normal file
View File

@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
This is the JRebel configuration file. It maps the running application to your IDE workspace, enabling JRebel reloading for this project.
Refer to https://manuals.jrebel.com/jrebel/standalone/config.html for more information.
-->
<application generated-by="intellij" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.zeroturnaround.com" xsi:schemaLocation="http://www.zeroturnaround.com http://update.zeroturnaround.com/jrebel/rebel-2_3.xsd">
<id>market</id>
<classpath>
<dir name="/Users/gs/Downloads/works/india_market_java/target/classes">
</dir>
</classpath>
</application>