Compare commits

...

41 Commits

Author SHA1 Message Date
vpckiet
d1f9600361 update money ipo check 2025-05-22 14:32:35 +07:00
vpckiet
ad95848c2c update ipo check 2025-05-22 14:00:18 +07:00
vu-tran
90a26823f0 update news 2025-05-13 10:10:42 +07:00
VoGiaHuy2058
c72c0ff4cd Revert "fix cron"
This reverts commit 85094c22b2.
2025-05-12 18:53:06 +07:00
VoGiaHuy2058
85094c22b2 fix cron 2025-05-12 18:44:16 +07:00
mush_ball
59684cccfc Edit .gitlab-ci.yml 2025-04-28 10:45:30 +00:00
vu-tran
57690ecb5e update logs 2025-04-28 17:38:16 +07:00
vu-tran
8cf511f1b7 update ipo 2025-04-24 13:41:59 +07:00
vu-tran
f612b5128e update issue size to long 2025-02-10 09:54:10 +07:00
vu-tran
07a69f29b9 update get data open ipo 2025-02-10 09:37:39 +07:00
vu-tran
3e1ff24bb6 update link ipo 2025-01-15 11:27:40 +07:00
vu-tran
b26d3962c4 update crawl ipo 2024-12-11 14:23:08 +07:00
vu-tran
d085d8de12 update crawl ipo 2024-12-11 12:39:15 +07:00
vpckiet
0476ad100b Merge branch 'bug/update_bse' into 'develop'
fix bug

See merge request india/india_market_java!55
2024-12-10 17:03:43 +00:00
Congkiet4695
f87df03715 fix bug 2024-12-11 00:01:57 +07:00
vpckiet
fc59d1f47c Merge branch 'bug/update_symbol' into 'develop'
fix bug chart day

See merge request india/india_market_java!54
2024-12-10 14:57:04 +00:00
vpckiet
6fdc13a72d fix bug chart day 2024-12-10 21:56:23 +07:00
vpckiet
6b5623dc33 Merge branch 'bug/update_symbol' into 'develop'
fix bug get chart

See merge request india/india_market_java!53
2024-12-10 13:15:40 +00:00
vpckiet
4a22fc0a95 fix bug get chart 2024-12-10 20:14:42 +07:00
vpckiet
563dae1c45 Merge branch 'feature/update_bseindia_detail' into 'develop'
update chart

See merge request india/india_market_java!52
2024-12-10 12:55:38 +00:00
vpckiet
fa02ebf03b update chart 2024-12-10 19:54:45 +07:00
vpckiet
0f5f25f744 Merge branch 'feature/update_bseindia_detail' into 'develop'
update get detail from bseindia

See merge request india/india_market_java!51
2024-12-10 09:38:33 +00:00
vpckiet
615aae7e1d update get detail from bseindia 2024-12-10 16:37:54 +07:00
vu-tran
c89345329d uncomment scheduler 2024-11-12 11:13:22 +07:00
vpckiet
f18521d190 Merge branch 'bug/nse_type' into 'develop'
fix bug

See merge request india/india_market_java!50
2024-10-28 10:40:50 +00:00
vpckiet
e84c212bf0 fix bug 2024-10-28 17:40:04 +07:00
vogiahuy2058
1ff1797b0b done new logic to add moneystock 2024-10-24 14:10:42 +07:00
vogiahuy2058
89c0fe0321 change cron 2024-10-23 16:39:41 +07:00
vogiahuy2058
b6052a46ff done save exchange type and save money stock 2024-10-23 16:35:59 +07:00
vpckiet
d40c30bcb5 Merge branch 'bug/crawl_news' into 'develop'
update crawl

See merge request india/india_market_java!48
2024-10-21 09:19:14 +00:00
vpckiet
5b16f1f2c9 update crawl 2024-10-21 16:14:22 +07:00
vpckiet
b8d4e0d3f2 Merge branch 'bug/update_kline_format' into 'develop'
update

See merge request india/india_market_java!46
2024-10-10 03:50:50 +00:00
vpckiet
c19277e390 update 2024-10-10 10:50:01 +07:00
vpckiet
514875878f Merge branch 'bug/update_kline_format' into 'develop'
fix bug kline

See merge request india/india_market_java!45
2024-10-10 01:47:17 +00:00
vpckiet
4257541342 fix bug kline 2024-10-09 17:18:26 +07:00
quangnguyen202
441adac15a Merge branch 'feature/optional-stock' into 'develop'
Add api for optional stock

See merge request india/india_market_java!44
2024-10-01 07:07:59 +00:00
quangnguyen202
ba29ab98c7 Add api for optional stock 2024-10-01 14:06:24 +07:00
Gavin g
94ec73874b Merge branch 'feature/kline-alternate-source' into 'develop'
Add alternate source

See merge request india/india_market_java!43
2024-09-26 09:57:06 +00:00
quangnguyen202
549345719a Add alternate source 2024-09-26 16:51:41 +07:00
gavin
3870400411 修改切换备用数据源的判断条件 2024-09-19 15:59:12 +08:00
Gavin g
94d2936ffb Merge branch 'feature/stock-kline-nseindia' into 'develop'
Add alternal source

See merge request india/india_market_java!42
2024-09-19 07:37:45 +00:00
20 changed files with 1262 additions and 236 deletions

View File

@@ -67,8 +67,8 @@ deploy-dev:
script:
- echo "Deploying application..."
- ls
- apt-get update
- apt-get install -y curl
- apk update
- apk add curl
- |
curl -X POST \
-H "Content-Type: application/json" \

View File

@@ -185,6 +185,12 @@
<version>30.1-jre</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>
</dependencies>
<build>

View File

@@ -0,0 +1,41 @@
package cn.stock.market.infrastructure.db.po;
import static com.querydsl.core.types.PathMetadataFactory.*;
import com.querydsl.core.types.dsl.*;
import com.querydsl.core.types.PathMetadata;
import javax.annotation.Generated;
import com.querydsl.core.types.Path;
/**
* QOptionalStockPO is a Querydsl query type for OptionalStockPO
*/
@Generated("com.querydsl.codegen.EntitySerializer")
public class QOptionalStockPO extends EntityPathBase<OptionalStockPO> {
private static final long serialVersionUID = 1161631810L;
public static final QOptionalStockPO optionalStockPO = new QOptionalStockPO("optionalStockPO");
public final StringPath company = createString("company");
public final NumberPath<Integer> id = createNumber("id", Integer.class);
public final StringPath symbol = createString("symbol");
public QOptionalStockPO(String variable) {
super(OptionalStockPO.class, forVariable(variable));
}
public QOptionalStockPO(Path<? extends OptionalStockPO> path) {
super(path.getType(), path.getMetadata());
}
public QOptionalStockPO(PathMetadata metadata) {
super(OptionalStockPO.class, metadata);
}
}

View File

@@ -38,6 +38,7 @@ public class QStockIpoPO extends EntityPathBase<StockIpoPO> {
public final StringPath stockCode = createString("stockCode");
public final StringPath stockName = createString("stockName");
public final StringPath exchangeType = createString("exchangeType");
public final NumberPath<java.math.BigDecimal> stockPrice = createNumber("stockPrice", java.math.BigDecimal.class);

View File

@@ -0,0 +1,16 @@
package cn.stock.market.domain.basic.convert;
import cn.qutaojing.common.domain.convert.SimpleEntityPOConvert;
import cn.qutaojing.common.utils.SpringUtils;
import cn.stock.market.domain.basic.entity.OptionalStock;
import cn.stock.market.infrastructure.db.po.OptionalStockPO;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@Component
@Lazy
public class OptionalStockConvert extends SimpleEntityPOConvert<OptionalStock, OptionalStockPO> {
public static OptionalStockConvert of() {
return SpringUtils.getBean(OptionalStockConvert.class);
}
}

View File

@@ -0,0 +1,16 @@
package cn.stock.market.domain.basic.entity;
import cn.stock.market.infrastructure.db.po.OptionalStockPO;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
@Data
@NoArgsConstructor
@SuperBuilder
@EqualsAndHashCode(
callSuper = false
)
public class OptionalStock extends OptionalStockPO {
}

View File

@@ -0,0 +1,32 @@
package cn.stock.market.domain.basic.repository;
import cn.qutaojing.common.domain.convert.IEntityPOConvert;
import cn.qutaojing.common.domain.respostory.SimplePoConvertEntityRepository;
import cn.stock.market.domain.basic.convert.OptionalStockConvert;
import cn.stock.market.domain.basic.entity.OptionalStock;
import cn.stock.market.infrastructure.db.po.OptionalStockPO;
import cn.stock.market.infrastructure.db.repo.OptionalStockRepo;
import com.rp.spring.jpa.GenericJpaRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
@Repository
@RequiredArgsConstructor(
onConstructor = @__(@Autowired)
)
public class OptionalStockRepository extends SimplePoConvertEntityRepository<OptionalStock, OptionalStockPO, Integer> {
final OptionalStockRepo repo;
final OptionalStockConvert convert;
@Override
public GenericJpaRepository<OptionalStockPO, Integer> repo() {
return repo;
}
@Override
public IEntityPOConvert<OptionalStock, OptionalStockPO> convert() {
return convert;
}
}

View File

@@ -249,40 +249,112 @@ public class StockService {
String result = "";
try {
// 使用Jsoup连接到网页
Document doc = Jsoup.connect("https://www.business-standard.com/markets/news")
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
.header("Referer", "https://www.business-standard.com/")
.header("Accept-Language", "en-US,en;q=0.9")
.get();
// Document doc = Jsoup.connect("https://www.business-standard.com/markets/news")
// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
// .header("Referer", "https://www.business-standard.com/")
// .header("Accept-Language", "en-US,en;q=0.9")
// .get();
String url = "https://www.business-standard.com/markets/news";
// Document doc = Jsoup.connect(url)
// .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
// .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
// .header("Accept-Language", "en-US,en;q=0.5")
// .header("Connection", "keep-alive")
// .referrer("https://www.google.com")
// .ignoreHttpErrors(true)
// .timeout(10000)
// .get();
String doGetNews = HttpClientRequest.doGetNews(url);
// result = doc.html().substring(doc.html().indexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">"),doc.html().lastIndexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">")+500);
Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57");
StringBuilder sb = new StringBuilder();
for (Element divElement : divElements) {
sb.append(divElement.outerHtml()).append("\n");
}
result = sb.toString();
// Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57");
// StringBuilder sb = new StringBuilder();
// for (Element divElement : divElements) {
// sb.append(divElement.outerHtml()).append("\n");
// }
// result = sb.toString();
result = extractNewsFromHtml(doGetNews);
} catch (Exception e) {
return e.toString();
}
return result;
}
private String extractNewsFromHtml(String rawHtml) {
Document doc = Jsoup.parse(rawHtml);
// Optional: extract raw block around <div class="listingstyle_shortvideoimg__0TWuX shortvideoimg">
int start = rawHtml.indexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">");
int end = rawHtml.lastIndexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">") + 500;
String result = (start != -1 && end > start) ? rawHtml.substring(start, end) : "";
// Extract article list using CSS selector
Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57");
StringBuilder sb = new StringBuilder();
for (Element divElement : divElements) {
sb.append(divElement.outerHtml()).append("\n");
}
// If you only want the div content:
result = sb.toString();
return result;
}
public List<String> getNewsInfo(String url) {
// String result = "";
// List<String> list = new ArrayList<>();
// try {
// // 使用Jsoup连接到网页
//// Document doc = Jsoup.connect(url)
//// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
//// .header("Referer", "https://www.business-standard.com/")
//// .header("Accept-Language", "en-US,en;q=0.9")
//// .get();
// Document doc = Jsoup.connect(url)
// .referrer("https://www.business-standard.com/")
// .header("Accept-Language", "en-US,en;q=0.9")
// .userAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1")
// .timeout(5000) // timeout 5 seconds
// .get();
// result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1);
// list.add(result);
// list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf("<meta property=\"og:url") - 5));
// } catch (Exception e) {
// list.add(e.toString());
// return list;
// }
// return list;
String result = "";
List<String> list = new ArrayList<>();
try {
// 使用Jsoup连接到网页
Document doc = Jsoup.connect(url)
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
.header("Referer", "https://www.business-standard.com/")
.header("Accept-Language", "en-US,en;q=0.9")
.get();
result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1);
// Step 1: Get raw HTML via HttpClient
String rawHtml = HttpClientRequest.doGetNews(url);
// Step 2: Parse HTML with Jsoup
Document doc = Jsoup.parse(rawHtml);
// Step 3: Extract content manually (like original)
int startIdx = rawHtml.indexOf("articleBody") + 14;
int endIdx = rawHtml.indexOf(",\"author\":") - 1;
if (startIdx > 0 && endIdx > startIdx) {
result = rawHtml.substring(startIdx, endIdx);
} else {
result = "[articleBody not found]";
}
list.add(result);
list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf("<meta property=\"og:url") - 5));
// Step 4: Extract og:title
int titleStart = rawHtml.indexOf("og:title") + 19;
int titleEnd = rawHtml.indexOf("<meta property=\"og:url") - 5;
if (titleStart > 0 && titleEnd > titleStart) {
list.add(rawHtml.substring(titleStart, titleEnd));
} else {
list.add("[title not found]");
}
} catch (Exception e) {
list.add(e.toString());
return list;
list.add("[Error] " + e.getMessage());
}
return list;
}

View File

@@ -0,0 +1,18 @@
package cn.stock.market.dto;
import lombok.Data;
@Data
public class OptionalStockResponse {
private String message;
private Integer code;
private DataResponse data = new DataResponse();
@Data
public static class DataResponse {
private String symbol;
private String company;
private Double pricecurrent;
private Float pricepercentchange;
}
}

View File

@@ -0,0 +1,48 @@
package cn.stock.market.dto.query;
import lombok.Data;
import java.time.LocalDate;
@Data
public class StockChartDto {
private Long timestamp;
private double open;
private double high;
private double low;
private double close;
private double volume;
public StockChartDto(Long date, double open, double high, double low, double close, double volume) {
this.timestamp = date;
this.open = open;
this.high = high;
this.low = low;
this.close = close;
this.volume = volume;
}
public double getVolume() {
return volume;
}
public Long getTimestamp() {
return timestamp;
}
public double getOpen() {
return open;
}
public double getHigh() {
return high;
}
public double getLow() {
return low;
}
public double getClose() {
return close;
}
}

View File

@@ -60,6 +60,10 @@ public class MoneyStockPO {
* NSE India的id */
String nseIndiaId;
/**
* NSE India Chart的id */
String nseIndiaChartId;
/**
* 自有self_url */
String selfUrl;
@@ -79,4 +83,6 @@ public class MoneyStockPO {
/**
* 是否展示 0是 1否 */
Integer isShow;
Boolean useFromBseindia = false;
}

View File

@@ -0,0 +1,35 @@
package cn.stock.market.infrastructure.db.po;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import org.hibernate.annotations.DynamicInsert;
import org.hibernate.annotations.DynamicUpdate;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.Table;
@SuperBuilder
@Data
@NoArgsConstructor
@AllArgsConstructor
@Entity
@DynamicInsert
@DynamicUpdate
@Table(
name = "optional_stock"
)
public class OptionalStockPO {
@Id
@GeneratedValue(
strategy = javax.persistence.GenerationType.IDENTITY
)
Integer id;
String symbol;
String company;
}

View File

@@ -61,11 +61,11 @@ public class StockIpoPO {
/**
* 是否显示【1 显示2 不显示】 */
Integer isShow;
Integer isShow = 1;
/**
* 是否上市【1 未上市2 已上市】 */
Integer isList;
Integer isList = 1;
Date createDate;
@@ -79,6 +79,8 @@ public class StockIpoPO {
* 申请总额 */
String apply;
String exchangeType;
/**
* 市盈率 */
BigDecimal peRatio;

View File

@@ -0,0 +1,7 @@
package cn.stock.market.infrastructure.db.repo;
import cn.stock.market.infrastructure.db.po.OptionalStockPO;
import com.rp.spring.jpa.GenericJpaRepository;
public interface OptionalStockRepo extends GenericJpaRepository<OptionalStockPO, Integer> {
}

View File

@@ -1,14 +1,20 @@
package cn.stock.market.infrastructure.job;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.stock.market.domain.basic.entity.BtodayStock;
import cn.stock.market.domain.basic.entity.MoneyStock;
import cn.stock.market.domain.basic.entity.StockIpo;
import cn.stock.market.domain.basic.repository.BtodayStockRepository;
import cn.stock.market.domain.basic.repository.MoneyStockRepository;
import cn.stock.market.domain.basic.repository.StockIpoRepository;
import cn.stock.market.dto.StockIpoDTO;
import cn.stock.market.infrastructure.db.po.QMoneyStockPO;
import cn.stock.market.infrastructure.db.po.QStockIpoPO;
import cn.stock.market.infrastructure.db.po.StockIpoPO;
import cn.stock.market.infrastructure.db.repo.BtodayStockRepo;
import com.alibaba.druid.sql.visitor.functions.Now;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
@@ -31,14 +37,11 @@ import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -54,6 +57,10 @@ public class Scraper {
private StockIpoRepository stockIpoRepository;
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
@Autowired
private MoneyStockRepository moneyStockRepository;
private boolean times = false;
@Scheduled(cron = "0 0 1 */2 * ?")
@@ -104,14 +111,16 @@ public class Scraper {
@Scheduled(cron = "0 0 18 * * ?")
// @PostConstruct
@RequestMapping("/testScraperGetMoneyControllerNewIPO")
public void getMoneyControllerNewIPOSchedule() {
log.info("定时任务执行获取新股ipo的方法开始执行");
// 目标 URL
String url = "https://www.moneycontrol.com/ipo/open-upcoming-ipos";
List<String> urls = Arrays.asList("https://www.moneycontrol.com/ipo/upcoming-ipos/", "https://www.moneycontrol.com/ipo/open-ipos/");
// 创建 HttpClient 实例
HttpClient client = HttpClients.createDefault();
// 创建 HttpGet 请求
for (String url : urls) {
HttpGet request = new HttpGet(url);
try {
// 执行请求
@@ -136,11 +145,18 @@ public class Scraper {
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
// 解析 openData 和 upcomingData
JSONArray openData = ipoTableData.getJSONArray("openData");
// if (openData.isEmpty()){
// openData = ipoTableData.getJSONArray("openUpcomingData");
// }
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
List<StockIpo> listStockIpoList = new ArrayList<>();
for (int i = 0; i < openData.size(); i++) {
@@ -153,8 +169,7 @@ public class Scraper {
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
Integer totalNumber = 0;
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
Integer issue_size = entry.getInteger("issue_size");
BigDecimal issueSize = new BigDecimal(issue_size);
BigDecimal issueSize = new BigDecimal(entry.getLong("issue_size"));
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
totalNumber = totalNumberBigDecimal.intValue();
}
@@ -163,7 +178,28 @@ public class Scraper {
stockIpo.setCreateDate(new Date());
stockIpo.setUpdateDate(new Date());
stockIpo.setSourceType("3");
listStockIpoList.add(stockIpo);
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
continue;
}
String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
for (String exchange : exchanges) {
StockIpo ipo = new StockIpo();
ipo.setStockCode(stockIpo.getStockCode());
ipo.setStockName(stockIpo.getStockName());
ipo.setStockPrice(stockIpo.getStockPrice());
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
ipo.setListingDate(stockIpo.getListingDate());
ipo.setTotalNumber(stockIpo.getTotalNumber());
ipo.setApply(stockIpo.getApply());
ipo.setCreateDate(stockIpo.getCreateDate());
ipo.setUpdateDate(stockIpo.getUpdateDate());
ipo.setExchangeType(exchange);
ipo.setSourceType("3");
listStockIpoList.add(ipo);
}
}
for (int i = 0; i < upcomingData.size(); i++) {
@@ -186,11 +222,37 @@ public class Scraper {
stockIpo.setCreateDate(new Date());
stockIpo.setUpdateDate(new Date());
stockIpo.setSourceType("3");
listStockIpoList.add(stockIpo);
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
continue;
}
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
for (String exchange : exchanges) {
StockIpo ipo = new StockIpo();
ipo.setStockCode(stockIpo.getStockCode());
ipo.setStockName(stockIpo.getStockName());
ipo.setStockPrice(stockIpo.getStockPrice());
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
ipo.setListingDate(stockIpo.getListingDate());
ipo.setTotalNumber(stockIpo.getTotalNumber());
ipo.setApply(stockIpo.getApply());
ipo.setCreateDate(stockIpo.getCreateDate());
ipo.setUpdateDate(stockIpo.getUpdateDate());
ipo.setExchangeType(exchange);
ipo.setSourceType("3");
listStockIpoList.add(ipo);
}
}
// stockIpoRepository.saveAll(listStockIpoList);
if (!listStockIpoList.isEmpty())
{
listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList());
}
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
if(CollectionUtil.isNotEmpty(scIdList)){
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
@@ -198,13 +260,27 @@ public class Scraper {
.map(StockIpo::getStockCode)
.collect(Collectors.toList());
listStockIpoList = listStockIpoList.stream()
.filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
.collect(Collectors.toList());
for (StockIpo stockIpo : listStockIpoList) {
StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().toLowerCase().trim().equals(stockIpo.getExchangeType().toLowerCase().trim()))).findFirst().orElse(null);
if (existIpo != null) {
if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
existIpo.setExchangeType(stockIpo.getExchangeType());
}
stockIpo.setId(existIpo.getId());
if (existIpo.getStockPrice() != null){
stockIpo.setStockPrice(existIpo.getStockPrice());
stockIpo.setApply(existIpo.getApply());
}
}
}
// listStockIpoList = listStockIpoList.stream()
// .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
// .collect(Collectors.toList());
//保存全部的新股
if(CollectionUtil.isNotEmpty(listStockIpoList)){
stockIpoRepository.saveAll(listStockIpoList);
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
}else {
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
@@ -222,7 +298,39 @@ public class Scraper {
} catch (IOException e) {
log.error("获取新股接口发生异常",e);
}
}
}
@Scheduled(cron = "0 0 19 * * ?")
@PostConstruct
@RequestMapping("/addMoneyStock")
public void addMoneyStock() {
log.info("定时任务执行获取新股ipo的方法开始执行");
List<StockIpo> stockIpoList;
if(!times){
stockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.listingDate.isNotNull()
.and(QStockIpoPO.stockIpoPO.listingDate.loe(DateTime.now()))
.and(QStockIpoPO.stockIpoPO.exchangeType.isNotNull())
.and(QStockIpoPO.stockIpoPO.exchangeType.isNotEmpty())
); // first time run
times = true;
}
else {
stockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.listingDate.isNotNull()
.and(QStockIpoPO.stockIpoPO.listingDate.loe(DateTime.now())
.and(QStockIpoPO.stockIpoPO.listingDate.goe(DateUtil.offsetDay(DateTime.now(), -3))))
.and(QStockIpoPO.stockIpoPO.exchangeType.isNotNull())
.and(QStockIpoPO.stockIpoPO.exchangeType.isNotEmpty())
); // from second run
}
List<MoneyStock> moneyStockList = convertToMoneyStock(stockIpoList);
moneyStockRepository.saveAll(moneyStockList);
}
@@ -236,12 +344,6 @@ public class Scraper {
}
}
private void processSubMap(Map<String, String> sefUrlList) {
for (Map.Entry<String, String> entry : sefUrlList.entrySet()) {
String companyName = entry.getKey();
@@ -378,4 +480,135 @@ public class Scraper {
log.info("Getting web detail URL: " + url + "...");
return Jsoup.connect(url).ignoreContentType(true).execute().body();
}
public List<String> getIpoExchange(String stockName, String stockCode) {
List<String> exchanges = new ArrayList<>();
String url = "https://www.moneycontrol.com/ipo/{stockName}-{stockCode}-ipodetail";
url = url.replace("{stockName}", stockName.toLowerCase().replace(" ", "-")).replace("{stockCode}", stockCode.toLowerCase());
org.apache.http.client.HttpClient client = HttpClients.createDefault();
HttpGet request = new HttpGet(url);
try {
org.apache.http.HttpResponse response = client.execute(request);
if (response.getStatusLine().getStatusCode() == 200) {
String responseBody = EntityUtils.toString(response.getEntity());
Document doc = Jsoup.parse(responseBody);
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
if (scriptTag != null) {
String jsonDataStr = scriptTag.html();
com.alibaba.fastjson.JSONObject jsonObject = com.alibaba.fastjson.JSONObject.parseObject(jsonDataStr);
log.info("获取到新股的json信息:" + jsonObject.toJSONString());
com.alibaba.fastjson.JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
com.alibaba.fastjson.JSONObject overview = pageProps.getJSONObject("ipoDetails").getJSONObject("ipoDetails").getJSONObject("overview");
String exchangeStr = overview.getString("listing_on");
if (exchangeStr != null && !exchangeStr.isEmpty() && !exchangeStr.equals("null")) {
exchanges.addAll(new ArrayList<>(Arrays.asList(exchangeStr.split(","))));
}
}
}
}
catch (Exception e) {
log.info(e.getMessage());
}
if (exchanges.isEmpty()) {
exchanges.add("");
}
return exchanges;
}
public List<MoneyStock> convertToMoneyStock(List<StockIpo> stockIpoList) {
// Lấy danh sách tất cả stockCode và exchangeType từ stockIpoList
Set<String> stockCodes = stockIpoList.stream().map(StockIpo::getStockCode).collect(Collectors.toSet());
// Set<String> exchangeTypes = stockIpoList.stream().map(StockIpo::getExchangeType).collect(Collectors.toSet());
Set<String> exchangeTypes = stockIpoList.stream().map(stockIpo -> stockIpo.getExchangeType().toUpperCase()).collect(Collectors.toSet());
// Lấy tất cả các MoneyStock tồn tại trong database với stockCode và exchangeType trong stockIpoList
List<MoneyStock> existingMoneyStocks = moneyStockRepository.findAll(
QMoneyStockPO.moneyStockPO.moneyScId.in(stockCodes)
.and(QMoneyStockPO.moneyStockPO.stockType.upper().in(exchangeTypes))
);
// Lưu các stockCode và exchangeType đã tồn tại
Set<String> existingKeys = existingMoneyStocks.stream()
.map(ms -> ms.getMoneyScId() + ms.getStockType().toUpperCase())
.collect(Collectors.toSet());
// Chỉ giữ những StockIpo chưa tồn tại
return stockIpoList.stream()
.filter(stockIpo -> !existingKeys.contains(stockIpo.getStockCode() + stockIpo.getExchangeType().toUpperCase()))
.map(this::convert)
.collect(Collectors.toList());
}
// public List<MoneyStock> convertToMoneyStock(List<StockIpo> stockIpoList) {
// return stockIpoList.stream()
// .filter(stockIpo -> !moneyStockRepository.exists(
// QMoneyStockPO.moneyStockPO.stockType.eq(stockIpo.getExchangeType())
// .and(QMoneyStockPO.moneyStockPO.moneyScId.eq(stockIpo.getStockCode()))
// )) // Chỉ giữ những StockIpo chưa tồn tại
// .map(this::convert)
// .collect(Collectors.toList());
// }
private MoneyStock convert(StockIpo stockIpo) {
MoneyStock moneyStock = new MoneyStock();
moneyStock.setStockName(stockIpo.getStockName());
moneyStock.setMoneyScId(stockIpo.getStockCode());
moneyStock.setSelfDispId(stockIpo.getStockCode());
moneyStock.setStockType(stockIpo.getExchangeType());
moneyStock.setSaveTime(DateTime.now());
moneyStock.setNseIndiaId(getNSEId(stockIpo));
return moneyStock;
}
private String getNSEId(StockIpo stockIpo){
String url = "https://priceapi.moneycontrol.com/pricefeed/" + stockIpo.getExchangeType().toLowerCase() + "/equitycash/" + stockIpo.getStockCode();
// 创建 HttpClient 实例
HttpClient client = HttpClients.createDefault();
// 创建 HttpGet 请求
HttpGet request = new HttpGet(url);
try {
// 执行请求
HttpResponse response = client.execute(request);
// 检查请求是否成功
if (response.getStatusLine().getStatusCode() == 200) {
// 获取响应体
String responseBody = EntityUtils.toString(response.getEntity());
if (responseBody != null) {
// 获取 JSON 数据
// 将 JSON 字符串解析为 Java JSONObject
JSONObject jsonObject = JSONObject.parseObject(responseBody);
log.info("获取到新股的json信息:" + jsonObject.toJSONString());
JSONObject jsonData = jsonObject.getJSONObject("data");
if(stockIpo.getExchangeType().equals("NSE")){
return jsonData.getString("NSEID");
}
else if(stockIpo.getExchangeType().equals("BSE")){
return jsonData.getString("BSEID");
}
}
}
}
catch (Exception e){
log.info(e.getMessage());
}
return "";
}
}

View File

@@ -265,6 +265,56 @@ public class HttpClientRequest {
return result;
}
public static String doGetNews(String url) {
CloseableHttpClient httpClient = null;
CloseableHttpResponse response = null;
String result = "";
try {
httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
// Spoof real browser headers
httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
httpGet.setHeader("Accept-Language", "en-US,en;q=0.9,vi;q=0.8,ug;q=0.7,fr;q=0.6");
httpGet.setHeader("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1");
httpGet.setHeader("Priority", "u=0, i");
httpGet.setHeader("Sec-Fetch-Dest", "document");
httpGet.setHeader("Sec-Fetch-Mode", "navigate");
httpGet.setHeader("Sec-Fetch-Site", "none");
httpGet.setHeader("Sec-Fetch-User", "?1");
httpGet.setHeader("Upgrade-Insecure-Requests", "1");
// Set cookies exactly like in curl
httpGet.setHeader("Cookie", "userUid=1747102033185-d570fba9-62fd-40be-93ca-ed08b4de57d4; _sid=MTc0NzEwMjAzMzE4Ni4ycTU%3D; _scor_uid=135c13065ff84620b5318b489af93e87; _gcl_au=1.1.249135292.1747102036; _ga=GA1.1.1031614211.1747102037; WZRK_G=28895afb56ff48dda59fe8de0af746bf; FCNEC=%5B%5B%22AKsRol8sHYeSYz_FYPkInYXN3P4ZDPfVKbsRfILfDuOMLhDtkTuoCJP5MlvT9gIbOe7IlDfY8ZeHszhwdVtAoKF1gWv0pLAq5EqpLpse8CEm_ZNv-bUSs6zEyqpOkeKWFI_Ei6VfNAvnZAO8PcXdF8_ncsaO902X7g%3D%3D%22%5D%5D; _ga_KRGL1M61LX=GS2.1.s1747105135$o2$g0$t1747105135$j60$l0$h0");
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(10000)
.setSocketTimeout(15000)
.setConnectionRequestTimeout(10000)
.build();
httpGet.setConfig(requestConfig);
response = httpClient.execute(httpGet);
HttpEntity entity = response.getEntity();
if (entity != null) {
result = EntityUtils.toString(entity);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (response != null) response.close();
if (httpClient != null) httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return result;
}
public static void main(String[] args) {
String url = "https://marketapi.intoday.in/widget/topgainer/view?exchange=nse";
String str = doGet(url);

View File

@@ -2,18 +2,21 @@ package cn.stock.market.utils;
import cn.stock.market.dto.StockHistoryRequest;
import cn.stock.market.dto.StockHistoryResponse;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.*;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class NseIndiaRequest {
private static final String NSE_INDIA_URL = "https://www.nseindia.com";
private static final String NSE_INDIA_CHART_URL = "https://charting.nseindia.com";
private static final OkHttpClient client;
private static final ObjectMapper objectMapper = new ObjectMapper();
static {
client = new OkHttpClient.Builder()
@@ -47,8 +50,8 @@ public class NseIndiaRequest {
return request;
}
private static void initCookie() {
Request request = createRequest(NSE_INDIA_URL);
private static void initCookie(String url) {
Request request = createRequest(url);
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Failed to fetch initial cookies");
@@ -58,8 +61,35 @@ public class NseIndiaRequest {
}
}
private static Integer getCode(String symbol) {
Request request = createRequest(NSE_INDIA_CHART_URL + "//Charts/GetEQMasters").newBuilder()
.addHeader("referer", NSE_INDIA_CHART_URL)
.addHeader("origin", NSE_INDIA_CHART_URL)
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Failed to get EQ code");
}
String result = response.body().string();
String regex = "(\\d+)\\|" + symbol + "\\|.*";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(result);
if (matcher.find()) {
return Integer.valueOf(matcher.group(1));
}
throw new IOException("No data found");
} catch (IOException e) {
throw new RuntimeException("Failed to get EQ code", e);
}
}
public static JSONObject stockByJYSFromHttp(String stockType, String symbol, String nseIndiaId) {
initCookie();
initCookie(NSE_INDIA_URL);
String url = NSE_INDIA_URL + "/api/quote-equity?symbol=" + nseIndiaId;
Request request = createRequest(url).newBuilder()
@@ -96,18 +126,44 @@ public class NseIndiaRequest {
}
}
public static StockHistoryResponse stockKLineFromHttp(StockHistoryRequest stockHistoryRequest) {
initCookie();
public static StockHistoryResponse stockKLineFromHttp(StockHistoryRequest stockHistoryRequest, String resolution) {
initCookie(NSE_INDIA_CHART_URL);
SimpleDateFormat sdf = new SimpleDateFormat("dd-MM-yyyy");
String fromDate = sdf.format(new Date(stockHistoryRequest.getFrom() * 1000));
String toDate = sdf.format(new Date(stockHistoryRequest.getTo() * 1000));
Integer code = getCode(stockHistoryRequest.getSymbol());
String url = String.format("%s/api/historical/cm/equity?symbol=%s&from=%s&to=%s", NSE_INDIA_URL, stockHistoryRequest.getSymbol(), fromDate, toDate);
int interval = 1;
if (StringUtils.equals("H", resolution)) {
resolution = "I";
interval = 60;
}
Request request = createRequest(url).newBuilder()
.addHeader("referer", NSE_INDIA_URL)
.addHeader("origin", NSE_INDIA_URL)
Map<String, Object> body = new HashMap<>();
body.put("chartPeriod", resolution);
body.put("chartStart", 0);
body.put("exch", "N");
body.put("fromDate", 0);
body.put("instrType", "C");
body.put("scripCode", code);
body.put("timeInterval", interval);
body.put("toDate", stockHistoryRequest.getTo() + 18000);
body.put("ulToken", code);
String payload;
try {
payload = objectMapper.writeValueAsString(body);
} catch (Exception e) {
throw new RuntimeException("Failed to serialize body", e);
}
RequestBody requestBody = RequestBody.create(
MediaType.get("application/json; charset=utf-8"),
payload
);
Request request = createRequest(NSE_INDIA_CHART_URL + "//Charts/symbolhistoricaldata/").newBuilder()
.addHeader("referer", NSE_INDIA_CHART_URL)
.addHeader("origin", NSE_INDIA_CHART_URL)
.post(requestBody)
.build();
try (Response response = client.newCall(request).execute()) {
@@ -115,52 +171,7 @@ public class NseIndiaRequest {
throw new IOException("Request failed with code: " + response.code());
}
JSONObject jsonData = JSONObject.parseObject(response.body().string());
JSONArray data =jsonData.getJSONArray("data");
StockHistoryResponse result = new StockHistoryResponse();
List<Long> tList = new ArrayList<>();
List<Double> oList = new ArrayList<>();
List<Double> hList = new ArrayList<>();
List<Double> lList = new ArrayList<>();
List<Double> cList = new ArrayList<>();
List<Long> vList = new ArrayList<>();
for (int i = 0; i < data.size(); i++) {
Long t, v;
Double o, h, l, c;
try {
JSONObject jsonObject = data.getJSONObject(i);
String timestampStr = jsonObject.getString("TIMESTAMP");
Instant instant = Instant.parse(timestampStr);
t = instant.toEpochMilli() / 1000;
o = jsonObject.getDouble("CH_OPENING_PRICE");
c = jsonObject.getDouble("CH_CLOSING_PRICE");
h = jsonObject.getDouble("CH_TRADE_HIGH_PRICE");
l = jsonObject.getDouble("CH_TRADE_LOW_PRICE");
v = jsonObject.getLong("CH_TOT_TRADED_VAL");
} catch (Exception e) {
continue;
}
tList.add(t);
oList.add(o);
hList.add(h);
lList.add(l);
cList.add(c);
vList.add(v);
}
result.setT(tList);
result.setO(oList);
result.setH(hList);
result.setL(lList);
result.setC(cList);
result.setV(vList);
StockHistoryResponse result = objectMapper.readValue(response.body().string(), StockHistoryResponse.class);
return result;
} catch (IOException e) {

View File

@@ -3,17 +3,25 @@ package cn.stock.market.web;
import cn.hutool.core.date.DateUtil;
import cn.stock.market.MoneyStockSuggestDTO;
import cn.stock.market.domain.basic.entity.MoneyStock;
import cn.stock.market.domain.basic.entity.OptionalStock;
import cn.stock.market.domain.basic.repository.MoneyStockRepository;
import cn.stock.market.dto.StockHistoryRequest;
import cn.stock.market.dto.StockHistoryResponse;
import cn.stock.market.domain.basic.repository.OptionalStockRepository;
import cn.stock.market.dto.*;
import cn.stock.market.dto.query.StockChartDto;
import cn.stock.market.infrastructure.db.po.QMoneyStockPO;
import cn.stock.market.utils.HttpRequest;
import cn.stock.market.utils.NseIndiaRequest;
import cn.stock.market.utils.ServerResponse;
import cn.stock.market.web.annotations.EncryptFilter;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
@@ -23,21 +31,36 @@ import io.swagger.annotations.ApiResponses;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.http.*;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import org.springframework.web.util.UriUtils;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.YearMonth;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -58,7 +81,14 @@ public class MoneyApiController {
@Autowired
private MoneyStockRepository moneyStockRepository;
@Autowired
private OptionalStockRepository optionalStockRepository;
@Autowired
private ObjectMapper objectMapper;
private static final String EXTERNAL_API_URL = "https://priceapi.moneycontrol.com/techCharts/indianMarket/stock/history";
private static final String OPTIONAL_STOCK_MONEYCONTROL_URL = "https://priceapi.moneycontrol.com/pricefeed/notapplicable/inidicesindia/";
@ApiOperation(value = "股票详情信息", httpMethod = "GET")
@ApiImplicitParams({
@@ -166,7 +196,6 @@ public class MoneyApiController {
@GetMapping({"/market/api/market/money/getStockDetail", "/api/market/money/getStockDetail"})
@ResponseBody
@EncryptFilter(decryptRequest = false)
public ServerResponse getStockDetail(@RequestParam String stockType, @RequestParam String symbol) {
MoneyStock moneyStock = moneyStockRepository.findOne(QMoneyStockPO.moneyStockPO.stockType.eq(stockType)
.and(QMoneyStockPO.moneyStockPO.moneyScId.eq(symbol))
@@ -190,8 +219,56 @@ public class MoneyApiController {
return ServerResponse.createBySuccess(json1);
}
String url = String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", stockType, symbol);
String url = String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", stockType.toLowerCase(), symbol);
int maxRetries = 3;
if (moneyStock.getUseFromBseindia()) {
String bseUrl = "https://api.bseindia.com/BseIndiaAPI/api/getScripHeaderData/w?Debtflag=&scripcode=" + moneyStock.getNseIndiaId() + "&seriesid=";
HttpHeaders headers = new HttpHeaders();
headers.add("accept", "application/json, text/plain, */*");
headers.add("accept-language", "en-US,en;q=0.9,vi;q=0.8");
headers.add("origin", "https://www.bseindia.com");
headers.add("referer", "https://www.bseindia.com/");
headers.add("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"");
headers.add("sec-ch-ua-mobile", "?0");
headers.add("sec-ch-ua-platform", "\"Windows\"");
headers.add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36");
HttpEntity<String> entity = new HttpEntity<>(headers);
try {
ResponseEntity<String> responseEntity = restTemplate.exchange(bseUrl, HttpMethod.GET, entity, String.class);
if (responseEntity.getStatusCode().value() == 200 && responseEntity.getBody() != null) {
JSONObject bseData = JSONObject.parseObject(responseEntity.getBody());
JSONObject json1 = new JSONObject();
json1.put("company", bseData.getJSONObject("Cmpname").getString("FullN"));
json1.put("pricepercentchange", bseData.getJSONObject("CurrRate").getString("PcChg"));
json1.put("stockType", stockType);
json1.put("pricechange", bseData.getJSONObject("CurrRate").getString("Chg"));
json1.put("pricecurrent", bseData.getJSONObject("CurrRate").getString("LTP"));
json1.put("priceprevclose", bseData.getJSONObject("Header").getString("PrevClose"));
json1.put("PREVDATE", "");
json1.put("dataSourceType", "3");
json1.put("symbol", symbol);
json1.put("BSEID", moneyStock.getNseIndiaId());
json1.put("NSEID", moneyStock.getNseIndiaId());
json1.put("LTH", bseData.getJSONObject("Header").getString("High"));
json1.put("LTL", bseData.getJSONObject("Header").getString("Low"));
json1.put("OPN", bseData.getJSONObject("Header").getString("Open"));
if (moneyStock != null) {
json1.put("id", moneyStock.getId());
}
json1.put("VOL", this.getVolume(moneyStock.getNseIndiaId()));
return ServerResponse.createBySuccess(json1);
}
} catch (Exception e) {
System.err.println("Error fetching data from BSE India: " + e.getMessage());
}
}
for (int retry = 1; retry <= maxRetries; retry++) {
try {
ResponseEntity<String> responseEntity = restTemplate.exchange(url, HttpMethod.GET, null, String.class);
@@ -252,6 +329,57 @@ public class MoneyApiController {
return null;
}
private String getVolume(String scripcode) {
String url = "https://api.bseindia.com/BseIndiaAPI/api/StockTrading/w?flag=&quotetype=EQ&scripcode=" + scripcode;
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.add("accept", "application/json, text/plain, */*");
headers.add("accept-language", "en-US,en;q=0.9,vi;q=0.8");
headers.add("origin", "https://www.bseindia.com");
headers.add("referer", "https://www.bseindia.com/");
headers.add("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"");
headers.add("sec-ch-ua-mobile", "?0");
headers.add("sec-ch-ua-platform", "\"Windows\"");
headers.add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36");
HttpEntity<String> entity = new HttpEntity<>(headers);
ResponseEntity<String> response = restTemplate.exchange(url, HttpMethod.GET, entity, String.class);
BigDecimal volume = BigDecimal.ZERO;
if (response.getStatusCode().value() == 200 && response.getBody() != null) {
JSONObject volumeData = JSONObject.parseObject(response.getBody());
if (volumeData.getString("TTQ") != null) {
volume = new BigDecimal(volumeData.getString("TTQ"));
}
}
return volume.toString();
}
private String getChartData(String scripcode, String resolution) {
String url = "https://charting.bseindia.com/charting/RestDataProvider.svc/getDat?exch=B&type=b&mode=bseL&fromdate=01-01-1991-01%3A01%3A00-AM&scode=" + scripcode;
if (resolution.equals("H")) {
url = "https://charting.bseindia.com/charting/RestDataProvider.svc/getDatI?exch=B&type=b&mode=bseL&fromdate=01-01-1991-01%3A01%3A00-AM&scode=" + scripcode;
}
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.add("accept", "application/json, text/plain, */*");
headers.add("accept-language", "en-US,en;q=0.9,vi;q=0.8");
headers.add("origin", "https://www.bseindia.com");
headers.add("referer", "https://www.bseindia.com/");
headers.add("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"");
headers.add("sec-ch-ua-mobile", "?0");
headers.add("sec-ch-ua-platform", "\"Windows\"");
headers.add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36");
HttpEntity<String> entity = new HttpEntity<>(headers);
ResponseEntity<String> response = restTemplate.exchange(url, HttpMethod.POST, entity, String.class);
return response.getBody();
}
private static List<MoneyStockSuggestDTO> nseActives() {
List<MoneyStockSuggestDTO> list = new ArrayList<>();
@@ -668,6 +796,45 @@ public class MoneyApiController {
return moneyStockSuggestDTOS;
}
private static void addToListDouble(List<Double> list, String value) {
String[] values = value.split(",");
for (String v : values) {
list.add(Double.parseDouble(v));
}
}
private static void addToList(List<Long> list, String value) {
String[] values = value.split(",");
for (String v : values) {
list.add(Math.round(Double.parseDouble(v)));
}
}
private static void addDatesToList(List<Long> list, String value) {
String[] dates = value.split(",");
SimpleDateFormat sdf = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a", Locale.US);
for (String dateStr : dates) {
try {
Date date = sdf.parse(dateStr);
list.add(date.getTime() / 1000);
} catch (ParseException e) {
e.printStackTrace();
}
}
}
private Long convertToTimestamp(String dateStr) {
try {
SimpleDateFormat sdf = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss a", Locale.US);
Date date = sdf.parse(dateStr);
return date.getTime() / 1000;
} catch (ParseException e) {
e.printStackTrace();
return 0L;
}
}
@GetMapping({"/market/api/market/money/history/kLine", "/api/market/money/history/kLine"})
@ApiOperation(value = "获取kline的money数据源", notes = "获取kline的money数据源", response = StockHistoryResponse.class)
@@ -684,6 +851,166 @@ public class MoneyApiController {
public ResponseEntity<StockHistoryResponse> getStockHistory(@RequestParam String symbol,
@RequestParam String resolution
) {
MoneyStock moneyStock1 = moneyStockRepository.findOne(
QMoneyStockPO.moneyStockPO.nseIndiaId.eq(symbol)
.and(QMoneyStockPO.moneyStockPO.isLock.eq(0))
.and(QMoneyStockPO.moneyStockPO.isShow.eq(0)))
.orElse(null);
if (moneyStock1 != null && moneyStock1.getUseFromBseindia()) {
String json = this.getChartData(moneyStock1.getNseIndiaId(), resolution);
Gson gson = new Gson();
JsonObject outerJson = gson.fromJson(json, JsonObject.class);
String innerJsonString;
if (resolution.equals("H")) {
innerJsonString = outerJson.get("getDatIResult").getAsString();
} else {
innerJsonString = outerJson.get("getDatResult").getAsString();
}
JsonObject innerJson = gson.fromJson(innerJsonString, JsonObject.class);
JsonArray dataInputValues = innerJson.getAsJsonArray("DataInputValues");
List<Double> openList = new ArrayList<>();
List<Double> highList = new ArrayList<>();
List<Double> lowList = new ArrayList<>();
List<Double> closeList = new ArrayList<>();
List<Long> volumeList = new ArrayList<>();
List<Long> dateList = new ArrayList<>();
Map<String, JsonObject> latestDataMap = new LinkedHashMap<>();
if (dataInputValues.size() > 0) {
JsonObject dataObject = dataInputValues.get(0).getAsJsonObject();
JsonArray openArray = dataObject.getAsJsonArray("OpenData");
JsonArray highArray = dataObject.getAsJsonArray("HighData");
JsonArray lowArray = dataObject.getAsJsonArray("LowData");
JsonArray closeArray = dataObject.getAsJsonArray("CloseData");
JsonArray volumeArray = dataObject.getAsJsonArray("VolumeData");
JsonArray dateArray = dataObject.getAsJsonArray("DateData");
if (resolution.equals("H")) {
openArray.forEach(o -> addToListDouble(openList, o.getAsJsonObject().get("Open").getAsString()));
highArray.forEach(h -> addToListDouble(highList, h.getAsJsonObject().get("High").getAsString()));
lowArray.forEach(l -> addToListDouble(lowList, l.getAsJsonObject().get("Low").getAsString()));
closeArray.forEach(c -> addToListDouble(closeList, c.getAsJsonObject().get("Close").getAsString()));
volumeArray.forEach(v -> addToList(volumeList, v.getAsJsonObject().get("Volume").getAsString()));
dateArray.forEach(d -> addDatesToList(dateList, d.getAsJsonObject().get("Date").getAsString()));
} else {
for (int i = 0; i < openArray.size(); i++) {
String open = openArray.get(i).getAsJsonObject().get("Open").getAsString();
String high = highArray.get(i).getAsJsonObject().get("High").getAsString();
String low = lowArray.get(i).getAsJsonObject().get("Low").getAsString();
String close = closeArray.get(i).getAsJsonObject().get("Close").getAsString();
String volume = volumeArray.get(i).getAsJsonObject().get("Volume").getAsString();
String dateStr = dateArray.get(i).getAsJsonObject().get("Date").getAsString();
Long timestamp = convertToTimestamp(dateStr);
String dateTimeKey = dateStr;
JsonObject dataJson = new JsonObject();
dataJson.addProperty("Open", open);
dataJson.addProperty("High", high);
dataJson.addProperty("Low", low);
dataJson.addProperty("Close", close);
dataJson.addProperty("Volume", volume);
dataJson.addProperty("Date", dateStr);
latestDataMap.put(dateTimeKey, dataJson);
}
latestDataMap.forEach((key, value) -> {
// Lấy giá trị các trường Open, High, Low, Close, Volume, Date từ dữ liệu
String openData = value.get("Open").getAsString();
String highData = value.get("High").getAsString();
String lowData = value.get("Low").getAsString();
String closeData = value.get("Close").getAsString();
String volumeData = value.get("Volume").getAsString();
String dateData = value.get("Date").getAsString();
// Tách các giá trị trong chuỗi bằng dấu phẩy
String[] openValues = openData.split(",");
String[] highValues = highData.split(",");
String[] lowValues = lowData.split(",");
String[] closeValues = closeData.split(",");
String[] volumeValues = volumeData.split(",");
String[] dateValues = dateData.split(",");
for (int i = 0; i < openValues.length; i++) {
double open = Double.parseDouble(openValues[i]);
double high = Double.parseDouble(highValues[i]);
double low = Double.parseDouble(lowValues[i]);
double close = Double.parseDouble(closeValues[i]);
long volume = Math.round(Double.parseDouble(volumeValues[i]));
long timestamp = convertToTimestamp(dateValues[i]);
int index = dateList.indexOf(timestamp);
if (index == -1) {
dateList.add(timestamp);
openList.add(open);
highList.add(high);
lowList.add(low);
closeList.add(close);
volumeList.add(volume);
} else {
openList.set(index, open);
highList.set(index, high);
lowList.set(index, low);
closeList.set(index, close);
volumeList.set(index, volume);
}
}
});
}
StockHistoryResponse response = new StockHistoryResponse();
response.setS("ok");
response.setT(dateList);
response.setO(openList);
response.setH(highList);
response.setL(lowList);
response.setC(closeList);
response.setV(volumeList);
// if (resolution.equals("H")) {
// Set<String> desiredTimes = new HashSet<>(Arrays.asList("09:30", "10:30", "11:30", "12:30", "13:30", "14:30"));
//
// // Filter data based on timestamps
// SimpleDateFormat sdf = new SimpleDateFormat("HH:mm");
// List<Integer> indicesToKeep = new ArrayList<>();
// for (int i = 0; i < dateList.size(); i++) {
// String time = sdf.format(new Date(dateList.get(i) * 1000));
// if (desiredTimes.contains(time)) {
// indicesToKeep.add(i);
// }
// }
//
// // Filter corresponding data
// List<Long> filteredT = indicesToKeep.stream().map(dateList::get).collect(Collectors.toList());
// List<Double> filteredO = indicesToKeep.stream().map(openList::get).collect(Collectors.toList());
// List<Double> filteredH = indicesToKeep.stream().map(highList::get).collect(Collectors.toList());
// List<Double> filteredL = indicesToKeep.stream().map(lowList::get).collect(Collectors.toList());
// List<Double> filteredC = indicesToKeep.stream().map(closeList::get).collect(Collectors.toList());
// List<Long> filteredV = indicesToKeep.stream().map(volumeList::get).collect(Collectors.toList());
// response.setT(filteredT);
// response.setO(filteredO);
// response.setH(filteredH);
// response.setL(filteredL);
// response.setC(filteredC);
// response.setV(filteredV);
// }
return ResponseEntity.ok(response);
}
}
// 向外部API发起请求并获取响应
StockHistoryRequest request = new StockHistoryRequest();
request.setSymbol(symbol);
@@ -692,24 +1019,24 @@ public class MoneyApiController {
int countback = 5;
if(StringUtils.equals("H",resolution)){
to = (long) (System.currentTimeMillis() / 1000);
from = to - (60 * 60);
countback = 60;
request.setResolution("1");
from = to - (10 * 60 * 60 );
countback = 328;
request.setResolution("60");
}else if(StringUtils.equals("D",resolution)){
to = (long) (System.currentTimeMillis() / 1000);
from = to - (24 * 60 * 60);
countback = 390;
request.setResolution("1");
from = to - (2 * 30 * 24 * 60 * 60 );
countback = 730;
request.setResolution("1D");
} else if (StringUtils.equals("W", resolution)) {
to = (long) (System.currentTimeMillis() / 1000);
from = to - (7 * 24 * 60 * 60);
countback = 471;
request.setResolution("5");
countback = 730;
request.setResolution("1W");
} else if (StringUtils.equals("M", resolution)) {
to = (long) (System.currentTimeMillis() / 1000);
from = to - (35 * 24 * 60 * 60);
countback = 328;
request.setResolution("30");
from = to - (15 * 30 * 24 * 60 * 60);
countback = 730;
request.setResolution("1D");
}
request.setFrom(from);
@@ -724,7 +1051,81 @@ public class MoneyApiController {
while (response == null && retryCount < maxRetries) {
try {
if (StringUtils.equals("M", resolution)) {
CloseableHttpClient client = HttpClients.createDefault();
HttpGet req = new HttpGet(apiUrl);
HttpResponse resp = client.execute(req);
String jsonResponse = EntityUtils.toString(resp.getEntity(), "UTF-8");
ObjectMapper mapper = new ObjectMapper();
JsonNode rootNode = mapper.readTree(jsonResponse);
JsonNode timeNode = rootNode.get("t");
JsonNode lowNode = rootNode.get("l");
JsonNode highNode = rootNode.get("h");
JsonNode openNode = rootNode.get("o");
JsonNode closeNode = rootNode.get("c");
JsonNode volumeNote = rootNode.get("v");
List<StockChartDto> stocks = new ArrayList<>();
for (int i = 0; i < timeNode.size(); i++) {
long timestamp = timeNode.get(i).asLong();
double closePrice = closeNode.get(i).asDouble();
double openPrice = openNode.get(i).asDouble();
double volume = volumeNote.get(i).asDouble();
double low = lowNode.get(i).asDouble();
double high = highNode.get(i).asDouble();
stocks.add(new StockChartDto(timestamp, openPrice, high, low, closePrice, volume));
}
Map<String, List<StockChartDto>> groupedByMonth = stocks.stream()
.collect(Collectors.groupingBy(
sp -> Instant.ofEpochSecond(sp.getTimestamp())
.atZone(ZoneId.systemDefault())
.toLocalDate()
.format(DateTimeFormatter.ofPattern("yyyy-MM")),
LinkedHashMap::new,
Collectors.toList()
));
List<Long> timestamps = new ArrayList<>();
List<Double> opens = new ArrayList<>();
List<Double> closes = new ArrayList<>();
List<Double> highs = new ArrayList<>();
List<Double> lows = new ArrayList<>();
List<Long> volumes = new ArrayList<>();
response = new StockHistoryResponse();
groupedByMonth.forEach((month, prices) -> {
double open = prices.get(0).getOpen();
double close = prices.get(prices.size() - 1).getClose();
double high = prices.stream().mapToDouble(StockChartDto::getHigh).max().orElse(0);
double low = prices.stream().mapToDouble(StockChartDto::getLow).min().orElse(0);
double volume = prices.stream().mapToDouble(StockChartDto::getVolume).sum();
long timestamp = YearMonth.parse(month, DateTimeFormatter.ofPattern("yyyy-MM")).atDay(1).atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli() / 1000;
timestamps.add(timestamp);
opens.add(open);
closes.add(close);
highs.add(high);
lows.add(low);
volumes.add((long) volume);
System.out.println("Month: " + month);
System.out.println("Open: " + open + ", Close: " + close + ", High: " + high + ", Low: " + low);
});
response.setS("ok");
response.setT(timestamps);
response.setL(lows);
response.setH(highs);
response.setO(opens);
response.setC(closes);
response.setV(volumes);
} else {
response = restTemplate.getForObject(apiUrl, StockHistoryResponse.class);
}
} catch (RestClientException e) {
// Log the exception or perform any other error handling
log.error("Error while making API request. Retrying... (Retry count: {})", retryCount + 1);
@@ -737,31 +1138,33 @@ public class MoneyApiController {
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
} catch (ClientProtocolException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (response != null) {
setResponse(response, resolution);
if (response != null && !response.getS().equals("error")) {
// setResponse(response, resolution);
// API request successful, return the response
return ResponseEntity.ok(response);
} else {
if (!StringUtils.equals("H", resolution)) {
try {
MoneyStock moneyStock = moneyStockRepository.findOne((QMoneyStockPO.moneyStockPO.moneyScId.eq(symbol))
.and(QMoneyStockPO.moneyStockPO.isLock.eq(0))
.and(QMoneyStockPO.moneyStockPO.isShow.eq(0)))
.orElse(null);
if (moneyStock != null && moneyStock.getNseIndiaId() != null && !moneyStock.getNseIndiaId().isEmpty()) {
request.setSymbol(moneyStock.getNseIndiaId());
response = NseIndiaRequest.stockKLineFromHttp(request);
if (moneyStock != null && moneyStock.getNseIndiaChartId() != null && !moneyStock.getNseIndiaChartId().isEmpty()) {
request.setSymbol(moneyStock.getNseIndiaChartId());
response = NseIndiaRequest.stockKLineFromHttp(request, resolution);
return ResponseEntity.ok(response);
}
} catch (Exception e) {
log.error("Failed to get data from nseindia.", e.getMessage());
return ResponseEntity.status(HttpStatus.BAD_REQUEST).build();
}
}
// All retries failed, return an error response
log.error("Failed to get a successful response after {} retries.", maxRetries);
@@ -770,6 +1173,33 @@ public class MoneyApiController {
// 返回响应
}
@GetMapping({"/market/api/market/stock/optional", "/api/market/stock/optional"})
@ResponseBody
@EncryptFilter(decryptRequest = false)
public ResponseEntity<List<Object>> getOptionalStock() {
List<OptionalStock> optionalStocks = optionalStockRepository.findAll();
List<Object> data = optionalStocks.stream().map(stock -> {
try {
String responseStr = HttpRequest.doGrabGet(OPTIONAL_STOCK_MONEYCONTROL_URL + URLEncoder.encode(stock.getSymbol(), "UTF-8"));
OptionalStockResponse response = objectMapper.readValue(responseStr, OptionalStockResponse.class);
if (response != null && response.getCode().equals(200)) {
return response.getData();
}
}
catch (Exception e) {
log.error("Failed to get optional stock from moneycontrol for " + stock.getSymbol(), e);
}
OptionalStockResponse response = new OptionalStockResponse();
response.getData().setSymbol(stock.getSymbol());
response.getData().setSymbol(stock.getCompany());
return response.getData();
}).collect(Collectors.toList());
return ResponseEntity.ok(data);
}
private void setResponse(StockHistoryResponse response, String resolution) {
if (!"ok".equals(response.getS())) {
return;

View File

@@ -26,13 +26,14 @@ public class MoneyProxyApiControl {
@GetMapping({"/market/api/proxy/queryStockMoneyProxy","/api/proxy/queryStockMoneyProxy"})
public JSONObject queryStockByMoneyIdAndExchangeType(String scId,String exchangeType){
log.info("queryStockByMoneyIdAndExchangeType");
return stockByMoneyIdAndExchangeTypeFromHttp(scId,exchangeType);
}
public static JSONObject stockByMoneyIdAndExchangeTypeFromHttp(String scId, String exchangeType) {
try {
String url = "https://priceapi.moneycontrol.com/pricefeed/"+exchangeType+"/equitycash/" + scId ;
String url = "https://moneycontrol.titanrwa.org/pricefeed/"+exchangeType+"/equitycash/" + scId ;
URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
con.setRequestMethod("GET");
@@ -53,7 +54,8 @@ public class MoneyProxyApiControl {
log.error("HTTP request failed with response code: " + responseCode);
}
} catch (Exception e) {
log.error("stockByMoneyIdAndExchangeTypeFromHttp",e);
e.printStackTrace();
log.error("stockByMoneyIdAndExchangeTypeFromHttp failed {}",e.getMessage());
}
return new JSONObject();
}

View File

@@ -3,9 +3,9 @@ spring:
show-sql: true
# Redis配置
redis:
host: lb-b6sze686-7db6hgvsx05ef8nz.clb.sg-tencentclb.net
host: 43.153.174.179
password: a5v8b86P4mVzFlUqJV
port: 56379
port: 30001
database: 1
lettuce:
pool:
@@ -17,7 +17,7 @@ spring:
datasource:
stock-market:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://lb-1ju43bvw-p0ecnv4by9ju8six.clb.ap-singapore.tencentclb.com:53306/india_stock?useUnicode=true&characterEncoding=utf-8
url: jdbc:mysql://43.153.174.179:30000/india_stock?useUnicode=true&characterEncoding=utf-8
username: root
password: uNejHIFQGJOUtYTmE
maxActive: 500