Compare commits
74 Commits
twelvedata
...
feature/up
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa796cf853 | ||
|
|
9a06979f07 | ||
|
|
d1f9600361 | ||
|
|
fcab660024 | ||
|
|
ad95848c2c | ||
|
|
90a26823f0 | ||
|
|
c72c0ff4cd | ||
|
|
85094c22b2 | ||
|
|
59684cccfc | ||
|
|
57690ecb5e | ||
|
|
8cf511f1b7 | ||
|
|
f612b5128e | ||
|
|
07a69f29b9 | ||
|
|
3e1ff24bb6 | ||
|
|
b26d3962c4 | ||
|
|
d085d8de12 | ||
|
|
0476ad100b | ||
|
|
f87df03715 | ||
|
|
fc59d1f47c | ||
|
|
6fdc13a72d | ||
|
|
6b5623dc33 | ||
|
|
4a22fc0a95 | ||
|
|
563dae1c45 | ||
|
|
fa02ebf03b | ||
|
|
0f5f25f744 | ||
|
|
615aae7e1d | ||
|
|
c89345329d | ||
|
|
f18521d190 | ||
|
|
e84c212bf0 | ||
|
|
1ff1797b0b | ||
|
|
89c0fe0321 | ||
|
|
b6052a46ff | ||
|
|
d40c30bcb5 | ||
|
|
5b16f1f2c9 | ||
|
|
b8d4e0d3f2 | ||
|
|
c19277e390 | ||
|
|
514875878f | ||
|
|
4257541342 | ||
|
|
441adac15a | ||
|
|
ba29ab98c7 | ||
|
|
94ec73874b | ||
|
|
549345719a | ||
|
|
3870400411 | ||
|
|
94d2936ffb | ||
|
|
59e4aa9e1a | ||
|
|
bae645e3c6 | ||
|
|
94d9aea2c8 | ||
|
|
5ae9762d45 | ||
|
|
ece52491dd | ||
|
|
6235d41711 | ||
|
|
c865eccaf4 | ||
|
|
136020a497 | ||
|
|
d7da59aca1 | ||
|
|
8427bed427 | ||
|
|
269f02b79d | ||
|
|
7cc456bcd8 | ||
|
|
be05d403af | ||
|
|
4a29bf6659 | ||
|
|
5b2eb8519a | ||
|
|
2e4449bd64 | ||
|
|
b321fd9d4e | ||
|
|
2f9e54bf4f | ||
| 3bb354bbc6 | |||
| a403cba964 | |||
|
|
99bde81670 | ||
|
|
bae941828e | ||
|
|
2f86571578 | ||
|
|
d491bef735 | ||
|
|
74c64eb7d4 | ||
|
|
f50b6b5519 | ||
| 2446d89b30 | |||
|
|
33a68eb7e8 | ||
|
|
7b0a08150d | ||
|
|
146e9d9bc0 |
@@ -2,7 +2,7 @@ stages:
|
||||
# 构建
|
||||
- build
|
||||
- dockerize
|
||||
- webhook
|
||||
- deploy
|
||||
|
||||
build:
|
||||
image: maven:3.6.3-openjdk-8
|
||||
@@ -60,16 +60,20 @@ dockerize:
|
||||
- docker push juyoutech/india_stock_market:latest
|
||||
- docker push juyoutech/india_stock_market:$CI_PIPELINE_ID
|
||||
- docker push juyoutech/india_stock_market:$CI_COMMIT_BRANCH
|
||||
webhook:
|
||||
stage: webhook
|
||||
deploy-dev:
|
||||
stage: deploy
|
||||
only:
|
||||
- develop
|
||||
script:
|
||||
- echo ">>>>>>Start Deploy<<<<<<"
|
||||
- echo "Deploying application..."
|
||||
- ls
|
||||
- apt-get update
|
||||
- apt-get install -y curl
|
||||
- curl --max-time 30 http://149.88.86.7:11113/api/webhook/justpull?project=india_market_java\&setenv_file=alpha.sh\&publish_env=alpha\&shell=deploy-test.sh
|
||||
- echo ">>>>>>Start Deploy Finish<<<<<<"
|
||||
- apk update
|
||||
- apk add curl
|
||||
- |
|
||||
curl -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"namespace":"yddev", "deployment_name":"india-stock-market"}' \
|
||||
https://updater-yddev.moneytj.com/restart-deployment
|
||||
- echo "Application successfully deployed."
|
||||
|
||||
|
||||
|
||||
6
pom.xml
6
pom.xml
@@ -185,6 +185,12 @@
|
||||
<version>30.1-jre</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.5.13</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package cn.stock.market.infrastructure.db.po;
|
||||
|
||||
import static com.querydsl.core.types.PathMetadataFactory.*;
|
||||
|
||||
import com.querydsl.core.types.dsl.*;
|
||||
|
||||
import com.querydsl.core.types.PathMetadata;
|
||||
import javax.annotation.Generated;
|
||||
import com.querydsl.core.types.Path;
|
||||
|
||||
|
||||
/**
|
||||
* QOptionalStockPO is a Querydsl query type for OptionalStockPO
|
||||
*/
|
||||
@Generated("com.querydsl.codegen.EntitySerializer")
|
||||
public class QOptionalStockPO extends EntityPathBase<OptionalStockPO> {
|
||||
|
||||
private static final long serialVersionUID = 1161631810L;
|
||||
|
||||
public static final QOptionalStockPO optionalStockPO = new QOptionalStockPO("optionalStockPO");
|
||||
|
||||
public final StringPath company = createString("company");
|
||||
|
||||
public final NumberPath<Integer> id = createNumber("id", Integer.class);
|
||||
|
||||
public final StringPath symbol = createString("symbol");
|
||||
|
||||
public QOptionalStockPO(String variable) {
|
||||
super(OptionalStockPO.class, forVariable(variable));
|
||||
}
|
||||
|
||||
public QOptionalStockPO(Path<? extends OptionalStockPO> path) {
|
||||
super(path.getType(), path.getMetadata());
|
||||
}
|
||||
|
||||
public QOptionalStockPO(PathMetadata metadata) {
|
||||
super(OptionalStockPO.class, metadata);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ public class QStockIpoPO extends EntityPathBase<StockIpoPO> {
|
||||
public final StringPath stockCode = createString("stockCode");
|
||||
|
||||
public final StringPath stockName = createString("stockName");
|
||||
public final StringPath exchangeType = createString("exchangeType");
|
||||
|
||||
public final NumberPath<java.math.BigDecimal> stockPrice = createNumber("stockPrice", java.math.BigDecimal.class);
|
||||
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
package cn.stock.market.infrastructure.db.po;
|
||||
|
||||
import static com.querydsl.core.types.PathMetadataFactory.*;
|
||||
|
||||
import com.querydsl.core.types.dsl.*;
|
||||
|
||||
import com.querydsl.core.types.PathMetadata;
|
||||
import javax.annotation.Generated;
|
||||
import com.querydsl.core.types.Path;
|
||||
|
||||
|
||||
/**
|
||||
* QTwelveStockPO is a Querydsl query type for TwelveStockPO
|
||||
*/
|
||||
@Generated("com.querydsl.codegen.EntitySerializer")
|
||||
public class QTwelveStockPO extends EntityPathBase<TwelveStockPO> {
|
||||
|
||||
private static final long serialVersionUID = -100071895L;
|
||||
|
||||
public static final QTwelveStockPO twelveStockPO = new QTwelveStockPO("twelveStockPO");
|
||||
|
||||
public final NumberPath<Integer> id = createNumber("id", Integer.class);
|
||||
|
||||
public final NumberPath<Integer> isLock = createNumber("isLock", Integer.class);
|
||||
|
||||
public final NumberPath<Integer> isShow = createNumber("isShow", Integer.class);
|
||||
|
||||
public final DateTimePath<java.util.Date> saveTime = createDateTime("saveTime", java.util.Date.class);
|
||||
|
||||
public final StringPath stockCode = createString("stockCode");
|
||||
|
||||
public final StringPath stockName = createString("stockName");
|
||||
|
||||
public final StringPath stockType = createString("stockType");
|
||||
|
||||
public QTwelveStockPO(String variable) {
|
||||
super(TwelveStockPO.class, forVariable(variable));
|
||||
}
|
||||
|
||||
public QTwelveStockPO(Path<? extends TwelveStockPO> path) {
|
||||
super(path.getType(), path.getMetadata());
|
||||
}
|
||||
|
||||
public QTwelveStockPO(PathMetadata metadata) {
|
||||
super(TwelveStockPO.class, metadata);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
package cn.stock.market.application.assembler;
|
||||
|
||||
import cn.qutaojing.common.utils.Beans;
|
||||
import cn.qutaojing.common.utils.SpringUtils;
|
||||
import cn.stock.market.domain.basic.entity.TwelveStock;
|
||||
import cn.stock.market.dto.TwelveStockDTO;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* TwelveStockAssembler
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Component
|
||||
@Lazy
|
||||
public class TwelveStockAssembler {
|
||||
public TwelveStockDTO toDTO(TwelveStock e) {
|
||||
TwelveStockDTO dto = Beans.mapper(e, TwelveStockDTO.class);
|
||||
if(dto == null) return dto;
|
||||
fill(e, dto);
|
||||
return dto;
|
||||
}
|
||||
|
||||
protected void fill(TwelveStock e, TwelveStockDTO dto) {
|
||||
if(dto == null) return;
|
||||
return;
|
||||
}
|
||||
|
||||
public static TwelveStockAssembler of() {
|
||||
return SpringUtils.getBean(TwelveStockAssembler.class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package cn.stock.market.domain.basic.convert;
|
||||
|
||||
import cn.qutaojing.common.domain.convert.SimpleEntityPOConvert;
|
||||
import cn.qutaojing.common.utils.SpringUtils;
|
||||
import cn.stock.market.domain.basic.entity.OptionalStock;
|
||||
import cn.stock.market.infrastructure.db.po.OptionalStockPO;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@Lazy
|
||||
public class OptionalStockConvert extends SimpleEntityPOConvert<OptionalStock, OptionalStockPO> {
|
||||
public static OptionalStockConvert of() {
|
||||
return SpringUtils.getBean(OptionalStockConvert.class);
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package cn.stock.market.domain.basic.convert;
|
||||
|
||||
import cn.qutaojing.common.domain.convert.SimpleEntityPOConvert;
|
||||
import cn.qutaojing.common.utils.SpringUtils;
|
||||
import cn.stock.market.domain.basic.entity.TwelveStock;
|
||||
import cn.stock.market.infrastructure.db.po.TwelveStockPO;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* TwelveStockConvert
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Component
|
||||
@Lazy
|
||||
public class TwelveStockConvert extends SimpleEntityPOConvert<TwelveStock, TwelveStockPO> {
|
||||
public static TwelveStockConvert of() {
|
||||
return SpringUtils.getBean(TwelveStockConvert.class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package cn.stock.market.domain.basic.entity;
|
||||
|
||||
import cn.stock.market.infrastructure.db.po.OptionalStockPO;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@SuperBuilder
|
||||
@EqualsAndHashCode(
|
||||
callSuper = false
|
||||
)
|
||||
public class OptionalStock extends OptionalStockPO {
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package cn.stock.market.domain.basic.entity;
|
||||
|
||||
import cn.qutaojing.common.utils.Beans;
|
||||
import cn.stock.market.dto.command.TwelveStockCreateCommand;
|
||||
import cn.stock.market.infrastructure.db.po.TwelveStockPO;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
/**
|
||||
* TwelveStock
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@SuperBuilder
|
||||
@EqualsAndHashCode(
|
||||
callSuper = false
|
||||
)
|
||||
public class TwelveStock extends TwelveStockPO {
|
||||
public void update(TwelveStockCreateCommand cmd) {
|
||||
Beans.copyProperties(cmd, this);
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package cn.stock.market.domain.basic.factory;
|
||||
|
||||
import cn.qutaojing.common.utils.SpringUtils;
|
||||
import cn.stock.market.domain.basic.entity.TwelveStock;
|
||||
import cn.stock.market.dto.command.TwelveStockCreateCommand;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* TwelveStockFactory
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Component
|
||||
@Lazy
|
||||
public class TwelveStockFactory {
|
||||
public static TwelveStockFactory of() {
|
||||
return SpringUtils.getBean(TwelveStockFactory.class);
|
||||
}
|
||||
|
||||
public TwelveStock from(TwelveStockCreateCommand cmd) {
|
||||
TwelveStock e = TwelveStock.builder().build();
|
||||
e.update(cmd);
|
||||
return e;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package cn.stock.market.domain.basic.repository;
|
||||
|
||||
import cn.qutaojing.common.domain.convert.IEntityPOConvert;
|
||||
import cn.qutaojing.common.domain.respostory.SimplePoConvertEntityRepository;
|
||||
import cn.stock.market.domain.basic.convert.OptionalStockConvert;
|
||||
import cn.stock.market.domain.basic.entity.OptionalStock;
|
||||
import cn.stock.market.infrastructure.db.po.OptionalStockPO;
|
||||
import cn.stock.market.infrastructure.db.repo.OptionalStockRepo;
|
||||
import com.rp.spring.jpa.GenericJpaRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
@RequiredArgsConstructor(
|
||||
onConstructor = @__(@Autowired)
|
||||
)
|
||||
public class OptionalStockRepository extends SimplePoConvertEntityRepository<OptionalStock, OptionalStockPO, Integer> {
|
||||
final OptionalStockRepo repo;
|
||||
|
||||
final OptionalStockConvert convert;
|
||||
|
||||
@Override
|
||||
public GenericJpaRepository<OptionalStockPO, Integer> repo() {
|
||||
return repo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IEntityPOConvert<OptionalStock, OptionalStockPO> convert() {
|
||||
return convert;
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package cn.stock.market.domain.basic.repository;
|
||||
|
||||
import cn.qutaojing.common.domain.convert.IEntityPOConvert;
|
||||
import cn.qutaojing.common.domain.respostory.SimplePoConvertEntityRepository;
|
||||
import cn.qutaojing.common.utils.SpringUtils;
|
||||
import cn.stock.market.domain.basic.convert.TwelveStockConvert;
|
||||
import cn.stock.market.domain.basic.entity.TwelveStock;
|
||||
import cn.stock.market.infrastructure.db.po.TwelveStockPO;
|
||||
import cn.stock.market.infrastructure.db.repo.TwelveStockRepo;
|
||||
import com.rp.spring.jpa.GenericJpaRepository;
|
||||
import java.lang.Integer;
|
||||
import java.lang.Override;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
/**
|
||||
* TwelveStockRepository
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Repository
|
||||
@RequiredArgsConstructor(
|
||||
onConstructor = @__(@Autowired)
|
||||
)
|
||||
public class TwelveStockRepository extends SimplePoConvertEntityRepository<TwelveStock, TwelveStockPO, Integer> {
|
||||
final TwelveStockRepo repo;
|
||||
|
||||
final TwelveStockConvert convert;
|
||||
|
||||
@Override
|
||||
public GenericJpaRepository<TwelveStockPO, Integer> repo() {
|
||||
return repo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IEntityPOConvert<TwelveStock, TwelveStockPO> convert() {
|
||||
return convert;
|
||||
}
|
||||
|
||||
public static TwelveStockRepository of() {
|
||||
return SpringUtils.getBean(TwelveStockRepository.class);
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,8 @@ import org.apache.commons.lang3.StringUtils;
|
||||
import cn.stock.market.infrastructure.db.po.QStockPO;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.http.HttpEntity;
|
||||
@@ -247,34 +249,112 @@ public class StockService {
|
||||
String result = "";
|
||||
try {
|
||||
// 使用Jsoup连接到网页
|
||||
Document doc = Jsoup.connect("https://www.business-standard.com/markets/news")
|
||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
|
||||
.header("Referer", "https://www.business-standard.com/")
|
||||
.header("Accept-Language", "en-US,en;q=0.9")
|
||||
.get();
|
||||
result = doc.html().substring(doc.html().indexOf("<div class=\"shortvideoimg\" style=\"position:relative\">"),doc.html().lastIndexOf("<div class=\"shortvideoimg\" style=\"position:relative\">")+1200);
|
||||
// Document doc = Jsoup.connect("https://www.business-standard.com/markets/news")
|
||||
// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
|
||||
// .header("Referer", "https://www.business-standard.com/")
|
||||
// .header("Accept-Language", "en-US,en;q=0.9")
|
||||
// .get();
|
||||
String url = "https://www.business-standard.com/markets/news";
|
||||
// Document doc = Jsoup.connect(url)
|
||||
// .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
// .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
// .header("Accept-Language", "en-US,en;q=0.5")
|
||||
// .header("Connection", "keep-alive")
|
||||
// .referrer("https://www.google.com")
|
||||
// .ignoreHttpErrors(true)
|
||||
// .timeout(10000)
|
||||
// .get();
|
||||
String doGetNews = HttpClientRequest.doGetNews(url);
|
||||
// result = doc.html().substring(doc.html().indexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">"),doc.html().lastIndexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">")+500);
|
||||
// Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57");
|
||||
// StringBuilder sb = new StringBuilder();
|
||||
// for (Element divElement : divElements) {
|
||||
// sb.append(divElement.outerHtml()).append("\n");
|
||||
// }
|
||||
// result = sb.toString();
|
||||
result = extractNewsFromHtml(doGetNews);
|
||||
} catch (Exception e) {
|
||||
return e.toString();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String extractNewsFromHtml(String rawHtml) {
|
||||
Document doc = Jsoup.parse(rawHtml);
|
||||
|
||||
// Optional: extract raw block around <div class="listingstyle_shortvideoimg__0TWuX shortvideoimg">
|
||||
int start = rawHtml.indexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">");
|
||||
int end = rawHtml.lastIndexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">") + 500;
|
||||
String result = (start != -1 && end > start) ? rawHtml.substring(start, end) : "";
|
||||
|
||||
// Extract article list using CSS selector
|
||||
Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57");
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Element divElement : divElements) {
|
||||
sb.append(divElement.outerHtml()).append("\n");
|
||||
}
|
||||
|
||||
// If you only want the div content:
|
||||
result = sb.toString();
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<String> getNewsInfo(String url) {
|
||||
// String result = "";
|
||||
// List<String> list = new ArrayList<>();
|
||||
// try {
|
||||
// // 使用Jsoup连接到网页
|
||||
//// Document doc = Jsoup.connect(url)
|
||||
//// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
|
||||
//// .header("Referer", "https://www.business-standard.com/")
|
||||
//// .header("Accept-Language", "en-US,en;q=0.9")
|
||||
//// .get();
|
||||
// Document doc = Jsoup.connect(url)
|
||||
// .referrer("https://www.business-standard.com/")
|
||||
// .header("Accept-Language", "en-US,en;q=0.9")
|
||||
// .userAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1")
|
||||
// .timeout(5000) // timeout 5 seconds
|
||||
// .get();
|
||||
// result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1);
|
||||
// list.add(result);
|
||||
// list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf("<meta property=\"og:url") - 5));
|
||||
// } catch (Exception e) {
|
||||
// list.add(e.toString());
|
||||
// return list;
|
||||
// }
|
||||
// return list;
|
||||
String result = "";
|
||||
List<String> list = new ArrayList<>();
|
||||
try {
|
||||
// 使用Jsoup连接到网页
|
||||
Document doc = Jsoup.connect(url)
|
||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
|
||||
.header("Referer", "https://www.business-standard.com/")
|
||||
.header("Accept-Language", "en-US,en;q=0.9")
|
||||
.get();
|
||||
result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1);
|
||||
// Step 1: Get raw HTML via HttpClient
|
||||
String rawHtml = HttpClientRequest.doGetNews(url);
|
||||
|
||||
// Step 2: Parse HTML with Jsoup
|
||||
Document doc = Jsoup.parse(rawHtml);
|
||||
|
||||
// Step 3: Extract content manually (like original)
|
||||
int startIdx = rawHtml.indexOf("articleBody") + 14;
|
||||
int endIdx = rawHtml.indexOf(",\"author\":") - 1;
|
||||
|
||||
if (startIdx > 0 && endIdx > startIdx) {
|
||||
result = rawHtml.substring(startIdx, endIdx);
|
||||
} else {
|
||||
result = "[articleBody not found]";
|
||||
}
|
||||
list.add(result);
|
||||
list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf("<meta property=\"og:url") - 5));
|
||||
|
||||
// Step 4: Extract og:title
|
||||
int titleStart = rawHtml.indexOf("og:title") + 19;
|
||||
int titleEnd = rawHtml.indexOf("<meta property=\"og:url") - 5;
|
||||
|
||||
if (titleStart > 0 && titleEnd > titleStart) {
|
||||
list.add(rawHtml.substring(titleStart, titleEnd));
|
||||
} else {
|
||||
list.add("[title not found]");
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
list.add(e.toString());
|
||||
return list;
|
||||
list.add("[Error] " + e.getMessage());
|
||||
}
|
||||
return list;
|
||||
}
|
||||
@@ -974,7 +1054,7 @@ public class StockService {
|
||||
String exchange = "bse";
|
||||
IndiaIndexVo vo1 = new IndiaIndexVo();
|
||||
String coCode = "20558";
|
||||
JSONObject object = TodayApis.getStockDetail(exchange, coCode);
|
||||
JSONObject object = TodayApis.getStockDetail("in%3BSEN", coCode);
|
||||
IndiaStockVO market = objToVo(object);
|
||||
market.setName("BSESENSEX指数");
|
||||
vo1.setIndexVo(market);
|
||||
@@ -1004,7 +1084,7 @@ public class StockService {
|
||||
String exchange = "nse";
|
||||
IndiaIndexVo vo1 = new IndiaIndexVo();
|
||||
String coCode = "20559";
|
||||
JSONObject object = TodayApis.getStockDetail(exchange, coCode);
|
||||
JSONObject object = TodayApis.getStockDetail("in%3BNSX", coCode);
|
||||
IndiaStockVO market = objToVo(object);
|
||||
market.setName("NIFTY50指数");
|
||||
vo1.setIndexVo(market);
|
||||
@@ -1034,23 +1114,23 @@ public class StockService {
|
||||
}
|
||||
private IndiaStockVO objToVo(JSONObject object){
|
||||
IndiaStockVO market = new IndiaStockVO();
|
||||
if(object.containsKey("prev_close")){
|
||||
market.setClose(object.getString("prev_close"));
|
||||
if(object.containsKey("priceprevclose")){
|
||||
market.setClose(object.getString("priceprevclose"));
|
||||
}
|
||||
if(object.containsKey("price")){
|
||||
market.setNowPrice(object.getString("price"));
|
||||
if(object.containsKey("pricecurrent")){
|
||||
market.setNowPrice(object.getString("pricecurrent"));
|
||||
}
|
||||
if(object.containsKey("perchg")){
|
||||
market.setRate(object.getString("perchg"));
|
||||
if(object.containsKey("pricepercentchange")){
|
||||
market.setRate(object.getString("pricepercentchange"));
|
||||
}
|
||||
if(object.containsKey("high_price")){
|
||||
market.setHigh(object.getString("high_price"));
|
||||
if(object.containsKey("HIGH")){
|
||||
market.setHigh(object.getString("HIGH"));
|
||||
}
|
||||
if(object.containsKey("low_price")){
|
||||
market.setLow(object.getString("low_price"));
|
||||
if(object.containsKey("LOW")){
|
||||
market.setLow(object.getString("LOW"));
|
||||
}
|
||||
if(object.containsKey("open_price")){
|
||||
market.setOpen(object.getString("open_price"));
|
||||
if(object.containsKey("OPEN")){
|
||||
market.setOpen(object.getString("OPEN"));
|
||||
}
|
||||
return market;
|
||||
// stockListVO.setName(jsonObject.getString("companyname"));
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
package cn.stock.market.domain.basic.service;
|
||||
|
||||
import cn.qutaojing.common.utils.SpringUtils;
|
||||
import cn.stock.market.domain.basic.factory.TwelveStockFactory;
|
||||
import cn.stock.market.domain.basic.repository.TwelveStockRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* TwelveStockService
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor(
|
||||
onConstructor = @__(@Autowired)
|
||||
)
|
||||
public class TwelveStockService {
|
||||
final TwelveStockRepository repository;
|
||||
|
||||
final TwelveStockFactory factory;
|
||||
|
||||
public TwelveStockRepository repository() {
|
||||
return repository;
|
||||
}
|
||||
|
||||
public static TwelveStockService of() {
|
||||
return SpringUtils.getBean(TwelveStockService.class);
|
||||
}
|
||||
}
|
||||
18
src/main/java/cn/stock/market/dto/OptionalStockResponse.java
Normal file
18
src/main/java/cn/stock/market/dto/OptionalStockResponse.java
Normal file
@@ -0,0 +1,18 @@
|
||||
package cn.stock.market.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class OptionalStockResponse {
|
||||
private String message;
|
||||
private Integer code;
|
||||
private DataResponse data = new DataResponse();
|
||||
|
||||
@Data
|
||||
public static class DataResponse {
|
||||
private String symbol;
|
||||
private String company;
|
||||
private Double pricecurrent;
|
||||
private Float pricepercentchange;
|
||||
}
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package cn.stock.market.dto;
|
||||
|
||||
import cn.stock.market.infrastructure.db.po.TwelveStockPO;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
/**
|
||||
* TwelveStockDTO
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@SuperBuilder
|
||||
@EqualsAndHashCode(
|
||||
callSuper = false
|
||||
)
|
||||
public class TwelveStockDTO extends TwelveStockPO {
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
package cn.stock.market.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class TwelveStockData {
|
||||
|
||||
private String symbol;
|
||||
|
||||
private String name;
|
||||
|
||||
private String currency;
|
||||
|
||||
private String exchange;
|
||||
|
||||
private String mic_code;
|
||||
|
||||
private String country;
|
||||
|
||||
private String type;
|
||||
}
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
package cn.stock.market.dto;
|
||||
|
||||
import io.swagger.annotations.ApiModel;
|
||||
import io.swagger.annotations.ApiModelProperty;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
/**
|
||||
* TwelveStockInfo类用于存储股票信息。
|
||||
* 该类包含了股票的基本信息和交易数据。
|
||||
*/
|
||||
@ApiModel("TwelveStockInfo")
|
||||
public class TwelveStockInfo {
|
||||
@ApiModelProperty("股票代码")
|
||||
private String stockCode;
|
||||
|
||||
@ApiModelProperty("交易代码")
|
||||
private String symbol;
|
||||
|
||||
@ApiModelProperty("股票名称")
|
||||
private String stockName;
|
||||
|
||||
@ApiModelProperty("股票类型")
|
||||
private String stockType;
|
||||
|
||||
@ApiModelProperty("当前价格")
|
||||
private String lastPrice;
|
||||
|
||||
@ApiModelProperty("变动百分比")
|
||||
private String perchg;
|
||||
@ApiModelProperty("变动值")
|
||||
private String change;
|
||||
|
||||
@ApiModelProperty("开盘价格")
|
||||
private String openPrice;
|
||||
|
||||
@ApiModelProperty("昨日收盘价格")
|
||||
private String previousPrice;
|
||||
|
||||
@ApiModelProperty("成交量")
|
||||
private String volume;
|
||||
|
||||
@ApiModelProperty("最高价格")
|
||||
private String highPrice;
|
||||
|
||||
@ApiModelProperty("最低价格")
|
||||
private String lowPrice;
|
||||
|
||||
@ApiModelProperty("52周最高价格")
|
||||
private String week52HighPrice;
|
||||
|
||||
@ApiModelProperty("52周最低价格")
|
||||
private String week52LowPrice;
|
||||
|
||||
@ApiModelProperty("股票当前的交易状态: 1-表示退市Closed, 2-Opening Auction, 3-Continuous Trading, 15-Intra-day Close (scheduled)")
|
||||
private String status;
|
||||
|
||||
@ApiModelProperty("股票id")
|
||||
private Integer id;
|
||||
}
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
package cn.stock.market.dto.command;
|
||||
|
||||
import java.lang.Integer;
|
||||
import java.lang.String;
|
||||
import java.util.Date;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
/**
|
||||
* TwelveStockCreateCommand
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Data
|
||||
@SuperBuilder
|
||||
@NoArgsConstructor
|
||||
public class TwelveStockCreateCommand {
|
||||
/**
|
||||
* 主键 */
|
||||
Integer id;
|
||||
|
||||
/**
|
||||
* 股票code */
|
||||
String stockCode;
|
||||
|
||||
/**
|
||||
* 股票名称 */
|
||||
String stockName;
|
||||
|
||||
/**
|
||||
* BSE or NSE */
|
||||
String stockType;
|
||||
|
||||
/**
|
||||
* 保存时间 */
|
||||
Date saveTime;
|
||||
|
||||
/**
|
||||
* 是否锁定 0否 1是 */
|
||||
Integer isLock;
|
||||
|
||||
/**
|
||||
* 是否展示 0是 1否 */
|
||||
Integer isShow;
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package cn.stock.market.dto.command;
|
||||
|
||||
import java.lang.Integer;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
|
||||
/**
|
||||
* TwelveStockModifyCommand
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@Data
|
||||
@SuperBuilder
|
||||
@NoArgsConstructor
|
||||
@EqualsAndHashCode(
|
||||
callSuper = false
|
||||
)
|
||||
public class TwelveStockModifyCommand extends TwelveStockCreateCommand {
|
||||
Integer id;
|
||||
}
|
||||
48
src/main/java/cn/stock/market/dto/query/StockChartDto.java
Normal file
48
src/main/java/cn/stock/market/dto/query/StockChartDto.java
Normal file
@@ -0,0 +1,48 @@
|
||||
package cn.stock.market.dto.query;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDate;
|
||||
|
||||
@Data
|
||||
public class StockChartDto {
|
||||
private Long timestamp;
|
||||
private double open;
|
||||
private double high;
|
||||
private double low;
|
||||
private double close;
|
||||
private double volume;
|
||||
|
||||
public StockChartDto(Long date, double open, double high, double low, double close, double volume) {
|
||||
this.timestamp = date;
|
||||
this.open = open;
|
||||
this.high = high;
|
||||
this.low = low;
|
||||
this.close = close;
|
||||
this.volume = volume;
|
||||
}
|
||||
|
||||
public double getVolume() {
|
||||
return volume;
|
||||
}
|
||||
|
||||
public Long getTimestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
public double getOpen() {
|
||||
return open;
|
||||
}
|
||||
|
||||
public double getHigh() {
|
||||
return high;
|
||||
}
|
||||
|
||||
public double getLow() {
|
||||
return low;
|
||||
}
|
||||
|
||||
public double getClose() {
|
||||
return close;
|
||||
}
|
||||
}
|
||||
@@ -187,18 +187,17 @@ public class TodayApis {
|
||||
|
||||
//获取详情
|
||||
public static JSONObject getStockDetail(String exchange,String coCode){
|
||||
String url = String.format("https://marketapi.intoday.in/widget/stockdetail/pullview?co_code=%s&exchange=%s",coCode,exchange);
|
||||
String url = String.format("https://priceapi.moneycontrol.com/pricefeed/notapplicable/inidicesindia/%s",exchange);
|
||||
String str = get(url);
|
||||
JSONObject object = JSON.parseObject(str);
|
||||
boolean bool = false;
|
||||
if(object.containsKey("success")){
|
||||
bool = object.getBoolean("success");
|
||||
if(object.containsKey("code")){
|
||||
bool = object.getInteger("code") == 200;
|
||||
}
|
||||
if(!bool){
|
||||
return new JSONObject();
|
||||
}
|
||||
JSONArray jsonArray = JSON.parseArray(object.getString("data"));
|
||||
JSONObject entity = JSON.parseObject(jsonArray.get(0).toString());
|
||||
JSONObject entity = JSON.parseObject(object.getString("data"));
|
||||
return entity;
|
||||
|
||||
}
|
||||
|
||||
@@ -56,6 +56,14 @@ public class MoneyStockPO {
|
||||
* 展示表示 */
|
||||
String selfDispId;
|
||||
|
||||
/**
|
||||
* NSE India的id */
|
||||
String nseIndiaId;
|
||||
|
||||
/**
|
||||
* NSE India Chart的id */
|
||||
String nseIndiaChartId;
|
||||
|
||||
/**
|
||||
* 自有self_url */
|
||||
String selfUrl;
|
||||
@@ -75,4 +83,6 @@ public class MoneyStockPO {
|
||||
/**
|
||||
* 是否展示 0是 1否 */
|
||||
Integer isShow;
|
||||
|
||||
Boolean useFromBseindia = false;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package cn.stock.market.infrastructure.db.po;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
import org.hibernate.annotations.DynamicInsert;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
|
||||
@SuperBuilder
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Entity
|
||||
@DynamicInsert
|
||||
@DynamicUpdate
|
||||
@Table(
|
||||
name = "optional_stock"
|
||||
)
|
||||
public class OptionalStockPO {
|
||||
@Id
|
||||
@GeneratedValue(
|
||||
strategy = javax.persistence.GenerationType.IDENTITY
|
||||
)
|
||||
Integer id;
|
||||
|
||||
String symbol;
|
||||
|
||||
String company;
|
||||
}
|
||||
@@ -61,11 +61,11 @@ public class StockIpoPO {
|
||||
|
||||
/**
|
||||
* 是否显示【1 显示,2 不显示】 */
|
||||
Integer isShow;
|
||||
Integer isShow = 1;
|
||||
|
||||
/**
|
||||
* 是否上市【1 未上市,2 已上市】 */
|
||||
Integer isList;
|
||||
Integer isList = 1;
|
||||
|
||||
Date createDate;
|
||||
|
||||
@@ -79,6 +79,8 @@ public class StockIpoPO {
|
||||
* 申请总额 */
|
||||
String apply;
|
||||
|
||||
String exchangeType;
|
||||
|
||||
/**
|
||||
* 市盈率 */
|
||||
BigDecimal peRatio;
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package cn.stock.market.infrastructure.db.po;
|
||||
|
||||
import java.lang.Integer;
|
||||
import java.lang.String;
|
||||
import java.util.Date;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.SuperBuilder;
|
||||
import org.hibernate.annotations.DynamicInsert;
|
||||
import org.hibernate.annotations.DynamicUpdate;
|
||||
|
||||
/**
|
||||
* TwelveStockPO
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
@SuperBuilder
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Entity
|
||||
@DynamicInsert
|
||||
@DynamicUpdate
|
||||
@Table(
|
||||
name = "twelve_stock"
|
||||
)
|
||||
public class TwelveStockPO {
|
||||
/**
|
||||
* 主键 */
|
||||
@Id
|
||||
@GeneratedValue(
|
||||
strategy = javax.persistence.GenerationType.IDENTITY
|
||||
)
|
||||
Integer id;
|
||||
|
||||
/**
|
||||
* 股票code */
|
||||
String stockCode;
|
||||
|
||||
/**
|
||||
* 股票名称 */
|
||||
String stockName;
|
||||
|
||||
/**
|
||||
* BSE or NSE */
|
||||
String stockType;
|
||||
|
||||
/**
|
||||
* 保存时间 */
|
||||
Date saveTime;
|
||||
|
||||
/**
|
||||
* 是否锁定 0否 1是 */
|
||||
Integer isLock;
|
||||
|
||||
/**
|
||||
* 是否展示 0是 1否 */
|
||||
Integer isShow;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package cn.stock.market.infrastructure.db.repo;
|
||||
|
||||
import cn.stock.market.infrastructure.db.po.OptionalStockPO;
|
||||
import com.rp.spring.jpa.GenericJpaRepository;
|
||||
|
||||
public interface OptionalStockRepo extends GenericJpaRepository<OptionalStockPO, Integer> {
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package cn.stock.market.infrastructure.db.repo;
|
||||
|
||||
import cn.stock.market.infrastructure.db.po.TwelveStockPO;
|
||||
import com.rp.spring.jpa.GenericJpaRepository;
|
||||
import java.lang.Integer;
|
||||
|
||||
/**
|
||||
* TwelveStockRepo
|
||||
*
|
||||
* @author rplees
|
||||
* @email rplees.i.ly@gmail.com
|
||||
* @created 2024/05/30
|
||||
*/
|
||||
public interface TwelveStockRepo extends GenericJpaRepository<TwelveStockPO, Integer> {
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import cn.stock.market.domain.basic.repository.StockRepository;
|
||||
import cn.stock.market.domain.basic.service.StockService;
|
||||
import cn.stock.market.infrastructure.api.investing.InvestingInvokerApis;
|
||||
import cn.stock.market.infrastructure.db.po.QSiteNewsPO;
|
||||
import cn.stock.market.utils.DateTimeUtil;
|
||||
import cn.stock.market.utils.Utils;
|
||||
import com.ag.utils.CollectionUtils;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
@@ -25,6 +26,8 @@ import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
@@ -118,8 +121,16 @@ public class InvestingTask {
|
||||
newsList.forEach( n -> {
|
||||
String contentUrl = n.substring(1, n.indexOf("class=\"img-smllnews\"") - 2);
|
||||
String id = contentUrl.substring(contentUrl.lastIndexOf("-") + 1, contentUrl.lastIndexOf("_"));
|
||||
String imgUrl = n.substring(n.indexOf("img loading=\"lazy\" src=") + 24, n.indexOf("?"));
|
||||
String time = n.substring(n.indexOf("Last Updated") + 23, n.indexOf("IST") - 9);
|
||||
// String imgUrl = n.substring(n.indexOf("img loading=\"lazy\" src=") + 24, n.indexOf("?"));
|
||||
String imgUrl = extractImgSrc(n);
|
||||
// String time = n.substring(n.indexOf("Last Updated") + 23, n.indexOf("IST") - 9);
|
||||
// Extract the date and time using regex
|
||||
Pattern pattern = Pattern.compile("Updated On :<!-- --> <!-- -->(.*?)<!-- -->");
|
||||
Matcher matcher = pattern.matcher(n);
|
||||
String time = "";
|
||||
if (matcher.find()) {
|
||||
time = DateTimeUtil.formatDateTimeFor24H(matcher.group(1).trim(),"dd MMM yyyy | hh:mm a");
|
||||
}
|
||||
|
||||
SiteNews siteNews = new SiteNews();
|
||||
siteNews.setAddTime(new Date());
|
||||
@@ -140,4 +151,22 @@ public class InvestingTask {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static String extractImgSrc(String htmlData) {
|
||||
String searchString = "<noscript>";
|
||||
int startPos = htmlData.indexOf(searchString);
|
||||
|
||||
if (startPos != -1) {
|
||||
String startTag = "src=";
|
||||
startPos = htmlData.indexOf(startTag, startPos) + startTag.length() + 1;
|
||||
|
||||
int endPos = htmlData.indexOf("?", startPos);
|
||||
|
||||
if (endPos != -1) {
|
||||
return htmlData.substring(startPos, endPos);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,15 +115,15 @@ public class MoneyScraper {
|
||||
if (ulElement != null) {
|
||||
for (Element aElement : ulElement.select("a")) {
|
||||
String exchangeValue = aElement.text().trim();
|
||||
if ("BSE".equals(exchangeValue) || "NSE".equals(exchangeValue)) {
|
||||
if ("BSE".equals(exchangeValue.toUpperCase()) || "NSE".equals(exchangeValue.toUpperCase())) {
|
||||
log.info(Thread.currentThread().getName() + ",the stock url: " + url +
|
||||
", the exchange Value: " + exchangeValue);
|
||||
MoneyStock build = MoneyStock.builder().stockName(name).stockType(exchangeValue.toLowerCase(Locale.ROOT))
|
||||
MoneyStock build = MoneyStock.builder().stockName(name).stockType(exchangeValue.toUpperCase(Locale.ROOT))
|
||||
.detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId))
|
||||
.selfUrl(url)
|
||||
.selfDispId(extractDispId(url))
|
||||
.moneyScId(companyCodeId).saveTime(new Date()).build();
|
||||
List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(name), QMoneyStockPO.moneyStockPO.stockType.eq(exchangeValue.toLowerCase(Locale.ROOT)));
|
||||
List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(name), QMoneyStockPO.moneyStockPO.stockType.upper().eq(exchangeValue.toUpperCase(Locale.ROOT)));
|
||||
if (CollectionUtil.isEmpty(all)) {
|
||||
moneyStockRepository.save(build);
|
||||
}
|
||||
@@ -264,15 +264,15 @@ public class MoneyScraper {
|
||||
for (Element aElement : ulElement.select("a")) {
|
||||
String exchangeValue = aElement.text().trim();
|
||||
|
||||
if ("BSE".equals(exchangeValue) || "NSE".equals(exchangeValue)) {
|
||||
if ("BSE".equals(exchangeValue.toUpperCase(Locale.ROOT)) || "NSE".equals(exchangeValue.toUpperCase(Locale.ROOT))) {
|
||||
log.info(Thread.currentThread().getName()+",stockName: " + textContent + ", self_link: " + linkAttribute +
|
||||
", the exchange Value: " + exchangeValue);
|
||||
MoneyStock build = MoneyStock.builder().stockName(textContent).stockType(exchangeValue.toLowerCase(Locale.ROOT))
|
||||
MoneyStock build = MoneyStock.builder().stockName(textContent).stockType(exchangeValue.toUpperCase(Locale.ROOT))
|
||||
.detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId))
|
||||
.selfUrl(linkAttribute)
|
||||
.selfDispId(extractDispId(linkAttribute))
|
||||
.moneyScId(companyCodeId).saveTime(new Date()).build();
|
||||
List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent), QMoneyStockPO.moneyStockPO.stockType.eq(exchangeValue.toLowerCase(Locale.ROOT)));
|
||||
List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent), QMoneyStockPO.moneyStockPO.stockType.upper().eq(exchangeValue.toUpperCase(Locale.ROOT)));
|
||||
if(CollectionUtil.isEmpty(all)){
|
||||
moneyStockRepository.save(build);
|
||||
}
|
||||
|
||||
@@ -1,14 +1,20 @@
|
||||
package cn.stock.market.infrastructure.job;
|
||||
|
||||
import cn.hutool.core.collection.CollectionUtil;
|
||||
import cn.hutool.core.date.DateTime;
|
||||
import cn.hutool.core.date.DateUtil;
|
||||
import cn.stock.market.domain.basic.entity.BtodayStock;
|
||||
import cn.stock.market.domain.basic.entity.MoneyStock;
|
||||
import cn.stock.market.domain.basic.entity.StockIpo;
|
||||
import cn.stock.market.domain.basic.repository.BtodayStockRepository;
|
||||
import cn.stock.market.domain.basic.repository.MoneyStockRepository;
|
||||
import cn.stock.market.domain.basic.repository.StockIpoRepository;
|
||||
import cn.stock.market.dto.StockIpoDTO;
|
||||
import cn.stock.market.infrastructure.db.po.QMoneyStockPO;
|
||||
import cn.stock.market.infrastructure.db.po.QStockIpoPO;
|
||||
import cn.stock.market.infrastructure.db.po.StockIpoPO;
|
||||
import cn.stock.market.infrastructure.db.repo.BtodayStockRepo;
|
||||
import com.alibaba.druid.sql.visitor.functions.Now;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
@@ -31,14 +37,11 @@ import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
@@ -54,6 +57,10 @@ public class Scraper {
|
||||
private StockIpoRepository stockIpoRepository;
|
||||
|
||||
private final ExecutorService executorService = Executors.newFixedThreadPool(5);
|
||||
@Autowired
|
||||
private MoneyStockRepository moneyStockRepository;
|
||||
|
||||
private boolean times = false;
|
||||
|
||||
|
||||
@Scheduled(cron = "0 0 1 */2 * ?")
|
||||
@@ -104,125 +111,226 @@ public class Scraper {
|
||||
|
||||
|
||||
@Scheduled(cron = "0 0 18 * * ?")
|
||||
// @PostConstruct
|
||||
@RequestMapping("/testScraperGetMoneyControllerNewIPO")
|
||||
public void getMoneyControllerNewIPOSchedule() {
|
||||
log.info("定时任务执行获取新股ipo的方法开始执行");
|
||||
// 目标 URL
|
||||
String url = "https://www.moneycontrol.com/ipo/open-upcoming-ipos";
|
||||
List<String> urls = Arrays.asList("https://www.moneycontrol.com/ipo/upcoming-ipos/", "https://www.moneycontrol.com/ipo/open-ipos/");
|
||||
// 创建 HttpClient 实例
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
// 创建 HttpGet 请求
|
||||
HttpGet request = new HttpGet(url);
|
||||
try {
|
||||
// 执行请求
|
||||
HttpResponse response = client.execute(request);
|
||||
for (String url : urls) {
|
||||
HttpGet request = new HttpGet(url);
|
||||
try {
|
||||
// 执行请求
|
||||
HttpResponse response = client.execute(request);
|
||||
|
||||
// 检查请求是否成功
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
// 获取响应体
|
||||
String responseBody = EntityUtils.toString(response.getEntity());
|
||||
// 检查请求是否成功
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
// 获取响应体
|
||||
String responseBody = EntityUtils.toString(response.getEntity());
|
||||
|
||||
// 使用 Jsoup 解析 HTML
|
||||
Document doc = Jsoup.parse(responseBody);
|
||||
// 使用 Jsoup 解析 HTML
|
||||
Document doc = Jsoup.parse(responseBody);
|
||||
|
||||
// 找到包含 JSON 数据的 <script> 标签
|
||||
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
|
||||
// 找到包含 JSON 数据的 <script> 标签
|
||||
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
|
||||
|
||||
if (scriptTag != null) {
|
||||
// 获取 JSON 数据
|
||||
String jsonDataStr = scriptTag.html();
|
||||
if (scriptTag != null) {
|
||||
// 获取 JSON 数据
|
||||
String jsonDataStr = scriptTag.html();
|
||||
|
||||
// 将 JSON 字符串解析为 Java JSONObject
|
||||
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
|
||||
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
|
||||
// 将 JSON 字符串解析为 Java JSONObject
|
||||
JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
|
||||
log.info("获取到新股的json信息:"+jsonObject.toJSONString());
|
||||
|
||||
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
|
||||
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
|
||||
|
||||
// 解析 openData 和 upcomingData
|
||||
JSONArray openData = ipoTableData.getJSONArray("openData");
|
||||
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
|
||||
List<StockIpo> listStockIpoList = new ArrayList<>();
|
||||
for (int i = 0; i < openData.size(); i++) {
|
||||
JSONObject entry = openData.getJSONObject(i);
|
||||
StockIpo stockIpo = new StockIpo();
|
||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||
stockIpo.setStockName(entry.getString("company_name"));
|
||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||
Integer totalNumber = 0;
|
||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||
Integer issue_size = entry.getInteger("issue_size");
|
||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||
totalNumber = totalNumberBigDecimal.intValue();
|
||||
|
||||
JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
|
||||
JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
|
||||
|
||||
|
||||
|
||||
// 解析 openData 和 upcomingData
|
||||
JSONArray openData = ipoTableData.getJSONArray("openData");
|
||||
// if (openData.isEmpty()){
|
||||
// openData = ipoTableData.getJSONArray("openUpcomingData");
|
||||
// }
|
||||
JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
|
||||
List<StockIpo> listStockIpoList = new ArrayList<>();
|
||||
for (int i = 0; i < openData.size(); i++) {
|
||||
JSONObject entry = openData.getJSONObject(i);
|
||||
StockIpo stockIpo = new StockIpo();
|
||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||
stockIpo.setStockName(entry.getString("company_name"));
|
||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||
Integer totalNumber = 0;
|
||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||
BigDecimal issueSize = new BigDecimal(entry.getLong("issue_size"));
|
||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||
totalNumber = totalNumberBigDecimal.intValue();
|
||||
}
|
||||
stockIpo.setTotalNumber(totalNumber);
|
||||
stockIpo.setApply(entry.getString("total_subs"));
|
||||
stockIpo.setCreateDate(new Date());
|
||||
stockIpo.setUpdateDate(new Date());
|
||||
stockIpo.setSourceType("3");
|
||||
|
||||
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
||||
continue;
|
||||
}
|
||||
String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
|
||||
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
|
||||
|
||||
for (String exchange : exchanges) {
|
||||
StockIpo ipo = new StockIpo();
|
||||
ipo.setStockCode(stockIpo.getStockCode());
|
||||
ipo.setStockName(stockIpo.getStockName());
|
||||
ipo.setStockPrice(stockIpo.getStockPrice());
|
||||
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
||||
ipo.setListingDate(stockIpo.getListingDate());
|
||||
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
||||
ipo.setApply(stockIpo.getApply());
|
||||
ipo.setCreateDate(stockIpo.getCreateDate());
|
||||
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
||||
ipo.setExchangeType(exchange);
|
||||
ipo.setSourceType("3");
|
||||
listStockIpoList.add(ipo);
|
||||
}
|
||||
}
|
||||
stockIpo.setTotalNumber(totalNumber);
|
||||
stockIpo.setApply(entry.getString("total_subs"));
|
||||
stockIpo.setCreateDate(new Date());
|
||||
stockIpo.setUpdateDate(new Date());
|
||||
stockIpo.setSourceType("3");
|
||||
listStockIpoList.add(stockIpo);
|
||||
}
|
||||
|
||||
for (int i = 0; i < upcomingData.size(); i++) {
|
||||
JSONObject entry = upcomingData.getJSONObject(i);
|
||||
StockIpo stockIpo = new StockIpo();
|
||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||
stockIpo.setStockName(entry.getString("company_name"));
|
||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||
Integer totalNumber = 0;
|
||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||
Integer issue_size = entry.getInteger("issue_size");
|
||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||
totalNumber = totalNumberBigDecimal.intValue();
|
||||
for (int i = 0; i < upcomingData.size(); i++) {
|
||||
JSONObject entry = upcomingData.getJSONObject(i);
|
||||
StockIpo stockIpo = new StockIpo();
|
||||
stockIpo.setStockCode(entry.getString("sc_id"));
|
||||
stockIpo.setStockName(entry.getString("company_name"));
|
||||
stockIpo.setStockPrice(entry.getBigDecimal("issue_price"));
|
||||
stockIpo.setSubscriptionDate(convertStringToTimestamp(entry.getString("open_date")));
|
||||
stockIpo.setListingDate(convertStringToTimestamp(entry.getString("listing_date")));
|
||||
Integer totalNumber = 0;
|
||||
if(null!=entry.getInteger("issue_size")&&entry.getBigDecimal("issue_price")!=null&&!entry.getBigDecimal("issue_price").equals(BigDecimal.valueOf(0))){
|
||||
Integer issue_size = entry.getInteger("issue_size");
|
||||
BigDecimal issueSize = new BigDecimal(issue_size);
|
||||
BigDecimal totalNumberBigDecimal = issueSize.divide(entry.getBigDecimal("issue_price"), 2, BigDecimal.ROUND_HALF_UP);
|
||||
totalNumber = totalNumberBigDecimal.intValue();
|
||||
}
|
||||
stockIpo.setTotalNumber(totalNumber);
|
||||
stockIpo.setApply(entry.getString("total_subs"));
|
||||
stockIpo.setCreateDate(new Date());
|
||||
stockIpo.setUpdateDate(new Date());
|
||||
stockIpo.setSourceType("3");
|
||||
|
||||
if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
|
||||
continue;
|
||||
}
|
||||
|
||||
List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
|
||||
|
||||
for (String exchange : exchanges) {
|
||||
StockIpo ipo = new StockIpo();
|
||||
ipo.setStockCode(stockIpo.getStockCode());
|
||||
ipo.setStockName(stockIpo.getStockName());
|
||||
ipo.setStockPrice(stockIpo.getStockPrice());
|
||||
ipo.setSubscriptionDate(stockIpo.getSubscriptionDate());
|
||||
ipo.setListingDate(stockIpo.getListingDate());
|
||||
ipo.setTotalNumber(stockIpo.getTotalNumber());
|
||||
ipo.setApply(stockIpo.getApply());
|
||||
ipo.setCreateDate(stockIpo.getCreateDate());
|
||||
ipo.setUpdateDate(stockIpo.getUpdateDate());
|
||||
ipo.setExchangeType(exchange);
|
||||
ipo.setSourceType("3");
|
||||
listStockIpoList.add(ipo);
|
||||
}
|
||||
}
|
||||
stockIpo.setTotalNumber(totalNumber);
|
||||
stockIpo.setApply(entry.getString("total_subs"));
|
||||
stockIpo.setCreateDate(new Date());
|
||||
stockIpo.setUpdateDate(new Date());
|
||||
stockIpo.setSourceType("3");
|
||||
listStockIpoList.add(stockIpo);
|
||||
}
|
||||
|
||||
// stockIpoRepository.saveAll(listStockIpoList);
|
||||
// stockIpoRepository.saveAll(listStockIpoList);
|
||||
|
||||
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
|
||||
if(CollectionUtil.isNotEmpty(scIdList)){
|
||||
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
|
||||
List<String> existingStockScIds = existStockIpoList.stream()
|
||||
.map(StockIpo::getStockCode)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
listStockIpoList = listStockIpoList.stream()
|
||||
.filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
//保存全部的新股
|
||||
if(CollectionUtil.isNotEmpty(listStockIpoList)){
|
||||
stockIpoRepository.saveAll(listStockIpoList);
|
||||
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
|
||||
}else {
|
||||
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
|
||||
if (!listStockIpoList.isEmpty())
|
||||
{
|
||||
listStockIpoList = listStockIpoList.stream().filter(a -> StringUtils.isNotBlank(a.getStockName())).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
List<String> scIdList = Lists.transform(listStockIpoList, StockIpo::getStockCode);
|
||||
if(CollectionUtil.isNotEmpty(scIdList)){
|
||||
List<StockIpo> existStockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.stockCode.in(scIdList));
|
||||
List<String> existingStockScIds = existStockIpoList.stream()
|
||||
.map(StockIpo::getStockCode)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (StockIpo stockIpo : listStockIpoList) {
|
||||
StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().toLowerCase().trim().equals(stockIpo.getExchangeType().toLowerCase().trim()))).findFirst().orElse(null);
|
||||
if (existIpo != null) {
|
||||
if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
|
||||
existIpo.setExchangeType(stockIpo.getExchangeType());
|
||||
}
|
||||
stockIpo.setId(existIpo.getId());
|
||||
if (existIpo.getStockPrice() != null){
|
||||
stockIpo.setStockPrice(existIpo.getStockPrice());
|
||||
stockIpo.setApply(existIpo.getApply());
|
||||
}
|
||||
}
|
||||
}
|
||||
// listStockIpoList = listStockIpoList.stream()
|
||||
// .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
|
||||
// .collect(Collectors.toList());
|
||||
|
||||
//保存全部的新股
|
||||
if(CollectionUtil.isNotEmpty(listStockIpoList)){
|
||||
stockIpoRepository.saveAll(listStockIpoList);
|
||||
|
||||
log.info("定时任务执行获取新股ipo的方法开始结束,保存了数据:{}", JSON.toJSONString(listStockIpoList));
|
||||
}else {
|
||||
log.info("定时任务执行获取新股ipo的方法开始结束,没有数据");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 输出整个 JSON 数据
|
||||
} else {
|
||||
log.info("未找到包含 JSON 数据的 <script> 标签");
|
||||
}
|
||||
|
||||
|
||||
// 输出整个 JSON 数据
|
||||
} else {
|
||||
log.info("未找到包含 JSON 数据的 <script> 标签");
|
||||
log.info("HTTP请求失败,状态码:" + response.getStatusLine().getStatusCode());
|
||||
}
|
||||
} else {
|
||||
log.info("HTTP请求失败,状态码:" + response.getStatusLine().getStatusCode());
|
||||
} catch (IOException e) {
|
||||
log.error("获取新股接口发生异常",e);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("获取新股接口发生异常",e);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Scheduled(cron = "0 0 19 * * ?")
|
||||
@PostConstruct
|
||||
@RequestMapping("/addMoneyStock")
|
||||
public void addMoneyStock() {
|
||||
log.info("定时任务执行获取新股ipo的方法开始执行");
|
||||
|
||||
List<StockIpo> stockIpoList;
|
||||
|
||||
if(!times){
|
||||
stockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.listingDate.isNotNull()
|
||||
.and(QStockIpoPO.stockIpoPO.listingDate.loe(DateTime.now()))
|
||||
.and(QStockIpoPO.stockIpoPO.exchangeType.isNotNull())
|
||||
.and(QStockIpoPO.stockIpoPO.exchangeType.isNotEmpty())
|
||||
); // first time run
|
||||
|
||||
times = true;
|
||||
}
|
||||
else {
|
||||
stockIpoList = stockIpoRepository.findAll(QStockIpoPO.stockIpoPO.listingDate.isNotNull()
|
||||
.and(QStockIpoPO.stockIpoPO.listingDate.loe(DateTime.now())
|
||||
.and(QStockIpoPO.stockIpoPO.listingDate.goe(DateUtil.offsetDay(DateTime.now(), -3))))
|
||||
.and(QStockIpoPO.stockIpoPO.exchangeType.isNotNull())
|
||||
.and(QStockIpoPO.stockIpoPO.exchangeType.isNotEmpty())
|
||||
); // from second run
|
||||
}
|
||||
|
||||
List<MoneyStock> moneyStockList = convertToMoneyStock(stockIpoList);
|
||||
moneyStockRepository.saveAll(moneyStockList);
|
||||
}
|
||||
|
||||
|
||||
@@ -236,12 +344,6 @@ public class Scraper {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private void processSubMap(Map<String, String> sefUrlList) {
|
||||
for (Map.Entry<String, String> entry : sefUrlList.entrySet()) {
|
||||
String companyName = entry.getKey();
|
||||
@@ -378,4 +480,135 @@ public class Scraper {
|
||||
log.info("Getting web detail URL: " + url + "...");
|
||||
return Jsoup.connect(url).ignoreContentType(true).execute().body();
|
||||
}
|
||||
|
||||
public List<String> getIpoExchange(String stockName, String stockCode) {
|
||||
List<String> exchanges = new ArrayList<>();
|
||||
String url = "https://www.moneycontrol.com/ipo/{stockName}-{stockCode}-ipodetail";
|
||||
url = url.replace("{stockName}", stockName.toLowerCase().replace(" ", "-")).replace("{stockCode}", stockCode.toLowerCase());
|
||||
|
||||
org.apache.http.client.HttpClient client = HttpClients.createDefault();
|
||||
HttpGet request = new HttpGet(url);
|
||||
try {
|
||||
org.apache.http.HttpResponse response = client.execute(request);
|
||||
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
String responseBody = EntityUtils.toString(response.getEntity());
|
||||
Document doc = Jsoup.parse(responseBody);
|
||||
|
||||
Element scriptTag = doc.selectFirst("script#__NEXT_DATA__");
|
||||
|
||||
if (scriptTag != null) {
|
||||
String jsonDataStr = scriptTag.html();
|
||||
|
||||
com.alibaba.fastjson.JSONObject jsonObject = com.alibaba.fastjson.JSONObject.parseObject(jsonDataStr);
|
||||
log.info("获取到新股的json信息:" + jsonObject.toJSONString());
|
||||
|
||||
com.alibaba.fastjson.JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
|
||||
com.alibaba.fastjson.JSONObject overview = pageProps.getJSONObject("ipoDetails").getJSONObject("ipoDetails").getJSONObject("overview");
|
||||
|
||||
String exchangeStr = overview.getString("listing_on");
|
||||
|
||||
if (exchangeStr != null && !exchangeStr.isEmpty() && !exchangeStr.equals("null")) {
|
||||
exchanges.addAll(new ArrayList<>(Arrays.asList(exchangeStr.split(","))));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
log.info(e.getMessage());
|
||||
}
|
||||
|
||||
if (exchanges.isEmpty()) {
|
||||
exchanges.add("");
|
||||
}
|
||||
|
||||
return exchanges;
|
||||
}
|
||||
|
||||
public List<MoneyStock> convertToMoneyStock(List<StockIpo> stockIpoList) {
|
||||
// Lấy danh sách tất cả stockCode và exchangeType từ stockIpoList
|
||||
Set<String> stockCodes = stockIpoList.stream().map(StockIpo::getStockCode).collect(Collectors.toSet());
|
||||
// Set<String> exchangeTypes = stockIpoList.stream().map(StockIpo::getExchangeType).collect(Collectors.toSet());
|
||||
Set<String> exchangeTypes = stockIpoList.stream().map(stockIpo -> stockIpo.getExchangeType().toUpperCase()).collect(Collectors.toSet());
|
||||
|
||||
// Lấy tất cả các MoneyStock tồn tại trong database với stockCode và exchangeType trong stockIpoList
|
||||
List<MoneyStock> existingMoneyStocks = moneyStockRepository.findAll(
|
||||
QMoneyStockPO.moneyStockPO.moneyScId.in(stockCodes)
|
||||
.and(QMoneyStockPO.moneyStockPO.stockType.upper().in(exchangeTypes))
|
||||
);
|
||||
|
||||
// Lưu các stockCode và exchangeType đã tồn tại
|
||||
Set<String> existingKeys = existingMoneyStocks.stream()
|
||||
.map(ms -> ms.getMoneyScId() + ms.getStockType().toUpperCase())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
// Chỉ giữ những StockIpo chưa tồn tại
|
||||
return stockIpoList.stream()
|
||||
.filter(stockIpo -> !existingKeys.contains(stockIpo.getStockCode() + stockIpo.getExchangeType().toUpperCase()))
|
||||
.map(this::convert)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
// public List<MoneyStock> convertToMoneyStock(List<StockIpo> stockIpoList) {
|
||||
// return stockIpoList.stream()
|
||||
// .filter(stockIpo -> !moneyStockRepository.exists(
|
||||
// QMoneyStockPO.moneyStockPO.stockType.eq(stockIpo.getExchangeType())
|
||||
// .and(QMoneyStockPO.moneyStockPO.moneyScId.eq(stockIpo.getStockCode()))
|
||||
// )) // Chỉ giữ những StockIpo chưa tồn tại
|
||||
// .map(this::convert)
|
||||
// .collect(Collectors.toList());
|
||||
// }
|
||||
|
||||
private MoneyStock convert(StockIpo stockIpo) {
|
||||
MoneyStock moneyStock = new MoneyStock();
|
||||
|
||||
moneyStock.setStockName(stockIpo.getStockName());
|
||||
moneyStock.setMoneyScId(stockIpo.getStockCode());
|
||||
moneyStock.setSelfDispId(stockIpo.getStockCode());
|
||||
moneyStock.setStockType(stockIpo.getExchangeType().toUpperCase());
|
||||
moneyStock.setSaveTime(DateTime.now());
|
||||
moneyStock.setNseIndiaId(getNSEId(stockIpo));
|
||||
|
||||
return moneyStock;
|
||||
}
|
||||
|
||||
private String getNSEId(StockIpo stockIpo){
|
||||
String url = "https://priceapi.moneycontrol.com/pricefeed/" + stockIpo.getExchangeType().toLowerCase() + "/equitycash/" + stockIpo.getStockCode();
|
||||
// 创建 HttpClient 实例
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
// 创建 HttpGet 请求
|
||||
HttpGet request = new HttpGet(url);
|
||||
try {
|
||||
// 执行请求
|
||||
HttpResponse response = client.execute(request);
|
||||
|
||||
// 检查请求是否成功
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
// 获取响应体
|
||||
String responseBody = EntityUtils.toString(response.getEntity());
|
||||
|
||||
if (responseBody != null) {
|
||||
// 获取 JSON 数据
|
||||
|
||||
// 将 JSON 字符串解析为 Java JSONObject
|
||||
JSONObject jsonObject = JSONObject.parseObject(responseBody);
|
||||
log.info("获取到新股的json信息:" + jsonObject.toJSONString());
|
||||
|
||||
JSONObject jsonData = jsonObject.getJSONObject("data");
|
||||
|
||||
if(stockIpo.getExchangeType().toUpperCase(Locale.ROOT).equals("NSE")){
|
||||
return jsonData.getString("NSEID");
|
||||
}
|
||||
else if(stockIpo.getExchangeType().toUpperCase(Locale.ROOT).equals("BSE")){
|
||||
return jsonData.getString("BSEID");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e){
|
||||
log.info(e.getMessage());
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
package cn.stock.market.infrastructure.job;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import cn.qutaojing.common.utils.SpringUtils;
|
||||
import cn.stock.market.domain.basic.entity.TwelveStock;
|
||||
import cn.stock.market.domain.basic.service.TwelveStockService;
|
||||
import cn.stock.market.dto.TwelveStockData;
|
||||
import cn.stock.market.utils.HttpRequest;
|
||||
import cn.stock.market.web.config.Config;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
@RestController
|
||||
public class TwelvedataTask {
|
||||
|
||||
@Autowired
|
||||
private TwelveStockService twelveStockService;
|
||||
|
||||
@GetMapping("/task/syncStockData")
|
||||
@Scheduled(cron = "0 0 18 * * ?")
|
||||
public void syncStockData() {
|
||||
log.info("每天18点定时同步Twelvedata股票数据开始");
|
||||
Stopwatch stopwatch = Stopwatch.createStarted();
|
||||
// 获取股票symbol列表
|
||||
List<TwelveStockData> stockList = getStockList();
|
||||
// 查询出数据库中所有股票
|
||||
List<TwelveStock> allStocks = twelveStockService.repository().findAll();
|
||||
List<String> savedSymbols = allStocks.stream().map(p-> p.getStockCode() + "#" + p.getStockType()).distinct().collect(Collectors.toList());
|
||||
// 去除数据库中已经存在的股票
|
||||
// 使用Iterator安全地移除元素
|
||||
if(stockList!=null && stockList.size()>0) {
|
||||
Iterator<TwelveStockData> iterator = stockList.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
TwelveStockData stockData = iterator.next();
|
||||
if (savedSymbols.contains(stockData.getSymbol() + "#" + stockData.getExchange())) { // 根据条件移除元素
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
//剩余数据入库
|
||||
List<TwelveStock> addList = Lists.newArrayList();
|
||||
if(stockList!=null && stockList.size()>0) {
|
||||
for(TwelveStockData stockData:stockList) {
|
||||
addList.add(cover(stockData));
|
||||
}
|
||||
twelveStockService.repository().saveAll(addList);
|
||||
}
|
||||
log.info("syncStockData执行, 受影响数{}, 耗时:{}毫秒", addList.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
|
||||
}
|
||||
|
||||
private TwelveStock cover(TwelveStockData data) {
|
||||
TwelveStock stock = new TwelveStock();
|
||||
stock.setStockCode(data.getSymbol());
|
||||
stock.setStockName(data.getName());
|
||||
stock.setStockType(data.getExchange());
|
||||
stock.setSaveTime(new Date());
|
||||
return stock;
|
||||
}
|
||||
|
||||
private List<TwelveStockData> getStockList() {
|
||||
try {
|
||||
String source = "docs";
|
||||
String country = "India";
|
||||
List<TwelveStockData> list = Lists.newArrayList();
|
||||
long start = System.currentTimeMillis();
|
||||
Config config = SpringUtils.getBean(Config.class);
|
||||
String url = String.format("https://api.twelvedata.com/stocks?source=%s&country=%s&apikey=%s",source,country,config.getTwelveApikey());
|
||||
String result = HttpRequest.doTwelveGet(url);
|
||||
log.info("getStockList:获取 {} {} 的股票全量列表结束,耗时: {} ms ",source,country,(System.currentTimeMillis()-start));
|
||||
JSONObject jsonObject = JSONObject.parseObject(result);
|
||||
if (jsonObject != null && !jsonObject.isEmpty()) {
|
||||
JSONArray jsonArray = jsonObject.getJSONArray("data");
|
||||
if(jsonArray!=null && jsonArray.size()>0) {
|
||||
for (int i = 0; i < jsonArray.size(); i++) {
|
||||
JSONObject item = jsonArray.getJSONObject(i);
|
||||
TwelveStockData stock = JSONObject.toJavaObject(item, TwelveStockData.class);
|
||||
list.add(stock);
|
||||
}
|
||||
}
|
||||
}
|
||||
return list;
|
||||
} catch (Exception e) {
|
||||
// TODO Auto-generated catch block
|
||||
log.error("getStockList同步twelvedata股票数据异常,Exception:{}", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import java.sql.Timestamp;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
import java.util.Calendar;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
@@ -186,6 +188,25 @@ public class DateTimeUtil {
|
||||
}
|
||||
|
||||
|
||||
public static String formatDateTimeFor24H(String inputDateTime,String inputFormatStr){
|
||||
String outputDateTime = "";
|
||||
// 1. 解析日期时间
|
||||
SimpleDateFormat inputFormat = new SimpleDateFormat(inputFormatStr, Locale.ENGLISH);
|
||||
Date date = null;
|
||||
try {
|
||||
date = inputFormat.parse(inputDateTime);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
if (date != null) {
|
||||
// 2. 转换为24小时制
|
||||
SimpleDateFormat outputFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
outputDateTime = outputFormat.format(date);
|
||||
}
|
||||
return outputDateTime;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
parseToDateByMinute(10);
|
||||
}
|
||||
|
||||
@@ -265,6 +265,56 @@ public class HttpClientRequest {
|
||||
return result;
|
||||
}
|
||||
|
||||
public static String doGetNews(String url) {
|
||||
CloseableHttpClient httpClient = null;
|
||||
CloseableHttpResponse response = null;
|
||||
String result = "";
|
||||
|
||||
try {
|
||||
httpClient = HttpClients.createDefault();
|
||||
|
||||
HttpGet httpGet = new HttpGet(url);
|
||||
|
||||
// Spoof real browser headers
|
||||
httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
|
||||
httpGet.setHeader("Accept-Language", "en-US,en;q=0.9,vi;q=0.8,ug;q=0.7,fr;q=0.6");
|
||||
httpGet.setHeader("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1");
|
||||
httpGet.setHeader("Priority", "u=0, i");
|
||||
httpGet.setHeader("Sec-Fetch-Dest", "document");
|
||||
httpGet.setHeader("Sec-Fetch-Mode", "navigate");
|
||||
httpGet.setHeader("Sec-Fetch-Site", "none");
|
||||
httpGet.setHeader("Sec-Fetch-User", "?1");
|
||||
httpGet.setHeader("Upgrade-Insecure-Requests", "1");
|
||||
|
||||
// Set cookies exactly like in curl
|
||||
httpGet.setHeader("Cookie", "userUid=1747102033185-d570fba9-62fd-40be-93ca-ed08b4de57d4; _sid=MTc0NzEwMjAzMzE4Ni4ycTU%3D; _scor_uid=135c13065ff84620b5318b489af93e87; _gcl_au=1.1.249135292.1747102036; _ga=GA1.1.1031614211.1747102037; WZRK_G=28895afb56ff48dda59fe8de0af746bf; FCNEC=%5B%5B%22AKsRol8sHYeSYz_FYPkInYXN3P4ZDPfVKbsRfILfDuOMLhDtkTuoCJP5MlvT9gIbOe7IlDfY8ZeHszhwdVtAoKF1gWv0pLAq5EqpLpse8CEm_ZNv-bUSs6zEyqpOkeKWFI_Ei6VfNAvnZAO8PcXdF8_ncsaO902X7g%3D%3D%22%5D%5D; _ga_KRGL1M61LX=GS2.1.s1747105135$o2$g0$t1747105135$j60$l0$h0");
|
||||
|
||||
RequestConfig requestConfig = RequestConfig.custom()
|
||||
.setConnectTimeout(10000)
|
||||
.setSocketTimeout(15000)
|
||||
.setConnectionRequestTimeout(10000)
|
||||
.build();
|
||||
httpGet.setConfig(requestConfig);
|
||||
|
||||
response = httpClient.execute(httpGet);
|
||||
HttpEntity entity = response.getEntity();
|
||||
|
||||
if (entity != null) {
|
||||
result = EntityUtils.toString(entity);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
if (response != null) response.close();
|
||||
if (httpClient != null) httpClient.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
String url = "https://marketapi.intoday.in/widget/topgainer/view?exchange=nse";
|
||||
String str = doGet(url);
|
||||
|
||||
@@ -2,7 +2,6 @@ package cn.stock.market.utils;
|
||||
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.HttpURLConnection;
|
||||
@@ -143,52 +142,6 @@ public class HttpRequest {
|
||||
return text;
|
||||
}
|
||||
|
||||
public static String doTwelveGet(String url) throws Exception{
|
||||
int maxRetries = 3;
|
||||
for (int retry = 1; retry <= maxRetries; retry++) {
|
||||
BufferedReader reader = null;
|
||||
StringBuilder response = new StringBuilder();
|
||||
try {
|
||||
URL obj = new URL(url);
|
||||
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
|
||||
con.setRequestMethod("GET");
|
||||
con.setRequestProperty("Accept", "application/json");
|
||||
con.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
|
||||
//模拟postman请求,否则返回403
|
||||
con.setRequestProperty("User-Agent", "PostmanRuntime/7.26.8");
|
||||
|
||||
int responseCode = con.getResponseCode();
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
reader = new BufferedReader(new InputStreamReader(con.getInputStream()));
|
||||
String inputLine;
|
||||
while ((inputLine = reader.readLine()) != null) {
|
||||
response.append(inputLine);
|
||||
}
|
||||
reader.close();
|
||||
return response.toString();
|
||||
} else {
|
||||
log.error("HTTP request failed with response code: " + responseCode);
|
||||
}
|
||||
}catch(Exception e) {
|
||||
log.error("请求twelvedata数据url:{},exception:{} ", url,e);
|
||||
}finally {
|
||||
if (reader != null) {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
// 如果不是最后一次重试,则等待一段时间再进行下一次重试
|
||||
if (retry < maxRetries) {
|
||||
try {
|
||||
// 1秒钟
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
String url = "";
|
||||
|
||||
181
src/main/java/cn/stock/market/utils/NseIndiaRequest.java
Normal file
181
src/main/java/cn/stock/market/utils/NseIndiaRequest.java
Normal file
@@ -0,0 +1,181 @@
|
||||
package cn.stock.market.utils;
|
||||
|
||||
import cn.stock.market.dto.StockHistoryRequest;
|
||||
import cn.stock.market.dto.StockHistoryResponse;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import okhttp3.*;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class NseIndiaRequest {
|
||||
private static final String NSE_INDIA_URL = "https://www.nseindia.com";
|
||||
private static final String NSE_INDIA_CHART_URL = "https://charting.nseindia.com";
|
||||
private static final OkHttpClient client;
|
||||
private static final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
static {
|
||||
client = new OkHttpClient.Builder()
|
||||
.cookieJar(new CookieJar() {
|
||||
private final Map<String, List<Cookie>> cookieStore = new HashMap<>();
|
||||
|
||||
@Override
|
||||
public void saveFromResponse(HttpUrl url, List<Cookie> cookies) {
|
||||
cookieStore.put(url.host(), cookies);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Cookie> loadForRequest(HttpUrl url) {
|
||||
List<Cookie> cookies = cookieStore.get(url.host());
|
||||
return cookies != null ? cookies : new ArrayList<Cookie>();
|
||||
}
|
||||
})
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Request createRequest(String url) {
|
||||
Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.header("accept", "application/json, text/plain, */*")
|
||||
.header("accept-language", "en-US,en;q=0.9")
|
||||
.header("cache-control", "no-cache")
|
||||
.header("pragma", "no-cache")
|
||||
.header("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
|
||||
.build();
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
private static void initCookie(String url) {
|
||||
Request request = createRequest(url);
|
||||
try (Response response = client.newCall(request).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
throw new IOException("Failed to fetch initial cookies");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to initialize cookies", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static Integer getCode(String symbol) {
|
||||
Request request = createRequest(NSE_INDIA_CHART_URL + "//Charts/GetEQMasters").newBuilder()
|
||||
.addHeader("referer", NSE_INDIA_CHART_URL)
|
||||
.addHeader("origin", NSE_INDIA_CHART_URL)
|
||||
.build();
|
||||
|
||||
try (Response response = client.newCall(request).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
throw new IOException("Failed to get EQ code");
|
||||
}
|
||||
|
||||
String result = response.body().string();
|
||||
|
||||
String regex = "(\\d+)\\|" + symbol + "\\|.*";
|
||||
Pattern pattern = Pattern.compile(regex);
|
||||
Matcher matcher = pattern.matcher(result);
|
||||
|
||||
if (matcher.find()) {
|
||||
return Integer.valueOf(matcher.group(1));
|
||||
}
|
||||
throw new IOException("No data found");
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to get EQ code", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static JSONObject stockByJYSFromHttp(String stockType, String symbol, String nseIndiaId) {
|
||||
initCookie(NSE_INDIA_URL);
|
||||
|
||||
String url = NSE_INDIA_URL + "/api/quote-equity?symbol=" + nseIndiaId;
|
||||
Request request = createRequest(url).newBuilder()
|
||||
.addHeader("referer", NSE_INDIA_URL)
|
||||
.addHeader("origin", NSE_INDIA_URL)
|
||||
.build();
|
||||
try (Response response = client.newCall(request).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
throw new IOException("Request failed with code: " + response.code());
|
||||
}
|
||||
|
||||
JSONObject jsonData = JSONObject.parseObject(response.body().string());
|
||||
JSONObject data =jsonData.getJSONObject("priceInfo");
|
||||
JSONObject json = new JSONObject();
|
||||
|
||||
json.put("pricepercentchange",data.getString("pChange"));
|
||||
json.put("stockType",stockType);
|
||||
json.put("pricechange",data.getString("change"));
|
||||
json.put("pricecurrent",data.getString("lastPrice"));
|
||||
json.put("priceprevclose",data.getString("previousClose"));
|
||||
json.put("PREVDATE","");
|
||||
json.put("VOL",jsonData.getJSONObject("preOpenMarket").getString("totalTradedVolume"));
|
||||
json.put("dataSourceType","3");
|
||||
json.put("symbol",symbol);
|
||||
json.put("BSEID",symbol);
|
||||
json.put("NSEID",symbol);
|
||||
json.put("LTH",data.getString("upperCP"));
|
||||
json.put("LTL",data.getString("lowerCP"));
|
||||
json.put("OPN",data.getString("open"));
|
||||
|
||||
return json;
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to fetch data", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static StockHistoryResponse stockKLineFromHttp(StockHistoryRequest stockHistoryRequest, String resolution) {
|
||||
initCookie(NSE_INDIA_CHART_URL);
|
||||
|
||||
Integer code = getCode(stockHistoryRequest.getSymbol());
|
||||
|
||||
int interval = 1;
|
||||
if (StringUtils.equals("H", resolution)) {
|
||||
resolution = "I";
|
||||
interval = 60;
|
||||
}
|
||||
|
||||
Map<String, Object> body = new HashMap<>();
|
||||
body.put("chartPeriod", resolution);
|
||||
body.put("chartStart", 0);
|
||||
body.put("exch", "N");
|
||||
body.put("fromDate", 0);
|
||||
body.put("instrType", "C");
|
||||
body.put("scripCode", code);
|
||||
body.put("timeInterval", interval);
|
||||
body.put("toDate", stockHistoryRequest.getTo() + 18000);
|
||||
body.put("ulToken", code);
|
||||
|
||||
String payload;
|
||||
try {
|
||||
payload = objectMapper.writeValueAsString(body);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Failed to serialize body", e);
|
||||
}
|
||||
|
||||
RequestBody requestBody = RequestBody.create(
|
||||
MediaType.get("application/json; charset=utf-8"),
|
||||
payload
|
||||
);
|
||||
|
||||
Request request = createRequest(NSE_INDIA_CHART_URL + "//Charts/symbolhistoricaldata/").newBuilder()
|
||||
.addHeader("referer", NSE_INDIA_CHART_URL)
|
||||
.addHeader("origin", NSE_INDIA_CHART_URL)
|
||||
.post(requestBody)
|
||||
.build();
|
||||
|
||||
try (Response response = client.newCall(request).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
throw new IOException("Request failed with code: " + response.code());
|
||||
}
|
||||
|
||||
StockHistoryResponse result = objectMapper.readValue(response.body().string(), StockHistoryResponse.class);
|
||||
|
||||
return result;
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Failed to fetch data", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.text.DecimalFormat;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -159,6 +160,19 @@ public class BTodayStockController {
|
||||
|
||||
String forObject = restTemplate.getForObject(apiUrl, String.class);
|
||||
JSONObject jsonObject = JSON.parseObject(forObject);
|
||||
JSONArray arryData = jsonObject.getJSONArray("data");
|
||||
if (arryData != null&&arryData.size()>0) {
|
||||
try {
|
||||
JSONObject jsonObject1 = arryData.getJSONObject(0);
|
||||
String price = jsonObject1.getString("price");
|
||||
double priceValue = Double.parseDouble(price);
|
||||
DecimalFormat df = new DecimalFormat("0.00");
|
||||
jsonObject1.put("price", df.format(priceValue));
|
||||
} catch (NumberFormatException e) {
|
||||
// Handle the case where price is not a valid double
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
jsonObject.put("id",btodayStock.getId());
|
||||
return jsonObject;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -26,13 +26,14 @@ public class MoneyProxyApiControl {
|
||||
|
||||
@GetMapping({"/market/api/proxy/queryStockMoneyProxy","/api/proxy/queryStockMoneyProxy"})
|
||||
public JSONObject queryStockByMoneyIdAndExchangeType(String scId,String exchangeType){
|
||||
log.info("queryStockByMoneyIdAndExchangeType");
|
||||
return stockByMoneyIdAndExchangeTypeFromHttp(scId,exchangeType);
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject stockByMoneyIdAndExchangeTypeFromHttp(String scId, String exchangeType) {
|
||||
try {
|
||||
String url = "https://priceapi.moneycontrol.com/pricefeed/"+exchangeType+"/equitycash/" + scId ;
|
||||
String url = "https://moneycontrol.titanrwa.org/pricefeed/"+exchangeType+"/equitycash/" + scId ;
|
||||
URL obj = new URL(url);
|
||||
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
|
||||
con.setRequestMethod("GET");
|
||||
@@ -53,7 +54,8 @@ public class MoneyProxyApiControl {
|
||||
log.error("HTTP request failed with response code: " + responseCode);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("stockByMoneyIdAndExchangeTypeFromHttp",e);
|
||||
e.printStackTrace();
|
||||
log.error("stockByMoneyIdAndExchangeTypeFromHttp failed {}",e.getMessage());
|
||||
}
|
||||
return new JSONObject();
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package cn.stock.market.web;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
@@ -12,6 +14,7 @@ import cn.qutaojing.common.jpa.ConditionBuilder;
|
||||
import cn.stock.market.domain.basic.entity.SiteNews;
|
||||
import cn.stock.market.domain.basic.repository.SiteNewsRepository;
|
||||
import cn.stock.market.infrastructure.db.po.QSiteNewsPO;
|
||||
import cn.stock.market.infrastructure.job.InvestingTask;
|
||||
import cn.stock.market.web.annotations.EncryptFilter;
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiImplicitParam;
|
||||
@@ -169,15 +172,22 @@ public class StockApiController {
|
||||
|
||||
@RequestMapping({"test.do"})
|
||||
@ResponseBody
|
||||
public ServerResponse test(@RequestParam("url") String url, @RequestParam("pageNum") Integer pageNum) {
|
||||
public ServerResponse test(/*@RequestParam("url") String url, @RequestParam("pageNum") Integer pageNum*/) {
|
||||
String news = stockService.getNews();
|
||||
List<String> newsList = Arrays.asList(news.split("<a href="));
|
||||
newsList = newsList.subList(1, newsList.size());
|
||||
newsList.forEach( n -> {
|
||||
String contentUrl = n.substring(1, n.indexOf("class=\"img-smllnews\"") - 2);
|
||||
String id = contentUrl.substring(contentUrl.lastIndexOf("-") + 1, contentUrl.lastIndexOf("_"));
|
||||
String imgUrl = n.substring(n.indexOf("img loading=\"lazy\" src=") + 24, n.indexOf("?"));
|
||||
String time = n.substring(n.indexOf("Last Updated") + 23, n.indexOf("IST") - 9);
|
||||
String imgUrl = InvestingTask.extractImgSrc(n);
|
||||
// String time = n.substring(n.indexOf("Last Updated") + 23, n.indexOf("IST") - 9);
|
||||
// Extract the date and time using regex
|
||||
Pattern pattern = Pattern.compile("Updated On :<!-- --> <!-- -->(.*?)<!-- -->");
|
||||
Matcher matcher = pattern.matcher(n);
|
||||
String time = "";
|
||||
if (matcher.find()) {
|
||||
time = matcher.group(1).trim();
|
||||
}
|
||||
|
||||
SiteNews siteNews = new SiteNews();
|
||||
siteNews.setAddTime(new Date());
|
||||
@@ -238,7 +248,7 @@ public class StockApiController {
|
||||
@ResponseBody
|
||||
public ServerResponse getIndiaIndexByToday() {
|
||||
String INDEX_CODE = "TODAY_INDEX";
|
||||
return RequestCacheUtils.cache("getIndiaIndexByToday.do", INDEX_CODE,60000, (string) -> {
|
||||
return RequestCacheUtils.cache("getIndiaIndexByToday.do", INDEX_CODE,6000, (string) -> {
|
||||
return this.stockService.getIndexByBtoday();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
package cn.stock.market.web;
|
||||
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiImplicitParam;
|
||||
import io.swagger.annotations.ApiImplicitParams;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.github.pagehelper.StringUtil;
|
||||
import com.google.common.cache.Cache;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import cn.qutaojing.common.utils.SpringUtils;
|
||||
import cn.stock.market.dto.StockHistoryResponse;
|
||||
import cn.stock.market.dto.TwelveStockInfo;
|
||||
import cn.stock.market.utils.HttpRequest;
|
||||
import cn.stock.market.utils.ServerResponse;
|
||||
import cn.stock.market.web.annotations.EncryptFilter;
|
||||
import cn.stock.market.web.config.Config;
|
||||
|
||||
/**
|
||||
* twelvedata数据源接口
|
||||
*
|
||||
*/
|
||||
@RestController
|
||||
@Api(value = "/TwelveApiController", tags = "twelvedata股票行情")
|
||||
@Slf4j
|
||||
@RequestMapping({"/api/market/twelvedata", "/api/hq/twelvedata"})
|
||||
public class TwelveApiController {
|
||||
|
||||
@ApiOperation(value = "股票详情信息", httpMethod = "GET")
|
||||
@ApiImplicitParams({
|
||||
@ApiImplicitParam(name = "stockCode",value = "股票对应代码",dataType ="String",required = true, paramType = "query"),
|
||||
@ApiImplicitParam(name = "stockType",value = "股票对应代码:BSE/NSE",dataType ="String",required = true, paramType = "query")
|
||||
})
|
||||
@GetMapping("/getStockDetail")
|
||||
@EncryptFilter(decryptRequest = false)
|
||||
public ServerResponse<?> getStockDetail(String stockCode,String stockType) {
|
||||
try {
|
||||
long start = System.currentTimeMillis();
|
||||
Config config = SpringUtils.getBean(Config.class);
|
||||
String url = String.format("https://api.twelvedata.com/quote?symbol=%s&exchange=%s&apikey=%s",stockCode,stockType,config.getTwelveApikey());
|
||||
String result = HttpRequest.doTwelveGet(url);
|
||||
if(StringUtil.isNotEmpty(result)) {
|
||||
JSONObject data = JSONObject.parseObject(result);
|
||||
TwelveStockInfo info = mapJsonToTwelveStock(data);
|
||||
log.info("getStockDetail:获取 {}, {} 的股票详情结束,耗时: {} ms ",stockCode,stockType,(System.currentTimeMillis()-start));
|
||||
return ServerResponse.createBySuccess("操作成功", info);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("获取股票详情参数:{},{}. Exception:{}" , stockCode,stockType, e);
|
||||
}
|
||||
return ServerResponse.createBySuccess("操作成功");
|
||||
}
|
||||
|
||||
private TwelveStockInfo mapJsonToTwelveStock(JSONObject jsonObject) {
|
||||
TwelveStockInfo info = new TwelveStockInfo();
|
||||
if (jsonObject != null && !jsonObject.isEmpty()) {
|
||||
info.setStockCode(jsonObject.getString("symbol"));
|
||||
info.setSymbol(jsonObject.getString("symbol"));
|
||||
info.setStockName(jsonObject.getString("name"));
|
||||
info.setStockType(jsonObject.getString("exchange"));
|
||||
info.setLastPrice(jsonObject.getString("close"));
|
||||
info.setPerchg(jsonObject.getString("percent_change"));
|
||||
info.setChange(jsonObject.getString("change"));
|
||||
info.setOpenPrice(jsonObject.getString("open"));
|
||||
info.setPreviousPrice(jsonObject.getString("previous_close"));
|
||||
info.setVolume(jsonObject.getString("volume"));
|
||||
info.setHighPrice(jsonObject.getString("high"));
|
||||
info.setLowPrice(jsonObject.getString("low"));
|
||||
JSONObject json = jsonObject.getJSONObject("fifty_two_week");
|
||||
if(json!=null && !json.isEmpty()) {
|
||||
info.setWeek52HighPrice(json.getString("high"));
|
||||
info.setWeek52LowPrice(json.getString("low"));
|
||||
}
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
@ApiOperation(value = "股票详情列表", httpMethod = "GET")
|
||||
@ApiImplicitParams({
|
||||
@ApiImplicitParam(name = "stockCodes",value = "股票对应代码, 多个时以英文逗号分隔",dataType ="String",required = true, paramType = "query"),
|
||||
@ApiImplicitParam(name = "stockType",value = "股票对应代码:BSE/NSE",dataType ="String",required = true, paramType = "query")
|
||||
})
|
||||
@GetMapping("/getStockDetailList")
|
||||
@EncryptFilter(decryptRequest = false)
|
||||
public ServerResponse<?> getStockDetailList(String stockCodes,String stockType) {
|
||||
try {
|
||||
long start = System.currentTimeMillis();
|
||||
Config config = SpringUtils.getBean(Config.class);
|
||||
String url = String.format("https://api.twelvedata.com/quote?symbol=%s&exchange=%s&apikey=%s",stockCodes,stockType,config.getTwelveApikey());
|
||||
String result = HttpRequest.doTwelveGet(url);
|
||||
if(StringUtil.isNotEmpty(result)) {
|
||||
JSONObject data = JSONObject.parseObject(result);
|
||||
List<TwelveStockInfo> list = mapJsonToTwelveList(stockCodes,data);
|
||||
log.info("getStockDetailList:获取 {}, {} 的股票列表详情结束,耗时: {} ms ",stockCodes,stockType,(System.currentTimeMillis()-start));
|
||||
return ServerResponse.createBySuccess("操作成功", list);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("获取股票详情参数:{},{}. Exception:{}" , stockCodes,stockType, e);
|
||||
}
|
||||
return ServerResponse.createBySuccess("操作成功");
|
||||
}
|
||||
|
||||
private List<TwelveStockInfo> mapJsonToTwelveList(String stockCodes,JSONObject jsonObject) {
|
||||
if (jsonObject == null || jsonObject.isEmpty()) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
List<TwelveStockInfo> list = new ArrayList<>();
|
||||
String[] items = stockCodes.split(",");
|
||||
if(items!=null && items.length>0) {
|
||||
if(items.length == 1) {
|
||||
TwelveStockInfo info = mapJsonToTwelveStock(jsonObject);
|
||||
list.add(info);
|
||||
return list;
|
||||
}
|
||||
for (String str : items) {
|
||||
JSONObject json = jsonObject.getJSONObject(str);
|
||||
TwelveStockInfo vo = mapJsonToTwelveStock(json);
|
||||
list.add(vo);
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -39,5 +39,4 @@ public class Config {
|
||||
String aliyunAccessKeyId;
|
||||
String aliyunAccessKeySecret;
|
||||
String aliyunAppCode;
|
||||
String twelveApikey;
|
||||
}
|
||||
|
||||
@@ -3,9 +3,9 @@ spring:
|
||||
show-sql: true
|
||||
# Redis配置
|
||||
redis:
|
||||
host: 43.132.212.180
|
||||
password: ruTZ9J3gaDhknJ
|
||||
port: 36379
|
||||
host: 43.153.174.179
|
||||
password: a5v8b86P4mVzFlUqJV
|
||||
port: 30001
|
||||
database: 1
|
||||
lettuce:
|
||||
pool:
|
||||
@@ -17,9 +17,9 @@ spring:
|
||||
datasource:
|
||||
stock-market:
|
||||
driver-class-name: com.mysql.cj.jdbc.Driver
|
||||
url: jdbc:mysql://149.88.86.7:33306/india_stock?useUnicode=true&characterEncoding=utf-8
|
||||
url: jdbc:mysql://43.153.174.179:30000/india_stock?useUnicode=true&characterEncoding=utf-8
|
||||
username: root
|
||||
password: 33BsUUcnXRYgwtIND
|
||||
password: uNejHIFQGJOUtYTmE
|
||||
maxActive: 500
|
||||
testWhileIdle: true
|
||||
validationQuery: SELECT 1
|
||||
@@ -42,7 +42,6 @@ gugudataAppKey: K5LZKV8KAYM4
|
||||
aliyunAccessKeyId: LTAI5tJi2z8cegG8fTW7BSQu
|
||||
aliyunAccessKeySecret: fnCI9LUcqLuH7D6nkhvSHQMob1JSm8
|
||||
aliyunAppCode: 75ef2615da614eaaa71e2e2058fc53b0
|
||||
twelveApikey: 4098a6fc2c3245a88c7da804f10223c2
|
||||
# 具体看类:CloudStorageConfig
|
||||
oss:
|
||||
type: 2 # 类型 1:七牛 2:阿里云 3:腾讯云
|
||||
|
||||
@@ -46,7 +46,6 @@ gugudataAppKey: K5LZKV8KAYM4
|
||||
aliyunAccessKeyId: LTAI5tCQRp7t158TVyoF2Yhe
|
||||
aliyunAccessKeySecret: glyzOno773Cv7HeWDliveu2H5gX486
|
||||
aliyunAppCode: 75ef2615da614eaaa71e2e2058fc53b0
|
||||
twelveApikey: 4098a6fc2c3245a88c7da804f10223c2
|
||||
# 具体看类:CloudStorageConfig
|
||||
oss:
|
||||
type: 2 # 类型 1:七牛 2:阿里云 3:腾讯云
|
||||
|
||||
@@ -31,9 +31,9 @@ public class JpaDDDGen {
|
||||
*/
|
||||
public static DataSource getMySQLDataSource() {
|
||||
MysqlDataSource mysqlDS = new MysqlDataSource();
|
||||
mysqlDS.setURL("jdbc:mysql://149.88.86.7:33306/india_stock?useUnicode=true&characterEncoding=utf-8");
|
||||
mysqlDS.setURL("jdbc:mysql://43.132.212.180:33306/india_stock?useUnicode=true&characterEncoding=utf-8");
|
||||
mysqlDS.setUser("root");
|
||||
mysqlDS.setPassword("33BsUUcnXRYgwtIND");
|
||||
mysqlDS.setPassword("33BsUUcnXRYgwt");
|
||||
return mysqlDS;
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ public class JpaDDDGen {
|
||||
/**
|
||||
* cs_statistic - 要生成的数据库表
|
||||
*/
|
||||
Cons.tableNameToEntiyMapping.put("twelve_stock", null);
|
||||
Cons.tableNameToEntiyMapping.put("money_stock", null);
|
||||
|
||||
ToolDDD.g(getMySQLDataSource().getConnection());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user