nhanvubk/ger-market-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into 'main'

Browse Source 
Develop

See merge request india/india_market_java!3
This commit is contained in:
rplees
2024-01-04 07:45:15 +00:00
parent 2087d8914b 15b654e425
commit 0a0c969e28
15 changed files with 704 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
pom.xml
View File

@@ -178,6 +178,13 @@
<artifactId>jsoup</artifactId>
<version>1.15.2</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>30.1-jre</version>
</dependency>
</dependencies>
<build>

49
src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java Normal file
View File

@@ -0,0 +1,49 @@
package cn.stock.market.infrastructure.db.po;
import static com.querydsl.core.types.PathMetadataFactory.*;
import com.querydsl.core.types.dsl.*;
import com.querydsl.core.types.PathMetadata;
import javax.annotation.Generated;
import com.querydsl.core.types.Path;
/**
* QMoneyStockPO is a Querydsl query type for MoneyStockPO
*/
@Generated("com.querydsl.codegen.EntitySerializer")
public class QMoneyStockPO extends EntityPathBase<MoneyStockPO> {
private static final long serialVersionUID = -279042648L;
public static final QMoneyStockPO moneyStockPO = new QMoneyStockPO("moneyStockPO");
public final StringPath detailUrl = createString("detailUrl");
public final NumberPath<Integer> id = createNumber("id", Integer.class);
public final StringPath moneyScId = createString("moneyScId");
public final DateTimePath<java.util.Date> saveTime = createDateTime("saveTime", java.util.Date.class);
public final StringPath selfUrl = createString("selfUrl");
public final StringPath stockName = createString("stockName");
public final StringPath stockType = createString("stockType");
public QMoneyStockPO(String variable) {
super(MoneyStockPO.class, forVariable(variable));
}
public QMoneyStockPO(Path<? extends MoneyStockPO> path) {
super(path.getType(), path.getMetadata());
}
public QMoneyStockPO(PathMetadata metadata) {
super(MoneyStockPO.class, metadata);
}
}

35
src/main/java/cn/stock/market/application/assembler/MoneyStockAssembler.java Normal file
View File

@@ -0,0 +1,35 @@
package cn.stock.market.application.assembler;
import cn.qutaojing.common.utils.Beans;
import cn.qutaojing.common.utils.SpringUtils;
import cn.stock.market.domain.basic.entity.MoneyStock;
import cn.stock.market.dto.MoneyStockDTO;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* MoneyStockAssembler
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Component
@Lazy
public class MoneyStockAssembler {
public MoneyStockDTO toDTO(MoneyStock e) {
MoneyStockDTO dto = Beans.mapper(e, MoneyStockDTO.class);
if(dto == null) return dto;
fill(e, dto);
return dto;
}
protected void fill(MoneyStock e, MoneyStockDTO dto) {
if(dto == null) return;
return;
}
public static MoneyStockAssembler of() {
return SpringUtils.getBean(MoneyStockAssembler.class);
}
}

23
src/main/java/cn/stock/market/domain/basic/convert/MoneyStockConvert.java Normal file
View File

@@ -0,0 +1,23 @@
package cn.stock.market.domain.basic.convert;
import cn.qutaojing.common.domain.convert.SimpleEntityPOConvert;
import cn.qutaojing.common.utils.SpringUtils;
import cn.stock.market.domain.basic.entity.MoneyStock;
import cn.stock.market.infrastructure.db.po.MoneyStockPO;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* MoneyStockConvert
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Component
@Lazy
public class MoneyStockConvert extends SimpleEntityPOConvert<MoneyStock, MoneyStockPO> {
public static MoneyStockConvert of() {
return SpringUtils.getBean(MoneyStockConvert.class);
}
}

28
src/main/java/cn/stock/market/domain/basic/entity/MoneyStock.java Normal file
View File

@@ -0,0 +1,28 @@
package cn.stock.market.domain.basic.entity;
import cn.qutaojing.common.utils.Beans;
import cn.stock.market.dto.command.MoneyStockCreateCommand;
import cn.stock.market.infrastructure.db.po.MoneyStockPO;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* MoneyStock
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Data
@NoArgsConstructor
@SuperBuilder
@EqualsAndHashCode(
callSuper = false
)
public class MoneyStock extends MoneyStockPO {
public void update(MoneyStockCreateCommand cmd) {
Beans.copyProperties(cmd, this);
}
}

28
src/main/java/cn/stock/market/domain/basic/factory/MoneyStockFactory.java Normal file
View File

@@ -0,0 +1,28 @@
package cn.stock.market.domain.basic.factory;
import cn.qutaojing.common.utils.SpringUtils;
import cn.stock.market.domain.basic.entity.MoneyStock;
import cn.stock.market.dto.command.MoneyStockCreateCommand;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
/**
* MoneyStockFactory
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Component
@Lazy
public class MoneyStockFactory {
public static MoneyStockFactory of() {
return SpringUtils.getBean(MoneyStockFactory.class);
}
public MoneyStock from(MoneyStockCreateCommand cmd) {
MoneyStock e = MoneyStock.builder().build();
e.update(cmd);
return e;
}
}

46
src/main/java/cn/stock/market/domain/basic/repository/MoneyStockRepository.java Normal file
View File

@@ -0,0 +1,46 @@
package cn.stock.market.domain.basic.repository;
import cn.qutaojing.common.domain.convert.IEntityPOConvert;
import cn.qutaojing.common.domain.respostory.SimplePoConvertEntityRepository;
import cn.qutaojing.common.utils.SpringUtils;
import cn.stock.market.domain.basic.convert.MoneyStockConvert;
import cn.stock.market.domain.basic.entity.MoneyStock;
import cn.stock.market.infrastructure.db.po.MoneyStockPO;
import cn.stock.market.infrastructure.db.repo.MoneyStockRepo;
import com.rp.spring.jpa.GenericJpaRepository;
import java.lang.Integer;
import java.lang.Override;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
/**
* MoneyStockRepository
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Repository
@RequiredArgsConstructor(
onConstructor = @__(@Autowired)
)
public class MoneyStockRepository extends SimplePoConvertEntityRepository<MoneyStock, MoneyStockPO, Integer> {
final MoneyStockRepo repo;
final MoneyStockConvert convert;
@Override
public GenericJpaRepository<MoneyStockPO, Integer> repo() {
return repo;
}
@Override
public IEntityPOConvert<MoneyStock, MoneyStockPO> convert() {
return convert;
}
public static MoneyStockRepository of() {
return SpringUtils.getBean(MoneyStockRepository.class);
}
}

33
src/main/java/cn/stock/market/domain/basic/service/MoneyStockService.java Normal file
View File

@@ -0,0 +1,33 @@
package cn.stock.market.domain.basic.service;
import cn.qutaojing.common.utils.SpringUtils;
import cn.stock.market.domain.basic.factory.MoneyStockFactory;
import cn.stock.market.domain.basic.repository.MoneyStockRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
* MoneyStockService
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Service
@RequiredArgsConstructor(
onConstructor = @__(@Autowired)
)
public class MoneyStockService {
final MoneyStockRepository repository;
final MoneyStockFactory factory;
public MoneyStockRepository repository() {
return repository;
}
public static MoneyStockService of() {
return SpringUtils.getBean(MoneyStockService.class);
}
}

23
src/main/java/cn/stock/market/dto/MoneyStockDTO.java Normal file
View File

@@ -0,0 +1,23 @@
package cn.stock.market.dto;
import cn.stock.market.infrastructure.db.po.MoneyStockPO;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* MoneyStockDTO
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Data
@NoArgsConstructor
@SuperBuilder
@EqualsAndHashCode(
callSuper = false
)
public class MoneyStockDTO extends MoneyStockPO {
}

48
src/main/java/cn/stock/market/dto/command/MoneyStockCreateCommand.java Normal file
View File

@@ -0,0 +1,48 @@
package cn.stock.market.dto.command;
import java.lang.Integer;
import java.lang.String;
import java.util.Date;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* MoneyStockCreateCommand
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Data
@SuperBuilder
@NoArgsConstructor
public class MoneyStockCreateCommand {
/**
* 主键 */
Integer id;
/**
* 股票名称 */
String stockName;
/**
* BSE or NSE */
String stockType;
/**
* money Control的id */
String moneyScId;
/**
* 自有self_url */
String selfUrl;
/**
* 细节url */
String detailUrl;
/**
* 保存时间 */
Date saveTime;
}

24
src/main/java/cn/stock/market/dto/command/MoneyStockModifyCommand.java Normal file
View File

@@ -0,0 +1,24 @@
package cn.stock.market.dto.command;
import java.lang.Integer;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
/**
* MoneyStockModifyCommand
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@Data
@SuperBuilder
@NoArgsConstructor
@EqualsAndHashCode(
callSuper = false
)
public class MoneyStockModifyCommand extends MoneyStockCreateCommand {
Integer id;
}

66
src/main/java/cn/stock/market/infrastructure/db/po/MoneyStockPO.java Normal file
View File

@@ -0,0 +1,66 @@
package cn.stock.market.infrastructure.db.po;
import java.lang.Integer;
import java.lang.String;
import java.util.Date;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import org.hibernate.annotations.DynamicInsert;
import org.hibernate.annotations.DynamicUpdate;
/**
* MoneyStockPO
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
@SuperBuilder
@Data
@NoArgsConstructor
@AllArgsConstructor
@Entity
@DynamicInsert
@DynamicUpdate
@Table(
name = "money_stock"
)
public class MoneyStockPO {
/**
* 主键 */
@Id
@GeneratedValue(
strategy = javax.persistence.GenerationType.IDENTITY
)
Integer id;
/**
* 股票名称 */
String stockName;
/**
* BSE or NSE */
String stockType;
/**
* money Control的id */
String moneyScId;
/**
* 自有self_url */
String selfUrl;
/**
* 细节url */
String detailUrl;
/**
* 保存时间 */
Date saveTime;
}

15
src/main/java/cn/stock/market/infrastructure/db/repo/MoneyStockRepo.java Normal file
View File

@@ -0,0 +1,15 @@
package cn.stock.market.infrastructure.db.repo;
import cn.stock.market.infrastructure.db.po.MoneyStockPO;
import com.rp.spring.jpa.GenericJpaRepository;
import java.lang.Integer;
/**
* MoneyStockRepo
*
* @author rplees
* @email rplees.i.ly@gmail.com
* @created 2024/01/03
*/
public interface MoneyStockRepo extends GenericJpaRepository<MoneyStockPO, Integer> {
}

278
src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java Normal file
View File

@@ -0,0 +1,278 @@
package cn.stock.market.infrastructure.job;
import cn.hutool.core.collection.CollectionUtil;
import cn.stock.market.domain.basic.entity.MoneyStock;
import cn.stock.market.domain.basic.repository.MoneyStockRepository;
import cn.stock.market.infrastructure.db.po.QMoneyStockPO;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* @author gs
* @date 2024/1/2 17:31
*/
@RestController
@Slf4j
public class MoneyScraper {
private static final int MAX_RETRY_ATTEMPTS = 10;
private static final int NUM_THREADS = 5;
@Autowired
private MoneyStockRepository moneyStockRepository;
@GetMapping("testScraperGetMoneyControlStock")
public void schedule(){
List<String> letters = new ArrayList<>();
for (char c = 'A'; c <= 'Z'; c++) {
letters.add(String.valueOf(c));
}
letters.add("others");
int tasksPerThread = (int) Math.ceil((double) letters.size() / NUM_THREADS);
// 手动创建线程池
ThreadPoolExecutor executorService = new ThreadPoolExecutor(
NUM_THREADS, NUM_THREADS,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>());
List<CompletableFuture<List<String>>> futures = new ArrayList<>();
for (int i = 0; i < NUM_THREADS; i++) {
int startIndex = i * tasksPerThread;
int endIndex = Math.min((i + 1) * tasksPerThread, letters.size());
List<String> letterRange = letters.subList(startIndex, endIndex);
CompletableFuture<List<String>> future = CompletableFuture.supplyAsync(() -> processLetters(letterRange), executorService);
futures.add(future);
}
List<String> results = new ArrayList<>();
for (CompletableFuture<List<String>> future : futures) {
try {
results.addAll(future.get());
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
executorService.shutdown();
log.error("All threads completed. Results:");
for (int i = 0; i < results.size(); i++) {
log.error("Thread " + (i + 1) + " processed letters: " + results.get(i));
}
}
/**
* 带有A B 分组的url
* @param url
* @param httpClient
* @param letter
* @return
* @throws IOException
*/
private List<String> sendHttpRequest(String url, HttpClient httpClient, String letter) throws IOException {
List<MoneyStock> allMoneyStock = moneyStockRepository.findAll();
Document document = fetchStockDetails(url);
extractExchangeDetails(document,allMoneyStock);
List<String> result = new ArrayList<>();
result.add("Thread " + Thread.currentThread().getName() + " processed letters: " + letter);
return result;
}
private List<String> processLetters(List<String> letterRange) {
log.error("Thread " + Thread.currentThread().getName() + " is processing letters: " + letterRange);
String urlBase = "https://www.moneycontrol.com/india/stockpricequote/%s";
HttpClient httpClient = HttpClients.createDefault();
List<String> results = new ArrayList<>();
for (String letter : letterRange) {
String detailUrl = String.format(urlBase, letter);
log.error("Thread " + Thread.currentThread().getName() + ",请求的url:" + detailUrl);
// 重试逻辑
int maxAttempts = 10;
int attempt = 1;
while (attempt <= maxAttempts) {
try {
List<String> response = sendHttpRequest(detailUrl, httpClient, letter) ;
results.addAll(response);
break; // 如果成功则跳出循环
} catch (IOException | RuntimeException e) {
// 处理异常的逻辑
e.printStackTrace();
log.error("Attempt " + attempt + " failed. Retrying...");
attempt++;
try {
Thread.sleep(1000); // 休眠1秒
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
}
}
}
}
return results;
}
/**
* 获取全部股票url_self
* @param url
* @return
*/
public static Document fetchStockDetails(String url) {
return fetchDocumentWithRetry(url);
}
public static Document fetchCompanyDetails(String url) {
return fetchDocumentWithRetry(url);
}
private static Document fetchDocumentWithRetry(String url) {
HttpClient httpClient = HttpClients.createDefault();
int retryAttempts = 0;
while (retryAttempts < MAX_RETRY_ATTEMPTS) {
try {
HttpGet request = new HttpGet(url);
request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
HttpResponse response = httpClient.execute(request);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode >= 200 && statusCode < 300) {
return Jsoup.parse(response.getEntity().getContent(), null, url);
} else {
retryAttempts++;
log.error("HTTP request failed with status code: " + statusCode);
}
} catch (IOException e) {
retryAttempts++;
log.error(Thread.currentThread().getName()+",Exception during HTTP request: " +retryAttempts);
}
}
return null; // Retry attempts exhausted
}
public void extractExchangeDetails(Document soup,List<MoneyStock> moneyStockHadSavedList) {
Elements companies = soup.select("table.pcq_tbl.MT10");
List<String> hadSaveUrl = moneyStockHadSavedList.stream().map(MoneyStock::getSelfUrl).collect(Collectors.toList());
for (Element company : companies) {
Elements elements = company.select("tr > td > a");
for (Element element : elements) {
String textContent = element.text().trim();
String linkAttribute = element.attr("href");
if(hadSaveUrl.contains(linkAttribute)){
log.error(Thread.currentThread().getName()+"已经存在了不需要重复保存,company_name: " + textContent + ", Link Attribute: " + linkAttribute);
continue;
}
log.info(Thread.currentThread().getName()+",Text Content: " + textContent + ", Link Attribute: " + linkAttribute);
Document soup2 = fetchCompanyDetails(linkAttribute);
if (soup2 != null) {
Element comIdInput = soup2.selectFirst("input[id=ap_sc_id]");
String companyCodeId = "";
if (comIdInput != null) {
companyCodeId = comIdInput.val();
log.info(Thread.currentThread().getName()+",the stockName: " + textContent + ", THE input id: " + companyCodeId);
} else {
log.error(Thread.currentThread().getName()+" No <input> with id='ap_sc_id' found on the website.");
}
if (soup2 != null) {
Element ulElement = soup2.selectFirst("ul[id=nseBseTab]");
if (ulElement != null) {
for (Element aElement : ulElement.select("a")) {
String exchangeValue = aElement.text().trim();
if ("BSE".equals(exchangeValue) || "NSE".equals(exchangeValue)) {
log.info(Thread.currentThread().getName()+",stockName: " + textContent + ", self_link: " + linkAttribute +
", the exchange Value: " + exchangeValue);
MoneyStock build = MoneyStock.builder().stockName(textContent).stockType(exchangeValue.toLowerCase(Locale.ROOT))
.detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId))
.selfUrl(linkAttribute)
.moneyScId(companyCodeId).saveTime(new Date()).build();
List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent), QMoneyStockPO.moneyStockPO.stockType.eq(exchangeValue.toLowerCase(Locale.ROOT)));
if(CollectionUtil.isEmpty(all)){
moneyStockRepository.save(build);
}
} else {
log.error(Thread.currentThread().getName()+", stockName: " + textContent + ", self_link: " + linkAttribute +
" is not a valid exchange type");
MoneyStock build = MoneyStock.builder().stockName(textContent)
.detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId))
.selfUrl(linkAttribute)
.moneyScId(companyCodeId).saveTime(new Date()).build();
List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent));
if(CollectionUtil.isEmpty(all)){
moneyStockRepository.save(build);
}
}
}
} else {
log.info("stockName: " + textContent + ", self_link: " + linkAttribute +
" has no current exchange types");
MoneyStock build = MoneyStock.builder().stockName(textContent).selfUrl(linkAttribute)
.moneyScId(companyCodeId).saveTime(new Date()).build();
List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent));
if(CollectionUtil.isEmpty(all)){
moneyStockRepository.save(build);
}
}
}
} else {
log.info(Thread.currentThread().getName()+",stockName: " + textContent + ", self_link: " + linkAttribute +
" cannot find corresponding stock id");
MoneyStock build = MoneyStock.builder().stockName(textContent).selfUrl(linkAttribute)
.saveTime(new Date()).build();
List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent));
if(CollectionUtil.isEmpty(all)){
moneyStockRepository.save(build);
}
}
}
}
}
}

2
src/test/java/rp/lee/jpa/JpaDDDGen.java
View File

@@ -50,7 +50,7 @@ public class JpaDDDGen {
/**
* cs_statistic - 要生成的数据库表
*/
Cons.tableNameToEntiyMapping.put("stock_ipo", null);
Cons.tableNameToEntiyMapping.put("money_stock", null);
ToolDDD.g(getMySQLDataSource().getConnection());
}