diff --git a/pom.xml b/pom.xml index 097f3d6..a25d07d 100644 --- a/pom.xml +++ b/pom.xml @@ -178,6 +178,13 @@ jsoup 1.15.2 + + + com.google.guava + guava + 30.1-jre + + diff --git a/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java b/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java new file mode 100644 index 0000000..b719747 --- /dev/null +++ b/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java @@ -0,0 +1,55 @@ +package cn.stock.market.infrastructure.db.po; + +import static com.querydsl.core.types.PathMetadataFactory.*; + +import com.querydsl.core.types.dsl.*; + +import com.querydsl.core.types.PathMetadata; +import javax.annotation.Generated; +import com.querydsl.core.types.Path; + + +/** + * QMoneyStockPO is a Querydsl query type for MoneyStockPO + */ +@Generated("com.querydsl.codegen.EntitySerializer") +public class QMoneyStockPO extends EntityPathBase { + + private static final long serialVersionUID = -279042648L; + + public static final QMoneyStockPO moneyStockPO = new QMoneyStockPO("moneyStockPO"); + + public final StringPath detailUrl = createString("detailUrl"); + + public final NumberPath id = createNumber("id", Integer.class); + + public final NumberPath isLock = createNumber("isLock", Integer.class); + + public final NumberPath isShow = createNumber("isShow", Integer.class); + + public final StringPath moneyScId = createString("moneyScId"); + + public final DateTimePath saveTime = createDateTime("saveTime", java.util.Date.class); + + public final StringPath selfDispId = createString("selfDispId"); + + public final StringPath selfUrl = createString("selfUrl"); + + public final StringPath stockName = createString("stockName"); + + public final StringPath stockType = createString("stockType"); + + public QMoneyStockPO(String variable) { + super(MoneyStockPO.class, forVariable(variable)); + } + + public QMoneyStockPO(Path path) { + super(path.getType(), path.getMetadata()); + } + + public QMoneyStockPO(PathMetadata metadata) { + super(MoneyStockPO.class, metadata); + } + +} + diff --git a/src/main/java/cn/stock/market/MoneyStockSuggestDTO.java b/src/main/java/cn/stock/market/MoneyStockSuggestDTO.java new file mode 100644 index 0000000..fcf019c --- /dev/null +++ b/src/main/java/cn/stock/market/MoneyStockSuggestDTO.java @@ -0,0 +1,25 @@ +package cn.stock.market; + +import lombok.Data; + +/** + * @author gs + * @date 2024/1/6 11:12 + */ +@Data +public class MoneyStockSuggestDTO { + + private String stockType; + private String stockName; + private String stockUrl; + private String highPrice; + private String lowPrice; + private String lastPrice; + private String prevClosePrice; + private String change; + private String changePercent; + private String dispId; + private String scId; + + +} diff --git a/src/main/java/cn/stock/market/application/assembler/MoneyStockAssembler.java b/src/main/java/cn/stock/market/application/assembler/MoneyStockAssembler.java new file mode 100644 index 0000000..3e000b7 --- /dev/null +++ b/src/main/java/cn/stock/market/application/assembler/MoneyStockAssembler.java @@ -0,0 +1,35 @@ +package cn.stock.market.application.assembler; + +import cn.qutaojing.common.utils.Beans; +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.dto.MoneyStockDTO; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * MoneyStockAssembler + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Component +@Lazy +public class MoneyStockAssembler { + public MoneyStockDTO toDTO(MoneyStock e) { + MoneyStockDTO dto = Beans.mapper(e, MoneyStockDTO.class); + if(dto == null) return dto; + fill(e, dto); + return dto; + } + + protected void fill(MoneyStock e, MoneyStockDTO dto) { + if(dto == null) return; + return; + } + + public static MoneyStockAssembler of() { + return SpringUtils.getBean(MoneyStockAssembler.class); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/convert/MoneyStockConvert.java b/src/main/java/cn/stock/market/domain/basic/convert/MoneyStockConvert.java new file mode 100644 index 0000000..3a0ad51 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/convert/MoneyStockConvert.java @@ -0,0 +1,23 @@ +package cn.stock.market.domain.basic.convert; + +import cn.qutaojing.common.domain.convert.SimpleEntityPOConvert; +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * MoneyStockConvert + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Component +@Lazy +public class MoneyStockConvert extends SimpleEntityPOConvert { + public static MoneyStockConvert of() { + return SpringUtils.getBean(MoneyStockConvert.class); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/entity/MoneyStock.java b/src/main/java/cn/stock/market/domain/basic/entity/MoneyStock.java new file mode 100644 index 0000000..fcd5321 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/entity/MoneyStock.java @@ -0,0 +1,28 @@ +package cn.stock.market.domain.basic.entity; + +import cn.qutaojing.common.utils.Beans; +import cn.stock.market.dto.command.MoneyStockCreateCommand; +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * MoneyStock + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Data +@NoArgsConstructor +@SuperBuilder +@EqualsAndHashCode( + callSuper = false +) +public class MoneyStock extends MoneyStockPO { + public void update(MoneyStockCreateCommand cmd) { + Beans.copyProperties(cmd, this); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/factory/MoneyStockFactory.java b/src/main/java/cn/stock/market/domain/basic/factory/MoneyStockFactory.java new file mode 100644 index 0000000..764db2b --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/factory/MoneyStockFactory.java @@ -0,0 +1,28 @@ +package cn.stock.market.domain.basic.factory; + +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.dto.command.MoneyStockCreateCommand; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * MoneyStockFactory + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Component +@Lazy +public class MoneyStockFactory { + public static MoneyStockFactory of() { + return SpringUtils.getBean(MoneyStockFactory.class); + } + + public MoneyStock from(MoneyStockCreateCommand cmd) { + MoneyStock e = MoneyStock.builder().build(); + e.update(cmd); + return e; + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/repository/MoneyStockRepository.java b/src/main/java/cn/stock/market/domain/basic/repository/MoneyStockRepository.java new file mode 100644 index 0000000..5cda981 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/repository/MoneyStockRepository.java @@ -0,0 +1,46 @@ +package cn.stock.market.domain.basic.repository; + +import cn.qutaojing.common.domain.convert.IEntityPOConvert; +import cn.qutaojing.common.domain.respostory.SimplePoConvertEntityRepository; +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.convert.MoneyStockConvert; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import cn.stock.market.infrastructure.db.repo.MoneyStockRepo; +import com.rp.spring.jpa.GenericJpaRepository; +import java.lang.Integer; +import java.lang.Override; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +/** + * MoneyStockRepository + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Repository +@RequiredArgsConstructor( + onConstructor = @__(@Autowired) +) +public class MoneyStockRepository extends SimplePoConvertEntityRepository { + final MoneyStockRepo repo; + + final MoneyStockConvert convert; + + @Override + public GenericJpaRepository repo() { + return repo; + } + + @Override + public IEntityPOConvert convert() { + return convert; + } + + public static MoneyStockRepository of() { + return SpringUtils.getBean(MoneyStockRepository.class); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/service/MoneyStockService.java b/src/main/java/cn/stock/market/domain/basic/service/MoneyStockService.java new file mode 100644 index 0000000..b524393 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/service/MoneyStockService.java @@ -0,0 +1,33 @@ +package cn.stock.market.domain.basic.service; + +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.factory.MoneyStockFactory; +import cn.stock.market.domain.basic.repository.MoneyStockRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * MoneyStockService + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Service +@RequiredArgsConstructor( + onConstructor = @__(@Autowired) +) +public class MoneyStockService { + final MoneyStockRepository repository; + + final MoneyStockFactory factory; + + public MoneyStockRepository repository() { + return repository; + } + + public static MoneyStockService of() { + return SpringUtils.getBean(MoneyStockService.class); + } +} diff --git a/src/main/java/cn/stock/market/dto/MoneyStockDTO.java b/src/main/java/cn/stock/market/dto/MoneyStockDTO.java new file mode 100644 index 0000000..1a6552c --- /dev/null +++ b/src/main/java/cn/stock/market/dto/MoneyStockDTO.java @@ -0,0 +1,23 @@ +package cn.stock.market.dto; + +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * MoneyStockDTO + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Data +@NoArgsConstructor +@SuperBuilder +@EqualsAndHashCode( + callSuper = false +) +public class MoneyStockDTO extends MoneyStockPO { +} diff --git a/src/main/java/cn/stock/market/dto/StockHistoryRequest.java b/src/main/java/cn/stock/market/dto/StockHistoryRequest.java new file mode 100644 index 0000000..ef4c781 --- /dev/null +++ b/src/main/java/cn/stock/market/dto/StockHistoryRequest.java @@ -0,0 +1,14 @@ +package cn.stock.market.dto; + +import lombok.Data; + +@Data +public class StockHistoryRequest { + private String symbol; + private String resolution; + private long from; + private long to; + private int countback; + private String currencyCode; + +} \ No newline at end of file diff --git a/src/main/java/cn/stock/market/dto/StockHistoryResponse.java b/src/main/java/cn/stock/market/dto/StockHistoryResponse.java new file mode 100644 index 0000000..66d27aa --- /dev/null +++ b/src/main/java/cn/stock/market/dto/StockHistoryResponse.java @@ -0,0 +1,17 @@ +package cn.stock.market.dto; + +import lombok.Data; + +import java.util.List; + +@Data +public class StockHistoryResponse { + private String s; + private List t; + private List o; + private List h; + private List l; + private List c; + private List v; + +} \ No newline at end of file diff --git a/src/main/java/cn/stock/market/dto/command/MoneyStockCreateCommand.java b/src/main/java/cn/stock/market/dto/command/MoneyStockCreateCommand.java new file mode 100644 index 0000000..d867ada --- /dev/null +++ b/src/main/java/cn/stock/market/dto/command/MoneyStockCreateCommand.java @@ -0,0 +1,60 @@ +package cn.stock.market.dto.command; + +import java.lang.Integer; +import java.lang.String; +import java.util.Date; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * MoneyStockCreateCommand + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Data +@SuperBuilder +@NoArgsConstructor +public class MoneyStockCreateCommand { + /** + * 主键 */ + Integer id; + + /** + * 股票名称 */ + String stockName; + + /** + * BSE or NSE */ + String stockType; + + /** + * money Control的id */ + String moneyScId; + + /** + * 展示表示 */ + String selfDispId; + + /** + * 自有self_url */ + String selfUrl; + + /** + * 细节url */ + String detailUrl; + + /** + * 保存时间 */ + Date saveTime; + + /** + * 是否锁定 0否 1是 */ + Integer isLock; + + /** + * 是否展示 0是 1否 */ + Integer isShow; +} diff --git a/src/main/java/cn/stock/market/dto/command/MoneyStockModifyCommand.java b/src/main/java/cn/stock/market/dto/command/MoneyStockModifyCommand.java new file mode 100644 index 0000000..ca74282 --- /dev/null +++ b/src/main/java/cn/stock/market/dto/command/MoneyStockModifyCommand.java @@ -0,0 +1,24 @@ +package cn.stock.market.dto.command; + +import java.lang.Integer; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * MoneyStockModifyCommand + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@Data +@SuperBuilder +@NoArgsConstructor +@EqualsAndHashCode( + callSuper = false +) +public class MoneyStockModifyCommand extends MoneyStockCreateCommand { + Integer id; +} diff --git a/src/main/java/cn/stock/market/infrastructure/db/po/MoneyStockPO.java b/src/main/java/cn/stock/market/infrastructure/db/po/MoneyStockPO.java new file mode 100644 index 0000000..7b6d464 --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/db/po/MoneyStockPO.java @@ -0,0 +1,78 @@ +package cn.stock.market.infrastructure.db.po; + +import java.lang.Integer; +import java.lang.String; +import java.util.Date; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.hibernate.annotations.DynamicInsert; +import org.hibernate.annotations.DynamicUpdate; + +/** + * MoneyStockPO + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +@SuperBuilder +@Data +@NoArgsConstructor +@AllArgsConstructor +@Entity +@DynamicInsert +@DynamicUpdate +@Table( + name = "money_stock" +) +public class MoneyStockPO { + /** + * 主键 */ + @Id + @GeneratedValue( + strategy = javax.persistence.GenerationType.IDENTITY + ) + Integer id; + + /** + * 股票名称 */ + String stockName; + + /** + * BSE or NSE */ + String stockType; + + /** + * money Control的id */ + String moneyScId; + + /** + * 展示表示 */ + String selfDispId; + + /** + * 自有self_url */ + String selfUrl; + + /** + * 细节url */ + String detailUrl; + + /** + * 保存时间 */ + Date saveTime; + + /** + * 是否锁定 0否 1是 */ + Integer isLock; + + /** + * 是否展示 0是 1否 */ + Integer isShow; +} diff --git a/src/main/java/cn/stock/market/infrastructure/db/repo/MoneyStockRepo.java b/src/main/java/cn/stock/market/infrastructure/db/repo/MoneyStockRepo.java new file mode 100644 index 0000000..eebe02a --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/db/repo/MoneyStockRepo.java @@ -0,0 +1,15 @@ +package cn.stock.market.infrastructure.db.repo; + +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import com.rp.spring.jpa.GenericJpaRepository; +import java.lang.Integer; + +/** + * MoneyStockRepo + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/06 + */ +public interface MoneyStockRepo extends GenericJpaRepository { +} diff --git a/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java b/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java new file mode 100644 index 0000000..f507632 --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java @@ -0,0 +1,337 @@ +package cn.stock.market.infrastructure.job; + +import cn.hutool.core.collection.CollectionUtil; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.domain.basic.repository.MoneyStockRepository; +import cn.stock.market.infrastructure.db.po.QMoneyStockPO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClients; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * @author gs + * @date 2024/1/2 17:31 + */ +@RestController +@Slf4j +public class MoneyScraper { + + private static final int MAX_RETRY_ATTEMPTS = 10; + private static final int NUM_THREADS = 5; + + + @Autowired + private MoneyStockRepository moneyStockRepository; + + @GetMapping("testScraperGetMoneyControlStock") + @Scheduled(cron = "0 0 2 */2 * ?") + public void schedule(){ + List letters = new ArrayList<>(); + for (char c = 'A'; c <= 'Z'; c++) { + letters.add(String.valueOf(c)); + } + letters.add("others"); + + int tasksPerThread = (int) Math.ceil((double) letters.size() / NUM_THREADS); + + // 手动创建线程池 + ThreadPoolExecutor executorService = new ThreadPoolExecutor( + NUM_THREADS, NUM_THREADS, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>()); + + List>> futures = new ArrayList<>(); + + for (int i = 0; i < NUM_THREADS; i++) { + int startIndex = i * tasksPerThread; + int endIndex = Math.min((i + 1) * tasksPerThread, letters.size()); + List letterRange = letters.subList(startIndex, endIndex); + CompletableFuture> future = CompletableFuture.supplyAsync(() -> processLetters(letterRange), executorService); + futures.add(future); + } + + List results = new ArrayList<>(); + + for (CompletableFuture> future : futures) { + try { + results.addAll(future.get()); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + } + + executorService.shutdown(); + + log.error("All threads completed. Results:"); + for (int i = 0; i < results.size(); i++) { + log.error("Thread " + (i + 1) + " processed letters: " + results.get(i)); + } + + } + + @GetMapping("testScraperGetMoneyControlStockNoScId") + public String testScraperGetMoneyControlStockNoScId(@RequestParam("url") String url) { + Document soup2 = fetchCompanyDetails(url); + + if (soup2 != null) { + Element comIdInput = soup2.selectFirst("input[id=ap_sc_id]"); + String companyCodeId = null; + if (comIdInput != null) { + companyCodeId = comIdInput.val(); + log.info(Thread.currentThread().getName() + ",the stock url: " + url + ", THE input id: " + companyCodeId); + } else { + log.error(Thread.currentThread().getName() + " No with id='ap_sc_id' found on the website."); + } + + if (soup2 != null) { + Element ulElement = soup2.selectFirst("ul[id=nseBseTab]"); + String name = soup2.selectFirst("#stockName > h1").text(); + if (ulElement != null) { + for (Element aElement : ulElement.select("a")) { + String exchangeValue = aElement.text().trim(); + if ("BSE".equals(exchangeValue) || "NSE".equals(exchangeValue)) { + log.info(Thread.currentThread().getName() + ",the stock url: " + url + + ", the exchange Value: " + exchangeValue); + MoneyStock build = MoneyStock.builder().stockName(name).stockType(exchangeValue.toLowerCase(Locale.ROOT)) + .detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId)) + .selfUrl(url) + .selfDispId(extractDispId(url)) + .moneyScId(companyCodeId).saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(name), QMoneyStockPO.moneyStockPO.stockType.eq(exchangeValue.toLowerCase(Locale.ROOT))); + if (CollectionUtil.isEmpty(all)) { + moneyStockRepository.save(build); + } + } + } + } + } + } + return "ok"; + } + + + /** + * 带有A B 分组的url + * @param url + * @param httpClient + * @param letter + * @return + * @throws IOException + */ + private List sendHttpRequest(String url, HttpClient httpClient, String letter) throws IOException { + List allMoneyStock = moneyStockRepository.findAll(); + Document document = fetchStockDetails(url); + extractExchangeDetails(document,allMoneyStock); + List result = new ArrayList<>(); + result.add("Thread " + Thread.currentThread().getName() + " processed letters: " + letter); + return result; + } + + + private List processLetters(List letterRange) { + log.error("Thread " + Thread.currentThread().getName() + " is processing letters: " + letterRange); + String urlBase = "https://www.moneycontrol.com/india/stockpricequote/%s"; + HttpClient httpClient = HttpClients.createDefault(); + List results = new ArrayList<>(); + + for (String letter : letterRange) { + String detailUrl = String.format(urlBase, letter); + log.error("Thread " + Thread.currentThread().getName() + ",请求的url:" + detailUrl); + + // 重试逻辑 + int maxAttempts = 10; + int attempt = 1; + + while (attempt <= maxAttempts) { + try { + List response = sendHttpRequest(detailUrl, httpClient, letter) ; + results.addAll(response); + break; // 如果成功则跳出循环 + } catch (IOException | RuntimeException e) { + // 处理异常的逻辑 + log.error("Attempt " + attempt + " failed. Retrying...",e); + attempt++; + try { + Thread.sleep(1000); // 休眠1秒 + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + } + } + + return results; + } + + + /** + * 获取全部股票url_self + * @param url + * @return + */ + public static Document fetchStockDetails(String url) { + return fetchDocumentWithRetry(url); + } + + public static Document fetchCompanyDetails(String url) { + return fetchDocumentWithRetry(url); + } + + private static Document fetchDocumentWithRetry(String url) { + HttpClient httpClient = HttpClients.createDefault(); + int retryAttempts = 0; + + while (retryAttempts < MAX_RETRY_ATTEMPTS) { + try { + HttpGet request = new HttpGet(url); + request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"); + + HttpResponse response = httpClient.execute(request); + int statusCode = response.getStatusLine().getStatusCode(); + + if (statusCode >= 200 && statusCode < 300) { + return Jsoup.parse(response.getEntity().getContent(), null, url); + } else { + retryAttempts++; + log.error("HTTP request failed with status code: " + statusCode); + } + } catch (IOException e) { + retryAttempts++; + log.error(Thread.currentThread().getName()+",Exception during HTTP request time: " +retryAttempts); + } + } + + return null; // Retry attempts exhausted + } + + + public void extractExchangeDetails(Document soup,List moneyStockHadSavedList) { + Elements companies = soup.select("table.pcq_tbl.MT10"); + List hadSaveUrl = moneyStockHadSavedList.stream().map(MoneyStock::getSelfUrl).collect(Collectors.toList()); + for (Element company : companies) { + Elements elements = company.select("tr > td > a"); + + for (Element element : elements) { + String textContent = element.text().trim(); + String linkAttribute = element.attr("href"); + if(hadSaveUrl.contains(linkAttribute)){ + log.error(Thread.currentThread().getName()+"已经存在了不需要重复保存,company_name: " + textContent + ", Link Attribute: " + linkAttribute); + continue; + } + log.info(Thread.currentThread().getName()+",Text Content: " + textContent + ", Link Attribute: " + linkAttribute); + Document soup2 = fetchCompanyDetails(linkAttribute); + + if (soup2 != null) { + Element comIdInput = soup2.selectFirst("input[id=ap_sc_id]"); + String companyCodeId = ""; + if (comIdInput != null) { + companyCodeId = comIdInput.val(); + log.info(Thread.currentThread().getName()+",the stockName: " + textContent + ", THE input id: " + companyCodeId); + } else { + log.error(Thread.currentThread().getName()+" No with id='ap_sc_id' found on the website."); + } + + if (soup2 != null) { + Element ulElement = soup2.selectFirst("ul[id=nseBseTab]"); + + if (ulElement != null) { + for (Element aElement : ulElement.select("a")) { + String exchangeValue = aElement.text().trim(); + + if ("BSE".equals(exchangeValue) || "NSE".equals(exchangeValue)) { + log.info(Thread.currentThread().getName()+",stockName: " + textContent + ", self_link: " + linkAttribute + + ", the exchange Value: " + exchangeValue); + MoneyStock build = MoneyStock.builder().stockName(textContent).stockType(exchangeValue.toLowerCase(Locale.ROOT)) + .detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId)) + .selfUrl(linkAttribute) + .selfDispId(extractDispId(linkAttribute)) + .moneyScId(companyCodeId).saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent), QMoneyStockPO.moneyStockPO.stockType.eq(exchangeValue.toLowerCase(Locale.ROOT))); + if(CollectionUtil.isEmpty(all)){ + moneyStockRepository.save(build); + } + } else { + log.error(Thread.currentThread().getName()+", stockName: " + textContent + ", self_link: " + linkAttribute + + " is not a valid exchange type"); + MoneyStock build = MoneyStock.builder().stockName(textContent) + .detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId)) + .selfUrl(linkAttribute) + .selfDispId(extractDispId(linkAttribute)) + .moneyScId(companyCodeId).saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent)); + if(CollectionUtil.isEmpty(all)){ + moneyStockRepository.save(build); + } + } + } + } else { + log.info("stockName: " + textContent + ", self_link: " + linkAttribute + + " has no current exchange types"); + MoneyStock build = MoneyStock.builder().stockName(textContent).selfUrl(linkAttribute) .selfDispId(extractDispId(linkAttribute)) + .moneyScId(companyCodeId).saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent)); + if(CollectionUtil.isEmpty(all)){ + moneyStockRepository.save(build); + } + } + } + } else { + log.info(Thread.currentThread().getName()+",stockName: " + textContent + ", self_link: " + linkAttribute + + " cannot find corresponding stock id"); + MoneyStock build = MoneyStock.builder().stockName(textContent).selfUrl(linkAttribute) .selfDispId(extractDispId(linkAttribute)) + .saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent)); + if(CollectionUtil.isEmpty(all)){ + moneyStockRepository.save(build); + } + } + } + } + } + + private static String extractDispId(String selfUrl) { + if(StringUtils.isBlank(selfUrl)||!StringUtils.startsWith(selfUrl,"http")){ + return null; + } + // 找到最后一个斜杠的位置 + int lastSlashIndex = selfUrl.lastIndexOf('/'); + + // 如果找到了斜杠,就提取它之后的部分 + if (lastSlashIndex != -1 && lastSlashIndex < selfUrl.length() - 1) { + return selfUrl.substring(lastSlashIndex + 1); + } + + // 如果没有斜杠,或者斜杠位于字符串的末尾,则返回原始字符串 + return selfUrl; + } + + + + +} diff --git a/src/main/java/cn/stock/market/web/MoneyApiController.java b/src/main/java/cn/stock/market/web/MoneyApiController.java new file mode 100644 index 0000000..7c6846f --- /dev/null +++ b/src/main/java/cn/stock/market/web/MoneyApiController.java @@ -0,0 +1,760 @@ +package cn.stock.market.web; + +import cn.stock.market.MoneyStockSuggestDTO; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.domain.basic.repository.MoneyStockRepository; +import cn.stock.market.dto.StockHistoryRequest; +import cn.stock.market.dto.StockHistoryResponse; +import cn.stock.market.infrastructure.db.po.QMoneyStockPO; +import cn.stock.market.utils.RequestCacheUtils; +import cn.stock.market.utils.ServerResponse; +import com.alibaba.fastjson.JSONObject; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Lists; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * @author gs + * @date 2024/1/4 10:25 + */ +@RestController +@Api(value = "/MoneyApiController", tags = "money股票行情") +@Slf4j +public class MoneyApiController { + + + @Autowired + private RestTemplate restTemplate; + + @Autowired + private MoneyStockRepository moneyStockRepository; + + private static final String EXTERNAL_API_URL = "https://priceapi.moneycontrol.com/techCharts/indianMarket/stock/history"; + + + + @ApiOperation(value = "股票详情信息",httpMethod = "GET") + @ApiImplicitParams({ + @ApiImplicitParam(name="stockType",value = "BSE或者NSE"), + @ApiImplicitParam(name="symbol",value = "scId值"), + @ApiImplicitParam(name="id",value = "id值"), + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "" + + "HN: 公司的名称(हिन्दी में - Hindi)\n" + + "HP: 当前股价\n" + + "DISPID: 行业/部门代码\n" + + "cl5yPerChange: 过去5年的百分比变化\n" + + "BIDQ 和 BIDP: 买入报价和数量\n" + + "newSubsector: 新的子部门\n" + + "cl1mDt: 过去1个月的数据日期\n" + + "P_C: 市盈率\n" + + "YR: 年份\n" + + "cl2yVal: 过去2年的数值\n" + + "SC_SUBSEC: 子部门\n" + + "sc_ttm_cons 和 SC_TTM: 市盈率\n" + + "clYtdVal: 本年度截至目前的数值\n" + + "cl6mDt: 过去6个月的数据日期\n" + + "cl1wVal: 过去1周的数值\n" + + "lastupd_epoch: 最后更新时间的 Epoch 时间戳\n" + + "cl1mChange: 过去1个月的数值变化\n" + + "cl3mChange: 过去3个月的数值变化\n" + + "pricecurrent: 当前股价\n" + + "150DayAvg: 过去150天的平均值\n" + + "clYtdChange: 本年度截至目前的数值变化\n" + + "tot_buy_qty: 总买入数量\n" + + "clYtdDt: 本年度截至目前的数据日期\n" + + "Group: 组别\n" + + "ACCOD: 账户代码\n" + + "cl3yDt: 过去3年的数据日期\n" + + "LTHDate: 最高值的日期\n" + + "cl3mDt: 过去3个月的数据日期\n" + + "DYCONS: 息率\n" + + "NSEID: NSE 股票代码\n" + + "52H: 52周最高价\n" + + "AvgDelVolPer_8day: 过去8天的平均成交量百分比\n" + + "52L: 52周最低价\n" + + "BV: 每股净值\n" + + "MKTLOT: 市场手数\n" + + "DVolAvg20: 过去20天的平均成交量\n" + + "OFFERP 和 OFFERQ: 卖出报价和数量\n" + + "cl6mVal: 过去6个月的数值\n" + + "AvgVolQtyDel_20day: 过去20天的平均成交量数量\n" + + "cl1mVal: 过去1个月的数值\n" + + "PBCONS: 市净率\n" + + "5DayAvg: 过去5天的平均值\n" + + "AvgDelVolPer_3day: 过去3天的平均成交量百分比\n" + + "sessionId: 会话ID\n" + + "TID: 交易ID\n" + + "SHRS: 股本总额\n" + + "50DayAvg: 过去50天的平均值\n" + + "cl3yChange: 过去3年的数值变化\n" + + "im_scid: 未知字段\n" + + "DVolAvg10: 过去10天的平均成交量\n" + + "AvgVolQtyTraded_20day: 过去20天的平均成交量交易数量\n" + + "symbol: 股票符号\n" + + "LP: 最后价格\n" + + "lower_circuit_limit 和 upper_circuit_limit: 跌停价和涨停价\n" + + "ty: 未知字段\n" + + "cl1wDt: 过去1周的数据日期\n" + + "cl5yDt: 过去5年的数据日期\n" + + "IND_PE: 行业市盈率\n" + + "cl1yPerChange: 过去1年的百分比变化\n" + + "cl3yVal: 过去3年的数值\n" + + "52HDate: 52周最高价的日期\n" + + "OPN: 开盘价\n" + + "cl1mPerChange: 过去1个月的百分比变化\n" + + "DTTIME: 交易时间\n" + + "SC_FULLNM: 完整的行业/部门名称\n" + + "DVolAvg5: 过去5天的平均成交量\n" + + "LTLDate: 最低值的日期\n" + + "DY: 息率\n" + + "etf: 是否为ETF\n" + + "clYtdPerChange: 本年度截至目前的百分比变化\n" + + "isinid: ISIN 编号\n" + + "cl2yDt: 过去2年的数据日期\n" + + "slug: 股票的 slug\n" + + "AvgDelVolPer_20day: 过去20天的平均成交量百分比\n" + + "cl5yVal: 过去5年的数值\n" + + "market_state: 市场状态\n" + + "cl2yPerChange: 过去2年的百分比变化\n" + + "pricechange: 价格变化\n" + + "AvgDelVolPer_5day: 过去5天的平均成交量百分比\n" + + "200DayAvg: 过去200天的平均值\n" + + "VOL: 成交量\n" + + "pricepercentchange: 价格百分比变化\n" + + "exchange: 交易所\n" + + "sc_mapindex: 行业/部门地图索引\n" + + "DVolAvg30: 过去30天的平均成交量\n" + + "cl1wChange: 过去1周的数值变化\n" + + "AVGP: 平均价格\n" + + "LTH: 最高价\n" + + "cl3yPerChange: 过去3年的百分比变化\n" + + "LTL: 最低价\n" + + "cl3mPerChange: 过去3个月的百分比变化\n" + + "PECONS: 市盈率\n" + + "priceprevclose: 前一交易日的收盘价\n" + + "30DayAvg: 过去30天的平均", response = JSONObject.class), + }) + @GetMapping("/api/market/money/getStockDetail") + @ResponseBody + public ServerResponse getStockDetail(@RequestParam String stockType, @RequestParam String symbol ) { + String url = String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s",stockType,symbol); + MoneyStock moneyStock = moneyStockRepository.findOne(QMoneyStockPO.moneyStockPO.stockType.eq(stockType).and(QMoneyStockPO.moneyStockPO.moneyScId.eq(symbol))).orElse(null); + /* if(moneyStock==null){ + return ServerResponse.createByErrorMsg("没有找到该股票"); + }*/ + // 设置重试次数 + int maxRetries = 3; + for (int retry = 1; retry <= maxRetries; retry++) { + try { + ResponseEntity responseEntity = restTemplate.exchange(url, HttpMethod.GET, null, String.class); + JSONObject json1 = new JSONObject(); + if (responseEntity.getStatusCode().value() == 200 && responseEntity.getBody() != null ) { + JSONObject data = JSONObject.parseObject(responseEntity.getBody()).getJSONObject("data"); + if(data!=null){ + json1.put("company",data.getString("company")); + json1.put("pricepercentchange",data.getString("pricepercentchange")); + json1.put("stockType",stockType); + json1.put("pricechange",data.getString("pricechange")); + json1.put("pricecurrent",data.getString("pricecurrent")); + json1.put("priceprevclose",data.getString("priceprevclose")); + json1.put("PREVDATE",data.getString("PREVDATE")); + json1.put("VOL",data.getString("VOL")); + json1.put("dataSourceType","3"); + json1.put("symbol",data.getString("symbol")); + json1.put("BSEID",data.getString("BSEID")); + json1.put("NSEID",data.getString("NSEID")); + json1.put("LTH",data.getString("HP")); + json1.put("LTL",data.getString("LP")); + json1.put("OPN",data.getString("OPN")); + if(null!=moneyStock){ + json1.put("id",moneyStock.getId()); + } + } + return ServerResponse.createBySuccess(json1); + } + + } catch (Exception e) { + } + // 如果不是最后一次重试,则等待一段时间再进行下一次重试 + if (retry < maxRetries) { + try { + // 1秒钟 + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + return null; + } + + + + private static List nseActives() { + List list = new ArrayList<>(); + String url = "https://www.moneycontrol.com/stocks/marketstats/nse-mostactive-stocks/nifty-50-9/"; + try { + Document doc = Jsoup.connect(url).get(); + int size = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div.bsr_table.hist_tbl_hm > table > tbody > tr").size(); + for (int i =1;i<=size;i++){ + MoneyStockSuggestDTO dto = new MoneyStockSuggestDTO(); + Element company_a = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child("+i+") > td.PR > span > a").first(); + if (company_a != null) { + String stockUrl = company_a.attr("href"); + String stockName = company_a.text(); + dto.setStockName(stockName); + dto.setStockUrl(stockUrl); + } + + String highPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child("+i+") > td:nth-child(2)").first()); + dto.setHighPrice(highPrice); + + String lowPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(3)").first()); + dto.setLowPrice(lowPrice); + + String lastPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(4)").first()); + dto.setLastPrice(lastPrice); + + String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first()); + + dto.setChangePercent(changePercent); + + dto.setStockType("nse"); + + list.add(dto); + log.info("---------------------------------------------" + i); + } + + } catch (IOException e) { + e.printStackTrace(); + } + return list; + } + + private static List bseActives() { + List list = new ArrayList<>(); + + String url = "https://www.moneycontrol.com/stocks/marketstats/bsemact1/index.php"; + try { + Document doc = Jsoup.connect(url).get(); + int size = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr").size(); + System.err.println(size); + for (int i =1;i<=size;i++){ + MoneyStockSuggestDTO dto = new MoneyStockSuggestDTO(); + Element company_a = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child("+i+") > td.PR > span > a").first(); + if (company_a != null) { + String stockUrl = company_a.attr("href"); + String stockName = company_a.text(); + dto.setStockName(stockName); + dto.setStockUrl(stockUrl); + } + + String highPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(2)").first()); + dto.setHighPrice(highPrice); + + String lowPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(3)").first()); + dto.setLowPrice(lowPrice); + + String lastPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(4)").first()); + dto.setLastPrice(lastPrice); + + String prevClosePrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first()); + dto.setPrevClosePrice(prevClosePrice); + + String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(6)").first()); + dto.setChangePercent(changePercent); + + dto.setStockType("bse"); + list.add(dto); + log.info("---------------------------------------------" + i); + } + + } catch (IOException e) { + log.error("occur Exception",e); + } + return list; + } + + private static List bseGainer() { + String url = "https://www.moneycontrol.com/stocks/marketstats/bse-gainer/sensex_4/"; + List list = Lists.newArrayList(); + try { + Document doc = Jsoup.connect(url).get(); + int size = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr").size(); + for (int i =1;i<=size;i++){ + Element company_a = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child("+i+") > td.PR > span > a").first(); + MoneyStockSuggestDTO dto = new MoneyStockSuggestDTO(); + if (company_a != null) { + String stockUrl = company_a.attr("href"); + String stockName = company_a.text(); + dto.setStockName(stockName); + dto.setStockUrl(stockUrl); + } + + String highPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(2)").first()); + dto.setHighPrice(highPrice); + + String lowPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(3)").first()); + dto.setLowPrice(lowPrice); + + String lastPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(4)").first()); + dto.setLastPrice(lastPrice); + + String prevClosePrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first()); + dto.setPrevClosePrice(prevClosePrice); + + String change = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(6)").first()); + dto.setChange(change); + + String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(7)").first()); + dto.setChangePercent(changePercent); + + dto.setStockType("bse"); + list.add(dto); + log.info("---------------------------------------------" + i); + } + + } catch (IOException e) { + e.printStackTrace(); + } + return list; + } + + private static List nseGainer() { + String url = "https://www.moneycontrol.com/stocks/marketstats/nsegainer/index.php"; + List list = Lists.newArrayList(); + try { + Document doc = Jsoup.connect(url).get(); + int size = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr").size(); + for (int i =1;i<=size;i++){ + MoneyStockSuggestDTO dto = new MoneyStockSuggestDTO(); + Element company_a = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child("+i+") > td.PR > span > h3 > a").first(); + if (company_a != null) { + String stockUrl = company_a.attr("href"); + String stockName = company_a.text(); + dto.setStockUrl(stockUrl); + dto.setStockName(stockName); + } + + String highPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(2)").first()); + dto.setHighPrice(highPrice); + + String lowPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(3)").first()); + dto.setLowPrice(lowPrice); + + String lastPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(4)").first()); + dto.setLastPrice(lastPrice); + + String prevClosePrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first()); + dto.setPrevClosePrice(prevClosePrice); + + String change = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(6)").first()); + dto.setChange(change); + + String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(7)").first()); + dto.setChangePercent(changePercent); + dto.setStockType("nse"); + list.add(dto); + } + + } catch (IOException e) { + e.printStackTrace(); + } + return list; + } + + private static List nseTopLoser() { + String url = "https://www.moneycontrol.com/stocks/marketstats/nseloser/index.php"; + List list = Lists.newArrayList(); + + try { + Document doc = Jsoup.connect(url).get(); + int size = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr").size(); + for (int i =1;i<=size;i++){ + MoneyStockSuggestDTO dto = new MoneyStockSuggestDTO(); + Element company_a = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child("+i+") > td.PR > span > h3 > a").first(); + if (company_a != null) { + String stockUrl = company_a.attr("href"); + String stockName = company_a.text(); + dto.setStockUrl(stockUrl); + dto.setStockName(stockName); + } + + String highPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(2)").first()); + dto.setHighPrice(highPrice); + String lowPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(3)").first()); + dto.setLowPrice(lowPrice); + String lastPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(4)").first()); + dto.setLastPrice(lastPrice); + + String prevClosePrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first()); + dto.setPrevClosePrice(prevClosePrice); + + String change = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(6)").first()); + dto.setChange(change); + + String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(7)").first()); + dto.setChangePercent(changePercent); + dto.setStockType("nse"); + list.add(dto); + log.info("---------------------------------------------" + i); + } + + } catch (IOException e) { + e.printStackTrace(); + } + return list; + } + + private static List bseTopLoser() { + String url = "https://www.moneycontrol.com/stocks/marketstats/bse-loser/sensex_4/"; + List list = Lists.newArrayList(); + + try { + Document doc = Jsoup.connect(url).get(); + int size = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr").size(); + for (int i =1;i<=size;i++){ + Element company_a = doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td.PR > span > a").first(); + MoneyStockSuggestDTO dto = new MoneyStockSuggestDTO(); + if (company_a != null) { + String stockUrl = company_a.attr("href"); + String stockName = company_a.text(); + dto.setStockName(stockName); + dto.setStockUrl(stockUrl); + + } + + String highPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(2)").first()); + dto.setHighPrice(highPrice); + + String lowPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(3)").first()); + dto.setLowPrice(lowPrice); + + String lastPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(4)").first()); + dto.setLastPrice(lastPrice); + + String prevClosePrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first()); + dto.setPrevClosePrice(prevClosePrice); + + String change = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(6)").first()); + dto.setChange(change); + + String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(7)").first()); + dto.setChangePercent(changePercent); + dto.setStockType("bse"); + list.add(dto); + log.info("---------------------------------------------" + i); + } + + } catch (IOException e) { + e.printStackTrace(); + } + return list; + } + + // 辅助方法,获取元素的文本或返回空字符串 + private static String getTextOrEmpty(Element element) { + return element != null ? element.text() : ""; + } + + + + + @ApiOperation(value = "股票推荐TopGainer",httpMethod = "GET") + @ApiImplicitParams({ + @ApiImplicitParam(name="stockType",value = "BSE或者NSE"), + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "" + + "股票推荐相关: top gainer", response = JSONObject.class), + }) + @GetMapping("/api/market/money/getTopGainer") + @ResponseBody + public List getTopGainer(@RequestParam String stockType) { + List moneyStockSuggestDTOS = null; + // 尝试从缓存中获取结果 + moneyStockSuggestDTOS = gainerStockSuggestCache.getIfPresent(stockType); + + if (moneyStockSuggestDTOS == null) { + // 缓存未命中,执行业务查询 + if (StringUtils.equals(stockType, "nse")) { + moneyStockSuggestDTOS = nseGainer(); + } else if (StringUtils.equals(stockType, "bse")) { + moneyStockSuggestDTOS = bseGainer(); + } + + moneyStockSuggestDTOS = moneyStockSuggestDTOS.stream().filter(f->StringUtils.isNotBlank(f.getStockName())).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(moneyStockSuggestDTOS)){ + List selfUlrList = moneyStockSuggestDTOS.stream().map(MoneyStockSuggestDTO::getStockName).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(selfUlrList)){ + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.in(selfUlrList)); + if(CollectionUtils.isNotEmpty(all)){ + moneyStockSuggestDTOS.stream().filter(f->all.stream().anyMatch(s->s.getStockName().equals(f.getStockName()))) + .forEach(f->f.setScId(all.stream().filter(s->s.getStockName().equals(f.getStockName())).findFirst().orElse(null).getMoneyScId())); + } + } + gainerStockSuggestCache.put(stockType, moneyStockSuggestDTOS); + } + // 将结果放入缓存 + } + return moneyStockSuggestDTOS; + } + + + @ApiOperation(value = "股票推荐TopLoser",httpMethod = "GET") + @ApiImplicitParams({ + @ApiImplicitParam(name="stockType",value = "BSE或者NSE"), + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "" + + "股票推荐相关: TopLoser", response = JSONObject.class), + }) + @GetMapping("/api/market/money/getTopLoser") + @ResponseBody + public List getTopLoser(@RequestParam String stockType) { + List moneyStockSuggestDTOS = null; + moneyStockSuggestDTOS = loserStockSuggestCache.getIfPresent(stockType); + if(null==moneyStockSuggestDTOS){ + if(StringUtils.equals(stockType,"nse")){ + moneyStockSuggestDTOS = nseTopLoser(); + }else if(StringUtils.equals(stockType,"bse")){ + moneyStockSuggestDTOS = bseTopLoser(); + } + moneyStockSuggestDTOS = moneyStockSuggestDTOS.stream().filter(f->StringUtils.isNotBlank(f.getStockName())).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(moneyStockSuggestDTOS)){ + moneyStockSuggestDTOS.stream().forEach(f->f.setDispId(extractLastSegment(f.getStockUrl()))); + List selfUlrList = moneyStockSuggestDTOS.stream().map(MoneyStockSuggestDTO::getStockName).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(selfUlrList)){ + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.in(selfUlrList)); + if(CollectionUtils.isNotEmpty(all)){ + moneyStockSuggestDTOS.stream().filter(f->all.stream().anyMatch(s->s.getStockName().equals(f.getStockName()))) + .forEach(f->f.setScId(all.stream().filter(s->s.getStockName().equals(f.getStockName())).findFirst().orElse(null).getMoneyScId())); + } + List noScIdList = moneyStockSuggestDTOS.stream().filter(f->StringUtils.isBlank(f.getScId())).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(noScIdList)){ + List dispIdList = noScIdList.stream().map(MoneyStockSuggestDTO::getDispId).collect(Collectors.toList()); + List all1 = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.selfDispId.in(dispIdList)); + if(CollectionUtils.isNotEmpty(all1)){ + moneyStockSuggestDTOS.stream().filter(f->all1.stream().anyMatch(s->s.getSelfDispId().equals(f.getDispId()))) + .forEach(f->f.setScId(all.stream().filter(s->s.getSelfDispId().equals(f.getDispId())).findFirst().orElse(null).getMoneyScId())); + } + } + } + loserStockSuggestCache.put(stockType, moneyStockSuggestDTOS); + } + } + return moneyStockSuggestDTOS; + } + + + + + @ApiOperation(value = "股票推荐TopActive",httpMethod = "GET") + @ApiImplicitParams({ + @ApiImplicitParam(name="stockType",value = "BSE或者NSE"), + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "" + + "股票推荐相关: top active", response = JSONObject.class), + }) + @GetMapping("/api/market/money/getTopActives") + @ResponseBody + public List getTopActive(@RequestParam String stockType) { + List moneyStockSuggestDTOS = null; + moneyStockSuggestDTOS = activesStockSuggestCache.getIfPresent(stockType); + if(moneyStockSuggestDTOS ==null){ + if(StringUtils.equals(stockType,"nse")){ + moneyStockSuggestDTOS = nseActives(); + }else if(StringUtils.equals(stockType,"bse")){ + moneyStockSuggestDTOS = bseActives(); + } + moneyStockSuggestDTOS = moneyStockSuggestDTOS.stream().filter(f->StringUtils.isNotBlank(f.getStockName())).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(moneyStockSuggestDTOS)){ + moneyStockSuggestDTOS.stream().forEach(f->f.setDispId(extractLastSegment(f.getStockUrl()))); + List selfUlrList = moneyStockSuggestDTOS.stream().map(MoneyStockSuggestDTO::getStockName).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(selfUlrList)){ + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.in(selfUlrList)); + if(CollectionUtils.isNotEmpty(all)){ + moneyStockSuggestDTOS.stream().filter(f->all.stream().anyMatch(s->s.getStockName().equals(f.getStockName()))) + .forEach(f->f.setScId(all.stream().filter(s->s.getStockName().equals(f.getStockName())).findFirst().orElse(null).getMoneyScId())); + } + List noScIdList = moneyStockSuggestDTOS.stream().filter(f->StringUtils.isBlank(f.getScId())).collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(noScIdList)){ + List dispIdList = noScIdList.stream().map(MoneyStockSuggestDTO::getDispId).collect(Collectors.toList()); + List all1 = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.selfDispId.in(dispIdList)); + if(CollectionUtils.isNotEmpty(all1)){ + moneyStockSuggestDTOS.stream().filter(f->all1.stream().anyMatch(s->s.getSelfDispId().equals(f.getDispId()))) + .forEach(f->f.setScId(all.stream().filter(s->s.getSelfDispId().equals(f.getDispId())).findFirst().orElse(null).getMoneyScId())); + } + } + } + activesStockSuggestCache.put(stockType, moneyStockSuggestDTOS); + } + } + + return moneyStockSuggestDTOS; + } + + + + @GetMapping("/api/market/money/history/kLine") + @ApiOperation(value = "获取kline的money数据源", notes = "获取kline的money数据源",response = StockHistoryResponse.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "symbol", value = "Stock symbol 对应的是NSEID 或者是BSEID", required = true, dataType = "String", paramType = "query"), + @ApiImplicitParam(name = "resolution", value = "单位:60 1D 1W 1D 对应H,D,W,Y", required = true, dataType = "String", paramType = "query"), + @ApiImplicitParam(name = "from", value = "Start timestamp", required = true, dataType = "long", paramType = "query"), + @ApiImplicitParam(name = "to", value = "End timestamp", required = true, dataType = "long", paramType = "query"), + @ApiImplicitParam(name = "countback", value = "开始时间和结束时间区间的计划展示的数量", required = true, dataType = "int", paramType = "query"), + @ApiImplicitParam(name = "currencyCode", value = "INR 不变", required = true, dataType = "String", paramType = "query") + }) + @ResponseBody + public ResponseEntity getStockHistory( @RequestParam String symbol, + @RequestParam String resolution + ) { + // 向外部API发起请求,并获取响应 + StockHistoryRequest request = new StockHistoryRequest(); + request.setSymbol(symbol); + Long to = null; + Long from = null; + int countback = 5; + if(StringUtils.equals("H",resolution)){ + to = (long) (System.currentTimeMillis() / 1000); + from = to - (1 * 60 * 60 ); + countback = 60; + request.setResolution("1"); + }else if(StringUtils.equals("D",resolution)){ + to = (long) (System.currentTimeMillis() / 1000); + from = to - (24 * 60 * 60 ); + countback = 9; + request.setResolution("60"); + }else if(StringUtils.equals("W",resolution)){ + to = (long) (System.currentTimeMillis() / 1000); + from = to - (10 * 24 * 60 * 60 ); + countback = 7; + request.setResolution("1D"); + }else if(StringUtils.equals("M",resolution)){ + to = (long) (System.currentTimeMillis() / 1000); + from = to - (35 * 24 * 60 * 60 ); + countback = 30; + request.setResolution("1D"); + } + + request.setFrom(from); + request.setTo(to); + request.setCountback(countback); + request.setCurrencyCode("INR"); + String apiUrl = buildApiUrl(request); + log.info("request url:"+apiUrl); + StockHistoryResponse response = null; + int maxRetries = 3; + int retryCount = 0; + + while (response == null && retryCount < maxRetries) { + try { + response = restTemplate.getForObject(apiUrl, StockHistoryResponse.class); + } catch (RestClientException e) { + // Log the exception or perform any other error handling + log.error("Error while making API request. Retrying... (Retry count: {})", retryCount + 1); + + // Increment the retry count + retryCount++; + // Add some delay before the next retry (you can adjust this as needed) + try { + Thread.sleep(300); // 1 second delay + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + } + + if (response != null) { + // API request successful, return the response + return ResponseEntity.ok(response); + } else { + // All retries failed, return an error response + log.error("Failed to get a successful response after {} retries.", maxRetries); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } + // 返回响应 + } + + + + + private String buildApiUrl(StockHistoryRequest request) { + // 构建外部API的URL + return String.format("%s?symbol=%s&resolution=%s&from=%d&to=%d&countback=%d¤cyCode=%s", + EXTERNAL_API_URL, request.getSymbol(), request.getResolution(), request.getFrom(), + request.getTo(), request.getCountback(), request.getCurrencyCode()); + }; + + + private static String extractLastSegment(String url) { + if (url == null) { + return null; + } + int lastSlashIndex = url.lastIndexOf('/'); + if (lastSlashIndex != -1 && lastSlashIndex < url.length() - 1) { + return url.substring(lastSlashIndex + 1); + } + return url; // 如果没有斜杠,或者斜杠位于字符串的末尾,则返回原始字符串 + } + + + public static void main(String[] args) { + nseGainer(); + nseActives(); + nseTopLoser(); + + bseActives(); + bseGainer(); + bseTopLoser(); + } + + private Cache> gainerStockSuggestCache = CacheBuilder.newBuilder() + .maximumSize(100) // 设置缓存的最大大小 + .expireAfterWrite(1, TimeUnit.HOURS) // 设置缓存条目的过期时间 + .build(); + + private Cache> loserStockSuggestCache = CacheBuilder.newBuilder() + .maximumSize(100) // 设置缓存的最大大小 + .expireAfterWrite(1, TimeUnit.HOURS) // 设置缓存条目的过期时间 + .build(); + + private Cache> activesStockSuggestCache = CacheBuilder.newBuilder() + .maximumSize(100) // 设置缓存的最大大小 + .expireAfterWrite(30, TimeUnit.MINUTES) // 设置缓存条目的过期时间 + .build(); + +} diff --git a/src/test/java/rp/lee/jpa/JpaDDDGen.java b/src/test/java/rp/lee/jpa/JpaDDDGen.java index 86c3255..98b0b7d 100644 --- a/src/test/java/rp/lee/jpa/JpaDDDGen.java +++ b/src/test/java/rp/lee/jpa/JpaDDDGen.java @@ -50,7 +50,7 @@ public class JpaDDDGen { /** * cs_statistic - 要生成的数据库表 */ - Cons.tableNameToEntiyMapping.put("stock_ipo", null); + Cons.tableNameToEntiyMapping.put("money_stock", null); ToolDDD.g(getMySQLDataSource().getConnection()); }