From 7f88f3be02e46e0d745dd1faabc23c61ca367f52 Mon Sep 17 00:00:00 2001 From: Achilles Date: Wed, 3 Jan 2024 21:20:05 +0800 Subject: [PATCH 1/2] =?UTF-8?q?money=20control=E4=BB=A3=E7=A0=81=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 7 + .../infrastructure/db/po/QMoneyStockPO.java | 49 ++++ .../assembler/MoneyStockAssembler.java | 35 +++ .../basic/convert/MoneyStockConvert.java | 23 ++ .../domain/basic/entity/MoneyStock.java | 28 ++ .../basic/factory/MoneyStockFactory.java | 28 ++ .../repository/MoneyStockRepository.java | 46 ++++ .../basic/service/MoneyStockService.java | 33 +++ .../cn/stock/market/dto/MoneyStockDTO.java | 23 ++ .../dto/command/MoneyStockCreateCommand.java | 48 ++++ .../dto/command/MoneyStockModifyCommand.java | 24 ++ .../infrastructure/db/po/MoneyStockPO.java | 66 +++++ .../db/repo/MoneyStockRepo.java | 15 + .../infrastructure/job/MoneyScraper.java | 260 ++++++++++++++++++ src/test/java/rp/lee/jpa/JpaDDDGen.java | 2 +- 15 files changed, 686 insertions(+), 1 deletion(-) create mode 100644 src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java create mode 100644 src/main/java/cn/stock/market/application/assembler/MoneyStockAssembler.java create mode 100644 src/main/java/cn/stock/market/domain/basic/convert/MoneyStockConvert.java create mode 100644 src/main/java/cn/stock/market/domain/basic/entity/MoneyStock.java create mode 100644 src/main/java/cn/stock/market/domain/basic/factory/MoneyStockFactory.java create mode 100644 src/main/java/cn/stock/market/domain/basic/repository/MoneyStockRepository.java create mode 100644 src/main/java/cn/stock/market/domain/basic/service/MoneyStockService.java create mode 100644 src/main/java/cn/stock/market/dto/MoneyStockDTO.java create mode 100644 src/main/java/cn/stock/market/dto/command/MoneyStockCreateCommand.java create mode 100644 src/main/java/cn/stock/market/dto/command/MoneyStockModifyCommand.java create mode 100644 src/main/java/cn/stock/market/infrastructure/db/po/MoneyStockPO.java create mode 100644 src/main/java/cn/stock/market/infrastructure/db/repo/MoneyStockRepo.java create mode 100644 src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java diff --git a/pom.xml b/pom.xml index 097f3d6..a25d07d 100644 --- a/pom.xml +++ b/pom.xml @@ -178,6 +178,13 @@ jsoup 1.15.2 + + + com.google.guava + guava + 30.1-jre + + diff --git a/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java b/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java new file mode 100644 index 0000000..6e9dc91 --- /dev/null +++ b/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java @@ -0,0 +1,49 @@ +package cn.stock.market.infrastructure.db.po; + +import static com.querydsl.core.types.PathMetadataFactory.*; + +import com.querydsl.core.types.dsl.*; + +import com.querydsl.core.types.PathMetadata; +import javax.annotation.Generated; +import com.querydsl.core.types.Path; + + +/** + * QMoneyStockPO is a Querydsl query type for MoneyStockPO + */ +@Generated("com.querydsl.codegen.EntitySerializer") +public class QMoneyStockPO extends EntityPathBase { + + private static final long serialVersionUID = -279042648L; + + public static final QMoneyStockPO moneyStockPO = new QMoneyStockPO("moneyStockPO"); + + public final StringPath detailUrl = createString("detailUrl"); + + public final NumberPath id = createNumber("id", Integer.class); + + public final StringPath moneyScId = createString("moneyScId"); + + public final DateTimePath saveTime = createDateTime("saveTime", java.util.Date.class); + + public final StringPath selfUrl = createString("selfUrl"); + + public final StringPath stockName = createString("stockName"); + + public final StringPath stockType = createString("stockType"); + + public QMoneyStockPO(String variable) { + super(MoneyStockPO.class, forVariable(variable)); + } + + public QMoneyStockPO(Path path) { + super(path.getType(), path.getMetadata()); + } + + public QMoneyStockPO(PathMetadata metadata) { + super(MoneyStockPO.class, metadata); + } + +} + diff --git a/src/main/java/cn/stock/market/application/assembler/MoneyStockAssembler.java b/src/main/java/cn/stock/market/application/assembler/MoneyStockAssembler.java new file mode 100644 index 0000000..e308295 --- /dev/null +++ b/src/main/java/cn/stock/market/application/assembler/MoneyStockAssembler.java @@ -0,0 +1,35 @@ +package cn.stock.market.application.assembler; + +import cn.qutaojing.common.utils.Beans; +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.dto.MoneyStockDTO; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * MoneyStockAssembler + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Component +@Lazy +public class MoneyStockAssembler { + public MoneyStockDTO toDTO(MoneyStock e) { + MoneyStockDTO dto = Beans.mapper(e, MoneyStockDTO.class); + if(dto == null) return dto; + fill(e, dto); + return dto; + } + + protected void fill(MoneyStock e, MoneyStockDTO dto) { + if(dto == null) return; + return; + } + + public static MoneyStockAssembler of() { + return SpringUtils.getBean(MoneyStockAssembler.class); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/convert/MoneyStockConvert.java b/src/main/java/cn/stock/market/domain/basic/convert/MoneyStockConvert.java new file mode 100644 index 0000000..b990c1e --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/convert/MoneyStockConvert.java @@ -0,0 +1,23 @@ +package cn.stock.market.domain.basic.convert; + +import cn.qutaojing.common.domain.convert.SimpleEntityPOConvert; +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * MoneyStockConvert + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Component +@Lazy +public class MoneyStockConvert extends SimpleEntityPOConvert { + public static MoneyStockConvert of() { + return SpringUtils.getBean(MoneyStockConvert.class); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/entity/MoneyStock.java b/src/main/java/cn/stock/market/domain/basic/entity/MoneyStock.java new file mode 100644 index 0000000..77a6a25 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/entity/MoneyStock.java @@ -0,0 +1,28 @@ +package cn.stock.market.domain.basic.entity; + +import cn.qutaojing.common.utils.Beans; +import cn.stock.market.dto.command.MoneyStockCreateCommand; +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * MoneyStock + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Data +@NoArgsConstructor +@SuperBuilder +@EqualsAndHashCode( + callSuper = false +) +public class MoneyStock extends MoneyStockPO { + public void update(MoneyStockCreateCommand cmd) { + Beans.copyProperties(cmd, this); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/factory/MoneyStockFactory.java b/src/main/java/cn/stock/market/domain/basic/factory/MoneyStockFactory.java new file mode 100644 index 0000000..8f9dfd2 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/factory/MoneyStockFactory.java @@ -0,0 +1,28 @@ +package cn.stock.market.domain.basic.factory; + +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.dto.command.MoneyStockCreateCommand; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * MoneyStockFactory + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Component +@Lazy +public class MoneyStockFactory { + public static MoneyStockFactory of() { + return SpringUtils.getBean(MoneyStockFactory.class); + } + + public MoneyStock from(MoneyStockCreateCommand cmd) { + MoneyStock e = MoneyStock.builder().build(); + e.update(cmd); + return e; + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/repository/MoneyStockRepository.java b/src/main/java/cn/stock/market/domain/basic/repository/MoneyStockRepository.java new file mode 100644 index 0000000..dd70882 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/repository/MoneyStockRepository.java @@ -0,0 +1,46 @@ +package cn.stock.market.domain.basic.repository; + +import cn.qutaojing.common.domain.convert.IEntityPOConvert; +import cn.qutaojing.common.domain.respostory.SimplePoConvertEntityRepository; +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.convert.MoneyStockConvert; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import cn.stock.market.infrastructure.db.repo.MoneyStockRepo; +import com.rp.spring.jpa.GenericJpaRepository; +import java.lang.Integer; +import java.lang.Override; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +/** + * MoneyStockRepository + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Repository +@RequiredArgsConstructor( + onConstructor = @__(@Autowired) +) +public class MoneyStockRepository extends SimplePoConvertEntityRepository { + final MoneyStockRepo repo; + + final MoneyStockConvert convert; + + @Override + public GenericJpaRepository repo() { + return repo; + } + + @Override + public IEntityPOConvert convert() { + return convert; + } + + public static MoneyStockRepository of() { + return SpringUtils.getBean(MoneyStockRepository.class); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/service/MoneyStockService.java b/src/main/java/cn/stock/market/domain/basic/service/MoneyStockService.java new file mode 100644 index 0000000..0c3b1e7 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/service/MoneyStockService.java @@ -0,0 +1,33 @@ +package cn.stock.market.domain.basic.service; + +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.factory.MoneyStockFactory; +import cn.stock.market.domain.basic.repository.MoneyStockRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * MoneyStockService + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Service +@RequiredArgsConstructor( + onConstructor = @__(@Autowired) +) +public class MoneyStockService { + final MoneyStockRepository repository; + + final MoneyStockFactory factory; + + public MoneyStockRepository repository() { + return repository; + } + + public static MoneyStockService of() { + return SpringUtils.getBean(MoneyStockService.class); + } +} diff --git a/src/main/java/cn/stock/market/dto/MoneyStockDTO.java b/src/main/java/cn/stock/market/dto/MoneyStockDTO.java new file mode 100644 index 0000000..0cf6efb --- /dev/null +++ b/src/main/java/cn/stock/market/dto/MoneyStockDTO.java @@ -0,0 +1,23 @@ +package cn.stock.market.dto; + +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * MoneyStockDTO + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Data +@NoArgsConstructor +@SuperBuilder +@EqualsAndHashCode( + callSuper = false +) +public class MoneyStockDTO extends MoneyStockPO { +} diff --git a/src/main/java/cn/stock/market/dto/command/MoneyStockCreateCommand.java b/src/main/java/cn/stock/market/dto/command/MoneyStockCreateCommand.java new file mode 100644 index 0000000..3306f01 --- /dev/null +++ b/src/main/java/cn/stock/market/dto/command/MoneyStockCreateCommand.java @@ -0,0 +1,48 @@ +package cn.stock.market.dto.command; + +import java.lang.Integer; +import java.lang.String; +import java.util.Date; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * MoneyStockCreateCommand + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Data +@SuperBuilder +@NoArgsConstructor +public class MoneyStockCreateCommand { + /** + * 主键 */ + Integer id; + + /** + * 股票名称 */ + String stockName; + + /** + * BSE or NSE */ + String stockType; + + /** + * money Control的id */ + String moneyScId; + + /** + * 自有self_url */ + String selfUrl; + + /** + * 细节url */ + String detailUrl; + + /** + * 保存时间 */ + Date saveTime; +} diff --git a/src/main/java/cn/stock/market/dto/command/MoneyStockModifyCommand.java b/src/main/java/cn/stock/market/dto/command/MoneyStockModifyCommand.java new file mode 100644 index 0000000..f6c4816 --- /dev/null +++ b/src/main/java/cn/stock/market/dto/command/MoneyStockModifyCommand.java @@ -0,0 +1,24 @@ +package cn.stock.market.dto.command; + +import java.lang.Integer; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * MoneyStockModifyCommand + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@Data +@SuperBuilder +@NoArgsConstructor +@EqualsAndHashCode( + callSuper = false +) +public class MoneyStockModifyCommand extends MoneyStockCreateCommand { + Integer id; +} diff --git a/src/main/java/cn/stock/market/infrastructure/db/po/MoneyStockPO.java b/src/main/java/cn/stock/market/infrastructure/db/po/MoneyStockPO.java new file mode 100644 index 0000000..78d83b7 --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/db/po/MoneyStockPO.java @@ -0,0 +1,66 @@ +package cn.stock.market.infrastructure.db.po; + +import java.lang.Integer; +import java.lang.String; +import java.util.Date; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.hibernate.annotations.DynamicInsert; +import org.hibernate.annotations.DynamicUpdate; + +/** + * MoneyStockPO + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +@SuperBuilder +@Data +@NoArgsConstructor +@AllArgsConstructor +@Entity +@DynamicInsert +@DynamicUpdate +@Table( + name = "money_stock" +) +public class MoneyStockPO { + /** + * 主键 */ + @Id + @GeneratedValue( + strategy = javax.persistence.GenerationType.IDENTITY + ) + Integer id; + + /** + * 股票名称 */ + String stockName; + + /** + * BSE or NSE */ + String stockType; + + /** + * money Control的id */ + String moneyScId; + + /** + * 自有self_url */ + String selfUrl; + + /** + * 细节url */ + String detailUrl; + + /** + * 保存时间 */ + Date saveTime; +} diff --git a/src/main/java/cn/stock/market/infrastructure/db/repo/MoneyStockRepo.java b/src/main/java/cn/stock/market/infrastructure/db/repo/MoneyStockRepo.java new file mode 100644 index 0000000..883bc4e --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/db/repo/MoneyStockRepo.java @@ -0,0 +1,15 @@ +package cn.stock.market.infrastructure.db.repo; + +import cn.stock.market.infrastructure.db.po.MoneyStockPO; +import com.rp.spring.jpa.GenericJpaRepository; +import java.lang.Integer; + +/** + * MoneyStockRepo + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2024/01/03 + */ +public interface MoneyStockRepo extends GenericJpaRepository { +} diff --git a/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java b/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java new file mode 100644 index 0000000..68cdeba --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java @@ -0,0 +1,260 @@ +package cn.stock.market.infrastructure.job; + +import cn.hutool.core.collection.CollectionUtil; +import cn.stock.market.domain.basic.entity.MoneyStock; +import cn.stock.market.domain.basic.repository.MoneyStockRepository; +import cn.stock.market.infrastructure.db.po.QMoneyStockPO; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClients; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author gs + * @date 2024/1/2 17:31 + */ +@RestController +@Slf4j +public class MoneyScraper { + + private static final int MAX_RETRY_ATTEMPTS = 10; + private static final int NUM_THREADS = 5; + + + @Autowired + private MoneyStockRepository moneyStockRepository; + + @GetMapping("testScraperGetMoneyControlStock") + public void schedule(){ + List letters = new ArrayList<>(); + for (char c = 'A'; c <= 'Z'; c++) { + letters.add(String.valueOf(c)); + } + letters.add("others"); + + int tasksPerThread = (int) Math.ceil((double) letters.size() / NUM_THREADS); + + // 手动创建线程池 + ThreadPoolExecutor executorService = new ThreadPoolExecutor( + NUM_THREADS, NUM_THREADS, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>()); + + List>> futures = new ArrayList<>(); + + for (int i = 0; i < NUM_THREADS; i++) { + int startIndex = i * tasksPerThread; + int endIndex = Math.min((i + 1) * tasksPerThread, letters.size()); + List letterRange = letters.subList(startIndex, endIndex); + CompletableFuture> future = CompletableFuture.supplyAsync(() -> processLetters(letterRange), executorService); + futures.add(future); + } + + List results = new ArrayList<>(); + + for (CompletableFuture> future : futures) { + try { + results.addAll(future.get()); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + } + + executorService.shutdown(); + + log.error("All threads completed. Results:"); + for (int i = 0; i < results.size(); i++) { + log.error("Thread " + (i + 1) + " processed letters: " + results.get(i)); + } + + } + + + private List sendHttpRequest(String url, HttpClient httpClient, String letter) throws IOException { + Document document = fetchStockDetails(url); + extractExchangeDetails(document); + List result = new ArrayList<>(); + result.add("Thread " + Thread.currentThread().getName() + " processed letters: " + letter); + return result; + } + + + private List processLetters(List letterRange) { + log.error("Thread " + Thread.currentThread().getName() + " is processing letters: " + letterRange); + String urlBase = "https://www.moneycontrol.com/india/stockpricequote/%s"; + HttpClient httpClient = HttpClients.createDefault(); + List results = new ArrayList<>(); + + for (String letter : letterRange) { + String detailUrl = String.format(urlBase, letter); + log.error("Thread " + Thread.currentThread().getName() + ",请求的url:" + detailUrl); + + // 重试逻辑 + int maxAttempts = 10; + int attempt = 1; + + while (attempt <= maxAttempts) { + try { + List response = sendHttpRequest(detailUrl, httpClient, letter) ; + results.addAll(response); + break; // 如果成功则跳出循环 + } catch (IOException | RuntimeException e) { + // 处理异常的逻辑 + e.printStackTrace(); + log.error("Attempt " + attempt + " failed. Retrying..."); + attempt++; + try { + Thread.sleep(1000); // 休眠1秒 + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + } + } + + return results; + } + + + public static Document fetchStockDetails(String url) { + return fetchDocumentWithRetry(url); + } + + public static Document fetchCompanyDetails(String url) { + return fetchDocumentWithRetry(url); + } + + private static Document fetchDocumentWithRetry(String url) { + HttpClient httpClient = HttpClients.createDefault(); + int retryAttempts = 0; + + while (retryAttempts < MAX_RETRY_ATTEMPTS) { + try { + HttpGet request = new HttpGet(url); + request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"); + + HttpResponse response = httpClient.execute(request); + int statusCode = response.getStatusLine().getStatusCode(); + + if (statusCode >= 200 && statusCode < 300) { + return Jsoup.parse(response.getEntity().getContent(), null, url); + } else { + retryAttempts++; + log.error("HTTP request failed with status code: " + statusCode); + } + } catch (IOException e) { + retryAttempts++; + log.error(Thread.currentThread().getName()+",Exception during HTTP request: " +retryAttempts); + } + } + + return null; // Retry attempts exhausted + } + + + public void extractExchangeDetails(Document soup) { + Elements companies = soup.select("table.pcq_tbl.MT10"); + + for (Element company : companies) { + Elements elements = company.select("tr > td > a"); + + for (Element element : elements) { + String textContent = element.text().trim(); + String linkAttribute = element.attr("href"); + + log.info(Thread.currentThread().getName()+",Text Content: " + textContent + ", Link Attribute: " + linkAttribute); + Document soup2 = fetchCompanyDetails(linkAttribute); + + if (soup2 != null) { + Element comIdInput = soup2.selectFirst("input[id=ap_sc_id]"); + String companyCodeId = ""; + if (comIdInput != null) { + companyCodeId = comIdInput.val(); + log.info(Thread.currentThread().getName()+",the stockName: " + textContent + ", THE input id: " + companyCodeId); + } else { + log.error(Thread.currentThread().getName()+" No with id='ap_sc_id' found on the website."); + } + + if (soup2 != null) { + Element ulElement = soup2.selectFirst("ul[id=nseBseTab]"); + + if (ulElement != null) { + for (Element aElement : ulElement.select("a")) { + String exchangeValue = aElement.text().trim(); + + if ("BSE".equals(exchangeValue) || "NSE".equals(exchangeValue)) { + log.info(Thread.currentThread().getName()+",stockName: " + textContent + ", self_link: " + linkAttribute + + ", the exchange Value: " + exchangeValue); + MoneyStock build = MoneyStock.builder().stockName(textContent).stockType(exchangeValue.toLowerCase(Locale.ROOT)) + .detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId)) + .selfUrl(linkAttribute) + .moneyScId(companyCodeId).saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent), QMoneyStockPO.moneyStockPO.stockType.eq(exchangeValue.toLowerCase(Locale.ROOT))); + if(CollectionUtil.isEmpty(all)){ + moneyStockRepository.save(build); + } + } else { + log.error(Thread.currentThread().getName()+", stockName: " + textContent + ", self_link: " + linkAttribute + + " is not a valid exchange type"); + MoneyStock build = MoneyStock.builder().stockName(textContent) + .detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId)) + .selfUrl(linkAttribute) + .moneyScId(companyCodeId).saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent)); + if(CollectionUtil.isEmpty(all)){ + moneyStockRepository.save(build); + } + } + } + } else { + log.info("stockName: " + textContent + ", self_link: " + linkAttribute + + " has no current exchange types"); + MoneyStock build = MoneyStock.builder().stockName(textContent).selfUrl(linkAttribute) + .moneyScId(companyCodeId).saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent)); + if(CollectionUtil.isEmpty(all)){ + moneyStockRepository.save(build); + } + } + } + } else { + log.info(Thread.currentThread().getName()+",stockName: " + textContent + ", self_link: " + linkAttribute + + " cannot find corresponding stock id"); + MoneyStock build = MoneyStock.builder().stockName(textContent).selfUrl(linkAttribute) + .saveTime(new Date()).build(); + List all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(textContent)); + if(CollectionUtil.isEmpty(all)){ + moneyStockRepository.save(build); + } + } + } + } + } + + + + + + +} diff --git a/src/test/java/rp/lee/jpa/JpaDDDGen.java b/src/test/java/rp/lee/jpa/JpaDDDGen.java index 86c3255..98b0b7d 100644 --- a/src/test/java/rp/lee/jpa/JpaDDDGen.java +++ b/src/test/java/rp/lee/jpa/JpaDDDGen.java @@ -50,7 +50,7 @@ public class JpaDDDGen { /** * cs_statistic - 要生成的数据库表 */ - Cons.tableNameToEntiyMapping.put("stock_ipo", null); + Cons.tableNameToEntiyMapping.put("money_stock", null); ToolDDD.g(getMySQLDataSource().getConnection()); } From 15b654e4255025c0fa66dc2be78c64c60fcec3e8 Mon Sep 17 00:00:00 2001 From: Achilles Date: Wed, 3 Jan 2024 22:27:54 +0800 Subject: [PATCH 2/2] =?UTF-8?q?money=20control=E4=BB=A3=E7=A0=81=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infrastructure/job/MoneyScraper.java | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java b/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java index 68cdeba..ef38799 100644 --- a/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java +++ b/src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java @@ -29,6 +29,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * @author gs @@ -91,9 +92,18 @@ public class MoneyScraper { } + /** + * 带有A B 分组的url + * @param url + * @param httpClient + * @param letter + * @return + * @throws IOException + */ private List sendHttpRequest(String url, HttpClient httpClient, String letter) throws IOException { + List allMoneyStock = moneyStockRepository.findAll(); Document document = fetchStockDetails(url); - extractExchangeDetails(document); + extractExchangeDetails(document,allMoneyStock); List result = new ArrayList<>(); result.add("Thread " + Thread.currentThread().getName() + " processed letters: " + letter); return result; @@ -137,6 +147,11 @@ public class MoneyScraper { } + /** + * 获取全部股票url_self + * @param url + * @return + */ public static Document fetchStockDetails(String url) { return fetchDocumentWithRetry(url); } @@ -173,16 +188,19 @@ public class MoneyScraper { } - public void extractExchangeDetails(Document soup) { + public void extractExchangeDetails(Document soup,List moneyStockHadSavedList) { Elements companies = soup.select("table.pcq_tbl.MT10"); - + List hadSaveUrl = moneyStockHadSavedList.stream().map(MoneyStock::getSelfUrl).collect(Collectors.toList()); for (Element company : companies) { Elements elements = company.select("tr > td > a"); for (Element element : elements) { String textContent = element.text().trim(); String linkAttribute = element.attr("href"); - + if(hadSaveUrl.contains(linkAttribute)){ + log.error(Thread.currentThread().getName()+"已经存在了不需要重复保存,company_name: " + textContent + ", Link Attribute: " + linkAttribute); + continue; + } log.info(Thread.currentThread().getName()+",Text Content: " + textContent + ", Link Attribute: " + linkAttribute); Document soup2 = fetchCompanyDetails(linkAttribute);