diff --git a/pom.xml b/pom.xml index bd70d8f..341677e 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,12 @@ 4.1.0 + + org.json + json + 20240303 + + io.springfox springfox-swagger2 diff --git a/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java b/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java index b719747..7e6ffb0 100644 --- a/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java +++ b/src/main/generated/cn/stock/market/infrastructure/db/po/QMoneyStockPO.java @@ -29,6 +29,10 @@ public class QMoneyStockPO extends EntityPathBase { public final StringPath moneyScId = createString("moneyScId"); + public final StringPath nseIndiaChartId = createString("nseIndiaChartId"); + + public final StringPath nseIndiaId = createString("nseIndiaId"); + public final DateTimePath saveTime = createDateTime("saveTime", java.util.Date.class); public final StringPath selfDispId = createString("selfDispId"); @@ -39,6 +43,8 @@ public class QMoneyStockPO extends EntityPathBase { public final StringPath stockType = createString("stockType"); + public final BooleanPath useFromBseindia = createBoolean("useFromBseindia"); + public QMoneyStockPO(String variable) { super(MoneyStockPO.class, forVariable(variable)); } diff --git a/src/main/generated/cn/stock/market/infrastructure/db/po/QStockIpoPO.java b/src/main/generated/cn/stock/market/infrastructure/db/po/QStockIpoPO.java index 657c8d4..3a39af1 100644 --- a/src/main/generated/cn/stock/market/infrastructure/db/po/QStockIpoPO.java +++ b/src/main/generated/cn/stock/market/infrastructure/db/po/QStockIpoPO.java @@ -23,6 +23,8 @@ public class QStockIpoPO extends EntityPathBase { public final DateTimePath createDate = createDateTime("createDate", java.util.Date.class); + public final StringPath exchangeType = createString("exchangeType"); + public final NumberPath id = createNumber("id", Integer.class); public final NumberPath isList = createNumber("isList", Integer.class); @@ -38,7 +40,6 @@ public class QStockIpoPO extends EntityPathBase { public final StringPath stockCode = createString("stockCode"); public final StringPath stockName = createString("stockName"); - public final StringPath exchangeType = createString("exchangeType"); public final NumberPath stockPrice = createNumber("stockPrice", java.math.BigDecimal.class); diff --git a/src/main/generated/cn/stock/market/infrastructure/db/po/QStockTempPO.java b/src/main/generated/cn/stock/market/infrastructure/db/po/QStockTempPO.java new file mode 100644 index 0000000..c4dd448 --- /dev/null +++ b/src/main/generated/cn/stock/market/infrastructure/db/po/QStockTempPO.java @@ -0,0 +1,43 @@ +package cn.stock.market.infrastructure.db.po; + +import static com.querydsl.core.types.PathMetadataFactory.*; + +import com.querydsl.core.types.dsl.*; + +import com.querydsl.core.types.PathMetadata; +import javax.annotation.Generated; +import com.querydsl.core.types.Path; + + +/** + * QStockTempPO is a Querydsl query type for StockTempPO + */ +@Generated("com.querydsl.codegen.EntitySerializer") +public class QStockTempPO extends EntityPathBase { + + private static final long serialVersionUID = 1973569078L; + + public static final QStockTempPO stockTempPO = new QStockTempPO("stockTempPO"); + + public final NumberPath id = createNumber("id", Integer.class); + + public final StringPath stockGid = createString("stockGid"); + + public final StringPath stockLogo = createString("stockLogo"); + + public final StringPath stockType = createString("stockType"); + + public QStockTempPO(String variable) { + super(StockTempPO.class, forVariable(variable)); + } + + public QStockTempPO(Path path) { + super(path.getType(), path.getMetadata()); + } + + public QStockTempPO(PathMetadata metadata) { + super(StockTempPO.class, metadata); + } + +} + diff --git a/src/main/java/cn/stock/market/domain/basic/convert/StockTempConvert.java b/src/main/java/cn/stock/market/domain/basic/convert/StockTempConvert.java new file mode 100644 index 0000000..8874d4a --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/convert/StockTempConvert.java @@ -0,0 +1,25 @@ +package cn.stock.market.domain.basic.convert; + +import cn.qutaojing.common.domain.convert.SimpleEntityPOConvert; +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.entity.Stock; +import cn.stock.market.domain.basic.entity.StockTemp; +import cn.stock.market.infrastructure.db.po.StockPO; +import cn.stock.market.infrastructure.db.po.StockTempPO; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * StockConvert + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2023/06/17 + */ +@Component +@Lazy +public class StockTempConvert extends SimpleEntityPOConvert { + public static StockTempConvert of() { + return SpringUtils.getBean(StockTempConvert.class); + } +} diff --git a/src/main/java/cn/stock/market/domain/basic/entity/StockTemp.java b/src/main/java/cn/stock/market/domain/basic/entity/StockTemp.java new file mode 100644 index 0000000..e81ad33 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/entity/StockTemp.java @@ -0,0 +1,26 @@ +package cn.stock.market.domain.basic.entity; + +import cn.qutaojing.common.utils.Beans; +import cn.stock.market.dto.command.StockCreateCommand; +import cn.stock.market.infrastructure.db.po.StockPO; +import cn.stock.market.infrastructure.db.po.StockTempPO; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * Stock + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2023/06/17 + */ +@Data +@NoArgsConstructor +@SuperBuilder +@EqualsAndHashCode( + callSuper = false +) +public class StockTemp extends StockTempPO { +} diff --git a/src/main/java/cn/stock/market/domain/basic/repository/StockRepository.java b/src/main/java/cn/stock/market/domain/basic/repository/StockRepository.java index bd02858..39ed937 100644 --- a/src/main/java/cn/stock/market/domain/basic/repository/StockRepository.java +++ b/src/main/java/cn/stock/market/domain/basic/repository/StockRepository.java @@ -54,6 +54,11 @@ public class StockRepository extends LocalCacheRepository cacheGidMap() { + Map map = cacheList().stream().collect(Collectors.toMap(Stock::getStockGid, val -> val, (u, v) -> u)); + return map; + } public Map cacheNameMap() { Map map = cacheList() diff --git a/src/main/java/cn/stock/market/domain/basic/repository/StockTempRepository.java b/src/main/java/cn/stock/market/domain/basic/repository/StockTempRepository.java new file mode 100644 index 0000000..1742f51 --- /dev/null +++ b/src/main/java/cn/stock/market/domain/basic/repository/StockTempRepository.java @@ -0,0 +1,63 @@ +package cn.stock.market.domain.basic.repository; + +import cn.qutaojing.common.domain.convert.IEntityPOConvert; +import cn.qutaojing.common.domain.respostory.LocalCacheBean; +import cn.qutaojing.common.domain.respostory.LocalCacheRepository; +import cn.qutaojing.common.jpa.ConditionBuilder; +import cn.qutaojing.common.utils.SpringUtils; +import cn.stock.market.domain.basic.convert.StockConvert; +import cn.stock.market.domain.basic.convert.StockTempConvert; +import cn.stock.market.domain.basic.entity.Stock; +import cn.stock.market.domain.basic.entity.StockTemp; +import cn.stock.market.infrastructure.db.po.QStockPO; +import cn.stock.market.infrastructure.db.po.QStockTempPO; +import cn.stock.market.infrastructure.db.po.StockPO; +import cn.stock.market.infrastructure.db.po.StockTempPO; +import cn.stock.market.infrastructure.db.repo.StockRepo; +import cn.stock.market.infrastructure.db.repo.StockTempRepo; +import com.google.common.cache.CacheBuilder; +import com.rp.spring.jpa.GenericJpaRepository; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * StockRepository + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2023/06/17 + */ +@Repository +@RequiredArgsConstructor(onConstructor = @__(@Autowired)) +public class StockTempRepository extends LocalCacheRepository { + final StockTempRepo repo; + final StockTempConvert convert; + final static QStockTempPO q = QStockTempPO.stockTempPO; + + public List findAllForCheck() { + return findAll(); + } + + @Override + public GenericJpaRepository repo() { + return repo; + } + + @Override + public IEntityPOConvert convert() { + return convert; + } + + public static StockTempRepository of() { + return SpringUtils.getBean(StockTempRepository.class); + } +} diff --git a/src/main/java/cn/stock/market/dto/model/Symbol.java b/src/main/java/cn/stock/market/dto/model/Symbol.java new file mode 100644 index 0000000..16990c4 --- /dev/null +++ b/src/main/java/cn/stock/market/dto/model/Symbol.java @@ -0,0 +1,28 @@ +package cn.stock.market.dto.model; + +import lombok.Data; + +import java.util.List; + +@Data +public class Symbol { + private String symbol; + private String description; + private String stockGid; + private String type; + private String exchange; + private String currencyCode; + private String currencyLogoid; + private String logoid; + private String providerId; + private String sourceLogoid; + private String sourceId; + private String country; + private boolean isPrimaryListing; + private List typespecs; + + // Getters and setters + // Bạn có thể dùng Lombok nếu muốn ngắn gọn hơn + // Hoặc generate getter/setter trong IDE + +} diff --git a/src/main/java/cn/stock/market/infrastructure/db/po/StockTempPO.java b/src/main/java/cn/stock/market/infrastructure/db/po/StockTempPO.java new file mode 100644 index 0000000..6801bc8 --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/db/po/StockTempPO.java @@ -0,0 +1,39 @@ +package cn.stock.market.infrastructure.db.po; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.hibernate.annotations.DynamicInsert; +import org.hibernate.annotations.DynamicUpdate; + +import javax.persistence.*; +import java.math.BigDecimal; +import java.util.Date; + +/** + * StockPO + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2023/06/17 + */ +@SuperBuilder +@Data +@NoArgsConstructor +@AllArgsConstructor +@Entity +@DynamicInsert +@DynamicUpdate +@Table(name = "stockTemp") +public class StockTempPO { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + Integer id; + + String stockType; + + String stockGid; + + String stockLogo; +} diff --git a/src/main/java/cn/stock/market/infrastructure/db/repo/StockTempRepo.java b/src/main/java/cn/stock/market/infrastructure/db/repo/StockTempRepo.java new file mode 100644 index 0000000..6ef0ad4 --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/db/repo/StockTempRepo.java @@ -0,0 +1,15 @@ +package cn.stock.market.infrastructure.db.repo; + +import cn.stock.market.infrastructure.db.po.StockPO; +import cn.stock.market.infrastructure.db.po.StockTempPO; +import com.rp.spring.jpa.GenericJpaRepository; + +/** + * StockRepo + * + * @author rplees + * @email rplees.i.ly@gmail.com + * @created 2023/06/17 + */ +public interface StockTempRepo extends GenericJpaRepository { +} diff --git a/src/main/java/cn/stock/market/infrastructure/job/StockNewTask.java b/src/main/java/cn/stock/market/infrastructure/job/StockNewTask.java new file mode 100644 index 0000000..9b8b1bb --- /dev/null +++ b/src/main/java/cn/stock/market/infrastructure/job/StockNewTask.java @@ -0,0 +1,104 @@ +package cn.stock.market.infrastructure.job; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; + +import cn.stock.market.domain.basic.entity.Stock; +import cn.stock.market.domain.basic.entity.StockTemp; +import cn.stock.market.domain.basic.repository.StockRepository; +import cn.stock.market.domain.basic.repository.StockTempRepository; +import cn.stock.market.dto.model.Symbol; +import cn.stock.market.infrastructure.db.po.StockPO; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.json.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class StockNewTask { + private static final OkHttpClient client = new OkHttpClient(); + private static final String BASE_URL = "https://symbol-search.tradingview.com/symbol_search/v3/"; + private static final String PARAMS = "?start={start}&hl=1&country=DE&lang=en&search_type=stocks&domain=production&sort_by_country=US&promo=true&exchange={exchange}"; + + @Autowired + StockRepository stockRepository; + + +// @PostConstruct + @Scheduled(cron = "0 01 22 * * ?") + public void syncStock() throws Exception { + + int limit = 20000; + List exchanges = Arrays.asList("BER", "DUS", "HAM", "HAN", "MUN", "SWB", "FWB", "XETR"); + Map stockGidMap = stockRepository.cacheGidMap(); + for (String exchange : exchanges) { + List newStocks = new ArrayList<>(); + int start = 0; + int symbolsRemaining; + do { + int finalStart = start; + if(start > 0){ + finalStart = start + 1; + } + String url = BASE_URL + PARAMS.replace("{start}", String.valueOf(finalStart)).replace("{exchange}", exchange); + Request request = new Request.Builder() + .url(url) + .addHeader("Origin", "https://www.tradingview.com") + .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)") + .addHeader("Accept", "application/json, text/plain, */*") + .addHeader("Referer", "https://www.tradingview.com/") + .build(); + + Response response = client.newCall(request).execute(); + + if (!response.isSuccessful()) { + throw new RuntimeException("Unexpected response code: " + response.code()); + } + + String responseBody = response.body().string(); + JSONObject json = new JSONObject(responseBody); + JSONArray symbols = json.getJSONArray("symbols"); + symbolsRemaining = json.getInt("symbols_remaining"); + + for (int i = 0; i < symbols.length(); i++) { + JSONObject s = symbols.getJSONObject(i); + if (s.optString("type").equals("stock") && !stockGidMap.containsKey(s.optString("exchange") + ":" + s.getString("symbol"))) { + Stock stock = new Stock(); + stock.setStockGid(s.optString("exchange") + ":" +s.getString("symbol")); + stock.setStockCode(stock.getStockGid()); + stock.setStockSpell(s.optString("symbol")); + stock.setStockName(s.optString("description")); + stock.setStockSymbol(s.optString("symbol")); + stock.setStockType(s.optString("exchange")); + stock.setIsLock(0); + stock.setIsShow(0); + stock.setAddTime(new Date()); + stock.setStockState(0); + stock.setStockPlate("https://s3-symbol-logo.tradingview.com/" + s.optString("source_logoid") + "--big.svg"); + newStocks.add(stock); + } + } + Thread.sleep(500); + + start += symbols.length(); + System.out.println("Fetched: " + symbols.length() + ", Remaining: " + symbolsRemaining); + if (!newStocks.isEmpty()) { + stockRepository.saveAll(newStocks); + } + if (start >= limit) { + break; + } + + } while (symbolsRemaining > 0); + } + } +}