From d085d8de121170d1f3e91f75dbb62ef7620c0f0c Mon Sep 17 00:00:00 2001
From: vu-tran <trannhanvu88@juyou88.com>
Date: Wed, 11 Dec 2024 12:39:15 +0700
Subject: [PATCH] update crawl ipo

---
 .../market/infrastructure/job/Scraper.java    | 27 ++++++++++++++-----
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/src/main/java/cn/stock/market/infrastructure/job/Scraper.java b/src/main/java/cn/stock/market/infrastructure/job/Scraper.java
index 54e0948..7e9ffbd 100644
--- a/src/main/java/cn/stock/market/infrastructure/job/Scraper.java
+++ b/src/main/java/cn/stock/market/infrastructure/job/Scraper.java
@@ -144,9 +144,13 @@ public class Scraper {
                     JSONObject jsonObject = JSONObject.parseObject(jsonDataStr);
                     log.info("获取到新股的json信息:"+jsonObject.toJSONString());
 
+
+
                     JSONObject pageProps = jsonObject.getJSONObject("props").getJSONObject("pageProps");
                     JSONObject ipoTableData = pageProps.getJSONObject("ipoTableData");
 
+
+
                     // 解析 openData 和 upcomingData
                     JSONArray openData = ipoTableData.getJSONArray("openData");
                     JSONArray upcomingData = ipoTableData.getJSONArray("upcomingData");
@@ -175,8 +179,8 @@ public class Scraper {
                         if (stockIpo.getStockCode() == null || stockIpo.getStockName() == null){
                             continue;
                         }
-
-                        List<String> exchanges = getIpoExchange(stockIpo.getStockName(), stockIpo.getStockCode());
+                        String codeDetail = entry.getString("url").substring(entry.getString("url").lastIndexOf('/') + 1);;
+                        List<String> exchanges = getIpoExchange(stockIpo.getStockName(), codeDetail);
 
                         for (String exchange : exchanges) {
                             StockIpo ipo = new StockIpo();
@@ -190,7 +194,7 @@ public class Scraper {
                             ipo.setCreateDate(stockIpo.getCreateDate());
                             ipo.setUpdateDate(stockIpo.getUpdateDate());
                             ipo.setExchangeType(exchange);
-
+                            ipo.setSourceType("3");
                             listStockIpoList.add(ipo);
                         }
                     }
@@ -234,7 +238,7 @@ public class Scraper {
                             ipo.setCreateDate(stockIpo.getCreateDate());
                             ipo.setUpdateDate(stockIpo.getUpdateDate());
                             ipo.setExchangeType(exchange);
-
+                            ipo.setSourceType("3");
                             listStockIpoList.add(ipo);
                         }
                     }
@@ -253,9 +257,18 @@ public class Scraper {
                                 .map(StockIpo::getStockCode)
                                 .collect(Collectors.toList());
 
-                        listStockIpoList = listStockIpoList.stream()
-                                .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
-                                .collect(Collectors.toList());
+                        for (StockIpo stockIpo : listStockIpoList) {
+                            StockIpo existIpo = existStockIpoList.stream().filter(a -> a.getStockCode().equals(stockIpo.getStockCode()) && (a.getExchangeType() == null || a.getExchangeType().isEmpty() || a.getExchangeType().equals(stockIpo.getExchangeType()) )).findFirst().orElse(null);
+                            if (existIpo != null) {
+                                if (existIpo.getExchangeType() == null || existIpo.getExchangeType().isEmpty()){
+                                    existIpo.setExchangeType(stockIpo.getExchangeType());
+                                }
+                                stockIpo.setId(existIpo.getId());
+                            }
+                        }
+//                        listStockIpoList = listStockIpoList.stream()
+//                                .filter(stockIpos -> !existingStockScIds.contains(stockIpos.getStockCode())).filter(stockIpo -> StringUtils.isNotBlank(stockIpo.getStockName()))
+//                                .collect(Collectors.toList());
 
                         //保存全部的新股
                         if(CollectionUtil.isNotEmpty(listStockIpoList)){