From 57690ecb5e9f7b4e996825d248f2d32e2bafda9f Mon Sep 17 00:00:00 2001 From: vu-tran Date: Mon, 28 Apr 2025 17:38:16 +0700 Subject: [PATCH 1/5] update logs --- src/main/java/cn/stock/market/web/MoneyProxyApiControl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/cn/stock/market/web/MoneyProxyApiControl.java b/src/main/java/cn/stock/market/web/MoneyProxyApiControl.java index 2bb0313..0e0c0a2 100644 --- a/src/main/java/cn/stock/market/web/MoneyProxyApiControl.java +++ b/src/main/java/cn/stock/market/web/MoneyProxyApiControl.java @@ -26,13 +26,14 @@ public class MoneyProxyApiControl { @GetMapping({"/market/api/proxy/queryStockMoneyProxy","/api/proxy/queryStockMoneyProxy"}) public JSONObject queryStockByMoneyIdAndExchangeType(String scId,String exchangeType){ + log.info("queryStockByMoneyIdAndExchangeType"); return stockByMoneyIdAndExchangeTypeFromHttp(scId,exchangeType); } public static JSONObject stockByMoneyIdAndExchangeTypeFromHttp(String scId, String exchangeType) { try { - String url = "https://priceapi.moneycontrol.com/pricefeed/"+exchangeType+"/equitycash/" + scId ; + String url = "https://moneycontrol.titanrwa.org/pricefeed/"+exchangeType+"/equitycash/" + scId ; URL obj = new URL(url); HttpURLConnection con = (HttpURLConnection) obj.openConnection(); con.setRequestMethod("GET"); @@ -53,7 +54,8 @@ public class MoneyProxyApiControl { log.error("HTTP request failed with response code: " + responseCode); } } catch (Exception e) { - log.error("stockByMoneyIdAndExchangeTypeFromHttp",e); + e.printStackTrace(); + log.error("stockByMoneyIdAndExchangeTypeFromHttp failed {}",e.getMessage()); } return new JSONObject(); } From 59684cccfc7e674204df825506d8ce265e005450 Mon Sep 17 00:00:00 2001 From: mush_ball Date: Mon, 28 Apr 2025 10:45:30 +0000 Subject: [PATCH 2/5] Edit .gitlab-ci.yml --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5119c46..3cf1e6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -67,8 +67,8 @@ deploy-dev: script: - echo "Deploying application..." - ls - - apt-get update - - apt-get install -y curl + - apk update + - apk add curl - | curl -X POST \ -H "Content-Type: application/json" \ From 85094c22b223a680795794b240f3feb9647200f9 Mon Sep 17 00:00:00 2001 From: VoGiaHuy2058 <39868777+vogiahuy2058@users.noreply.github.com> Date: Mon, 12 May 2025 18:44:16 +0700 Subject: [PATCH 3/5] fix cron --- .../domain/basic/service/SiteNewsService.java | 82 ++++++++++--------- .../market/infrastructure/job/JobBoot.java | 75 +++++++++-------- src/main/resources/application-base-alpha.yml | 4 +- 3 files changed, 87 insertions(+), 74 deletions(-) diff --git a/src/main/java/cn/stock/market/domain/basic/service/SiteNewsService.java b/src/main/java/cn/stock/market/domain/basic/service/SiteNewsService.java index 08d39db..0628f6f 100644 --- a/src/main/java/cn/stock/market/domain/basic/service/SiteNewsService.java +++ b/src/main/java/cn/stock/market/domain/basic/service/SiteNewsService.java @@ -5,6 +5,7 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.stereotype.Service; @@ -39,23 +40,26 @@ import net.sf.json.JSONObject; @SuppressWarnings("rawtypes") @RequiredArgsConstructor(onConstructor = @__(@Autowired)) public class SiteNewsService { - final SiteNewsRepository repository; - final SiteNewsFactory factory; - - /*新闻资讯-查询列表*/ + final SiteNewsRepository repository; + final SiteNewsFactory factory; + + @Value("${news.main.url}") + private String mainUrl; + + /*新闻资讯-查询列表*/ public ServerResponse getList(int pageNum, int pageSize, Integer type, String sort, String keyword, HttpServletRequest request){ - SiteNewsQuery query = new SiteNewsQuery(); - query.setStatus(1); - query.setType(type); - query.setOrderSpecifier(sort); - query.setKeyword(keyword); - Page page = repository.findAll(query.where(), PageParam.of(pageNum, pageSize), query.orderSpecifier()); - + SiteNewsQuery query = new SiteNewsQuery(); + query.setStatus(1); + query.setType(type); + query.setOrderSpecifier(sort); + query.setKeyword(keyword); + Page page = repository.findAll(query.where(), PageParam.of(pageNum, pageSize), query.orderSpecifier()); + return ServerResponse.createBySuccess(Utils.toPageHelperInfo(page)); } /*新闻资讯-查询详情*/ - public ServerResponse getDetail(int id) { + public ServerResponse getDetail(int id) { return ServerResponse.createBySuccess(repository.find(id)); } @@ -66,48 +70,48 @@ public class SiteNewsService { /*新闻资讯-top最新新闻资讯*/ public ServerResponse getTopNewsList(int pageSize){ - SiteNewsQuery query = new SiteNewsQuery(); - query.setStatus(1); - - List list = repository.findAll(query.where(), pageSize, query.orderSpecifier()); - + SiteNewsQuery query = new SiteNewsQuery(); + query.setStatus(1); + + List list = repository.findAll(query.where(), pageSize, query.orderSpecifier()); + PageInfo pageInfo = new PageInfo(); pageInfo.setList(list); return ServerResponse.createBySuccess(pageInfo); } - - /*新闻资讯-抓取*/ + + /*新闻资讯-抓取*/ public int grabNews() { int ret = 0; //新闻类型：1、财经要闻，2、经济数据，3、全球股市，4、7*24全球，5、商品资讯，6、上市公司，7、全球央行 - ret = addNews(1, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetImportantNewsList"); + ret = addNews(1, mainUrl + "/pc_news/FastNews/GetImportantNewsList"); log.info("财经要闻-抓取条数：" + ret); - ret = addNews(2, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=125&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(2, mainUrl + "/pc_news/FastNews/GetInfoList?code=125&pageNumber=1&pagesize=20&condition=&r="); log.info("经济数据-抓取条数：" + ret); - ret = addNews(3, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=105&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(3, mainUrl + "/pc_news/FastNews/GetInfoList?code=105&pageNumber=1&pagesize=20&condition=&r="); log.info("全球股市-抓取条数：" + ret); - ret = addNews(4, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=100&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(4, mainUrl + "/pc_news/FastNews/GetInfoList?code=100&pageNumber=1&pagesize=20&condition=&r="); log.info("7*24全球-抓取条数：" + ret); - ret = addNews(5, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=106&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(5, mainUrl + "/pc_news/FastNews/GetInfoList?code=106&pageNumber=1&pagesize=20&condition=&r="); log.info("商品资讯-抓取条数：" + ret); - ret = addNews(6, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=103&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(6, mainUrl + "/pc_news/FastNews/GetInfoList?code=103&pageNumber=1&pagesize=20&condition=&r="); log.info("上市公司-抓取条数：" + ret); - ret = addNews(7, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=118&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(7, mainUrl + "/pc_news/FastNews/GetInfoList?code=118&pageNumber=1&pagesize=20&condition=&r="); log.info("全球央行-抓取条数：" + ret); return ret; } /* - *抓取新闻专用 - * type：新闻类型：1、财经要闻，2、经济数据，3、全球股市，4、7*24全球，5、商品资讯，6、上市公司，7、全球央行 - * */ + *抓取新闻专用 + * type：新闻类型：1、财经要闻，2、经济数据，3、全球股市，4、7*24全球，5、商品资讯，6、上市公司，7、全球央行 + * */ private int addNews(Integer type, String url){ int k = 0; try { @@ -124,7 +128,7 @@ public class SiteNewsService { //新闻不存在则添加 if(repository.getNewsBySourceIdCount(newsId) == 0){ //获取新闻详情 - String newdata = HttpRequest.doGrabGet(PropertiesUtil.getProperty("news.main.url") + "/PC_News/Detail/GetDetailContent?id="+ newsId +"&type=1"); + String newdata = HttpRequest.doGrabGet(mainUrl + "/PC_News/Detail/GetDetailContent?id="+ newsId +"&type=1"); newdata = newdata.substring(1,newdata.length()-1).replace("\\\\\\\"","\""); newdata = newdata.replace("\\\"","\""); newdata = StringUtils.UnicodeToCN(newdata); @@ -145,9 +149,9 @@ public class SiteNewsService { siteNews.setStatus(1); siteNews.setType(type); try { - repository.saveAndFlush(siteNews); + repository.saveAndFlush(siteNews); } catch(Exception e) { - log.warn("siteNewsMapper insert error: {}", e.getLocalizedMessage()); + log.warn("siteNewsMapper insert error: {}", e.getLocalizedMessage()); } k++; } @@ -159,12 +163,12 @@ public class SiteNewsService { } return k; } - - public SiteNewsRepository repository() { - return repository; - } - public static SiteNewsService of() { - return SpringUtils.getBean(SiteNewsService.class); - } + public SiteNewsRepository repository() { + return repository; + } + + public static SiteNewsService of() { + return SpringUtils.getBean(SiteNewsService.class); + } } diff --git a/src/main/java/cn/stock/market/infrastructure/job/JobBoot.java b/src/main/java/cn/stock/market/infrastructure/job/JobBoot.java index a3aa98f..fec3dbe 100644 --- a/src/main/java/cn/stock/market/infrastructure/job/JobBoot.java +++ b/src/main/java/cn/stock/market/infrastructure/job/JobBoot.java @@ -2,6 +2,7 @@ package cn.stock.market.infrastructure.job; import java.util.concurrent.TimeUnit; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -12,21 +13,29 @@ import cn.stock.market.domain.basic.service.SiteArticleService; import cn.stock.market.domain.basic.service.SiteNewsService; import lombok.extern.slf4j.Slf4j; +import javax.annotation.PostConstruct; + @Slf4j -//@Component +@Component public class JobBoot { + + @Autowired + private SiteNewsService siteNewsService; + + @Autowired + private SiteArticleService siteArticleService; /** * cronExpression表达式定义：  - 字段   允许值   允许的特殊字符 - 秒   0-59   , - * / - 分   0-59   , - * / - 小时   0-23   , - * / - 日期   1-31   , - * ? / L W C - 月份   1-12 或者 JAN-DEC   , - * / - 星期   1-7 或者 SUN-SAT   , - * ? / L C # - 年（可选）   留空, 1970-2099   , - * / - - https://blog.csdn.net/ClementAD/article/details/42042111 + 字段   允许值   允许的特殊字符 + 秒   0-59   , - * / + 分   0-59   , - * / + 小时   0-23   , - * / + 日期   1-31   , - * ? / L W C + 月份   1-12 或者 JAN-DEC   , - * / + 星期   1-7 或者 SUN-SAT   , - * ? / L C # + 年（可选）   留空, 1970-2099   , - * / + + https://blog.csdn.net/ClementAD/article/details/42042111 */ // @Scheduled(cron="0 0/1 * * * ? ") @Scheduled(cron="0/30 * * * * ? ") @@ -35,33 +44,33 @@ public class JobBoot { // Stopwatch stopwatch = Stopwatch.createStarted(); // log.info("batchStockTrust执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); } - + @Scheduled(cron="0/30 * * * * ? ") public void batchStockPosition() { MdcUtil.setTraceIdIfAbsent(); // Stopwatch stopwatch = Stopwatch.createStarted(); // log.info("batchStockPosition执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); } - - /* - * 新闻资讯抓取 - * */ + + /* + * 新闻资讯抓取 + * */ @Scheduled(cron = "0 0/30 9-20 * * ?") - public void artInfoTask() { - MdcUtil.setTraceIdIfAbsent(); - Stopwatch stopwatch = Stopwatch.createStarted(); - int count = SiteArticleService.of().grabArticle(); - log.info("artInfoTask执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); - } - - /* - * 新闻资讯抓取 - * */ - @Scheduled(cron = "0 0/30 9-20 * * ?") - public void newsInfoTask() { - MdcUtil.setTraceIdIfAbsent(); - Stopwatch stopwatch = Stopwatch.createStarted(); - int count = SiteNewsService.of().grabNews(); - log.info("newsInfoTask执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); - } + public void artInfoTask() { + MdcUtil.setTraceIdIfAbsent(); + Stopwatch stopwatch = Stopwatch.createStarted(); + int count = siteArticleService.grabArticle(); + log.info("artInfoTask执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } + + /* + * 新闻资讯抓取 + * */ + @Scheduled(cron = "0 0/30 9-20 * * ?") + public void newsInfoTask() { + MdcUtil.setTraceIdIfAbsent(); + Stopwatch stopwatch = Stopwatch.createStarted(); + int count = siteNewsService.grabNews(); + log.info("newsInfoTask执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } } diff --git a/src/main/resources/application-base-alpha.yml b/src/main/resources/application-base-alpha.yml index 590859e..b92fe23 100644 --- a/src/main/resources/application-base-alpha.yml +++ b/src/main/resources/application-base-alpha.yml @@ -17,7 +17,7 @@ spring: datasource: stock-market: driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://43.156.40.39:30030/india_stock?useUnicode=true&characterEncoding=utf-8 + url: jdbc:mysql://43.153.174.179:30000/india_stock?useUnicode=true&characterEncoding=utf-8 username: root password: uNejHIFQGJOUtYTmE maxActive: 500 @@ -36,7 +36,7 @@ spring: validationQuery: SELECT 1 timeBetweenEvictionRunsMillis: 6000 type: com.alibaba.druid.pool.DruidDataSource - + ig507Licence: 4CEF3CDC-3A2E-76AB-9BF1-BE856F3D4468 gugudataAppKey: K5LZKV8KAYM4 aliyunAccessKeyId: LTAI5tJi2z8cegG8fTW7BSQu From c72c0ff4cde72663872d21583937b9bcc7c9df40 Mon Sep 17 00:00:00 2001 From: VoGiaHuy2058 <39868777+vogiahuy2058@users.noreply.github.com> Date: Mon, 12 May 2025 18:53:06 +0700 Subject: [PATCH 4/5] Revert "fix cron" This reverts commit 85094c22b223a680795794b240f3feb9647200f9. --- .../domain/basic/service/SiteNewsService.java | 82 +++++++++---------- .../market/infrastructure/job/JobBoot.java | 75 ++++++++--------- src/main/resources/application-base-alpha.yml | 4 +- 3 files changed, 74 insertions(+), 87 deletions(-) diff --git a/src/main/java/cn/stock/market/domain/basic/service/SiteNewsService.java b/src/main/java/cn/stock/market/domain/basic/service/SiteNewsService.java index 0628f6f..08d39db 100644 --- a/src/main/java/cn/stock/market/domain/basic/service/SiteNewsService.java +++ b/src/main/java/cn/stock/market/domain/basic/service/SiteNewsService.java @@ -5,7 +5,6 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.stereotype.Service; @@ -40,26 +39,23 @@ import net.sf.json.JSONObject; @SuppressWarnings("rawtypes") @RequiredArgsConstructor(onConstructor = @__(@Autowired)) public class SiteNewsService { - final SiteNewsRepository repository; - final SiteNewsFactory factory; - - @Value("${news.main.url}") - private String mainUrl; - - /*新闻资讯-查询列表*/ + final SiteNewsRepository repository; + final SiteNewsFactory factory; + + /*新闻资讯-查询列表*/ public ServerResponse getList(int pageNum, int pageSize, Integer type, String sort, String keyword, HttpServletRequest request){ - SiteNewsQuery query = new SiteNewsQuery(); - query.setStatus(1); - query.setType(type); - query.setOrderSpecifier(sort); - query.setKeyword(keyword); - Page page = repository.findAll(query.where(), PageParam.of(pageNum, pageSize), query.orderSpecifier()); - + SiteNewsQuery query = new SiteNewsQuery(); + query.setStatus(1); + query.setType(type); + query.setOrderSpecifier(sort); + query.setKeyword(keyword); + Page page = repository.findAll(query.where(), PageParam.of(pageNum, pageSize), query.orderSpecifier()); + return ServerResponse.createBySuccess(Utils.toPageHelperInfo(page)); } /*新闻资讯-查询详情*/ - public ServerResponse getDetail(int id) { + public ServerResponse getDetail(int id) { return ServerResponse.createBySuccess(repository.find(id)); } @@ -70,48 +66,48 @@ public class SiteNewsService { /*新闻资讯-top最新新闻资讯*/ public ServerResponse getTopNewsList(int pageSize){ - SiteNewsQuery query = new SiteNewsQuery(); - query.setStatus(1); - - List list = repository.findAll(query.where(), pageSize, query.orderSpecifier()); - + SiteNewsQuery query = new SiteNewsQuery(); + query.setStatus(1); + + List list = repository.findAll(query.where(), pageSize, query.orderSpecifier()); + PageInfo pageInfo = new PageInfo(); pageInfo.setList(list); return ServerResponse.createBySuccess(pageInfo); } - - /*新闻资讯-抓取*/ + + /*新闻资讯-抓取*/ public int grabNews() { int ret = 0; //新闻类型：1、财经要闻，2、经济数据，3、全球股市，4、7*24全球，5、商品资讯，6、上市公司，7、全球央行 - ret = addNews(1, mainUrl + "/pc_news/FastNews/GetImportantNewsList"); + ret = addNews(1, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetImportantNewsList"); log.info("财经要闻-抓取条数：" + ret); - ret = addNews(2, mainUrl + "/pc_news/FastNews/GetInfoList?code=125&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(2, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=125&pageNumber=1&pagesize=20&condition=&r="); log.info("经济数据-抓取条数：" + ret); - ret = addNews(3, mainUrl + "/pc_news/FastNews/GetInfoList?code=105&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(3, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=105&pageNumber=1&pagesize=20&condition=&r="); log.info("全球股市-抓取条数：" + ret); - ret = addNews(4, mainUrl + "/pc_news/FastNews/GetInfoList?code=100&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(4, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=100&pageNumber=1&pagesize=20&condition=&r="); log.info("7*24全球-抓取条数：" + ret); - ret = addNews(5, mainUrl + "/pc_news/FastNews/GetInfoList?code=106&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(5, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=106&pageNumber=1&pagesize=20&condition=&r="); log.info("商品资讯-抓取条数：" + ret); - ret = addNews(6, mainUrl + "/pc_news/FastNews/GetInfoList?code=103&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(6, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=103&pageNumber=1&pagesize=20&condition=&r="); log.info("上市公司-抓取条数：" + ret); - ret = addNews(7, mainUrl + "/pc_news/FastNews/GetInfoList?code=118&pageNumber=1&pagesize=20&condition=&r="); + ret = addNews(7, PropertiesUtil.getProperty("news.main.url") + "/pc_news/FastNews/GetInfoList?code=118&pageNumber=1&pagesize=20&condition=&r="); log.info("全球央行-抓取条数：" + ret); return ret; } /* - *抓取新闻专用 - * type：新闻类型：1、财经要闻，2、经济数据，3、全球股市，4、7*24全球，5、商品资讯，6、上市公司，7、全球央行 - * */ + *抓取新闻专用 + * type：新闻类型：1、财经要闻，2、经济数据，3、全球股市，4、7*24全球，5、商品资讯，6、上市公司，7、全球央行 + * */ private int addNews(Integer type, String url){ int k = 0; try { @@ -128,7 +124,7 @@ public class SiteNewsService { //新闻不存在则添加 if(repository.getNewsBySourceIdCount(newsId) == 0){ //获取新闻详情 - String newdata = HttpRequest.doGrabGet(mainUrl + "/PC_News/Detail/GetDetailContent?id="+ newsId +"&type=1"); + String newdata = HttpRequest.doGrabGet(PropertiesUtil.getProperty("news.main.url") + "/PC_News/Detail/GetDetailContent?id="+ newsId +"&type=1"); newdata = newdata.substring(1,newdata.length()-1).replace("\\\\\\\"","\""); newdata = newdata.replace("\\\"","\""); newdata = StringUtils.UnicodeToCN(newdata); @@ -149,9 +145,9 @@ public class SiteNewsService { siteNews.setStatus(1); siteNews.setType(type); try { - repository.saveAndFlush(siteNews); + repository.saveAndFlush(siteNews); } catch(Exception e) { - log.warn("siteNewsMapper insert error: {}", e.getLocalizedMessage()); + log.warn("siteNewsMapper insert error: {}", e.getLocalizedMessage()); } k++; } @@ -163,12 +159,12 @@ public class SiteNewsService { } return k; } + + public SiteNewsRepository repository() { + return repository; + } - public SiteNewsRepository repository() { - return repository; - } - - public static SiteNewsService of() { - return SpringUtils.getBean(SiteNewsService.class); - } + public static SiteNewsService of() { + return SpringUtils.getBean(SiteNewsService.class); + } } diff --git a/src/main/java/cn/stock/market/infrastructure/job/JobBoot.java b/src/main/java/cn/stock/market/infrastructure/job/JobBoot.java index fec3dbe..a3aa98f 100644 --- a/src/main/java/cn/stock/market/infrastructure/job/JobBoot.java +++ b/src/main/java/cn/stock/market/infrastructure/job/JobBoot.java @@ -2,7 +2,6 @@ package cn.stock.market.infrastructure.job; import java.util.concurrent.TimeUnit; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -13,29 +12,21 @@ import cn.stock.market.domain.basic.service.SiteArticleService; import cn.stock.market.domain.basic.service.SiteNewsService; import lombok.extern.slf4j.Slf4j; -import javax.annotation.PostConstruct; - @Slf4j -@Component +//@Component public class JobBoot { - - @Autowired - private SiteNewsService siteNewsService; - - @Autowired - private SiteArticleService siteArticleService; /** * cronExpression表达式定义：  - 字段   允许值   允许的特殊字符 - 秒   0-59   , - * / - 分   0-59   , - * / - 小时   0-23   , - * / - 日期   1-31   , - * ? / L W C - 月份   1-12 或者 JAN-DEC   , - * / - 星期   1-7 或者 SUN-SAT   , - * ? / L C # - 年（可选）   留空, 1970-2099   , - * / - - https://blog.csdn.net/ClementAD/article/details/42042111 + 字段   允许值   允许的特殊字符 + 秒   0-59   , - * / + 分   0-59   , - * / + 小时   0-23   , - * / + 日期   1-31   , - * ? / L W C + 月份   1-12 或者 JAN-DEC   , - * / + 星期   1-7 或者 SUN-SAT   , - * ? / L C # + 年（可选）   留空, 1970-2099   , - * / + + https://blog.csdn.net/ClementAD/article/details/42042111 */ // @Scheduled(cron="0 0/1 * * * ? ") @Scheduled(cron="0/30 * * * * ? ") @@ -44,33 +35,33 @@ public class JobBoot { // Stopwatch stopwatch = Stopwatch.createStarted(); // log.info("batchStockTrust执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); } - + @Scheduled(cron="0/30 * * * * ? ") public void batchStockPosition() { MdcUtil.setTraceIdIfAbsent(); // Stopwatch stopwatch = Stopwatch.createStarted(); // log.info("batchStockPosition执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); } - - /* - * 新闻资讯抓取 - * */ + + /* + * 新闻资讯抓取 + * */ @Scheduled(cron = "0 0/30 9-20 * * ?") - public void artInfoTask() { - MdcUtil.setTraceIdIfAbsent(); - Stopwatch stopwatch = Stopwatch.createStarted(); - int count = siteArticleService.grabArticle(); - log.info("artInfoTask执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); - } - - /* - * 新闻资讯抓取 - * */ - @Scheduled(cron = "0 0/30 9-20 * * ?") - public void newsInfoTask() { - MdcUtil.setTraceIdIfAbsent(); - Stopwatch stopwatch = Stopwatch.createStarted(); - int count = siteNewsService.grabNews(); - log.info("newsInfoTask执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); - } + public void artInfoTask() { + MdcUtil.setTraceIdIfAbsent(); + Stopwatch stopwatch = Stopwatch.createStarted(); + int count = SiteArticleService.of().grabArticle(); + log.info("artInfoTask执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } + + /* + * 新闻资讯抓取 + * */ + @Scheduled(cron = "0 0/30 9-20 * * ?") + public void newsInfoTask() { + MdcUtil.setTraceIdIfAbsent(); + Stopwatch stopwatch = Stopwatch.createStarted(); + int count = SiteNewsService.of().grabNews(); + log.info("newsInfoTask执行, 受影响数{}, 耗时:{}毫秒", count, stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } } diff --git a/src/main/resources/application-base-alpha.yml b/src/main/resources/application-base-alpha.yml index b92fe23..590859e 100644 --- a/src/main/resources/application-base-alpha.yml +++ b/src/main/resources/application-base-alpha.yml @@ -17,7 +17,7 @@ spring: datasource: stock-market: driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://43.153.174.179:30000/india_stock?useUnicode=true&characterEncoding=utf-8 + url: jdbc:mysql://43.156.40.39:30030/india_stock?useUnicode=true&characterEncoding=utf-8 username: root password: uNejHIFQGJOUtYTmE maxActive: 500 @@ -36,7 +36,7 @@ spring: validationQuery: SELECT 1 timeBetweenEvictionRunsMillis: 6000 type: com.alibaba.druid.pool.DruidDataSource - + ig507Licence: 4CEF3CDC-3A2E-76AB-9BF1-BE856F3D4468 gugudataAppKey: K5LZKV8KAYM4 aliyunAccessKeyId: LTAI5tJi2z8cegG8fTW7BSQu From 90a26823f0ba28318d1be89569868c9cbca44d07 Mon Sep 17 00:00:00 2001 From: vu-tran Date: Tue, 13 May 2025 10:10:42 +0700 Subject: [PATCH 5/5] update news --- .../domain/basic/service/StockService.java | 117 +++++++++++++----- .../stock/market/utils/HttpClientRequest.java | 50 ++++++++ src/main/resources/application-base-alpha.yml | 6 +- 3 files changed, 141 insertions(+), 32 deletions(-) diff --git a/src/main/java/cn/stock/market/domain/basic/service/StockService.java b/src/main/java/cn/stock/market/domain/basic/service/StockService.java index c21f982..b0ad5c3 100644 --- a/src/main/java/cn/stock/market/domain/basic/service/StockService.java +++ b/src/main/java/cn/stock/market/domain/basic/service/StockService.java @@ -255,47 +255,106 @@ public class StockService { // .header("Accept-Language", "en-US,en;q=0.9") // .get(); String url = "https://www.business-standard.com/markets/news"; - Document doc = Jsoup.connect(url) - .referrer("https://www.business-standard.com/") - .header("Accept-Language", "en-US,en;q=0.9") - .userAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1") - .timeout(5000) // timeout 5 seconds - .get(); - // result = doc.html().substring(doc.html().indexOf("
"),doc.html().lastIndexOf("
")+500); - Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57"); - StringBuilder sb = new StringBuilder(); - for (Element divElement : divElements) { - sb.append(divElement.outerHtml()).append("\n"); - } - result = sb.toString(); +// Document doc = Jsoup.connect(url) +// .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") +// .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") +// .header("Accept-Language", "en-US,en;q=0.5") +// .header("Connection", "keep-alive") +// .referrer("https://www.google.com") +// .ignoreHttpErrors(true) +// .timeout(10000) +// .get(); + String doGetNews = HttpClientRequest.doGetNews(url); +// result = doc.html().substring(doc.html().indexOf("
"),doc.html().lastIndexOf("
")+500); +// Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57"); +// StringBuilder sb = new StringBuilder(); +// for (Element divElement : divElements) { +// sb.append(divElement.outerHtml()).append("\n"); +// } +// result = sb.toString(); + result = extractNewsFromHtml(doGetNews); } catch (Exception e) { return e.toString(); } return result; } + private String extractNewsFromHtml(String rawHtml) { + Document doc = Jsoup.parse(rawHtml); + + // Optional: extract raw block around
+ int start = rawHtml.indexOf("
"); + int end = rawHtml.lastIndexOf("
") + 500; + String result = (start != -1 && end > start) ? rawHtml.substring(start, end) : ""; + + // Extract article list using CSS selector + Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57"); + StringBuilder sb = new StringBuilder(); + for (Element divElement : divElements) { + sb.append(divElement.outerHtml()).append("\n"); + } + + // If you only want the div content: + result = sb.toString(); + return result; + } + public List getNewsInfo(String url) { +// String result = ""; +// List list = new ArrayList<>(); +// try { +// // 使用Jsoup连接到网页 +//// Document doc = Jsoup.connect(url) +//// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36") +//// .header("Referer", "https://www.business-standard.com/") +//// .header("Accept-Language", "en-US,en;q=0.9") +//// .get(); +// Document doc = Jsoup.connect(url) +// .referrer("https://www.business-standard.com/") +// .header("Accept-Language", "en-US,en;q=0.9") +// .userAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1") +// .timeout(5000) // timeout 5 seconds +// .get(); +// result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1); +// list.add(result); +// list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf(" list = new ArrayList<>(); try { - // 使用Jsoup连接到网页 -// Document doc = Jsoup.connect(url) -// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36") -// .header("Referer", "https://www.business-standard.com/") -// .header("Accept-Language", "en-US,en;q=0.9") -// .get(); - Document doc = Jsoup.connect(url) - .referrer("https://www.business-standard.com/") - .header("Accept-Language", "en-US,en;q=0.9") - .userAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1") - .timeout(5000) // timeout 5 seconds - .get(); - result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1); + // Step 1: Get raw HTML via HttpClient + String rawHtml = HttpClientRequest.doGetNews(url); + + // Step 2: Parse HTML with Jsoup + Document doc = Jsoup.parse(rawHtml); + + // Step 3: Extract content manually (like original) + int startIdx = rawHtml.indexOf("articleBody") + 14; + int endIdx = rawHtml.indexOf(",\"author\":") - 1; + + if (startIdx > 0 && endIdx > startIdx) { + result = rawHtml.substring(startIdx, endIdx); + } else { + result = "[articleBody not found]"; + } list.add(result); - list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf(" 0 && titleEnd > titleStart) { + list.add(rawHtml.substring(titleStart, titleEnd)); + } else { + list.add("[title not found]"); + } + } catch (Exception e) { - list.add(e.toString()); - return list; + list.add("[Error] " + e.getMessage()); } return list; } diff --git a/src/main/java/cn/stock/market/utils/HttpClientRequest.java b/src/main/java/cn/stock/market/utils/HttpClientRequest.java index e6c7c9c..50498da 100644 --- a/src/main/java/cn/stock/market/utils/HttpClientRequest.java +++ b/src/main/java/cn/stock/market/utils/HttpClientRequest.java @@ -265,6 +265,56 @@ public class HttpClientRequest { return result; } + public static String doGetNews(String url) { + CloseableHttpClient httpClient = null; + CloseableHttpResponse response = null; + String result = ""; + + try { + httpClient = HttpClients.createDefault(); + + HttpGet httpGet = new HttpGet(url); + + // Spoof real browser headers + httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"); + httpGet.setHeader("Accept-Language", "en-US,en;q=0.9,vi;q=0.8,ug;q=0.7,fr;q=0.6"); + httpGet.setHeader("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1"); + httpGet.setHeader("Priority", "u=0, i"); + httpGet.setHeader("Sec-Fetch-Dest", "document"); + httpGet.setHeader("Sec-Fetch-Mode", "navigate"); + httpGet.setHeader("Sec-Fetch-Site", "none"); + httpGet.setHeader("Sec-Fetch-User", "?1"); + httpGet.setHeader("Upgrade-Insecure-Requests", "1"); + + // Set cookies exactly like in curl + httpGet.setHeader("Cookie", "userUid=1747102033185-d570fba9-62fd-40be-93ca-ed08b4de57d4; _sid=MTc0NzEwMjAzMzE4Ni4ycTU%3D; _scor_uid=135c13065ff84620b5318b489af93e87; _gcl_au=1.1.249135292.1747102036; _ga=GA1.1.1031614211.1747102037; WZRK_G=28895afb56ff48dda59fe8de0af746bf; FCNEC=%5B%5B%22AKsRol8sHYeSYz_FYPkInYXN3P4ZDPfVKbsRfILfDuOMLhDtkTuoCJP5MlvT9gIbOe7IlDfY8ZeHszhwdVtAoKF1gWv0pLAq5EqpLpse8CEm_ZNv-bUSs6zEyqpOkeKWFI_Ei6VfNAvnZAO8PcXdF8_ncsaO902X7g%3D%3D%22%5D%5D; _ga_KRGL1M61LX=GS2.1.s1747105135$o2$g0$t1747105135$j60$l0$h0"); + + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(10000) + .setSocketTimeout(15000) + .setConnectionRequestTimeout(10000) + .build(); + httpGet.setConfig(requestConfig); + + response = httpClient.execute(httpGet); + HttpEntity entity = response.getEntity(); + + if (entity != null) { + result = EntityUtils.toString(entity); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (response != null) response.close(); + if (httpClient != null) httpClient.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return result; + } + public static void main(String[] args) { String url = "https://marketapi.intoday.in/widget/topgainer/view?exchange=nse"; String str = doGet(url); diff --git a/src/main/resources/application-base-alpha.yml b/src/main/resources/application-base-alpha.yml index 590859e..9b896bb 100644 --- a/src/main/resources/application-base-alpha.yml +++ b/src/main/resources/application-base-alpha.yml @@ -3,9 +3,9 @@ spring: show-sql: true # Redis配置 redis: - host: 43.156.40.39 + host: 43.153.174.179 password: a5v8b86P4mVzFlUqJV - port: 30031 + port: 30001 database: 1 lettuce: pool: @@ -17,7 +17,7 @@ spring: datasource: stock-market: driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://43.156.40.39:30030/india_stock?useUnicode=true&characterEncoding=utf-8 + url: jdbc:mysql://43.153.174.179:30000/india_stock?useUnicode=true&characterEncoding=utf-8 username: root password: uNejHIFQGJOUtYTmE maxActive: 500