新闻定时任务

Develop

See merge request india/india_market_java!4

src/main/java/cn/stock/market/domain/basic/service/StockService.java

@@ -272,8 +272,9 @@ public class StockService {
 		return result;
 	}
 
-	public String getNewsInfo(String url) {
+	public List<String> getNewsInfo(String url) {
 		String result = "";
+		List<String> list = new ArrayList<>();
 		try {
 			// 使用Jsoup连接到网页
 			Document doc = Jsoup.connect(url)
@@ -281,11 +282,14 @@ public class StockService {
 					.header("Referer", "https://www.business-standard.com/")
 					.header("Accept-Language", "en-US,en;q=0.9")
 					.get();
-			result = doc.html().substring(doc.html().indexOf("articleBody") + 15, doc.html().indexOf(",\"author\":") - 1);
+			result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1);
+			list.add(result);
+			list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf("<meta property=\"og:url") - 5));
 		} catch (Exception e) {
-			return e.toString();
+			list.add(e.toString());
+			return list;
 		}
-		return result;
+		return list;
 	}
 
 	public ServerResponse getStock(int pageNum, int pageSize, String keyWords, String stockPlate, String stockType,

src/main/java/cn/stock/market/infrastructure/job/InvestingTask.java

@@ -117,15 +117,15 @@ public class InvestingTask {
             String id = contentUrl.substring(contentUrl.lastIndexOf("-") + 1, contentUrl.lastIndexOf("_"));
             String imgUrl = n.substring(n.indexOf("img src=") + 9, n.indexOf("?"));
             String time = n.substring(n.indexOf("Last Updated") + 23, n.indexOf("IST") - 9);
-            String title = n.substring(n.indexOf("html\">") + 6, n.indexOf("<div class=\"short-video-img\">") - 47);
 
             SiteNews siteNews = new SiteNews();
             siteNews.setAddTime(new Date());
             siteNews.setSourceId(id);
-            siteNews.setTitle(title);
             siteNews.setDescription(time);
             siteNews.setImgurl(imgUrl);
-            siteNews.setContent(stockService.getNewsInfo(contentUrl));
+            List<String> newsInfo = stockService.getNewsInfo(contentUrl);
+            siteNews.setContent(newsInfo.get(0));
+            siteNews.setTitle(newsInfo.get(1));
             List<SiteNews> list = newsRepository.findAll(QSiteNewsPO.siteNewsPO.sourceId.eq(id));
             if (list.size() == 0) {
                 newsRepository.save(siteNews);

src/main/java/cn/stock/market/infrastructure/job/MoneyScraper.java

@@ -16,6 +16,7 @@ import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 import java.io.BufferedReader;
@@ -93,9 +94,8 @@ public class MoneyScraper {
     }
 
     @GetMapping("testScraperGetMoneyControlStockNoScId")
-    public String testScraperGetMoneyControlStockNoScId() {
+    public String testScraperGetMoneyControlStockNoScId(@RequestParam("url") String url) {
-        String url = "http://www.moneycontrol.com/india/stockpricequote/foodprocessing/nestleindia/NI";
+       Document soup2 = fetchCompanyDetails(url);
-      /*  Document soup2 = fetchCompanyDetails(url);
 
         if (soup2 != null) {
             Element comIdInput = soup2.selectFirst("input[id=ap_sc_id]");
@@ -109,19 +109,19 @@ public class MoneyScraper {
 
             if (soup2 != null) {
                 Element ulElement = soup2.selectFirst("ul[id=nseBseTab]");
-
+                String name = soup2.selectFirst("#stockName > h1").text();
                 if (ulElement != null) {
                     for (Element aElement : ulElement.select("a")) {
                         String exchangeValue = aElement.text().trim();
                         if ("BSE".equals(exchangeValue) || "NSE".equals(exchangeValue)) {
                             log.info(Thread.currentThread().getName() + ",the stock url: " + url +
                                     ", the exchange Value: " + exchangeValue);
-                            MoneyStock build = MoneyStock.builder().stockName("sss").stockType(exchangeValue.toLowerCase(Locale.ROOT))
+                            MoneyStock build = MoneyStock.builder().stockName(name).stockType(exchangeValue.toLowerCase(Locale.ROOT))
                                     .detailUrl(String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s", exchangeValue.toLowerCase(), companyCodeId))
                                     .selfUrl(url)
                                     .selfDispId(extractDispId(url))
                                     .moneyScId(companyCodeId).saveTime(new Date()).build();
-                            List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq("Nestle"), QMoneyStockPO.moneyStockPO.stockType.eq(exchangeValue.toLowerCase(Locale.ROOT)));
+                            List<MoneyStock> all = moneyStockRepository.findAll(QMoneyStockPO.moneyStockPO.stockName.eq(name), QMoneyStockPO.moneyStockPO.stockType.eq(exchangeValue.toLowerCase(Locale.ROOT)));
                             if (CollectionUtil.isEmpty(all)) {
                                 moneyStockRepository.save(build);
                             }
@@ -129,7 +129,7 @@ public class MoneyScraper {
                     }
                 }
             }
-        }*/
+        }
             return "ok";
     }
 

src/main/java/cn/stock/market/web/MoneyApiController.java

@@ -6,8 +6,11 @@ import cn.stock.market.domain.basic.repository.MoneyStockRepository;
 import cn.stock.market.dto.StockHistoryRequest;
 import cn.stock.market.dto.StockHistoryResponse;
 import cn.stock.market.infrastructure.db.po.QMoneyStockPO;
+import cn.stock.market.utils.RequestCacheUtils;
 import cn.stock.market.utils.ServerResponse;
 import com.alibaba.fastjson.JSONObject;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Lists;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
@@ -23,17 +26,20 @@ import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.client.RestClientException;
 import org.springframework.web.client.RestTemplate;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 /**
@@ -60,6 +66,7 @@ public class MoneyApiController {
     @ApiImplicitParams({
             @ApiImplicitParam(name="stockType",value = "BSE或者NSE"),
             @ApiImplicitParam(name="symbol",value = "scId值"),
+            @ApiImplicitParam(name="id",value = "id值"),
     })
     @ApiResponses(value = {
             @ApiResponse(code = 200, message = "" +
@@ -162,14 +169,39 @@ public class MoneyApiController {
     @ResponseBody
     public ServerResponse getStockDetail(@RequestParam String stockType, @RequestParam String symbol ) {
         String url = String.format("https://priceapi.moneycontrol.com/pricefeed/%s/equitycash/%s",stockType,symbol);
+        MoneyStock moneyStock = moneyStockRepository.findOne(QMoneyStockPO.moneyStockPO.stockType.eq(stockType).and(QMoneyStockPO.moneyStockPO.moneyScId.eq(symbol))).orElse(null);
+     /*   if(moneyStock==null){
+            return ServerResponse.createByErrorMsg("没有找到该股票");
+        }*/
         // 设置重试次数
         int maxRetries = 3;
         for (int retry = 1; retry <= maxRetries; retry++) {
             try {
                 ResponseEntity<String> responseEntity = restTemplate.exchange(url, HttpMethod.GET, null, String.class);
-
+                JSONObject json1 = new JSONObject();
                 if (responseEntity.getStatusCode().value() == 200 && responseEntity.getBody() != null ) {
-                    return ServerResponse.createBySuccess(JSONObject.parseObject(responseEntity.getBody()).getJSONObject("data"));
+                    JSONObject data = JSONObject.parseObject(responseEntity.getBody()).getJSONObject("data");
+                    if(data!=null){
+                        json1.put("company",data.getString("company"));
+                        json1.put("pricepercentchange",data.getString("pricepercentchange"));
+                        json1.put("stockType",stockType);
+                        json1.put("pricechange",data.getString("pricechange"));
+                        json1.put("pricecurrent",data.getString("pricecurrent"));
+                        json1.put("priceprevclose",data.getString("priceprevclose"));
+                        json1.put("PREVDATE",data.getString("PREVDATE"));
+                        json1.put("VOL",data.getString("VOL"));
+                        json1.put("dataSourceType","3");
+                        json1.put("symbol",data.getString("symbol"));
+                        json1.put("BSEID",data.getString("BSEID"));
+                        json1.put("NSEID",data.getString("NSEID"));
+                        json1.put("LTH",data.getString("HP"));
+                        json1.put("LTL",data.getString("LP"));
+                        json1.put("OPN",data.getString("OPN"));
+                        if(null!=moneyStock){
+                            json1.put("id",moneyStock.getId());
+                        }
+                    }
+                    return ServerResponse.createBySuccess(json1);
                 }
 
             } catch (Exception e) {
@@ -214,10 +246,8 @@ public class MoneyApiController {
                 String lastPrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div >  div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(4)").first());
                 dto.setLastPrice(lastPrice);
 
-                String change = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div >  div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first());
+                String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div >  div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first());
-                dto.setChange(change);
 
-                String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div  > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(6)").first());
                 dto.setChangePercent(changePercent);
 
                 dto.setStockType("nse");
@@ -262,10 +292,7 @@ public class MoneyApiController {
                 String prevClosePrice = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(5)").first());
                 dto.setPrevClosePrice(prevClosePrice);
 
-                String change = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(6)").first());
+                String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(6)").first());
-                dto.setChange(change);
-
-                String changePercent = getTextOrEmpty(doc.select("#mc_content > section > section > div.clearfix.stat_container > div.columnst.FR.wbg.brdwht > div > div > div.bsr_table.hist_tbl_hm > table > tbody > tr:nth-child(" + i + ") > td:nth-child(7)").first());
                 dto.setChangePercent(changePercent);
 
                 dto.setStockType("bse");
@@ -476,11 +503,18 @@ public class MoneyApiController {
     @ResponseBody
     public List<MoneyStockSuggestDTO> getTopGainer(@RequestParam String stockType) {
         List<MoneyStockSuggestDTO> moneyStockSuggestDTOS = null;
+        // 尝试从缓存中获取结果
+        moneyStockSuggestDTOS = gainerStockSuggestCache.getIfPresent(stockType);
+
+        if (moneyStockSuggestDTOS == null) {
+            // 缓存未命中，执行业务查询
             if (StringUtils.equals(stockType, "nse")) {
                 moneyStockSuggestDTOS = nseGainer();
             } else if (StringUtils.equals(stockType, "bse")) {
                 moneyStockSuggestDTOS = bseGainer();
             }
+
+            moneyStockSuggestDTOS = moneyStockSuggestDTOS.stream().filter(f->StringUtils.isNotBlank(f.getStockName())).collect(Collectors.toList());
             if(CollectionUtils.isNotEmpty(moneyStockSuggestDTOS)){
                 List<String> selfUlrList = moneyStockSuggestDTOS.stream().map(MoneyStockSuggestDTO::getStockName).collect(Collectors.toList());
                 if(CollectionUtils.isNotEmpty(selfUlrList)){
@@ -490,6 +524,9 @@ public class MoneyApiController {
                                 .forEach(f->f.setScId(all.stream().filter(s->s.getStockName().equals(f.getStockName())).findFirst().orElse(null).getMoneyScId()));
                     }
                 }
+                gainerStockSuggestCache.put(stockType, moneyStockSuggestDTOS);
+            }
+            // 将结果放入缓存
         }
         return moneyStockSuggestDTOS;
     }
@@ -507,11 +544,14 @@ public class MoneyApiController {
     @ResponseBody
     public List<MoneyStockSuggestDTO> getTopLoser(@RequestParam String stockType) {
         List<MoneyStockSuggestDTO> moneyStockSuggestDTOS = null;
+        moneyStockSuggestDTOS = loserStockSuggestCache.getIfPresent(stockType);
+        if(null==moneyStockSuggestDTOS){
             if(StringUtils.equals(stockType,"nse")){
                 moneyStockSuggestDTOS = nseTopLoser();
             }else if(StringUtils.equals(stockType,"bse")){
                 moneyStockSuggestDTOS = bseTopLoser();
             }
+            moneyStockSuggestDTOS = moneyStockSuggestDTOS.stream().filter(f->StringUtils.isNotBlank(f.getStockName())).collect(Collectors.toList());
             if(CollectionUtils.isNotEmpty(moneyStockSuggestDTOS)){
                 moneyStockSuggestDTOS.stream().forEach(f->f.setDispId(extractLastSegment(f.getStockUrl())));
                 List<String> selfUlrList = moneyStockSuggestDTOS.stream().map(MoneyStockSuggestDTO::getStockName).collect(Collectors.toList());
@@ -531,8 +571,9 @@ public class MoneyApiController {
                         }
                     }
                 }
+                loserStockSuggestCache.put(stockType, moneyStockSuggestDTOS);
+            }
         }
-
         return moneyStockSuggestDTOS;
     }
 
@@ -551,11 +592,14 @@ public class MoneyApiController {
     @ResponseBody
     public List<MoneyStockSuggestDTO> getTopActive(@RequestParam String stockType) {
         List<MoneyStockSuggestDTO> moneyStockSuggestDTOS = null;
+        moneyStockSuggestDTOS = activesStockSuggestCache.getIfPresent(stockType);
+        if(moneyStockSuggestDTOS ==null){
             if(StringUtils.equals(stockType,"nse")){
                 moneyStockSuggestDTOS = nseActives();
             }else if(StringUtils.equals(stockType,"bse")){
                 moneyStockSuggestDTOS = bseActives();
             }
+            moneyStockSuggestDTOS = moneyStockSuggestDTOS.stream().filter(f->StringUtils.isNotBlank(f.getStockName())).collect(Collectors.toList());
             if(CollectionUtils.isNotEmpty(moneyStockSuggestDTOS)){
                 moneyStockSuggestDTOS.stream().forEach(f->f.setDispId(extractLastSegment(f.getStockUrl())));
                 List<String> selfUlrList = moneyStockSuggestDTOS.stream().map(MoneyStockSuggestDTO::getStockName).collect(Collectors.toList());
@@ -575,6 +619,8 @@ public class MoneyApiController {
                         }
                     }
                 }
+                activesStockSuggestCache.put(stockType, moneyStockSuggestDTOS);
+            }
         }
 
         return moneyStockSuggestDTOS;
@@ -585,36 +631,85 @@ public class MoneyApiController {
     @GetMapping("/api/market/money/history/kLine")
     @ApiOperation(value = "获取kline的money数据源", notes = "获取kline的money数据源",response = StockHistoryResponse.class)
     @ApiImplicitParams({
-            @ApiImplicitParam(name = "symbol", value = "Stock symbol", required = true, dataType = "String", paramType = "query"),
+            @ApiImplicitParam(name = "symbol", value = "Stock symbol 对应的是NSEID 或者是BSEID", required = true, dataType = "String", paramType = "query"),
-            @ApiImplicitParam(name = "resolution", value = "Resolution", required = true, dataType = "String", paramType = "query"),
+            @ApiImplicitParam(name = "resolution", value = "单位:60 1D 1W 1D 对应H,D,W,Y", required = true, dataType = "String", paramType = "query"),
             @ApiImplicitParam(name = "from", value = "Start timestamp", required = true, dataType = "long", paramType = "query"),
             @ApiImplicitParam(name = "to", value = "End timestamp", required = true, dataType = "long", paramType = "query"),
-            @ApiImplicitParam(name = "countback", value = "Countback", required = true, dataType = "int", paramType = "query"),
+            @ApiImplicitParam(name = "countback", value = "开始时间和结束时间区间的计划展示的数量", required = true, dataType = "int", paramType = "query"),
-            @ApiImplicitParam(name = "currencyCode", value = "Currency code", required = true, dataType = "String", paramType = "query")
+            @ApiImplicitParam(name = "currencyCode", value = "INR 不变", required = true, dataType = "String", paramType = "query")
     })
     @ResponseBody
     public ResponseEntity<StockHistoryResponse> getStockHistory( @RequestParam String symbol,
-                                                                 @RequestParam String resolution,
+                                                                 @RequestParam String resolution
-                                                                 @RequestParam long from,
+                                                               ) {
-                                                                 @RequestParam long to,
-                                                                 @RequestParam int countback,
-                                                                 @RequestParam String currencyCode) {
-        // 创建一个RestTemplate实例
-        RestTemplate restTemplate = new RestTemplate();
         // 向外部API发起请求，并获取响应
         StockHistoryRequest request = new StockHistoryRequest();
         request.setSymbol(symbol);
-        request.setResolution(resolution);
+        Long to = null;
+        Long from = null;
+        int countback = 5;
+        if(StringUtils.equals("H",resolution)){
+            to  = (long) (System.currentTimeMillis() / 1000);
+            from = to - (1 * 60 * 60 );
+            countback = 60;
+            request.setResolution("1");
+        }else if(StringUtils.equals("D",resolution)){
+            to  = (long) (System.currentTimeMillis() / 1000);
+            from = to - (24 * 60 * 60 );
+            countback = 9;
+            request.setResolution("60");
+        }else if(StringUtils.equals("W",resolution)){
+            to  = (long) (System.currentTimeMillis() / 1000);
+            from = to - (10 * 24 * 60 * 60 );
+            countback = 7;
+            request.setResolution("1D");
+        }else if(StringUtils.equals("M",resolution)){
+            to  = (long) (System.currentTimeMillis() / 1000);
+            from = to - (35 * 24 * 60 * 60 );
+            countback = 30;
+            request.setResolution("1D");
+        }
+
         request.setFrom(from);
         request.setTo(to);
         request.setCountback(countback);
-        request.setCurrencyCode(currencyCode);
+        request.setCurrencyCode("INR");
         String apiUrl = buildApiUrl(request);
-        StockHistoryResponse response = restTemplate.getForObject(apiUrl, StockHistoryResponse.class);
+        log.info("request url:"+apiUrl);
+        StockHistoryResponse response = null;
+        int maxRetries = 3;
+        int retryCount = 0;
 
-        // 返回响应
+        while (response == null && retryCount < maxRetries) {
-        return ResponseEntity.ok(response);
+            try {
+                response = restTemplate.getForObject(apiUrl, StockHistoryResponse.class);
+            } catch (RestClientException e) {
+                // Log the exception or perform any other error handling
+                log.error("Error while making API request. Retrying... (Retry count: {})", retryCount + 1);
+
+                // Increment the retry count
+                retryCount++;
+                // Add some delay before the next retry (you can adjust this as needed)
+                try {
+                    Thread.sleep(300); // 1 second delay
+                } catch (InterruptedException ex) {
+                    Thread.currentThread().interrupt();
                 }
+            }
+        }
+
+        if (response != null) {
+            // API request successful, return the response
+            return ResponseEntity.ok(response);
+        } else {
+            // All retries failed, return an error response
+            log.error("Failed to get a successful response after {} retries.", maxRetries);
+            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
+        }
+        // 返回响应
+    }
+
+
 
 
     private String buildApiUrl(StockHistoryRequest request) {
@@ -647,4 +742,19 @@ public class MoneyApiController {
         bseTopLoser();
     }
 
+    private Cache<String, List<MoneyStockSuggestDTO>> gainerStockSuggestCache = CacheBuilder.newBuilder()
+            .maximumSize(100) // 设置缓存的最大大小
+            .expireAfterWrite(1, TimeUnit.HOURS) // 设置缓存条目的过期时间
+            .build();
+
+    private Cache<String, List<MoneyStockSuggestDTO>> loserStockSuggestCache = CacheBuilder.newBuilder()
+            .maximumSize(100) // 设置缓存的最大大小
+            .expireAfterWrite(1, TimeUnit.HOURS) // 设置缓存条目的过期时间
+            .build();
+
+    private Cache<String, List<MoneyStockSuggestDTO>> activesStockSuggestCache = CacheBuilder.newBuilder()
+            .maximumSize(100) // 设置缓存的最大大小
+            .expireAfterWrite(30, TimeUnit.MINUTES) // 设置缓存条目的过期时间
+            .build();
+
 }