diff --git a/src/main/java/cn/stock/market/infrastructure/job/InvestingTask.java b/src/main/java/cn/stock/market/infrastructure/job/InvestingTask.java index 3fb5442..6d446f3 100644 --- a/src/main/java/cn/stock/market/infrastructure/job/InvestingTask.java +++ b/src/main/java/cn/stock/market/infrastructure/job/InvestingTask.java @@ -17,8 +17,16 @@ import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestTemplate; import java.io.IOException; import java.util.Arrays; @@ -31,12 +39,16 @@ import java.util.regex.Pattern; @Slf4j @Component +@RestController +@RequestMapping("/api/market/investing") public class InvestingTask { @Autowired StockService stockService; @Autowired SiteNewsRepository newsRepository; + @Autowired + RestTemplate restTemplate; // @Scheduled(cron = "0 0 6 * * ?") public void syncIndiaData(){ @@ -169,4 +181,138 @@ public class InvestingTask { return null; } + + /*德国新闻接口*/ + @Scheduled(cron = "0 0 0/3 * * ?") + public void saveGerNews() { + log.info("德国股票新闻数据同步开始"); + int savedCount = 0; + int totalCount = 0; + try { + // API URL for getting news list + String newsListUrl = "https://api.boerse-frankfurt.de/v1/data/category_news?newsType=ALL&lang=de&offset=0&limit=50"; + + // Headers for the API request + HttpHeaders headers = new HttpHeaders(); + headers.add("accept", "application/json, text/plain, */*"); + headers.add("accept-language", "en-US,en;q=0.9,vi;q=0.8,ug;q=0.7,fr;q=0.6"); + headers.add("origin", "https://www.boerse-frankfurt.de"); + headers.add("priority", "u=1, i"); + headers.add("referer", "https://www.boerse-frankfurt.de/"); + headers.add("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"); + + HttpEntity entity = new HttpEntity<>(headers); + + // Get news list + ResponseEntity response = restTemplate.exchange( + newsListUrl, + HttpMethod.GET, + entity, + String.class + ); + + if (response.getStatusCode().value() == 200 && response.getBody() != null) { + JSONObject newsListResponse = JSON.parseObject(response.getBody()); + JSONArray newsData = newsListResponse.getJSONArray("data"); + + if (newsData != null && newsData.size() > 0) { + totalCount = newsData.size(); + log.info("Found {} German news items to process", totalCount); + + for (int i = 0; i < newsData.size(); i++) { + try { + JSONObject newsItem = newsData.getJSONObject(i); + String newsId = newsItem.getString("id"); + String headline = newsItem.getString("headline"); + String time = newsItem.getString("time"); + String source = newsItem.getString("source"); + String teaserText = newsItem.getString("teaserText"); + String teaserImageUrl = newsItem.getString("teaserImageUrl"); + + // Check if news already exists + List existingNews = newsRepository.findAll(QSiteNewsPO.siteNewsPO.sourceId.eq(newsId)); + if (existingNews.size() == 0) { + // Get news detail + String newsDetailUrl = "https://api.boerse-frankfurt.de/v1/data/news?id=" + newsId + "&lang=de"; + HttpEntity detailEntity = new HttpEntity<>(headers); + + ResponseEntity detailResponse = restTemplate.exchange( + newsDetailUrl, + HttpMethod.GET, + detailEntity, + String.class + ); + + if (detailResponse.getStatusCode().value() == 200 && detailResponse.getBody() != null) { + JSONObject newsDetail = JSON.parseObject(detailResponse.getBody()); + String body = newsDetail.getString("body"); + + // Create SiteNews entity + SiteNews siteNews = new SiteNews(); + siteNews.setAddTime(new Date()); + siteNews.setSourceId(newsId); + siteNews.setTitle(headline); + siteNews.setSourceName(source); + siteNews.setDescription(teaserText != null ? teaserText : ""); + siteNews.setImgurl(teaserImageUrl); + siteNews.setContent(body != null ? body : ""); + siteNews.setStatus(1); + siteNews.setType(1); // Set as financial news type + siteNews.setViews(0); + + // Parse and set show time + if (time != null && !time.isEmpty()) { + try { + // Parse ISO 8601 format: "2025-06-19T08:37:58+02:00" + // Remove timezone offset and convert to standard format + String timeStr = time.replace("+02:00", "").replace("T", " "); + siteNews.setShowTime(DateTimeUtil.strToDate(timeStr, "yyyy-MM-dd HH:mm:ss")); + } catch (Exception e) { + log.warn("Failed to parse time for news {}: {}", newsId, time); + siteNews.setShowTime(new Date()); + } + } else { + siteNews.setShowTime(new Date()); + } + + try { + newsRepository.save(siteNews); + savedCount++; + log.info("Saved German news [{}/{}]: {}", savedCount, totalCount, headline); + } catch (Exception e) { + log.warn("Failed to save German news {}: {}", newsId, e.getMessage()); + } + } else { + log.warn("Failed to get news detail for {}: HTTP {}", newsId, detailResponse.getStatusCode()); + } + } else { + log.debug("News {} already exists, skipping", newsId); + } + } catch (Exception e) { + log.warn("Error processing news item {}: {}", i, e.getMessage()); + } + } + } else { + log.warn("No news data found in API response"); + } + } else { + log.error("Failed to get news list: HTTP {}", response.getStatusCode()); + } + log.info("德国股票新闻数据同步完成，处理了 {} 条新闻，保存了 {} 条新闻", totalCount, savedCount); + } catch (Exception e) { + log.error("德国新闻数据同步异常，异常信息: {}", e.getMessage(), e); + } + } + + /** + * Test method to manually trigger German news sync + * This can be called via REST API or scheduled task + */ + @GetMapping("/test-ger-news") + public String testSaveGerNews() { + log.info("Testing German news sync..."); + saveGerNews(); + log.info("German news sync test completed"); + return "German news sync test completed. Check logs for details."; + } } diff --git a/src/main/java/cn/stock/market/infrastructure/job/StockNewTask.java b/src/main/java/cn/stock/market/infrastructure/job/StockNewTask.java index 2b25a68..33d5990 100644 --- a/src/main/java/cn/stock/market/infrastructure/job/StockNewTask.java +++ b/src/main/java/cn/stock/market/infrastructure/job/StockNewTask.java @@ -38,75 +38,79 @@ public class StockNewTask { // @PostConstruct @Scheduled(cron = "0 01 22 * * ?") public void syncStock() throws Exception { - - int limit = 20000; - List exchanges = Arrays.asList("BER", "DUS", "HAM", "HAN", "MUN", "SWB", "FWB", "XETR"); - Map stockGidMap = stockRepository.cacheGidMap(); - for (String exchange : exchanges) { - List newStocks = new ArrayList<>(); - int start = 0; - int symbolsRemaining; - do { - int finalStart = start; - if(start > 0){ - finalStart = start + 1; - } - String url = BASE_URL + PARAMS.replace("{start}", String.valueOf(finalStart)).replace("{exchange}", exchange); - Request request = new Request.Builder() - .url(url) - .addHeader("Origin", "https://www.tradingview.com") - .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)") - .addHeader("Accept", "application/json, text/plain, */*") - .addHeader("Referer", "https://www.tradingview.com/") - .build(); - - Response response = client.newCall(request).execute(); - - if (!response.isSuccessful()) { - throw new RuntimeException("Unexpected response code: " + response.code()); - } - - String responseBody = response.body().string(); - JSONObject json = new JSONObject(responseBody); - JSONArray symbols = json.getJSONArray("symbols"); - symbolsRemaining = json.getInt("symbols_remaining"); - - for (int i = 0; i < symbols.length(); i++) { - JSONObject s = symbols.getJSONObject(i); - if (s.optString("type").equals("stock") && !stockGidMap.containsKey(s.optString("exchange") + ":" + s.getString("symbol"))) { - Stock stock = new Stock(); - stock.setStockGid(s.optString("exchange") + ":" +s.getString("symbol")); - stock.setStockCode(stock.getStockGid()); - stock.setStockSpell(s.optString("symbol")); - stock.setStockName(s.optString("description")); - stock.setStockSymbol(s.optString("symbol")); - stock.setStockType(s.optString("exchange")); - stock.setIsLock(0); - stock.setIsShow(0); - stock.setAddTime(new Date()); - stock.setStockState(0); - stock.setStockPlate("https://s3-symbol-logo.tradingview.com/" + s.optString("source_logoid") + "--big.svg"); - newStocks.add(stock); - stockGidMap.put(s.optString("exchange") + ":" + s.getString("symbol"), stock); + try { + int limit = 20000; + List exchanges = Arrays.asList("BER", "DUS", "HAM", "HAN", "MUN", "SWB", "FWB", "XETR"); + Map stockGidMap = stockRepository.cacheGidMap(); + for (String exchange : exchanges) { + List newStocks = new ArrayList<>(); + int start = 0; + int symbolsRemaining; + do { + int finalStart = start; + if (start > 0) { + finalStart = start + 1; } - } - Thread.sleep(500); + String url = BASE_URL + PARAMS.replace("{start}", String.valueOf(finalStart)).replace("{exchange}", exchange); + Request request = new Request.Builder() + .url(url) + .addHeader("Origin", "https://www.tradingview.com") + .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)") + .addHeader("Accept", "application/json, text/plain, */*") + .addHeader("Referer", "https://www.tradingview.com/") + .build(); - start += symbols.length(); - System.out.println("Fetched: " + symbols.length() + ", Remaining: " + symbolsRemaining); - try { - if (!newStocks.isEmpty()) { - stockRepository.saveAll(newStocks); + Response response = client.newCall(request).execute(); + + if (!response.isSuccessful()) { + throw new RuntimeException("Unexpected response code: " + response.code()); } - }catch (Exception e) { - log.error("Insert stock failed: {}", e.getMessage()); - } - if (start >= limit) { - break; - } + String responseBody = response.body().string(); + JSONObject json = new JSONObject(responseBody); + JSONArray symbols = json.getJSONArray("symbols"); + symbolsRemaining = json.getInt("symbols_remaining"); - } while (symbolsRemaining > 0); + for (int i = 0; i < symbols.length(); i++) { + JSONObject s = symbols.getJSONObject(i); + if (s.optString("type").equals("stock") && !stockGidMap.containsKey(s.optString("exchange") + ":" + s.getString("symbol"))) { + Stock stock = new Stock(); + stock.setStockGid(s.optString("exchange") + ":" + s.getString("symbol")); + stock.setStockCode(stock.getStockGid()); + stock.setStockSpell(s.optString("symbol")); + stock.setStockName(s.optString("description")); + stock.setStockSymbol(s.optString("symbol")); + stock.setStockType(s.optString("exchange")); + stock.setIsLock(0); + stock.setIsShow(0); + stock.setAddTime(new Date()); + stock.setStockState(0); + stock.setStockPlate("https://s3-symbol-logo.tradingview.com/" + s.optString("source_logoid") + "--big.svg"); + newStocks.add(stock); + stockGidMap.put(s.optString("exchange") + ":" + s.getString("symbol"), stock); + } + } + Thread.sleep(500); + + start += symbols.length(); + System.out.println("Fetched: " + symbols.length() + ", Remaining: " + symbolsRemaining); + try { + if (!newStocks.isEmpty()) { + stockRepository.saveAll(newStocks); + } + } catch (Exception e) { + log.error("Insert stock failed: {}", e.getMessage()); + } + + if (start >= limit) { + break; + } + + } while (symbolsRemaining > 0); + } + }catch (Exception e) { + log.error("Insert stock failed: {}", e.getMessage()); + e.printStackTrace(); } } }