nhanvubk/ger-market-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

get germany news

Browse Source
This commit is contained in:
vu-tran
2025-06-19 14:14:13 +07:00
parent 5b3504f32a
commit e423e3fc19
2 changed files with 214 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
src/main/java/cn/stock/market/infrastructure/job/InvestingTask.java
View File

@@ -17,8 +17,16 @@ import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
import java.io.IOException;
import java.util.Arrays;
@@ -31,12 +39,16 @@ import java.util.regex.Pattern;
@Slf4j
@Component
@RestController
@RequestMapping("/api/market/investing")
public class InvestingTask {
@Autowired
StockService stockService;
@Autowired
SiteNewsRepository newsRepository;
@Autowired
RestTemplate restTemplate;
// @Scheduled(cron = "0 0 6 * * ?")
public void syncIndiaData(){
@@ -169,4 +181,138 @@ public class InvestingTask {
return null;
}
/*德国新闻接口*/
@Scheduled(cron = "0 0 0/3 * * ?")
public void saveGerNews() {
log.info("德国股票新闻数据同步开始");
int savedCount = 0;
int totalCount = 0;
try {
// API URL for getting news list
String newsListUrl = "https://api.boerse-frankfurt.de/v1/data/category_news?newsType=ALL&lang=de&offset=0&limit=50";
// Headers for the API request
HttpHeaders headers = new HttpHeaders();
headers.add("accept", "application/json, text/plain, */*");
headers.add("accept-language", "en-US,en;q=0.9,vi;q=0.8,ug;q=0.7,fr;q=0.6");
headers.add("origin", "https://www.boerse-frankfurt.de");
headers.add("priority", "u=1, i");
headers.add("referer", "https://www.boerse-frankfurt.de/");
headers.add("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36");
HttpEntity<String> entity = new HttpEntity<>(headers);
// Get news list
ResponseEntity<String> response = restTemplate.exchange(
newsListUrl,
HttpMethod.GET,
entity,
String.class
);
if (response.getStatusCode().value() == 200 && response.getBody() != null) {
JSONObject newsListResponse = JSON.parseObject(response.getBody());
JSONArray newsData = newsListResponse.getJSONArray("data");
if (newsData != null && newsData.size() > 0) {
totalCount = newsData.size();
log.info("Found {} German news items to process", totalCount);
for (int i = 0; i < newsData.size(); i++) {
try {
JSONObject newsItem = newsData.getJSONObject(i);
String newsId = newsItem.getString("id");
String headline = newsItem.getString("headline");
String time = newsItem.getString("time");
String source = newsItem.getString("source");
String teaserText = newsItem.getString("teaserText");
String teaserImageUrl = newsItem.getString("teaserImageUrl");
// Check if news already exists
List<SiteNews> existingNews = newsRepository.findAll(QSiteNewsPO.siteNewsPO.sourceId.eq(newsId));
if (existingNews.size() == 0) {
// Get news detail
String newsDetailUrl = "https://api.boerse-frankfurt.de/v1/data/news?id=" + newsId + "&lang=de";
HttpEntity<String> detailEntity = new HttpEntity<>(headers);
ResponseEntity<String> detailResponse = restTemplate.exchange(
newsDetailUrl,
HttpMethod.GET,
detailEntity,
String.class
);
if (detailResponse.getStatusCode().value() == 200 && detailResponse.getBody() != null) {
JSONObject newsDetail = JSON.parseObject(detailResponse.getBody());
String body = newsDetail.getString("body");
// Create SiteNews entity
SiteNews siteNews = new SiteNews();
siteNews.setAddTime(new Date());
siteNews.setSourceId(newsId);
siteNews.setTitle(headline);
siteNews.setSourceName(source);
siteNews.setDescription(teaserText != null ? teaserText : "");
siteNews.setImgurl(teaserImageUrl);
siteNews.setContent(body != null ? body : "");
siteNews.setStatus(1);
siteNews.setType(1); // Set as financial news type
siteNews.setViews(0);
// Parse and set show time
if (time != null && !time.isEmpty()) {
try {
// Parse ISO 8601 format: "2025-06-19T08:37:58+02:00"
// Remove timezone offset and convert to standard format
String timeStr = time.replace("+02:00", "").replace("T", " ");
siteNews.setShowTime(DateTimeUtil.strToDate(timeStr, "yyyy-MM-dd HH:mm:ss"));
} catch (Exception e) {
log.warn("Failed to parse time for news {}: {}", newsId, time);
siteNews.setShowTime(new Date());
}
} else {
siteNews.setShowTime(new Date());
}
try {
newsRepository.save(siteNews);
savedCount++;
log.info("Saved German news [{}/{}]: {}", savedCount, totalCount, headline);
} catch (Exception e) {
log.warn("Failed to save German news {}: {}", newsId, e.getMessage());
}
} else {
log.warn("Failed to get news detail for {}: HTTP {}", newsId, detailResponse.getStatusCode());
}
} else {
log.debug("News {} already exists, skipping", newsId);
}
} catch (Exception e) {
log.warn("Error processing news item {}: {}", i, e.getMessage());
}
}
} else {
log.warn("No news data found in API response");
}
} else {
log.error("Failed to get news list: HTTP {}", response.getStatusCode());
}
log.info("德国股票新闻数据同步完成，处理了 {} 条新闻，保存了 {} 条新闻", totalCount, savedCount);
} catch (Exception e) {
log.error("德国新闻数据同步异常，异常信息: {}", e.getMessage(), e);
}
}
/**
* Test method to manually trigger German news sync
* This can be called via REST API or scheduled task
*/
@GetMapping("/test-ger-news")
public String testSaveGerNews() {
log.info("Testing German news sync...");
saveGerNews();
log.info("German news sync test completed");
return "German news sync test completed. Check logs for details.";
}
}

132
src/main/java/cn/stock/market/infrastructure/job/StockNewTask.java
View File

@@ -38,75 +38,79 @@ public class StockNewTask {
// @PostConstruct
@Scheduled(cron = "0 01 22 * * ?")
public void syncStock() throws Exception {
int limit = 20000;
List<String> exchanges = Arrays.asList("BER", "DUS", "HAM", "HAN", "MUN", "SWB", "FWB", "XETR");
Map<String, Stock> stockGidMap = stockRepository.cacheGidMap();
for (String exchange : exchanges) {
List<Stock> newStocks = new ArrayList<>();
int start = 0;
int symbolsRemaining;
do {
int finalStart = start;
if(start > 0){
finalStart = start + 1;
}
String url = BASE_URL + PARAMS.replace("{start}", String.valueOf(finalStart)).replace("{exchange}", exchange);
Request request = new Request.Builder()
.url(url)
.addHeader("Origin", "https://www.tradingview.com")
.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.addHeader("Accept", "application/json, text/plain, */*")
.addHeader("Referer", "https://www.tradingview.com/")
.build();
Response response = client.newCall(request).execute();
if (!response.isSuccessful()) {
throw new RuntimeException("Unexpected response code: " + response.code());
}
String responseBody = response.body().string();
JSONObject json = new JSONObject(responseBody);
JSONArray symbols = json.getJSONArray("symbols");
symbolsRemaining = json.getInt("symbols_remaining");
for (int i = 0; i < symbols.length(); i++) {
JSONObject s = symbols.getJSONObject(i);
if (s.optString("type").equals("stock") && !stockGidMap.containsKey(s.optString("exchange") + ":" + s.getString("symbol"))) {
Stock stock = new Stock();
stock.setStockGid(s.optString("exchange") + ":" +s.getString("symbol"));
stock.setStockCode(stock.getStockGid());
stock.setStockSpell(s.optString("symbol"));
stock.setStockName(s.optString("description"));
stock.setStockSymbol(s.optString("symbol"));
stock.setStockType(s.optString("exchange"));
stock.setIsLock(0);
stock.setIsShow(0);
stock.setAddTime(new Date());
stock.setStockState(0);
stock.setStockPlate("https://s3-symbol-logo.tradingview.com/" + s.optString("source_logoid") + "--big.svg");
newStocks.add(stock);
stockGidMap.put(s.optString("exchange") + ":" + s.getString("symbol"), stock);
try {
int limit = 20000;
List<String> exchanges = Arrays.asList("BER", "DUS", "HAM", "HAN", "MUN", "SWB", "FWB", "XETR");
Map<String, Stock> stockGidMap = stockRepository.cacheGidMap();
for (String exchange : exchanges) {
List<Stock> newStocks = new ArrayList<>();
int start = 0;
int symbolsRemaining;
do {
int finalStart = start;
if (start > 0) {
finalStart = start + 1;
}
}
Thread.sleep(500);
String url = BASE_URL + PARAMS.replace("{start}", String.valueOf(finalStart)).replace("{exchange}", exchange);
Request request = new Request.Builder()
.url(url)
.addHeader("Origin", "https://www.tradingview.com")
.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.addHeader("Accept", "application/json, text/plain, */*")
.addHeader("Referer", "https://www.tradingview.com/")
.build();
start += symbols.length();
System.out.println("Fetched: " + symbols.length() + ", Remaining: " + symbolsRemaining);
try {
if (!newStocks.isEmpty()) {
stockRepository.saveAll(newStocks);
Response response = client.newCall(request).execute();
if (!response.isSuccessful()) {
throw new RuntimeException("Unexpected response code: " + response.code());
}
}catch (Exception e) {
log.error("Insert stock failed: {}", e.getMessage());
}
if (start >= limit) {
break;
}
String responseBody = response.body().string();
JSONObject json = new JSONObject(responseBody);
JSONArray symbols = json.getJSONArray("symbols");
symbolsRemaining = json.getInt("symbols_remaining");
} while (symbolsRemaining > 0);
for (int i = 0; i < symbols.length(); i++) {
JSONObject s = symbols.getJSONObject(i);
if (s.optString("type").equals("stock") && !stockGidMap.containsKey(s.optString("exchange") + ":" + s.getString("symbol"))) {
Stock stock = new Stock();
stock.setStockGid(s.optString("exchange") + ":" + s.getString("symbol"));
stock.setStockCode(stock.getStockGid());
stock.setStockSpell(s.optString("symbol"));
stock.setStockName(s.optString("description"));
stock.setStockSymbol(s.optString("symbol"));
stock.setStockType(s.optString("exchange"));
stock.setIsLock(0);
stock.setIsShow(0);
stock.setAddTime(new Date());
stock.setStockState(0);
stock.setStockPlate("https://s3-symbol-logo.tradingview.com/" + s.optString("source_logoid") + "--big.svg");
newStocks.add(stock);
stockGidMap.put(s.optString("exchange") + ":" + s.getString("symbol"), stock);
}
}
Thread.sleep(500);
start += symbols.length();
System.out.println("Fetched: " + symbols.length() + ", Remaining: " + symbolsRemaining);
try {
if (!newStocks.isEmpty()) {
stockRepository.saveAll(newStocks);
}
} catch (Exception e) {
log.error("Insert stock failed: {}", e.getMessage());
}
if (start >= limit) {
break;
}
} while (symbolsRemaining > 0);
}
}catch (Exception e) {
log.error("Insert stock failed: {}", e.getMessage());
e.printStackTrace();
}
}
}