nhanvubk/ger-market-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into 'main'

Browse Source 
Revert "fix cron"

See merge request india/india_market_java!59
This commit is contained in:
mush_ball
2025-05-21 14:34:33 +00:00
parent d4dd23ba34 90a26823f0
commit d2c03b32f0
5 changed files with 147 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitlab-ci.yml
View File

@@ -67,8 +67,8 @@ deploy-dev:
script: script:
- echo "Deploying application..." - echo "Deploying application..."
- ls - ls
- apt-get update - apk update
- apt-get install -y curl - apk add curl
- | - |
curl -X POST \ curl -X POST \
-H "Content-Type: application/json" \ -H "Content-Type: application/json" \

115
src/main/java/cn/stock/market/domain/basic/service/StockService.java
View File

@@ -255,47 +255,106 @@ public class StockService {
// .header("Accept-Language", "en-US,en;q=0.9") // .header("Accept-Language", "en-US,en;q=0.9")
// .get(); // .get();
String url = "https://www.business-standard.com/markets/news"; String url = "https://www.business-standard.com/markets/news";
Document doc = Jsoup.connect(url) // Document doc = Jsoup.connect(url)
.referrer("https://www.business-standard.com/") // .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
.header("Accept-Language", "en-US,en;q=0.9") // .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.userAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1") // .header("Accept-Language", "en-US,en;q=0.5")
.timeout(5000) // timeout 5 seconds // .header("Connection", "keep-alive")
.get(); // .referrer("https://www.google.com")
// .ignoreHttpErrors(true)
// .timeout(10000)
// .get();
String doGetNews = HttpClientRequest.doGetNews(url);
// result = doc.html().substring(doc.html().indexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">"),doc.html().lastIndexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">")+500); // result = doc.html().substring(doc.html().indexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">"),doc.html().lastIndexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">")+500);
Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57"); // Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57");
StringBuilder sb = new StringBuilder(); // StringBuilder sb = new StringBuilder();
for (Element divElement : divElements) { // for (Element divElement : divElements) {
sb.append(divElement.outerHtml()).append("\n"); // sb.append(divElement.outerHtml()).append("\n");
} // }
result = sb.toString(); // result = sb.toString();
result = extractNewsFromHtml(doGetNews);
} catch (Exception e) { } catch (Exception e) {
return e.toString(); return e.toString();
} }
return result; return result;
} }
private String extractNewsFromHtml(String rawHtml) {
Document doc = Jsoup.parse(rawHtml);
// Optional: extract raw block around <div class="listingstyle_shortvideoimg__0TWuX shortvideoimg">
int start = rawHtml.indexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">");
int end = rawHtml.lastIndexOf("<div class=\"listingstyle_shortvideoimg__0TWuX shortvideoimg\">") + 500;
String result = (start != -1 && end > start) ? rawHtml.substring(start, end) : "";
// Extract article list using CSS selector
Elements divElements = doc.select("div.listingstyle_cardlistlist__dfq57");
StringBuilder sb = new StringBuilder();
for (Element divElement : divElements) {
sb.append(divElement.outerHtml()).append("\n");
}
// If you only want the div content:
result = sb.toString();
return result;
}
public List<String> getNewsInfo(String url) { public List<String> getNewsInfo(String url) {
// String result = "";
// List<String> list = new ArrayList<>();
// try {
// // 使用Jsoup连接到网页
//// Document doc = Jsoup.connect(url)
//// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
//// .header("Referer", "https://www.business-standard.com/")
//// .header("Accept-Language", "en-US,en;q=0.9")
//// .get();
// Document doc = Jsoup.connect(url)
// .referrer("https://www.business-standard.com/")
// .header("Accept-Language", "en-US,en;q=0.9")
// .userAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1")
// .timeout(5000) // timeout 5 seconds
// .get();
// result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1);
// list.add(result);
// list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf("<meta property=\"og:url") - 5));
// } catch (Exception e) {
// list.add(e.toString());
// return list;
// }
// return list;
String result = ""; String result = "";
List<String> list = new ArrayList<>(); List<String> list = new ArrayList<>();
try { try {
// 使用Jsoup连接到网页 // Step 1: Get raw HTML via HttpClient
// Document doc = Jsoup.connect(url) String rawHtml = HttpClientRequest.doGetNews(url);
// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
// .header("Referer", "https://www.business-standard.com/") // Step 2: Parse HTML with Jsoup
// .header("Accept-Language", "en-US,en;q=0.9") Document doc = Jsoup.parse(rawHtml);
// .get();
Document doc = Jsoup.connect(url) // Step 3: Extract content manually (like original)
.referrer("https://www.business-standard.com/") int startIdx = rawHtml.indexOf("articleBody") + 14;
.header("Accept-Language", "en-US,en;q=0.9") int endIdx = rawHtml.indexOf(",\"author\":") - 1;
.userAgent("Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1")
.timeout(5000) // timeout 5 seconds if (startIdx > 0 && endIdx > startIdx) {
.get(); result = rawHtml.substring(startIdx, endIdx);
result = doc.html().substring(doc.html().indexOf("articleBody") + 14, doc.html().indexOf(",\"author\":") - 1); } else {
result = "[articleBody not found]";
}
list.add(result); list.add(result);
list.add(doc.html().substring(doc.html().indexOf("og:title") + 19, doc.html().indexOf("<meta property=\"og:url") - 5));
// Step 4: Extract og:title
int titleStart = rawHtml.indexOf("og:title") + 19;
int titleEnd = rawHtml.indexOf("<meta property=\"og:url") - 5;
if (titleStart > 0 && titleEnd > titleStart) {
list.add(rawHtml.substring(titleStart, titleEnd));
} else {
list.add("[title not found]");
}
} catch (Exception e) { } catch (Exception e) {
list.add(e.toString()); list.add("[Error] " + e.getMessage());
return list;
} }
return list; return list;
} }

50
src/main/java/cn/stock/market/utils/HttpClientRequest.java
View File

@@ -265,6 +265,56 @@ public class HttpClientRequest {
return result; return result;
} }
public static String doGetNews(String url) {
CloseableHttpClient httpClient = null;
CloseableHttpResponse response = null;
String result = "";
try {
httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
// Spoof real browser headers
httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
httpGet.setHeader("Accept-Language", "en-US,en;q=0.9,vi;q=0.8,ug;q=0.7,fr;q=0.6");
httpGet.setHeader("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1");
httpGet.setHeader("Priority", "u=0, i");
httpGet.setHeader("Sec-Fetch-Dest", "document");
httpGet.setHeader("Sec-Fetch-Mode", "navigate");
httpGet.setHeader("Sec-Fetch-Site", "none");
httpGet.setHeader("Sec-Fetch-User", "?1");
httpGet.setHeader("Upgrade-Insecure-Requests", "1");
// Set cookies exactly like in curl
httpGet.setHeader("Cookie", "userUid=1747102033185-d570fba9-62fd-40be-93ca-ed08b4de57d4; _sid=MTc0NzEwMjAzMzE4Ni4ycTU%3D; _scor_uid=135c13065ff84620b5318b489af93e87; _gcl_au=1.1.249135292.1747102036; _ga=GA1.1.1031614211.1747102037; WZRK_G=28895afb56ff48dda59fe8de0af746bf; FCNEC=%5B%5B%22AKsRol8sHYeSYz_FYPkInYXN3P4ZDPfVKbsRfILfDuOMLhDtkTuoCJP5MlvT9gIbOe7IlDfY8ZeHszhwdVtAoKF1gWv0pLAq5EqpLpse8CEm_ZNv-bUSs6zEyqpOkeKWFI_Ei6VfNAvnZAO8PcXdF8_ncsaO902X7g%3D%3D%22%5D%5D; _ga_KRGL1M61LX=GS2.1.s1747105135$o2$g0$t1747105135$j60$l0$h0");
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(10000)
.setSocketTimeout(15000)
.setConnectionRequestTimeout(10000)
.build();
httpGet.setConfig(requestConfig);
response = httpClient.execute(httpGet);
HttpEntity entity = response.getEntity();
if (entity != null) {
result = EntityUtils.toString(entity);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (response != null) response.close();
if (httpClient != null) httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return result;
}
public static void main(String[] args) { public static void main(String[] args) {
String url = "https://marketapi.intoday.in/widget/topgainer/view?exchange=nse"; String url = "https://marketapi.intoday.in/widget/topgainer/view?exchange=nse";
String str = doGet(url); String str = doGet(url);

6
src/main/java/cn/stock/market/web/MoneyProxyApiControl.java
View File

@@ -26,13 +26,14 @@ public class MoneyProxyApiControl {
@GetMapping({"/market/api/proxy/queryStockMoneyProxy","/api/proxy/queryStockMoneyProxy"}) @GetMapping({"/market/api/proxy/queryStockMoneyProxy","/api/proxy/queryStockMoneyProxy"})
public JSONObject queryStockByMoneyIdAndExchangeType(String scId,String exchangeType){ public JSONObject queryStockByMoneyIdAndExchangeType(String scId,String exchangeType){
log.info("queryStockByMoneyIdAndExchangeType");
return stockByMoneyIdAndExchangeTypeFromHttp(scId,exchangeType); return stockByMoneyIdAndExchangeTypeFromHttp(scId,exchangeType);
} }
public static JSONObject stockByMoneyIdAndExchangeTypeFromHttp(String scId, String exchangeType) { public static JSONObject stockByMoneyIdAndExchangeTypeFromHttp(String scId, String exchangeType) {
try { try {
String url = "https://priceapi.moneycontrol.com/pricefeed/"+exchangeType+"/equitycash/" + scId ; String url = "https://moneycontrol.titanrwa.org/pricefeed/"+exchangeType+"/equitycash/" + scId ;
URL obj = new URL(url); URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection) obj.openConnection(); HttpURLConnection con = (HttpURLConnection) obj.openConnection();
con.setRequestMethod("GET"); con.setRequestMethod("GET");
@@ -53,7 +54,8 @@ public class MoneyProxyApiControl {
log.error("HTTP request failed with response code: " + responseCode); log.error("HTTP request failed with response code: " + responseCode);
} }
} catch (Exception e) { } catch (Exception e) {
log.error("stockByMoneyIdAndExchangeTypeFromHttp",e); e.printStackTrace();
log.error("stockByMoneyIdAndExchangeTypeFromHttp failed {}",e.getMessage());
} }
return new JSONObject(); return new JSONObject();
} }

6
src/main/resources/application-base-alpha.yml
View File

@@ -3,9 +3,9 @@ spring:
show-sql: true show-sql: true
# Redis配置 # Redis配置
redis: redis:
host: 43.156.40.39 host: 43.153.174.179
password: a5v8b86P4mVzFlUqJV password: a5v8b86P4mVzFlUqJV
port: 30031 port: 30001
database: 1 database: 1
lettuce: lettuce:
pool: pool:
@@ -17,7 +17,7 @@ spring:
datasource: datasource:
stock-market: stock-market:
driver-class-name: com.mysql.cj.jdbc.Driver driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://43.156.40.39:30030/india_stock?useUnicode=true&characterEncoding=utf-8 url: jdbc:mysql://43.153.174.179:30000/india_stock?useUnicode=true&characterEncoding=utf-8
username: root username: root
password: uNejHIFQGJOUtYTmE password: uNejHIFQGJOUtYTmE
maxActive: 500 maxActive: 500