爬取新股数据测试

src/main/java/cn/stock/market/domain/basic/service/StockService.java

@@ -31,6 +31,10 @@ import okhttp3.Request;
 import okhttp3.Request.Builder;
 import org.apache.commons.lang3.StringUtils;
 import cn.stock.market.infrastructure.db.po.QStockPO;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.http.HttpEntity;
@@ -252,27 +256,18 @@ public class StockService {
 	public ServerResponse getNews() {
 		String result = "";
 		try {
-			// 创建URL对象
-			URL url = new URL("https://www.business-standard.com/markets/news");
-
-			// 打开URL连接
-			URLConnection connection = url.openConnection();
-
-			// 获取输入流
-			BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
-
-			// 读取数据
-			StringBuilder data = new StringBuilder();
-			String line;
-			while ((line = reader.readLine()) != null) {
-				data.append(line);
+			// 使用Jsoup连接到网页
+			Document doc = Jsoup.connect("https://www.business-standard.com/markets/news")
+					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
+					.header("Referer", "https://www.business-standard.com/")
+					.header("Accept-Language", "en-US,en;q=0.9")
+					.get();
+			// 使用选择器提取所需的数据
+			Elements newsHeadlines = doc.select(".article-listing");
+			// 遍历提取到的数据并输出
+			for (Element headline : newsHeadlines) {
+				System.out.println(headline.text());
 			}
-
-			// 关闭输入流
-			reader.close();
-
-			// 输出抓取到的数据
-			result = data.toString();
 		} catch (Exception e) {
 			return ServerResponse.createByErrorMsg(e.toString());
 		}

src/main/java/cn/stock/market/infrastructure/api/investing/InvestingInvokerApis.java

@@ -209,7 +209,7 @@ public class InvestingInvokerApis {
 	public static void main(String[] args) throws IOException {
 		JSONObject __market = of().__page(1,100);
 		System.out.println(__market);
-		
+
 		JSONObject __page = of().__page(1, 10);
 		JSONObject __page_nifty100 = of().__page_nifty100(1, 10);