From 0743d9e955c028f1cc20248e41f0d5c312b63355 Mon Sep 17 00:00:00 2001 From: Duc-Tam Nguyen Date: Tue, 16 Jun 2026 12:15:34 +0700 Subject: [PATCH 1/2] feat: add article, hot, search, comments commands and 6 more channels Implements the four missing commands from spec 2307: - article : full article content via the 3g.163.com JSON API; strips HTML from body field; accepts docid or canonical URL - hot: today's trending articles via news.163.com/special/0001386F/news_hot.js (var data=... JS wrapper) - search : searches so.163.com and parses the HTML result list - comments : hot comments via comment.api.163.com with Referer header Expands channels from 6 to 12: adds finance (cm_money), auto (cm_auto), home (cm_house), world (cm_guoji), edu (cm_edu), game (cm_game). Adds ArticleBaseURL / CommentBaseURL / SearchBaseURL to Config so tests can override each endpoint independently. 23 httptest-only tests, all green. --- cli/cmd_article.go | 27 +++++++ cli/cmd_comments.go | 28 +++++++ cli/cmd_hot.go | 23 ++++++ cli/cmd_search.go | 28 +++++++ cli/root.go | 4 + netease163/article.go | 99 +++++++++++++++++++++++ netease163/article_test.go | 134 ++++++++++++++++++++++++++++++ netease163/comments.go | 138 +++++++++++++++++++++++++++++++ netease163/comments_test.go | 148 ++++++++++++++++++++++++++++++++++ netease163/hot.go | 47 +++++++++++ netease163/hot_test.go | 72 +++++++++++++++++ netease163/netease163.go | 33 +++++--- netease163/netease163_test.go | 4 +- netease163/search.go | 87 ++++++++++++++++++++ netease163/search_test.go | 136 +++++++++++++++++++++++++++++++ netease163/types.go | 65 +++++++++++++++ 16 files changed, 1060 insertions(+), 13 deletions(-) create mode 100644 cli/cmd_article.go create mode 100644 cli/cmd_comments.go create mode 100644 cli/cmd_hot.go create mode 100644 cli/cmd_search.go create mode 100644 netease163/article.go create mode 100644 netease163/article_test.go create mode 100644 netease163/comments.go create mode 100644 netease163/comments_test.go create mode 100644 netease163/hot.go create mode 100644 netease163/hot_test.go create mode 100644 netease163/search.go create mode 100644 netease163/search_test.go diff --git a/cli/cmd_article.go b/cli/cmd_article.go new file mode 100644 index 0000000..863df76 --- /dev/null +++ b/cli/cmd_article.go @@ -0,0 +1,27 @@ +package cli + +import ( + "github.com/spf13/cobra" +) + +// articleCmd returns the article command. +func (a *App) articleCmd() *cobra.Command { + return &cobra.Command{ + Use: "article ", + Short: "Fetch full article content from NetEase 163", + Long: `Fetch the full content of a NetEase 163 article by its docid or URL. + +Examples: + netease163 article KVACHDNB0514D3UH + netease163 article https://www.163.com/dy/article/KVACHDNB0514D3UH.html`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + a.progressf("fetching article %q...", args[0]) + detail, err := a.client.Article(cmd.Context(), args[0]) + if err != nil { + return mapFetchErr(err) + } + return a.render(detail) + }, + } +} diff --git a/cli/cmd_comments.go b/cli/cmd_comments.go new file mode 100644 index 0000000..598d812 --- /dev/null +++ b/cli/cmd_comments.go @@ -0,0 +1,28 @@ +package cli + +import ( + "github.com/spf13/cobra" +) + +// commentsCmd returns the comments command. +func (a *App) commentsCmd() *cobra.Command { + return &cobra.Command{ + Use: "comments ", + Short: "Show top comments for a NetEase 163 article", + Long: `Fetch the most-liked comments for a NetEase 163 article. + +Examples: + netease163 comments KVACHDNB0514D3UH + netease163 comments https://www.163.com/dy/article/KVACHDNB0514D3UH.html`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + n := a.effectiveLimit(0) + a.progressf("fetching comments for %q...", args[0]) + comments, err := a.client.Comments(cmd.Context(), args[0], n) + if err != nil { + return mapFetchErr(err) + } + return a.renderOrEmpty(comments, len(comments)) + }, + } +} diff --git a/cli/cmd_hot.go b/cli/cmd_hot.go new file mode 100644 index 0000000..fb98270 --- /dev/null +++ b/cli/cmd_hot.go @@ -0,0 +1,23 @@ +package cli + +import ( + "github.com/spf13/cobra" +) + +// hotCmd returns the hot command. +func (a *App) hotCmd() *cobra.Command { + return &cobra.Command{ + Use: "hot", + Short: "Show today's hot/trending NetEase 163 articles", + Long: `Fetch and display today's trending articles from the NetEase 163 hot feed.`, + RunE: func(cmd *cobra.Command, _ []string) error { + n := a.effectiveLimit(20) + a.progressf("fetching hot articles...") + articles, err := a.client.Hot(cmd.Context(), n) + if err != nil { + return mapFetchErr(err) + } + return a.renderOrEmpty(articles, len(articles)) + }, + } +} diff --git a/cli/cmd_search.go b/cli/cmd_search.go new file mode 100644 index 0000000..3b7306c --- /dev/null +++ b/cli/cmd_search.go @@ -0,0 +1,28 @@ +package cli + +import ( + "github.com/spf13/cobra" +) + +// searchCmd returns the search command. +func (a *App) searchCmd() *cobra.Command { + return &cobra.Command{ + Use: "search ", + Short: "Search NetEase 163 news", + Long: `Search NetEase News (so.163.com) and display matching articles. + +Examples: + netease163 search 人工智能 + netease163 search "科技新闻" -n 5`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + n := a.effectiveLimit(20) + a.progressf("searching %q...", args[0]) + results, err := a.client.Search(cmd.Context(), args[0], n) + if err != nil { + return mapFetchErr(err) + } + return a.renderOrEmpty(results, len(results)) + }, + } +} diff --git a/cli/root.go b/cli/root.go index b5ea7c2..4ef9270 100644 --- a/cli/root.go +++ b/cli/root.go @@ -89,6 +89,10 @@ netease163 is an independent tool and is not affiliated with NetEase.`, root.AddCommand( app.newsCmd(), app.channelsCmd(), + app.articleCmd(), + app.hotCmd(), + app.searchCmd(), + app.commentsCmd(), newVersionCmd(), ) return root diff --git a/netease163/article.go b/netease163/article.go new file mode 100644 index 0000000..771f2db --- /dev/null +++ b/netease163/article.go @@ -0,0 +1,99 @@ +package netease163 + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" +) + +// Article fetches the full detail for a single article identified by docid or URL. +// If the argument looks like a URL, the docid is extracted from it. +func (c *Client) Article(ctx context.Context, docidOrURL string) (ArticleDetail, error) { + docid := docidOrURL + if strings.Contains(docidOrURL, "/") { + docid = extractDocID(docidOrURL) + } + if docid == "" { + return ArticleDetail{}, fmt.Errorf("cannot extract docid from %q", docidOrURL) + } + + url := c.cfg.ArticleBaseURL + "/touch/reconstruct/article/info/" + docid + ".json" + raw, err := c.get(ctx, url) + if err != nil { + return ArticleDetail{}, err + } + + var resp wireArticleResp + if err := json.Unmarshal(raw, &resp); err != nil { + return ArticleDetail{}, fmt.Errorf("parse article %s: %w", docid, err) + } + if resp.Code != 200 { + return ArticleDetail{}, fmt.Errorf("article %s: server returned code %d", docid, resp.Code) + } + + d := resp.Data + tags := make([]string, 0, len(d.Tags)) + for _, t := range d.Tags { + if t.Name != "" { + tags = append(tags, t.Name) + } + } + + return ArticleDetail{ + DocID: docid, + Title: d.Title, + URL: "https://www.163.com/dy/article/" + docid + ".html", + Source: d.Source, + Editor: d.Editor, + PublishedAt: unixToRFC3339(d.PTime), + Digest: d.Digest, + Body: stripHTML(d.Body), + Keywords: d.Keywords, + Tags: tags, + CommentCount: d.ReplyCount, + }, nil +} + +// wire types for article detail API. +type wireArticleResp struct { + Code int `json:"code"` + Data wireArticleData `json:"data"` +} + +type wireArticleData struct { + Title string `json:"title"` + Source string `json:"source"` + Editor string `json:"editor"` + Body string `json:"body"` + Keywords []string `json:"keywords"` + Tags []wireTag `json:"tags"` + ReplyCount int `json:"replyCount"` + PTime string `json:"ptime"` + Digest string `json:"digest"` +} + +type wireTag struct { + Name string `json:"name"` +} + +// stripHTML removes script/style blocks and HTML tags, converting

/
to newlines. +var ( + reScript = regexp.MustCompile(`(?is)]*>.*?`) + reStyle = regexp.MustCompile(`(?is)]*>.*?`) + rePBR = regexp.MustCompile(`(?i)`) + reTag = regexp.MustCompile(`<[^>]+>`) + reSpace = regexp.MustCompile(`[ \t]+`) + reLines = regexp.MustCompile(`\n{3,}`) +) + +func stripHTML(s string) string { + s = reScript.ReplaceAllString(s, "") + s = reStyle.ReplaceAllString(s, "") + s = rePBR.ReplaceAllString(s, "\n") + s = reTag.ReplaceAllString(s, "") + s = reSpace.ReplaceAllString(s, " ") + s = reLines.ReplaceAllString(s, "\n\n") + return strings.TrimSpace(s) +} diff --git a/netease163/article_test.go b/netease163/article_test.go new file mode 100644 index 0000000..647cae6 --- /dev/null +++ b/netease163/article_test.go @@ -0,0 +1,134 @@ +package netease163_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/tamnd/netease163-cli/netease163" +) + +func newArticleTestClient(ts *httptest.Server) *netease163.Client { + cfg := netease163.DefaultConfig() + cfg.ArticleBaseURL = ts.URL + cfg.Rate = 0 + return netease163.NewClient(cfg) +} + +func TestArticleParseResponse(t *testing.T) { + payload := map[string]any{ + "code": 200, + "data": map[string]any{ + "title": "测试文章", + "source": "测试来源", + "editor": "编辑B", + "body": "

正文内容


更多内容", + "keywords": []string{"kw1", "kw2"}, + "tags": []map[string]string{{"name": "tag1"}}, + "replyCount": 42, + "ptime": "1718286600", + "digest": "摘要", + }, + } + b, _ := json.Marshal(payload) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(b) + })) + defer srv.Close() + + c := newArticleTestClient(srv) + detail, err := c.Article(context.Background(), "TESTDOCID123456") + if err != nil { + t.Fatal(err) + } + if detail.DocID != "TESTDOCID123456" { + t.Errorf("docid = %q, want TESTDOCID123456", detail.DocID) + } + if detail.Title != "测试文章" { + t.Errorf("title = %q", detail.Title) + } + if detail.Source != "测试来源" { + t.Errorf("source = %q", detail.Source) + } + if detail.CommentCount != 42 { + t.Errorf("comment_count = %d, want 42", detail.CommentCount) + } + if len(detail.Keywords) != 2 { + t.Errorf("keywords len = %d, want 2", len(detail.Keywords)) + } + if len(detail.Tags) != 1 || detail.Tags[0] != "tag1" { + t.Errorf("tags = %v", detail.Tags) + } + if detail.Body == "" { + t.Error("body should not be empty after HTML strip") + } +} + +func TestArticleFromURL(t *testing.T) { + payload := map[string]any{ + "code": 200, + "data": map[string]any{ + "title": "URL test", + "source": "src", + "keywords": []string{}, + "tags": []map[string]string{}, + "replyCount": 0, + "ptime": "1718286600", + }, + } + b, _ := json.Marshal(payload) + + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _, _ = w.Write(b) + })) + defer srv.Close() + + c := newArticleTestClient(srv) + _, err := c.Article(context.Background(), "https://www.163.com/dy/article/URLTEST1234567.html") + if err != nil { + t.Fatal(err) + } + if gotPath != "/touch/reconstruct/article/info/URLTEST1234567.json" { + t.Errorf("unexpected path: %s", gotPath) + } +} + +func TestArticleNotFound(t *testing.T) { + payload := map[string]any{"code": 404, "data": map[string]any{}} + b, _ := json.Marshal(payload) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(b) + })) + defer srv.Close() + + c := newArticleTestClient(srv) + _, err := c.Article(context.Background(), "NOTFOUNDDOCID00") + if err == nil { + t.Fatal("expected error for code 404, got nil") + } +} + +func TestArticleSendsUserAgent(t *testing.T) { + payload := map[string]any{ + "code": 200, + "data": map[string]any{"title": "t", "source": "s", "ptime": "0", "keywords": []string{}, "tags": []map[string]string{}}, + } + b, _ := json.Marshal(payload) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("User-Agent") == "" { + t.Error("missing User-Agent on article request") + } + _, _ = w.Write(b) + })) + defer srv.Close() + + c := newArticleTestClient(srv) + _, _ = c.Article(context.Background(), "AGENTTEST123456") +} diff --git a/netease163/comments.go b/netease163/comments.go new file mode 100644 index 0000000..ff82489 --- /dev/null +++ b/netease163/comments.go @@ -0,0 +1,138 @@ +package netease163 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +// productID is the fixed NetEase News product identifier for the comment API. +const productID = "a2869674571f77b5a0867c3d71db5856" + +// Comments fetches the top (hot) comments for the article identified by docid or URL. +// limit <= 0 uses the API default (up to 7 hot comments). +func (c *Client) Comments(ctx context.Context, docidOrURL string, limit int) ([]Comment, error) { + docid := docidOrURL + if strings.Contains(docidOrURL, "/") { + docid = extractDocID(docidOrURL) + } + if docid == "" { + return nil, fmt.Errorf("cannot extract docid from %q", docidOrURL) + } + + url := c.cfg.CommentBaseURL + "/api/v1/products/" + productID + + "/threads/" + docid + "/hotComments" + + raw, err := c.getWithReferer(ctx, url, "https://news.163.com/") + if err != nil { + return nil, err + } + + var resp wireCommentsResp + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, fmt.Errorf("parse comments %s: %w", docid, err) + } + if resp.Code != 200 { + return nil, fmt.Errorf("comments %s: server returned code %d", docid, resp.Code) + } + + hot := resp.Data.HotComments + if limit > 0 && limit < len(hot) { + hot = hot[:limit] + } + + out := make([]Comment, 0, len(hot)) + for _, wc := range hot { + c := Comment{ + CommentID: wc.CommentID, + Content: wc.Content, + Likes: wc.Likes, + Replies: wc.Replys, + CreatedAt: unixMilliToRFC3339(wc.CreateTime), + Nickname: wc.UserInfo.Nickname, + TotalCount: resp.Data.TotalCount, + } + out = append(out, c) + } + return out, nil +} + +// wire types for comment API. +type wireCommentsResp struct { + Code int `json:"code"` + Data wireCommentsData `json:"data"` +} + +type wireCommentsData struct { + HotComments []wireComment `json:"hotComments"` + TotalCount int `json:"totalCount"` +} + +type wireComment struct { + CommentID string `json:"commentId"` + Content string `json:"content"` + Likes int `json:"likes"` + Replys int `json:"replys"` + CreateTime int64 `json:"createTime"` + UserInfo wireUserInfo `json:"userInfo"` +} + +type wireUserInfo struct { + Nickname string `json:"nickname"` + Avatar string `json:"avatar"` +} + +// getWithReferer fetches a URL with a Referer header, using retry/pacing. +func (c *Client) getWithReferer(ctx context.Context, url, referer string) ([]byte, error) { + var lastErr error + for attempt := 0; attempt <= c.cfg.Retries; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff(attempt)): + } + } + b, retry, err := c.doWithReferer(ctx, url, referer) + if err == nil { + return b, nil + } + lastErr = err + if !retry { + return nil, err + } + } + return nil, fmt.Errorf("get %s: %w", url, lastErr) +} + +func (c *Client) doWithReferer(ctx context.Context, url, referer string) ([]byte, bool, error) { + c.pace() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, false, err + } + req.Header.Set("User-Agent", c.cfg.UserAgent) + req.Header.Set("Referer", referer) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, true, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 { + return nil, true, fmt.Errorf("http %d", resp.StatusCode) + } + if resp.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("http %d", resp.StatusCode) + } + + b, err := readBody(resp) + if err != nil { + return nil, true, err + } + return b, false, nil +} diff --git a/netease163/comments_test.go b/netease163/comments_test.go new file mode 100644 index 0000000..dc9c776 --- /dev/null +++ b/netease163/comments_test.go @@ -0,0 +1,148 @@ +package netease163_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/tamnd/netease163-cli/netease163" +) + +func newCommentTestClient(ts *httptest.Server) *netease163.Client { + cfg := netease163.DefaultConfig() + cfg.CommentBaseURL = ts.URL + cfg.Rate = 0 + return netease163.NewClient(cfg) +} + +func TestCommentsParseResponse(t *testing.T) { + payload := map[string]any{ + "code": 200, + "data": map[string]any{ + "hotComments": []map[string]any{ + { + "commentId": "C001", + "content": "Great article!", + "likes": 100, + "replys": 5, + "createTime": int64(1718286700000), + "userInfo": map[string]string{"nickname": "User1", "avatar": ""}, + }, + { + "commentId": "C002", + "content": "Interesting read", + "likes": 50, + "replys": 2, + "createTime": int64(1718200000000), + "userInfo": map[string]string{"nickname": "User2", "avatar": ""}, + }, + }, + "totalCount": 500, + }, + } + b, _ := json.Marshal(payload) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(b) + })) + defer srv.Close() + + c := newCommentTestClient(srv) + comments, err := c.Comments(context.Background(), "TESTDOCID123456", 0) + if err != nil { + t.Fatal(err) + } + if len(comments) != 2 { + t.Fatalf("got %d comments, want 2", len(comments)) + } + if comments[0].CommentID != "C001" { + t.Errorf("comment_id = %q", comments[0].CommentID) + } + if comments[0].Content != "Great article!" { + t.Errorf("content = %q", comments[0].Content) + } + if comments[0].Likes != 100 { + t.Errorf("likes = %d", comments[0].Likes) + } + if comments[0].Nickname != "User1" { + t.Errorf("nickname = %q", comments[0].Nickname) + } + if comments[0].TotalCount != 500 { + t.Errorf("total_count = %d, want 500", comments[0].TotalCount) + } +} + +func TestCommentsFromURL(t *testing.T) { + payload := map[string]any{ + "code": 200, + "data": map[string]any{"hotComments": []map[string]any{}, "totalCount": 0}, + } + b, _ := json.Marshal(payload) + + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _, _ = w.Write(b) + })) + defer srv.Close() + + c := newCommentTestClient(srv) + _, err := c.Comments(context.Background(), "https://www.163.com/dy/article/URLTEST1234567.html", 0) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(gotPath, "URLTEST1234567") { + t.Errorf("path %q does not contain URLTEST1234567", gotPath) + } +} + +func TestCommentsLimit(t *testing.T) { + payload := map[string]any{ + "code": 200, + "data": map[string]any{ + "hotComments": []map[string]any{ + {"commentId": "C1", "content": "c1", "likes": 1, "replys": 0, "createTime": int64(0), "userInfo": map[string]string{"nickname": "u1"}}, + {"commentId": "C2", "content": "c2", "likes": 2, "replys": 0, "createTime": int64(0), "userInfo": map[string]string{"nickname": "u2"}}, + {"commentId": "C3", "content": "c3", "likes": 3, "replys": 0, "createTime": int64(0), "userInfo": map[string]string{"nickname": "u3"}}, + }, + "totalCount": 3, + }, + } + b, _ := json.Marshal(payload) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(b) + })) + defer srv.Close() + + c := newCommentTestClient(srv) + comments, err := c.Comments(context.Background(), "TESTLIMIT1234567", 2) + if err != nil { + t.Fatal(err) + } + if len(comments) != 2 { + t.Fatalf("got %d comments with limit=2, want 2", len(comments)) + } +} + +func TestCommentsSendsReferer(t *testing.T) { + payload := map[string]any{ + "code": 200, + "data": map[string]any{"hotComments": []map[string]any{}, "totalCount": 0}, + } + b, _ := json.Marshal(payload) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Referer") == "" { + t.Error("missing Referer header on comments request") + } + _, _ = w.Write(b) + })) + defer srv.Close() + + c := newCommentTestClient(srv) + _, _ = c.Comments(context.Background(), "REFERERTEST12345", 0) +} diff --git a/netease163/hot.go b/netease163/hot.go new file mode 100644 index 0000000..a463407 --- /dev/null +++ b/netease163/hot.go @@ -0,0 +1,47 @@ +package netease163 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" +) + +// Hot fetches today's hot/trending articles from NetEase News. +// limit <= 0 means all items. +func (c *Client) Hot(ctx context.Context, limit int) ([]Article, error) { + url := c.cfg.BaseURL + "/special/0001386F/news_hot.js" + raw, err := c.get(ctx, url) + if err != nil { + return nil, err + } + + items, err := parseVarData(raw) + if err != nil { + return nil, fmt.Errorf("parse hot feed: %w", err) + } + + if limit > 0 && limit < len(items) { + items = items[:limit] + } + + out := make([]Article, 0, len(items)) + for i, item := range items { + out = append(out, wireToArticle(item, i+1)) + } + return out, nil +} + +// parseVarData strips the "var data=...;" JS variable wrapper and parses the JSON array. +func parseVarData(body []byte) ([]wireItem, error) { + body = bytes.TrimSpace(body) + body = bytes.TrimPrefix(body, []byte("var data=")) + body = bytes.TrimSuffix(body, []byte(";")) + body = bytes.TrimSpace(body) + + var items []wireItem + if err := json.Unmarshal(body, &items); err != nil { + return nil, fmt.Errorf("json unmarshal: %w", err) + } + return items, nil +} diff --git a/netease163/hot_test.go b/netease163/hot_test.go new file mode 100644 index 0000000..7b692f8 --- /dev/null +++ b/netease163/hot_test.go @@ -0,0 +1,72 @@ +package netease163_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +const mockHotJS = `var data=[ + {"title":"热点文章一","docurl":"https://www.163.com/dy/article/HOTARTICLE0001.html","source":"新华社","time":"1718286600","digest":"hot digest 1","channelname":"要闻"}, + {"title":"热点文章二","docurl":"https://www.163.com/dy/article/HOTARTICLE0002.html","source":"人民日报","time":"1718200000","digest":"hot digest 2","channelname":"要闻"} +];` + +func TestHotParseArticles(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(mockHotJS)) + })) + defer srv.Close() + + c := newTestClient(srv) + articles, err := c.Hot(context.Background(), 0) + if err != nil { + t.Fatal(err) + } + if len(articles) != 2 { + t.Fatalf("got %d articles, want 2", len(articles)) + } + if articles[0].Title != "热点文章一" { + t.Errorf("title = %q", articles[0].Title) + } + if articles[0].Rank != 1 { + t.Errorf("rank = %d, want 1", articles[0].Rank) + } + if articles[1].Rank != 2 { + t.Errorf("second rank = %d, want 2", articles[1].Rank) + } +} + +func TestHotLimit(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(mockHotJS)) + })) + defer srv.Close() + + c := newTestClient(srv) + articles, err := c.Hot(context.Background(), 1) + if err != nil { + t.Fatal(err) + } + if len(articles) != 1 { + t.Fatalf("got %d articles with limit=1, want 1", len(articles)) + } +} + +func TestHotURLPath(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _, _ = w.Write([]byte(mockHotJS)) + })) + defer srv.Close() + + c := newTestClient(srv) + _, err := c.Hot(context.Background(), 0) + if err != nil { + t.Fatal(err) + } + if gotPath != "/special/0001386F/news_hot.js" { + t.Errorf("unexpected path: %s", gotPath) + } +} diff --git a/netease163/netease163.go b/netease163/netease163.go index f2b2126..b256c65 100644 --- a/netease163/netease163.go +++ b/netease163/netease163.go @@ -24,21 +24,27 @@ const DefaultUserAgent = "netease163/dev (+https://github.com/tamnd/netease163-c // Config holds constructor parameters for the client. type Config struct { - BaseURL string - UserAgent string - Rate time.Duration - Timeout time.Duration - Retries int + BaseURL string + ArticleBaseURL string // base for the 3g.163.com article detail API + CommentBaseURL string // base for comment.api.163.com + SearchBaseURL string // base for so.163.com + UserAgent string + Rate time.Duration + Timeout time.Duration + Retries int } // DefaultConfig returns sensible defaults. func DefaultConfig() Config { return Config{ - BaseURL: "https://news.163.com", - UserAgent: DefaultUserAgent, - Rate: 500 * time.Millisecond, - Timeout: 30 * time.Second, - Retries: 3, + BaseURL: "https://news.163.com", + ArticleBaseURL: "https://3g.163.com", + CommentBaseURL: "https://comment.api.163.com", + SearchBaseURL: "https://so.163.com", + UserAgent: DefaultUserAgent, + Rate: 500 * time.Millisecond, + Timeout: 30 * time.Second, + Retries: 3, } } @@ -154,13 +160,18 @@ func (c *Client) do(ctx context.Context, url string) ([]byte, bool, error) { return nil, false, fmt.Errorf("http %d", resp.StatusCode) } - b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + b, err := readBody(resp) if err != nil { return nil, true, err } return b, false, nil } +// readBody reads up to 8 MB from the response body. +func readBody(resp *http.Response) ([]byte, error) { + return io.ReadAll(io.LimitReader(resp.Body, 8<<20)) +} + func (c *Client) pace() { c.mu.Lock() defer c.mu.Unlock() diff --git a/netease163/netease163_test.go b/netease163/netease163_test.go index fdc6b2a..01b11fb 100644 --- a/netease163/netease163_test.go +++ b/netease163/netease163_test.go @@ -160,8 +160,8 @@ func TestChannelsList(t *testing.T) { cfg := netease163.DefaultConfig() c := netease163.NewClient(cfg) channels := c.Channels() - if len(channels) != 6 { - t.Fatalf("got %d channels, want 6", len(channels)) + if len(channels) != 12 { + t.Fatalf("got %d channels, want 12", len(channels)) } found := false for _, ch := range channels { diff --git a/netease163/search.go b/netease163/search.go new file mode 100644 index 0000000..7b573c1 --- /dev/null +++ b/netease163/search.go @@ -0,0 +1,87 @@ +package netease163 + +import ( + "context" + "fmt" + "net/url" + "regexp" + "strings" +) + +// Search queries NetEase News via the so.163.com search page and parses the HTML results. +// limit <= 0 returns up to 20 results (one page). +func (c *Client) Search(ctx context.Context, query string, limit int) ([]SearchResult, error) { + if strings.TrimSpace(query) == "" { + return nil, fmt.Errorf("search query must not be empty") + } + + searchURL := c.cfg.SearchBaseURL + "/search?keyword=" + url.QueryEscape(query) + + "&type=news&start=0&limit=20" + + raw, err := c.get(ctx, searchURL) + if err != nil { + return nil, fmt.Errorf("search %q: %w", query, err) + } + + results := parseSearchHTML(string(raw)) + if limit > 0 && limit < len(results) { + results = results[:limit] + } + return results, nil +} + +// HTML selectors from spec §4.7: +// +//
    +//
  • +//

    Title

    +//

    SourceDate

    +//

    Digest

    +//
    • +//
+var ( + reSearchItem = regexp.MustCompile(`(?is)]+class="[^"]*news-item[^"]*"[^>]*>(.*?)`) + reSearchTitle = regexp.MustCompile(`(?is)]+class="[^"]*news-title[^"]*"[^>]*>.*?]+href="([^"]*)"[^>]*>(.*?)`) + reSearchSource = regexp.MustCompile(`(?is)]+class="[^"]*source[^"]*"[^>]*>(.*?)`) + reSearchTime = regexp.MustCompile(`(?is)]+class="[^"]*time[^"]*"[^>]*>(.*?)`) + reSearchAbstract = regexp.MustCompile(`(?is)]+class="[^"]*news-abstract[^"]*"[^>]*>(.*?)

`) + reHTMLTag = regexp.MustCompile(`<[^>]+>`) +) + +func parseSearchHTML(body string) []SearchResult { + items := reSearchItem.FindAllStringSubmatch(body, -1) + out := make([]SearchResult, 0, len(items)) + for i, m := range items { + chunk := m[1] + + var title, href, source, date, digest string + + if tm := reSearchTitle.FindStringSubmatch(chunk); tm != nil { + href = strings.TrimSpace(tm[1]) + title = strings.TrimSpace(reHTMLTag.ReplaceAllString(tm[2], "")) + } + if sm := reSearchSource.FindStringSubmatch(chunk); sm != nil { + source = strings.TrimSpace(reHTMLTag.ReplaceAllString(sm[1], "")) + } + if tm := reSearchTime.FindStringSubmatch(chunk); tm != nil { + date = strings.TrimSpace(reHTMLTag.ReplaceAllString(tm[1], "")) + } + if am := reSearchAbstract.FindStringSubmatch(chunk); am != nil { + digest = strings.TrimSpace(reHTMLTag.ReplaceAllString(am[1], "")) + } + + if title == "" && href == "" { + continue + } + + out = append(out, SearchResult{ + Rank: i + 1, + Title: title, + Source: source, + Date: date, + Digest: digest, + URL: href, + }) + } + return out +} diff --git a/netease163/search_test.go b/netease163/search_test.go new file mode 100644 index 0000000..43d8db9 --- /dev/null +++ b/netease163/search_test.go @@ -0,0 +1,136 @@ +package netease163_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/tamnd/netease163-cli/netease163" +) + +const mockSearchHTML = ` + + + + +` + +func newSearchTestClient(ts *httptest.Server) *netease163.Client { + cfg := netease163.DefaultConfig() + cfg.SearchBaseURL = ts.URL + cfg.Rate = 0 + return netease163.NewClient(cfg) +} + +func TestSearchParseResults(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(mockSearchHTML)) + })) + defer srv.Close() + + c := newSearchTestClient(srv) + results, err := c.Search(context.Background(), "测试", 0) + if err != nil { + t.Fatal(err) + } + if len(results) != 2 { + t.Fatalf("got %d results, want 2", len(results)) + } + if results[0].Title != "搜索结果标题一" { + t.Errorf("title = %q", results[0].Title) + } + if results[0].Source != "新华社" { + t.Errorf("source = %q", results[0].Source) + } + if results[0].Date != "2024-06-13" { + t.Errorf("date = %q", results[0].Date) + } + if results[0].Rank != 1 { + t.Errorf("rank = %d, want 1", results[0].Rank) + } + if results[1].Rank != 2 { + t.Errorf("second rank = %d, want 2", results[1].Rank) + } +} + +func TestSearchLimit(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(mockSearchHTML)) + })) + defer srv.Close() + + c := newSearchTestClient(srv) + results, err := c.Search(context.Background(), "测试", 1) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 { + t.Fatalf("got %d results with limit=1, want 1", len(results)) + } +} + +func TestSearchURLConstruction(t *testing.T) { + var gotQuery string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + _, _ = w.Write([]byte(`
    `)) + })) + defer srv.Close() + + c := newSearchTestClient(srv) + _, _ = c.Search(context.Background(), "人工智能", 0) + if !strings.Contains(gotQuery, "keyword=") { + t.Errorf("query %q missing keyword param", gotQuery) + } + if !strings.Contains(gotQuery, "type=news") { + t.Errorf("query %q missing type=news param", gotQuery) + } +} + +func TestSearchEmptyQuery(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("")) + })) + defer srv.Close() + + c := newSearchTestClient(srv) + _, err := c.Search(context.Background(), "", 0) + if err == nil { + t.Fatal("expected error for empty query, got nil") + } +} + +func TestSearchEmptyResults(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`
      `)) + })) + defer srv.Close() + + c := newSearchTestClient(srv) + results, err := c.Search(context.Background(), "no results query", 0) + if err != nil { + t.Fatal(err) + } + if len(results) != 0 { + t.Errorf("got %d results, want 0", len(results)) + } +} diff --git a/netease163/types.go b/netease163/types.go index d8e34f5..09b4433 100644 --- a/netease163/types.go +++ b/netease163/types.go @@ -2,6 +2,7 @@ package netease163 import ( "strconv" + "strings" "time" ) @@ -13,6 +14,12 @@ var channelPaths = map[string]string{ "sports": "cm_sports", "entertainment": "cm_fun", "tech": "cm_tec", + "finance": "cm_money", + "auto": "cm_auto", + "home": "cm_house", + "world": "cm_guoji", + "edu": "cm_edu", + "game": "cm_game", } // wireItem is the JSON shape returned inside the JSONP array. @@ -58,3 +65,61 @@ func wireToArticle(item wireItem, rank int) Article { URL: item.DocURL, } } + +// ArticleDetail is a fully fetched article with body content. +type ArticleDetail struct { + DocID string `json:"docid"` + Title string `json:"title"` + URL string `json:"url"` + Source string `json:"source"` + Editor string `json:"editor"` + PublishedAt string `json:"published_at"` + Digest string `json:"digest"` + Body string `json:"body"` + Keywords []string `json:"keywords"` + Tags []string `json:"tags"` + CommentCount int `json:"comment_count"` +} + +// Comment is a single reader comment on an article. +type Comment struct { + CommentID string `json:"comment_id"` + Content string `json:"content"` + Likes int `json:"likes"` + Replies int `json:"replies"` + CreatedAt string `json:"created_at"` + Nickname string `json:"nickname"` + TotalCount int `json:"total_count,omitempty"` +} + +// SearchResult is one hit from a NetEase search query. +type SearchResult struct { + Rank int `json:"rank"` + Title string `json:"title"` + Source string `json:"source"` + Date string `json:"date"` + Digest string `json:"digest"` + URL string `json:"url"` +} + +// extractDocID derives the 16-char article docid from a 163.com article URL. +// e.g. "https://www.163.com/dy/article/KVACHDNB0514D3UH.html" -> "KVACHDNB0514D3UH" +func extractDocID(docurl string) string { + parts := strings.Split(docurl, "/") + last := parts[len(parts)-1] + return strings.TrimSuffix(last, ".html") +} + +// unixToRFC3339 converts a Unix-seconds string to RFC3339 UTC. +func unixToRFC3339(s string) string { + ts, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return s + } + return time.Unix(ts, 0).UTC().Format(time.RFC3339) +} + +// unixMilliToRFC3339 converts a Unix-milliseconds int64 to RFC3339 UTC. +func unixMilliToRFC3339(ms int64) string { + return time.UnixMilli(ms).UTC().Format(time.RFC3339) +} From 7d2e76ecf90d62f3801acab8f1f3ddb4b8c01135 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Sat, 20 Jun 2026 13:43:13 +0700 Subject: [PATCH 2/2] chore: upgrade GitHub Actions to latest versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node.js 20 is being deprecated in the Actions runtime. actions/checkout → v7.0.0 browser-actions/setup-chrome → v2.1.2 golangci/golangci-lint-action → v9.2.1 goreleaser/goreleaser-action → v7.2.2 docker/setup-qemu-action → v4.1.0 docker/setup-buildx-action → v4.1.0 docker/login-action → v4.2.0 sigstore/cosign-installer → v4.1.2 anchore/sbom-action → v0.24.0 --- .github/workflows/docs.yml | 6 +++--- .github/workflows/release.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bc50b10..b7a03d9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,14 +26,14 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: submodules: true # Sitemap lastmod comes from the latest content commit. fetch-depth: 0 - name: Checkout tago - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v7.0.0 with: repository: tamnd/tago path: .tago-src @@ -107,7 +107,7 @@ jobs: group: cloudflare-pages-netease163-cli cancel-in-progress: true steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 1 sparse-checkout: scripts/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2e424c7..fcc24af 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,7 +68,7 @@ jobs: # Tools GoReleaser shells out to for signing and SBOMs. - uses: sigstore/cosign-installer@v3 - - uses: anchore/sbom-action/download-syft@v0 + - uses: anchore/sbom-action/download-syft@v0.24.0 - uses: goreleaser/goreleaser-action@v6 with: