【instagram】爬虫照片墙PC端数据采集

﹏ヽ暗。殇╰゛Y 2024-04-17 11:27 140阅读 0赞

喜欢一朵花，未必要将它摘下

在这里插入图片描述

采集案例

  1. package main
  3. import (
  4.     "context"
  5.     "encoding/json"
  6.     "fmt"
  7.     "go.mongodb.org/mongo-driver/mongo"
  8.     "go.mongodb.org/mongo-driver/mongo/options"
  9.     "io"
  10.     "log"
  11.     "net/http"
  12.     "net/url"
  13.     "strings"
  14.     "sync"
  15.     "time"
  16. )
  18. const (
  19.     proxyURL = "http://127.0.0.1:2333"
  20.     mongoURL = "mongodb://233.233.233.233:2333"
  21. )
  23. type blogInfo struct {
  24.     UserID       string    `json:"userId" bson:"userId"`
  25.     Title        string    `json:"title" bson:"title"`
  26.     CommentCount float64   `json:"commentCount" bson:"commentCount"`
  27.     LikeCount    float64   `json:"likeCount" bson:"likeCount"`
  28.     Shortcode    string    `json:"shortcode" bson:"shortcode"`
  29.     PublishTime  time.Time `json:"publishTime" bson:"publishTime"`
  30.     CreateTime   time.Time `json:"createTime" bson:"createTime"`
  31. }
  33. type UserProfile struct {
  34.     UserID     string    `json:"userId" bson:"userId"`
  35.     UserName   string    `json:"userName" bson:"userName"`
  36.     Followers  float64   `json:"followers" bson:"followers"`
  37.     Biography  string    `json:"biography" bson:"biography"`
  38.     Follow     float64   `json:"follow" bson:"follow"`
  39.     Blog       float64   `json:"blog" bson:"blog"`
  40.     CreateTime time.Time `json:"createTime" bson:"createTime"`
  41. }
  43. func crawlBlogInfo(userId string, endCursor string, mgClient *mongo.Client) (bool, string, error) {
  44.     proxyURL, err := url.Parse(proxyURL)
  45.     if err != nil {
  46.         fmt.Println("Error parsing proxy URL:", err)
  47.         return false, "", err
  48.     }
  49.     transport := &http.Transport{
  50.         Proxy: http.ProxyURL(proxyURL),
  51.     }
  52.     client := &http.Client{
  53.         Transport: transport,
  54.     }
  55.     params := `%7B"id"%3A"` + userId + `"%2C"after"%3A"` + endCursor + `"%2C"first"%3A12%7D`
  56.     urlStr := fmt.Sprintf("https://www.instagram.com/graphql/query/?doc_id=%s&variables=%s", "自定义doc_id", params)
  57.     req, err := http.NewRequest("GET", urlStr, nil)
  58.     if err != nil {
  59.         fmt.Println("Error creating request:", err)
  60.         return false, "", err
  61.     }
  62.     req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
  63.     req.Header.Set("x-ig-app-id", "app_id")
  64.     resp, err := client.Do(req)
  65.     if err != nil {
  66.         fmt.Println("Error fetching the page:", err)
  67.         return false, "", err
  68.     }
  69.     defer resp.Body.Close()
  70.     body, err := io.ReadAll(resp.Body)
  72.     if err != nil {
  73.         fmt.Println("Error reading response body:", err)
  74.         return false, "", err
  75.     }
  76.     var jsonData map[string]interface{}
  77.     err = json.Unmarshal(body, &jsonData)
  78.     if err != nil {
  79.         fmt.Println("Error parsing JSON:", err)
  80.         return false, "", err
  81.     }
  82.     var rootPath []interface{}
  83.     if data, ok := jsonData["data"].(map[string]interface{}); ok {
  84.         if user, ok := data["user"].(map[string]interface{}); ok {
  85.             if edgeOwnerToTimelineMedia, ok := user["edge_owner_to_timeline_media"].(map[string]interface{}); ok {
  86.                 if edges, ok := edgeOwnerToTimelineMedia["edges"].([]interface{}); ok {
  87.                     rootPath = edges
  88.                 }
  89.             }
  90.         }
  91.     }
  92.     if rootPath == nil {
  93.         fmt.Println("Unexpected JSON structure")
  94.         return false, "", nil
  95.     }
  96.     for _, item := range rootPath {
  97.         titleNode := item.(map[string]interface{})["node"].(map[string]interface{})["edge_media_to_caption"].(map[string]interface{})["edges"].([]interface{})
  98.         var title string
  99.         if len(titleNode) == 0 {
  100.             log.Println("title : No data available in edges")
  101.         } else {
  102.             title = item.(map[string]interface{})["node"].(map[string]interface{})["edge_media_to_caption"].(map[string]interface{})["edges"].([]interface{})[0].(map[string]interface{})["node"].(map[string]interface{})["text"].(string)
  103.         }
  104.         if title != "" {
  105.             commentCount := item.(map[string]interface{})["node"].(map[string]interface{})["edge_media_to_comment"].(map[string]interface{})["count"].(float64)
  106.             likeCount := item.(map[string]interface{})["node"].(map[string]interface{})["edge_media_preview_like"].(map[string]interface{})["count"].(float64)
  107.             shortcode := item.(map[string]interface{})["node"].(map[string]interface{})["shortcode"].(string)
  108.             tm := item.(map[string]interface{})["node"].(map[string]interface{})["taken_at_timestamp"].(float64)
  109.             timestamp := time.Unix(int64(tm), 0)
  110.             insBlogInfo := mgClient.Database("crawlers").Collection("ins_blog_info_v1")
  111.             insBlogItem := blogInfo{
  112.                 UserID:       userId,
  113.                 Title:        title,
  114.                 CommentCount: commentCount,
  115.                 LikeCount:    likeCount,
  116.                 Shortcode:    shortcode,
  117.                 PublishTime:  timestamp,
  118.                 CreateTime:   time.Now(),
  119.             }
  120.             insertOneRes, insertErr := insBlogInfo.InsertOne(context.TODO(), insBlogItem)
  121.             if insertErr != nil {
  122.                 insertFlag := strings.Index(insertErr.Error(), "[E11000 duplicate key error collection")
  123.                 if insertFlag != -1 {
  124.                     fmt.Println("DATA REPEAT!!!")
  125.                     continue
  126.                 } else {
  127.                     fmt.Println("CODE ERROR!!!")
  128.                     fmt.Println(insertErr.Error())
  129.                     continue
  130.                 }
  131.             } else {
  132.                 fmt.Printf("写入成功:%v\n", insertOneRes.InsertedID)
  133.             }
  134.         } else {
  135.             fmt.Println("title is empty")
  136.             continue
  137.         }
  138.     }
  139.     hasNextPage := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_owner_to_timeline_media"].(map[string]interface{})["page_info"].(map[string]interface{})["has_next_page"].(bool)
  140.     endCursorNext := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_owner_to_timeline_media"].(map[string]interface{})["page_info"].(map[string]interface{})["end_cursor"].(string)
  141.     return hasNextPage, endCursorNext, nil
  142. }
  144. func crawlUserProfile(wg *sync.WaitGroup, urlStr string, mgClient *mongo.Client) {
  145.     defer wg.Done()
  146.     proxyURL, err := url.Parse(proxyURL)
  147.     if err != nil {
  148.         fmt.Println("Error parsing proxy URL:", err)
  149.         return
  150.     }
  151.     transport := &http.Transport{
  152.         Proxy: http.ProxyURL(proxyURL),
  153.     }
  154.     client := &http.Client{
  155.         Transport: transport,
  156.     }
  157.     req, err := http.NewRequest("GET", urlStr, nil)
  158.     if err != nil {
  159.         fmt.Println("Error creating request:", err)
  160.         return
  161.     }
  162.     req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
  163.     req.Header.Set("x-ig-app-id", "自定义app_id")
  164.     resp, err := client.Do(req)
  165.     if err != nil {
  166.         fmt.Println("Error fetching the page:", err)
  167.         return
  168.     }
  169.     defer resp.Body.Close()
  170.     body, err := io.ReadAll(resp.Body)
  171.     if err != nil {
  172.         fmt.Println("Error reading response body:", err)
  173.         return
  174.     }
  176.     var jsonData map[string]interface{}
  177.     err = json.Unmarshal(body, &jsonData)
  178.     if err != nil {
  179.         fmt.Println("Error parsing JSON:", err)
  180.         return
  181.     }
  183.     var biography string = jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["biography"].(string)
  184.     followers := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_followed_by"].(map[string]interface{})["count"].(float64)
  185.     fllow := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_follow"].(map[string]interface{})["count"].(float64)
  186.     blog := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_owner_to_timeline_media"].(map[string]interface{})["count"].(float64)
  187.     userName := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_owner_to_timeline_media"].(map[string]interface{})["edges"].([]interface{})[0].(map[string]interface{})["node"].(map[string]interface{})["owner"].(map[string]interface{})["username"].(string)
  188.     userId := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_owner_to_timeline_media"].(map[string]interface{})["edges"].([]interface{})[0].(map[string]interface{})["node"].(map[string]interface{})["owner"].(map[string]interface{})["id"].(string)
  189.     insUserProfile := mgClient.Database("crawlers").Collection("ins_user_profile")
  190.     insUserItem := UserProfile{
  191.         UserID:     userId,
  192.         UserName:   userName,
  193.         Followers:  followers,
  194.         Biography:  biography,
  195.         Follow:     fllow,
  196.         Blog:       blog,
  197.         CreateTime: time.Now(),
  198.     }
  199.     fmt.Println(insUserItem)
  200.     insertOneRes, insertErr := insUserProfile.InsertOne(context.TODO(), insUserItem)
  201.     if insertErr != nil {
  202.         insertFlag := strings.Index(insertErr.Error(), "[E11000 duplicate key error collection")
  203.         if insertFlag != -1 {
  204.             fmt.Println("DATA REPEAT!!!")
  205.         } else {
  206.             fmt.Println("CODE ERROR!!!")
  207.             fmt.Println(insertErr.Error())
  208.         }
  209.     } else {
  210.         fmt.Printf("写入成功:%v\n", insertOneRes.InsertedID)
  211.     }
  213.     rootPath := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_owner_to_timeline_media"].(map[string]interface{})["edges"].([]interface{})
  214.     for _, item := range rootPath {
  215.         titleNode := item.(map[string]interface{})["node"].(map[string]interface{})["edge_media_to_caption"].(map[string]interface{})["edges"].([]interface{})
  216.         var title string
  217.         if len(titleNode) == 0 {
  218.             log.Println("title : No data available in edges")
  219.         } else {
  220.             title = item.(map[string]interface{})["node"].(map[string]interface{})["edge_media_to_caption"].(map[string]interface{})["edges"].([]interface{})[0].(map[string]interface{})["node"].(map[string]interface{})["text"].(string)
  221.         }
  223.         commentCount := item.(map[string]interface{})["node"].(map[string]interface{})["edge_media_to_comment"].(map[string]interface{})["count"].(float64)
  224.         likeCount := item.(map[string]interface{})["node"].(map[string]interface{})["edge_media_preview_like"].(map[string]interface{})["count"].(float64)
  225.         shortcode := item.(map[string]interface{})["node"].(map[string]interface{})["shortcode"].(string)
  226.         tm := item.(map[string]interface{})["node"].(map[string]interface{})["taken_at_timestamp"].(float64)
  227.         timestamp := time.Unix(int64(tm), 0)
  228.         insBlogInfo := mgClient.Database("crawlers").Collection("ins_blog_info")
  229.         insBlogItem := blogInfo{
  230.             UserID:       userId,
  231.             Title:        title,
  232.             CommentCount: commentCount,
  233.             LikeCount:    likeCount,
  234.             Shortcode:    shortcode,
  235.             PublishTime:  timestamp,
  236.             CreateTime:   time.Now(),
  237.         }
  238.         insertOneRes, insertErr := insBlogInfo.InsertOne(context.TODO(), insBlogItem)
  239.         if insertErr != nil {
  240.             insertFlag := strings.Index(insertErr.Error(), "[E11000 duplicate key error collection")
  241.             if insertFlag != -1 {
  242.                 fmt.Println("DATA REPEAT!!!")
  243.             } else {
  244.                 fmt.Println("CODE ERROR!!!")
  245.                 fmt.Println(insertErr.Error())
  246.             }
  247.         } else {
  248.             fmt.Printf("写入成功:%v\n", insertOneRes.InsertedID)
  249.         }
  251.     }
  252.     hasNextPage := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_owner_to_timeline_media"].(map[string]interface{})["page_info"].(map[string]interface{})["has_next_page"].(bool)
  253.     endCursor := jsonData["data"].(map[string]interface{})["user"].(map[string]interface{})["edge_owner_to_timeline_media"].(map[string]interface{})["page_info"].(map[string]interface{})["end_cursor"].(string)
  255.     for hasNextPage {
  256.         hasNextPage, endCursor, err = crawlBlogInfo(userId, endCursor, mgClient)
  257.         if err != nil {
  258.             fmt.Println("Error fetching the page:", err)
  260.         } else {
  261.             fmt.Println("NextPage:", endCursor)
  262.         }
  263.     }
  265. }
  267. func main() {
  268.     clientOptions := options.Client().ApplyURI(mongoURL)
  269.     mgClient, cllectErr := mongo.Connect(context.TODO(), clientOptions)
  270.     if cllectErr != nil {
  271.         log.Fatal(cllectErr)
  272.     }
  273.     pingErr := mgClient.Ping(context.TODO(), nil)
  274.     if pingErr != nil {
  275.         log.Fatal(pingErr)
  276.     }
  277.     urls := []string{
  278.         "https://www.instagram.com/api/v1/users/web_profile_info/?username=visit_nanchang",
  279.     }
  280.     var wg sync.WaitGroup
  282.     for _, url := range urls {
  283.         wg.Add(1)
  284.         go crawlUserProfile(&wg, url, mgClient)
  285.     }
  287.     wg.Wait() // 等待所有goroutine完成
  288.     fmt.Println("所有请求完成")
  289. }

发表评论

表情：
评论列表 （有 0 条评论，140人围观）

还没有评论，来说两句吧...

相关阅读

    相关 CSS照片墙

    开发工具与关键技术：DW 作者：文泽钦 撰写时间：2019年1月31日 今天跟大家分享一个css照片墙，代码也很简单，能省略的效果也省略了，大家也不要学我，不然学校也

    Myth丶恋晨Myth丶恋晨/ 0 赞/ 445 阅读

    相关 照片墙

    开发工具与关键技术：Adobe Dreamweaver CC 2017 CSS 作者：廖亚星 撰写时间：2019年2月15日 下面是让多张照片错乱分布在页面上，当移

    痛定思痛。痛定思痛。/ 0 赞/ 468 阅读