better

Query.js

@@ -1,8 +1,7 @@
-// 这里是一些常用的查询语句
+// 这里是分析数据时用的的查询语句
 
 
 // 房价均价查询语句
-
 db.lianjia.aggregate([
  {'$match': {"address.0": {$exists: true}}},
  {
@@ -29,7 +28,6 @@ db.lianjia.aggregate([
 
 
 // 平均薪资查询语句
-
 db.zhilian.aggregate([
  {'$match': {"workingExp.name": "1-3年"}},
  {
@@ -60,4 +58,30 @@ db.lianjia.aggregate([
  {
  '$sort': {detailCrawlTime: -1}
  }
-], {allowDiskUse: true});
+], {allowDiskUse: true});
+
+
+// 租房数据
+db.lianjia_zufang.aggregate([
+ {
+ $group: {
+ _id: "$city",
+ count: {$sum: 1},
+ avg: {$avg: "$price"},
+ std: {$stdDevPop: "$price"},
+ unitPrice: {$avg: {$divide: ["$price", "$mianji"]}}
+ }
+ },
+ {
+ $project: {
+ unitPrice: 1, // 单位价格
+ count: 1, //总数
+ avg: 1, //每平米均价
+ std: 1, //标准差
+ ratio: {$divide: ["$std", "$avg"]} //标准差与均价的比值
+ }
+ },
+ {
+ '$sort': {count: -1}
+ }
+]);

README.md

@@ -65,13 +65,40 @@ option支持：lianjia_ershou、zhilian、lianjia_zufang
 方便定时脚本记录抓取情况，使用info命令可以输出当前抓取数据量到文件
 
 ```
-getAwayBSG -info
+getAwayBSG -info -info_save_to=./numLog.txt
 ```
 
+使用-info_save_to参数指定文件保存位置，默认为当前目录的numLog.txt文件中
+
 3.help
 
 输出支持的全部命令列表
 
 ```
 getAwayBSG -help
-```
+```
+
+
+## 数据分析
+
+分析用的MongoDB语句在[Query.js](./Query.js)文件中，使用MongoDB执行即可
+
+## 编译
+
+编译使用xgo，需要先安装docker
+
+```
+git clone https://github.com/jiangwei1995910/getAwayBSG
+
+docker pull karalabe/xgo-latest
+
+go get github.com/karalabe/xgo
+
+cd getAwayBSG
+
+sh ./build.sh
+```
+
+## 部署
+
+如果需要分布式或者多进程抓取，在不同机器或者多个进程中指定相同的MongoDB源即可，程序已经支持分布式多进程抓取了。已抓取的链接和状态会通过MongoDB共享

config-all.yaml

@@ -1515,10 +1515,6 @@ zlCityList:
  url: https://www.zhaopin.com/maoming/
  code: 771
  pinyin: maoming
- - name: 蒙自市
- url: https://www.zhaopin.com/mengzishi/
- code:
- pinyin: mengzishi
  - name: 满洲里
  url: https://www.zhaopin.com/manzhouli/
  code: 10157

config.yaml

@@ -1515,10 +1515,6 @@ zlCityList:
 # url: https://www.zhaopin.com/maoming/
 # code: 771
 # pinyin: maoming
-# - name: 蒙自市
-# url: https://www.zhaopin.com/mengzishi/
-# code:
-# pinyin: mengzishi
 # - name: 满洲里
 # url: https://www.zhaopin.com/manzhouli/
 # code: 10157

entrance/info.go

@@ -10,14 +10,14 @@ import (
  "time"
 )
 
-func Start_info() {
+func Start_info(path string) {
 
- fd, _ := os.OpenFile("./numLog.txt", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
+ fd, _ := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
  fd_time := time.Now().Format("2006-01-02 15:04:05")
  fd_content := strings.Join([]string{
- fd_time, ":\n",
- getLianjiaErShouFangStatus(), "\n",
- getLianJiaZuFangStatus(), "\n",
+ fd_time, ":",
+ getLianjiaErShouFangStatus(), " ",
+ getLianJiaZuFangStatus(), " ",
  getZhiLianStatus(), "\n",
  }, "")
  buf := []byte(fd_content)

entrance/lianjia.go

@@ -156,6 +156,7 @@ func crawlDetail() (sucnum int) {
  c := colly.NewCollector()
  configInfo := configs.Config()
 
+ //设置延时
  if configInfo["crawlDelay"] != nil {
  delay, _ := configInfo["crawlDelay"].(json.Number).Int64()
  if delay > 0 {
@@ -166,6 +167,7 @@ func crawlDetail() (sucnum int) {
  }
  }
 
+ //设置代理
  if configInfo["proxyList"] != nil && len(configInfo["proxyList"].([]interface{})) > 0 {
  var proxyList []string
  for _, v := range configInfo["proxyList"].([]interface{}) {
@@ -181,8 +183,11 @@ func crawlDetail() (sucnum int) {
  }
  }
 
+ //随机UA
  extensions.RandomUserAgent(c)
+ //自动referer
  extensions.Referer(c)
+ //设置MongoDB存储状态信息
  storage := &cachemongo.Storage{
  Database: "colly",
  URI: configInfo["dburl"].(string) + "/colly",
@@ -240,6 +245,7 @@ func crawlDetail() (sucnum int) {
  odb := client.Database(configInfo["dbDatabase"].(string))
  lianjia := odb.Collection(configInfo["dbCollection"].(string))
 
+ //读取出全部需要抓取详情的数据
  cur, err := lianjia.Find(ctx, bson.M{"detailCrawlTime": bson.M{"$exists": false}})
 
  if err != nil {
@@ -263,28 +269,29 @@ func crawlDetail() (sucnum int) {
 }
 
 func Start_lianjia_ershou() {
- listFlag := make(chan int)
- detailFlag := make(chan int)
+ listFlag := make(chan int) //记录列表抓取是否完成
+ detailFlag := make(chan int) //记录详情是否抓取完成
 
  go func() {
  listCrawler()
- listFlag <- 1
+ listFlag <- 1 //列表抓取完成
  }()
 
  go func() {
  zeroNum := 0
  for i := 0; i < 1; i = 0 {
  if crawlDetail() == 0 {
  zeroNum++
- if zeroNum > 3 {
+ if zeroNum > 3 { //尝试3次都没有详情需要抓取，结束详情抓取
  break
  }
- time.Sleep(300 * time.Second)
+ time.Sleep(300 * time.Second) //没有详情需要抓取了，等待5分钟再尝试
  }
  }
- detailFlag <- 1
+ detailFlag <- 1 //详情抓取完成
  }()
 
+ //详情抓取与列表抓取都完成了，结束主线程
  <-listFlag
  <-detailFlag
 }

entrance/lianjia_zufang.go

@@ -63,12 +63,13 @@ func TcrawlerOneCityZuFang(cityUrl string, cityname string) {
  })
 
  c.OnHTML(".content__list--item", func(element *colly.HTMLElement) {
-
+ var err error
  var link string
  var title string
  var address string
  var area string
  var price int
+ var mianji int
  element.ForEach(".twoline a", func(i int, element *colly.HTMLElement) {
  link = "https://" + element.Request.URL.Host + element.Attr("href")
  title = strings.TrimSpace(element.Text)
@@ -82,6 +83,22 @@ func TcrawlerOneCityZuFang(cityUrl string, cityname string) {
  }
  })
 
+ desc := element.ChildText(".content__list--item--des")
+ desc = strings.ReplaceAll(desc, " ", "")
+ desc = strings.ReplaceAll(desc, "\n", "")
+ fmt.Println(desc)
+ re, _ := regexp.Compile("(\\d+)㎡/")
+ indexs := re.FindStringIndex(desc)
+ if len(indexs) == 2 {
+
+ mianji, err = strconv.Atoi(desc[indexs[0] : indexs[1]-4])
+ if err != nil {
+ mianji = 0
+ }
+ } else {
+ mianji = 0
+ }
+
  element.ForEach(".content__list--item-price em", func(i int, element *colly.HTMLElement) {
  var err error
  price, err = strconv.Atoi(element.Text)
@@ -90,26 +107,27 @@ func TcrawlerOneCityZuFang(cityUrl string, cityname string) {
  }
  })
 
- fmt.Println(price)
- fmt.Println(link)
- fmt.Println(title)
- fmt.Println(address)
- fmt.Println(area)
- fmt.Println(cityname)
+ //fmt.Println(price)
+ //fmt.Println(link)
+ //fmt.Println(title)
+ //fmt.Println(address)
+ //fmt.Println(area)
+ //fmt.Println(cityname)
  fmt.Println("--------------------")
 
  client := db.GetClient()
  ctx := db.GetCtx()
 
  db := client.Database(configInfo["dbDatabase"].(string))
  lianjia := db.Collection(configInfo["zufangCollection"].(string))
- _, err := lianjia.InsertOne(ctx, bson.M{
+ _, err = lianjia.InsertOne(ctx, bson.M{
  "Link": link,
  "title": title,
  "address": address,
  "area": area,
  "price": price,
  "city": cityname,
+ "mianji": mianji,
  "crawl_time": time.Now(),
  })
  if err != nil {

entrance/zhilian.go

@@ -26,6 +26,9 @@ func Start_zhilian() {
  var total int = 1000
  for start := 0; start < total; start += 50 {
  cityid := cityList[j].(map[string]interface{})["code"]
+ if cityid == nil {
+ fmt.Println(cityList[j])
+ }
  icityid, err := cityid.(json.Number).Int64()
  if err != nil {
  icityid = 530

main.go

@@ -7,7 +7,7 @@ import (
  "getAwayBSG/entrance"
 )
 
-// 实际中应该用更好的变量名
+// 申明配置变量
 var (
  help bool
  config string
@@ -16,6 +16,7 @@ var (
  zhilian bool
  clean bool
  info bool
+ infoSaveTo string
 )
 
 func init() {
@@ -26,15 +27,19 @@ func init() {
  flag.BoolVar(&zhilian, "zhilian", false, "抓取智联招聘数据")
  flag.BoolVar(&clean, "clean", false, "清理缓存")
  flag.BoolVar(&info, "info", false, "保存抓取状态")
+ flag.StringVar(&infoSaveTo, "info_save_to", "./numlog.txt", "输入状态文件保存位置")
 }
 
 func main() {
  flag.Parse()
+ //初始化配置信息，同时输出配置信息
  if config != "" {
  configs.SetConfig(config)
  }
  fmt.Println(configs.Config())
 
+
+ //进入不同入口
  if help {
  flag.Usage()
  } else if lianjia_ershou {
@@ -46,7 +51,7 @@ func main() {
  } else if clean {
  entrance.Start_clean()
  } else if info {
- entrance.Start_info()
+ entrance.Start_info(infoSaveTo)
  } else {
  flag.Usage()
  }