Skip to content

Commit 9a07e33

Browse files
npownpow
committed
scripts for DUPROPRIO
1 parent 19cfa0d commit 9a07e33

File tree

2 files changed

+16
-0
lines changed

2 files changed

+16
-0
lines changed

‎scraper/fetch_curr_DUPROPRIO.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
OUTPUT_DIR=tmp/DUPROPRIO/curr
4+
5+
for x in `seq 224`; do
6+
echo $x
7+
curl -o $OUTPUT_DIR/$x.html "http://duproprio.com/search/?hash=/g-re=6/s-hide-sold=/s-mode=list/p-con=main/p-ord=date/p-dir=DESC/pa-ge=$x" -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://duproprio.com/montreal-island' -H 'Cookie: _sp_id.c781=23269ac288be84a8.1417119277.1.1417191204.1417119277; _sp_ses.c781=*; dp_shared_session=a39kdqdnp0535o55q269inqmd4; overlaydisplayed=true; dp_session2[cookie]=a%3A12%3A%7Bs%3A4%3A%22lang%22%3Bs%3A9%3A%22s%3A2%3A%22en%22%3B%22%3Bs%3A11%3A%22province_id%22%3Bs%3A8%3A%22s%3A1%3A%221%22%3B%22%3Bs%3A16%3A%22search_main_hash%22%3Bs%3A82%3A%22s%3A74%3A%22%2Fg-re%3D6%2Fs-hide-sold%3D%2Fs-mode%3Dlist%2Fp-con%3Dmain%2Fp-ord%3Ddate%2Fp-dir%3DDESC%2Fpa-ge%3D1%2F%22%3B%22%3Bs%3A18%3A%22infosession_banner%22%3Bs%3A39%3A%22a%3A3%3A%7Bi%3A1754%3Bi%3A6%3Bi%3A1756%3Bi%3A6%3Bi%3A1758%3Bi%3A6%3B%7D%22%3Bs%3A23%3A%22mobile_banner_app_count%22%3Bs%3A4%3A%22i%3A5%3B%22%3Bs%3A18%3A%22seo_search_results%22%3Bs%3A11%3A%22s%3A4%3A%22true%22%3B%22%3Bs%3A15%3A%22alert_ove_close%22%3Bs%3A4%3A%22i%3A2%3B%22%3Bs%3A35%3A%22qualification_popup_decision_choice%22%3Bs%3A2%3A%22N%3B%22%3Bs%3A38%3A%22qualification_popup_decision_timestamp%22%3Bs%3A2%3A%22N%3B%22%3Bs%3A41%3A%22qualification_popup_timestamp_shown_array%22%3Bs%3A37%3A%22a%3A1%3A%7Bi%3A0%3Bs%3A19%3A%222014-11-28+10%3A58%3A18%22%3B%7D%22%3Bs%3A27%3A%22map_neighborhood_categories%22%3Bs%3A6%3A%22a%3A0%3A%7B%7D%22%3Bs%3A16%3A%22search_code_hash%22%3Bs%3A76%3A%22s%3A68%3A%22%2Fs-mode%3Dmap%2Fm-codes%3D556390%2Fp-con%3Dcode%2Fp-ord%3Ddate%2Fp-dir%3DDESC%2Fpa-ge%3D1%2F%22%3B%22%3B%7D; dp_intelligencer=a%3A3%3A%7Bs%3A3%3A%22uid%22%3Bi%3A101829702%3Bs%3A6%3A%22pepper%22%3Bs%3A32%3A%226cec6144837c530543fc13c97b3e12e0%22%3Bs%3A8%3A%22data_bag%22%3Ba%3A0%3A%7B%7D%7D; _sp_id.c781=23269ac288be84a8.1417119277.3.1417191566.1417122899; _sp_ses.c781=*' -H 'Connection: keep-alive' --compressed
8+
done

‎scraper/fetch_hist_DUPROPRIO.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
OUTPUT_DIR=tmp/DUPROPRIO/hist
4+
5+
for x in `seq 882`; do
6+
echo $x
7+
curl -o $OUTPUT_DIR/$x.html "http://duproprio.com/search/?hash=/g-re=6/s-pmin=0/s-pmax=99999999/s-build=1/s-filter=sold/s-hide-sold=/s-mode=list/p-con=main/p-ord=date/p-dir=DESC/pa-ge=$x" -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Referer: http://duproprio.com/search/?hash=/g-re=6/s-pmin=0/s-pmax=99999999/s-build=1/s-filter=sold/s-hide-sold=/s-mode=list/p-con=main/p-ord=date/p-dir=DESC/pa-ge=1' -H 'Cookie: _sp_id.c781=23269ac288be84a8.1417119277.1.1417190746.1417119277; _sp_ses.c781=*; dp_shared_session=a39kdqdnp0535o55q269inqmd4; overlaydisplayed=true; _sp_id.c781=23269ac288be84a8.1417119277.3.1417190628.1417122899; _sp_ses.c781=*; dp_session2[cookie]=a%3A12%3A%7Bs%3A4%3A%22lang%22%3Bs%3A9%3A%22s%3A2%3A%22en%22%3B%22%3Bs%3A11%3A%22province_id%22%3Bs%3A8%3A%22s%3A1%3A%221%22%3B%22%3Bs%3A16%3A%22search_main_hash%22%3Bs%3A132%3A%22s%3A123%3A%22%2Fg-re%3D6%2Fs-pmin%3D0%2Fs-pmax%3D99999999%2Fs-build%3D1%2Fs-filter%3Dsold%2Fs-hide-sold%3D%2Fs-mode%3Dlist%2Fp-con%3Dmain%2Fp-ord%3Ddate%2Fp-dir%3DDESC%2Fpa-ge%3D1%2F%22%3B%22%3Bs%3A18%3A%22infosession_banner%22%3Bs%3A39%3A%22a%3A3%3A%7Bi%3A1754%3Bi%3A6%3Bi%3A1756%3Bi%3A6%3Bi%3A1758%3Bi%3A6%3B%7D%22%3Bs%3A23%3A%22mobile_banner_app_count%22%3Bs%3A4%3A%22i%3A5%3B%22%3Bs%3A18%3A%22seo_search_results%22%3Bs%3A12%3A%22s%3A5%3A%22false%22%3B%22%3Bs%3A15%3A%22alert_ove_close%22%3Bs%3A4%3A%22i%3A2%3B%22%3Bs%3A35%3A%22qualification_popup_decision_choice%22%3Bs%3A2%3A%22N%3B%22%3Bs%3A38%3A%22qualification_popup_decision_timestamp%22%3Bs%3A2%3A%22N%3B%22%3Bs%3A41%3A%22qualification_popup_timestamp_shown_array%22%3Bs%3A37%3A%22a%3A1%3A%7Bi%3A0%3Bs%3A19%3A%222014-11-28+10%3A58%3A18%22%3B%7D%22%3Bs%3A27%3A%22map_neighborhood_categories%22%3Bs%3A6%3A%22a%3A0%3A%7B%7D%22%3Bs%3A16%3A%22search_code_hash%22%3Bs%3A76%3A%22s%3A68%3A%22%2Fs-mode%3Dmap%2Fm-codes%3D556390%2Fp-con%3Dcode%2Fp-ord%3Ddate%2Fp-dir%3DDESC%2Fpa-ge%3D1%2F%22%3B%22%3B%7D; dp_intelligencer=a%3A3%3A%7Bs%3A3%3A%22uid%22%3Bi%3A101829702%3Bs%3A6%3A%22pepper%22%3Bs%3A32%3A%226cec6144837c530543fc13c97b3e12e0%22%3Bs%3A8%3A%22data_bag%22%3Ba%3A0%3A%7B%7D%7D' -H 'Connection: keep-alive' --compressed
8+
done

0 commit comments

Comments
 (0)