#cavat, a general spider based on scrapy
make sure you are in PROJECT/crawler folder
set up virtualenv
virtualenv env
source env/bin/activate
if it's ubuntu, set like this, detail
virtualenv --system-site-packages env
source env/bin/activate
install dependencies
pip install -r requirements.txt
copy alembic.ini.example alembic.ini, then copy configs/dev.config.example.ini configs/dev.config.ini
change the sqlalchemy.url to your own
init database
alembic upgrade head
scrapy crawl 36kr