diff --git a/config.example.json b/config.example.json index 4bfc714..adca8fa 100644 --- a/config.example.json +++ b/config.example.json @@ -2,12 +2,19 @@ "clientId": "", "clientSecret": "", "userId": "", - "playlistId": "", - "radioTrackserviceUrl": "http://www.novaplanet.com/radionova/cetaitquoicetitre/", - "radioEntrySelector": "div.cestquoicetitre_results>div.resultat>div.info-col", - "radioTitleSelector": "h3.titre", - "radioArtistSelector": "h2.artiste", - "searchLinear": false, - "fm4Api": false, - "localEnvironment": true + "localEnvironment": true, + "playlists": { + "radiox": { + "playlistId": "", + "radioTrackserviceUrl": "http://www.radiox.co.uk/playlist/", + "radioEntrySelector": "div.playlist_entry_info", + "radioTitleSelector": ".track", + "radioArtistSelector": ".artist" + }, + "fm4": { + "playlistId": "", + "radioTrackserviceUrl": "https://audioapi.orf.at/fm4/api/json/current/broadcasts", + "fm4Api": true + } + } } diff --git a/logger.js b/logger.js index 9bff7be..d602770 100644 --- a/logger.js +++ b/logger.js @@ -1,11 +1,18 @@ +"use strict"; var fs = require('fs'); var util = require('util'); var log_stdout = process.stdout; module.exports = { - log: function(message){ - log_stdout.write(util.format(message) + '\n'); - message = new Date().toString() + ': ' + message; - fs.appendFileSync('./application.log', util.format(message) + '\n'); + /** + * writes to log file + * @param {String} message + * @param {String} [prefix] + */ + log: function(message, prefix){ + let prefixedMessage = (prefix) ? prefix+': '+message : message; + log_stdout.write(util.format(prefixedMessage) + '\n'); + let datedMessage = new Date().toString() + ': ' + prefixedMessage; + fs.appendFileSync('./application.log', util.format(datedMessage) + '\n'); } }; diff --git a/main.js b/main.js index 4fa94cd..e6c90b1 100644 --- a/main.js +++ b/main.js @@ -6,16 +6,23 @@ var logger = require('./logger'); var spotifyPlaylist = require('./spotifyPlaylist'); var radioCrawler = require('./radioCrawler'); var spotifySearch = require('./spotifySearch'); +var fs = require('fs'); +var config = JSON.parse(fs.readFileSync('config.json', 'utf8')); // starting the procedure start(); function start(){ - spotifyPlaylist.getAllTracks() - .then(radioCrawler.getTracks) + let playlistName = process.argv[2]; + if(!playlistName || !config.playlists[playlistName]){ + logger.log('Error: playlist not found'); + process.exit(); + } + spotifyPlaylist.getAllTracks(playlistName) + .then(() => radioCrawler.getTracks(playlistName)) .then(radioTracks => radioCrawler.cleanTracks(radioTracks)) .then(cleanedTracks => spotifySearch.searchTracks(cleanedTracks)) - .then(newTracks => spotifyPlaylist.addTracks(newTracks)) + .then(newTracks => spotifyPlaylist.addTracks(playlistName, newTracks)) .then(process.exit); } diff --git a/package.json b/package.json index 31fc230..1aa203c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "spotifyRadioPlaylist", - "version": "2.3.3", + "version": "2.4.0", "dependencies": { "bluebird": "^3.5.0", "cheerio": "^0.19.0" diff --git a/radioCrawler.js b/radioCrawler.js index 9a5184d..cf91eb4 100644 --- a/radioCrawler.js +++ b/radioCrawler.js @@ -15,17 +15,22 @@ String.prototype.isEmpty = function() {return (!this || !this.length)}; /** * getTracks + * @param {string} [playlistName] * @param {string} [trackserviceUrl] * @returns {Promise} */ -function getTracks(trackserviceUrl){ - var url = trackserviceUrl || config.radioTrackserviceUrl; - if(config.fm4Api){ +function getTracks(playlistName, trackserviceUrl){ + let playlistConfig = config.playlists[playlistName]; + let url = trackserviceUrl || playlistConfig.radioTrackserviceUrl; + if(playlistConfig.fm4Api){ return getFm4Broadcasts(url) - .then(broadcasts => broadcasts.map(broadcast => getFm4BroadcastTracks(broadcast))) + .then(broadcasts => { + console.log('getting tracks from API for '+broadcasts.length+' broadcasts'); + return broadcasts.map(broadcast => getFm4BroadcastTracks(broadcast)); + }) .then(AllBroadcastsWithTracks => Promise.all(AllBroadcastsWithTracks)) .then(broadcasts => { - var tracks = []; + let tracks = []; broadcasts.forEach(broadcast => { broadcast.forEach(track => tracks.push(track)); }); @@ -35,17 +40,17 @@ function getTracks(trackserviceUrl){ return new Promise((resolve, reject) => { console.log('getting tracks from radio trackservice'); - var trackserviceReq = http.request(url, function(res) { - var html = ''; + let trackserviceReq = http.request(url, function(res) { + let html = ''; if(res.statusCode === 302){ console.log('following redirect to ' + res.headers.location); - resolve(getTracks(res.headers.location)); + resolve(getTracks(playlistName, res.headers.location)); return; } if(res.statusCode !== 200){ - var error = 'Trackservice Error: Status '+res.statusCode; - logger.log(error); + let error = 'Trackservice Error: Status '+res.statusCode; + logger.log(error, playlistName); reject(error); process.exit(1); return; @@ -56,25 +61,24 @@ function getTracks(trackserviceUrl){ html += chunk; }); res.on('end', function() { - var $ = cheerio.load(html), + let $ = cheerio.load(html), tracks = []; - $(config.radioEntrySelector).each(function(i, elem){ - var $entry = $(this), - isUnique = true, + $(playlistConfig.radioEntrySelector).each(function(i, elem){ + let $entry = $(this), $title, // cheerio-object $artist, // cheerio-object title, // string artist; // string - if (config.searchLinear) { - // Stations like ORF FM4 have strange markup and need linear search - $title = $entry.nextAll(config.radioTitleSelector).first(); - $artist = $entry.nextAll(config.radioArtistSelector).first(); + if (playlistConfig.searchLinear) { + // Stations like the old page of ORF FM4 have strange markup and need linear search + $title = $entry.nextAll(playlistConfig.radioTitleSelector).first(); + $artist = $entry.nextAll(playlistConfig.radioArtistSelector).first(); } else { // Most other station playlists feature nested markup - $title = $entry.find(config.radioTitleSelector); - $artist = $entry.find(config.radioArtistSelector); + $title = $entry.find(playlistConfig.radioTitleSelector); + $artist = $entry.find(playlistConfig.radioArtistSelector); } title = $title.text(); @@ -87,7 +91,7 @@ function getTracks(trackserviceUrl){ }); if(tracks.length === 0){ - logger.log('no tracks found on radio trackservice.'); + logger.log('no tracks found on radio trackservice.', playlistName); return; process.exit(1); } @@ -97,7 +101,7 @@ function getTracks(trackserviceUrl){ }); trackserviceReq.on('error', function(e) { - logger.log('problem with trackservice request: ' + e.message); + logger.log('problem with trackservice request: ' + e.message, playlistName); process.exit(1); }); @@ -117,8 +121,8 @@ function getFm4Broadcasts(broadcastsUrl){ res.on('data', (chunk) => { rawData += chunk; }); res.on('end', () => { try { - var days = JSON.parse(rawData); - var allBroadcasts = []; + let days = JSON.parse(rawData); + let allBroadcasts = []; days.map(day => day.broadcasts).map(broadcasts => { broadcasts.map(broadcast => allBroadcasts.push(broadcast)) }); diff --git a/readme.md b/readme.md index 03afeed..a28007f 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # spotifyRadioPlaylist -Putting the latest tracks from a radio station with online trackservice (like [this one](http://www.novaplanet.com/radionova/cetaitquoicetitre/)) onto a spotify playlist. +Putting the latest tracks from a radio station with online trackservice (like [this one](http://www.novaplanet.com/radionova/cetaitquoicetitre/)) onto a spotify playlist by crawling the radio website. More on the background of this project in my [blog post](http://blog.chrisrohrer.de/radio-to-spotify-playlist/). @@ -13,14 +13,14 @@ Currently running hourly via cronjob on [this playlist](https://play.spotify.com Before running it on the server you need to authenticate via oAuth (as the user who owns the specified playlist) locally. 1. Clone this repo into a local directory -2. Copy `config.example.json` to `config.json` and insert your own credentials & radio station. +2. Copy `config.example.json` to `config.json` and insert your own credentials & playlistIds for your desired radio stations. You can obtain a clientId & clientSecret by [registering a new application](https://developer.spotify.com/my-applications/#!/applications). There are some pre-defined and tested radio stations, they can be found in `station-examples.md`. You can define your own schemes, all you need is the URL of the playlist page of the station and three jQuery selectors: one for the playlist entry element, one for the title and one for the artist. Some radio stations (like FM4) have special markup that requires linear instead of nested search; this behaviour can be set with the `searchLinear` flag. 3. Run `npm install` -4. Run `npm start` +4. Run `npm start ` (stationIdentifier is the name of the playlist config inside your config file.) 5. Open the displayed URL in your browser & grant permission for your app to change your playlists. This will open a page on localhost which you can close. Now you find two new files: `accessToken` and `refreshToken`. They contain the secret information to authenticate the user with spotify, so handle with care! -This should have added the first tracks to your playlist already. Every time you run `node main.js` again, the script will check if there are new tracks that have not been added to the playlist yet. +This should have added the first tracks to your playlist already. Every time you run `npm start ` again, the script will check if there are new tracks that have not been added to the playlist yet. You may want to run this on a server via cronjob every X minutes or so (depending on how many results your trackservice delivers in one page). @@ -28,7 +28,7 @@ You may want to run this on a server via cronjob every X minutes or so (dependin 2. Copy your local `accessToken` and `refreshToken` onto the server 3. Copy your local `config.json` onto the server and change the last entry to `"localEnvironment": false` 4. Run `npm install` -5. Configure your cronjob to run `node main.js` every X minutes (don't forget to change to the correct directory first! - this can be done with a bash script) +5. Configure your cronjob to run `node main.js ` every X minutes (don't forget to change to the correct directory first! - this can be done with a bash script) ## Updates diff --git a/spotifyPlaylist.js b/spotifyPlaylist.js index fa3e8ba..0df72b1 100644 --- a/spotifyPlaylist.js +++ b/spotifyPlaylist.js @@ -16,20 +16,22 @@ var spotifyPlaylist = { tracks: [] }; -function getAllTracks(){ - return getTracks(0); +function getAllTracks(playlistName){ + console.log('getting tracks from playlist '+playlistName); + return getTracks(playlistName, 0); } /** * getTracks + * @param {String} playlistName like in config * @param {int} offset - 0 for first page * @returns {Promise} */ -function getTracks(offset){ +function getTracks(playlistName, offset){ return new Promise((resolve, reject) => { - var LIMIT = 100; - console.log('getting next '+LIMIT+' tracks from playlist...'); - var accessToken = spotifyOAuth.getAccessToken(), + let playlistId = config.playlists[playlistName].playlistId; + let LIMIT = 100; + let accessToken = spotifyOAuth.getAccessToken(), playlistRequest; if(accessToken === false || accessToken === ''){ @@ -38,23 +40,23 @@ function getTracks(offset){ playlistRequest = https.request({ hostname: 'api.spotify.com', - path: '/v1/users/'+config.userId+'/playlists/'+config.playlistId+'/tracks?fields=next,items.track.uri&limit='+LIMIT+'&offset='+offset, + path: '/v1/users/'+config.userId+'/playlists/'+playlistId+'/tracks?fields=next,items.track.uri&limit='+LIMIT+'&offset='+offset, method: 'GET', headers: { 'Authorization': 'Bearer '+ accessToken, 'Accept': 'application/json' } }, function(res){ - var data = ''; + let data = ''; if(res.statusCode !== 200) { - spotifyHelper.checkForRateLimit(res, 'requesting playlist tracks', () => spotifyPlaylist.getTracks(offset)) + spotifyHelper.checkForRateLimit(res, 'requesting playlist tracks', () => spotifyPlaylist.getTracks(playlistName, offset)) .then(() => { if(res.statusCode === 401){ spotifyOAuth.refresh(); } else { var error = "Error getting tracks from playlist. Status "+res.statusCode; - logger.log(error); + logger.log(error, playlistName); reject(error); process.exit(1); } @@ -74,7 +76,7 @@ function getTracks(offset){ if(jsonData.next === null){ resolve(); } else if(typeof jsonData.next === 'string'){ - resolve(spotifyPlaylist.getTracks(offset + LIMIT)); + resolve(spotifyPlaylist.getTracks(playlistName, offset + LIMIT)); } else { reject('error getting data from playlist request'); } @@ -89,18 +91,19 @@ function getTracks(offset){ /** * addTracks - * @param results - * @param [lastCall] use this if this is the last call to this function, so the program can be stopped afterwards. + * @param {String} playlistName + * @param {Array} results */ -function addTracks(results){ - var accessToken = spotifyOAuth.getAccessToken(), +function addTracks(playlistName, results){ + let playlistConfig = config.playlists[playlistName]; + let accessToken = spotifyOAuth.getAccessToken(), LIMIT = 40; // limit how many tracks will be added in one request if(accessToken === false){ return; } if(results.length === 0){ - logger.log('no new tracks to add'); + logger.log('no new tracks to add', playlistName); process.exit(); return; } @@ -118,7 +121,7 @@ function addTracks(results){ setTimeout(() => { var addRequest = https.request({ hostname: 'api.spotify.com', - path: '/v1/users/'+config.userId+'/playlists/'+config.playlistId+'/tracks?position=0&uris='+items.join(), // join all 40 track ids for the query string + path: '/v1/users/'+config.userId+'/playlists/'+playlistConfig.playlistId+'/tracks?position=0&uris='+items.join(), // join all 40 track ids for the query string method: 'POST', headers: { 'Authorization': 'Bearer '+ accessToken, @@ -126,16 +129,16 @@ function addTracks(results){ } }, function(res){ if(res.statusCode === 201){ - logger.log('Success! Added '+ items.length + ' tracks.'); + logger.log('Success! Added '+ items.length + ' tracks.', playlistName); resolve(); return; } else { - spotifyHelper.checkForRateLimit(res, 'adding to playlist', () => resolve(spotifyPlaylist.addTracks(results))) + spotifyHelper.checkForRateLimit(res, 'adding to playlist', () => resolve(spotifyPlaylist.addTracks(playlistName, results))) .then(() => { if(res.statusCode === 401){ spotifyOAuth.refresh(); } else { - logger.log("Error adding to playlist. Status "+res.statusCode); + logger.log("Error adding to playlist. Status "+res.statusCode, playlistName); process.exit(1); } }); diff --git a/spotifySearch.js b/spotifySearch.js index f3ec91d..08cf50f 100644 --- a/spotifySearch.js +++ b/spotifySearch.js @@ -15,6 +15,7 @@ var Promise = require('bluebird'); function searchTracks(tracks){ var searchRequests = []; + console.log('Searching spotify for '+tracks.length+' tracks'); tracks.forEach(function(track, i){ searchRequests.push(sendSearchRequest(track, i * 100)); // timeout so we don't run into limits that fast }); diff --git a/station-examples.md b/station-examples.md index 25feaf8..caa1199 100644 --- a/station-examples.md +++ b/station-examples.md @@ -12,25 +12,15 @@ Works for *radio1*, *1xtra*, *radio2*, *6music* and *radioscotland* (adjust the "radioEntrySelector": "div.pll-playlist-item-details", "radioTitleSelector": "div.pll-playlist-item-title", "radioArtistSelector": "div.pll-playlist-item-artist > a", - "searchLinear": false, - "fm4Api": false, ## ORF FM4 "radioTrackserviceUrl": "https://audioapi.orf.at/fm4/api/json/current/broadcasts", - "radioEntrySelector": "", - "radioTitleSelector": "", - "radioArtistSelector": "", - "searchLinear": true, "fm4Api": true, ## ORF Ö1 "radioTrackserviceUrl": "https://audioapi.orf.at/oe1/api/json/current/broadcasts", - "radioEntrySelector": "", - "radioTitleSelector": "", - "radioArtistSelector": "", - "searchLinear": true, "fm4Api": true, ## RadioX @@ -39,7 +29,6 @@ Works for *radio1*, *1xtra*, *radio2*, *6music* and *radioscotland* (adjust the "radioEntrySelector": "div.playlist_entry_info", "radioTitleSelector": ".track", "radioArtistSelector": ".artist", - "searchLinear": false, "fm4Api": false, ## Radio Nova @@ -48,8 +37,6 @@ Works for *radio1*, *1xtra*, *radio2*, *6music* and *radioscotland* (adjust the "radioEntrySelector": "div.cestquoicetitre_results>div.resultat>div.info-col", "radioTitleSelector": "h3.titre", "radioArtistSelector": "h2.artiste", - "searchLinear": false, - "fm4Api": false, ## NDR2 @@ -57,8 +44,6 @@ Works for *radio1*, *1xtra*, *radio2*, *6music* and *radioscotland* (adjust the "radioEntrySelector": "div#playlist>ul>li.program>div.details>h3", "radioTitleSelector": "span.title", "radioArtistSelector": "span.artist", - "searchLinear": false, - "fm4Api": false, ## FluxFM @@ -66,8 +51,6 @@ Works for *radio1*, *1xtra*, *radio2*, *6music* and *radioscotland* (adjust the "radioEntrySelector": "table#songs>tr>td.title>div", "radioTitleSelector": "span.song", "radioArtistSelector": "span.artist", - "searchLinear": false, - "fm4Api": false, ## SWR1 @@ -75,8 +58,6 @@ Works for *radio1*, *1xtra*, *radio2*, *6music* and *radioscotland* (adjust the "radioEntrySelector": ".musicItemText", "radioTitleSelector": "h3", "radioArtistSelector": "p", - "searchLinear": false, - "fm4Api": false, ## Radio Swiss Jazz @@ -84,5 +65,3 @@ Works for *radio1*, *1xtra*, *radio2*, *6music* and *radioscotland* (adjust the "radioEntrySelector": ".playlist>.item-row", "radioTitleSelector": ".titletag", "radioArtistSelector": ".artist", - "searchLinear": false, - "fm4Api": false,