From b78a7a0cc4a1d6015d161c34869050fea7f4e403 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 17:27:27 -0400 Subject: [PATCH 01/71] Add link header parser --- Gemfile | 1 + Gemfile.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Gemfile b/Gemfile index ebed2475a..137f4487f 100644 --- a/Gemfile +++ b/Gemfile @@ -73,6 +73,7 @@ gem "aws-sdk-s3" gem "csv" gem "excon" gem "faraday", "~> 0.17.4" +gem 'link-header-parser', '~> 6.0', '>= 6.0.1' gem "fiddle" gem "hyperresource" gem "mutex_m" diff --git a/Gemfile.lock b/Gemfile.lock index aee601b34..befdc4b50 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -257,6 +257,7 @@ GEM kaminari-core (= 1.2.2) kaminari-core (1.2.2) language_server-protocol (3.17.0.3) + link-header-parser (6.0.1) local_time (2.1.0) logger (1.6.1) lograge (0.14.0) @@ -774,6 +775,7 @@ DEPENDENCIES importmap-rails jbuilder kaminari + link-header-parser (~> 6.0, >= 6.0.1) local_time logger lograge From 0fd257f3a24a077ace94a8e8b84bf3439395e80f Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 17:28:10 -0400 Subject: [PATCH 02/71] Add in active record encryption config --- config/application.rb | 4 ++++ env-example | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/config/application.rb b/config/application.rb index 8fabd2812..0dadf0e2a 100644 --- a/config/application.rb +++ b/config/application.rb @@ -151,5 +151,9 @@ def fmt.call(data) config.log_tags = [:request_id] config.active_record.schema_format = :ruby + + config.active_record.encryption.primary_key = ENV['ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY'] + config.active_record.encryption.deterministic_key = ENV['ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY'] + config.active_record.encryption.key_derivation_salt = ENV['ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT'] end end diff --git a/env-example b/env-example index e6ac8233d..e985a1fcc 100644 --- a/env-example +++ b/env-example @@ -84,3 +84,8 @@ APPLE_API_BRIDGE_URL= # slack notifications SLACK_SNS_TOPIC= SLACK_CHANNEL_ID= + +# encrypt active record attributes +ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY= +ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY= +ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT= From 03e5b7e363eb0bb39843315b682b2031d4e663ad Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 17:29:25 -0400 Subject: [PATCH 03/71] Add in megaphone feed and config --- app/models/feeds/megaphone_feed.rb | 7 +++++++ app/models/megaphone.rb | 5 +++++ app/models/megaphone/config.rb | 10 ++++++++++ .../20240929210828_create_megaphone_configs.rb | 13 +++++++++++++ db/schema.rb | 12 +++++++++++- 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 app/models/feeds/megaphone_feed.rb create mode 100644 app/models/megaphone.rb create mode 100644 app/models/megaphone/config.rb create mode 100644 db/migrate/20240929210828_create_megaphone_configs.rb diff --git a/app/models/feeds/megaphone_feed.rb b/app/models/feeds/megaphone_feed.rb new file mode 100644 index 000000000..cb86c0b17 --- /dev/null +++ b/app/models/feeds/megaphone_feed.rb @@ -0,0 +1,7 @@ +class Feeds::MegaphoneFeed < Feed + has_one :megaphone_config, class_name: "::Megaphone::Config", inverse_of: :feed + + def self.model_name + Feed.model_name + end +end diff --git a/app/models/megaphone.rb b/app/models/megaphone.rb new file mode 100644 index 000000000..bad25999f --- /dev/null +++ b/app/models/megaphone.rb @@ -0,0 +1,5 @@ +module Megaphone + def self.table_name_prefix + "megaphone_" + end +end diff --git a/app/models/megaphone/config.rb b/app/models/megaphone/config.rb new file mode 100644 index 000000000..3d42b0cbe --- /dev/null +++ b/app/models/megaphone/config.rb @@ -0,0 +1,10 @@ +module Megaphone + class Config < ApplicationRecord + belongs_to :feed + + validates_presence_of :token, :network_id, :feed_id + + encrypts :token + encrypts :network_id + end +end diff --git a/db/migrate/20240929210828_create_megaphone_configs.rb b/db/migrate/20240929210828_create_megaphone_configs.rb new file mode 100644 index 000000000..5ef0b15cc --- /dev/null +++ b/db/migrate/20240929210828_create_megaphone_configs.rb @@ -0,0 +1,13 @@ +class CreateMegaphoneConfigs < ActiveRecord::Migration[7.2] + def change + create_table :megaphone_configs do |t| + t.string :token + t.string :network_id + t.string :network_name + t.boolean "publish_enabled", default: false, null: false + t.bigint "feed_id", null: false + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 119ed4999..86bf24b51 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_09_28_150204) do +ActiveRecord::Schema[7.2].define(version: 2024_09_29_210828) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -317,6 +317,16 @@ t.index ["episode_id"], name: "index_media_versions_on_episode_id" end + create_table "megaphone_configs", force: :cascade do |t| + t.string "token" + t.string "network_id" + t.string "network_name" + t.boolean "publish_enabled", default: false, null: false + t.bigint "feed_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "podcast_imports", force: :cascade do |t| t.integer "podcast_id" t.string "url" From 8cb9e11ebfe65a0f481c417e7df336586a897305 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 17:40:43 -0400 Subject: [PATCH 04/71] starter tests for megaphone config --- test/factories/megaphone_config_factory.rb | 7 +++++++ test/models/megaphone/config_test.rb | 13 +++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 test/factories/megaphone_config_factory.rb create mode 100644 test/models/megaphone/config_test.rb diff --git a/test/factories/megaphone_config_factory.rb b/test/factories/megaphone_config_factory.rb new file mode 100644 index 000000000..40352cb78 --- /dev/null +++ b/test/factories/megaphone_config_factory.rb @@ -0,0 +1,7 @@ +FactoryBot.define do + factory :megaphone_config, class: Megaphone::Config do + token { "thisisatokenforacessingtheapi" } + network_id { "this-is-a-network-id" } + feed + end +end diff --git a/test/models/megaphone/config_test.rb b/test/models/megaphone/config_test.rb new file mode 100644 index 000000000..a6922f662 --- /dev/null +++ b/test/models/megaphone/config_test.rb @@ -0,0 +1,13 @@ +require "test_helper" + +describe Megaphone::Config do + describe "#valid?" do + it "must have required attributes" do + config = build(:megaphone_config) + assert_not_nil config.token + assert_not_nil config.network_id + assert_not_nil config.feed_id + assert config.valid? + end + end +end From 90083a2fbad36e1e4da2f4e53a1f8265504e02ef Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 17:43:25 -0400 Subject: [PATCH 05/71] Initial megaphone api support --- app/models/megaphone/api.rb | 121 ++++++++++++++++++++++++++++++ test/models/megaphone/api_test.rb | 13 ++++ 2 files changed, 134 insertions(+) create mode 100644 app/models/megaphone/api.rb create mode 100644 test/models/megaphone/api_test.rb diff --git a/app/models/megaphone/api.rb b/app/models/megaphone/api.rb new file mode 100644 index 000000000..b694bdb7e --- /dev/null +++ b/app/models/megaphone/api.rb @@ -0,0 +1,121 @@ +module Megaphone + class Api + attr_accessor :token, :network_id, :endpoint_url + + DEFAULT_ENDPOINT = "https://cms.megaphone.fm/api" + + PAGINATION_HEADERS = %w(link x-per-page x-page x-total) + PAGINATION_LINKS = %w(first last next previous) + + def initialize(token:, network_id:, endpoint_url: nil) + @token = token + @network_id = network_id + @endpoint_url = endpoint_url + end + + def get(path, params={}, headers={}) + request = {url: join_url(path), headers: headers, params: params} + response = get_url(request) + data = incoming_body_filter(response.body) + if data.kind_of?(Array) + pagination = pagination_from_headers(response.env.response_headers) + {items: data, pagination: pagination, request: request, response: response} + else + {items: [data], pagination: {}, request: request, response: response} + end + end + + def post(path, body, headers={}) + response = connection({url: join_url(path), headers: headers}).post() do |req| + req.body = outgoing_body_filter(body) + end + incoming_body_filter(response.body) + end + + def put(path, body, headers={}) + connection({url: join_url(path), headers: headers}).put() do |req| + req.body = outgoing_body_filter(body) + end + incoming_body_filter(response.body) + end + + # TODO: and we need delete... + + def api_base + @endpoint_url || DEFAULT_ENDPOINT + end + + def pagination_from_headers(headers) + paging = (headers || {}).slice(*PAGINATION_HEADERS).transform_keys do |h| + h.sub(/^x-/, "").gsub(/-/, "_").to_sym + end + + [:page, :per_page, :total].each do |k| + paging[k] = paging[k].to_i if paging.key?(k) + end + + paging[:link] = parse_links(paging[:link]) + + paging + end + + def parse_links(link_headers) + return {} unless link_headers.present? + collection = LinkHeaderParser.parse(link_headers, base: mp.api.api_base) + links = collection.group_by_relation_type + PAGINATION_LINKS.inject({}) do |map, key| + if link = (links[key] || []).first + map[key] = link.target_uri + end + map + end + end + + def get_url(options) + response = connection(options).get() + end + + def join_url(*path) + File.join(api_base, "networks", network_id, *path) + end + + def incoming_body_filter(str) + result = JSON.parse(str || "") + transform_keys(result) + end + + def transform_keys(result) + if result.kind_of?(Array) + result.map { |r| transform_keys(r) } + elsif result.respond_to?(:deep_transform_keys) + result.deep_transform_keys { |key| key.to_s.underscore.to_sym } + else + result + end + end + + def outgoing_body_filter(attr) + (attr || {}).deep_transform_keys { |key| key.to_s.camelize(:lower) }.to_json + end + + def default_headers + { + "Content-Type" => "application/json", + "Accept" => "application/json", + "User-Agent" => "PRX-Feeder-Megaphone/1.0 (Rails-#{Rails.env})" + } + end + + def connection(options) + url = options[:url] + headers = default_headers.merge(options[:headers] || {}) + params = options[:params] || {} + Faraday.new(url: url, headers: headers, params: params) do |builder| + builder.request :token_auth, token + builder.response :raise_error + builder.response :logger + builder.adapter :excon + end + end + end +end diff --git a/test/models/megaphone/api_test.rb b/test/models/megaphone/api_test.rb new file mode 100644 index 000000000..fef6095c9 --- /dev/null +++ b/test/models/megaphone/api_test.rb @@ -0,0 +1,13 @@ +require "test_helper" +require "securerandom" + +describe Megaphone::Api do + + let(:token) { SecureRandom.uuid } + let(:network_id) { "this-is-a-network-id" } + let(:api) { Megaphone::Api.new(token: token, network_id: network_id) } + + it "assigns the api key" do + assert_equal api.token, token + end +end From 972f67c93f16421c9be9d7cda0e43a54dc77ab5c Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 17:44:08 -0400 Subject: [PATCH 06/71] Podcast megaphone api implementation --- app/models/megaphone/paged_collection.rb | 19 ++++ app/models/megaphone/podcast.rb | 117 +++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 app/models/megaphone/paged_collection.rb create mode 100644 app/models/megaphone/podcast.rb diff --git a/app/models/megaphone/paged_collection.rb b/app/models/megaphone/paged_collection.rb new file mode 100644 index 000000000..a852f08f4 --- /dev/null +++ b/app/models/megaphone/paged_collection.rb @@ -0,0 +1,19 @@ +class Megaphone::PagedCollection + attr_accessor :model, :items, :per_page, :page, :total, :links + + def initialize(model, result) + @model = model + + @items = (result[:items] || []).map { |i| model.new(i) } + + paging = result[:pagination] || {} + @per_page = paging[:per_page] + @page = paging[:page] + @total = paging[:total] + @links = paging[:link] + end + + def count + items.length + end +end diff --git a/app/models/megaphone/podcast.rb b/app/models/megaphone/podcast.rb new file mode 100644 index 000000000..325f2eed4 --- /dev/null +++ b/app/models/megaphone/podcast.rb @@ -0,0 +1,117 @@ +module Megaphone + class Podcast + include ActiveModel::Model + attr_accessor :feed, :api + + # Required attributes for a create + # external_id is not required by megaphone, but we need it to be set! + CREATE_REQUIRED = %w(title subtitle summary itunes_categories language external_id) + + # Other attributes available on create + CREATE_ATTRIBUTES = CREATE_REQUIRED + %w(link copyright author background_image_file_url + explicit owner_name owner_email slug original_rss_url itunes_identifier podtrac_enabled + google_play_identifier episode_limit podcast_type advertising_tags excluded_categories) + + # Update also allows the span opt in + UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES + %w(span_opt_in) + + # Deprecated, so we shouldn't rely on these, but they show up as attributes + DEPRECATED = %w(category redirect_url itunes_active redirected_at itunes_rating + google_podcasts_identifier stitcher_identifier) + + # All other attributes we might expect back from the Megaphone API + # (some documented, others not so much) + OTHER_ATTRIBUTES = %w(id created_at updated_at image_file uid network_id recurring_import + episodes_count spotify_identifier default_ad_settings iheart_identifier feed_url + default_pre_count default_post_count cloned_feed_urls ad_free_feed_urls main_feed ad_free) + + ALL_ATTRIBUTES = (UPDATE_ATTRIBUTES + DEPRECATED + OTHER_ATTRIBUTES) + + attr_accessor *ALL_ATTRIBUTES + + validates_presence_of CREATE_REQUIRED + + validates_presence_of :id, on: :update + + validates_absence_of :id, on: :create + + # initialize from attributes + def initialize(attributes={}) + super + end + + def self.new_from_feed(feed) + podcast = Megaphone::Podcast.new(attributes_from_feed(feed)) + podcast.feed = feed + podcast + end + + def self.attributes_from_feed(feed) + podcast = feed.podcast + itunes_categories = feed.itunes_categories.present? ? feed.itunes_categories : podcast.itunes_categories + { + title: feed.title || podcast.title, + subtitle: feed.subtitle || podcast.subtitle, + summary: feed.description || podcast.description, + itunes_categories: (itunes_categories || []).map(&:name), + language: (podcast.language || "en-us").split("-").first, + link: podcast.link, + copyright: podcast.copyright, + author: podcast.author_name, + background_image_file_url: feed.ready_itunes_image || podcast.ready_itunes_image, + explicit: podcast.explicit, + owner_name: podcast.owner_name, + owner_email: podcast.owner_email, + slug: feed.slug, + # itunes_identifier: ????? TBD, + # handle prefix values in dt rss rendering of enclosure urls + podtrac_enabled: false, + episode_limit: feed.display_episodes_count, + external_id: podcast.guid, + podcast_type: podcast.itunes_type, + advertising_tags: podcast.categories, + # set in augury, can we get it here? + # excluded_categories: ????? TBD, + } + end + + def list + result = api.get("podcasts") + Megaphone::PagedCollection.new(Megaphone::Podcast, result) + end + + def find_by_guid + result = api.get("podcasts", externalId: feed.podcast.guid) + Megaphone::PagedCollection.new(Megaphone::Podcast, result) + end + + def find_by_megaphone_id(mpid = self.id) + result = api.get("podcasts/#{mpid}") + (result[:items] || []).first + end + + def create! + validate!(:create) + body = as_json.slice(*Megaphone::Podcast::CREATE_ATTRIBUTES) + result = api.post("podcasts", body) + self.attributes = result.slice(*Megaphone::Podcast::ALL_ATTRIBUTES) + self + end + + def update! + validate!(:update) + body = as_json.slice(*Megaphone::Podcast::UPDATE_ATTRIBUTES).to_json + result = api.put("podcasts/#{id}", body) + self.attributes = result.slice(*Megaphone::Podcast::ALL_ATTRIBUTES) + self + end + + def config + feed.megaphone_config + end + + def api + @api ||= Megaphone::Api.new(token: config.token, network_id: config.network_id) + end + end +end From 4c76d449bf6d3fee4ba6e1e3cd1cbb47eebd6c48 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 18:03:38 -0400 Subject: [PATCH 07/71] Fix test --- Gemfile | 2 +- test/models/megaphone/config_test.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 137f4487f..a87612c95 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem "aws-sdk-s3" gem "csv" gem "excon" gem "faraday", "~> 0.17.4" -gem 'link-header-parser', '~> 6.0', '>= 6.0.1' +gem "link-header-parser", "~> 6.0", ">= 6.0.1" gem "fiddle" gem "hyperresource" gem "mutex_m" diff --git a/test/models/megaphone/config_test.rb b/test/models/megaphone/config_test.rb index a6922f662..3e1da3bed 100644 --- a/test/models/megaphone/config_test.rb +++ b/test/models/megaphone/config_test.rb @@ -3,7 +3,7 @@ describe Megaphone::Config do describe "#valid?" do it "must have required attributes" do - config = build(:megaphone_config) + config = build(:megaphone_config, feed_id: 123) assert_not_nil config.token assert_not_nil config.network_id assert_not_nil config.feed_id From 9ca679ca40250494443243ac600832e8c5dc5aef Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 18:05:19 -0400 Subject: [PATCH 08/71] Linting fixes --- app/models/megaphone/api.rb | 27 +++++++++++++-------------- app/models/megaphone/podcast.rb | 27 ++++++++++++++------------- config/application.rb | 6 +++--- test/models/megaphone/api_test.rb | 1 - 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/app/models/megaphone/api.rb b/app/models/megaphone/api.rb index b694bdb7e..0dafbba3e 100644 --- a/app/models/megaphone/api.rb +++ b/app/models/megaphone/api.rb @@ -4,8 +4,8 @@ class Api DEFAULT_ENDPOINT = "https://cms.megaphone.fm/api" - PAGINATION_HEADERS = %w(link x-per-page x-page x-total) - PAGINATION_LINKS = %w(first last next previous) + PAGINATION_HEADERS = %w[link x-per-page x-page x-total] + PAGINATION_LINKS = %w[first last next previous] def initialize(token:, network_id:, endpoint_url: nil) @token = token @@ -13,11 +13,11 @@ def initialize(token:, network_id:, endpoint_url: nil) @endpoint_url = endpoint_url end - def get(path, params={}, headers={}) + def get(path, params = {}, headers = {}) request = {url: join_url(path), headers: headers, params: params} response = get_url(request) data = incoming_body_filter(response.body) - if data.kind_of?(Array) + if data.is_a?(Array) pagination = pagination_from_headers(response.env.response_headers) {items: data, pagination: pagination, request: request, response: response} else @@ -25,15 +25,15 @@ def get(path, params={}, headers={}) end end - def post(path, body, headers={}) - response = connection({url: join_url(path), headers: headers}).post() do |req| + def post(path, body, headers = {}) + response = connection({url: join_url(path), headers: headers}).post do |req| req.body = outgoing_body_filter(body) end incoming_body_filter(response.body) end - def put(path, body, headers={}) - connection({url: join_url(path), headers: headers}).put() do |req| + def put(path, body, headers = {}) + connection({url: join_url(path), headers: headers}).put do |req| req.body = outgoing_body_filter(body) end incoming_body_filter(response.body) @@ -47,7 +47,7 @@ def api_base def pagination_from_headers(headers) paging = (headers || {}).slice(*PAGINATION_HEADERS).transform_keys do |h| - h.sub(/^x-/, "").gsub(/-/, "_").to_sym + h.sub(/^x-/, "").tr("-", "_").to_sym end [:page, :per_page, :total].each do |k| @@ -63,16 +63,15 @@ def parse_links(link_headers) return {} unless link_headers.present? collection = LinkHeaderParser.parse(link_headers, base: mp.api.api_base) links = collection.group_by_relation_type - PAGINATION_LINKS.inject({}) do |map, key| - if link = (links[key] || []).first + PAGINATION_LINKS.each_with_object({}) do |key, map| + if (link = (links[key] || []).first) map[key] = link.target_uri end - map end end def get_url(options) - response = connection(options).get() + connection(options).get end def join_url(*path) @@ -85,7 +84,7 @@ def incoming_body_filter(str) end def transform_keys(result) - if result.kind_of?(Array) + if result.is_a?(Array) result.map { |r| transform_keys(r) } elsif result.respond_to?(:deep_transform_keys) result.deep_transform_keys { |key| key.to_s.underscore.to_sym } diff --git a/app/models/megaphone/podcast.rb b/app/models/megaphone/podcast.rb index 325f2eed4..be9211f7d 100644 --- a/app/models/megaphone/podcast.rb +++ b/app/models/megaphone/podcast.rb @@ -1,33 +1,34 @@ module Megaphone class Podcast include ActiveModel::Model - attr_accessor :feed, :api + attr_accessor :feed + attr_writer :api # Required attributes for a create # external_id is not required by megaphone, but we need it to be set! - CREATE_REQUIRED = %w(title subtitle summary itunes_categories language external_id) + CREATE_REQUIRED = %w[title subtitle summary itunes_categories language external_id] # Other attributes available on create - CREATE_ATTRIBUTES = CREATE_REQUIRED + %w(link copyright author background_image_file_url + CREATE_ATTRIBUTES = CREATE_REQUIRED + %w[link copyright author background_image_file_url explicit owner_name owner_email slug original_rss_url itunes_identifier podtrac_enabled - google_play_identifier episode_limit podcast_type advertising_tags excluded_categories) + google_play_identifier episode_limit podcast_type advertising_tags excluded_categories] # Update also allows the span opt in - UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES + %w(span_opt_in) + UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES + %w[span_opt_in] # Deprecated, so we shouldn't rely on these, but they show up as attributes - DEPRECATED = %w(category redirect_url itunes_active redirected_at itunes_rating - google_podcasts_identifier stitcher_identifier) + DEPRECATED = %w[category redirect_url itunes_active redirected_at itunes_rating + google_podcasts_identifier stitcher_identifier] # All other attributes we might expect back from the Megaphone API # (some documented, others not so much) - OTHER_ATTRIBUTES = %w(id created_at updated_at image_file uid network_id recurring_import + OTHER_ATTRIBUTES = %w[id created_at updated_at image_file uid network_id recurring_import episodes_count spotify_identifier default_ad_settings iheart_identifier feed_url - default_pre_count default_post_count cloned_feed_urls ad_free_feed_urls main_feed ad_free) + default_pre_count default_post_count cloned_feed_urls ad_free_feed_urls main_feed ad_free] ALL_ATTRIBUTES = (UPDATE_ATTRIBUTES + DEPRECATED + OTHER_ATTRIBUTES) - attr_accessor *ALL_ATTRIBUTES + attr_accessor(*ALL_ATTRIBUTES) validates_presence_of CREATE_REQUIRED @@ -36,7 +37,7 @@ class Podcast validates_absence_of :id, on: :create # initialize from attributes - def initialize(attributes={}) + def initialize(attributes = {}) super end @@ -69,7 +70,7 @@ def self.attributes_from_feed(feed) episode_limit: feed.display_episodes_count, external_id: podcast.guid, podcast_type: podcast.itunes_type, - advertising_tags: podcast.categories, + advertising_tags: podcast.categories # set in augury, can we get it here? # excluded_categories: ????? TBD, } @@ -85,7 +86,7 @@ def find_by_guid Megaphone::PagedCollection.new(Megaphone::Podcast, result) end - def find_by_megaphone_id(mpid = self.id) + def find_by_megaphone_id(mpid = id) result = api.get("podcasts/#{mpid}") (result[:items] || []).first end diff --git a/config/application.rb b/config/application.rb index 0dadf0e2a..b70d9a9a1 100644 --- a/config/application.rb +++ b/config/application.rb @@ -152,8 +152,8 @@ def fmt.call(data) config.log_tags = [:request_id] config.active_record.schema_format = :ruby - config.active_record.encryption.primary_key = ENV['ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY'] - config.active_record.encryption.deterministic_key = ENV['ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY'] - config.active_record.encryption.key_derivation_salt = ENV['ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT'] + config.active_record.encryption.primary_key = ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"] + config.active_record.encryption.deterministic_key = ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"] + config.active_record.encryption.key_derivation_salt = ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"] end end diff --git a/test/models/megaphone/api_test.rb b/test/models/megaphone/api_test.rb index fef6095c9..e2ec20643 100644 --- a/test/models/megaphone/api_test.rb +++ b/test/models/megaphone/api_test.rb @@ -2,7 +2,6 @@ require "securerandom" describe Megaphone::Api do - let(:token) { SecureRandom.uuid } let(:network_id) { "this-is-a-network-id" } let(:api) { Megaphone::Api.new(token: token, network_id: network_id) } From 844fb2024690c06f3152336ffbcafb920853b175 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 1 Oct 2024 18:17:09 -0400 Subject: [PATCH 09/71] fix that test with an actual feed --- test/models/megaphone/config_test.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/models/megaphone/config_test.rb b/test/models/megaphone/config_test.rb index 3e1da3bed..3e1e62dee 100644 --- a/test/models/megaphone/config_test.rb +++ b/test/models/megaphone/config_test.rb @@ -3,7 +3,8 @@ describe Megaphone::Config do describe "#valid?" do it "must have required attributes" do - config = build(:megaphone_config, feed_id: 123) + feed = create(:feed) + config = build(:megaphone_config, feed: feed) assert_not_nil config.token assert_not_nil config.network_id assert_not_nil config.feed_id From 6341784f3a4a961f4fedb79dd245986351763089 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 3 Oct 2024 10:23:57 -0400 Subject: [PATCH 10/71] Start of episode, basic tests --- app/models/megaphone/episode.rb | 38 +++++++++++++++++++++++++++ app/models/megaphone/model.rb | 15 +++++++++++ app/models/megaphone/podcast.rb | 14 +--------- test/factories/feed_factory.rb | 10 +++++++ test/models/megaphone/episode_test.rb | 18 +++++++++++++ test/models/megaphone/podcast_test.rb | 16 +++++++++++ 6 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 app/models/megaphone/episode.rb create mode 100644 app/models/megaphone/model.rb create mode 100644 test/models/megaphone/episode_test.rb create mode 100644 test/models/megaphone/podcast_test.rb diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb new file mode 100644 index 000000000..c36d0c63a --- /dev/null +++ b/app/models/megaphone/episode.rb @@ -0,0 +1,38 @@ +module Megaphone + class Episode < Megaphone::Model + attr_accessor :episode + + # Required attributes for a create + # external_id is not required by megaphone, but we need it to be set! + CREATE_REQUIRED = %w[title external_id] + + # All other attributes we might expect back from the Megaphone API + # (some documented, others not so much) + OTHER_ATTRIBUTES = %w[id created_at updated_at] + + DEPRECATED = %w[] + + ALL_ATTRIBUTES = (CREATE_REQUIRED + DEPRECATED + OTHER_ATTRIBUTES) + + attr_accessor(*ALL_ATTRIBUTES) + + validates_presence_of CREATE_REQUIRED + + validates_presence_of :id, on: :update + + validates_absence_of :id, on: :create + + def self.new_from_episode(dt_episode, feed = nil) + episode = Megaphone::Episode.new(attributes_from_episode(dt_episode)) + episode.episode = dt_episode + episode.feed = feed + episode + end + + def self.attributes_from_episode(dte) + { + title: dte.title + } + end + end +end diff --git a/app/models/megaphone/model.rb b/app/models/megaphone/model.rb new file mode 100644 index 000000000..a53b28696 --- /dev/null +++ b/app/models/megaphone/model.rb @@ -0,0 +1,15 @@ +module Megaphone + class Model + include ActiveModel::Model + attr_accessor :feed + attr_writer :api + + def config + feed.megaphone_config + end + + def api + @api ||= Megaphone::Api.new(token: config.token, network_id: config.network_id) + end + end +end diff --git a/app/models/megaphone/podcast.rb b/app/models/megaphone/podcast.rb index be9211f7d..d489a9872 100644 --- a/app/models/megaphone/podcast.rb +++ b/app/models/megaphone/podcast.rb @@ -1,9 +1,5 @@ module Megaphone - class Podcast - include ActiveModel::Model - attr_accessor :feed - attr_writer :api - + class Podcast < Megaphone::Model # Required attributes for a create # external_id is not required by megaphone, but we need it to be set! CREATE_REQUIRED = %w[title subtitle summary itunes_categories language external_id] @@ -106,13 +102,5 @@ def update! self.attributes = result.slice(*Megaphone::Podcast::ALL_ATTRIBUTES) self end - - def config - feed.megaphone_config - end - - def api - @api ||= Megaphone::Api.new(token: config.token, network_id: config.network_id) - end end end diff --git a/test/factories/feed_factory.rb b/test/factories/feed_factory.rb index ce70babe9..e50cc60f2 100644 --- a/test/factories/feed_factory.rb +++ b/test/factories/feed_factory.rb @@ -54,5 +54,15 @@ feed.apple_config = build(:apple_config) end end + + factory :megaphone_feed, class: "Feeds::MegaphoneFeed" do + type { "Feeds::MegaphoneFeed" } + private { false } + sequence(:slug) { |n| "mp-feed-#{n}" } + + after(:build) do |feed, _evaluator| + feed.megaphone_config = build(:megaphone_config) + end + end end end diff --git a/test/models/megaphone/episode_test.rb b/test/models/megaphone/episode_test.rb new file mode 100644 index 000000000..5e10ebcdf --- /dev/null +++ b/test/models/megaphone/episode_test.rb @@ -0,0 +1,18 @@ +require "test_helper" + +describe Megaphone::Episode do + let(:podcast) { create(:podcast) } + let(:feed) { create(:megaphone_feed, podcast: podcast) } + let(:dt_episode) { create(:episode, podcast: podcast) } + + describe "#valid?" do + it "must have required attributes" do + episode = Megaphone::Episode.new_from_episode(dt_episode, feed) + assert_not_nil episode + assert_not_nil episode.episode + assert_not_nil episode.feed + assert_equal episode.title, dt_episode.title + assert episode.valid? + end + end +end diff --git a/test/models/megaphone/podcast_test.rb b/test/models/megaphone/podcast_test.rb new file mode 100644 index 000000000..45e8a2ea8 --- /dev/null +++ b/test/models/megaphone/podcast_test.rb @@ -0,0 +1,16 @@ +require "test_helper" + +describe Megaphone::Podcast do + let(:dt_podcast) { create(:podcast) } + let(:feed) { create(:megaphone_feed, podcast: dt_podcast) } + + describe "#valid?" do + it "must have required attributes" do + podcast = Megaphone::Podcast.new_from_feed(feed) + assert_not_nil podcast + assert_not_nil podcast.feed + assert_equal podcast.title, feed.title + assert podcast.valid? + end + end +end From 82f2bee69a22d83a8a33ba331a2ea15ba23915ed Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 3 Oct 2024 11:07:11 -0400 Subject: [PATCH 11/71] set external id to dt episode guid --- app/models/megaphone/episode.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index c36d0c63a..d8ba10b89 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -31,7 +31,8 @@ def self.new_from_episode(dt_episode, feed = nil) def self.attributes_from_episode(dte) { - title: dte.title + title: dte.title, + external_id: dte.guid } end end From e2fa41da466320ed4e10ba84e6dcf446dea33beb Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 8 Oct 2024 17:56:31 -0400 Subject: [PATCH 12/71] Fix some bugs on link header handling --- app/models/megaphone/api.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/models/megaphone/api.rb b/app/models/megaphone/api.rb index 0dafbba3e..f646b0ea0 100644 --- a/app/models/megaphone/api.rb +++ b/app/models/megaphone/api.rb @@ -5,7 +5,7 @@ class Api DEFAULT_ENDPOINT = "https://cms.megaphone.fm/api" PAGINATION_HEADERS = %w[link x-per-page x-page x-total] - PAGINATION_LINKS = %w[first last next previous] + PAGINATION_LINKS = %i[first last next previous] def initialize(token:, network_id:, endpoint_url: nil) @token = token @@ -61,12 +61,13 @@ def pagination_from_headers(headers) def parse_links(link_headers) return {} unless link_headers.present? - collection = LinkHeaderParser.parse(link_headers, base: mp.api.api_base) + collection = LinkHeaderParser.parse(link_headers, base: api_base) links = collection.group_by_relation_type PAGINATION_LINKS.each_with_object({}) do |key, map| if (link = (links[key] || []).first) map[key] = link.target_uri end + map end end From 03ef603c9c4d7775d20f0b88649856477726a27f Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 8 Oct 2024 17:57:01 -0400 Subject: [PATCH 13/71] Initial episode attribute mapping --- app/models/megaphone/episode.rb | 46 ++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index d8ba10b89..0886d7c9b 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -6,6 +6,11 @@ class Episode < Megaphone::Model # external_id is not required by megaphone, but we need it to be set! CREATE_REQUIRED = %w[title external_id] + CREATE_ATTRIBUTES = CREATE_REQUIRED + %w[pubdate pubdate_timezone author link explicit draft + subtitle summary background_image_file_url background_audio_file_url pre_count post_count + insertion_points guid pre_offset post_offset expected_adhash original_filename original_url + episode_number season_number retain_ad_locations advertising_tags] + # All other attributes we might expect back from the Megaphone API # (some documented, others not so much) OTHER_ATTRIBUTES = %w[id created_at updated_at] @@ -26,14 +31,49 @@ def self.new_from_episode(dt_episode, feed = nil) episode = Megaphone::Episode.new(attributes_from_episode(dt_episode)) episode.episode = dt_episode episode.feed = feed + episode.set_audio_attributes episode end - def self.attributes_from_episode(dte) + def self.attributes_from_episode(e) { - title: dte.title, - external_id: dte.guid + title: e.title, + external_id: e.guid, + guid: e.item_guid, + pubdate: e.published_at, + pubdate_timezone: e.published_at.zone, + author: e.author_name, + link: e.url, + explicit: e.explicit, + draft: e.draft?, + subtitle: e.subtitle, + summary: e.description, + background_image_file_url: e.ready_image&.href, + episode_number: e.episode_number, + season_number: e.season_number, + advertising_tags: e.categories + # pre_count: e.pre_count, + # post_count: e.post_count, + # expected_adhash: e.expected_adhash, + # original_filename: e.original_filename, + # original_url: e.original_url, } end + + def set_audio_attributes + return unless episode.feed_ready? + self.background_audio_file_url = enclosure_url + self.insertion_points = timings + self.retain_ad_locations = true + end + + def enclosure_url + url = EnclosureUrlBuilder.new.base_enclosure_url(episode.podcast, episode, feed) + EnclosureUrlBuilder.mark_authorized(url, feed) + end + + def timings + episode.media[0..-2].map(&:duration) + end end end From b405ab2ffcb2a2925c56bf960bc6b806200ce88d Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 8 Oct 2024 17:57:23 -0400 Subject: [PATCH 14/71] Set some defaults for encryption in test --- test/test_helper.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_helper.rb b/test/test_helper.rb index 0efc19100..42ccf3fb5 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -19,6 +19,9 @@ "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUhHWUUvUVBZVWtkVUFmczcyZ1FUQkE5aTVBNkRndklFOGlpV3RrQzFScDdvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFaHFJSFVZUDN3QmxMdnMvQVpLM1ZHdW0vai8rMkhnVVF6dDc4TFQ0blMrckkxSlZJT0ZyVQpSVUZ6NmtSZ0pFeGxyZjdvSGZxZkxZanZGM0JvT3pmbWx3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQ==" ENV["APPLE_API_BRIDGE_URL"] = "http://localhost:3000" ENV["SLACK_CHANNEL_ID"] = "" +ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"] = "thisisamadeupkeythisisamadeupkey" +ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"] = "andthisisanotheronetoothankyouuu" +ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"] = "maybeyouarefeelingsaltyaboutthis" ENV["RAILS_ENV"] ||= "test" require_relative "../config/environment" From 0eff44c3243f1247fac6be117aee5267a6054f75 Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Mon, 21 Oct 2024 10:12:01 -0500 Subject: [PATCH 15/71] Prototype integrations interface --- app/models/apple/episode.rb | 10 +++- app/models/apple/publisher.rb | 48 +--------------- app/models/apple/show.rb | 53 +----------------- app/models/integrations/base/episode.rb | 40 +++++++++++++ .../base/episode_set_operations.rb | 56 +++++++++++++++++++ app/models/integrations/base/publisher.rb | 29 ++++++++++ app/models/integrations/base/show.rb | 50 +++++++++++++++++ test/models/apple/publisher_test.rb | 4 +- 8 files changed, 190 insertions(+), 100 deletions(-) create mode 100644 app/models/integrations/base/episode.rb create mode 100644 app/models/integrations/base/episode_set_operations.rb create mode 100644 app/models/integrations/base/publisher.rb create mode 100644 app/models/integrations/base/show.rb diff --git a/app/models/apple/episode.rb b/app/models/apple/episode.rb index 9d8a2c801..26e0cc293 100644 --- a/app/models/apple/episode.rb +++ b/app/models/apple/episode.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Apple - class Episode + class Episode < Integrations::Base::Episode include Apple::ApiWaiting include Apple::ApiResponse attr_accessor :show, @@ -221,6 +221,14 @@ def initialize(show:, feeder_episode:, api:) @api = api || Apple::Api.from_env end + def synced_with_integration? + synced_with_apple? + end + + def integration_new? + apple_new? + end + def api_response feeder_episode.apple_sync_log&.api_response end diff --git a/app/models/apple/publisher.rb b/app/models/apple/publisher.rb index bfb84474a..af744cf76 100644 --- a/app/models/apple/publisher.rb +++ b/app/models/apple/publisher.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Apple - class Publisher + class Publisher < Integrations::Base::Publisher PUBLISH_CHUNK_LEN = 25 attr_reader :public_feed, @@ -35,52 +35,6 @@ def podcast public_feed.podcast end - def filter_episodes_to_sync(eps) - # Reject episodes if the audio is marked as uploaded/complete - # or if the episode is a video - eps - .reject(&:synced_with_apple?) - .reject(&:video_content_type?) - end - - def episodes_to_sync - # only look at the private delegated delivery feed - filter_episodes_to_sync(show.apple_private_feed_episodes) - end - - def filter_episodes_to_archive(eps) - eps_in_private_feed = Set.new(show.apple_private_feed_episodes) - - # Episodes to archive can include: - # - episodes that are now excluded from the feed - # - episodes that are deleted or unpublished - # - episodes that have fallen off the end of the feed (Feed#display_episodes_count) - eps - .reject { |ep| eps_in_private_feed.include?(ep) } - .reject(&:apple_new?) - .reject(&:archived?) - end - - def episodes_to_archive - # look at the global list of episodes, not just the private feed - filter_episodes_to_archive(show.podcast_episodes) - end - - def filter_episodes_to_unarchive(eps) - eps.filter(&:archived?) - end - - def episodes_to_unarchive - # only look at the private delegated delivery feed - filter_episodes_to_unarchive(show.apple_private_feed_episodes) - end - - def only_episodes_with_apple_state(eps) - # Only select episodes that have an remote apple state, - # as determined by the sync log - eps.reject(&:apple_new?) - end - def poll_all_episodes! poll_episodes!(show.podcast_episodes) end diff --git a/app/models/apple/show.rb b/app/models/apple/show.rb index 61b0ab3a8..3ac67fd25 100644 --- a/app/models/apple/show.rb +++ b/app/models/apple/show.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Apple - class Show + class Show < Integrations::Base::Show include Apple::ApiResponse attr_reader :public_feed, @@ -165,55 +165,8 @@ def get_show self.class.get_show(api, apple_id) end - # In the case where there are duplicate guids in the feeds, we want to make - # sure that the most "current" episode is the one that maps to the remote state. - def sort_by_episode_properties(eps) - # Sort the episodes by: - # 1. Non-deleted episodes first - # 2. Published episodes first - # 3. Published date most recent first - # 4. Created date most recent first - eps = - eps.sort_by do |e| - [ - e.deleted_at.nil? ? 1 : -1, - e.published_at.present? ? 1 : -1, - e.published_at || e.created_at, - e.created_at - ] - end - - # return sorted list, reversed - # modeling a priority queue -- most important first - eps.reverse - end - - def podcast_feeder_episodes - @podcast_feeder_episodes ||= - podcast.episodes - .reset - .with_deleted - .group_by(&:item_guid) - .values - .map { |eps| sort_by_episode_properties(eps) } - .map(&:first) - end - - # All the episodes -- including deleted and unpublished - def podcast_episodes - @podcast_episodes ||= podcast_feeder_episodes.map { |e| Apple::Episode.new(api: api, show: self, feeder_episode: e) } - end - - # Does not include deleted episodes - def episodes - raise "Missing apple show id" unless apple_id.present? - - @episodes ||= begin - feed_episode_ids = Set.new(private_feed.feed_episodes.feed_ready.map(&:id)) - - podcast_episodes - .filter { |e| feed_episode_ids.include?(e.feeder_episode.id) } - end + def build_integration_episode(feeder_episode) + Apple::Episode.new(api: api, show: self, feeder_episode: feeder_episode) end def apple_private_feed_episodes diff --git a/app/models/integrations/base/episode.rb b/app/models/integrations/base/episode.rb new file mode 100644 index 000000000..59156c533 --- /dev/null +++ b/app/models/integrations/base/episode.rb @@ -0,0 +1,40 @@ +module Integrations + module Base + class Episode + attr_reader :feeder_episode + + def initialize(feeder_episode) + @feeder_episode = feeder_episode + end + + def synced_with_integration? + raise NotImplementedError, "Subclasses must implement synced_with_integration?" + end + + def integration_new? + raise NotImplementedError, "Subclasses must implement integration_new?" + end + + def archived? + raise NotImplementedError, "Subclasses must implement archived?" + end + + def video_content_type? + feeder_episode.video_content_type? + end + + # Delegate methods to feeder_episode + def method_missing(method_name, *arguments, &block) + if feeder_episode.respond_to?(method_name) + feeder_episode.send(method_name, *arguments, &block) + else + super + end + end + + def respond_to_missing?(method_name, include_private = false) + feeder_episode.respond_to?(method_name) || super + end + end + end +end diff --git a/app/models/integrations/base/episode_set_operations.rb b/app/models/integrations/base/episode_set_operations.rb new file mode 100644 index 000000000..fcdfc2c7f --- /dev/null +++ b/app/models/integrations/base/episode_set_operations.rb @@ -0,0 +1,56 @@ +module Integrations + module Base + module EpisodeSetOperations + # In the case where there are duplicate guids in the feeds, we want to make + # sure that the most "current" episode is the one that maps to the remote state. + def sort_by_episode_properties(eps) + # Sort the episodes by: + # 1. Non-deleted episodes first + # 2. Published episodes first + # 3. Published date most recent first + # 4. Created date most recent first + eps = + eps.sort_by do |e| + [ + e.deleted_at.nil? ? 1 : -1, + e.published_at.present? ? 1 : -1, + e.published_at || e.created_at, + e.created_at + ] + end + + # return sorted list, reversed + # modeling a priority queue -- most important first + eps.reverse + end + + def filter_episodes_to_sync(eps) + # Reject episodes if the audio is marked as uploaded/complete + # or if the episode is a video + eps + .reject(&:synced_with_integration?) + .reject(&:video_content_type?) + end + + def filter_episodes_to_archive(eps, eps_in_feed) + # Episodes to archive can include: + # - episodes that are now excluded from the feed + # - episodes that are deleted or unpublished + # - episodes that have fallen off the end of the feed (Feed#display_episodes_count) + eps + .reject { |ep| eps_in_feed.include?(ep) } + .reject(&:integration_new?) + .reject(&:archived?) + end + + def filter_episodes_to_unarchive(eps) + eps.filter(&:archived?) + end + + # Only select episodes that have an remote integration state + def only_episodes_with_integration_state(eps) + eps.reject(&:integration_new?) + end + end + end +end diff --git a/app/models/integrations/base/publisher.rb b/app/models/integrations/base/publisher.rb new file mode 100644 index 000000000..d2d298d53 --- /dev/null +++ b/app/models/integrations/base/publisher.rb @@ -0,0 +1,29 @@ +module Integrations + module Base + class Publisher + include EpisodeSetOperations + + attr_reader :show + + def initialize(show:) + @show = show + end + + def episodes_to_sync + filter_episodes_to_sync(show.episodes) + end + + def episodes_to_archive + filter_episodes_to_archive(show.podcast_episodes, Set.new(show.episodes)) + end + + def episodes_to_unarchive + filter_episodes_to_unarchive(show.episodes) + end + + def publish! + raise NotImplementedError, "Subclasses must implement publish!" + end + end + end +end diff --git a/app/models/integrations/base/show.rb b/app/models/integrations/base/show.rb new file mode 100644 index 000000000..f65ce0f05 --- /dev/null +++ b/app/models/integrations/base/show.rb @@ -0,0 +1,50 @@ +module Integrations + module Base + class Show + include EpisodeSetOperations + + attr_reader :feed + + def initialize(public_feed:, private_feed:) + @public_feed = public_feed + @private_feed = private_feed + end + + def podcast + private_feed.podcast + end + + def podcast_feeder_episodes + @podcast_feeder_episodes ||= + podcast.episodes + .reset + .with_deleted + .group_by(&:item_guid) + .values + .map { |eps| sort_by_episode_properties(eps) } + .map(&:first) + end + + # All the episodes -- including deleted and unpublished + def podcast_episodes + @podcast_episodes ||= podcast_feeder_episodes.map { |e| build_integration_episode(e) } + end + + # Does not include deleted episodes + def episodes + @episodes ||= begin + feed_episode_ids = Set.new(private_feed.feed_episodes.feed_ready.map(&:id)) + + podcast_episodes + .filter { |e| feed_episode_ids.include?(e.feeder_episode.id) } + end + end + + private + + def build_integration_episode(feeder_episode) + raise NotImplementedError, "Subclasses must implement create_integration_episode" + end + end + end +end diff --git a/test/models/apple/publisher_test.rb b/test/models/apple/publisher_test.rb index c8d3885db..cac56e7b2 100644 --- a/test/models/apple/publisher_test.rb +++ b/test/models/apple/publisher_test.rb @@ -38,10 +38,10 @@ it "should only return episodes that have an apple state" do episode.stub(:apple_new?, true) do - assert_equal apple_publisher.only_episodes_with_apple_state([episode]), [] + assert_equal apple_publisher.only_episodes_with_integration_state([episode]), [] end episode.stub(:apple_new?, false) do - assert_equal apple_publisher.only_episodes_with_apple_state([episode]), [episode] + assert_equal apple_publisher.only_episodes_with_integration_state([episode]), [episode] end end end From c5873e0bd331991b3f17a1b52f0ae8a089ba50a4 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 22 Oct 2024 08:24:39 -0400 Subject: [PATCH 16/71] Move prx access to an api module --- .../placements_preview_controller.rb | 24 +-- app/controllers/podcasts_controller.rb | 2 +- app/helpers/embed_player_helper.rb | 2 +- app/models/concerns/import_utils.rb | 2 +- app/models/prx/api.rb | 144 ++++++++++++++++++ app/models/prx/augury.rb | 27 ++++ app/models/task.rb | 2 +- app/representers/api/base_representer.rb | 4 +- lib/prx_access.rb | 142 ----------------- test/models/apple/api_test.rb | 1 - test/models/episode_test.rb | 3 - test/models/podcast_test.rb | 3 - .../prx/api_test.rb} | 18 +-- test/models/sync_log_test.rb | 1 - 14 files changed, 188 insertions(+), 187 deletions(-) create mode 100644 app/models/prx/api.rb create mode 100644 app/models/prx/augury.rb delete mode 100644 lib/prx_access.rb rename test/{lib/prx_access_test.rb => models/prx/api_test.rb} (57%) diff --git a/app/controllers/placements_preview_controller.rb b/app/controllers/placements_preview_controller.rb index a670cc9e7..bf88f0df8 100644 --- a/app/controllers/placements_preview_controller.rb +++ b/app/controllers/placements_preview_controller.rb @@ -1,5 +1,5 @@ class PlacementsPreviewController < ApplicationController - include PrxAccess + include Prx::Api before_action :set_podcast @@ -16,27 +16,9 @@ def set_podcast authorize @podcast, :show? end - def placements_href - "/api/v1/podcasts/#{@podcast.id}/placements" - end - - def fetch_placements - if ENV["AUGURY_HOST"].present? - api(root: augury_root, account: "*").tap { |a| a.href = placements_href }.get - end - rescue HyperResource::ClientError, HyperResource::ServerError, NotImplementedError => e - Rails.logger.error("Error fetching placements", error: e.message) - nil - end - - def cached_placements - Rails.cache.fetch(placements_href, expires_in: 1.minute) do - fetch_placements - end - end - def get_placement(original_count) - cached_placements&.find { |i| i.original_count == original_count } + placements = Prx::Augury.new.placements(@podcast) + placements&.find { |i| i.original_count == original_count } end def get_zones(original_count) diff --git a/app/controllers/podcasts_controller.rb b/app/controllers/podcasts_controller.rb index 6f4839903..7160fb035 100644 --- a/app/controllers/podcasts_controller.rb +++ b/app/controllers/podcasts_controller.rb @@ -1,5 +1,5 @@ class PodcastsController < ApplicationController - include PrxAccess + include Prx::Api include SlackHelper before_action :set_podcast, only: %i[show edit update destroy] diff --git a/app/helpers/embed_player_helper.rb b/app/helpers/embed_player_helper.rb index eeb41336c..05b26b441 100644 --- a/app/helpers/embed_player_helper.rb +++ b/app/helpers/embed_player_helper.rb @@ -1,5 +1,5 @@ module EmbedPlayerHelper - include PrxAccess + include Prx::Api EMBED_PLAYER_LANDING_PATH = "/listen" EMBED_PLAYER_PATH = "/e" diff --git a/app/models/concerns/import_utils.rb b/app/models/concerns/import_utils.rb index 11ba70dbe..547223962 100644 --- a/app/models/concerns/import_utils.rb +++ b/app/models/concerns/import_utils.rb @@ -1,5 +1,5 @@ require "active_support/concern" -require "prx_access" +require "prx/api" require "net/http" require "uri" require "text_sanitizer" diff --git a/app/models/prx/api.rb b/app/models/prx/api.rb new file mode 100644 index 000000000..1569cfce1 --- /dev/null +++ b/app/models/prx/api.rb @@ -0,0 +1,144 @@ +module Prx + module Api + class PrxHyperResource < HyperResource + def incoming_body_filter(hash) + super(hash.deep_transform_keys { |key| key.to_s.underscore }) + end + + def outgoing_body_filter(hash) + super(hash.deep_transform_keys { |key| key.to_s.camelize(:lower) }) + end + + def to_hash + attributes || {} + end + + class Link < HyperResource::Link + attr_accessor :type, :profile + + def initialize(resource, link_spec = {}) + super + self.type = link_spec["type"] + self.profile = link_spec["profile"] + end + + def where(params) + super.tap do |res| + res.type = type + res.profile = profile + end + end + + def headers(*args) + super.tap do |res| + if args.count > 0 + res.type = type + res.profile = profile + end + end + end + + def post_response(attrs = nil) + attrs ||= resource.attributes + attrs = (resource.default_attributes || {}).merge(attrs) + + # adding this line to call outgoing_body_filter + attrs = resource.outgoing_body_filter(attrs) + + faraday_connection.post do |req| + req.body = resource.adapter.serialize(attrs) + end + end + + def put_response(attrs = nil) + attrs ||= resource.attributes + attrs = (resource.default_attributes || {}).merge(attrs) + + # adding this line to call outgoing_body_filter + attrs = resource.outgoing_body_filter(attrs) + + faraday_connection.put do |req| + req.body = resource.adapter.serialize(attrs) + end + end + + def patch_response(attrs = nil) + attrs ||= resource.attributes.changed_attributes + attrs = (resource.default_attributes || {}).merge(attrs) + + # adding this line to call outgoing_body_filter + attrs = resource.outgoing_body_filter(attrs) + + faraday_connection.patch do |req| + req.body = resource.adapter.serialize(attrs) + end + end + end + end + + def default_headers + { + "Content-Type" => "application/json", + "Accept" => "application/json" + } + end + + def api(options = {}) + opts = {root: id_root, headers: default_headers}.merge(options) + if (account = opts.delete(:account)) + token = get_account_token(account) + opts[:headers]["Authorization"] = "Bearer #{token}" + end + + PrxHyperResource.new(opts) + end + + def api_resource(body, root = id_root) + href = body.dig(:_links, :self, :href) + resource = api(root: root) + link = PrxHyperResource::Link.new(resource, href: href) + PrxHyperResource.new_from(body: body, resource: resource, link: link) + end + + def get_account_token(account) + id = ENV["PRX_CLIENT_ID"] + se = ENV["PRX_SECRET"] + oauth_options = {site: id_root, token_url: "/token"} + client = OAuth2::Client.new(id, se, oauth_options) do |faraday| + faraday.request :url_encoded + faraday.adapter :excon + end + client.client_credentials.get_token(account: account).token + end + + def id_root + root_uri ENV["ID_HOST"] + end + + def play_root + root_uri ENV["PLAY_HOST"] + end + + private + + def method_missing(method, *args) + if /_root$/.match?(method) + root_uri ENV[method.to_s.sub(/_root$/, "_HOST").upcase], "/api/v1" + else + super + end + end + + def respond_to_missing?(method, include_private = false) + method.to_s.ends_with?("_root") || super + end + + def root_uri(host, path = "") + if /\.org|\.tech/.match?(host) + URI::HTTPS.build(host: host, path: path).to_s + else + URI::HTTP.build(host: host, path: path).to_s + end + end + end +end diff --git a/app/models/prx/augury.rb b/app/models/prx/augury.rb new file mode 100644 index 000000000..7a2ce6088 --- /dev/null +++ b/app/models/prx/augury.rb @@ -0,0 +1,27 @@ +module PRX + class Augury + include Prx::Api + API_PATH = "/api/v1" + + attr_accessor :enabled, :root, :expiration + + def initialize(options = {}) + @expiration = (options[:expiration] || 1.minute).to_i + @root = options[:root] || augury_root + @enabled = @root.present? + end + + def placements(podcast, options = {}) + path = "#{API_PATH}/podcasts/#{podcast.id}/placements" + expires = (options[:expiration] || expiration).to_i + Rails.cache.fetch(path, expires_in: expires) { get(root, path) } + end + + def get(root, path) + api(root: root, account: "*").tap { |a| a.href = path }.get + rescue HyperResource::ClientError, HyperResource::ServerError, NotImplementedError => e + Rails.logger.error("Error: GET #{path}", error: e.message) + nil + end + end +end diff --git a/app/models/task.rb b/app/models/task.rb index fdc14b2a1..074eef8f4 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -1,5 +1,5 @@ require "hash_serializer" -require "prx_access" +require "prx/api" class Task < ApplicationRecord include PorterCallback diff --git a/app/representers/api/base_representer.rb b/app/representers/api/base_representer.rb index f221ac120..0d9a3aa0e 100644 --- a/app/representers/api/base_representer.rb +++ b/app/representers/api/base_representer.rb @@ -1,10 +1,10 @@ require "api" require "hal_api/representer" -require "prx_access" +require "prx/api" require "text_sanitizer" class Api::BaseRepresenter < HalApi::Representer - include PrxAccess + include Prx::Api include TextSanitizer self.alternate_host = ENV["PRX_HOST"] || "www.prx.org" diff --git a/lib/prx_access.rb b/lib/prx_access.rb deleted file mode 100644 index 453914000..000000000 --- a/lib/prx_access.rb +++ /dev/null @@ -1,142 +0,0 @@ -module PrxAccess - class PrxHyperResource < HyperResource - def incoming_body_filter(hash) - super(hash.deep_transform_keys { |key| key.to_s.underscore }) - end - - def outgoing_body_filter(hash) - super(hash.deep_transform_keys { |key| key.to_s.camelize(:lower) }) - end - - def to_hash - attributes || {} - end - - class Link < HyperResource::Link - attr_accessor :type, :profile - - def initialize(resource, link_spec = {}) - super - self.type = link_spec["type"] - self.profile = link_spec["profile"] - end - - def where(params) - super.tap do |res| - res.type = type - res.profile = profile - end - end - - def headers(*args) - super.tap do |res| - if args.count > 0 - res.type = type - res.profile = profile - end - end - end - - def post_response(attrs = nil) - attrs ||= resource.attributes - attrs = (resource.default_attributes || {}).merge(attrs) - - # adding this line to call outgoing_body_filter - attrs = resource.outgoing_body_filter(attrs) - - faraday_connection.post do |req| - req.body = resource.adapter.serialize(attrs) - end - end - - def put_response(attrs = nil) - attrs ||= resource.attributes - attrs = (resource.default_attributes || {}).merge(attrs) - - # adding this line to call outgoing_body_filter - attrs = resource.outgoing_body_filter(attrs) - - faraday_connection.put do |req| - req.body = resource.adapter.serialize(attrs) - end - end - - def patch_response(attrs = nil) - attrs ||= resource.attributes.changed_attributes - attrs = (resource.default_attributes || {}).merge(attrs) - - # adding this line to call outgoing_body_filter - attrs = resource.outgoing_body_filter(attrs) - - faraday_connection.patch do |req| - req.body = resource.adapter.serialize(attrs) - end - end - end - end - - def default_headers - { - "Content-Type" => "application/json", - "Accept" => "application/json" - } - end - - def api(options = {}) - opts = {root: id_root, headers: default_headers}.merge(options) - if (account = opts.delete(:account)) - token = get_account_token(account) - opts[:headers]["Authorization"] = "Bearer #{token}" - end - - PrxHyperResource.new(opts) - end - - def api_resource(body, root = id_root) - href = body.dig(:_links, :self, :href) - resource = api(root: root) - link = PrxHyperResource::Link.new(resource, href: href) - PrxHyperResource.new_from(body: body, resource: resource, link: link) - end - - def get_account_token(account) - id = ENV["PRX_CLIENT_ID"] - se = ENV["PRX_SECRET"] - oauth_options = {site: id_root, token_url: "/token"} - client = OAuth2::Client.new(id, se, oauth_options) do |faraday| - faraday.request :url_encoded - faraday.adapter :excon - end - client.client_credentials.get_token(account: account).token - end - - def id_root - root_uri ENV["ID_HOST"] - end - - def play_root - root_uri ENV["PLAY_HOST"] - end - - private - - def method_missing(method, *args) - if /_root$/.match?(method) - root_uri ENV[method.to_s.sub(/_root$/, "_HOST").upcase], "/api/v1" - else - super - end - end - - def respond_to_missing?(method, include_private = false) - method.to_s.ends_with?("_root") || super - end - - def root_uri(host, path = "") - if /\.org|\.tech/.match?(host) - URI::HTTPS.build(host: host, path: path).to_s - else - URI::HTTP.build(host: host, path: path).to_s - end - end -end diff --git a/test/models/apple/api_test.rb b/test/models/apple/api_test.rb index 99d2d5f82..22b307ec0 100644 --- a/test/models/apple/api_test.rb +++ b/test/models/apple/api_test.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "test_helper" -require "prx_access" require "base64" require "securerandom" diff --git a/test/models/episode_test.rb b/test/models/episode_test.rb index d484f82d8..9bf6d09fd 100644 --- a/test/models/episode_test.rb +++ b/test/models/episode_test.rb @@ -1,9 +1,6 @@ require "test_helper" -require "prx_access" describe Episode do - include PrxAccess - let(:episode) { create(:episode_with_media) } it "initializes guid and overrides" do diff --git a/test/models/podcast_test.rb b/test/models/podcast_test.rb index be55bc7be..e018a3c07 100644 --- a/test/models/podcast_test.rb +++ b/test/models/podcast_test.rb @@ -1,9 +1,6 @@ require "test_helper" -require "prx_access" describe Podcast do - include PrxAccess - let(:podcast) { create(:podcast) } it "has episodes" do diff --git a/test/lib/prx_access_test.rb b/test/models/prx/api_test.rb similarity index 57% rename from test/lib/prx_access_test.rb rename to test/models/prx/api_test.rb index a795a7d83..4e82521f9 100644 --- a/test/lib/prx_access_test.rb +++ b/test/models/prx/api_test.rb @@ -1,20 +1,18 @@ -require "prx_access" - -class PrxAccessTest - include PrxAccess +class PrxApiTest + include Prx::Api end -describe PrxAccess do - let(:prx_access) { PrxAccessTest.new } - let(:resource) { PrxAccess::PrxHyperResource.new } +describe PrxApiTest do + let(:prx_api) { PrxApiTest.new } + let(:resource) { Prx::Api::PrxHyperResource.new } it "returns an api" do - refute_nil prx_access.api + refute_nil prx_api.api end it "returns root uri" do - refute_nil prx_access.id_root - refute_nil prx_access.feeder_root + refute_nil prx_api.id_root + refute_nil prx_api.feeder_root end it "underscores incoming hash keys" do diff --git a/test/models/sync_log_test.rb b/test/models/sync_log_test.rb index 06777dae6..83dec9437 100644 --- a/test/models/sync_log_test.rb +++ b/test/models/sync_log_test.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "test_helper" -require "prx_access" describe SyncLog do describe "indexes" do From 1c8a5e46c5534bbb8fbf41e3b194432437264b38 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 22 Oct 2024 08:25:06 -0400 Subject: [PATCH 17/71] Move prx access, remove announce --- app/workers/application_worker.rb | 35 +------------------------ test/workers/application_worker_test.rb | 9 ------- 2 files changed, 1 insertion(+), 43 deletions(-) diff --git a/app/workers/application_worker.rb b/app/workers/application_worker.rb index c5020ca0a..1e0749d3a 100644 --- a/app/workers/application_worker.rb +++ b/app/workers/application_worker.rb @@ -1,21 +1,12 @@ -require "prx_access" +require "prx/api" class ApplicationWorker - include PrxAccess include Shoryuken::Worker - attr_accessor :message - def self.prefix_name(name) [prefix, application, name].join("_") end - def self.announce_queues(model, actions) - actions.map do |action| - [prefix, "announce", application, model, action].join("_") - end - end - def self.application "feeder" end @@ -24,30 +15,6 @@ def self.prefix Rails.configuration.active_job.queue_name_prefix end - def announce_perform(event) - self.message = event.deep_symbolize_keys - method = delegate_method(message) - - if respond_to?(method) - public_send(method, message[:body]) - else - raise "`#{self.class.name}` subscribed, but doesn't implement " \ - "`#{delegate_method}` for '#{event.inspect}'" - end - end - - def action - (message || {})[:action] - end - - def subject - (message || {})[:subject] - end - - def delegate_method(message) - ["receive", message[:subject], message[:action]].join("_") - end - def logger Shoryuken.logger end diff --git a/test/workers/application_worker_test.rb b/test/workers/application_worker_test.rb index 756e58afd..c740ab8d0 100644 --- a/test/workers/application_worker_test.rb +++ b/test/workers/application_worker_test.rb @@ -8,15 +8,6 @@ assert_equal ApplicationWorker.prefix_name("foo"), "test_feeder_foo" end - it "lists announce queues" do - assert_equal ApplicationWorker.announce_queues("foo", ["bar", "wat"]), - ["test_announce_feeder_foo_bar", "test_announce_feeder_foo_wat"] - end - - it "determines the delegate method" do - assert_equal worker.delegate_method(msg), "receive_foo_bar" - end - it "has a logger" do assert_not_nil worker.logger assert worker.logger.is_a?(::Logger) From 29f5463b6e0896e2d76291c2355902295c141cf9 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 22 Oct 2024 08:25:31 -0400 Subject: [PATCH 18/71] Use private_feed arg --- app/models/apple/episode.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/apple/episode.rb b/app/models/apple/episode.rb index 9d8a2c801..6144e46d2 100644 --- a/app/models/apple/episode.rb +++ b/app/models/apple/episode.rb @@ -247,7 +247,7 @@ def podcast def enclosure_url url = EnclosureUrlBuilder.new.base_enclosure_url(podcast, feeder_episode, private_feed) - EnclosureUrlBuilder.mark_authorized(url, show.private_feed) + EnclosureUrlBuilder.mark_authorized(url, private_feed) end def enclosure_filename From 44d5f296a6e2faa1feccc42c85f160746b5a4374 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 22 Oct 2024 08:25:56 -0400 Subject: [PATCH 19/71] Include all the attributes --- app/models/megaphone/episode.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index 0886d7c9b..19deffafd 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -17,7 +17,7 @@ class Episode < Megaphone::Model DEPRECATED = %w[] - ALL_ATTRIBUTES = (CREATE_REQUIRED + DEPRECATED + OTHER_ATTRIBUTES) + ALL_ATTRIBUTES = (CREATE_ATTRIBUTES + DEPRECATED + OTHER_ATTRIBUTES) attr_accessor(*ALL_ATTRIBUTES) From cb95dbb20da6431a8909b9120def489c45e482f7 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 22 Oct 2024 08:26:20 -0400 Subject: [PATCH 20/71] Update gems --- Gemfile.lock | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index befdc4b50..4e277daa5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -205,12 +205,20 @@ GEM fuzzyurl (0.2.2) globalid (1.2.1) activesupport (>= 6.1) - google-protobuf (3.25.3-aarch64-linux) - google-protobuf (3.25.3-arm64-darwin) - google-protobuf (3.25.3-x86_64-darwin) - google-protobuf (3.25.3-x86_64-linux) - googleapis-common-protos-types (1.14.0) - google-protobuf (~> 3.18) + google-protobuf (4.28.2-aarch64-linux) + bigdecimal + rake (>= 13) + google-protobuf (4.28.2-arm64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.28.2-x86_64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.28.2-x86_64-linux) + bigdecimal + rake (>= 13) + googleapis-common-protos-types (1.16.0) + google-protobuf (>= 3.18, < 5.a) hal_api-rails (1.2.2) activemodel activesupport @@ -320,8 +328,8 @@ GEM opentelemetry-api (1.2.5) opentelemetry-common (0.21.0) opentelemetry-api (~> 1.0) - opentelemetry-exporter-otlp (0.26.3) - google-protobuf (~> 3.14) + opentelemetry-exporter-otlp (0.29.0) + google-protobuf (>= 3.18) googleapis-common-protos-types (~> 1.3) opentelemetry-api (~> 1.1) opentelemetry-common (~> 0.20) @@ -549,7 +557,7 @@ GEM psych (5.1.2) stringio public_suffix (5.0.0) - puma (5.6.8) + puma (5.6.9) nio4r (~> 2.0) pundit (2.3.0) activesupport (>= 3.0.0) From b24c264db78dcc6e76ca0c0245ae793ca11ba6c0 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 29 Oct 2024 16:45:15 -0400 Subject: [PATCH 21/71] In progress work on saving episodes to megaphone --- .../placements_preview_controller.rb | 2 +- app/models/apple/podcast_delivery.rb | 2 +- app/models/megaphone/episode.rb | 66 ++++++++++++++++--- app/models/prx/augury.rb | 6 +- env-example | 6 +- test/models/prx/api_test.rb | 2 + test/models/prx/augury_test.rb | 9 +++ 7 files changed, 76 insertions(+), 17 deletions(-) create mode 100644 test/models/prx/augury_test.rb diff --git a/app/controllers/placements_preview_controller.rb b/app/controllers/placements_preview_controller.rb index bf88f0df8..364303186 100644 --- a/app/controllers/placements_preview_controller.rb +++ b/app/controllers/placements_preview_controller.rb @@ -17,7 +17,7 @@ def set_podcast end def get_placement(original_count) - placements = Prx::Augury.new.placements(@podcast) + placements = Prx::Augury.new.placements(@podcast.id) placements&.find { |i| i.original_count == original_count } end diff --git a/app/models/apple/podcast_delivery.rb b/app/models/apple/podcast_delivery.rb index 9b083ae01..1d43b37c6 100644 --- a/app/models/apple/podcast_delivery.rb +++ b/app/models/apple/podcast_delivery.rb @@ -76,7 +76,7 @@ def self.create_podcast_deliveries(api, episodes) # An alternative workflow would be to swap out the existing delivery and # upload different audio. # - # The overall publishing workflow dependes on the assumption that there is + # The overall publishing workflow depends on the assumption that there is # a delivery present. If we don't create a delivery here, we short-circuit # subsequent steps (no uploads, no audio linking). episodes = select_episodes_for_delivery(episodes) diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index 19deffafd..fb030f58a 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -2,6 +2,9 @@ module Megaphone class Episode < Megaphone::Model attr_accessor :episode + # Used to form the adhash value + ADHASH_VALUES = {"pre" => "0", "mid" => "1", "post" => "2"}.freeze + # Required attributes for a create # external_id is not required by megaphone, but we need it to be set! CREATE_REQUIRED = %w[title external_id] @@ -9,7 +12,7 @@ class Episode < Megaphone::Model CREATE_ATTRIBUTES = CREATE_REQUIRED + %w[pubdate pubdate_timezone author link explicit draft subtitle summary background_image_file_url background_audio_file_url pre_count post_count insertion_points guid pre_offset post_offset expected_adhash original_filename original_url - episode_number season_number retain_ad_locations advertising_tags] + episode_number season_number retain_ad_locations advertising_tags ad_free] # All other attributes we might expect back from the Megaphone API # (some documented, others not so much) @@ -51,22 +54,57 @@ def self.attributes_from_episode(e) background_image_file_url: e.ready_image&.href, episode_number: e.episode_number, season_number: e.season_number, - advertising_tags: e.categories - # pre_count: e.pre_count, - # post_count: e.post_count, - # expected_adhash: e.expected_adhash, - # original_filename: e.original_filename, - # original_url: e.original_url, + advertising_tags: e.categories, + ad_free: e.categories.include?("adfree") } end + def set_placement_attributes + placement = get_placement(episode.segment_count) + self.expected_adhash = adhash_for_placement(placement) + self.pre_count = expected_adhash.count("0") + self.post_count = expected_adhash.count("2") + end + + def adhash_for_placement(placement) + placement + .zones + .filter { |z| z["type"] == "ad" } + .map { |z| ADHASH_VALUES[z["section"]] } + .join("") + end + + def get_placement(original_count) + placements = Prx::Augury.new.placements(@podcast.id) + placements&.find { |i| i.original_count == original_count } + end + def set_audio_attributes - return unless episode.feed_ready? - self.background_audio_file_url = enclosure_url + return unless episode.complete_media? + self.background_audio_file_url = upload_url self.insertion_points = timings self.retain_ad_locations = true end + def upload_url + resp = Faraday.head(enclosure_url) + if resp.status == 302 + media_version = resp.env.response_headers["x-episode-media-version"] + if media_version == episode.media_version_id + location = resp.env.response_headers["location"] + arrangement_version_url(location, media_version) + end + end + end + + def arrangement_version_url(location, media_version) + uri = URI.parse(location) + path = uri.path.split("/") + ext = File.extname(path.last) + filename = File.basename(path.last, ext) + "_" + media_version + File.extname(path.last) + uri.path = (path[0..-2] + [filename]).join("/") + end + def enclosure_url url = EnclosureUrlBuilder.new.base_enclosure_url(episode.podcast, episode, feed) EnclosureUrlBuilder.mark_authorized(url, feed) @@ -75,5 +113,15 @@ def enclosure_url def timings episode.media[0..-2].map(&:duration) end + + def pre_after_original?(placement) + sections = placement.zones.split { |z| z[:type] == "original" } + sections[1].any? { |z| %w[ad house].include?(z[:type]) && z[:id].match(/pre/) } + end + + def post_before_original?(placement) + sections = placement.zones.split { |z| z[:type] == "original" } + sections[-2].any? { |z| %w[ad house].include?(z[:type]) && z[:id].match(/post/) } + end end end diff --git a/app/models/prx/augury.rb b/app/models/prx/augury.rb index 7a2ce6088..10376bd49 100644 --- a/app/models/prx/augury.rb +++ b/app/models/prx/augury.rb @@ -1,4 +1,4 @@ -module PRX +module Prx class Augury include Prx::Api API_PATH = "/api/v1" @@ -11,8 +11,8 @@ def initialize(options = {}) @enabled = @root.present? end - def placements(podcast, options = {}) - path = "#{API_PATH}/podcasts/#{podcast.id}/placements" + def placements(podcast_id, options = {}) + path = "#{API_PATH}/podcasts/#{podcast_id}/placements" expires = (options[:expiration] || expiration).to_i Rails.cache.fetch(path, expires_in: expires) { get(root, path) } end diff --git a/env-example b/env-example index e985a1fcc..ecd0caa22 100644 --- a/env-example +++ b/env-example @@ -86,6 +86,6 @@ SLACK_SNS_TOPIC= SLACK_CHANNEL_ID= # encrypt active record attributes -ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY= -ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY= -ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT= +ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=0123456789abcdefghijklmnopqrstuv +ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=abcdefghijklmnopqrstuv0123456789 +ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=abcdefghijk0123456789lmnopqrstuv diff --git a/test/models/prx/api_test.rb b/test/models/prx/api_test.rb index 4e82521f9..8bdd0b327 100644 --- a/test/models/prx/api_test.rb +++ b/test/models/prx/api_test.rb @@ -1,3 +1,5 @@ +require "test_helper" + class PrxApiTest include Prx::Api end diff --git a/test/models/prx/augury_test.rb b/test/models/prx/augury_test.rb new file mode 100644 index 000000000..8f9e3db83 --- /dev/null +++ b/test/models/prx/augury_test.rb @@ -0,0 +1,9 @@ +require "test_helper" + +describe Prx::Augury do + let(augury) { Prx::Augury.new } + + it "retrieves placements" do + assert augury.placements(1234) + end +end From 7670baef9ff8836268e688acb4c92787e59ba34c Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 29 Oct 2024 18:00:27 -0400 Subject: [PATCH 22/71] Get the basic augury test working --- test/fixtures/placements.json | 214 +++++++++++++++++++++++++++++++++ test/models/prx/augury_test.rb | 15 ++- 2 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/placements.json diff --git a/test/fixtures/placements.json b/test/fixtures/placements.json new file mode 100644 index 000000000..267a0017a --- /dev/null +++ b/test/fixtures/placements.json @@ -0,0 +1,214 @@ +{ + "count": 3, + "total": 3, + "_embedded": { + "prx:items": [ + { + "id": 3063, + "name": "One Segment", + "originalCount": 1, + "zones": [ + { + "id": "original_1", + "name": "Original 1", + "type": "original", + "section": "original" + }, + { + "id": "sonic_id", + "name": "Sonic", + "type": "sonic_id", + "section": "post" + } + ], + "_links": { + "self": { + "href": "/api/v1/inventory/5678/placements/3063", + "profile": "http://meta.prx.org/model/application-record/placement" + }, + "profile": { + "href": "http://meta.prx.org/model/application-record/placement" + } + } + }, + { + "id": 3129, + "name": "Two Segments", + "originalCount": 2, + "zones": [ + { + "id": "house_pre", + "name": "House Preroll", + "type": "ad", + "section": "pre" + }, + { + "id": "original_1", + "name": "Original 1", + "type": "original", + "section": "original" + }, + { + "id": "mid_1", + "name": "Midroll 1", + "type": "ad", + "section": "mid" + }, + { + "id": "mid_2", + "name": "Midroll 2", + "type": "ad", + "section": "mid" + }, + { + "id": "original_2", + "name": "Original 2", + "type": "original", + "section": "original" + }, + { + "id": "post_1", + "name": "Postroll 1", + "type": "ad", + "section": "post" + }, + { + "id": "post_2", + "name": "Postroll 2", + "type": "ad", + "section": "post" + }, + { + "id": "house_post", + "name": "House Postroll", + "type": "ad", + "section": "post" + }, + { + "id": "sonic_id", + "name": "Sonic", + "type": "sonic_id", + "section": "post" + } + ], + "_links": { + "self": { + "href": "/api/v1/inventory/5678/placements/3129", + "profile": "http://meta.prx.org/model/application-record/placement" + }, + "profile": { + "href": "http://meta.prx.org/model/application-record/placement" + } + } + }, + { + "id": 3064, + "name": "Three Segments", + "originalCount": 3, + "zones": [ + { + "id": "house_pre", + "name": "House Preroll", + "type": "ad", + "section": "pre" + }, + { + "id": "original_1", + "name": "Original 1 (Intro)", + "type": "original", + "section": "original" + }, + { + "id": "pre_1", + "name": "Preroll 1", + "type": "ad", + "section": "pre" + }, + { + "id": "pre_2", + "name": "Preroll 2", + "type": "ad", + "section": "pre" + }, + { + "id": "original_2", + "name": "Original 2", + "type": "original", + "section": "original" + }, + { + "id": "mid_1", + "name": "Midroll 1", + "type": "ad", + "section": "mid" + }, + { + "id": "mid_2", + "name": "Midroll 2", + "type": "ad", + "section": "mid" + }, + { + "id": "original_3", + "name": "Original 3", + "type": "original", + "section": "original" + }, + { + "id": "post_1", + "name": "Postroll 1", + "type": "ad", + "section": "post" + }, + { + "id": "post_2", + "name": "Postroll 2", + "type": "ad", + "section": "post" + }, + { + "id": "house_post", + "name": "House Postroll", + "type": "ad", + "section": "post" + }, + { + "id": "sonic_id", + "name": "Sonic", + "type": "sonic_id", + "section": "post" + } + ], + "_links": { + "self": { + "href": "/api/v1/inventory/5678/placements/3064", + "profile": "http://meta.prx.org/model/application-record/placement" + }, + "profile": { + "href": "http://meta.prx.org/model/application-record/placement" + } + } + } + ] + }, + "_links": { + "self": { + "href": "/api/v1/podcasts/1234/placements", + "profile": "http://meta.prx.org/model/collection/application-record/placement" + }, + "prx:vary": { + "href": "/api/v1/podcasts/1234/placements{?page,per,zoom,filters,sorts}", + "templated": true + }, + "profile": { + "href": "http://meta.prx.org/model/collection/application-record/placement" + }, + "curies": [ + { + "name": "prx", + "href": "http://meta.prx.org/relation/{rel}", + "templated": true + } + ] + } +} \ No newline at end of file diff --git a/test/models/prx/augury_test.rb b/test/models/prx/augury_test.rb index 8f9e3db83..ed5274c8f 100644 --- a/test/models/prx/augury_test.rb +++ b/test/models/prx/augury_test.rb @@ -1,9 +1,20 @@ require "test_helper" describe Prx::Augury do - let(augury) { Prx::Augury.new } + let(:augury) { Prx::Augury.new } + + before { + stub_request(:post, 'https://id.prx.org/token') + .to_return(status: 200, + body: '{"access_token":"thisisnotatoken","token_type":"bearer"}', + headers: {"Content-Type" => "application/json; charset=utf-8"}) + + stub_request(:get, "https://inventory.dovetail.prx.org/api/v1/podcasts/1234/placements") + .to_return(status: 200, body: json_file(:placements), headers: {}) + } it "retrieves placements" do - assert augury.placements(1234) + placements = augury.placements(1234) + assert placements.collect { |p| p.original_count } == [1, 2, 3] end end From ced221945238539944f2e294c2b7e179f23f4bc2 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 29 Oct 2024 18:04:07 -0400 Subject: [PATCH 23/71] prettier --- test/fixtures/placements.json | 424 ++++++++++++++++----------------- test/models/prx/augury_test.rb | 2 +- 2 files changed, 213 insertions(+), 213 deletions(-) diff --git a/test/fixtures/placements.json b/test/fixtures/placements.json index 267a0017a..a54f6b048 100644 --- a/test/fixtures/placements.json +++ b/test/fixtures/placements.json @@ -1,214 +1,214 @@ { - "count": 3, - "total": 3, - "_embedded": { - "prx:items": [ - { - "id": 3063, - "name": "One Segment", - "originalCount": 1, - "zones": [ - { - "id": "original_1", - "name": "Original 1", - "type": "original", - "section": "original" - }, - { - "id": "sonic_id", - "name": "Sonic", - "type": "sonic_id", - "section": "post" - } - ], - "_links": { - "self": { - "href": "/api/v1/inventory/5678/placements/3063", - "profile": "http://meta.prx.org/model/application-record/placement" - }, - "profile": { - "href": "http://meta.prx.org/model/application-record/placement" - } - } - }, - { - "id": 3129, - "name": "Two Segments", - "originalCount": 2, - "zones": [ - { - "id": "house_pre", - "name": "House Preroll", - "type": "ad", - "section": "pre" - }, - { - "id": "original_1", - "name": "Original 1", - "type": "original", - "section": "original" - }, - { - "id": "mid_1", - "name": "Midroll 1", - "type": "ad", - "section": "mid" - }, - { - "id": "mid_2", - "name": "Midroll 2", - "type": "ad", - "section": "mid" - }, - { - "id": "original_2", - "name": "Original 2", - "type": "original", - "section": "original" - }, - { - "id": "post_1", - "name": "Postroll 1", - "type": "ad", - "section": "post" - }, - { - "id": "post_2", - "name": "Postroll 2", - "type": "ad", - "section": "post" - }, - { - "id": "house_post", - "name": "House Postroll", - "type": "ad", - "section": "post" - }, - { - "id": "sonic_id", - "name": "Sonic", - "type": "sonic_id", - "section": "post" - } - ], - "_links": { - "self": { - "href": "/api/v1/inventory/5678/placements/3129", - "profile": "http://meta.prx.org/model/application-record/placement" - }, - "profile": { - "href": "http://meta.prx.org/model/application-record/placement" - } - } - }, - { - "id": 3064, - "name": "Three Segments", - "originalCount": 3, - "zones": [ - { - "id": "house_pre", - "name": "House Preroll", - "type": "ad", - "section": "pre" - }, - { - "id": "original_1", - "name": "Original 1 (Intro)", - "type": "original", - "section": "original" - }, - { - "id": "pre_1", - "name": "Preroll 1", - "type": "ad", - "section": "pre" - }, - { - "id": "pre_2", - "name": "Preroll 2", - "type": "ad", - "section": "pre" - }, - { - "id": "original_2", - "name": "Original 2", - "type": "original", - "section": "original" - }, - { - "id": "mid_1", - "name": "Midroll 1", - "type": "ad", - "section": "mid" - }, - { - "id": "mid_2", - "name": "Midroll 2", - "type": "ad", - "section": "mid" - }, - { - "id": "original_3", - "name": "Original 3", - "type": "original", - "section": "original" - }, - { - "id": "post_1", - "name": "Postroll 1", - "type": "ad", - "section": "post" - }, - { - "id": "post_2", - "name": "Postroll 2", - "type": "ad", - "section": "post" - }, - { - "id": "house_post", - "name": "House Postroll", - "type": "ad", - "section": "post" - }, - { - "id": "sonic_id", - "name": "Sonic", - "type": "sonic_id", - "section": "post" - } - ], - "_links": { - "self": { - "href": "/api/v1/inventory/5678/placements/3064", - "profile": "http://meta.prx.org/model/application-record/placement" - }, - "profile": { - "href": "http://meta.prx.org/model/application-record/placement" - } - } - } - ] + "count": 3, + "total": 3, + "_embedded": { + "prx:items": [ + { + "id": 3063, + "name": "One Segment", + "originalCount": 1, + "zones": [ + { + "id": "original_1", + "name": "Original 1", + "type": "original", + "section": "original" + }, + { + "id": "sonic_id", + "name": "Sonic", + "type": "sonic_id", + "section": "post" + } + ], + "_links": { + "self": { + "href": "/api/v1/inventory/5678/placements/3063", + "profile": "http://meta.prx.org/model/application-record/placement" + }, + "profile": { + "href": "http://meta.prx.org/model/application-record/placement" + } + } + }, + { + "id": 3129, + "name": "Two Segments", + "originalCount": 2, + "zones": [ + { + "id": "house_pre", + "name": "House Preroll", + "type": "ad", + "section": "pre" + }, + { + "id": "original_1", + "name": "Original 1", + "type": "original", + "section": "original" + }, + { + "id": "mid_1", + "name": "Midroll 1", + "type": "ad", + "section": "mid" + }, + { + "id": "mid_2", + "name": "Midroll 2", + "type": "ad", + "section": "mid" + }, + { + "id": "original_2", + "name": "Original 2", + "type": "original", + "section": "original" + }, + { + "id": "post_1", + "name": "Postroll 1", + "type": "ad", + "section": "post" + }, + { + "id": "post_2", + "name": "Postroll 2", + "type": "ad", + "section": "post" + }, + { + "id": "house_post", + "name": "House Postroll", + "type": "ad", + "section": "post" + }, + { + "id": "sonic_id", + "name": "Sonic", + "type": "sonic_id", + "section": "post" + } + ], + "_links": { + "self": { + "href": "/api/v1/inventory/5678/placements/3129", + "profile": "http://meta.prx.org/model/application-record/placement" + }, + "profile": { + "href": "http://meta.prx.org/model/application-record/placement" + } + } + }, + { + "id": 3064, + "name": "Three Segments", + "originalCount": 3, + "zones": [ + { + "id": "house_pre", + "name": "House Preroll", + "type": "ad", + "section": "pre" + }, + { + "id": "original_1", + "name": "Original 1 (Intro)", + "type": "original", + "section": "original" + }, + { + "id": "pre_1", + "name": "Preroll 1", + "type": "ad", + "section": "pre" + }, + { + "id": "pre_2", + "name": "Preroll 2", + "type": "ad", + "section": "pre" + }, + { + "id": "original_2", + "name": "Original 2", + "type": "original", + "section": "original" + }, + { + "id": "mid_1", + "name": "Midroll 1", + "type": "ad", + "section": "mid" + }, + { + "id": "mid_2", + "name": "Midroll 2", + "type": "ad", + "section": "mid" + }, + { + "id": "original_3", + "name": "Original 3", + "type": "original", + "section": "original" + }, + { + "id": "post_1", + "name": "Postroll 1", + "type": "ad", + "section": "post" + }, + { + "id": "post_2", + "name": "Postroll 2", + "type": "ad", + "section": "post" + }, + { + "id": "house_post", + "name": "House Postroll", + "type": "ad", + "section": "post" + }, + { + "id": "sonic_id", + "name": "Sonic", + "type": "sonic_id", + "section": "post" + } + ], + "_links": { + "self": { + "href": "/api/v1/inventory/5678/placements/3064", + "profile": "http://meta.prx.org/model/application-record/placement" + }, + "profile": { + "href": "http://meta.prx.org/model/application-record/placement" + } + } + } + ] + }, + "_links": { + "self": { + "href": "/api/v1/podcasts/1234/placements", + "profile": "http://meta.prx.org/model/collection/application-record/placement" }, - "_links": { - "self": { - "href": "/api/v1/podcasts/1234/placements", - "profile": "http://meta.prx.org/model/collection/application-record/placement" - }, - "prx:vary": { - "href": "/api/v1/podcasts/1234/placements{?page,per,zoom,filters,sorts}", - "templated": true - }, - "profile": { - "href": "http://meta.prx.org/model/collection/application-record/placement" - }, - "curies": [ - { - "name": "prx", - "href": "http://meta.prx.org/relation/{rel}", - "templated": true - } - ] - } -} \ No newline at end of file + "prx:vary": { + "href": "/api/v1/podcasts/1234/placements{?page,per,zoom,filters,sorts}", + "templated": true + }, + "profile": { + "href": "http://meta.prx.org/model/collection/application-record/placement" + }, + "curies": [ + { + "name": "prx", + "href": "http://meta.prx.org/relation/{rel}", + "templated": true + } + ] + } +} diff --git a/test/models/prx/augury_test.rb b/test/models/prx/augury_test.rb index ed5274c8f..17738eeb2 100644 --- a/test/models/prx/augury_test.rb +++ b/test/models/prx/augury_test.rb @@ -4,7 +4,7 @@ let(:augury) { Prx::Augury.new } before { - stub_request(:post, 'https://id.prx.org/token') + stub_request(:post, "https://id.prx.org/token") .to_return(status: 200, body: '{"access_token":"thisisnotatoken","token_type":"bearer"}', headers: {"Content-Type" => "application/json; charset=utf-8"}) From a2e0ff8352dfe3225779e70a222653290364b591 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 29 Oct 2024 18:39:23 -0400 Subject: [PATCH 24/71] Use env vars for urls --- test/models/prx/augury_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/models/prx/augury_test.rb b/test/models/prx/augury_test.rb index 17738eeb2..080ff5304 100644 --- a/test/models/prx/augury_test.rb +++ b/test/models/prx/augury_test.rb @@ -4,12 +4,12 @@ let(:augury) { Prx::Augury.new } before { - stub_request(:post, "https://id.prx.org/token") + stub_request(:post, "https://#{ENV["ID_HOST"]}/token") .to_return(status: 200, body: '{"access_token":"thisisnotatoken","token_type":"bearer"}', headers: {"Content-Type" => "application/json; charset=utf-8"}) - stub_request(:get, "https://inventory.dovetail.prx.org/api/v1/podcasts/1234/placements") + stub_request(:get, "https://#{ENV["AUGURY_HOST"]}/api/v1/podcasts/1234/placements") .to_return(status: 200, body: json_file(:placements), headers: {}) } From 59d192a954854e1b2dd47d1582e699dbd48f8bee Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Mon, 4 Nov 2024 17:17:40 -0600 Subject: [PATCH 25/71] Clarify naming and extract episode status instance methods --- app/models/apple/config.rb | 2 +- app/models/apple/episode_delivery_status.rb | 8 ++++++++ app/models/apple/publisher.rb | 2 +- app/models/concerns/apple_delivery.rb | 10 ++++------ ...230920153421_add_needs_apple_delivery_to_episode.rb | 4 ++-- test/factories/apple_episode_factory.rb | 2 +- test/models/apple/episode_delivery_status_test.rb | 2 +- 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/app/models/apple/config.rb b/app/models/apple/config.rb index a1a4a2db7..87ba16b27 100644 --- a/app/models/apple/config.rb +++ b/app/models/apple/config.rb @@ -41,7 +41,7 @@ def self.build_apple_config(podcast, key) def self.mark_as_delivered!(apple_publisher) apple_publisher.episodes_to_sync.each do |episode| if episode.podcast_container&.needs_delivery? == false - episode.feeder_episode.apple_has_delivery! + episode.feeder_episode.apple_mark_as_delivered! end end end diff --git a/app/models/apple/episode_delivery_status.rb b/app/models/apple/episode_delivery_status.rb index 5781c180c..c78e58a20 100644 --- a/app/models/apple/episode_delivery_status.rb +++ b/app/models/apple/episode_delivery_status.rb @@ -21,5 +21,13 @@ def increment_asset_wait def reset_asset_wait self.class.update_status(episode, asset_processing_attempts: 0) end + + def mark_as_delivered! + self.class.update_status(episode, delivered: true) + end + + def mark_as_not_delivered! + self.class.update_status(episode, delivered: false) + end end end diff --git a/app/models/apple/publisher.rb b/app/models/apple/publisher.rb index 65ee96901..81e7ef81a 100644 --- a/app/models/apple/publisher.rb +++ b/app/models/apple/publisher.rb @@ -367,7 +367,7 @@ def mark_delivery_files_uploaded!(eps) # update the feeder episode to indicate that delivery is no longer needed eps.each do |ep| Rails.logger.info("Marking episode as no longer needing delivery", {episode_id: ep.feeder_episode.id}) - ep.feeder_episode.apple_has_delivery! + ep.feeder_episode.apple_mark_as_delivered! end Rails.logger.info("Updated remote container references for episodes.", {count: res.length}) diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb index 0083481b7..2be30403b 100644 --- a/app/models/concerns/apple_delivery.rb +++ b/app/models/concerns/apple_delivery.rb @@ -34,17 +34,15 @@ def apple_episode_delivery_status end def apple_needs_delivery? - return true if apple_episode_delivery_status.nil? - apple_episode_delivery_status.delivered == false end - def apple_needs_delivery! - apple_update_delivery_status(delivered: false) + def apple_mark_as_not_delivered! + apple_episode_delivery_status.mark_as_not_delivered! end - def apple_has_delivery! - apple_update_delivery_status(delivered: true) + def apple_mark_as_delivered! + apple_episode_delivery_status.mark_as_delivered! end def measure_asset_processing_duration diff --git a/db/migrate/20230920153421_add_needs_apple_delivery_to_episode.rb b/db/migrate/20230920153421_add_needs_apple_delivery_to_episode.rb index 3b3d01d52..c61370edf 100644 --- a/db/migrate/20230920153421_add_needs_apple_delivery_to_episode.rb +++ b/db/migrate/20230920153421_add_needs_apple_delivery_to_episode.rb @@ -20,10 +20,10 @@ def change needs_delivery_episodes = [] apple_episodes.each do |apple_episode| if (apple_episode.podcast_container.nil? || apple_episode.podcast_container.needs_delivery?) || apple_episode.apple_hosted_audio_asset_container_id.blank? - apple_episode.feeder_episode.apple_needs_delivery! + apple_episode.feeder_episode.apple_mark_as_not_delivered! needs_delivery_episodes << apple_episode.feeder_episode else - apple_episode.feeder_episode.apple_has_delivery! + apple_episode.feeder_episode.apple_mark_as_delivered! end end diff --git a/test/factories/apple_episode_factory.rb b/test/factories/apple_episode_factory.rb index ef8458758..1d2cb31e1 100644 --- a/test/factories/apple_episode_factory.rb +++ b/test/factories/apple_episode_factory.rb @@ -13,7 +13,7 @@ factory :uploaded_apple_episode do feeder_episode do ep = create(:episode) - ep.apple_has_delivery! + ep.apple_mark_as_delivered! ep end transient do diff --git a/test/models/apple/episode_delivery_status_test.rb b/test/models/apple/episode_delivery_status_test.rb index 04cf8e3e0..ed1cc51cb 100644 --- a/test/models/apple/episode_delivery_status_test.rb +++ b/test/models/apple/episode_delivery_status_test.rb @@ -14,7 +14,7 @@ class Apple::EpisodeDeliveryStatusTest < ActiveSupport::TestCase episode.destroy assert_equal episode, delivery_status.episode assert_difference "Apple::EpisodeDeliveryStatus.count", +1 do - episode.apple_needs_delivery! + episode.apple_mark_as_not_delivered! end assert_equal episode, episode.apple_episode_delivery_statuses.first.episode end From 9da9dd2774c218f983a922d5ffe0740700b15bc0 Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Tue, 29 Oct 2024 14:13:16 -0500 Subject: [PATCH 26/71] Fix quirk on soft deleted dependent destroy --- app/models/concerns/apple_delivery.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb index 2be30403b..261acdc1a 100644 --- a/app/models/concerns/apple_delivery.rb +++ b/app/models/concerns/apple_delivery.rb @@ -11,7 +11,7 @@ module AppleDelivery class_name: "Apple::PodcastDelivery" has_many :apple_podcast_delivery_files, through: :apple_podcast_deliveries, source: :podcast_delivery_files, class_name: "Apple::PodcastDeliveryFile" - has_many :apple_episode_delivery_statuses, -> { order(created_at: :desc) }, dependent: :destroy, class_name: "Apple::EpisodeDeliveryStatus" + has_many :apple_episode_delivery_statuses, -> { order(created_at: :desc) }, class_name: "Apple::EpisodeDeliveryStatus" alias_method :podcast_container, :apple_podcast_container alias_method :apple_status, :apple_episode_delivery_status From 2aa0e4d07f1f0d42893e1ce6e02e91e91d7e0651 Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Tue, 29 Oct 2024 14:13:38 -0500 Subject: [PATCH 27/71] Move in related methods --- app/models/concerns/apple_delivery.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb index 261acdc1a..fe09c9f2d 100644 --- a/app/models/concerns/apple_delivery.rb +++ b/app/models/concerns/apple_delivery.rb @@ -45,6 +45,26 @@ def apple_mark_as_delivered! apple_episode_delivery_status.mark_as_delivered! end + def apple_mark_as_uploaded! + apple_episode_delivery_status.mark_as_uploaded! + end + + def apple_mark_as_not_uploaded! + apple_episode_delivery_status.mark_as_not_uploaded! + end + + def apple_prepare_for_delivery! + # remove the previous delivery attempt (soft delete) + apple_podcast_deliveries.map(&:destroy) + apple_podcast_deliveries.reset + apple_podcast_delivery_files.reset + apple_podcast_container&.podcast_deliveries&.reset + end + + def apple_mark_for_reupload! + apple_mark_as_not_delivered! + end + def measure_asset_processing_duration statuses = apple_episode_delivery_statuses.to_a From fd6c5cf9c98a38101749cb5ccf218d2563677b37 Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Tue, 29 Oct 2024 14:21:27 -0500 Subject: [PATCH 28/71] Break up method refactor --- app/models/apple/publisher.rb | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/app/models/apple/publisher.rb b/app/models/apple/publisher.rb index 81e7ef81a..4cfc6d192 100644 --- a/app/models/apple/publisher.rb +++ b/app/models/apple/publisher.rb @@ -143,9 +143,12 @@ def deliver_and_publish!(eps) sync_podcast_deliveries!(eps) sync_podcast_delivery_files!(eps) - # upload and mark as uploaded + # upload and mark as uploaded, then update the audio container reference execute_upload_operations!(eps) mark_delivery_files_uploaded!(eps) + update_audio_container_reference!(eps) + + mark_as_delivered!(eps) wait_for_upload_processing(eps) @@ -361,16 +364,24 @@ def mark_delivery_files_uploaded!(eps) Rails.logger.tagged("##{__method__}") do pdfs = eps.map(&:podcast_delivery_files).flatten ::Apple::PodcastDeliveryFile.mark_uploaded(api, pdfs) + end + end + def update_audio_container_reference!(eps) + Rails.logger.tagged("##{__method__}") do # link the podcast container with the audio to the episode res = Apple::Episode.update_audio_container_reference(api, eps) - # update the feeder episode to indicate that delivery is no longer needed + + Rails.logger.info("Updated remote container references for episodes.", {count: res.length}) + end + end + + def mark_as_delivered!(eps) + Rails.logger.tagged("##{__method__}") do eps.each do |ep| Rails.logger.info("Marking episode as no longer needing delivery", {episode_id: ep.feeder_episode.id}) ep.feeder_episode.apple_mark_as_delivered! end - - Rails.logger.info("Updated remote container references for episodes.", {count: res.length}) end end From 7ea78961fb1d381fa03e77f8bc993e5195e66e7b Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Tue, 29 Oct 2024 14:31:06 -0500 Subject: [PATCH 29/71] Add the uploaded state marker --- app/models/apple/episode_delivery_status.rb | 8 ++++++++ db/migrate/20241029181938_new_file_uploaded_state.rb | 5 +++++ db/schema.rb | 3 ++- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20241029181938_new_file_uploaded_state.rb diff --git a/app/models/apple/episode_delivery_status.rb b/app/models/apple/episode_delivery_status.rb index c78e58a20..42416239f 100644 --- a/app/models/apple/episode_delivery_status.rb +++ b/app/models/apple/episode_delivery_status.rb @@ -22,6 +22,14 @@ def reset_asset_wait self.class.update_status(episode, asset_processing_attempts: 0) end + def mark_as_uploaded! + self.class.update_status(episode, uploaded: true) + end + + def mark_as_not_uploaded! + self.class.update_status(episode, uploaded: false) + end + def mark_as_delivered! self.class.update_status(episode, delivered: true) end diff --git a/db/migrate/20241029181938_new_file_uploaded_state.rb b/db/migrate/20241029181938_new_file_uploaded_state.rb new file mode 100644 index 000000000..c5ca9b5e5 --- /dev/null +++ b/db/migrate/20241029181938_new_file_uploaded_state.rb @@ -0,0 +1,5 @@ +class NewFileUploadedState < ActiveRecord::Migration[7.2] + def change + add_column :apple_episode_delivery_statuses, :uploaded, :boolean, default: false + end +end diff --git a/db/schema.rb b/db/schema.rb index 235f715d7..45e036f4a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_10_02_215949) do +ActiveRecord::Schema[7.2].define(version: 2024_10_29_181938) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" enable_extension "uuid-ossp" @@ -37,6 +37,7 @@ t.integer "source_fetch_count", default: 0 t.bigint "source_media_version_id" t.integer "asset_processing_attempts", default: 0, null: false + t.boolean "uploaded", default: false t.index ["episode_id", "created_at"], name: "index_apple_episode_delivery_statuses_on_episode_id_created_at", include: ["delivered", "id"] t.index ["episode_id"], name: "index_apple_episode_delivery_statuses_on_episode_id" end From f2fc12904ff86630fe57cec240964435932d6d8a Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Wed, 30 Oct 2024 09:40:26 -0500 Subject: [PATCH 30/71] Break out the upload flow --- app/models/apple/podcast_delivery.rb | 6 ---- app/models/apple/publisher.rb | 47 +++++++++++++++++----------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/app/models/apple/podcast_delivery.rb b/app/models/apple/podcast_delivery.rb index 9b083ae01..861bdc5d6 100644 --- a/app/models/apple/podcast_delivery.rb +++ b/app/models/apple/podcast_delivery.rb @@ -73,12 +73,6 @@ def self.create_podcast_deliveries(api, episodes) end # Don't create deliveries for containers that already have deliveries. - # An alternative workflow would be to swap out the existing delivery and - # upload different audio. - # - # The overall publishing workflow dependes on the assumption that there is - # a delivery present. If we don't create a delivery here, we short-circuit - # subsequent steps (no uploads, no audio linking). episodes = select_episodes_for_delivery(episodes) podcast_containers = episodes.map(&:podcast_container) diff --git a/app/models/apple/publisher.rb b/app/models/apple/publisher.rb index 4cfc6d192..a4e680ad1 100644 --- a/app/models/apple/publisher.rb +++ b/app/models/apple/publisher.rb @@ -131,30 +131,35 @@ def publish! def deliver_and_publish!(eps) Rails.logger.tagged("Apple::Publisher#deliver_and_publish!") do eps.each_slice(PUBLISH_CHUNK_LEN) do |eps| - # Soft delete any existing delivery and delivery files - prepare_for_delivery!(eps) + # First upload the audio files + eps.filter(&:apple_needs_upload?).tap do |eps| + # Soft delete any existing delivery and delivery files + prepare_for_delivery!(eps) - # only create if needed - sync_episodes!(eps) - sync_podcast_containers!(eps) + # only create if needed + sync_episodes!(eps) + sync_podcast_containers!(eps) - wait_for_versioned_source_metadata(eps) + wait_for_versioned_source_metadata(eps) - sync_podcast_deliveries!(eps) - sync_podcast_delivery_files!(eps) + sync_podcast_deliveries!(eps) + sync_podcast_delivery_files!(eps) - # upload and mark as uploaded, then update the audio container reference - execute_upload_operations!(eps) - mark_delivery_files_uploaded!(eps) - update_audio_container_reference!(eps) + # upload and mark as uploaded, then update the audio container reference + execute_upload_operations!(eps) + mark_delivery_files_uploaded!(eps) + update_audio_container_reference!(eps) + # finally mark as uploaded + mark_as_uploaded!(eps) + end - mark_as_delivered!(eps) + increment_asset_wait!(eps) wait_for_upload_processing(eps) - - increment_asset_wait!(eps) wait_for_asset_state(eps) + mark_as_delivered!(eps) + publish_drafting!(eps) reset_asset_wait!(eps) @@ -234,9 +239,6 @@ def wait_for_upload_processing(eps) def increment_asset_wait!(eps) Rails.logger.tagged("##{__method__}") do - eps = eps.filter { |e| e.podcast_delivery_files.any?(&:api_marked_as_uploaded?) } - - # Mark the episodes as waiting again for asset processing eps.each { |ep| ep.apple_episode_delivery_status.increment_asset_wait } end end @@ -385,6 +387,15 @@ def mark_as_delivered!(eps) end end + def mark_as_uploaded!(eps) + Rails.logger.tagged("##{__method__}") do + eps.each do |ep| + Rails.logger.info("Marking episode as no longer needing delivery", {episode_id: ep.feeder_episode.id}) + ep.feeder_episode.apple_mark_as_uploaded! + end + end + end + def publish_drafting!(eps) Rails.logger.tagged("##{__method__}") do eps = eps.select { |ep| ep.drafting? && ep.container_upload_complete? } From 7d7ead79f0cd4e831a19ce605be4c857935b3c1c Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Thu, 31 Oct 2024 10:44:06 -0500 Subject: [PATCH 31/71] Split out coverage for update reference --- test/factories/apple_episode_api_response.rb | 3 ++- test/factories/apple_episode_factory.rb | 6 +++++- test/models/apple/publisher_test.rb | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/test/factories/apple_episode_api_response.rb b/test/factories/apple_episode_api_response.rb index cf46ca04f..1021120e8 100644 --- a/test/factories/apple_episode_api_response.rb +++ b/test/factories/apple_episode_api_response.rb @@ -15,13 +15,14 @@ after(:build) do |response_container, evaluator| response_container["api_response"] = - {"request_metadata" => {"apple_episode_id" => evaluator.apple_episode_id, "item_guid" => evaluator.item_guid}, + {"request_metadata" => {"apple_episode_id" => evaluator.apple_episode_id, "guid" => evaluator.item_guid}, "api_url" => evaluator.api_url, "api_parameters" => {}, "api_response" => {"ok" => evaluator.ok, "err" => evaluator.err, "val" => {"data" => {"id" => "123", "attributes" => { + "appleHostedAudioAssetContainerId" => nil, "appleHostedAudioAssetVendorId" => evaluator.apple_hosted_audio_asset_container_id, "publishingState" => evaluator.publishing_state, "guid" => evaluator.item_guid, diff --git a/test/factories/apple_episode_factory.rb b/test/factories/apple_episode_factory.rb index 1d2cb31e1..97052d6ca 100644 --- a/test/factories/apple_episode_factory.rb +++ b/test/factories/apple_episode_factory.rb @@ -6,7 +6,8 @@ # set up transient api_response transient do feeder_episode { create(:episode) } - api_response { build(:apple_episode_api_response) } + api_response { build(:apple_episode_api_response, item_guid: feeder_episode.item_guid) } + apple_hosted_audio_asset_container_id { "456" } end # set a complete episode factory varient @@ -17,9 +18,12 @@ ep end transient do + apple_hosted_audio_asset_container_id { "456" } api_response do build(:apple_episode_api_response, publishing_state: "PUBLISH", + item_guid: feeder_episode.item_guid, + apple_hosted_audio_asset_container_id: apple_hosted_audio_asset_container_id, apple_hosted_audio_state: Apple::Episode::AUDIO_ASSET_SUCCESS) end end diff --git a/test/models/apple/publisher_test.rb b/test/models/apple/publisher_test.rb index 20ef7da77..3be35700f 100644 --- a/test/models/apple/publisher_test.rb +++ b/test/models/apple/publisher_test.rb @@ -527,4 +527,21 @@ end end end + + describe "#update_audio_container_reference!" do + let(:episode) { build(:uploaded_apple_episode, show: apple_publisher.show, apple_hosted_audio_asset_container_id: nil) } + + it "updates container references for episodes" do + assert episode.has_unlinked_container? + + mock_result = episode.apple_sync_log.api_response.deep_dup + mock_result["api_response"]["val"]["data"]["attributes"]["appleHostedAudioAssetContainerId"] = "456" + + apple_publisher.api.stub(:bridge_remote_and_retry, [[mock_result], []]) do + apple_publisher.update_audio_container_reference!([episode]) + end + + refute episode.has_unlinked_container? + end + end end From 0a94b59a7850378af96317c0287639e9a3f24de4 Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Thu, 31 Oct 2024 10:46:47 -0500 Subject: [PATCH 32/71] fixup! Add the uploaded state marker --- test/models/apple/publisher_test.rb | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/models/apple/publisher_test.rb b/test/models/apple/publisher_test.rb index 3be35700f..7ed6019d6 100644 --- a/test/models/apple/publisher_test.rb +++ b/test/models/apple/publisher_test.rb @@ -528,6 +528,42 @@ end end + describe "#mark_as_uploaded!" do + let(:episode1) { build(:uploaded_apple_episode, show: apple_publisher.show) } + let(:episode2) { build(:uploaded_apple_episode, show: apple_publisher.show) } + let(:episodes) { [episode1, episode2] } + + it "marks episodes as uploaded" do + episodes.each do |ep| + refute ep.delivery_status.uploaded + end + + apple_publisher.mark_as_uploaded!(episodes) + + episodes.each do |ep| + assert ep.delivery_status.uploaded + end + end + end + + describe "#mark_as_delivered!" do + let(:episode1) { build(:apple_episode, show: apple_publisher.show) } + let(:episode2) { build(:apple_episode, show: apple_publisher.show) } + let(:episodes) { [episode1, episode2] } + + it "marks episodes as delivered" do + episodes.each do |ep| + refute ep.delivery_status.delivered + end + + apple_publisher.mark_as_delivered!(episodes) + + episodes.each do |ep| + assert ep.delivery_status.delivered + end + end + end + describe "#update_audio_container_reference!" do let(:episode) { build(:uploaded_apple_episode, show: apple_publisher.show, apple_hosted_audio_asset_container_id: nil) } From f9793b27783b70753e9b3fee0b0d215f34f81162 Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Thu, 31 Oct 2024 11:08:44 -0500 Subject: [PATCH 33/71] Break out methods for testing --- app/models/apple/episode.rb | 18 +++++-- app/models/apple/episode_delivery_status.rb | 6 ++- app/models/apple/publisher.rb | 59 ++++++++++++--------- app/models/concerns/apple_delivery.rb | 4 ++ test/models/apple/episode_test.rb | 17 +----- test/models/apple/publisher_test.rb | 44 ++++++++++++--- 6 files changed, 96 insertions(+), 52 deletions(-) diff --git a/app/models/apple/episode.rb b/app/models/apple/episode.rb index 9d8a2c801..b90668707 100644 --- a/app/models/apple/episode.rb +++ b/app/models/apple/episode.rb @@ -15,10 +15,8 @@ class Episode EPISODE_ASSET_WAIT_TIMEOUT = 15.minutes.freeze EPISODE_ASSET_WAIT_INTERVAL = 10.seconds.freeze - # Cleans up old delivery/delivery files iff the episode is to be delivered + # Cleans up old delivery/delivery files iff the episode is to be uploaded def self.prepare_for_delivery(episodes) - episodes = episodes.select { |ep| ep.needs_delivery? } - episodes.map do |ep| Rails.logger.info("Preparing episode #{ep.feeder_id} for delivery", {episode_id: ep.feeder_id}) ep.feeder_episode.apple_prepare_for_delivery! @@ -567,5 +565,19 @@ def apple_episode_delivery_statuses alias_method :delivery_files, :podcast_delivery_files alias_method :delivery_status, :apple_episode_delivery_status alias_method :delivery_statuses, :apple_episode_delivery_statuses + alias_method :apple_status, :apple_episode_delivery_status + + # Delegate methods to feeder_episode + def method_missing(method_name, *arguments, &block) + if feeder_episode.respond_to?(method_name) + feeder_episode.send(method_name, *arguments, &block) + else + super + end + end + + def respond_to_missing?(method_name, include_private = false) + feeder_episode.respond_to?(method_name) || super + end end end diff --git a/app/models/apple/episode_delivery_status.rb b/app/models/apple/episode_delivery_status.rb index 42416239f..eef9e84bb 100644 --- a/app/models/apple/episode_delivery_status.rb +++ b/app/models/apple/episode_delivery_status.rb @@ -30,12 +30,14 @@ def mark_as_not_uploaded! self.class.update_status(episode, uploaded: false) end + # Whether the media file has been uploaded to Apple + # is a subset of whether the episode has been delivered def mark_as_delivered! - self.class.update_status(episode, delivered: true) + self.class.update_status(episode, delivered: true, uploaded: true) end def mark_as_not_delivered! - self.class.update_status(episode, delivered: false) + self.class.update_status(episode, delivered: false, uploaded: false) end end end diff --git a/app/models/apple/publisher.rb b/app/models/apple/publisher.rb index a4e680ad1..43b30e540 100644 --- a/app/models/apple/publisher.rb +++ b/app/models/apple/publisher.rb @@ -131,41 +131,49 @@ def publish! def deliver_and_publish!(eps) Rails.logger.tagged("Apple::Publisher#deliver_and_publish!") do eps.each_slice(PUBLISH_CHUNK_LEN) do |eps| - # First upload the audio files eps.filter(&:apple_needs_upload?).tap do |eps| - # Soft delete any existing delivery and delivery files - prepare_for_delivery!(eps) + upload_media!(eps) + end - # only create if needed - sync_episodes!(eps) - sync_podcast_containers!(eps) + process_and_deliver!(eps) - wait_for_versioned_source_metadata(eps) + raise_delivery_processing_errors(eps) + end + end + end - sync_podcast_deliveries!(eps) - sync_podcast_delivery_files!(eps) + def upload_media!(eps) + # Soft delete any existing delivery and delivery files + prepare_for_delivery!(eps) - # upload and mark as uploaded, then update the audio container reference - execute_upload_operations!(eps) - mark_delivery_files_uploaded!(eps) - update_audio_container_reference!(eps) - # finally mark as uploaded - mark_as_uploaded!(eps) - end + # only create if needed + sync_episodes!(eps) + sync_podcast_containers!(eps) - increment_asset_wait!(eps) + wait_for_versioned_source_metadata(eps) - wait_for_upload_processing(eps) - wait_for_asset_state(eps) + sync_podcast_deliveries!(eps) + sync_podcast_delivery_files!(eps) - mark_as_delivered!(eps) + # upload and mark as uploaded, then update the audio container reference + execute_upload_operations!(eps) + mark_delivery_files_uploaded!(eps) + update_audio_container_reference!(eps) - publish_drafting!(eps) - reset_asset_wait!(eps) + # finally mark the episode as uploaded + mark_as_uploaded!(eps) + end - raise_delivery_processing_errors(eps) - end - end + def process_and_deliver!(eps) + increment_asset_wait!(eps) + + wait_for_upload_processing(eps) + wait_for_asset_state(eps) + + mark_as_delivered!(eps) + + publish_drafting!(eps) + reset_asset_wait!(eps) end def prepare_for_delivery!(eps) @@ -239,6 +247,7 @@ def wait_for_upload_processing(eps) def increment_asset_wait!(eps) Rails.logger.tagged("##{__method__}") do + eps = eps.filter { |e| e.feeder_episode.apple_status.uploaded? } eps.each { |ep| ep.apple_episode_delivery_status.increment_asset_wait } end end diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb index fe09c9f2d..02d3afbe3 100644 --- a/app/models/concerns/apple_delivery.rb +++ b/app/models/concerns/apple_delivery.rb @@ -37,6 +37,10 @@ def apple_needs_delivery? apple_episode_delivery_status.delivered == false end + def apple_needs_upload? + apple_episode_delivery_status.uploaded == false + end + def apple_mark_as_not_delivered! apple_episode_delivery_status.mark_as_not_delivered! end diff --git a/test/models/apple/episode_test.rb b/test/models/apple/episode_test.rb index 30e8d8feb..2cefff029 100644 --- a/test/models/apple/episode_test.rb +++ b/test/models/apple/episode_test.rb @@ -274,22 +274,7 @@ describe ".prepare_for_delivery" do it "should filter for episodes that need delivery" do - mock = Minitest::Mock.new - mock.expect(:call, true, []) - - apple_episode.feeder_episode.stub(:apple_prepare_for_delivery!, mock) do - apple_episode.stub(:needs_delivery?, true) do - assert_equal [apple_episode], Apple::Episode.prepare_for_delivery([apple_episode]) - end - end - - mock.verify - end - - it "should reject delivered episodes" do - apple_episode.stub(:needs_delivery?, false) do - assert_equal [], Apple::Episode.prepare_for_delivery([apple_episode]) - end + assert_equal [apple_episode], Apple::Episode.prepare_for_delivery([apple_episode]) end describe "soft deleting the delivery files" do diff --git a/test/models/apple/publisher_test.rb b/test/models/apple/publisher_test.rb index 7ed6019d6..390dfe60c 100644 --- a/test/models/apple/publisher_test.rb +++ b/test/models/apple/publisher_test.rb @@ -413,6 +413,7 @@ it "should increment asset wait count for each episode" do episodes.each do |ep| assert_equal 0, ep.apple_episode_delivery_status.asset_processing_attempts + ep.feeder_episode.apple_mark_as_uploaded! end apple_publisher.increment_asset_wait!(episodes) @@ -422,15 +423,12 @@ end end - it "should only increment the episodes that are still waiting" do + it "only increments the episodes that are still waiting" do assert 1, episode1.podcast_delivery_files.length assert 1, episode2.podcast_delivery_files.length - episode2.podcast_delivery_files.first.stub(:api_marked_as_uploaded?, false) do - episode1.podcast_delivery_files.first.stub(:api_marked_as_uploaded?, true) do - apple_publisher.increment_asset_wait!(episodes) - end - end + episode1.feeder_episode.apple_mark_as_uploaded! + apple_publisher.increment_asset_wait!(episodes) assert_equal [1, 0], [episode1, episode2].map { |ep| ep.apple_episode_delivery_status.asset_processing_attempts } end @@ -580,4 +578,38 @@ refute episode.has_unlinked_container? end end + + describe "#deliver_and_publish!" do + let(:episode) { build(:uploaded_apple_episode, show: apple_publisher.show) } + + it "skips upload for already uploaded episodes" do + episode.feeder_episode.apple_mark_as_uploaded! + + mock = Minitest::Mock.new + episode.feeder_episode.stub(:apple_prepare_for_delivery!, ->(*) { raise "Should not be called" }) do + mock.expect(:call, nil, [[]]) + apple_publisher.stub(:upload_media!, mock) do + apple_publisher.stub(:process_and_deliver!, ->(*) {}) do + apple_publisher.deliver_and_publish!([episode]) + end + end + end + + assert mock.verify + end + + it "processes uploads for non-uploaded episodes" do + refute episode.delivery_status.uploaded + + mock = Minitest::Mock.new + mock.expect(:call, nil, [[episode]]) + apple_publisher.stub(:upload_media!, mock) do + apple_publisher.stub(:process_and_deliver!, ->(*) {}) do + apple_publisher.deliver_and_publish!([episode]) + end + end + + mock.verify + end + end end From a6dfab1e70a1d01381e93941c8af45f3bdd83f5b Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Tue, 5 Nov 2024 09:28:34 -0600 Subject: [PATCH 34/71] Fix return value --- app/models/concerns/apple_delivery.rb | 2 +- test/models/concerns/apple_delivery_test.rb | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb index 02d3afbe3..fd99e1f8f 100644 --- a/app/models/concerns/apple_delivery.rb +++ b/app/models/concerns/apple_delivery.rb @@ -18,7 +18,7 @@ module AppleDelivery end def publish_to_apple? - podcast.apple_config&.publish_to_apple? + podcast.apple_config&.publish_to_apple? || false end def apple_update_delivery_status(attrs) diff --git a/test/models/concerns/apple_delivery_test.rb b/test/models/concerns/apple_delivery_test.rb index 285fdf858..c077587eb 100644 --- a/test/models/concerns/apple_delivery_test.rb +++ b/test/models/concerns/apple_delivery_test.rb @@ -75,4 +75,24 @@ class AppleDeliveryTest < ActiveSupport::TestCase assert_equal episode.apple_episode_delivery_statuses.last, result end end + + describe "#publish_to_apple?" do + let(:episode) { create(:episode) } + + it "returns false when podcast has no apple config" do + refute episode.publish_to_apple? + end + + it "returns false when apple config exists but publishing disabled" do + create(:apple_config, feed: create(:private_feed, podcast: episode.podcast), publish_enabled: false) + refute episode.publish_to_apple? + end + + it "returns true when apple config exists and publishing enabled" do + assert episode.publish_to_apple? == false + create(:apple_config, feed: create(:private_feed, podcast: episode.podcast), publish_enabled: true) + episode.podcast.reload + assert episode.publish_to_apple? + end + end end From 7ba1c29fa18fa6bdfd8be6c772490852aa39f19b Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Tue, 5 Nov 2024 09:29:04 -0600 Subject: [PATCH 35/71] Remove dupe methods --- app/models/concerns/apple_delivery.rb | 12 ------- test/models/concerns/apple_delivery_test.rb | 35 +++++++++++++++++++-- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb index fd99e1f8f..0ed1730a5 100644 --- a/app/models/concerns/apple_delivery.rb +++ b/app/models/concerns/apple_delivery.rb @@ -81,18 +81,6 @@ def measure_asset_processing_duration Time.now - start_status.created_at end - def apple_prepare_for_delivery! - # remove the previous delivery attempt (soft delete) - apple_podcast_deliveries.map(&:destroy) - apple_podcast_deliveries.reset - apple_podcast_delivery_files.reset - apple_podcast_container&.podcast_deliveries&.reset - end - - def apple_mark_for_reupload! - apple_needs_delivery! - end - def apple_episode return nil if !persisted? || !publish_to_apple? diff --git a/test/models/concerns/apple_delivery_test.rb b/test/models/concerns/apple_delivery_test.rb index c077587eb..239603576 100644 --- a/test/models/concerns/apple_delivery_test.rb +++ b/test/models/concerns/apple_delivery_test.rb @@ -18,16 +18,45 @@ class AppleDeliveryTest < ActiveSupport::TestCase end it "can be set to false" do - episode.apple_has_delivery! + episode.apple_mark_as_delivered! refute episode.apple_needs_delivery? end it "can be set to true" do - episode.apple_has_delivery! + episode.apple_mark_as_delivered! refute episode.apple_needs_delivery? # now set it to true - episode.apple_needs_delivery! + episode.apple_mark_as_not_delivered! + assert episode.apple_needs_delivery? + end + end + + describe "#apple_mark_as_delivered!" do + let(:episode) { create(:episode) } + + it "supercedes the uploaded status" do + episode.apple_mark_as_not_delivered! + + assert episode.apple_needs_upload? + assert episode.apple_needs_delivery? + + episode.apple_mark_as_delivered! + + refute episode.apple_needs_upload? + refute episode.apple_needs_delivery? + end + end + + describe "#apple_mark_as_uploaded!" do + it "sets the uploaded status" do + episode.apple_mark_as_uploaded! + assert episode.apple_episode_delivery_status.uploaded + refute episode.apple_needs_upload? + end + + it "does not interact with the delivery status" do + episode.apple_mark_as_uploaded! assert episode.apple_needs_delivery? end end From 53a95a065000d38aee3391f23352f031239f58bc Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Tue, 5 Nov 2024 09:29:34 -0600 Subject: [PATCH 36/71] Coverage for apple_prepare_for_delivery --- test/models/concerns/apple_delivery_test.rb | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/models/concerns/apple_delivery_test.rb b/test/models/concerns/apple_delivery_test.rb index 239603576..51bea7100 100644 --- a/test/models/concerns/apple_delivery_test.rb +++ b/test/models/concerns/apple_delivery_test.rb @@ -124,4 +124,29 @@ class AppleDeliveryTest < ActiveSupport::TestCase assert episode.publish_to_apple? end end + + describe "#apple_prepare_for_delivery!" do + let(:episode) { create(:episode) } + let(:container) { create(:apple_podcast_container, episode: episode) } + let(:delivery) { create(:apple_podcast_delivery, episode: episode, podcast_container: container) } + let(:delivery_file) { create(:apple_podcast_delivery_file, episode: episode, podcast_delivery: delivery) } + + before do + delivery_file # Create the delivery file + end + + it "soft deletes existing deliveries" do + assert_equal 1, episode.apple_podcast_deliveries.count + episode.apple_prepare_for_delivery! + assert_equal 0, episode.apple_podcast_deliveries.count + assert_equal 1, episode.apple_podcast_deliveries.with_deleted.count + end + + it "resets associations" do + episode.apple_prepare_for_delivery! + refute episode.apple_podcast_deliveries.loaded? + refute episode.apple_podcast_delivery_files.loaded? + refute episode.apple_podcast_container.podcast_deliveries.loaded? + end + end end From 3cc10c8efec75a3df249ae493329e2f40f917022 Mon Sep 17 00:00:00 2001 From: Sam Vevang Date: Tue, 5 Nov 2024 09:40:48 -0600 Subject: [PATCH 37/71] Bifucated uploaded/delivered states match --- db/migrate/20241029181938_new_file_uploaded_state.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db/migrate/20241029181938_new_file_uploaded_state.rb b/db/migrate/20241029181938_new_file_uploaded_state.rb index c5ca9b5e5..593f6593d 100644 --- a/db/migrate/20241029181938_new_file_uploaded_state.rb +++ b/db/migrate/20241029181938_new_file_uploaded_state.rb @@ -1,5 +1,12 @@ class NewFileUploadedState < ActiveRecord::Migration[7.2] def change add_column :apple_episode_delivery_statuses, :uploaded, :boolean, default: false + + # Set the new column to match the delivered column + execute(<<~SQL + UPDATE apple_episode_delivery_statuses + SET uploaded = delivered + SQL + ) end end From dd8a552aa1c47f1af3c34fde607e2449df165322 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 26 Nov 2024 11:58:37 -0500 Subject: [PATCH 38/71] Refactor pipeline and episode status for use in multiple integrations --- app/jobs/publish_apple_job.rb | 20 ------- app/jobs/publish_feed_job.rb | 29 ++++++---- app/models/apple/episode_delivery_status.rb | 43 -------------- app/models/concerns/apple_delivery.rb | 11 ++-- app/models/episode.rb | 3 +- app/models/feed.rb | 14 +++++ app/models/feeds/apple_subscription.rb | 20 +++++++ app/models/feeds/megaphone_feed.rb | 20 +++++++ app/models/integrations.rb | 7 +++ .../integrations/episode_delivery_status.rb | 46 +++++++++++++++ .../integrations/episode_integrations.rb | 17 ++++++ app/models/megaphone/config.rb | 4 ++ app/models/megaphone/publisher.rb | 6 ++ app/models/publishing_pipeline_state.rb | 17 +++--- .../20231101154016_backfill_media_versions.rb | 2 +- ...ivery_status_to_episode_delivery_status.rb | 16 ++++++ db/schema.rb | 37 ++++++------ .../apple_episode_delivery_status_factory.rb | 14 ----- test/factories/apple_episode_factory.rb | 4 +- .../episode_delivery_status_factory.rb | 43 ++++++++++++++ test/jobs/publish_apple_job_test.rb | 27 --------- test/jobs/publish_feed_job_test.rb | 57 ++++++++++--------- test/models/concerns/apple_delivery_test.rb | 2 +- .../episode_delivery_status_test.rb | 41 +++++++------ test/models/publishing_pipeline_state_test.rb | 16 +++--- 25 files changed, 313 insertions(+), 203 deletions(-) delete mode 100644 app/jobs/publish_apple_job.rb delete mode 100644 app/models/apple/episode_delivery_status.rb create mode 100644 app/models/integrations.rb create mode 100644 app/models/integrations/episode_delivery_status.rb create mode 100644 app/models/integrations/episode_integrations.rb create mode 100644 app/models/megaphone/publisher.rb create mode 100644 db/migrate/20241123174426_rename_apple_episode_delivery_status_to_episode_delivery_status.rb delete mode 100644 test/factories/apple_episode_delivery_status_factory.rb create mode 100644 test/factories/episode_delivery_status_factory.rb delete mode 100644 test/jobs/publish_apple_job_test.rb rename test/models/{apple => integrations}/episode_delivery_status_test.rb (85%) diff --git a/app/jobs/publish_apple_job.rb b/app/jobs/publish_apple_job.rb deleted file mode 100644 index ba43a635d..000000000 --- a/app/jobs/publish_apple_job.rb +++ /dev/null @@ -1,20 +0,0 @@ -class PublishAppleJob < ApplicationJob - queue_as :feeder_publishing - - def self.publish_to_apple(apple_config) - apple_config.build_publisher.publish! - end - - def self.do_perform(apple_config) - if !apple_config.publish_to_apple? - logger.info "Skipping publish to apple for #{apple_config.class.name} #{apple_config.id}" - return - end - - publish_to_apple(apple_config) - end - - def perform(apple_config) - self.class.do_perform(apple_config) - end -end diff --git a/app/jobs/publish_feed_job.rb b/app/jobs/publish_feed_job.rb index d25f9e271..eba1f5e2d 100644 --- a/app/jobs/publish_feed_job.rb +++ b/app/jobs/publish_feed_job.rb @@ -12,11 +12,17 @@ def perform(podcast, pub_item) # grab the current publishing pipeline. return :null if null_publishing_item?(podcast, pub_item) return :mismatched if mismatched_publishing_item?(podcast, pub_item) + Rails.logger.info("Starting publishing pipeline via PublishFeedJob", {podcast_id: podcast.id, publishing_queue_item_id: pub_item.id}) PublishingPipelineState.start!(podcast) - podcast.feeds.each { |feed| publish_apple(podcast, feed) } + + # Publish each integration for each feed (e.g. apple, megaphone) + podcast.feeds.each { |feed| publish_integration(podcast, feed) } + + # After integrations, publish RSS, if appropriate podcast.feeds.each { |feed| publish_rss(podcast, feed) } + PublishingPipelineState.complete!(podcast) rescue Apple::AssetStateTimeoutError => e fail_state(podcast, "apple_timeout", e) @@ -26,18 +32,17 @@ def perform(podcast, pub_item) PublishingPipelineState.settle_remaining!(podcast) end - def publish_apple(podcast, feed) - return unless feed.publish_to_apple? - - res = PublishAppleJob.do_perform(podcast.apple_config) - PublishingPipelineState.publish_apple!(podcast) + def publish_integration(podcast, feed) + return unless feed.publish_integration? + res = feed.publish_integration! + PublishingPipelineState.publish_integration!(podcast) res rescue => e - if podcast.apple_config.sync_blocks_rss - fail_state(podcast, "apple", e) + if feed.config.sync_blocks_rss + fail_state(podcast, feed.integration_type, e) else - Rails.logger.error("Error publishing to Apple, but continuing to publish RSS", {podcast_id: podcast.id, error: e.message}) - PublishingPipelineState.error_apple!(podcast) + Rails.logger.error("Error publishing to #{feed.integration_type}, but continuing to publish RSS", {podcast_id: podcast.id, error: e.message}) + PublishingPipelineState.error_integration!(podcast) end end @@ -51,9 +56,9 @@ def publish_rss(podcast, feed) def fail_state(podcast, type, error) (pipeline_method, log_level) = case type - when "apple" then [:error_apple!, :warn] - when "rss" then [:error_rss!, :warn] when "apple_timeout", "error" then [:error!, :error] + when "rss" then [:error_rss!, :warn] + else [:error_integration!, :warn] end PublishingPipelineState.public_send(pipeline_method, podcast) diff --git a/app/models/apple/episode_delivery_status.rb b/app/models/apple/episode_delivery_status.rb deleted file mode 100644 index eef9e84bb..000000000 --- a/app/models/apple/episode_delivery_status.rb +++ /dev/null @@ -1,43 +0,0 @@ -module Apple - class EpisodeDeliveryStatus < ApplicationRecord - belongs_to :episode, -> { with_deleted }, class_name: "::Episode" - - def self.update_status(episode, attrs) - new_status = (episode.apple_episode_delivery_status&.dup || default_status(episode)) - new_status.assign_attributes(attrs) - new_status.save! - episode.apple_episode_delivery_statuses.reset - new_status - end - - def self.default_status(episode) - new(episode: episode) - end - - def increment_asset_wait - self.class.update_status(episode, asset_processing_attempts: (asset_processing_attempts || 0) + 1) - end - - def reset_asset_wait - self.class.update_status(episode, asset_processing_attempts: 0) - end - - def mark_as_uploaded! - self.class.update_status(episode, uploaded: true) - end - - def mark_as_not_uploaded! - self.class.update_status(episode, uploaded: false) - end - - # Whether the media file has been uploaded to Apple - # is a subset of whether the episode has been delivered - def mark_as_delivered! - self.class.update_status(episode, delivered: true, uploaded: true) - end - - def mark_as_not_delivered! - self.class.update_status(episode, delivered: false, uploaded: false) - end - end -end diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb index 0ed1730a5..394f2568d 100644 --- a/app/models/concerns/apple_delivery.rb +++ b/app/models/concerns/apple_delivery.rb @@ -11,7 +11,6 @@ module AppleDelivery class_name: "Apple::PodcastDelivery" has_many :apple_podcast_delivery_files, through: :apple_podcast_deliveries, source: :podcast_delivery_files, class_name: "Apple::PodcastDeliveryFile" - has_many :apple_episode_delivery_statuses, -> { order(created_at: :desc) }, class_name: "Apple::EpisodeDeliveryStatus" alias_method :podcast_container, :apple_podcast_container alias_method :apple_status, :apple_episode_delivery_status @@ -22,15 +21,19 @@ def publish_to_apple? end def apple_update_delivery_status(attrs) - Apple::EpisodeDeliveryStatus.update_status(self, attrs) + update_episode_delivery_status(:apple, attrs) + end + + def apple_episode_delivery_statuses + episode_delivery_statuses.apple end def build_initial_delivery_status - Apple::EpisodeDeliveryStatus.default_status(self) + Integrations::EpisodeDeliveryStatus.default_status(:apple, self) end def apple_episode_delivery_status - apple_episode_delivery_statuses.order(created_at: :desc).first || build_initial_delivery_status + episode_delivery_status(:apple) || build_initial_delivery_status end def apple_needs_delivery? diff --git a/app/models/episode.rb b/app/models/episode.rb index 36e9826e1..d9dde682d 100644 --- a/app/models/episode.rb +++ b/app/models/episode.rb @@ -8,6 +8,7 @@ class Episode < ApplicationRecord include EpisodeFilters include EpisodeHasFeeds include EpisodeMedia + include Integrations::EpisodeIntegrations include PublishingStatus include TextSanitizer include EmbedPlayerHelper @@ -245,7 +246,7 @@ def copy_media(force = false) def publish! Rails.logger.tagged("Episode#publish!") do - apple_mark_for_reupload! + feeds.each { |f| f.mark_as_not_delivered!(self) } podcast&.publish! end end diff --git a/app/models/feed.rb b/app/models/feed.rb index 8538400ae..77f6c2afc 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -70,6 +70,20 @@ def self.enclosure_template_default "https://#{ENV["DOVETAIL_HOST"]}{/podcast_id,feed_slug,guid,original_basename}{feed_extension}" end + def mark_as_not_delivered!(episode) + # for default / RSS feeds, don't do anything + # TODO: we could mark an episode needing to pulished in this RSS feed file + # then later check to see if it is published in the feed yet + # a la "where's my episode?" publish tracking + end + + def publish_integration? + false + end + + def publish_integration! + end + def set_defaults self.file_name ||= DEFAULT_FILE_NAME self.enclosure_template ||= Feed.enclosure_template_default diff --git a/app/models/feeds/apple_subscription.rb b/app/models/feeds/apple_subscription.rb index 6e1d9a96b..016ca28f4 100644 --- a/app/models/feeds/apple_subscription.rb +++ b/app/models/feeds/apple_subscription.rb @@ -25,6 +25,8 @@ class Feeds::AppleSubscription < Feed validate :must_be_private validate :must_have_token + alias_method :config, :apple_config + # for soft delete, need a unique slug to be able to make another def paranoia_destroy_attributes { @@ -33,6 +35,10 @@ def paranoia_destroy_attributes } end + def mark_as_not_delivered!(episode) + episode.apple_episode_delivery_status.mark_as_not_delivered! + end + def set_defaults self.slug ||= DEFAULT_FEED_SLUG self.title ||= DEFAULT_TITLE @@ -113,6 +119,20 @@ def apple? true end + def integration_type + :apple + end + + def publish_integration! + if publish_integration? + apple_config.build_publisher.publish! + end + end + + def publish_integration? + publish_to_apple? + end + def publish_to_apple? !!apple_config&.publish_to_apple? end diff --git a/app/models/feeds/megaphone_feed.rb b/app/models/feeds/megaphone_feed.rb index cb86c0b17..7d9b18fde 100644 --- a/app/models/feeds/megaphone_feed.rb +++ b/app/models/feeds/megaphone_feed.rb @@ -1,7 +1,27 @@ class Feeds::MegaphoneFeed < Feed has_one :megaphone_config, class_name: "::Megaphone::Config", inverse_of: :feed + alias_method :config, :megaphone_config + def self.model_name Feed.model_name end + + def integration_type + :megaphone + end + + def publish_integration? + megaphone_config&.publish_to_megaphone? + end + + def publish_integration! + if publish_integration? + megaphone_config.build_publisher.publish! + end + end + + def mark_as_not_delivered!(episode) + episode.episode_delivery_statuses.megaphone.first&.mark_as_not_delivered! + end end diff --git a/app/models/integrations.rb b/app/models/integrations.rb new file mode 100644 index 000000000..f52e375f3 --- /dev/null +++ b/app/models/integrations.rb @@ -0,0 +1,7 @@ +module Integrations + def self.table_name_prefix + "integrations_" + end + + INTEGRATIONS = %i[apple megaphone] +end diff --git a/app/models/integrations/episode_delivery_status.rb b/app/models/integrations/episode_delivery_status.rb new file mode 100644 index 000000000..ceaa58f44 --- /dev/null +++ b/app/models/integrations/episode_delivery_status.rb @@ -0,0 +1,46 @@ +module Integrations + class EpisodeDeliveryStatus < ApplicationRecord + belongs_to :episode, -> { with_deleted }, class_name: "::Episode" + + enum :integration, Integrations::INTEGRATIONS + + def self.update_status(integration, episode, attrs) + new_status = (episode.episode_delivery_status(integration)&.dup || default_status(integration, episode)) + attrs[:integration] = integration + new_status.assign_attributes(attrs) + new_status.save! + episode.episode_delivery_statuses.reset + new_status + end + + def self.default_status(integration, episode) + new(episode: episode, integration: integration) + end + + def increment_asset_wait + self.class.update_status(integration, episode, asset_processing_attempts: (asset_processing_attempts || 0) + 1) + end + + def reset_asset_wait + self.class.update_status(integration, episode, asset_processing_attempts: 0) + end + + def mark_as_uploaded! + self.class.update_status(integration, episode, uploaded: true) + end + + def mark_as_not_uploaded! + self.class.update_status(integration, episode, uploaded: false) + end + + # Whether the media file has been uploaded to the Integration + # is a subset of whether the episode has been delivered + def mark_as_delivered! + self.class.update_status(integration, episode, delivered: true, uploaded: true) + end + + def mark_as_not_delivered! + self.class.update_status(integration, episode, delivered: false, uploaded: false) + end + end +end diff --git a/app/models/integrations/episode_integrations.rb b/app/models/integrations/episode_integrations.rb new file mode 100644 index 000000000..28dc229d3 --- /dev/null +++ b/app/models/integrations/episode_integrations.rb @@ -0,0 +1,17 @@ +require "active_support/concern" + +module Integrations::EpisodeIntegrations + extend ActiveSupport::Concern + + included do + has_many :episode_delivery_statuses, -> { order(created_at: :desc) }, class_name: "Integrations::EpisodeDeliveryStatus" + end + + def episode_delivery_status(integration) + episode_delivery_statuses.order(created_at: :desc).send(integration.intern).first + end + + def update_episode_delivery_status(integration, attrs) + Integrations::EpisodeDeliveryStatus.update_status(integration, self, attrs) + end +end diff --git a/app/models/megaphone/config.rb b/app/models/megaphone/config.rb index 3d42b0cbe..f676b2e33 100644 --- a/app/models/megaphone/config.rb +++ b/app/models/megaphone/config.rb @@ -6,5 +6,9 @@ class Config < ApplicationRecord encrypts :token encrypts :network_id + + def publish_to_megaphone? + valid? && publish_enabled? + end end end diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb new file mode 100644 index 000000000..cbf95a6fc --- /dev/null +++ b/app/models/megaphone/publisher.rb @@ -0,0 +1,6 @@ +module Megaphone + class Publisher < Integrations::Base::Publisher + def publish! + end + end +end diff --git a/app/models/publishing_pipeline_state.rb b/app/models/publishing_pipeline_state.rb index 3fcb5baf7..b7d38ebff 100644 --- a/app/models/publishing_pipeline_state.rb +++ b/app/models/publishing_pipeline_state.rb @@ -48,11 +48,11 @@ class PublishingPipelineState < ApplicationRecord :created, :started, :published_rss, - :published_apple, + :published_integration, :complete, :error, :expired, - :error_apple, + :error_integration, :error_rss ] @@ -146,16 +146,17 @@ def self.publish_rss!(podcast) state_transition(podcast, :published_rss) end - def self.publish_apple!(podcast) - state_transition(podcast, :published_apple) + def self.error_rss!(podcast) + state_transition(podcast, :error_rss) end - def self.error_apple!(podcast) - state_transition(podcast, :error_apple) + # TODO: do something with the integration type? + def self.publish_integration!(podcast) + state_transition(podcast, :published_integration) end - def self.error_rss!(podcast) - state_transition(podcast, :error_rss) + def self.error_integration!(podcast) + state_transition(podcast, :error_integration) end def self.complete!(podcast) diff --git a/db/migrate/20231101154016_backfill_media_versions.rb b/db/migrate/20231101154016_backfill_media_versions.rb index 825de6378..04f127985 100644 --- a/db/migrate/20231101154016_backfill_media_versions.rb +++ b/db/migrate/20231101154016_backfill_media_versions.rb @@ -2,7 +2,7 @@ class BackfillMediaVersions < ActiveRecord::Migration[7.0] def change reversible do |dir| dir.up do - episodes = Episode.where(id: Apple::EpisodeDeliveryStatus.distinct.pluck(:episode_id)) + episodes = Episode.where(id: Integrations::EpisodeDeliveryStatus.distinct.pluck(:episode_id)) episodes.each do |episode| ds = episode.apple_episode_delivery_status diff --git a/db/migrate/20241123174426_rename_apple_episode_delivery_status_to_episode_delivery_status.rb b/db/migrate/20241123174426_rename_apple_episode_delivery_status_to_episode_delivery_status.rb new file mode 100644 index 000000000..e25a59463 --- /dev/null +++ b/db/migrate/20241123174426_rename_apple_episode_delivery_status_to_episode_delivery_status.rb @@ -0,0 +1,16 @@ +class RenameAppleEpisodeDeliveryStatusToEpisodeDeliveryStatus < ActiveRecord::Migration[7.2] + def up + rename_table :apple_episode_delivery_statuses, :integrations_episode_delivery_statuses + add_column :integrations_episode_delivery_statuses, :integration, :integer + execute(<<~SQL + UPDATE integrations_episode_delivery_statuses + SET integration = 0 + SQL + ) + end + + def down + rename_table :integrations_episode_delivery_statuses, :apple_episode_delivery_statuses + remove_column :apple_episode_delivery_statuses, :integration + end +end diff --git a/db/schema.rb b/db/schema.rb index 0c454fc14..07665e27d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_11_20_165556) do +ActiveRecord::Schema[7.2].define(version: 2024_11_23_174426) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" enable_extension "uuid-ossp" @@ -26,22 +26,6 @@ t.index ["key_id"], name: "index_apple_configs_on_key_id" end - create_table "apple_episode_delivery_statuses", force: :cascade do |t| - t.bigint "episode_id", null: false - t.boolean "delivered", default: false - t.datetime "created_at", null: false - t.string "source_url" - t.string "source_filename" - t.bigint "source_size" - t.text "enclosure_url" - t.integer "source_fetch_count", default: 0 - t.bigint "source_media_version_id" - t.integer "asset_processing_attempts", default: 0, null: false - t.boolean "uploaded", default: false - t.index ["episode_id", "created_at"], name: "index_apple_episode_delivery_statuses_on_episode_id_created_at", include: ["delivered", "id"] - t.index ["episode_id"], name: "index_apple_episode_delivery_statuses_on_episode_id" - end - create_table "apple_keys", force: :cascade do |t| t.string "provider_id" t.string "key_id" @@ -249,6 +233,23 @@ t.index ["podcast_id"], name: "index_feeds_on_podcast_id_default", unique: true, where: "(slug IS NULL)" end + create_table "integrations_episode_delivery_statuses", force: :cascade do |t| + t.bigint "episode_id", null: false + t.boolean "delivered", default: false + t.datetime "created_at", null: false + t.string "source_url" + t.string "source_filename" + t.bigint "source_size" + t.text "enclosure_url" + t.integer "source_fetch_count", default: 0 + t.bigint "source_media_version_id" + t.integer "asset_processing_attempts", default: 0, null: false + t.boolean "uploaded", default: false + t.integer "integration" + t.index ["episode_id", "created_at"], name: "index_apple_episode_delivery_statuses_on_episode_id_created_at", include: ["delivered", "id"] + t.index ["episode_id"], name: "index_integrations_episode_delivery_statuses_on_episode_id" + end + create_table "itunes_categories", id: :serial, force: :cascade do |t| t.datetime "created_at", precision: nil t.datetime "updated_at", precision: nil @@ -493,11 +494,11 @@ end add_foreign_key "apple_configs", "feeds" - add_foreign_key "apple_episode_delivery_statuses", "episodes" add_foreign_key "episode_imports", "podcast_imports" add_foreign_key "feed_images", "feeds" add_foreign_key "feed_tokens", "feeds" add_foreign_key "feeds", "podcasts" + add_foreign_key "integrations_episode_delivery_statuses", "episodes" add_foreign_key "itunes_images", "feeds" add_foreign_key "media_version_resources", "media_resources" add_foreign_key "media_version_resources", "media_versions" diff --git a/test/factories/apple_episode_delivery_status_factory.rb b/test/factories/apple_episode_delivery_status_factory.rb deleted file mode 100644 index 45a927b33..000000000 --- a/test/factories/apple_episode_delivery_status_factory.rb +++ /dev/null @@ -1,14 +0,0 @@ -FactoryBot.define do - factory :apple_episode_delivery_status, class: "Apple::EpisodeDeliveryStatus" do - association :episode - - delivered { false } - asset_processing_attempts { 0 } - source_url { "http://example.com/audio.mp3" } - source_size { 1_048_576 } # 1 MB - source_filename { "episode_audio.mp3" } - enclosure_url { "http://cdn.example.com/audio.mp3" } - source_media_version_id { 1 } - source_fetch_count { 0 } - end -end diff --git a/test/factories/apple_episode_factory.rb b/test/factories/apple_episode_factory.rb index 97052d6ca..5f05e87c3 100644 --- a/test/factories/apple_episode_factory.rb +++ b/test/factories/apple_episode_factory.rb @@ -13,9 +13,7 @@ # set a complete episode factory varient factory :uploaded_apple_episode do feeder_episode do - ep = create(:episode) - ep.apple_mark_as_delivered! - ep + create(:episode) end transient do apple_hosted_audio_asset_container_id { "456" } diff --git a/test/factories/episode_delivery_status_factory.rb b/test/factories/episode_delivery_status_factory.rb new file mode 100644 index 000000000..75d376a3d --- /dev/null +++ b/test/factories/episode_delivery_status_factory.rb @@ -0,0 +1,43 @@ +FactoryBot.define do + factory :episode_delivery_status, class: "Integrations::EpisodeDeliveryStatus" do + association :episode + + integration { Integrations::EpisodeDeliveryStatus.integrations[:apple] } + delivered { false } + asset_processing_attempts { 0 } + source_url { "http://example.com/audio.mp3" } + source_size { 1_048_576 } # 1 MB + source_filename { "episode_audio.mp3" } + enclosure_url { "http://cdn.example.com/audio.mp3" } + source_media_version_id { 1 } + source_fetch_count { 0 } + end + + factory :apple_episode_delivery_status, class: "Integrations::EpisodeDeliveryStatus" do + association :episode + + integration { Integrations::EpisodeDeliveryStatus.integrations[:apple] } + delivered { false } + asset_processing_attempts { 0 } + source_url { "http://example.com/audio.mp3" } + source_size { 1_048_576 } # 1 MB + source_filename { "episode_audio.mp3" } + enclosure_url { "http://cdn.example.com/audio.mp3" } + source_media_version_id { 1 } + source_fetch_count { 0 } + end + + factory :megaphone_episode_delivery_status, class: "Integrations::EpisodeDeliveryStatus" do + association :episode + + integration { Integrations::EpisodeDeliveryStatus.integrations[:megaphone] } + delivered { false } + asset_processing_attempts { 0 } + source_url { "http://example.com/audio.mp3" } + source_size { 1_048_576 } # 1 MB + source_filename { "episode_audio.mp3" } + enclosure_url { "http://cdn.example.com/audio.mp3" } + source_media_version_id { 1 } + source_fetch_count { 0 } + end +end diff --git a/test/jobs/publish_apple_job_test.rb b/test/jobs/publish_apple_job_test.rb deleted file mode 100644 index c5d6c7b10..000000000 --- a/test/jobs/publish_apple_job_test.rb +++ /dev/null @@ -1,27 +0,0 @@ -require "test_helper" - -describe PublishAppleJob do - let(:episode) { create(:episode, prx_uri: "/api/v1/stories/87683") } - let(:podcast) { episode.podcast } - let(:feed) { podcast.default_feed } - let(:apple_feed) { create(:apple_feed, podcast: podcast) } - - describe "publishing to apple" do - it "does not publish to apple unless publish_enabled?" do - apple_feed.apple_config.update(publish_enabled: false) - - # test that the `publish_to_apple` method is not called - PublishAppleJob.stub(:publish_to_apple, ->(x) { raise "should not be called" }) do - assert_nil PublishAppleJob.perform_now(apple_feed.apple_config) - end - end - - it "does publish to apple if publish_enabled?" do - apple_feed.apple_config.update(publish_enabled: true) - - PublishAppleJob.stub(:publish_to_apple, :it_published!) do - assert_equal :it_published!, PublishAppleJob.perform_now(apple_feed.apple_config) - end - end - end -end diff --git a/test/jobs/publish_feed_job_test.rb b/test/jobs/publish_feed_job_test.rb index f21445bbf..0110be627 100644 --- a/test/jobs/publish_feed_job_test.rb +++ b/test/jobs/publish_feed_job_test.rb @@ -76,7 +76,7 @@ pub_item = PublishingQueueItem.create(podcast: podcast) assert job.null_publishing_item?(podcast, pub_item) - PublishAppleJob.stub(:do_perform, :publishing_apple!) do + private_feed.stub(:publish_integration!, true) do assert_equal :null, job.perform(podcast, pub_item) end @@ -90,7 +90,7 @@ queue_item = PublishingPipelineState.start_pipeline!(podcast) refute job.null_publishing_item?(podcast, queue_item) - PublishAppleJob.stub(:do_perform, :publishing_apple!) do + private_feed.stub(:publish_integration!, true) do refute_equal :null, job.perform(podcast, queue_item) end end @@ -112,36 +112,38 @@ job.stub(:s3_client, stub_client) do pqi = PublishingPipelineState.start_pipeline!(podcast) # Simulate some method blowing up - PublishAppleJob.stub(:do_perform, ->(*, **) { raise "some apple error" }) do - assert_raises(RuntimeError) { job.perform(podcast, pqi) } - assert_equal ["created", "started", "error_apple", "error"], PublishingPipelineState.where(podcast: podcast).latest_pipelines.order(id: :asc).pluck(:status) + private_feed.stub(:publish_integration!, -> { raise "random apple error" }) do + podcast.stub(:feeds, [private_feed]) do + assert_raises(RuntimeError) { job.perform(podcast, pqi) } + assert_equal ["created", "started", "error_integration", "error"], PublishingPipelineState.where(podcast: podcast).latest_pipelines.order(id: :asc).pluck(:status) + end end end end end it "does not schedule publishing to apple if the apple config prevents it" do - podcast.apple_config.update!(publish_enabled: false) - assert_nil job.publish_apple(podcast, apple_feed) + apple_feed.apple_config.update!(publish_enabled: false) + assert_nil job.publish_integration(podcast, apple_feed) end it "does not schedule publishing to apple if the apple config is disabled" do - apple_config.update!(publish_enabled: false) - assert_nil job.publish_apple(podcast, apple_feed) + apple_feed.apple_config.update!(publish_enabled: false) + assert_nil job.publish_integration(podcast, apple_feed) end describe "when the apple config is present" do it "does not schedule publishing to apple if the config is marked as not publishable" do - podcast.apple_config.update!(publish_enabled: false) + apple_feed.apple_config.update!(publish_enabled: false) - assert_nil job.publish_apple(podcast, apple_feed) + assert_nil job.publish_integration(podcast, apple_feed) end it "does run the apple publishing if the config is present and marked as publishable" do assert apple_feed.apple_config.present? assert apple_feed.apple_config.publish_enabled - PublishAppleJob.stub(:do_perform, :publishing_apple!) do - assert_equal :publishing_apple!, job.publish_apple(podcast, apple_feed) + private_feed.stub(:publish_integration!, :publishing_apple!) do + assert_equal :publishing_apple!, job.publish_integration(podcast, apple_feed) end end @@ -155,10 +157,11 @@ assert apple_feed.apple_config.present? assert apple_feed.apple_config.publish_enabled - PublishAppleJob.stub(:do_perform, ->(*, **) { raise "some apple error" }) do - assert_raises(RuntimeError) { PublishingPipelineState.attempt!(feed.podcast, perform_later: false) } - - assert_equal ["created", "started", "error", "error_apple"].sort, PublishingPipelineState.where(podcast: feed.podcast).latest_pipelines.pluck(:status).sort + private_feed.stub(:publish_integration!, -> { raise "some apple error" }) do + feed.podcast.stub(:feeds, [podcast.public_feed, private_feed, feed]) do + assert_raises(RuntimeError) { PublishingPipelineState.attempt!(feed.podcast, perform_later: false) } + assert_equal ["created", "started", "error", "error_integration"].sort, PublishingPipelineState.where(podcast: feed.podcast).latest_pipelines.pluck(:status).sort + end end end @@ -169,11 +172,12 @@ feed.reload PublishFeedJob.stub(:s3_client, stub_client) do - PublishAppleJob.stub(:do_perform, ->(*, **) { raise "some apple error" }) do - # no error raised - PublishingPipelineState.attempt!(feed.podcast, perform_later: false) - - assert_equal ["created", "started", "error_apple", "published_rss", "published_rss", "published_rss", "complete"].sort, PublishingPipelineState.where(podcast: feed.podcast).latest_pipelines.pluck(:status).sort + private_feed.stub(:publish_integration!, -> { raise "some apple error" }) do + feed.podcast.stub(:feeds, [podcast.public_feed, private_feed, feed]) do + # no error raised + PublishingPipelineState.attempt!(feed.podcast, perform_later: false) + assert_equal ["created", "started", "error_integration", "published_rss", "published_rss", "published_rss", "complete"].sort, PublishingPipelineState.where(podcast: feed.podcast).latest_pipelines.pluck(:status).sort + end end end end @@ -182,10 +186,11 @@ assert apple_feed.apple_config.present? assert apple_feed.apple_config.publish_enabled - PublishAppleJob.stub(:do_perform, ->(*, **) { raise Apple::AssetStateTimeoutError.new([]) }) do - assert_raises(Apple::AssetStateTimeoutError) { PublishingPipelineState.attempt!(feed.podcast, perform_later: false) } - - assert_equal ["created", "started", "error", "error_apple"].sort, PublishingPipelineState.where(podcast: feed.podcast).latest_pipelines.pluck(:status).sort + private_feed.stub(:publish_integration!, -> { raise Apple::AssetStateTimeoutError.new([]) }) do + podcast.stub(:feeds, [private_feed]) do + assert_raises(Apple::AssetStateTimeoutError) { PublishingPipelineState.attempt!(feed.podcast, perform_later: false) } + assert_equal ["created", "started", "error", "error_integration"].sort, PublishingPipelineState.where(podcast: feed.podcast).latest_pipelines.pluck(:status).sort + end end end end diff --git a/test/models/concerns/apple_delivery_test.rb b/test/models/concerns/apple_delivery_test.rb index 51bea7100..da4ca1446 100644 --- a/test/models/concerns/apple_delivery_test.rb +++ b/test/models/concerns/apple_delivery_test.rb @@ -100,7 +100,7 @@ class AppleDeliveryTest < ActiveSupport::TestCase it "returns the new status" do result = episode.apple_status.increment_asset_wait - assert_instance_of Apple::EpisodeDeliveryStatus, result + assert_instance_of Integrations::EpisodeDeliveryStatus, result assert_equal episode.apple_episode_delivery_statuses.last, result end end diff --git a/test/models/apple/episode_delivery_status_test.rb b/test/models/integrations/episode_delivery_status_test.rb similarity index 85% rename from test/models/apple/episode_delivery_status_test.rb rename to test/models/integrations/episode_delivery_status_test.rb index ed1cc51cb..9c4709b66 100644 --- a/test/models/apple/episode_delivery_status_test.rb +++ b/test/models/integrations/episode_delivery_status_test.rb @@ -1,7 +1,7 @@ require "test_helper" -class Apple::EpisodeDeliveryStatusTest < ActiveSupport::TestCase - describe Apple::EpisodeDeliveryStatus do +class Integrations::EpisodeDeliveryStatusTest < ActiveSupport::TestCase + describe Integrations::EpisodeDeliveryStatus do let(:episode) { create(:episode) } let(:delivery_status) { create(:apple_episode_delivery_status, episode: episode) } @@ -13,7 +13,7 @@ class Apple::EpisodeDeliveryStatusTest < ActiveSupport::TestCase it "can belong to deleted episodes" do episode.destroy assert_equal episode, delivery_status.episode - assert_difference "Apple::EpisodeDeliveryStatus.count", +1 do + assert_difference "Integrations::EpisodeDeliveryStatus.count", +1 do episode.apple_mark_as_not_delivered! end assert_equal episode, episode.apple_episode_delivery_statuses.first.episode @@ -31,32 +31,37 @@ class Apple::EpisodeDeliveryStatusTest < ActiveSupport::TestCase describe "default values" do it "sets asset_processing_attempts to 0 by default" do - new_status = Apple::EpisodeDeliveryStatus.new + new_status = Integrations::EpisodeDeliveryStatus.new assert_equal 0, new_status.asset_processing_attempts end end describe ".update_status" do it "creates a new status when none exists" do - episode.apple_episode_delivery_statuses.destroy_all - assert_difference "Apple::EpisodeDeliveryStatus.count", 1 do - Apple::EpisodeDeliveryStatus.update_status(episode, delivered: true) + episode.episode_delivery_statuses.destroy_all + assert_difference "Integrations::EpisodeDeliveryStatus.count", 1 do + Integrations::EpisodeDeliveryStatus.update_status(:apple, episode, delivered: true) end end it "creates a new status even when one already exists" do _existing_status = create(:apple_episode_delivery_status, episode: episode) - assert_difference "Apple::EpisodeDeliveryStatus.count", 1 do - Apple::EpisodeDeliveryStatus.update_status(episode, delivered: false) + assert_difference "Integrations::EpisodeDeliveryStatus.count", 1 do + Integrations::EpisodeDeliveryStatus.update_status(:apple, episode, delivered: false) end end it "updates attributes of the new status" do - new_status = Apple::EpisodeDeliveryStatus.update_status(episode, - delivered: true, - source_url: "http://example.com/audio.mp3", - source_size: 1024, - source_filename: "audio.mp3") + new_status = Integrations::EpisodeDeliveryStatus.update_status( + :apple, + episode, + { + delivered: true, + source_url: "http://example.com/audio.mp3", + source_size: 1024, + source_filename: "audio.mp3" + } + ) assert new_status.delivered assert_equal "http://example.com/audio.mp3", new_status.source_url @@ -66,8 +71,8 @@ class Apple::EpisodeDeliveryStatusTest < ActiveSupport::TestCase it "resets the episode's apple_episode_delivery_statuses association" do episode.apple_episode_delivery_statuses.load - Apple::EpisodeDeliveryStatus.update_status(episode, delivered: true) - refute episode.apple_episode_delivery_statuses.loaded? + Integrations::EpisodeDeliveryStatus.update_status(:apple, episode, delivered: true) + refute episode.episode_delivery_statuses.loaded? end end @@ -90,7 +95,7 @@ class Apple::EpisodeDeliveryStatusTest < ActiveSupport::TestCase it "creates a new status entry" do assert delivery_status.asset_processing_attempts.zero? - assert_difference "Apple::EpisodeDeliveryStatus.count", 1 do + assert_difference "Integrations::EpisodeDeliveryStatus.count", 1 do delivery_status.increment_asset_wait end end @@ -112,7 +117,7 @@ class Apple::EpisodeDeliveryStatusTest < ActiveSupport::TestCase it "creates a new status entry" do assert delivery_status.asset_processing_attempts.zero? - assert_difference "Apple::EpisodeDeliveryStatus.count", 1 do + assert_difference "Integrations::EpisodeDeliveryStatus.count", 1 do delivery_status.reset_asset_wait end end diff --git a/test/models/publishing_pipeline_state_test.rb b/test/models/publishing_pipeline_state_test.rb index 820524a6d..a0f0687b0 100644 --- a/test/models/publishing_pipeline_state_test.rb +++ b/test/models/publishing_pipeline_state_test.rb @@ -253,7 +253,7 @@ it 'sets the status to "error"' do pqi = nil PublishFeedJob.stub_any_instance(:save_file, nil) do - PublishFeedJob.stub_any_instance(:publish_apple, ->(*args) { raise "error" }) do + PublishFeedJob.stub_any_instance(:publish_integration, ->(*args) { raise "error" }) do pqi = PublishingQueueItem.ensure_queued!(podcast) assert_raises(RuntimeError) { PublishingPipelineState.attempt!(podcast, perform_later: false) } @@ -276,7 +276,7 @@ describe "complete!" do it 'sets the status to "complete"' do PublishFeedJob.stub_any_instance(:save_file, nil) do - PublishFeedJob.stub_any_instance(:publish_apple, "pub!") do + PublishFeedJob.stub_any_instance(:publish_integration, "pub!") do PublishFeedJob.stub_any_instance(:publish_rss, "pub!") do PublishingPipelineState.attempt!(podcast, perform_later: false) end @@ -291,7 +291,7 @@ PublishingQueueItem.create!(podcast: podcast) PublishFeedJob.stub_any_instance(:save_file, nil) do - PublishFeedJob.stub_any_instance(:publish_apple, "pub!") do + PublishFeedJob.stub_any_instance(:publish_integration, "pub!") do PublishFeedJob.stub_any_instance(:publish_rss, "pub!") do PublishingPipelineState.attempt!(podcast, perform_later: false) end @@ -315,14 +315,16 @@ it "can publish via the apple configs" do assert [f1, f2, f3] - PublishAppleJob.stub(:do_perform, "published apple!") do - PublishFeedJob.stub_any_instance(:save_file, "saved rss!") do - PublishingPipelineState.attempt!(podcast, perform_later: false) + f3.stub(:publish_integration!, "published apple!") do + podcast.stub(:feeds, [f1, f2, f3]) do + PublishFeedJob.stub_any_instance(:save_file, "saved rss!") do + PublishingPipelineState.attempt!(podcast, perform_later: false) + end end end PublishingPipelineState.complete!(podcast) assert_equal( - ["complete", "published_rss", "published_rss", "published_rss", "published_apple", "started", "created"], + ["complete", "published_rss", "published_rss", "published_rss", "published_integration", "started", "created"], PublishingPipelineState.order(id: :desc).pluck(:status) ) end From 1c01abb9bc46e9346c46b07033d9938d17ca47e1 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 26 Nov 2024 18:22:08 -0500 Subject: [PATCH 39/71] Initial megaphone feed UI --- app/controllers/feeds_controller.rb | 12 +++- app/helpers/feeds_helper.rb | 4 ++ app/models/feed.rb | 4 ++ app/models/feeds/megaphone_feed.rb | 10 ++- app/policies/feed_policy.rb | 4 ++ app/policies/megaphone/config_policy.rb | 17 +++++ app/views/feeds/_form.html.erb | 8 ++- .../feeds/_form_megaphone_config.html.erb | 66 +++++++++++++++++++ app/views/feeds/_tabs.html.erb | 9 ++- config/routes.rb | 1 + ...dd_sync_blocks_rss_to_megaphone_configs.rb | 5 ++ db/schema.rb | 3 +- 12 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 app/policies/megaphone/config_policy.rb create mode 100644 app/views/feeds/_form_megaphone_config.html.erb create mode 100644 db/migrate/20241126230612_add_sync_blocks_rss_to_megaphone_configs.rb diff --git a/app/controllers/feeds_controller.rb b/app/controllers/feeds_controller.rb index 151cb23ad..1e3e9d729 100644 --- a/app/controllers/feeds_controller.rb +++ b/app/controllers/feeds_controller.rb @@ -39,6 +39,15 @@ def new_apple render "new" end + def new_megaphone + @feed = Feeds::MegaphoneFeed.new(podcast: @podcast, private: true) + @feed.build_megaphone_config + authorize @feed + + @feed.assign_attributes(feed_params) + render "new" + end + # POST /feeds def create @feed = @podcast.feeds.new(feed_params) @@ -153,7 +162,8 @@ def nilified_feed_params feed_tokens_attributes: %i[id label token _destroy], feed_images_attributes: %i[id original_url size alt_text caption credit _destroy _retry], itunes_images_attributes: %i[id original_url size alt_text caption credit _destroy _retry], - apple_config_attributes: [:id, :publish_enabled, :sync_blocks_rss, {key_attributes: %i[id provider_id key_id key_pem_b64]}] + apple_config_attributes: [:id, :publish_enabled, :sync_blocks_rss, {key_attributes: %i[id provider_id key_id key_pem_b64]}], + megaphone_config_attributes: [:id, :publish_enabled, :sync_blocks_rss, :network_id, :network_name, :token] ) end end diff --git a/app/helpers/feeds_helper.rb b/app/helpers/feeds_helper.rb index c101bbd6d..926308846 100644 --- a/app/helpers/feeds_helper.rb +++ b/app/helpers/feeds_helper.rb @@ -72,4 +72,8 @@ def feed_retry_image_path(feed, form) def apple_feed?(feed) feed.type == "Feeds::AppleSubscription" end + + def megaphone_feed?(feed) + feed.type == "Feeds::MegaphoneFeed" + end end diff --git a/app/models/feed.rb b/app/models/feed.rb index 77f6c2afc..a5767ec7b 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -77,6 +77,10 @@ def mark_as_not_delivered!(episode) # a la "where's my episode?" publish tracking end + def integration_type + nil + end + def publish_integration? false end diff --git a/app/models/feeds/megaphone_feed.rb b/app/models/feeds/megaphone_feed.rb index 7d9b18fde..81bee6f9b 100644 --- a/app/models/feeds/megaphone_feed.rb +++ b/app/models/feeds/megaphone_feed.rb @@ -1,8 +1,12 @@ class Feeds::MegaphoneFeed < Feed - has_one :megaphone_config, class_name: "::Megaphone::Config", inverse_of: :feed + has_one :megaphone_config, class_name: "::Megaphone::Config", dependent: :destroy, autosave: true, validate: true, inverse_of: :feed + + after_initialize :set_defaults alias_method :config, :megaphone_config + accepts_nested_attributes_for :megaphone_config, allow_destroy: true, reject_if: :all_blank + def self.model_name Feed.model_name end @@ -11,6 +15,10 @@ def integration_type :megaphone end + def set_defaults + self.slug ||= "megaphone" + end + def publish_integration? megaphone_config&.publish_to_megaphone? end diff --git a/app/policies/feed_policy.rb b/app/policies/feed_policy.rb index 5b60bf395..34552abda 100644 --- a/app/policies/feed_policy.rb +++ b/app/policies/feed_policy.rb @@ -15,6 +15,10 @@ def new_apple? update? end + def new_megaphone? + update? + end + def update? PodcastPolicy.new(token, resource.podcast).update? && !resource.edit_locked? end diff --git a/app/policies/megaphone/config_policy.rb b/app/policies/megaphone/config_policy.rb new file mode 100644 index 000000000..1ab3e04b5 --- /dev/null +++ b/app/policies/megaphone/config_policy.rb @@ -0,0 +1,17 @@ +class Megaphone::ConfigPolicy < ApplicationPolicy + def new? + create? + end + + def show? + FeedPolicy.new(token, resource.feed).show? + end + + def create? + FeedPolicy.new(token, resource.feed).create? + end + + def update? + FeedPolicy.new(token, resource.feed).update? + end +end diff --git a/app/views/feeds/_form.html.erb b/app/views/feeds/_form.html.erb index 8575394ce..d4a13ea46 100644 --- a/app/views/feeds/_form.html.erb +++ b/app/views/feeds/_form.html.erb @@ -24,6 +24,12 @@ <%= render "form_audio_format", podcast: podcast, feed: feed, form: form %> <%= render "form_ad_zones", podcast: podcast, feed: feed, form: form %> <% end %> + <% elsif megaphone_feed?(feed) %> + <%= render "form_megaphone_config", podcast: podcast, feed: feed, form: form %> + <% if feed.persisted? %> + <%= render "form_audio_format", podcast: podcast, feed: feed, form: form %> + <%= render "form_ad_zones", podcast: podcast, feed: feed, form: form %> + <% end %> <% else %> <%# custom feeds %> <%= render "form_main", podcast: podcast, feed: feed, form: form %> <%= render "form_auth", podcast: podcast, feed: feed, form: form %> @@ -38,7 +44,7 @@
<%= render "form_status", podcast: podcast, feed: feed, form: form %> - <% unless apple_feed?(feed) %> + <% unless (apple_feed?(feed) || megaphone_feed?(feed)) %> <%= render "form_distribution", podcast: podcast, feed: feed, form: form %> <% end %>
diff --git a/app/views/feeds/_form_megaphone_config.html.erb b/app/views/feeds/_form_megaphone_config.html.erb new file mode 100644 index 000000000..2936d0b2b --- /dev/null +++ b/app/views/feeds/_form_megaphone_config.html.erb @@ -0,0 +1,66 @@ +
+
+
+

<%= t(".title") %>

+
+
+
+

<%= t(".description") %>

+ <%= t(".guide_link").html_safe %> +
+ <%= form.fields_for :megaphone_config do |config_form| %> +
+
+ <%= config_form.text_field :token %> + <%= config_form.label :token, "API Token", required: true %> + <%= field_help_text t(".help.token") %> +
+
+
+
+ <%= config_form.text_field :network_id %> + <%= config_form.label :network_id, "Network ID", required: true %> + <%= field_help_text t(".help.network_id") %> +
+
+
+
+ <%= config_form.text_field :network_name %> + <%= config_form.label :network_name, "Network Name"%> + <%= field_help_text t(".help.network_name") %> +
+
+ <% if config_form.object.persisted? %> +
+
+ <%= config_form.check_box :publish_enabled %> +
+ <%= config_form.label :publish_enabled %> + <%= help_text t(".help.publish_enabled") %> +
+
+
+ <%= config_form.check_box :sync_blocks_rss %> +
+ <%= config_form.label :sync_blocks_rss %> + <%= help_text t(".help.sync_blocks_rss") %> +
+
+
+ <% end %> + <% end %> + <% if form.object.megaphone_config&.persisted? %> +
+
+ <%= form.number_field :display_episodes_count %> + <%= form.label :display_episodes_count %> + <%= field_help_text t(".help.display_episodes_count") %> +
+
+ <% end %> + + <%= form.hidden_field :type, value: "Feeds::MegaphoneFeed" %> + +
+
+
diff --git a/app/views/feeds/_tabs.html.erb b/app/views/feeds/_tabs.html.erb index 81d2e1f05..144ecdbb2 100644 --- a/app/views/feeds/_tabs.html.erb +++ b/app/views/feeds/_tabs.html.erb @@ -12,12 +12,17 @@
<%= link_to "Add a Feed", new_podcast_feed_path(podcast), class: "btn btn-success flex-grow-1" %> - <% if @feeds.none?(&:apple?) %> + <% if @feeds.none?(&:integration_type) %>
    -
  • <%= link_to "Add Apple Feed", new_apple_podcast_feeds_path(podcast), class: "btn btn-success bg-success text-white shadow w-100" %>
    • + <% if @feeds.none? { |f| f.integration_type == :apple} %> +
  • <%= link_to "Add Apple Feed", new_apple_podcast_feeds_path(podcast), class: "btn btn-success bg-success text-white shadow w-100" %>
    • + <% end %> + <% if @feeds.none? { |f| f.integration_type == :megaphone} %> +
  • <%= link_to "Add Megaphone Feed", new_megaphone_podcast_feeds_path(podcast), class: "btn btn-success bg-success text-white shadow w-100" %>
    • + <% end %>
<% end %>
diff --git a/config/routes.rb b/config/routes.rb index 13510e6d8..8f9e6b400 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -6,6 +6,7 @@ resource :planner, only: [:show, :create], controller: :podcast_planner resources :feeds, except: [:edit] do get "new_apple", on: :collection + get "new_megaphone", on: :collection end resources :episodes, only: [:index, :create, :new] resources :placements_preview, only: [:show] diff --git a/db/migrate/20241126230612_add_sync_blocks_rss_to_megaphone_configs.rb b/db/migrate/20241126230612_add_sync_blocks_rss_to_megaphone_configs.rb new file mode 100644 index 000000000..ec0ef6de5 --- /dev/null +++ b/db/migrate/20241126230612_add_sync_blocks_rss_to_megaphone_configs.rb @@ -0,0 +1,5 @@ +class AddSyncBlocksRssToMegaphoneConfigs < ActiveRecord::Migration[7.2] + def change + add_column :megaphone_configs, :sync_blocks_rss, :boolean, default: false, null: false + end +end diff --git a/db/schema.rb b/db/schema.rb index 07665e27d..71bd21355 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_11_23_174426) do +ActiveRecord::Schema[7.2].define(version: 2024_11_26_230612) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" enable_extension "uuid-ossp" @@ -333,6 +333,7 @@ t.bigint "feed_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.boolean "sync_blocks_rss", default: false, null: false end create_table "podcast_imports", force: :cascade do |t| From e2114cadf83a1abbee05d6d3c056470183a6b869 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 26 Nov 2024 18:43:49 -0500 Subject: [PATCH 40/71] Fix tests --- app/models/megaphone/config.rb | 2 +- test/factories/feed_factory.rb | 5 ++--- test/factories/megaphone_config_factory.rb | 3 +++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/models/megaphone/config.rb b/app/models/megaphone/config.rb index f676b2e33..77a440dce 100644 --- a/app/models/megaphone/config.rb +++ b/app/models/megaphone/config.rb @@ -2,7 +2,7 @@ module Megaphone class Config < ApplicationRecord belongs_to :feed - validates_presence_of :token, :network_id, :feed_id + validates_presence_of :token, :network_id encrypts :token encrypts :network_id diff --git a/test/factories/feed_factory.rb b/test/factories/feed_factory.rb index 6d8ce2c36..d2255eaff 100644 --- a/test/factories/feed_factory.rb +++ b/test/factories/feed_factory.rb @@ -56,11 +56,10 @@ factory :megaphone_feed, class: "Feeds::MegaphoneFeed" do type { "Feeds::MegaphoneFeed" } - private { false } - sequence(:slug) { |n| "mp-feed-#{n}" } + private { true } after(:build) do |feed, _evaluator| - feed.megaphone_config = build(:megaphone_config) + feed.megaphone_config = build(:megaphone_config, feed: feed) end end end diff --git a/test/factories/megaphone_config_factory.rb b/test/factories/megaphone_config_factory.rb index 40352cb78..9aafc0783 100644 --- a/test/factories/megaphone_config_factory.rb +++ b/test/factories/megaphone_config_factory.rb @@ -1,7 +1,10 @@ FactoryBot.define do factory :megaphone_config, class: Megaphone::Config do + publish_enabled { true } + sync_blocks_rss { true } token { "thisisatokenforacessingtheapi" } network_id { "this-is-a-network-id" } + network_name { "test network" } feed end end From 9f5bfc1bc2587577b191e4805b37b05835a57dcf Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 26 Nov 2024 18:54:05 -0500 Subject: [PATCH 41/71] Linting issue --- app/views/feeds/_form_megaphone_config.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/feeds/_form_megaphone_config.html.erb b/app/views/feeds/_form_megaphone_config.html.erb index 2936d0b2b..0fa2d1ef4 100644 --- a/app/views/feeds/_form_megaphone_config.html.erb +++ b/app/views/feeds/_form_megaphone_config.html.erb @@ -26,7 +26,7 @@
<%= config_form.text_field :network_name %> - <%= config_form.label :network_name, "Network Name"%> + <%= config_form.label :network_name, "Network Name" %> <%= field_help_text t(".help.network_name") %>
From 484c193e28078059862e8fb33a5e8072c8d115d6 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 26 Nov 2024 19:17:56 -0500 Subject: [PATCH 42/71] Fix saving and deleting megaphone feed config --- app/models/feeds/megaphone_feed.rb | 6 +++- app/views/feeds/_form.html.erb | 4 --- .../feeds/_form_megaphone_config.html.erb | 35 +++++++------------ 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/app/models/feeds/megaphone_feed.rb b/app/models/feeds/megaphone_feed.rb index 81bee6f9b..63c7ee913 100644 --- a/app/models/feeds/megaphone_feed.rb +++ b/app/models/feeds/megaphone_feed.rb @@ -16,7 +16,11 @@ def integration_type end def set_defaults - self.slug ||= "megaphone" + self.slug ||= "Megaphone" + self.title ||= "Megaphone" + self.private = true + + super end def publish_integration? diff --git a/app/views/feeds/_form.html.erb b/app/views/feeds/_form.html.erb index d4a13ea46..6583d3cd6 100644 --- a/app/views/feeds/_form.html.erb +++ b/app/views/feeds/_form.html.erb @@ -26,10 +26,6 @@ <% end %> <% elsif megaphone_feed?(feed) %> <%= render "form_megaphone_config", podcast: podcast, feed: feed, form: form %> - <% if feed.persisted? %> - <%= render "form_audio_format", podcast: podcast, feed: feed, form: form %> - <%= render "form_ad_zones", podcast: podcast, feed: feed, form: form %> - <% end %> <% else %> <%# custom feeds %> <%= render "form_main", podcast: podcast, feed: feed, form: form %> <%= render "form_auth", podcast: podcast, feed: feed, form: form %> diff --git a/app/views/feeds/_form_megaphone_config.html.erb b/app/views/feeds/_form_megaphone_config.html.erb index 0fa2d1ef4..b1c77f3a0 100644 --- a/app/views/feeds/_form_megaphone_config.html.erb +++ b/app/views/feeds/_form_megaphone_config.html.erb @@ -30,31 +30,20 @@ <%= field_help_text t(".help.network_name") %>
- <% if config_form.object.persisted? %> -
-
- <%= config_form.check_box :publish_enabled %> -
- <%= config_form.label :publish_enabled %> - <%= help_text t(".help.publish_enabled") %> -
-
-
- <%= config_form.check_box :sync_blocks_rss %> -
- <%= config_form.label :sync_blocks_rss %> - <%= help_text t(".help.sync_blocks_rss") %> -
+
+
+ <%= config_form.check_box :publish_enabled %> +
+ <%= config_form.label :publish_enabled %> + <%= help_text t(".help.publish_enabled") %>
- <% end %> - <% end %> - <% if form.object.megaphone_config&.persisted? %> -
-
- <%= form.number_field :display_episodes_count %> - <%= form.label :display_episodes_count %> - <%= field_help_text t(".help.display_episodes_count") %> +
+ <%= config_form.check_box :sync_blocks_rss %> +
+ <%= config_form.label :sync_blocks_rss %> + <%= help_text t(".help.sync_blocks_rss") %> +
<% end %> From 732dee89c5badcc00c18911800d3205bbc7d273d Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Wed, 27 Nov 2024 15:26:14 -0500 Subject: [PATCH 43/71] Refactor sync log to include integration type --- app/models/apple/config.rb | 2 +- app/models/apple/episode.rb | 10 ++++++++-- app/models/apple/podcast_container.rb | 4 ++-- app/models/apple/podcast_delivery.rb | 4 ++-- app/models/apple/podcast_delivery_file.rb | 6 +++--- app/models/apple/publisher.rb | 8 +++++++- app/models/apple/show.rb | 17 +++++++++++++---- app/models/concerns/apple_delivery.rb | 4 ++-- app/models/feed.rb | 2 +- app/models/feeds/megaphone_feed.rb | 8 ++++++-- app/models/megaphone/podcast.rb | 12 ++++++++++++ app/models/megaphone/publisher.rb | 5 +++++ app/models/sync_log.rb | 15 ++++++++++++--- ...241127171040_add_integration_to_sync_logs.rb | 14 ++++++++++++++ db/schema.rb | 5 +++-- .../apple_podcast_container_factory.rb | 7 ++++++- .../apple_podcast_delivery_file_factory.rb | 7 ++++++- test/models/apple/episode_test.rb | 4 ++-- test/models/apple/podcast_container_test.rb | 10 +++++----- test/models/apple/podcast_delivery_file_test.rb | 4 ++-- test/models/apple/publisher_test.rb | 2 +- test/models/sync_log_test.rb | 13 +++++++------ 22 files changed, 120 insertions(+), 43 deletions(-) create mode 100644 db/migrate/20241127171040_add_integration_to_sync_logs.rb diff --git a/app/models/apple/config.rb b/app/models/apple/config.rb index 87ba16b27..8bdfa007a 100644 --- a/app/models/apple/config.rb +++ b/app/models/apple/config.rb @@ -96,7 +96,7 @@ def apple_key def apple_data episode_data = [ - SyncLog.where(feeder_type: "episodes", feeder_id: podcast.episodes.pluck(:id)), + SyncLog.apple.where(feeder_type: "episodes", feeder_id: podcast.episodes.pluck(:id)), Apple::PodcastContainer.where(episode: podcast.episodes) ].flatten.compact diff --git a/app/models/apple/episode.rb b/app/models/apple/episode.rb index 553d549b0..ad457382d 100644 --- a/app/models/apple/episode.rb +++ b/app/models/apple/episode.rb @@ -203,7 +203,13 @@ def self.upsert_sync_log(ep, res) apple_id = res.dig("api_response", "val", "data", "id") raise "Missing remote apple id" unless apple_id.present? - sl = SyncLog.log!(feeder_id: ep.feeder_episode.id, feeder_type: :episodes, external_id: apple_id, api_response: res) + sl = SyncLog.log!( + integration: :apple, + feeder_id: ep.feeder_episode.id, + feeder_type: :episodes, + external_id: apple_id, + api_response: res + ) # reload local state if ep.feeder_episode.apple_sync_log.nil? ep.feeder_episode.reload @@ -262,7 +268,7 @@ def enclosure_filename end def sync_log - SyncLog.episodes.find_by(feeder_id: feeder_episode.id, feeder_type: :episodes) + SyncLog.apple.episodes.find_by(feeder_id: feeder_episode.id, feeder_type: :episodes) end def self.get_episode_bridge_params(api, feeder_id, apple_id) diff --git a/app/models/apple/podcast_container.rb b/app/models/apple/podcast_container.rb index 30c09a59a..60c1c379b 100644 --- a/app/models/apple/podcast_container.rb +++ b/app/models/apple/podcast_container.rb @@ -9,7 +9,7 @@ class PodcastContainer < ApplicationRecord default_scope { includes(:apple_sync_log) } - has_one :apple_sync_log, -> { podcast_containers }, foreign_key: :feeder_id, class_name: "SyncLog", dependent: :destroy + has_one :apple_sync_log, -> { podcast_containers.apple }, foreign_key: :feeder_id, class_name: "SyncLog", dependent: :destroy has_many :podcast_deliveries, dependent: :destroy has_many :podcast_delivery_files, through: :podcast_deliveries belongs_to :episode, -> { with_deleted }, class_name: "::Episode" @@ -145,7 +145,7 @@ def self.upsert_podcast_container(episode, row) external_id: external_id, feeder_episode_id: episode.feeder_id}) - SyncLog.log!(feeder_id: pc.id, feeder_type: :podcast_containers, external_id: external_id, api_response: row) + SyncLog.log!(integration: :apple, feeder_id: pc.id, feeder_type: :podcast_containers, external_id: external_id, api_response: row) # reset the episode's podcast container cached value pc.reload if action == :updated diff --git a/app/models/apple/podcast_delivery.rb b/app/models/apple/podcast_delivery.rb index 861bdc5d6..f6f74d188 100644 --- a/app/models/apple/podcast_delivery.rb +++ b/app/models/apple/podcast_delivery.rb @@ -8,7 +8,7 @@ class PodcastDelivery < ApplicationRecord default_scope { includes(:apple_sync_log) } - has_one :apple_sync_log, -> { podcast_deliveries }, foreign_key: :feeder_id, class_name: "SyncLog", dependent: :destroy + has_one :apple_sync_log, -> { podcast_deliveries.apple }, foreign_key: :feeder_id, class_name: "SyncLog", dependent: :destroy has_many :podcast_delivery_files, dependent: :destroy belongs_to :episode, -> { with_deleted }, class_name: "::Episode" belongs_to :podcast_container, class_name: "::Apple::PodcastContainer" @@ -143,7 +143,7 @@ def self.upsert_podcast_delivery(podcast_container, row) feeder_episode_id: podcast_container.episode.id, podcast_delivery_id: delivery.id}) - SyncLog.log!(feeder_id: pd.id, feeder_type: :podcast_deliveries, external_id: external_id, api_response: row) + SyncLog.log!(integration: :apple, feeder_id: pd.id, feeder_type: :podcast_deliveries, external_id: external_id, api_response: row) # Flush the cache on the podcast container podcast_container.podcast_deliveries.reset diff --git a/app/models/apple/podcast_delivery_file.rb b/app/models/apple/podcast_delivery_file.rb index 854b5e04a..2a6baf2ae 100644 --- a/app/models/apple/podcast_delivery_file.rb +++ b/app/models/apple/podcast_delivery_file.rb @@ -11,7 +11,7 @@ class DeliveryFileError < StandardError; end default_scope { includes(:apple_sync_log) } - has_one :apple_sync_log, -> { podcast_delivery_files }, foreign_key: :feeder_id, class_name: "SyncLog", autosave: true, dependent: :delete + has_one :apple_sync_log, -> { podcast_delivery_files.apple }, foreign_key: :feeder_id, class_name: "SyncLog", autosave: true, dependent: :delete belongs_to :podcast_delivery has_one :podcast_container, through: :podcast_delivery belongs_to :episode, -> { with_deleted }, class_name: "::Episode" @@ -94,7 +94,7 @@ def self.mark_uploaded(api, pdfs) join_on(PODCAST_DELIVERY_FILE_ID_ATTR, pdfs, episode_bridge_results).each do |(pdf, row)| external_id = row.dig("api_response", "val", "data", "id") pdf.update!(api_marked_as_uploaded: true) - SyncLog.log!(feeder_id: pdf.id, feeder_type: :podcast_delivery_files, external_id: external_id, api_response: row) + SyncLog.log!(integration: :apple, feeder_id: pdf.id, feeder_type: :podcast_delivery_files, external_id: external_id, api_response: row) end api.raise_bridge_api_error(errs) if errs.present? @@ -287,7 +287,7 @@ def self.upsert_podcast_delivery_file(podcast_delivery, row) feeder_episode_id: pdf.episode.id, podcast_delivery_file_id: pdf.podcast_delivery.id}) - SyncLog.log!(feeder_id: pdf.id, feeder_type: :podcast_delivery_files, external_id: external_id, api_response: row) + SyncLog.log!(integration: :apple, feeder_id: pdf.id, feeder_type: :podcast_delivery_files, external_id: external_id, api_response: row) # Flush the cache on the podcast container podcast_delivery.delivery_files.reset diff --git a/app/models/apple/publisher.rb b/app/models/apple/publisher.rb index d6bdf767f..ffe4ce6ae 100644 --- a/app/models/apple/publisher.rb +++ b/app/models/apple/publisher.rb @@ -79,7 +79,13 @@ def publish! deliver_and_publish!(episodes_to_sync) # success - SyncLog.log!(feeder_id: public_feed.id, feeder_type: :feeds, external_id: show.apple_id, api_response: {success: true}) + SyncLog.log!( + integration: :apple, + feeder_id: public_feed.id, + feeder_type: :feeds, + external_id: show.apple_id, + api_response: {success: true} + ) end def deliver_and_publish!(eps) diff --git a/app/models/apple/show.rb b/app/models/apple/show.rb index 43ea0322b..a32573b47 100644 --- a/app/models/apple/show.rb +++ b/app/models/apple/show.rb @@ -17,17 +17,20 @@ def self.apple_episode_json(api, show_id) end def self.connect_existing(apple_show_id, apple_config) - if (sl = SyncLog.find_by(feeder_id: apple_config.public_feed.id, feeder_type: :feeds)) + if (sl = SyncLog.apple.find_by(feeder_id: apple_config.public_feed.id, feeder_type: :feeds)) if apple_show_id.blank? return sl.destroy! elsif sl.external_id != apple_show_id sl.update!(external_id: apple_show_id) end else - SyncLog.log!(feeder_id: apple_config.public_feed.id, + SyncLog.log!( + integration: :apple, + feeder_id: apple_config.public_feed.id, feeder_type: :feeds, sync_completed_at: Time.now.utc, - external_id: apple_show_id) + external_id: apple_show_id + ) end api = Apple::Api.from_apple_config(apple_config) @@ -137,7 +140,13 @@ def sync! Rails.logger.tagged("Apple::Show#sync!") do apple_json = create_or_update_show(sync_log) public_feed.reload - SyncLog.log!(feeder_id: public_feed.id, feeder_type: :feeds, external_id: apple_json["api_response"]["val"]["data"]["id"], api_response: apple_json) + SyncLog.log!( + integration: :apple, + feeder_id: public_feed.id, + feeder_type: :feeds, + external_id: apple_json["api_response"]["val"]["data"]["id"], + api_response: apple_json + ) end end diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb index 394f2568d..daea990f6 100644 --- a/app/models/concerns/apple_delivery.rb +++ b/app/models/concerns/apple_delivery.rb @@ -4,7 +4,7 @@ module AppleDelivery extend ActiveSupport::Concern included do - has_one :apple_sync_log, -> { episodes }, foreign_key: :feeder_id, class_name: "SyncLog" + has_one :apple_sync_log, -> { episodes.apple }, foreign_key: :feeder_id, class_name: "SyncLog" has_one :apple_podcast_delivery, class_name: "Apple::PodcastDelivery" has_one :apple_podcast_container, class_name: "Apple::PodcastContainer" has_many :apple_podcast_deliveries, through: :apple_podcast_container, source: :podcast_deliveries, @@ -17,7 +17,7 @@ module AppleDelivery end def publish_to_apple? - podcast.apple_config&.publish_to_apple? || false + !!podcast.apple_config&.publish_to_apple? end def apple_update_delivery_status(attrs) diff --git a/app/models/feed.rb b/app/models/feed.rb index a5767ec7b..ec55c5261 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -34,7 +34,7 @@ class Feed < ApplicationRecord has_many :itunes_images, -> { order("created_at DESC") }, autosave: true, dependent: :destroy, inverse_of: :feed has_many :itunes_categories, validate: true, autosave: true, dependent: :destroy - has_one :apple_sync_log, -> { feeds }, foreign_key: :feeder_id, class_name: "SyncLog" + has_one :apple_sync_log, -> { feeds.apple }, foreign_key: :feeder_id, class_name: "SyncLog" accepts_nested_attributes_for :feed_images, allow_destroy: true, reject_if: ->(i) { i[:id].blank? && i[:original_url].blank? } accepts_nested_attributes_for :itunes_images, allow_destroy: true, reject_if: ->(i) { i[:id].blank? && i[:original_url].blank? } diff --git a/app/models/feeds/megaphone_feed.rb b/app/models/feeds/megaphone_feed.rb index 63c7ee913..d1a1ead3b 100644 --- a/app/models/feeds/megaphone_feed.rb +++ b/app/models/feeds/megaphone_feed.rb @@ -24,15 +24,19 @@ def set_defaults end def publish_integration? - megaphone_config&.publish_to_megaphone? + publish_to_megaphone? end def publish_integration! if publish_integration? - megaphone_config.build_publisher.publish! + Publisher.new(self).publish! end end + def publish_to_megaphone? + valid? && persisted? && config&.publish_to_megaphone? + end + def mark_as_not_delivered!(episode) episode.episode_delivery_statuses.megaphone.first&.mark_as_not_delivered! end diff --git a/app/models/megaphone/podcast.rb b/app/models/megaphone/podcast.rb index d489a9872..15b7268bc 100644 --- a/app/models/megaphone/podcast.rb +++ b/app/models/megaphone/podcast.rb @@ -37,6 +37,18 @@ def initialize(attributes = {}) super end + def self.find_by_feed(feed) + return nil unless feed.podcast&.guid + podcast = Megaphone::Podcast.new(feed: feed) + podcast.find_by_guid(feed.podcast.guid).items.first + end + + # def self.find_by_megaphone_id(feed) + # return nil unless feed.podcast&.guid + # podcast = Megaphone::Podcast.new(feed: feed) + # podcast.find_by_guid(feed.podcast.guid).items.first + # end + def self.new_from_feed(feed) podcast = Megaphone::Podcast.new(attributes_from_feed(feed)) podcast.feed = feed diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index cbf95a6fc..96a85b21e 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -1,6 +1,11 @@ module Megaphone class Publisher < Integrations::Base::Publisher + def initialize(feed) + @feed = feed + end + def publish! + # megaphone_podcast = Megaphone::Podcast.find_or_create!(feed) end end end diff --git a/app/models/sync_log.rb b/app/models/sync_log.rb index e443b1619..f3ee16e8e 100644 --- a/app/models/sync_log.rb +++ b/app/models/sync_log.rb @@ -1,9 +1,12 @@ -# frozen_string_literal: true - class SyncLog < ApplicationRecord + enum :integration, Integrations::INTEGRATIONS + + # kinda like an AR polymorphic relation, but not using that enum :feeder_type, { + # common feeds: "feeds", episodes: "episodes", + # apple podcast_containers: "containers", podcast_deliveries: "deliveries", podcast_delivery_files: "delivery_files" @@ -23,12 +26,18 @@ def complete? end def self.log!(attrs) + integration = attrs.delete(:integration) feeder_type = attrs.delete(:feeder_type) feeder_id = attrs.delete(:feeder_id) external_id = attrs.delete(:external_id) api_response = attrs.delete(:api_response) - sync_log = SyncLog.find_or_initialize_by(feeder_type: feeder_type, feeder_id: feeder_id, external_id: external_id) + sync_log = SyncLog.find_or_initialize_by( + integration: integration, + feeder_type: feeder_type, + feeder_id: feeder_id, + external_id: external_id + ) sync_log.update!(api_response: api_response) sync_log end diff --git a/db/migrate/20241127171040_add_integration_to_sync_logs.rb b/db/migrate/20241127171040_add_integration_to_sync_logs.rb new file mode 100644 index 000000000..9e2cf4d1a --- /dev/null +++ b/db/migrate/20241127171040_add_integration_to_sync_logs.rb @@ -0,0 +1,14 @@ +class AddIntegrationToSyncLogs < ActiveRecord::Migration[7.2] + def change + add_column :sync_logs, :integration, :integer + + execute(<<~SQL + UPDATE sync_logs + SET integration = 0 + SQL + ) + + remove_index :sync_logs, [:feeder_type, :feeder_id], unique: true + add_index :sync_logs, [:integration, :feeder_type, :feeder_id], unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 71bd21355..7eb2b51ec 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_11_26_230612) do +ActiveRecord::Schema[7.2].define(version: 2024_11_27_171040) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" enable_extension "uuid-ossp" @@ -459,7 +459,8 @@ t.datetime "updated_at" t.datetime "created_at" t.text "api_response" - t.index ["feeder_type", "feeder_id"], name: "index_sync_logs_on_feeder_type_and_feeder_id", unique: true + t.integer "integration" + t.index ["integration", "feeder_type", "feeder_id"], name: "index_sync_logs_on_integration_and_feeder_type_and_feeder_id", unique: true end create_table "tasks", id: :serial, force: :cascade do |t| diff --git a/test/factories/apple_podcast_container_factory.rb b/test/factories/apple_podcast_container_factory.rb index e64a67c40..643c0049e 100644 --- a/test/factories/apple_podcast_container_factory.rb +++ b/test/factories/apple_podcast_container_factory.rb @@ -10,7 +10,12 @@ after(:build) do |podcast_container, evaluator| api_response = build(:podcast_container_api_response, podcast_container_id: evaluator.external_id) - podcast_container.apple_sync_log = SyncLog.new(external_id: evaluator.external_id, feeder_type: :podcast_containers, **api_response) + podcast_container.apple_sync_log = SyncLog.new( + integration: :apple, + external_id: evaluator.external_id, + feeder_type: :podcast_containers, + **api_response + ) end end end diff --git a/test/factories/apple_podcast_delivery_file_factory.rb b/test/factories/apple_podcast_delivery_file_factory.rb index 93ff47779..cb02929b2 100644 --- a/test/factories/apple_podcast_delivery_file_factory.rb +++ b/test/factories/apple_podcast_delivery_file_factory.rb @@ -6,7 +6,12 @@ after(:build) do |delivery_file, evaluator| api_response = build(:podcast_delivery_file_api_response, external_id: evaluator.external_id) - delivery_file.apple_sync_log = SyncLog.new(external_id: evaluator.external_id, feeder_type: :podcast_delivery_files, **api_response) + delivery_file.apple_sync_log = SyncLog.new( + integration: :apple, + external_id: evaluator.external_id, + feeder_type: :podcast_delivery_files, + **api_response + ) end end end diff --git a/test/models/apple/episode_test.rb b/test/models/apple/episode_test.rb index 2cefff029..c560d3391 100644 --- a/test/models/apple/episode_test.rb +++ b/test/models/apple/episode_test.rb @@ -72,7 +72,7 @@ let(:delivery_file) do pdf = Apple::PodcastDeliveryFile.new(episode: episode, podcast_delivery: delivery) - pdf.update(apple_sync_log: SyncLog.new(**build(:podcast_delivery_file_api_response).merge(external_id: "123"), feeder_type: :podcast_delivery_files)) + pdf.update(apple_sync_log: SyncLog.new(**build(:podcast_delivery_file_api_response).merge(external_id: "123"), feeder_type: :podcast_delivery_files, integration: :apple)) pdf.save! pdf end @@ -288,7 +288,7 @@ let(:delivery_file) do pdf = Apple::PodcastDeliveryFile.new(episode: episode, podcast_delivery: delivery) - pdf.update(apple_sync_log: SyncLog.new(**build(:podcast_delivery_file_api_response).merge(external_id: "123"), feeder_type: :podcast_delivery_files)) + pdf.update(apple_sync_log: SyncLog.new(**build(:podcast_delivery_file_api_response).merge(external_id: "123"), feeder_type: :podcast_delivery_files, integration: :apple)) pdf.save! pdf end diff --git a/test/models/apple/podcast_container_test.rb b/test/models/apple/podcast_container_test.rb index 7c07e3b68..53eb3490b 100644 --- a/test/models/apple/podcast_container_test.rb +++ b/test/models/apple/podcast_container_test.rb @@ -172,19 +172,19 @@ class Apple::PodcastContainerTest < ActiveSupport::TestCase it "should create logs based on a returned row value" do apple_episode.stub(:apple_id, apple_episode_id) do apple_episode.stub(:audio_asset_vendor_id, apple_audio_asset_vendor_id) do - assert_equal SyncLog.podcast_containers.count, 0 + assert_equal SyncLog.apple.podcast_containers.count, 0 assert_equal Apple::PodcastContainer.count, 0 Apple::PodcastContainer.upsert_podcast_container(apple_episode, podcast_container_json_row) - assert_equal SyncLog.podcast_containers.count, 1 + assert_equal SyncLog.apple.podcast_containers.count, 1 assert_equal Apple::PodcastContainer.count, 1 Apple::PodcastContainer.upsert_podcast_container(apple_episode, podcast_container_json_row) # The second call should not create a new log or podcast container - assert_equal SyncLog.podcast_containers.count, 1 + assert_equal SyncLog.apple.podcast_containers.count, 1 assert_equal Apple::PodcastContainer.count, 1 end end @@ -251,7 +251,7 @@ class Apple::PodcastContainerTest < ActiveSupport::TestCase describe ".poll_podcast_container_state(api, episodes)" do it "creates new records if they dont exist" do - assert_equal SyncLog.podcast_containers.count, 0 + assert_equal SyncLog.apple.podcast_containers.count, 0 Apple::PodcastContainer.stub(:get_podcast_containers_via_episodes, [podcast_container_json_row]) do apple_episode.stub(:apple_id, apple_episode_id) do @@ -262,7 +262,7 @@ class Apple::PodcastContainerTest < ActiveSupport::TestCase end end end - assert_equal SyncLog.podcast_containers.count, 1 + assert_equal SyncLog.apple.podcast_containers.count, 1 end end diff --git a/test/models/apple/podcast_delivery_file_test.rb b/test/models/apple/podcast_delivery_file_test.rb index 41f872224..89177b0e1 100644 --- a/test/models/apple/podcast_delivery_file_test.rb +++ b/test/models/apple/podcast_delivery_file_test.rb @@ -38,7 +38,7 @@ class ApplePodcastDeliveryFileTest < ActiveSupport::TestCase let(:asset_delivery_state) { "COMPLETE" } let(:pdf_resp_container) { build(:podcast_delivery_file_api_response, asset_delivery_state: asset_delivery_state, asset_processing_state: asset_processing_state) } - let(:apple_id) { {external_id: "123"} } + let(:apple_id) { {external_id: "123", integration: :apple} } let(:pdf) { Apple::PodcastDeliveryFile.new(apple_sync_log: SyncLog.new(**pdf_resp_container.merge(apple_id))) } describe "#delivery_awaiting_upload?" do @@ -117,7 +117,7 @@ class ApplePodcastDeliveryFileTest < ActiveSupport::TestCase it "should soft delete the delivery" do pdf_resp_container = build(:podcast_delivery_file_api_response) pdf = Apple::PodcastDeliveryFile.create!(podcast_delivery: podcast_delivery, episode: podcast_container.episode) - pdf.create_apple_sync_log!(**pdf_resp_container.merge(external_id: "123")) + pdf.create_apple_sync_log!(**pdf_resp_container.merge(external_id: "123", integration: :apple)) assert_equal [pdf], podcast_delivery.podcast_delivery_files.reset diff --git a/test/models/apple/publisher_test.rb b/test/models/apple/publisher_test.rb index 5c78ed9f5..56646d870 100644 --- a/test/models/apple/publisher_test.rb +++ b/test/models/apple/publisher_test.rb @@ -485,7 +485,7 @@ let(:asset_delivery_state) { "COMPLETE" } let(:pdf_resp_container) { build(:podcast_delivery_file_api_response, asset_delivery_state: asset_delivery_state, asset_processing_state: asset_processing_state) } - let(:apple_id) { {external_id: "123"} } + let(:apple_id) { {external_id: "123", integration: :apple} } let(:podcast_container) { create(:apple_podcast_container, episode: apple_episode.feeder_episode) } let(:podcast_delivery) { Apple::PodcastDelivery.create!(podcast_container: podcast_container, episode: apple_episode.feeder_episode) } diff --git a/test/models/sync_log_test.rb b/test/models/sync_log_test.rb index 83dec9437..7aa5833b1 100644 --- a/test/models/sync_log_test.rb +++ b/test/models/sync_log_test.rb @@ -5,9 +5,9 @@ describe SyncLog do describe "indexes" do it "prevents the same feeder_type, feeder_id combination from being saved" do - s = SyncLog.new(feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) + s = SyncLog.new(integration: :apple, feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) s.save! - s2 = SyncLog.new(feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) + s2 = SyncLog.new(integration: :apple, feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) assert_raises ActiveRecord::RecordNotUnique do s2.save! end @@ -15,7 +15,7 @@ end describe ".feeds" do it "filters records by a feeds enum" do - s = SyncLog.new(feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) + s = SyncLog.new(integration: :apple, feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) s.save! assert_equal SyncLog.feeds, [s] end @@ -24,10 +24,11 @@ describe ".log!" do it "creates a new record" do assert_difference "SyncLog.count", 1 do - SyncLog.log!(feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) + SyncLog.log!(integration: :apple, feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) end s = SyncLog.last + assert_equal s.integration, "apple" assert_equal s.feeder_type, "feeds" assert_equal s.feeder_id, 123 assert_equal s.external_id, "456" @@ -35,9 +36,9 @@ end it "updates an existing record" do - s = SyncLog.create!(feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) + s = SyncLog.create!(integration: :apple, feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "bar"}) assert_no_difference "SyncLog.count" do - SyncLog.log!(feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "baz"}) + SyncLog.log!(integration: :apple, feeder_type: :feeds, feeder_id: 123, external_id: 456, api_response: {foo: "baz"}) end assert_equal s.reload.api_response, {foo: "baz"}.as_json end From 5fb3048f0377cc3a1e3f0498118fa14c17abb435 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sat, 30 Nov 2024 09:17:00 -0500 Subject: [PATCH 44/71] MP publisher start, publishes show --- app/models/apple/publisher.rb | 4 -- app/models/apple/show.rb | 9 ++- app/models/feed.rb | 4 ++ app/models/integrations/base/episode.rb | 6 +- app/models/integrations/base/publisher.rb | 3 +- app/models/integrations/base/show.rb | 7 +- app/models/megaphone/api.rb | 32 +++++---- app/models/megaphone/episode.rb | 40 ++++++----- app/models/megaphone/model.rb | 19 +++-- app/models/megaphone/podcast.rb | 88 +++++++++++++---------- app/models/megaphone/publisher.rb | 60 +++++++++++++++- app/models/sync_log.rb | 2 +- test/models/megaphone/episode_test.rb | 11 +-- test/models/megaphone/podcast_test.rb | 2 +- test/models/megaphone/publisher_test.rb | 62 ++++++++++++++++ 15 files changed, 247 insertions(+), 102 deletions(-) create mode 100644 test/models/megaphone/publisher_test.rb diff --git a/app/models/apple/publisher.rb b/app/models/apple/publisher.rb index ffe4ce6ae..8728946c7 100644 --- a/app/models/apple/publisher.rb +++ b/app/models/apple/publisher.rb @@ -1,9 +1,5 @@ -# frozen_string_literal: true - module Apple class Publisher < Integrations::Base::Publisher - PUBLISH_CHUNK_LEN = 25 - attr_reader :public_feed, :private_feed, :api, diff --git a/app/models/apple/show.rb b/app/models/apple/show.rb index a32573b47..e8e1c957f 100644 --- a/app/models/apple/show.rb +++ b/app/models/apple/show.rb @@ -4,9 +4,12 @@ module Apple class Show < Integrations::Base::Show include Apple::ApiResponse - attr_reader :public_feed, - :private_feed, - :api + attr_reader :api + + def initialize(public_feed:, private_feed:) + @public_feed = public_feed + @private_feed = private_feed + end def self.apple_shows_json(api) api.get_paged_collection("shows") diff --git a/app/models/feed.rb b/app/models/feed.rb index ec55c5261..51756b42a 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -88,6 +88,10 @@ def publish_integration? def publish_integration! end + def sync_log(integration) + SyncLog.latest.find_by(integration: integration, feeder_id: id, feeder_type: :feeds) + end + def set_defaults self.file_name ||= DEFAULT_FILE_NAME self.enclosure_template ||= Feed.enclosure_template_default diff --git a/app/models/integrations/base/episode.rb b/app/models/integrations/base/episode.rb index 59156c533..b1b2ae181 100644 --- a/app/models/integrations/base/episode.rb +++ b/app/models/integrations/base/episode.rb @@ -1,11 +1,7 @@ module Integrations module Base class Episode - attr_reader :feeder_episode - - def initialize(feeder_episode) - @feeder_episode = feeder_episode - end + attr_accessor :feeder_episode def synced_with_integration? raise NotImplementedError, "Subclasses must implement synced_with_integration?" diff --git a/app/models/integrations/base/publisher.rb b/app/models/integrations/base/publisher.rb index d2d298d53..36f106c95 100644 --- a/app/models/integrations/base/publisher.rb +++ b/app/models/integrations/base/publisher.rb @@ -1,9 +1,10 @@ module Integrations module Base class Publisher + PUBLISH_CHUNK_LEN = 25 include EpisodeSetOperations - attr_reader :show + attr_accessor :show def initialize(show:) @show = show diff --git a/app/models/integrations/base/show.rb b/app/models/integrations/base/show.rb index f65ce0f05..4b1e8851e 100644 --- a/app/models/integrations/base/show.rb +++ b/app/models/integrations/base/show.rb @@ -3,12 +3,7 @@ module Base class Show include EpisodeSetOperations - attr_reader :feed - - def initialize(public_feed:, private_feed:) - @public_feed = public_feed - @private_feed = private_feed - end + attr_accessor :public_feed, :private_feed def podcast private_feed.podcast diff --git a/app/models/megaphone/api.rb b/app/models/megaphone/api.rb index f646b0ea0..40316161d 100644 --- a/app/models/megaphone/api.rb +++ b/app/models/megaphone/api.rb @@ -16,27 +16,29 @@ def initialize(token:, network_id:, endpoint_url: nil) def get(path, params = {}, headers = {}) request = {url: join_url(path), headers: headers, params: params} response = get_url(request) - data = incoming_body_filter(response.body) - if data.is_a?(Array) - pagination = pagination_from_headers(response.env.response_headers) - {items: data, pagination: pagination, request: request, response: response} - else - {items: [data], pagination: {}, request: request, response: response} - end + result(request, response) end def post(path, body, headers = {}) - response = connection({url: join_url(path), headers: headers}).post do |req| - req.body = outgoing_body_filter(body) - end - incoming_body_filter(response.body) + request = {url: join_url(path), headers: headers, body: outgoing_body_filter(body)} + response = connection(request).post + result(request, response) end def put(path, body, headers = {}) - connection({url: join_url(path), headers: headers}).put do |req| - req.body = outgoing_body_filter(body) + request = {url: join_url(path), headers: headers, body: outgoing_body_filter(body)} + response = connection(request).put + result(request, response) + end + + def result(request, response) + data = incoming_body_filter(response.body) + if data.is_a?(Array) + pagination = pagination_from_headers(response.env.response_headers) + {items: data, pagination: pagination, request: request, response: response} + else + {items: [data], pagination: {}, request: request, response: response} end - incoming_body_filter(response.body) end # TODO: and we need delete... @@ -113,7 +115,7 @@ def connection(options) Faraday.new(url: url, headers: headers, params: params) do |builder| builder.request :token_auth, token builder.response :raise_error - builder.response :logger + builder.response :logger, Rails.logger builder.adapter :excon end end diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index fb030f58a..c05c4af66 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -1,24 +1,25 @@ module Megaphone - class Episode < Megaphone::Model - attr_accessor :episode + class Episode < Integrations::Base::Episode + include Megaphone::Model + attr_accessor :private_feed # Used to form the adhash value ADHASH_VALUES = {"pre" => "0", "mid" => "1", "post" => "2"}.freeze # Required attributes for a create # external_id is not required by megaphone, but we need it to be set! - CREATE_REQUIRED = %w[title external_id] + CREATE_REQUIRED = %i[title external_id] - CREATE_ATTRIBUTES = CREATE_REQUIRED + %w[pubdate pubdate_timezone author link explicit draft + CREATE_ATTRIBUTES = CREATE_REQUIRED + %i[pubdate pubdate_timezone author link explicit draft subtitle summary background_image_file_url background_audio_file_url pre_count post_count insertion_points guid pre_offset post_offset expected_adhash original_filename original_url episode_number season_number retain_ad_locations advertising_tags ad_free] # All other attributes we might expect back from the Megaphone API # (some documented, others not so much) - OTHER_ATTRIBUTES = %w[id created_at updated_at] + OTHER_ATTRIBUTES = %i[id created_at updated_at] - DEPRECATED = %w[] + DEPRECATED = %i[] ALL_ATTRIBUTES = (CREATE_ATTRIBUTES + DEPRECATED + OTHER_ATTRIBUTES) @@ -30,11 +31,11 @@ class Episode < Megaphone::Model validates_absence_of :id, on: :create - def self.new_from_episode(dt_episode, feed = nil) - episode = Megaphone::Episode.new(attributes_from_episode(dt_episode)) - episode.episode = dt_episode - episode.feed = feed - episode.set_audio_attributes + def self.new_from_episode(feed, feeder_episode) + episode = Megaphone::Episode.new(attributes_from_episode(feeder_episode)) + episode.feeder_episode = feeder_episode + episode.private_feed = feed + episode.config = feed.config episode end @@ -60,7 +61,7 @@ def self.attributes_from_episode(e) end def set_placement_attributes - placement = get_placement(episode.segment_count) + placement = get_placement(feeder_episode.segment_count) self.expected_adhash = adhash_for_placement(placement) self.pre_count = expected_adhash.count("0") self.post_count = expected_adhash.count("2") @@ -79,8 +80,9 @@ def get_placement(original_count) placements&.find { |i| i.original_count == original_count } end + # call this before create or update, yah def set_audio_attributes - return unless episode.complete_media? + return unless feeder_episode.complete_media? self.background_audio_file_url = upload_url self.insertion_points = timings self.retain_ad_locations = true @@ -90,7 +92,7 @@ def upload_url resp = Faraday.head(enclosure_url) if resp.status == 302 media_version = resp.env.response_headers["x-episode-media-version"] - if media_version == episode.media_version_id + if media_version == feeder_episode.media_version_id location = resp.env.response_headers["location"] arrangement_version_url(location, media_version) end @@ -106,12 +108,16 @@ def arrangement_version_url(location, media_version) end def enclosure_url - url = EnclosureUrlBuilder.new.base_enclosure_url(episode.podcast, episode, feed) - EnclosureUrlBuilder.mark_authorized(url, feed) + url = EnclosureUrlBuilder.new.base_enclosure_url( + feeder_episode.podcast, + feeder_episode, + private_feed + ) + EnclosureUrlBuilder.mark_authorized(url, private_feed) end def timings - episode.media[0..-2].map(&:duration) + feeder_episode.media[0..-2].map(&:duration) end def pre_after_original?(placement) diff --git a/app/models/megaphone/model.rb b/app/models/megaphone/model.rb index a53b28696..a9f3b6bec 100644 --- a/app/models/megaphone/model.rb +++ b/app/models/megaphone/model.rb @@ -1,15 +1,22 @@ +require "active_support/concern" + module Megaphone - class Model - include ActiveModel::Model - attr_accessor :feed - attr_writer :api + module Model + extend ActiveSupport::Concern - def config - feed.megaphone_config + included do + include ActiveModel::Model + attr_accessor :config + attr_writer :api + attr_accessor :api_response end def api @api ||= Megaphone::Api.new(token: config.token, network_id: config.network_id) end + + def api_response_log_item + api_response&.slice(:request, :items, :pagination) + end end end diff --git a/app/models/megaphone/podcast.rb b/app/models/megaphone/podcast.rb index 15b7268bc..869604ce3 100644 --- a/app/models/megaphone/podcast.rb +++ b/app/models/megaphone/podcast.rb @@ -1,24 +1,26 @@ module Megaphone - class Podcast < Megaphone::Model + class Podcast < Integrations::Base::Show + include Megaphone::Model + # Required attributes for a create # external_id is not required by megaphone, but we need it to be set! - CREATE_REQUIRED = %w[title subtitle summary itunes_categories language external_id] + CREATE_REQUIRED = %i[title subtitle summary itunes_categories language external_id] # Other attributes available on create - CREATE_ATTRIBUTES = CREATE_REQUIRED + %w[link copyright author background_image_file_url + CREATE_ATTRIBUTES = CREATE_REQUIRED + %i[link copyright author background_image_file_url explicit owner_name owner_email slug original_rss_url itunes_identifier podtrac_enabled google_play_identifier episode_limit podcast_type advertising_tags excluded_categories] # Update also allows the span opt in - UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES + %w[span_opt_in] + UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES + %i[span_opt_in] # Deprecated, so we shouldn't rely on these, but they show up as attributes - DEPRECATED = %w[category redirect_url itunes_active redirected_at itunes_rating + DEPRECATED = %i[category redirect_url itunes_active redirected_at itunes_rating google_podcasts_identifier stitcher_identifier] # All other attributes we might expect back from the Megaphone API # (some documented, others not so much) - OTHER_ATTRIBUTES = %w[id created_at updated_at image_file uid network_id recurring_import + OTHER_ATTRIBUTES = %i[id created_at updated_at image_file uid network_id recurring_import episodes_count spotify_identifier default_ad_settings iheart_identifier feed_url default_pre_count default_post_count cloned_feed_urls ad_free_feed_urls main_feed ad_free] @@ -32,26 +34,19 @@ class Podcast < Megaphone::Model validates_absence_of :id, on: :create - # initialize from attributes - def initialize(attributes = {}) - super - end - def self.find_by_feed(feed) - return nil unless feed.podcast&.guid - podcast = Megaphone::Podcast.new(feed: feed) - podcast.find_by_guid(feed.podcast.guid).items.first + podcast = new_from_feed(feed) + public_feed = feed.podcast.public_feed + sync_log = public_feed.sync_log(:megaphone) + mp = podcast.find_by_megaphone_id(sync_log&.external_id) + mp ||= podcast.find_by_guid(feed.podcast.guid) + mp end - # def self.find_by_megaphone_id(feed) - # return nil unless feed.podcast&.guid - # podcast = Megaphone::Podcast.new(feed: feed) - # podcast.find_by_guid(feed.podcast.guid).items.first - # end - def self.new_from_feed(feed) podcast = Megaphone::Podcast.new(attributes_from_feed(feed)) - podcast.feed = feed + podcast.private_feed = feed + podcast.config = feed.config podcast end @@ -84,35 +79,54 @@ def self.attributes_from_feed(feed) } end + def build_integration_episode(feeder_episode) + Megaphone::Episode.new_from_episode(private_feed, feeder_episode) + end + + def updated_at=(d) + d = Time.parse(d) if d.is_a?(String) + @updated_at = d + end + def list - result = api.get("podcasts") - Megaphone::PagedCollection.new(Megaphone::Podcast, result) + self.api_response = api.get("podcasts") + Megaphone::PagedCollection.new(Megaphone::Podcast, api_response) end - def find_by_guid - result = api.get("podcasts", externalId: feed.podcast.guid) - Megaphone::PagedCollection.new(Megaphone::Podcast, result) + def find_by_guid(guid = podcast.guid) + return nil if guid.blank? + self.api_response = api.get("podcasts", externalId: guid) + handle_response(api_response) end def find_by_megaphone_id(mpid = id) - result = api.get("podcasts/#{mpid}") - (result[:items] || []).first + return nil if mpid.blank? + self.api_response = api.get("podcasts/#{mpid}") + handle_response(api_response) end def create! validate!(:create) - body = as_json.slice(*Megaphone::Podcast::CREATE_ATTRIBUTES) - result = api.post("podcasts", body) - self.attributes = result.slice(*Megaphone::Podcast::ALL_ATTRIBUTES) - self + body = as_json(only: CREATE_ATTRIBUTES.map(&:to_s)) + self.api_response = api.post("podcasts", body) + handle_response(api_response) end - def update! + def update!(feed = nil) + if feed + self.attributes = self.class.attributes_from_feed(feed) + end validate!(:update) - body = as_json.slice(*Megaphone::Podcast::UPDATE_ATTRIBUTES).to_json - result = api.put("podcasts/#{id}", body) - self.attributes = result.slice(*Megaphone::Podcast::ALL_ATTRIBUTES) - self + body = as_json(only: UPDATE_ATTRIBUTES.map(&:to_s)) + self.api_response = api.put("podcasts/#{id}", body) + handle_response(api_response) + end + + def handle_response(api_response) + if (item = (api_response[:items] || []).first) + self.attributes = item.slice(*ALL_ATTRIBUTES) + self + end end end end diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index 96a85b21e..ee7a26cfc 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -1,11 +1,69 @@ module Megaphone class Publisher < Integrations::Base::Publisher + attr_reader :feed, :megaphone_podcast + + alias_method :private_feed, :feed + def initialize(feed) @feed = feed end + def show + @megaphone_podcast ||= Megaphone::Podcast.find_by_feed(feed) + end + def publish! - # megaphone_podcast = Megaphone::Podcast.find_or_create!(feed) + sync_podcast! + raise "Missing Megaphone Podcast!" unless megaphone_podcast&.id.present? + + # success + SyncLog.log!( + integration: :megaphone, + feeder_id: public_feed.id, + feeder_type: :feeds, + external_id: megaphone_podcast.id, + api_response: {success: true} + ) + end + + def deliver_and_publish! + episodes = episodes_to_sync + puts "deliver_and_publish!: #{episodes.count}" + end + + def sync_podcast! + if (@megaphone_podcast = Megaphone::Podcast.find_by_feed(feed)) + # see if we need to update by comparing dates + # - there is no episode delivery status ;) + if @megaphone_podcast.updated_at < podcast.updated_at + @megaphone_podcast.update!(feed) + end + end + + @megaphone_podcast ||= if @megaphone_podcast.blank? + Megaphone::Podcast.new_from_feed(feed).create! + end + + SyncLog.log!( + integration: :megaphone, + feeder_id: public_feed.id, + feeder_type: :feeds, + external_id: @megaphone_podcast.id, + api_response: @megaphone_podcast.api_response_log_item + ) + @megaphone_podcast + end + + def config + feed&.config + end + + def public_feed + podcast&.public_feed + end + + def podcast + feed&.podcast end end end diff --git a/app/models/sync_log.rb b/app/models/sync_log.rb index f3ee16e8e..ede1522a9 100644 --- a/app/models/sync_log.rb +++ b/app/models/sync_log.rb @@ -15,7 +15,7 @@ class SyncLog < ApplicationRecord scope :latest, -> do joins("JOIN LATERAL ( SELECT max(id) as max_id FROM sync_logs - GROUP BY feeder_type, feeder_id, external_id ) q + GROUP BY integration, feeder_type, feeder_id, external_id ) q ON id = max_id") end diff --git a/test/models/megaphone/episode_test.rb b/test/models/megaphone/episode_test.rb index 5e10ebcdf..834a85ae9 100644 --- a/test/models/megaphone/episode_test.rb +++ b/test/models/megaphone/episode_test.rb @@ -3,15 +3,16 @@ describe Megaphone::Episode do let(:podcast) { create(:podcast) } let(:feed) { create(:megaphone_feed, podcast: podcast) } - let(:dt_episode) { create(:episode, podcast: podcast) } + let(:feeder_episode) { create(:episode, podcast: podcast) } describe "#valid?" do it "must have required attributes" do - episode = Megaphone::Episode.new_from_episode(dt_episode, feed) + episode = Megaphone::Episode.new_from_episode(feed, feeder_episode) assert_not_nil episode - assert_not_nil episode.episode - assert_not_nil episode.feed - assert_equal episode.title, dt_episode.title + assert_equal episode.feeder_episode, feeder_episode + assert_equal episode.private_feed, feed + assert_equal episode.config, feed.config + assert_equal episode.title, feeder_episode.title assert episode.valid? end end diff --git a/test/models/megaphone/podcast_test.rb b/test/models/megaphone/podcast_test.rb index 45e8a2ea8..5d2e9258c 100644 --- a/test/models/megaphone/podcast_test.rb +++ b/test/models/megaphone/podcast_test.rb @@ -8,7 +8,7 @@ it "must have required attributes" do podcast = Megaphone::Podcast.new_from_feed(feed) assert_not_nil podcast - assert_not_nil podcast.feed + assert_not_nil podcast.config assert_equal podcast.title, feed.title assert podcast.valid? end diff --git a/test/models/megaphone/publisher_test.rb b/test/models/megaphone/publisher_test.rb new file mode 100644 index 000000000..98fcc4bb2 --- /dev/null +++ b/test/models/megaphone/publisher_test.rb @@ -0,0 +1,62 @@ +require "test_helper" + +describe Megaphone::Publisher do + let(:podcast) { create(:podcast) } + let(:public_feed) { podcast.default_feed } + let(:feed) { create(:megaphone_feed, podcast: podcast, private: true) } + + let(:publisher) do + Megaphone::Publisher.new(feed) + end + + describe ".initialize" do + it "should build a publisher with the correct feeds" do + assert_equal publisher.feed, feed + assert_equal publisher.podcast, podcast + assert_equal publisher.public_feed, public_feed + end + end + + describe "#sync_podcast!" do + it "should create a new podcast" do + stub_request(:get, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts?externalId=95ebfb22-0002-5f78-a7aa-5acb5ac7daa9") + .to_return(status: 200, body: [].to_json, headers: {}) + + stub_request(:post, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts") + .to_return(status: 200, body: {id: "ABC12345"}.to_json, headers: {}) + + megaphone_podcast = publisher.sync_podcast! + assert_not_nil megaphone_podcast + assert_equal megaphone_podcast.id, "ABC12345" + end + + it "should find a podcast by megaphone id" do + stub_request(:get, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts/A1B2C4D5E6F7G8") + .to_return(status: 200, body: {id: "A1B2C4D5E6F7G8", updatedAt: (Time.now + 1.minute).utc.iso8601}.to_json, headers: {}) + + SyncLog.log!( + integration: :megaphone, + feeder_id: public_feed.id, + feeder_type: :feeds, + external_id: "A1B2C4D5E6F7G8", + api_response: {request: {}, items: {}} + ) + + megaphone_podcast = publisher.sync_podcast! + assert_not_nil megaphone_podcast + assert_equal megaphone_podcast.id, "A1B2C4D5E6F7G8" + end + + it "should find a podcast by guid" do + stub_request(:get, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts?externalId=95ebfb22-0002-5f78-a7aa-5acb5ac7daa9") + .to_return(status: 200, body: [{id: "DEF67890", updatedAt: "2024-11-03T14:54:02.690Z"}].to_json, headers: {}) + + stub_request(:put, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts/DEF67890") + .to_return(status: 200, body: [{id: "DEF67890", updatedAt: Time.now.utc.iso8601}].to_json, headers: {}) + + megaphone_podcast = publisher.sync_podcast! + assert_not_nil megaphone_podcast + assert_equal megaphone_podcast.id, "DEF67890" + end + end +end From f5628665498a8d3d507f4c9c61f3b82409cbdc2e Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 3 Dec 2024 17:09:48 -0500 Subject: [PATCH 45/71] Start episode publishing and tests --- app/models/apple/show.rb | 5 ---- app/models/megaphone/episode.rb | 16 ++++++++++++ app/models/megaphone/publisher.rb | 4 +-- test/models/megaphone/publisher_test.rb | 34 ++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/app/models/apple/show.rb b/app/models/apple/show.rb index e8e1c957f..df336445d 100644 --- a/app/models/apple/show.rb +++ b/app/models/apple/show.rb @@ -6,11 +6,6 @@ class Show < Integrations::Base::Show attr_reader :api - def initialize(public_feed:, private_feed:) - @public_feed = public_feed - @private_feed = private_feed - end - def self.apple_shows_json(api) api.get_paged_collection("shows") end diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index c05c4af66..7dc80f390 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -60,6 +60,22 @@ def self.attributes_from_episode(e) } end + def synced_with_integration? + delivery_status&.delivered? + end + + def integration_new? + false + end + + def archived? + false + end + + def delivery_status + feeder_episode&.episode_delivery_status(:megaphone) + end + def set_placement_attributes placement = get_placement(feeder_episode.segment_count) self.expected_adhash = adhash_for_placement(placement) diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index ee7a26cfc..e22d4e1c8 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -14,7 +14,7 @@ def show def publish! sync_podcast! - raise "Missing Megaphone Podcast!" unless megaphone_podcast&.id.present? + sync_episodes! # success SyncLog.log!( @@ -26,7 +26,7 @@ def publish! ) end - def deliver_and_publish! + def sync_episodes! episodes = episodes_to_sync puts "deliver_and_publish!: #{episodes.count}" end diff --git a/test/models/megaphone/publisher_test.rb b/test/models/megaphone/publisher_test.rb index 98fcc4bb2..2a1dbb4dc 100644 --- a/test/models/megaphone/publisher_test.rb +++ b/test/models/megaphone/publisher_test.rb @@ -17,6 +17,38 @@ end end + describe "sync_episodes!" do + let(:episode) { create(:episode, podcast: podcast) } + + before do + stub_request(:get, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts/A1B2C4D5E6F7G8") + .to_return(status: 200, body: {id: "A1B2C4D5E6F7G8", updatedAt: (Time.now + 1.minute).utc.iso8601}.to_json, headers: {}) + + SyncLog.log!( + integration: :megaphone, + feeder_id: public_feed.id, + feeder_type: :feeds, + external_id: "A1B2C4D5E6F7G8", + api_response: {request: {}, items: {}} + ) + end + + it "should create new draft episodes" do + assert episode + puts episode.inspect + publisher.sync_episodes! + end + + it "should update episodes" do + end + + it "should update episodes with audio" do + end + + it "should update episodes with incomplete audio" do + end + end + describe "#sync_podcast!" do it "should create a new podcast" do stub_request(:get, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts?externalId=95ebfb22-0002-5f78-a7aa-5acb5ac7daa9") @@ -47,7 +79,7 @@ assert_equal megaphone_podcast.id, "A1B2C4D5E6F7G8" end - it "should find a podcast by guid" do + it "should find and update a podcast by guid" do stub_request(:get, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts?externalId=95ebfb22-0002-5f78-a7aa-5acb5ac7daa9") .to_return(status: 200, body: [{id: "DEF67890", updatedAt: "2024-11-03T14:54:02.690Z"}].to_json, headers: {}) From 90312918f23a042d13194cde113125bd3f4fda53 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Fri, 6 Dec 2024 20:30:38 -0500 Subject: [PATCH 46/71] New unfinished scope for episode integrations --- .../integrations/episode_integrations.rb | 20 +++++++++++++++++++ app/models/megaphone/publisher.rb | 12 +++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/app/models/integrations/episode_integrations.rb b/app/models/integrations/episode_integrations.rb index 28dc229d3..1f255cd25 100644 --- a/app/models/integrations/episode_integrations.rb +++ b/app/models/integrations/episode_integrations.rb @@ -5,6 +5,26 @@ module Integrations::EpisodeIntegrations included do has_many :episode_delivery_statuses, -> { order(created_at: :desc) }, class_name: "Integrations::EpisodeDeliveryStatus" + has_many :sync_logs, -> { episodes }, foreign_key: "feeder_id" + + scope :unfinished, ->(integration) do + int = Integrations::EpisodeDeliveryStatus.integrations[integration] + frag = <<~SQL + left join lateral ( + select "integrations_episode_delivery_statuses".* + from "integrations_episode_delivery_statuses" + where "episodes"."id" = "integrations_episode_delivery_statuses"."episode_id" + order by "integrations_episode_delivery_statuses"."created_at" desc + limit 1 + ) eds on true + SQL + joins(frag) + .where('("eds"."episode_id" is null) or (("eds"."delivered" = false or "eds"."uploaded" = false) and "eds"."integration" = ?)', int) + end + end + + def sync_log(integration) + sync_logs.send(integration.intern).order(updated_at: :desc).first end def episode_delivery_status(integration) diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index e22d4e1c8..fdfc79023 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -27,8 +27,16 @@ def publish! end def sync_episodes! - episodes = episodes_to_sync - puts "deliver_and_publish!: #{episodes.count}" + Rails.logger.tagged("Megaphone::Publisher#sync_episodes!") do + # start with drafts, make sure they have been at least created + private_feed.episodes.unfinished(:megaphone).each do |ep| + puts ep.class.name + puts ep.inspect + # if it is new, create it - meaning no external_id + # create episodes that don't have an external id + # or can't be found by guid? + end + end end def sync_podcast! From f23eed25157f7035e57667c8553bfa8b6d0777ee Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sat, 7 Dec 2024 12:38:18 -0500 Subject: [PATCH 47/71] Refactor Feed#apple? to Feed#integration_type --- app/controllers/feeds_controller.rb | 2 +- app/models/apple/episode.rb | 4 ---- app/models/concerns/episode_has_feeds.rb | 2 +- app/models/feed.rb | 8 ++------ app/models/feeds/apple_subscription.rb | 4 ---- app/views/podcast_planner/_form_draft_settings.html.erb | 2 +- config/locales/en.yml | 1 + 7 files changed, 6 insertions(+), 17 deletions(-) diff --git a/app/controllers/feeds_controller.rb b/app/controllers/feeds_controller.rb index 1e3e9d729..a2bbccfc9 100644 --- a/app/controllers/feeds_controller.rb +++ b/app/controllers/feeds_controller.rb @@ -24,7 +24,7 @@ def new end def get_apple_show_options(feed) - if feed.apple? && feed.apple_config&.key + if feed.integration_type == :apple && feed.apple_config&.key feed.apple_show_options end end diff --git a/app/models/apple/episode.rb b/app/models/apple/episode.rb index ad457382d..6973e6b9a 100644 --- a/app/models/apple/episode.rb +++ b/app/models/apple/episode.rb @@ -432,10 +432,6 @@ def publishing_state_parameters(state) self.class.publishing_state_params(apple_id, state) end - def apple? - feeder_episode.apple? - end - def apple_json return nil unless api_response.present? diff --git a/app/models/concerns/episode_has_feeds.rb b/app/models/concerns/episode_has_feeds.rb index f2e5eb205..d5afbeef3 100644 --- a/app/models/concerns/episode_has_feeds.rb +++ b/app/models/concerns/episode_has_feeds.rb @@ -24,7 +24,7 @@ module EpisodeHasFeeds def set_default_feeds if feeds.blank? self.feeds = (podcast&.feeds || []).filter_map do |feed| - feed if feed.default? || feed.apple? + feed if feed.default? || feed.integration_type end end end diff --git a/app/models/feed.rb b/app/models/feed.rb index 51756b42a..5bda742df 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -106,8 +106,8 @@ def sanitize_text def friendly_title if default? I18n.t("helpers.label.feed.friendly_titles.default") - elsif apple? - I18n.t("helpers.label.feed.friendly_titles.apple") + elsif integration_type + I18n.t("helpers.label.feed.friendly_titles.#{integration_type}") else title end @@ -168,10 +168,6 @@ def public? !private? end - def apple? - false - end - def default_runtime_settings? default? && public? && include_zones.nil? && audio_format.blank? end diff --git a/app/models/feeds/apple_subscription.rb b/app/models/feeds/apple_subscription.rb index 016ca28f4..31aa07485 100644 --- a/app/models/feeds/apple_subscription.rb +++ b/app/models/feeds/apple_subscription.rb @@ -115,10 +115,6 @@ def must_have_token end end - def apple? - true - end - def integration_type :apple end diff --git a/app/views/podcast_planner/_form_draft_settings.html.erb b/app/views/podcast_planner/_form_draft_settings.html.erb index f328804b8..aef3b6bdf 100644 --- a/app/views/podcast_planner/_form_draft_settings.html.erb +++ b/app/views/podcast_planner/_form_draft_settings.html.erb @@ -34,7 +34,7 @@
<% feeds = @podcast.feeds.tab_order %> - <% defaults = feeds.filter_map { |f| f.id if f.default? || f.apple? } %> + <% defaults = feeds.filter_map { |f| f.id if f.default? || f.integration_type } %> <% data = {planner_target: "feedIds", action: "planner#setFeedIds"} %> <%= form.collection_check_boxes(:feed_ids, feeds, :id, :friendly_title, {checked: defaults}) do |b| %> diff --git a/config/locales/en.yml b/config/locales/en.yml index 4e9753270..5f2cbd466 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -416,6 +416,7 @@ en: friendly_titles: apple: Apple Subscriptions default: Default RSS Feed + megaphone: Megaphone Integration house: House Ads include_donation_url: Include Donation Form include_podcast_value: Include Micropayments Wallet From 9f4bed867ada7ab89a5b7bba0571c805bd17fd95 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sat, 7 Dec 2024 12:39:49 -0500 Subject: [PATCH 48/71] Fix megaphone api setting post/put body --- app/models/megaphone/api.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/megaphone/api.rb b/app/models/megaphone/api.rb index 40316161d..f8494e8f1 100644 --- a/app/models/megaphone/api.rb +++ b/app/models/megaphone/api.rb @@ -21,13 +21,17 @@ def get(path, params = {}, headers = {}) def post(path, body, headers = {}) request = {url: join_url(path), headers: headers, body: outgoing_body_filter(body)} - response = connection(request).post + response = connection(request.slice(:url, :headers)).post do |req| + req.body = request[:body] + end result(request, response) end def put(path, body, headers = {}) request = {url: join_url(path), headers: headers, body: outgoing_body_filter(body)} - response = connection(request).put + response = connection(request.slice(:url, :headers)).put do |req| + req.body = request[:body] + end result(request, response) end From f13bffe20e5041f0a67728c679c958fa551eaac4 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sat, 7 Dec 2024 12:40:25 -0500 Subject: [PATCH 49/71] More sensible, and largely unused, feed defaults --- app/models/feeds/megaphone_feed.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/feeds/megaphone_feed.rb b/app/models/feeds/megaphone_feed.rb index d1a1ead3b..54e012f12 100644 --- a/app/models/feeds/megaphone_feed.rb +++ b/app/models/feeds/megaphone_feed.rb @@ -16,8 +16,8 @@ def integration_type end def set_defaults - self.slug ||= "Megaphone" - self.title ||= "Megaphone" + self.slug = "prx-#{id}" + self.title = podcast&.title self.private = true super From 40f57dd61bcb750c89242b01c4f68dcb11676b63 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sat, 7 Dec 2024 12:45:31 -0500 Subject: [PATCH 50/71] Basic megaphone episode crud methods --- app/models/megaphone/episode.rb | 83 +++++++++++++++++++++++---- test/models/megaphone/episode_test.rb | 18 ++++-- 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index 7dc80f390..8daed9ddb 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -1,7 +1,7 @@ module Megaphone class Episode < Integrations::Base::Episode include Megaphone::Model - attr_accessor :private_feed + attr_accessor :podcast # Used to form the adhash value ADHASH_VALUES = {"pre" => "0", "mid" => "1", "post" => "2"}.freeze @@ -17,7 +17,7 @@ class Episode < Integrations::Base::Episode # All other attributes we might expect back from the Megaphone API # (some documented, others not so much) - OTHER_ATTRIBUTES = %i[id created_at updated_at] + OTHER_ATTRIBUTES = %i[id podcast_id created_at updated_at] DEPRECATED = %i[] @@ -31,11 +31,27 @@ class Episode < Integrations::Base::Episode validates_absence_of :id, on: :create - def self.new_from_episode(feed, feeder_episode) + def self.find_by_episode(megaphone_podcast, feeder_episode) + episode = new_from_episode(megaphone_podcast, feeder_episode) + sync_log = feeder_episode.sync_log(:megaphone) + mp = episode.find_by_megaphone_id(sync_log&.external_id) + mp ||= episode.find_by_guid(feeder_episode.guid) + mp + end + + def self.new_from_episode(megaphone_podcast, feeder_episode) + # start with basic attributes copied from the feeder episode episode = Megaphone::Episode.new(attributes_from_episode(feeder_episode)) + + # set relations to the feeder episode and megaphone podcast episode.feeder_episode = feeder_episode - episode.private_feed = feed - episode.config = feed.config + episode.podcast = megaphone_podcast + episode.config = megaphone_podcast.config + + # we should always be able to set these, published or not + # this does make a remote call to get the placements from augury + episode.set_placement_attributes + episode end @@ -60,6 +76,46 @@ def self.attributes_from_episode(e) } end + def find_by_guid(guid = feeder_episode.guid) + return nil if guid.blank? + self.api_response = api.get("podcasts/#{podcast.id}/episodes", externalId: guid) + handle_response(api_response) + end + + def find_by_megaphone_id(mpid = id) + return nil if mpid.blank? + self.api_response = api.get("podcasts/#{podcast.id}/episodes/#{mpid}") + handle_response(api_response) + end + + def create! + validate!(:create) + body = as_json(only: CREATE_ATTRIBUTES.map(&:to_s)) + self.api_response = api.post("podcasts/#{podcast.id}/episodes", body) + handle_response(api_response) + end + + def update!(feed = nil) + if feed + self.attributes = self.class.attributes_from_feed(feed) + end + validate!(:update) + body = as_json(only: UPDATE_ATTRIBUTES.map(&:to_s)) + self.api_response = api.put("podcasts/#{podcast.id}/episodes/#{id}", body) + handle_response(api_response) + end + + def handle_response(api_response) + if (item = (api_response[:items] || []).first) + self.attributes = item.slice(*ALL_ATTRIBUTES) + self + end + end + + def private_feed + podcast.private_feed + end + def synced_with_integration? delivery_status&.delivered? end @@ -72,15 +128,20 @@ def archived? false end + def feeder_podcast + feeder_episode.podcast + end + def delivery_status feeder_episode&.episode_delivery_status(:megaphone) end def set_placement_attributes - placement = get_placement(feeder_episode.segment_count) - self.expected_adhash = adhash_for_placement(placement) - self.pre_count = expected_adhash.count("0") - self.post_count = expected_adhash.count("2") + if (placement = get_placement(feeder_episode.segment_count)) + self.expected_adhash = adhash_for_placement(placement) + self.pre_count = expected_adhash.count("0") + self.post_count = expected_adhash.count("2") + end end def adhash_for_placement(placement) @@ -92,7 +153,7 @@ def adhash_for_placement(placement) end def get_placement(original_count) - placements = Prx::Augury.new.placements(@podcast.id) + placements = Prx::Augury.new.placements(feeder_podcast.id) placements&.find { |i| i.original_count == original_count } end @@ -125,7 +186,7 @@ def arrangement_version_url(location, media_version) def enclosure_url url = EnclosureUrlBuilder.new.base_enclosure_url( - feeder_episode.podcast, + feeder_podcast, feeder_episode, private_feed ) diff --git a/test/models/megaphone/episode_test.rb b/test/models/megaphone/episode_test.rb index 834a85ae9..a6ebd20a8 100644 --- a/test/models/megaphone/episode_test.rb +++ b/test/models/megaphone/episode_test.rb @@ -1,13 +1,23 @@ require "test_helper" describe Megaphone::Episode do - let(:podcast) { create(:podcast) } - let(:feed) { create(:megaphone_feed, podcast: podcast) } - let(:feeder_episode) { create(:episode, podcast: podcast) } + let(:feeder_podcast) { create(:podcast) } + let(:feed) { create(:megaphone_feed, podcast: feeder_podcast) } + let(:feeder_episode) { create(:episode, podcast: feeder_podcast, segment_count: 2) } + let(:podcast) { Megaphone::Podcast.new_from_feed(feed) } describe "#valid?" do + before { + stub_request(:post, "https://#{ENV["ID_HOST"]}/token") + .to_return(status: 200, + body: '{"access_token":"thisisnotatoken","token_type":"bearer"}', + headers: {"Content-Type" => "application/json; charset=utf-8"}) + + stub_request(:get, "https://#{ENV["AUGURY_HOST"]}/api/v1/podcasts/#{feeder_podcast.id}/placements") + .to_return(status: 200, body: json_file(:placements), headers: {}) + } it "must have required attributes" do - episode = Megaphone::Episode.new_from_episode(feed, feeder_episode) + episode = Megaphone::Episode.new_from_episode(podcast, feeder_episode) assert_not_nil episode assert_equal episode.feeder_episode, feeder_episode assert_equal episode.private_feed, feed From d579e5c74465e2b503361c6b1a306e297e81286d Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sat, 7 Dec 2024 12:46:11 -0500 Subject: [PATCH 51/71] Instantiate megaphone episodes with a megaphone podcast --- app/models/megaphone/podcast.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/megaphone/podcast.rb b/app/models/megaphone/podcast.rb index 869604ce3..affdfd73f 100644 --- a/app/models/megaphone/podcast.rb +++ b/app/models/megaphone/podcast.rb @@ -80,7 +80,7 @@ def self.attributes_from_feed(feed) end def build_integration_episode(feeder_episode) - Megaphone::Episode.new_from_episode(private_feed, feeder_episode) + Megaphone::Episode.new_from_episode(self, feeder_episode) end def updated_at=(d) From 6a41795443005c1ccced047ddf30f9a290941d6a Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sat, 7 Dec 2024 12:47:13 -0500 Subject: [PATCH 52/71] Get publisher to create megaphone episode drafts --- app/models/megaphone/publisher.rb | 49 ++++++++++++++----------- test/models/megaphone/publisher_test.rb | 19 +++++++++- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index fdfc79023..55a455990 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -1,17 +1,18 @@ module Megaphone class Publisher < Integrations::Base::Publisher - attr_reader :feed, :megaphone_podcast - - alias_method :private_feed, :feed + attr_reader :feed def initialize(feed) @feed = feed end - def show + def megaphone_podcast @megaphone_podcast ||= Megaphone::Podcast.find_by_feed(feed) end + alias_method :show, :megaphone_podcast + alias_method :private_feed, :feed + def publish! sync_podcast! sync_episodes! @@ -30,36 +31,42 @@ def sync_episodes! Rails.logger.tagged("Megaphone::Publisher#sync_episodes!") do # start with drafts, make sure they have been at least created private_feed.episodes.unfinished(:megaphone).each do |ep| - puts ep.class.name - puts ep.inspect - # if it is new, create it - meaning no external_id - # create episodes that don't have an external id - # or can't be found by guid? + # see if we can find it by guid or megaphone id + if (megaphone_episode = Megaphone::Episode.find_by_episode(megaphone_podcast, ep)) + megaphone_episode.update!(ep) + end + + megaphone_episode ||= Megaphone::Episode.new_from_episode(megaphone_podcast, ep).create! + + SyncLog.log!( + integration: :megaphone, + feeder_id: ep.id, + feeder_type: :episodes, + external_id: megaphone_episode.id, + api_response: megaphone_episode.api_response_log_item + ) end end end def sync_podcast! - if (@megaphone_podcast = Megaphone::Podcast.find_by_feed(feed)) - # see if we need to update by comparing dates - # - there is no episode delivery status ;) - if @megaphone_podcast.updated_at < podcast.updated_at - @megaphone_podcast.update!(feed) - end + # see if we need to update by comparing dates + # - there is no episode delivery status ;) + if megaphone_podcast && (megaphone_podcast.updated_at < podcast.updated_at) + megaphone_podcast.update!(feed) end - @megaphone_podcast ||= if @megaphone_podcast.blank? - Megaphone::Podcast.new_from_feed(feed).create! - end + # if that didn't find & update a podcast, create it and set it + @megaphone_podcast ||= Megaphone::Podcast.new_from_feed(feed).create! SyncLog.log!( integration: :megaphone, feeder_id: public_feed.id, feeder_type: :feeds, - external_id: @megaphone_podcast.id, - api_response: @megaphone_podcast.api_response_log_item + external_id: megaphone_podcast.id, + api_response: megaphone_podcast.api_response_log_item ) - @megaphone_podcast + megaphone_podcast end def config diff --git a/test/models/megaphone/publisher_test.rb b/test/models/megaphone/publisher_test.rb index 2a1dbb4dc..17bcbad06 100644 --- a/test/models/megaphone/publisher_test.rb +++ b/test/models/megaphone/publisher_test.rb @@ -1,7 +1,7 @@ require "test_helper" describe Megaphone::Publisher do - let(:podcast) { create(:podcast) } + let(:podcast) { create(:podcast, id: 1234) } let(:public_feed) { podcast.default_feed } let(:feed) { create(:megaphone_feed, podcast: podcast, private: true) } @@ -21,6 +21,7 @@ let(:episode) { create(:episode, podcast: podcast) } before do + # setup the megaphone podcast stub_request(:get, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts/A1B2C4D5E6F7G8") .to_return(status: 200, body: {id: "A1B2C4D5E6F7G8", updatedAt: (Time.now + 1.minute).utc.iso8601}.to_json, headers: {}) @@ -31,11 +32,25 @@ external_id: "A1B2C4D5E6F7G8", api_response: {request: {}, items: {}} ) + + # setup augury placements for the podcast + stub_request(:post, "https://#{ENV["ID_HOST"]}/token") + .to_return(status: 200, + body: '{"access_token":"thisisnotatoken","token_type":"bearer"}', + headers: {"Content-Type" => "application/json; charset=utf-8"}) + + stub_request(:get, "https://#{ENV["AUGURY_HOST"]}/api/v1/podcasts/#{podcast.id}/placements") + .to_return(status: 200, body: json_file(:placements), headers: {}) + + stub_request(:get, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts/A1B2C4D5E6F7G8/episodes?externalId=#{episode.guid}") + .to_return(status: 200, body: [].to_json, headers: {}) + + stub_request(:post, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts/A1B2C4D5E6F7G8/episodes") + .to_return(status: 200, body: {id: "megaphone-episode-guid"}.to_json, headers: {}) end it "should create new draft episodes" do assert episode - puts episode.inspect publisher.sync_episodes! end From f070ccc2a0d61b229531da59fe4ab27d918acb3a Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sun, 8 Dec 2024 12:55:59 -0500 Subject: [PATCH 53/71] Reuse methods in the super class --- app/models/apple/episode.rb | 10 ---------- app/models/integrations/base/episode.rb | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/models/apple/episode.rb b/app/models/apple/episode.rb index 6973e6b9a..dcd1ec957 100644 --- a/app/models/apple/episode.rb +++ b/app/models/apple/episode.rb @@ -506,16 +506,6 @@ def audio_asset_state_success? audio_asset_state == AUDIO_ASSET_SUCCESS end - def has_media_version? - return false unless delivery_status.present? && delivery_status.source_media_version_id.present? - - delivery_status.source_media_version_id == feeder_episode.media_version_id - end - - def needs_media_version? - !has_media_version? - end - def needs_delivery? return true if missing_container? diff --git a/app/models/integrations/base/episode.rb b/app/models/integrations/base/episode.rb index b1b2ae181..ed069fd92 100644 --- a/app/models/integrations/base/episode.rb +++ b/app/models/integrations/base/episode.rb @@ -19,6 +19,16 @@ def video_content_type? feeder_episode.video_content_type? end + def has_media_version? + return false unless delivery_status.present? && delivery_status.source_media_version_id.present? + + delivery_status.source_media_version_id == feeder_episode.media_version_id + end + + def needs_media_version? + !has_media_version? + end + # Delegate methods to feeder_episode def method_missing(method_name, *arguments, &block) if feeder_episode.respond_to?(method_name) From 03c0757a93502bdc83755aaf7ee708b0a24a7e73 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sun, 8 Dec 2024 12:57:05 -0500 Subject: [PATCH 54/71] Update episode delivery status access to optionally create a default record --- app/models/integrations/episode_integrations.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/models/integrations/episode_integrations.rb b/app/models/integrations/episode_integrations.rb index 1f255cd25..3d8d1d8a5 100644 --- a/app/models/integrations/episode_integrations.rb +++ b/app/models/integrations/episode_integrations.rb @@ -27,8 +27,13 @@ def sync_log(integration) sync_logs.send(integration.intern).order(updated_at: :desc).first end - def episode_delivery_status(integration) - episode_delivery_statuses.order(created_at: :desc).send(integration.intern).first + def episode_delivery_status(integration, with_default = false) + status = episode_delivery_statuses.order(created_at: :desc).send(integration.intern).first + if !status && with_default + Integrations::EpisodeDeliveryStatus.default_status(integration, self) + else + status + end end def update_episode_delivery_status(integration, attrs) From 2c709b66c199f91d57d7609eff29476a276e4a59 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sun, 8 Dec 2024 13:30:30 -0500 Subject: [PATCH 55/71] Megaphone episode audio publishing --- app/models/megaphone/episode.rb | 92 +++++++++++++++++++++---- app/models/megaphone/publisher.rb | 98 ++++++++++++++++++--------- test/models/megaphone/episode_test.rb | 9 +++ 3 files changed, 153 insertions(+), 46 deletions(-) diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index 8daed9ddb..814029fba 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -3,6 +3,10 @@ class Episode < Integrations::Base::Episode include Megaphone::Model attr_accessor :podcast + # track upload source data + SOURCE_ATTRIBUTES = [:source_media_version_id, :source_size, :source_fetch_count, :source_url, :source_filename] + attr_accessor(*SOURCE_ATTRIBUTES) + # Used to form the adhash value ADHASH_VALUES = {"pre" => "0", "mid" => "1", "post" => "2"}.freeze @@ -17,7 +21,8 @@ class Episode < Integrations::Base::Episode # All other attributes we might expect back from the Megaphone API # (some documented, others not so much) - OTHER_ATTRIBUTES = %i[id podcast_id created_at updated_at] + OTHER_ATTRIBUTES = %i[id podcast_id created_at updated_at status + download_url audio_file_processing audio_file_status audio_file_updated_at] DEPRECATED = %i[] @@ -52,6 +57,9 @@ def self.new_from_episode(megaphone_podcast, feeder_episode) # this does make a remote call to get the placements from augury episode.set_placement_attributes + # may move this later, but let's also see about audio + episode.set_audio_attributes + episode end @@ -93,6 +101,8 @@ def create! body = as_json(only: CREATE_ATTRIBUTES.map(&:to_s)) self.api_response = api.post("podcasts/#{podcast.id}/episodes", body) handle_response(api_response) + update_delivery_status + self end def update!(feed = nil) @@ -103,6 +113,8 @@ def update!(feed = nil) body = as_json(only: UPDATE_ATTRIBUTES.map(&:to_s)) self.api_response = api.put("podcasts/#{podcast.id}/episodes/#{id}", body) handle_response(api_response) + update_delivery_status + self end def handle_response(api_response) @@ -112,6 +124,19 @@ def handle_response(api_response) end end + # update delivery status after a create or update + def update_delivery_status + # if there's audio and we just uploaded it successfully, set attr, then check status + if feeder_episode.complete_media? && background_audio_file_url + attrs = source_attributes.merge(uploaded: true) + feeder_episode.update_episode_delivery_status(:megaphone, attrs) + # or if there's not audio yet or it didn't change + else + # we're done, mark it as delivered! + delivery_status(true).mark_as_delivered! + end + end + def private_feed podcast.private_feed end @@ -132,8 +157,8 @@ def feeder_podcast feeder_episode.podcast end - def delivery_status - feeder_episode&.episode_delivery_status(:megaphone) + def delivery_status(with_default = false) + feeder_episode&.episode_delivery_status(:megaphone, with_default) end def set_placement_attributes @@ -160,30 +185,67 @@ def get_placement(original_count) # call this before create or update, yah def set_audio_attributes return unless feeder_episode.complete_media? - self.background_audio_file_url = upload_url - self.insertion_points = timings - self.retain_ad_locations = true + + # check if the version is different from what was saved before + if !has_media_version? + media_info = get_media_info(enclosure_url) + + # if dovetail has the right media info, we can update + if media_info[:media_version] == feeder_episode.media_version_id + self.source_media_version_id = media_info[:media_version] + self.source_size = media_info[:size] + self.source_fetch_count = (delivery_status&.source_fetch_count || 0) + 1 + self.source_url = arrangement_version_url(media_info[:location], media_info[:media_version], source_fetch_count) + self.source_filename = url_filename(source_url) + self.background_audio_file_url = source_url + self.insertion_points = timings + self.retain_ad_locations = true + else + # if not, mark it as not uploaded and move on + delivery_status.mark_as_not_uploaded! + end + end + end + + def source_attributes + attributes.slice(*SOURCE_ATTRIBUTES) end - def upload_url - resp = Faraday.head(enclosure_url) + def get_media_info(enclosure) + info = { + enclosure_url: enclosure, + media_version: nil, + location: nil, + size: nil + } + resp = Faraday.head(enclosure) if resp.status == 302 - media_version = resp.env.response_headers["x-episode-media-version"] - if media_version == feeder_episode.media_version_id - location = resp.env.response_headers["location"] - arrangement_version_url(location, media_version) - end + info[:media_version] = resp.env.response_headers["x-episode-media-version"] + info[:location] = resp.env.response_headers["location"] + info[:size] = resp.env.response_headers["content-length"] + else + logger.error("DTR media redirect not returned: #{resp.status}", enclosure: enclosure, resp: resp) + raise("DTR media redirect not returned: #{resp.status}") end + info + rescue err + logger.error("Error getting DTR media info", enclosure: enclosure, err: err) + raise err end - def arrangement_version_url(location, media_version) + def arrangement_version_url(location, media_version, count) uri = URI.parse(location) path = uri.path.split("/") ext = File.extname(path.last) - filename = File.basename(path.last, ext) + "_" + media_version + File.extname(path.last) + base = File.basename(path.last, ext) + filename = "#{base}_#{media_version}_#{count}#{ext}" uri.path = (path[0..-2] + [filename]).join("/") end + def url_filename(url) + URI.parse(url).path.split("/").last + end + def enclosure_url url = EnclosureUrlBuilder.new.base_enclosure_url( feeder_podcast, diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index 55a455990..13b9d6d0b 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -29,44 +29,80 @@ def publish! def sync_episodes! Rails.logger.tagged("Megaphone::Publisher#sync_episodes!") do - # start with drafts, make sure they have been at least created - private_feed.episodes.unfinished(:megaphone).each do |ep| - # see if we can find it by guid or megaphone id - if (megaphone_episode = Megaphone::Episode.find_by_episode(megaphone_podcast, ep)) - megaphone_episode.update!(ep) - end - - megaphone_episode ||= Megaphone::Episode.new_from_episode(megaphone_podcast, ep).create! - - SyncLog.log!( - integration: :megaphone, - feeder_id: ep.id, - feeder_type: :episodes, - external_id: megaphone_episode.id, - api_response: megaphone_episode.api_response_log_item - ) + # delete or unpublish episodes we aren't including in the feed anymore + unpublish_and_delete_episodes! + + # start with create and update, make sure they have been created at least + create_and_update_episodes! + + # check if the upload has completed and the audio has finished processing + check_status_episodes! + end + end + + # For status + # :uploaded = the latest media version was ready on dtr, and was saved to mp + # :delivered = attributes saved, media finished processing + # if not uploaded, and media ready, try to set that on the next update + # if uploaded and not delivered, check mp status, see if processing done + def check_status_episodes! + private_feed.episodes.unfinished(:megaphone).each do |ep| + megaphone_episode = Megaphone::Episode.find_by_episode(megaphone_podcast, ep) + next unless megaphone_episode + end + end + + def create_and_update_episodes! + megaphone_episodes = [] + private_feed.episodes.unfinished(:megaphone).each do |ep| + # see if we can find it by guid or megaphone id + if (megaphone_episode = Megaphone::Episode.find_by_episode(megaphone_podcast, ep)) + megaphone_episode.update!(ep) end + + megaphone_episode ||= create_episode!(megaphone_podcast, ep) + + SyncLog.log!( + integration: :megaphone, + feeder_id: ep.id, + feeder_type: :episodes, + external_id: megaphone_episode.id, + api_response: megaphone_episode.api_response_log_item + ) + megaphone_episodes << megaphone_episode end + + megaphone_episodes + end + + def create_episode!(megaphone_podcast, ep) + me = Megaphone::Episode.new_from_episode(megaphone_podcast, ep) + me.create! + end + + def unpublish_and_delete_episodes! end def sync_podcast! - # see if we need to update by comparing dates - # - there is no episode delivery status ;) - if megaphone_podcast && (megaphone_podcast.updated_at < podcast.updated_at) - megaphone_podcast.update!(feed) - end + Rails.logger.tagged("Megaphone::Publisher#sync_podcast!") do + # see if we need to update by comparing dates + # - there is no episode delivery status ;) + if megaphone_podcast && (megaphone_podcast.updated_at < podcast.updated_at) + megaphone_podcast.update!(feed) + end - # if that didn't find & update a podcast, create it and set it - @megaphone_podcast ||= Megaphone::Podcast.new_from_feed(feed).create! + # if that didn't find & update a podcast, create it and set it + @megaphone_podcast ||= Megaphone::Podcast.new_from_feed(feed).create! - SyncLog.log!( - integration: :megaphone, - feeder_id: public_feed.id, - feeder_type: :feeds, - external_id: megaphone_podcast.id, - api_response: megaphone_podcast.api_response_log_item - ) - megaphone_podcast + SyncLog.log!( + integration: :megaphone, + feeder_id: public_feed.id, + feeder_type: :feeds, + external_id: megaphone_podcast.id, + api_response: megaphone_podcast.api_response_log_item + ) + megaphone_podcast + end end def config diff --git a/test/models/megaphone/episode_test.rb b/test/models/megaphone/episode_test.rb index a6ebd20a8..f60d856d1 100644 --- a/test/models/megaphone/episode_test.rb +++ b/test/models/megaphone/episode_test.rb @@ -26,4 +26,13 @@ assert episode.valid? end end + + describe "#create!" do + it "can create a draft with no audio" do + end + it "can create a published episodes with audio" do + end + it "can create a published episodes with the wrong media vetsion from DTR" do + end + end end From f4ea9180d5ce00387de2990bf22b5ff522e578b3 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sun, 8 Dec 2024 16:22:05 -0500 Subject: [PATCH 56/71] Move some sync log updates to integration models --- app/models/megaphone/episode.rb | 12 +++++++++ app/models/megaphone/podcast.rb | 21 +++++++++++++-- app/models/megaphone/publisher.rb | 18 ------------- test/models/megaphone/episode_test.rb | 37 +++++++++++++++++++-------- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index 814029fba..c7a9fbbda 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -101,6 +101,7 @@ def create! body = as_json(only: CREATE_ATTRIBUTES.map(&:to_s)) self.api_response = api.post("podcasts/#{podcast.id}/episodes", body) handle_response(api_response) + update_sync_log update_delivery_status self end @@ -113,10 +114,21 @@ def update!(feed = nil) body = as_json(only: UPDATE_ATTRIBUTES.map(&:to_s)) self.api_response = api.put("podcasts/#{podcast.id}/episodes/#{id}", body) handle_response(api_response) + update_sync_log update_delivery_status self end + def update_sync_log + SyncLog.log!( + integration: :megaphone, + feeder_id: feeder_episode.id, + feeder_type: :episodes, + external_id: id, + api_response: api_response_log_item + ) + end + def handle_response(api_response) if (item = (api_response[:items] || []).first) self.attributes = item.slice(*ALL_ATTRIBUTES) diff --git a/app/models/megaphone/podcast.rb b/app/models/megaphone/podcast.rb index affdfd73f..82118b659 100644 --- a/app/models/megaphone/podcast.rb +++ b/app/models/megaphone/podcast.rb @@ -36,8 +36,7 @@ class Podcast < Integrations::Base::Show def self.find_by_feed(feed) podcast = new_from_feed(feed) - public_feed = feed.podcast.public_feed - sync_log = public_feed.sync_log(:megaphone) + sync_log = podcast.public_feed.sync_log(:megaphone) mp = podcast.find_by_megaphone_id(sync_log&.external_id) mp ||= podcast.find_by_guid(feed.podcast.guid) mp @@ -79,6 +78,10 @@ def self.attributes_from_feed(feed) } end + def public_feed + private_feed.podcast&.public_feed + end + def build_integration_episode(feeder_episode) Megaphone::Episode.new_from_episode(self, feeder_episode) end @@ -110,6 +113,8 @@ def create! body = as_json(only: CREATE_ATTRIBUTES.map(&:to_s)) self.api_response = api.post("podcasts", body) handle_response(api_response) + update_sync_log + self end def update!(feed = nil) @@ -120,6 +125,8 @@ def update!(feed = nil) body = as_json(only: UPDATE_ATTRIBUTES.map(&:to_s)) self.api_response = api.put("podcasts/#{id}", body) handle_response(api_response) + update_sync_log + self end def handle_response(api_response) @@ -128,5 +135,15 @@ def handle_response(api_response) self end end + + def update_sync_log + SyncLog.log!( + integration: :megaphone, + feeder_type: :feeds, + feeder_id: public_feed.id, + external_id: id, + api_response: api_response_log_item + ) + end end end diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index 13b9d6d0b..8884de183 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -16,15 +16,6 @@ def megaphone_podcast def publish! sync_podcast! sync_episodes! - - # success - SyncLog.log!( - integration: :megaphone, - feeder_id: public_feed.id, - feeder_type: :feeds, - external_id: megaphone_podcast.id, - api_response: {success: true} - ) end def sync_episodes! @@ -59,16 +50,7 @@ def create_and_update_episodes! if (megaphone_episode = Megaphone::Episode.find_by_episode(megaphone_podcast, ep)) megaphone_episode.update!(ep) end - megaphone_episode ||= create_episode!(megaphone_podcast, ep) - - SyncLog.log!( - integration: :megaphone, - feeder_id: ep.id, - feeder_type: :episodes, - external_id: megaphone_episode.id, - api_response: megaphone_episode.api_response_log_item - ) megaphone_episodes << megaphone_episode end diff --git a/test/models/megaphone/episode_test.rb b/test/models/megaphone/episode_test.rb index f60d856d1..0aef7ccc6 100644 --- a/test/models/megaphone/episode_test.rb +++ b/test/models/megaphone/episode_test.rb @@ -4,18 +4,19 @@ let(:feeder_podcast) { create(:podcast) } let(:feed) { create(:megaphone_feed, podcast: feeder_podcast) } let(:feeder_episode) { create(:episode, podcast: feeder_podcast, segment_count: 2) } - let(:podcast) { Megaphone::Podcast.new_from_feed(feed) } + let(:podcast) { Megaphone::Podcast.new_from_feed(feed).tap { |p| p.id = "mp-123-456" } } - describe "#valid?" do - before { - stub_request(:post, "https://#{ENV["ID_HOST"]}/token") - .to_return(status: 200, - body: '{"access_token":"thisisnotatoken","token_type":"bearer"}', - headers: {"Content-Type" => "application/json; charset=utf-8"}) + before { + stub_request(:post, "https://#{ENV["ID_HOST"]}/token") + .to_return(status: 200, + body: '{"access_token":"thisisnotatoken","token_type":"bearer"}', + headers: {"Content-Type" => "application/json; charset=utf-8"}) - stub_request(:get, "https://#{ENV["AUGURY_HOST"]}/api/v1/podcasts/#{feeder_podcast.id}/placements") - .to_return(status: 200, body: json_file(:placements), headers: {}) - } + stub_request(:get, "https://#{ENV["AUGURY_HOST"]}/api/v1/podcasts/#{feeder_podcast.id}/placements") + .to_return(status: 200, body: json_file(:placements), headers: {}) + } + + describe "#valid?" do it "must have required attributes" do episode = Megaphone::Episode.new_from_episode(podcast, feeder_episode) assert_not_nil episode @@ -28,7 +29,21 @@ end describe "#create!" do - it "can create a draft with no audio" do + before { + stub_request(:post, "https://cms.megaphone.fm/api/networks/this-is-a-network-id/podcasts/mp-123-456/episodes") + .to_return(status: 200, body: {id: "megaphone-episode-guid"}.to_json, headers: {}) + } + it "can create a draft with no audio and mark delivered" do + refute feeder_episode.complete_media? + assert_nil feeder_episode.episode_delivery_status(:megaphone) + assert_nil feeder_episode.sync_log(:megaphone) + + episode = Megaphone::Episode.new_from_episode(podcast, feeder_episode) + episode.create! + + assert_nil episode.background_audio_file_url + assert feeder_episode.sync_log(:megaphone).external_id + assert feeder_episode.episode_delivery_status(:megaphone) end it "can create a published episodes with audio" do end From ad179a7a29110b59d2baa8b7620de25e6027b9bf Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sun, 8 Dec 2024 16:24:50 -0500 Subject: [PATCH 57/71] Move podcast sync log to model as well --- app/models/megaphone/publisher.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index 8884de183..8ab35603b 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -75,14 +75,6 @@ def sync_podcast! # if that didn't find & update a podcast, create it and set it @megaphone_podcast ||= Megaphone::Podcast.new_from_feed(feed).create! - - SyncLog.log!( - integration: :megaphone, - feeder_id: public_feed.id, - feeder_type: :feeds, - external_id: megaphone_podcast.id, - api_response: megaphone_podcast.api_response_log_item - ) megaphone_podcast end end From 74c371ab603205eceacf615e893184a625446a04 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sun, 8 Dec 2024 16:55:03 -0500 Subject: [PATCH 58/71] Make migration reversible --- .../20241127171040_add_integration_to_sync_logs.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/db/migrate/20241127171040_add_integration_to_sync_logs.rb b/db/migrate/20241127171040_add_integration_to_sync_logs.rb index 9e2cf4d1a..26e8c8de0 100644 --- a/db/migrate/20241127171040_add_integration_to_sync_logs.rb +++ b/db/migrate/20241127171040_add_integration_to_sync_logs.rb @@ -1,5 +1,5 @@ class AddIntegrationToSyncLogs < ActiveRecord::Migration[7.2] - def change + def up add_column :sync_logs, :integration, :integer execute(<<~SQL @@ -11,4 +11,11 @@ def change remove_index :sync_logs, [:feeder_type, :feeder_id], unique: true add_index :sync_logs, [:integration, :feeder_type, :feeder_id], unique: true end + + def down + remove_index :sync_logs, [:integration, :feeder_type, :feeder_id] + remove_column :sync_logs, :integration + + add_index :sync_logs, [:feeder_type, :feeder_id], unique: true + end end From 84e6b566a566b04f0f7531e1035ed75d368b3371 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 00:20:24 -0500 Subject: [PATCH 59/71] Initial cuepoint impl --- .../integrations/episode_delivery_status.rb | 2 +- .../integrations/episode_integrations.rb | 2 +- app/models/megaphone/cuepoint.rb | 61 +++++++++++++ app/models/megaphone/episode.rb | 85 ++++++++++++++++--- app/models/megaphone/publisher.rb | 12 +++ 5 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 app/models/megaphone/cuepoint.rb diff --git a/app/models/integrations/episode_delivery_status.rb b/app/models/integrations/episode_delivery_status.rb index ceaa58f44..9f5c4e017 100644 --- a/app/models/integrations/episode_delivery_status.rb +++ b/app/models/integrations/episode_delivery_status.rb @@ -36,7 +36,7 @@ def mark_as_not_uploaded! # Whether the media file has been uploaded to the Integration # is a subset of whether the episode has been delivered def mark_as_delivered! - self.class.update_status(integration, episode, delivered: true, uploaded: true) + self.class.update_status(integration, episode, delivered: true, uploaded: true, asset_processing_attempts: 0) end def mark_as_not_delivered! diff --git a/app/models/integrations/episode_integrations.rb b/app/models/integrations/episode_integrations.rb index 3d8d1d8a5..ad28f1e72 100644 --- a/app/models/integrations/episode_integrations.rb +++ b/app/models/integrations/episode_integrations.rb @@ -28,7 +28,7 @@ def sync_log(integration) end def episode_delivery_status(integration, with_default = false) - status = episode_delivery_statuses.order(created_at: :desc).send(integration.intern).first + status = episode_delivery_statuses.reset.order(created_at: :desc).send(integration.intern).first if !status && with_default Integrations::EpisodeDeliveryStatus.default_status(integration, self) else diff --git a/app/models/megaphone/cuepoint.rb b/app/models/megaphone/cuepoint.rb new file mode 100644 index 000000000..7689f4add --- /dev/null +++ b/app/models/megaphone/cuepoint.rb @@ -0,0 +1,61 @@ +module Megaphone + class Cuepoint + include Megaphone::Model + + CUEPOINT_TYPES = %i[preroll midroll postroll remove] + + AD_SOURCES = %i[auto promo span] + + CREATE_REQUIRED = %i[cuepoint_type ad_count start_time end_time ad_sources action] + + CREATE_ATTRIBUTES = CREATE_REQUIRED + %i[title end_time is_active offset notes] + + ALL_ATTRIBUTES = (CREATE_ATTRIBUTES + DEPRECATED + OTHER_ATTRIBUTES) + + attr_accessor(*ALL_ATTRIBUTES) + + validates_presence_of CREATE_REQUIRED + + def self.from_placement(zones) + cuepoints = [] + current_cuepoint = nil + original_duration = 0 + original_count = 0 + zones.each do |zone| + # if this is an ad zone, add it to the cue point + if zone[:type] == "ad" + if current_cuepoint + current_cuepoint.ad_count = current_cuepoint.ad_count + 1 + current_cuepoint.ad_sources << source_for_zone(zone) + else + current_cuepoint = new( + cuepoint_type: "#{zone[:section]}roll", + ad_count: 1, + start_time: original_duration, + ad_sources: [source_for_zone(zone)], + action: :insert, + is_active: true + ) + cuepoints << current_cuepoint + end + elsif zone[:type] == "original" + current_cuepoint = nil + original_duration += feeder_episode.media[original_count].duration + original_count += 1 + end + end + end + + def source_for_zone(zone) + if zone[:id].match?(/^house/) + :promo + else + :auto + end + end + + def as_json_for_create + as_json(only: CREATE_ATTRIBUTES.map(&:to_s)) + end + end +end diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index c7a9fbbda..46e4ca5b1 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -119,6 +119,61 @@ def update!(feed = nil) self end + # call this when we need to update the audio on mp + # like when dtr wasn't ready at first + # so we can make that update and then mark uploaded + def upload_audio! + set_audio_attributes + update! + end + + # call this when audio has been updated on mp + # and we're checking to see if mp is done processing + # so we can update cuepoints and mark delivered + def check_audio! + # Re-request the megaphone api + find_by_megaphone_id + + # get the audip attributes + set_audio_attributes + + # check to see if the audio on mp matches + if original_filename == source_filename + if !audio_file_processing && audio_file_status == "success" + reset_asset_wait replace_cuepoints!(episode) + delivery_status(true).mark_as_delivered! + else + # still waiting - increment asset state + delivery_status(true).increment_asset_wait + end + else + # this would be a weird timing thing maybe, but ... + # if the files don't match, we need to go back and upload + delivery_status(true).mark_as_not_uploaded! + end + end + + def replace_cuepoints!(episode) + # retrieve the placement info from augury + zones = get_placement_zones(feeder_episode.segment_count) + + # create cuepoint instances from that + cuepoints = Megaphone::Cuepoint.from_placement(zones) + + # put those as a list to the mp api + cuepoints_batch!(cuepoints) + end + + def cuepoints_batch!(cuepoints) + # validate all the cuepoints about to be created + cuepoints.all? { |cp| cp.validate!(:create) } + body = cuepoints.map { |cp| cp.as_json_for_create } + self.api_response = api.put("podcasts/#{podcast.id}/episodes/#{id}/cuepoints_batch", body) + update_sync_log + update_delivery_status + self + end + def update_sync_log SyncLog.log!( integration: :megaphone, @@ -147,6 +202,7 @@ def update_delivery_status # we're done, mark it as delivered! delivery_status(true).mark_as_delivered! end + feeder_episode.episode_delivery_statuses.reset end def private_feed @@ -174,24 +230,27 @@ def delivery_status(with_default = false) end def set_placement_attributes - if (placement = get_placement(feeder_episode.segment_count)) - self.expected_adhash = adhash_for_placement(placement) + if (zones = get_placement_zones(feeder_episode.segment_count)) + self.expected_adhash = adhash_for_placement(zones) self.pre_count = expected_adhash.count("0") self.post_count = expected_adhash.count("2") end end - def adhash_for_placement(placement) - placement - .zones - .filter { |z| z["type"] == "ad" } - .map { |z| ADHASH_VALUES[z["section"]] } + def adhash_for_placement(zones) + zones + .filter { |z| z[:type] == "ad" } + .map { |z| ADHASH_VALUES[z[:section]] } .join("") end - def get_placement(original_count) + def get_placement_zones(original_count = nil) + if original_count.to_i < 1 + original_count = (feeder_episode&.segment_count || 1).to_i + end placements = Prx::Augury.new.placements(feeder_podcast.id) - placements&.find { |i| i.original_count == original_count } + placement = placements&.find { |i| i.original_count == original_count } + (placement&.zones || []).map(&:with_indifferent_access) end # call this before create or update, yah @@ -271,13 +330,13 @@ def timings feeder_episode.media[0..-2].map(&:duration) end - def pre_after_original?(placement) - sections = placement.zones.split { |z| z[:type] == "original" } + def pre_after_original?(zones) + sections = zones.split { |z| z[:type] == "original" } sections[1].any? { |z| %w[ad house].include?(z[:type]) && z[:id].match(/pre/) } end - def post_before_original?(placement) - sections = placement.zones.split { |z| z[:type] == "original" } + def post_before_original?(zones) + sections = zones.split { |z| z[:type] == "original" } sections[-2].any? { |z| %w[ad house].include?(z[:type]) && z[:id].match(/post/) } end end diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index 8ab35603b..477ba5545 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -40,6 +40,18 @@ def check_status_episodes! private_feed.episodes.unfinished(:megaphone).each do |ep| megaphone_episode = Megaphone::Episode.find_by_episode(megaphone_podcast, ep) next unless megaphone_episode + + # check if it is uploaded yet + # if not go looking for the DTR media version + if !megaphone_episode.delivery_status.uploaded? + megaphone_episode.upload_audio! + end + + # check if it is uploaded, but not delivered - see if megaphone has processed + status = megaphone_episode.delivery_status + if !status.delivered? && status.uploaded? + megaphone_episode.check_audio! + end end end From 04499182661673a6f94b912d9ace389b3503882b Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 00:56:19 -0500 Subject: [PATCH 60/71] Retry checking audio status, raise error when failing --- app/models/megaphone/episode.rb | 2 +- app/models/megaphone/publisher.rb | 29 +++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index 46e4ca5b1..725f23dc8 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -140,7 +140,7 @@ def check_audio! # check to see if the audio on mp matches if original_filename == source_filename if !audio_file_processing && audio_file_status == "success" - reset_asset_wait replace_cuepoints!(episode) + replace_cuepoints!(episode) delivery_status(true).mark_as_delivered! else # still waiting - increment asset state diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index 477ba5545..573517579 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -1,5 +1,8 @@ module Megaphone class Publisher < Integrations::Base::Publisher + WAIT_INTERVAL = 5.seconds + WAIT_TIMEOUT = 5.minutes + attr_reader :feed def initialize(feed) @@ -37,9 +40,27 @@ def sync_episodes! # if not uploaded, and media ready, try to set that on the next update # if uploaded and not delivered, check mp status, see if processing done def check_status_episodes! - private_feed.episodes.unfinished(:megaphone).each do |ep| + episodes = private_feed.episodes.unfinished(:megaphone) + timeout_at = Time.now.utc + WAIT_TIMEOUT + + while episode.size > 0 && Time.now.utc < timeout_at + sleep(wait_interval) + episodes = check_episodes(episodes) + end + + # if after all those checks, still incomplete? throw an error + if episodes.size > 0 + msg = "Megaphone::Publisher.check_status_episodes! timed out on: #{episodes.map(&:id)}" + Logger.error(msg) + raise msg + end + end + + def check_episodes(episodes) + remaining = [] + + episodes.each do |ep| megaphone_episode = Megaphone::Episode.find_by_episode(megaphone_podcast, ep) - next unless megaphone_episode # check if it is uploaded yet # if not go looking for the DTR media version @@ -52,6 +73,10 @@ def check_status_episodes! if !status.delivered? && status.uploaded? megaphone_episode.check_audio! end + + if !ep.episode_delivery_status(:megaphone).delivered? + remaining << ep + end end end From 3676a78f551e7e09a1fa84fe22ebf9d3263b7a51 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 00:58:37 -0500 Subject: [PATCH 61/71] Fix episodes typo --- app/models/megaphone/publisher.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/megaphone/publisher.rb b/app/models/megaphone/publisher.rb index 573517579..f6534bf18 100644 --- a/app/models/megaphone/publisher.rb +++ b/app/models/megaphone/publisher.rb @@ -43,7 +43,7 @@ def check_status_episodes! episodes = private_feed.episodes.unfinished(:megaphone) timeout_at = Time.now.utc + WAIT_TIMEOUT - while episode.size > 0 && Time.now.utc < timeout_at + while episodes.size > 0 && Time.now.utc < timeout_at sleep(wait_interval) episodes = check_episodes(episodes) end From 8812b73feb9af846dc62d513710dd8371169494f Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 01:36:55 -0500 Subject: [PATCH 62/71] cuipoint test --- app/models/megaphone/cuepoint.rb | 13 ++--- app/models/megaphone/episode.rb | 3 +- test/fixtures/zones.json | 74 ++++++++++++++++++++++++++ test/models/megaphone/cuepoint_test.rb | 30 +++++++++++ 4 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/zones.json create mode 100644 test/models/megaphone/cuepoint_test.rb diff --git a/app/models/megaphone/cuepoint.rb b/app/models/megaphone/cuepoint.rb index 7689f4add..96ed58a19 100644 --- a/app/models/megaphone/cuepoint.rb +++ b/app/models/megaphone/cuepoint.rb @@ -10,20 +10,20 @@ class Cuepoint CREATE_ATTRIBUTES = CREATE_REQUIRED + %i[title end_time is_active offset notes] - ALL_ATTRIBUTES = (CREATE_ATTRIBUTES + DEPRECATED + OTHER_ATTRIBUTES) + ALL_ATTRIBUTES = CREATE_ATTRIBUTES attr_accessor(*ALL_ATTRIBUTES) validates_presence_of CREATE_REQUIRED - def self.from_placement(zones) + def self.from_zones_and_media(zones, media) cuepoints = [] current_cuepoint = nil original_duration = 0 original_count = 0 zones.each do |zone| # if this is an ad zone, add it to the cue point - if zone[:type] == "ad" + if ["ad", "sonic_id"].include?(zone[:type]) if current_cuepoint current_cuepoint.ad_count = current_cuepoint.ad_count + 1 current_cuepoint.ad_sources << source_for_zone(zone) @@ -40,14 +40,15 @@ def self.from_placement(zones) end elsif zone[:type] == "original" current_cuepoint = nil - original_duration += feeder_episode.media[original_count].duration + original_duration += media[original_count].duration original_count += 1 end end + cuepoints end - def source_for_zone(zone) - if zone[:id].match?(/^house/) + def self.source_for_zone(zone) + if zone[:id].match?(/^house/) || zone[:type] == "sonic_id" :promo else :auto diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index 725f23dc8..1b7c15b79 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -156,9 +156,10 @@ def check_audio! def replace_cuepoints!(episode) # retrieve the placement info from augury zones = get_placement_zones(feeder_episode.segment_count) + media = feeder_episode.media # create cuepoint instances from that - cuepoints = Megaphone::Cuepoint.from_placement(zones) + cuepoints = Megaphone::Cuepoint.from_zones_and_media(zones, media) # put those as a list to the mp api cuepoints_batch!(cuepoints) diff --git a/test/fixtures/zones.json b/test/fixtures/zones.json new file mode 100644 index 000000000..70efc6a3c --- /dev/null +++ b/test/fixtures/zones.json @@ -0,0 +1,74 @@ +[ + { + "id": "original_1", + "name": "Original 1 (Intro)", + "type": "original", + "section": "original" + }, + { + "id": "house_pre", + "name": "House Preroll", + "type": "ad", + "section": "pre" + }, + { + "id": "pre_1", + "name": "Preroll 1", + "type": "ad", + "section": "pre" + }, + { + "id": "pre_2", + "name": "Preroll 2", + "type": "ad", + "section": "pre" + }, + { + "id": "original_2", + "name": "Original 2", + "type": "original", + "section": "original" + }, + { + "id": "mid_1", + "name": "Midroll 1", + "type": "ad", + "section": "mid" + }, + { + "id": "mid_2", + "name": "Midroll 2", + "type": "ad", + "section": "mid" + }, + { + "id": "original_3", + "name": "Original 3", + "type": "original", + "section": "original" + }, + { + "id": "post_1", + "name": "Postroll 1", + "type": "ad", + "section": "post" + }, + { + "id": "post_2", + "name": "Postroll 2", + "type": "ad", + "section": "post" + }, + { + "id": "house_post", + "name": "House Postroll", + "type": "ad", + "section": "post" + }, + { + "id": "sonic_id", + "name": "Sonic", + "type": "sonic_id", + "section": "post" + } +] diff --git a/test/models/megaphone/cuepoint_test.rb b/test/models/megaphone/cuepoint_test.rb new file mode 100644 index 000000000..5a03a967a --- /dev/null +++ b/test/models/megaphone/cuepoint_test.rb @@ -0,0 +1,30 @@ +require "test_helper" +describe Megaphone::Cuepoint do + let(:episode) { create(:episode) } + let(:zones) { JSON.parse(json_file(:zones)).map(&:with_indifferent_access) } + let(:media) { + [ + create(:content, episode: episode, position: 1, duration: 120.00), + create(:content, episode: episode, position: 2, duration: 300.00), + create(:content, episode: episode, position: 3, duration: 120.00) + ] + } + + describe "#valid?" do + it "must have required attributes" do + cuepoints = Megaphone::Cuepoint.from_zones_and_media(zones, media) + assert cuepoints.length == 3 + assert cuepoints[0].ad_count == 3 + assert cuepoints[0].ad_sources == [:promo, :auto, :auto] + assert cuepoints[0].start_time.to_i == 120 + + assert cuepoints[1].ad_count == 2 + assert cuepoints[1].ad_sources == [:auto, :auto] + assert cuepoints[1].start_time.to_i == (120 + 300) + + assert cuepoints[2].ad_count == 4 + assert cuepoints[2].ad_sources == [:auto, :auto, :promo, :promo] + assert cuepoints[2].start_time.to_i == (120 + 300 + 120) + end + end +end From 008d087c4d255e78792c0cef7d02f38c584dd83f Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 13:19:20 -0500 Subject: [PATCH 63/71] Add megaphone form labels --- config/locales/en.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/locales/en.yml b/config/locales/en.yml index f5fe2ed74..9f7301b22 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -333,6 +333,9 @@ en: apple/config: publish_enabled: Enable publishing to Apple Podcasts Connect sync_blocks_rss: Episodes must publish to Apple before the public feed + megaphone/config: + publish_enabled: Enable publishing to Megaphone + sync_blocks_rss: Episodes must publish to Megaphone before the public feed episode: ad_breaks: Ad Breaks author_email: Author Email @@ -814,6 +817,12 @@ en: private: Your feed can be public or private. Leave unchecked to keep it a Public Feed. If you check this box this feed becomes Private and you'll need to generate an authorization token. Be careful who you share your private URL with - anyone who has access to the URL will be able to use it. url: If you already have a public URL for your podcast feed (e.g., feedburner), enter it here. It should point to your private feed URL. title: Distribution + form_megaphone_config: + title: Megaphone Integration Configuration + description: Below configure the API token and network ID to integrate Dovetail publishing episode audio to Megaphone. + help: + publish_enabled: Enable automatically publishing and updating episodes in this feed to Megaphone? + sync_blocks_rss: Block publishing to the default feed RSS until the Megaphone episode is published? form_main: confirm: file_name: Woh there! Changing your PRX feed url could break things for your subscribers - are you sure? From f1177ff051521daa4e7694439067e6289c971211 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 13:20:09 -0500 Subject: [PATCH 64/71] Episode form status for megaphone --- app/views/episodes/_form_status.html.erb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/views/episodes/_form_status.html.erb b/app/views/episodes/_form_status.html.erb index d29098d70..1f01c9817 100644 --- a/app/views/episodes/_form_status.html.erb +++ b/app/views/episodes/_form_status.html.erb @@ -47,6 +47,22 @@

<% end %> + <% if episode.persisted? && episode.publish_to_integration?(:megaphone) %> + <% integration = :megaphone %> + <% integration_status = episode_integration_status(integration, episode) %> +
+

+ <%= "#{integration.to_s.titleize} Status" %>: + + <%= t("helpers.label.episode.media_statuses.#{integration_status}") %> + +
+ <%= t(".last_updated") %> +
+ <%= local_time_ago(episode_integration_updated_at(integration, episode)) %> +

+
+ <% end %>
<%= render "layouts/stale_record_field", form: form %> From a6d133db24bb580c6b782b467f4b52bb0f4e7d76 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 13:20:47 -0500 Subject: [PATCH 65/71] Podcast form megaphone status --- app/views/podcasts/_form_status.html.erb | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/app/views/podcasts/_form_status.html.erb b/app/views/podcasts/_form_status.html.erb index 8a57f62f5..6b4d57e50 100644 --- a/app/views/podcasts/_form_status.html.erb +++ b/app/views/podcasts/_form_status.html.erb @@ -3,11 +3,33 @@

<%= t(".title") %>

+ <% if podcast.persisted? && podcast.publish_to_integration?(:megaphone) %> +
+ <% integration = :megaphone %> + <% integration_status = podcast_integration_status(integration, podcast) %> +
+

+ <%= "#{integration.to_s.titleize} Status" %>: + + <%= t("helpers.label.episode.media_statuses.#{integration_status}") %> + +
+ <%= t(".last_updated") %> +
+ <%= local_time_ago(podcast_integration_updated_at(integration, podcast)) %> +

+
+
+ <% end %> + <%= render "layouts/stale_record_field", form: form %>
<% if podcast.persisted? %> -

<%= t(".updated_at_hint") %>
<%= local_time_ago(podcast.updated_at) %>

+

<%= t(".updated_at_hint") %> +
+ <%= local_time_ago(podcast.updated_at) %> +

<% if policy(podcast).update? %> <% data = {action: "unsaved#discard", unsaved_target: "discard"} %> From d49da30f98bbcac16c4fc72145cd973965a1f76d Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 13:21:41 -0500 Subject: [PATCH 66/71] New integration status helper methods --- app/helpers/episodes_helper.rb | 23 +++++++++++++++++++++++ app/helpers/podcasts_helper.rb | 17 +++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/app/helpers/episodes_helper.rb b/app/helpers/episodes_helper.rb index 69ac00c2a..b2670cbfb 100644 --- a/app/helpers/episodes_helper.rb +++ b/app/helpers/episodes_helper.rb @@ -15,6 +15,29 @@ def episode_explicit_options end end + def episode_integration_status(integration, episode) + status = episode.episode_delivery_status(integration, true) + if !status + "not_found" + elsif status.new_record? + "new" + elsif !status.uploaded? + "incomplete" + elsif !status.delivered? + "processing" + elsif status.delivered? + "complete" + else + "not_found" + end + end + + def episode_integration_updated_at(integration, episode) + episode.sync_log(integration)&.updated_at || + episode.episode_delivery_status(integration)&.created_at || + episode.updated_at + end + def episode_apple_status(episode) apple_episode = episode.apple_episode if !apple_episode diff --git a/app/helpers/podcasts_helper.rb b/app/helpers/podcasts_helper.rb index ef3024ddf..0e0ad2e87 100644 --- a/app/helpers/podcasts_helper.rb +++ b/app/helpers/podcasts_helper.rb @@ -5,6 +5,23 @@ module PodcastsHelper RSS_LANGUAGE_CODES = %w[af sq eu be bg ca zh-cn zh-tw hr cs da nl nl-be nl-nl en en-au en-bz en-ca en-ie en-jm en-nz en-ph en-za en-tt en-gb en-us en-zw et fo fi fr fr-be fr-ca fr-fr fr-lu fr-mc fr-ch gl gd de de-at de-de de-li de-lu de-ch el haw hu is in ga it it-it it-ch ja ko mk no pl pt pt-br pt-pt ro ro-mo ro-ro ru ru-mo ru-ru sr sk sl es es-ar es-bo es-cl es-co es-cr es-do es-ec es-sv es-gt es-hn es-mx es-ni es-pa es-py es-pe es-pr es-es es-uy es-ve sv sv-fi sv-se tr uk] + def podcast_integration_status(integration, podcast) + sync = podcast.public_feed.sync_log(integration) + if !sync + "not_found" + elsif !sync.external_id + "new" + elsif sync.updated_at <= podcast.updated_at + "incomplete" + else + "complete" + end + end + + def podcast_integration_updated_at(integration, podcast) + podcast.public_feed.sync_log(integration)&.updated_at || podcast.updated_at + end + def feed_description(feed, podcast) [feed.description, podcast.description].detect { |d| d.present? } || "" end From 889d52f5a75c1fe5c8beec81e2d006b015d4ed5b Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 13:22:50 -0500 Subject: [PATCH 67/71] Add new podcast integrations concern --- app/models/integrations/podcast_integrations.rb | 13 +++++++++++++ app/models/podcast.rb | 1 + 2 files changed, 14 insertions(+) create mode 100644 app/models/integrations/podcast_integrations.rb diff --git a/app/models/integrations/podcast_integrations.rb b/app/models/integrations/podcast_integrations.rb new file mode 100644 index 000000000..5ca65e8ad --- /dev/null +++ b/app/models/integrations/podcast_integrations.rb @@ -0,0 +1,13 @@ +require "active_support/concern" + +module Integrations::PodcastIntegrations + extend ActiveSupport::Concern + + # included do + # end + + def publish_to_integration?(integration) + # see if there is an integration + feeds.any? { |f| f.integration_type == integration && f.publish_integration? } + end +end diff --git a/app/models/podcast.rb b/app/models/podcast.rb index 238019921..55b5c0d03 100644 --- a/app/models/podcast.rb +++ b/app/models/podcast.rb @@ -15,6 +15,7 @@ class Podcast < ApplicationRecord include EmbedPlayerHelper include PodcastFilters include ReleaseEpisodes + include Integrations::PodcastIntegrations acts_as_paranoid From d8712576e3e7f011dcb378d924a74475fd02941b Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 13:23:17 -0500 Subject: [PATCH 68/71] Have episode know if it is integrated --- app/models/integrations/episode_integrations.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/models/integrations/episode_integrations.rb b/app/models/integrations/episode_integrations.rb index ad28f1e72..bf0a7dea4 100644 --- a/app/models/integrations/episode_integrations.rb +++ b/app/models/integrations/episode_integrations.rb @@ -23,6 +23,11 @@ module Integrations::EpisodeIntegrations end end + def publish_to_integration?(integration) + # see if there is an integration + podcast.feeds.any? { |f| f.integration_type == integration && f.publish_integration? } + end + def sync_log(integration) sync_logs.send(integration.intern).order(updated_at: :desc).first end From f7ebfc509e09e233bb3d55c7b88f96d9721022a2 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 13:23:42 -0500 Subject: [PATCH 69/71] No permission to optin to span --- app/models/megaphone/podcast.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/models/megaphone/podcast.rb b/app/models/megaphone/podcast.rb index 82118b659..f4bc3f03a 100644 --- a/app/models/megaphone/podcast.rb +++ b/app/models/megaphone/podcast.rb @@ -11,8 +11,9 @@ class Podcast < Integrations::Base::Show explicit owner_name owner_email slug original_rss_url itunes_identifier podtrac_enabled google_play_identifier episode_limit podcast_type advertising_tags excluded_categories] - # Update also allows the span opt in - UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES + %i[span_opt_in] + # Update also allows the span opt in, but we don't have permissions + # UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES + %i[span_opt_in] + UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES # Deprecated, so we shouldn't rely on these, but they show up as attributes DEPRECATED = %i[category redirect_url itunes_active redirected_at itunes_rating From 94ae8aca2affb422a905e29d30ce479796dcf289 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Thu, 12 Dec 2024 13:34:15 -0500 Subject: [PATCH 70/71] Include house zones better --- app/models/megaphone/cuepoint.rb | 4 ++-- test/fixtures/zones.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/models/megaphone/cuepoint.rb b/app/models/megaphone/cuepoint.rb index 96ed58a19..6a0aaeaa1 100644 --- a/app/models/megaphone/cuepoint.rb +++ b/app/models/megaphone/cuepoint.rb @@ -23,7 +23,7 @@ def self.from_zones_and_media(zones, media) original_count = 0 zones.each do |zone| # if this is an ad zone, add it to the cue point - if ["ad", "sonic_id"].include?(zone[:type]) + if ["ad", "sonic_id", "house"].include?(zone[:type]) if current_cuepoint current_cuepoint.ad_count = current_cuepoint.ad_count + 1 current_cuepoint.ad_sources << source_for_zone(zone) @@ -48,7 +48,7 @@ def self.from_zones_and_media(zones, media) end def self.source_for_zone(zone) - if zone[:id].match?(/^house/) || zone[:type] == "sonic_id" + if ["sonic_id", "house"].include?(zone[:type]) :promo else :auto diff --git a/test/fixtures/zones.json b/test/fixtures/zones.json index 70efc6a3c..89a72655c 100644 --- a/test/fixtures/zones.json +++ b/test/fixtures/zones.json @@ -8,7 +8,7 @@ { "id": "house_pre", "name": "House Preroll", - "type": "ad", + "type": "house", "section": "pre" }, { @@ -62,7 +62,7 @@ { "id": "house_post", "name": "House Postroll", - "type": "ad", + "type": "house", "section": "post" }, { From e81280ae20ab9d1ff43d3c3fd0e7c158f132660d Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Fri, 13 Dec 2024 00:54:18 -0500 Subject: [PATCH 71/71] Fix update episode --- app/controllers/placements_preview_controller.rb | 2 +- app/models/megaphone/episode.rb | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/controllers/placements_preview_controller.rb b/app/controllers/placements_preview_controller.rb index 364303186..3356141f8 100644 --- a/app/controllers/placements_preview_controller.rb +++ b/app/controllers/placements_preview_controller.rb @@ -5,8 +5,8 @@ class PlacementsPreviewController < ApplicationController # GET /podcasts/1/placements_preview/2 def show - @fetch_error = cached_placements.nil? @zones = get_zones(params[:id].to_i) + @fetch_error = @zones.nil? end private diff --git a/app/models/megaphone/episode.rb b/app/models/megaphone/episode.rb index 1b7c15b79..5cab9e362 100644 --- a/app/models/megaphone/episode.rb +++ b/app/models/megaphone/episode.rb @@ -19,6 +19,8 @@ class Episode < Integrations::Base::Episode insertion_points guid pre_offset post_offset expected_adhash original_filename original_url episode_number season_number retain_ad_locations advertising_tags ad_free] + UPDATE_ATTRIBUTES = CREATE_ATTRIBUTES + # All other attributes we might expect back from the Megaphone API # (some documented, others not so much) OTHER_ATTRIBUTES = %i[id podcast_id created_at updated_at status @@ -69,7 +71,7 @@ def self.attributes_from_episode(e) external_id: e.guid, guid: e.item_guid, pubdate: e.published_at, - pubdate_timezone: e.published_at.zone, + pubdate_timezone: e.published_at, author: e.author_name, link: e.url, explicit: e.explicit, @@ -106,9 +108,11 @@ def create! self end - def update!(feed = nil) - if feed - self.attributes = self.class.attributes_from_feed(feed) + def update!(episode = nil) + if episode + self.attributes = self.class.attributes_from_episode(episode) + set_placement_attributes + set_audio_attributes end validate!(:update) body = as_json(only: UPDATE_ATTRIBUTES.map(&:to_s)) @@ -240,7 +244,7 @@ def set_placement_attributes def adhash_for_placement(zones) zones - .filter { |z| z[:type] == "ad" } + .filter { |z| ["ad", "sonic_id", "house"].include?(z[:type]) } .map { |z| ADHASH_VALUES[z[:section]] } .join("") end