Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved remove duplicates #1034

Open
wants to merge 11 commits into
base: development
Choose a base branch
from
+100 −3
Open

Improved remove duplicates #1034

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
9bdfc46
Improve remove duplicates
girishpanchal30 Dec 17, 2024
ea40097
fix: update upsell link
girishpanchal30 Dec 17, 2024
c12e36e
Add multiple tags support
girishpanchal30 Dec 18, 2024
4722e02
Encode the tag value if the value type is an object
girishpanchal30 Dec 18, 2024
b19b716
Create unique meta key
girishpanchal30 Dec 18, 2024
7d24918
e2e: check upsell link
girishpanchal30 Dec 19, 2024
892fa2a
Add phpunit tests
girishpanchal30 Dec 19, 2024
1e6c0d1
Improve PHPUnit testcases
girishpanchal30 Dec 19, 2024
50e8d5c
fix: change meta key
girishpanchal30 Dec 20, 2024
5d2cb2a
fix: import count
girishpanchal30 Dec 20, 2024
8dc01a3
fix: php warning
girishpanchal30 Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions includes/admin/feedzy-rss-feeds-import.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ public function feedzy_import_feed_options() {
$import_auto_translation = get_post_meta( $post->ID, 'import_auto_translation', true );
$import_auto_translation = 'yes' === $import_auto_translation ? 'checked' : '';
$import_translation_lang = get_post_meta( $post->ID, 'import_auto_translation_lang', true );
$mark_duplicate_tag = get_post_meta( $post->ID, 'mark_duplicate_tag', true );
$import_post_author = get_post_meta( $post->ID, 'import_post_author', true );

/**
Expand Down Expand Up @@ -1022,6 +1023,9 @@ private function get_complete_import_status( $post_id ) {
if ( $items ) {
$status['cumulative'] = count( $items );
}
if ( ! empty( $status['duplicates'] ) ) {
$status['total'] = absint( $status['total'] - count( $status['duplicates'] ) );
}

return $status;
}
Expand Down Expand Up @@ -1377,6 +1381,8 @@ private function run_job( $job, $max ) {
$import_auto_translation = $this->feedzy_is_agency() && 'yes' === $import_auto_translation ? true : false;
$import_translation_lang = get_post_meta( $job->ID, 'import_auto_translation_lang', true );
$import_post_author = get_post_meta( $job->ID, 'import_post_author', true );
$mark_duplicate_tag = get_post_meta( $job->ID, 'mark_duplicate_tag', true );
$mark_duplicate_tag = feedzy_is_pro() && ! empty( $mark_duplicate_tag ) ? preg_replace( '/[\[\]#]/', '', $mark_duplicate_tag ) : '';
$max = $import_feed_limit;

if ( metadata_exists( 'post', $job->ID, 'import_post_status' ) ) {
Expand Down Expand Up @@ -1524,11 +1530,36 @@ private function run_job( $job, $max ) {
$is_duplicate = $use_new_hash ? in_array( $item_hash, $imported_items_new, true ) : in_array( $item_hash, $imported_items_old, true );
$items_found[ $item['item_url'] ] = $item['item_title'];

$duplicate_tag_value = array();
$mark_duplicate_key = 'item_url';
if ( 'yes' === $import_remove_duplicates && ! $is_duplicate ) {
$is_duplicate_post = $this->is_duplicate_post( $import_post_type, 'feedzy_item_url', esc_url_raw( $item['item_url'] ) );
if ( ! empty( $mark_duplicate_tag ) ) {
$mark_duplicate_tag = is_string( $mark_duplicate_tag ) ? explode( ',', $mark_duplicate_tag ) : $mark_duplicate_tag;
$mark_duplicate_tag = array_map( 'trim', $mark_duplicate_tag );
$duplicate_tag_value = array_map(
function ( $tag ) use ( $item_obj, $item ) {
if ( str_contains( $tag, 'item_custom' ) && $this->feedzy_is_business() ) {
$tag = apply_filters( 'feedzy_parse_custom_tags', "[#$tag]", $item_obj );
} elseif ( isset( $item[ $tag ] ) ) {
$tag = isset( $item[ $tag ] ) ? is_object( $item[ $tag ] ) ? wp_json_encode( $item[ $tag ] ) : $item[ $tag ] : '';
}
return $tag;
},
$mark_duplicate_tag
);
}
if ( ! empty( $duplicate_tag_value ) ) {
$duplicate_tag_value = implode( ' ', $duplicate_tag_value );
$duplicate_tag_value = substr( sanitize_key( wp_strip_all_tags( $duplicate_tag_value ) ), 0, apply_filters( 'feedzy_mark_duplicate_content_limit', 256 ) );
$mark_duplicate_key = 'mark_duplicate';
} else {
$duplicate_tag_value = esc_url_raw( $item['item_url'] );
}
$is_duplicate_post = $this->is_duplicate_post( $import_post_type, 'feedzy_' . $mark_duplicate_key, $duplicate_tag_value );
if ( ! empty( $is_duplicate_post ) ) {
foreach ( $is_duplicate_post as $p ) {
$found_duplicates[] = get_post_meta( $p, 'feedzy_item_url', true );
$found_duplicates[ $item_hash ] = get_post_meta( $p, 'feedzy_' . $mark_duplicate_key, true );
$duplicates[ $item['item_url'] ] = $item['item_title'];
wp_delete_post( $p, true );
}
}
Expand Down Expand Up @@ -2063,6 +2094,11 @@ function( $term ) {
update_post_meta( $new_post_id, 'feedzy_job', $job->ID );
update_post_meta( $new_post_id, 'feedzy_item_author', sanitize_text_field( $author ) );

// Verify that the `$mark_duplicate_key` does not match `'item_url'` to ensure the condition applies only when a different tag is specified.
if ( $mark_duplicate_key && 'item_url' !== $mark_duplicate_key ) {
update_post_meta( $new_post_id, 'feedzy_' . $mark_duplicate_key, $duplicate_tag_value );
}

// we can use this to associate the items that were imported in a particular run.
update_post_meta( $new_post_id, 'feedzy_job_time', $last_run );

Expand Down
16 changes: 16 additions & 0 deletions includes/views/import-metabox-edit.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -812,6 +812,22 @@ class="fz-switch-toggle" type="checkbox" value="yes"
</div>
</div>

<div class="form-block form-block-two-column <?php echo esc_attr( apply_filters( 'feedzy_upsell_class', '' ) ); ?>">
<?php echo wp_kses_post( apply_filters( 'feedzy_upsell_content', '', 'remove-duplicates', 'import' ) ); ?>
<div class="fz-left">
<h4 class="h4"><?php esc_html_e( 'Mark Duplicates By', 'feedzy-rss-feeds' ); ?> <?php echo ! feedzy_is_pro() ? ' <span class="pro-label">PRO</span>' : ''; ?></h4>
</div>
<div class="fz-right">
<div class="fz-form-group">
<label class="form-label"><?php esc_html_e( 'Mark the duplicate items using the magic tag value', 'feedzy-rss-feeds' ); ?></label>
<input type="text" id="feedzy_mark_duplicate" name="feedzy_meta_data[mark_duplicate_tag]" class="form-control" value="<?php echo esc_attr( $mark_duplicate_tag ); ?>"<?php disabled( true, 'checked' !== $import_remove_duplicates ); ?> />
<div class="help-text pt-8">
<?php esc_html_e( 'Helpful if you want to identify and remove duplicate items. Enter a magic tag to mark duplicates based on its value, e.g: [#item_title], [#item_content], [#item_url] etc.', 'feedzy-rss-feeds' ); ?>
</div>
</div>
</div>
</div>

<div class="form-block form-block-two-column">
<div class="fz-left">
<h4 class="h4"><?php esc_html_e( 'Items Count', 'feedzy-rss-feeds' ); ?></h4>
Expand Down
8 changes: 8 additions & 0 deletions includes/views/js/import-metabox-edit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,14 @@
tagList.find( 'input:text' ).val( '' );
}
} );

$( document ).on( 'change', 'input#remove-duplicates', function() {
if ( $(this).is(':checked') ) {
$('input#feedzy_mark_duplicate').attr( 'disabled', false );
} else {
$('input#feedzy_mark_duplicate').attr( 'disabled', true );
}
} );
}

function initSummary() {
Expand Down
4 changes: 4 additions & 0 deletions tests/e2e/specs/upsell.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,5 +69,9 @@ test.describe( 'Upsell', () => {
await page.locator('.fz-form-group:has( #fz-event-execution )').hover({ force: true });
upgradeAlert = page.locator('#feedzy-import-form a[href*="utm_campaign=schedule-import-job"]');
await expect( upgradeAlert ).toBeVisible();

await page.locator('.fz-form-group:has( #feedzy_mark_duplicate )').hover({ force: true });
upgradeAlert = page.locator('#feedzy-import-form a[href*="utm_campaign=remove-duplicates"]');
await expect( upgradeAlert ).toBeVisible();
} );
});
35 changes: 34 additions & 1 deletion tests/test-import.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public function setUp(): void {
* @access public
* @dataProvider importDataProvider
*/
public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic_tags = '[#item_content]', $use_filter = false, $type = 'post' ) {
public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic_tags = '[#item_content]', $use_filter = false, $type = 'post', $check_duplicate = false ) {
do_action( 'init' );

$num_items = $this->import_limit;
Expand Down Expand Up @@ -93,6 +93,11 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic
$_POST['custom_vars_key'] = array();
$_POST['custom_vars_value'] = array();

if ( $check_duplicate ) {
$_POST['feedzy_meta_data']['import_remove_duplicates'] = 'yes';
$_POST['feedzy_meta_data']['mark_duplicate_tag'] = '[#item_author]';
}

do_action( 'save_post_feedzy_imports', $p->ID, $p );
$this->assertEquals( $p->post_title, $random_name2 );
$this->assertEquals( $p->post_type, 'feedzy_imports' );
Expand All @@ -110,6 +115,11 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic
$import_custom_fields = get_post_meta( $p->ID, 'imports_custom_fields', true );
$import_feed_limit = get_post_meta( $p->ID, 'import_feed_limit', true );

if ( $check_duplicate ) {
$remove_duplicates = get_post_meta( $p->ID, 'import_remove_duplicates', true );
$mark_duplicate_tag = get_post_meta( $p->ID, 'mark_duplicate_tag', true );
}

$this->assertEquals( $type, $import_post_type );
$this->assertEquals( 'category_' . $category_id, $import_post_term );
$this->assertEquals( 'publish', $import_post_status );
Expand All @@ -123,6 +133,11 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic
$this->assertEquals( '', $import_custom_fields );
$this->assertEquals( $num_items, $import_feed_limit );

if ( $check_duplicate ) {
$this->assertEquals( 'yes', $remove_duplicates );
$this->assertEquals( '[#item_author]', $mark_duplicate_tag );
}

$feed_src = str_replace( PHP_EOL, '', $urls );

// Do Cron
Expand Down Expand Up @@ -232,6 +247,11 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic
$this->assertEquals( 1, get_post_meta( $created[0]->ID, 'feedzy', true ) );
$this->assertNotEmpty( get_post_meta( $created[0]->ID, 'feedzy_item_url', true ) );

// Check found duplicates items.
if ( $check_duplicate ) {
$this->assertNotEmpty( get_post_meta( $created[0]->ID, 'feedzy_mark_duplicate', true ) );
}

// Check Post Delete
$this->assertNotEquals( false, wp_delete_post( $p->ID ) );
return $created[0];
Expand Down Expand Up @@ -370,6 +390,19 @@ function( $date ) {
update_option( 'gmt_offset', 0 ); // reset UTC
}

/**
* Test check duplicate.
*/
public function test_check_duplicate() {
define( 'FEEDZY_PRO_ABSPATH', '' );
$function = new ReflectionFunction( 'feedzy_is_pro' );
$function->invokeArgs( array( false ) );

$post = $this->test_feedzy_imports( $this->get_rand_name(), $this->get_rand_name(), $this->get_two_rand_feeds(), '[#item_content]', false, 'post', true );

$this->assertNotEmpty( $post );
}

/**
* Utility method to generate a random 5 char string.
*
Expand Down
Loading