Merge branch 'release/5.1.0' into v5

.github/workflows/code-analysis.yaml

@@ -17,7 +17,7 @@ jobs:
           - name: 'PHPStan'
             run: composer phpstan
           - name: 'Coding Standards'
-            run: composer fix-cs
+            run: composer check-cs
     name: ${{ matrix.actions.name }}
     runs-on: ubuntu-latest
     steps:

CHANGELOG.md

@@ -1,5 +1,9 @@
 # SEOmatic Changelog
 
+## 5.1.0 - 2024.07.12
+### Added
+* Remove queue generated sitemaps, switch to paginated sitemaps to allow them to be rendered at web response time, but still be managable in size
+
 ## 5.0.4 - 2024.05.13
 ### Added
 * Added a setting in Plugin Settings -> Tags to specify which site should be used as the `x-default` for `hreflang` tags ([1162](https://github.com/nystudio107/craft-seomatic/issues/1162))

composer.json

@@ -2,7 +2,7 @@
   "name": "nystudio107/craft-seomatic",
   "description": "SEOmatic facilitates modern SEO best practices & implementation for Craft CMS 5. It is a turnkey SEO system that is comprehensive, powerful, and flexible.",
   "type": "craft-plugin",
-  "version": "5.0.4",
+  "version": "5.1.0",
   "minimum-stability": "dev",
   "prefer-stable": true,
   "keywords": [
@@ -68,7 +68,7 @@
     "verbb/doxter": "^6.0.0"
   },
   "scripts": {
-    "phpstan": "phpstan --ansi --memory-limit=1G",
+    "phpstan": "phpstan --ansi --memory-limit=2G",
     "check-cs": "ecs check --ansi",
     "fix-cs": "ecs check --fix --ansi",
     "test": "codecept run unit --coverage-xml"

docs/docs/configuring/content-seo.md

@@ -40,21 +40,19 @@ SEOmatic automatically creates a sitemap index for each of your Site Groups. Thi
 
 Instead of one massive sitemap that must be updated any time anything changes, only the sitemap for the Section, Category Group, or Commerce Product Type will be updated when something changes in it.
 
+Additionally, sitemaps are paginated according to the **Sitemap Page Size** setting (which defaults to `500` per page), to allow for a large number of entries in the sitemaps without hurting performance.
+
 SEOmatic can automatically include files such as `.pdf`, `.xls`, `.doc` and other indexable file types in Asset fields or Asset fields in matrix or Neo blocks.
 
 In addition, SEOmatic can automatically create [Image sitemaps](https://support.google.com/webmasters/answer/178636?hl=en) and [Video sitemaps](https://developers.google.com/webmasters/videosearch/sitemaps) from images & videos in Asset fields or Asset fields in matrix or Neo blocks.
 
-Sitemap Indexes are automatically submitted to search engines whenever a new Section, Category Group, or Commerce Product Type is added.
-
-Section Sitemaps are automatically submitted to search engines whenever a new Element in that Section, Category Group, or Commerce Product Type is added.
+Search engines no longer allow for sitemap submission via, but rather will re-crawl your sitemaps on a regular basis to find new pages.
 
 ### Sitemap Generation
 
-Because XML sitemaps can be time-intensive to generate with a growing number of entries, SEOmatic creates your sitemaps via a queue job and caches the result. The cache is automatically broken whenever something in that sitemap is changed, and a new queue job is created to regenerate it.
-
-If `runQueueAutomatically` is set to `false` in [General Config Settings](https://craftcms.com/docs/5.x/reference/config/general.html#runqueueautomatically) the Queue job to create the sitemap will not be run during the http request for the sitemap. You’ll need to run it manually via whatever means you use to run the Queue.
+SEOmatic generates sitemaps on-demmand when requested on the frontend, and then caches the sitemap for future requests so they are fast.
 
-Normally SEOmatic will regenerate the sitemap for a Section, Category Group, or Product any time you save an element. However, if you are importing a large number of elements, or prefer to regenerate the sitemap manually you can set disable the **Regenerate Sitemaps Automatically** option in SEOmatic’s Plugin Settings.
+Normally SEOmatic will invalidate the cache for a sitemap for a Section, Category Group, or Product any time you save an element. However,  if you prefer to regenerate the sitemap manually you can set disable the **Regenerate Sitemaps Automatically** option in SEOmatic’s Plugin Settings.
 
 ![Screenshot of a console running the following command to generate a blog sitemap: `./craft seomatic/sitemap/generate --siteId=1 --handle=blog`](../resources/screenshots/seomatic-sitemap-console-command.png)
 

docs/docs/configuring/plugin-settings.md

@@ -10,7 +10,6 @@ Plugin Settings let you control various SEOmatic settings across all sites/langu
 * **Automatic Render Enabled** – Controls whether SEOmatic automatically renders metadata on your pages. If you turn this off, you will need to manually render the metadata via `seomatic.tag.render()`, `seomatic.link.render()`, etc. Selectively disable rendering via Twig with `{% do seomatic.config.renderEnabled(false) %}`.
 * **Sitemaps Enabled** – Controls whether SEOmatic will automatically render front-end sitemaps for your site.
 * **Regenerate Sitemaps Automatically** – Controls whether sitemaps will automatically be regenerated when entries are saved.
-* **Submit Sitemap Changes** – Should sitemaps be submitted to search engines automatically whenever there are changes?
 * **Include Homepage in Breadcrumbs** – Should the homepage be included in the generated Breadcrumbs JSON-LD?
 * **Manually Set SEOmatic Environment** – If off, SEOmatic will automatically attempt to determine the current environment. Turn this on to manually set the environment.
 * **Environment** – The server environment, either `live`, `staging`, or `local`. If `devMode` is on, SEOmatic will override this setting to local Development. This setting controls whether certain things render; for instance only in the `live` production environment will Google Analytics and other tracking tags send analytics data. SEOmatic also automatically sets the `robots` tag to `none` for everything but the `live` production environment.

phpstan.neon

@@ -5,9 +5,6 @@ parameters:
     level: 5
     paths:
         - src
-    excludePaths:
-    # Until the plugins are ported, exclude their integrations
-        - src/seoelements/SeoEvent.php
     ignoreErrors:
     # Until the plugins are ported, exclude their classes
         - '*besteadfast\\preparsefield\\*'

src/base/SitemapInterface.php

@@ -21,5 +21,5 @@ interface SitemapInterface
     // Constants
     // =========================================================================
 
-    public const GLOBAL_SITEMAP_CACHE_TAG = 'seomatic_sitemap';
+    public const GLOBAL_SITEMAP_CACHE_TAG = 'seomatic_sitemap_v2';
 }

src/console/controllers/SitemapController.php

@@ -83,6 +83,7 @@ public function actionGenerate()
         foreach ($siteIds as $siteId) {
             $metaBundles = Seomatic::$plugin->metaBundles->getContentMetaBundlesForSiteId($siteId);
             Seomatic::$plugin->metaBundles->pruneVestigialMetaBundles($metaBundles);
+
             /** @var MetaBundle $metaBundle */
             foreach ($metaBundles as $metaBundle) {
                 $process = false;
@@ -97,17 +98,33 @@ public function actionGenerate()
                         . ', siteId '
                         . $siteId
                         . PHP_EOL;
-                    // Generate the sitemap so it is in the cache
+
+                    $seoElement = Seomatic::$plugin->seoElements->getSeoElementByMetaBundleType($metaBundle->sourceBundleType);
+                    $elementQuery = $seoElement::sitemapElementsQuery($metaBundle);
+                    $pageSize = (int) $metaBundle->metaSitemapVars->sitemapPageSize;
+                    $sitemapLimit = (int) $metaBundle->metaSitemapVars->sitemapLimit;
+
+                    if (!empty($pageSize)) {
+                        $total = empty($sitemapLimit) ? $elementQuery->count() : min($elementQuery->count(), $sitemapLimit);
+                        $pageCount = ceil($total / $pageSize);
+                    } else {
+                        $pageCount = 1;
+                    }
+
                     $site = Craft::$app->getSites()->getSiteById($metaBundle->sourceSiteId);
+                    $sitemap = SitemapTemplate::create();
                     if ($site) {
-                        $sitemap = SitemapTemplate::create();
-                        $sitemap->render([
-                            'groupId' => $site->groupId,
-                            'siteId' => $metaBundle->sourceSiteId,
-                            'handle' => $metaBundle->sourceHandle,
-                            'type' => $metaBundle->sourceBundleType,
-                            'immediately' => true,
-                        ]);
+                        for ($pageNum = 1; $pageNum <= $pageCount; $pageNum++) {
+                            echo sprintf('Generating page %d of %d' . PHP_EOL, $pageNum, $pageCount);
+                            $sitemap->render([
+                                'groupId' => $site->groupId,
+                                'siteId' => $metaBundle->sourceSiteId,
+                                'handle' => $metaBundle->sourceHandle,
+                                'type' => $metaBundle->sourceBundleType,
+                                'page' => $pageNum,
+                            ]);
+                        }
+                        // Generate the sitemap so it is in the cache
                     }
 
                     echo '---' . PHP_EOL;

src/controllers/SitemapController.php

@@ -120,15 +120,22 @@ public function actionSitemapEmptyStyles(): Response
      *
      * @return Response
      */
-    public function actionSitemap(int $groupId, string $type, string $handle, int $siteId): Response
+    public function actionSitemap(int $groupId, string $type, string $handle, int $siteId, string $file): Response
     {
+        $page = null;
+
+        if (preg_match('/sitemap-p([\d]+)\.xml/i', $file, $matches)) {
+            $page = $matches[1];
+        }
+
         $xml = Seomatic::$plugin->sitemaps->renderTemplate(
             Sitemaps::SEOMATIC_SITEMAP_CONTAINER,
             [
                 'groupId' => $groupId,
                 'type' => $type,
                 'handle' => $handle,
                 'siteId' => $siteId,
+                'page' => $page,
             ]
         );
         $headers = Craft::$app->response->headers;

src/debug/views/seomatic/render-tag.php

@@ -53,8 +53,8 @@
                     <?php foreach ($value as $subName => $subValue): ?>
                         <?php $meta['PROPERTY_NAME'] = [$subName]; ?>
                         <?php if ($subName === '__errors') {
-                            continue;
-                        } ?>
+    continue;
+} ?>
                         <tr>
                             <th class="seomatic-property"><?= Html::encode($subName) ?><?= $this->render('render-copy-menu', [
                                     'value' => $subValue ?? '',

src/helpers/Field.php

@@ -111,8 +111,7 @@ public static function fieldsOfTypeFromLayout(
         string      $fieldClassKey,
         FieldLayout $layout,
         bool        $keysOnly = true,
-    ): array
-    {
+    ): array {
         $foundFields = [];
         if (!empty(self::FIELD_CLASSES[$fieldClassKey])) {
             // Cache me if you can
@@ -155,8 +154,7 @@ public static function fieldsOfTypeFromElement(
         Element $element,
         string  $fieldClassKey,
         bool    $keysOnly = true,
-    ): array
-    {
+    ): array {
         $foundFields = [];
         $layout = $element->getFieldLayout();
         if ($layout !== null) {
@@ -267,8 +265,7 @@ public static function fieldsOfTypeFromSource(
         string $sourceHandle,
         string $fieldClassKey,
         bool   $keysOnly = true,
-    ): array
-    {
+    ): array {
         $foundFields = [];
         $layouts = [];
         // Get the layouts