From 8dffb4e345790ba76ce959acf3bd2ab083f89288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kuna?= <1282324+codecalm@users.noreply.github.com> Date: Wed, 5 Feb 2025 22:04:57 +0100 Subject: [PATCH] Update robots.txt handling and improve sitemap URL generation (#2130) * Update robots.txt handling and improve sitemap URL generation * Add dynamic page data handling and update sitemap generation * Remove debug output from sitemap generation * Create thin-clouds-glow.md --- .changeset/thin-clouds-glow.md | 5 +++ preview/eleventy.config.mjs | 13 ++++++- preview/package.json | 2 +- preview/pages/_data/{pages.json => urls.json} | 0 .../_includes/cards/most-visited-pages.html | 12 +++---- .../layout/header-actions/breadcrumb.html | 6 ++-- preview/pages/_includes/ui/breadcrumb.html | 4 +-- preview/pages/robots.liquid | 13 +++++++ preview/pages/robots.txt | 4 --- preview/pages/sitemap.liquid | 14 ++++++++ preview/pages/sitemap.xml | 34 ------------------- 11 files changed, 56 insertions(+), 51 deletions(-) create mode 100644 .changeset/thin-clouds-glow.md rename preview/pages/_data/{pages.json => urls.json} (100%) create mode 100644 preview/pages/robots.liquid delete mode 100644 preview/pages/robots.txt create mode 100644 preview/pages/sitemap.liquid delete mode 100644 preview/pages/sitemap.xml diff --git a/.changeset/thin-clouds-glow.md b/.changeset/thin-clouds-glow.md new file mode 100644 index 000000000..f86ddfd87 --- /dev/null +++ b/.changeset/thin-clouds-glow.md @@ -0,0 +1,5 @@ +--- +"preview": patch +--- + +Update `robots.txt` handling and improve sitemap URL generation diff --git a/preview/eleventy.config.mjs b/preview/eleventy.config.mjs index e2fccef79..0de650b6f 100644 --- a/preview/eleventy.config.mjs +++ b/preview/eleventy.config.mjs @@ -1,6 +1,7 @@ import { readFileSync } from 'node:fs'; import { EleventyRenderPlugin } from "@11ty/eleventy"; import { join, dirname } from 'node:path'; +import { sync } from 'glob'; /* * Copy list @@ -75,6 +76,16 @@ export default function (eleventyConfig) { eleventyConfig.addGlobalData("license", readFileSync(join("..", "LICENSE"), "utf-8")); eleventyConfig.addGlobalData("changelog", readFileSync(join("..", "CHANGELOG.md"), "utf-8")); + eleventyConfig.addGlobalData("pages", () => { + return sync('pages/**/*.html').filter((file) => { + return !file.includes('pages/_') && !file.includes('pages/docs/index.html'); + }).map((file) => { + return { + url: file.replace(/^pages\//, '/') + } +}); + }); + eleventyConfig.addGlobalData("site", { title: "Tabler", description: "Premium and Open Source dashboard template with responsive and high quality UI.", @@ -86,7 +97,7 @@ export default function (eleventyConfig) { githubSponsorsUrl: "https://github.com/sponsors/codecalm", changelogUrl: "https://github.com/tabler/tabler/releases", sponsorUrl: "https://github.com/sponsors/codecalm", - previewUrl: "https://tabler.io/demo", + previewUrl: "https://preview.tabler.io", docsUrl: "https://tabler.io/docs", mapboxKey: "pk.eyJ1IjoidGFibGVyIiwiYSI6ImNscHh3dnhndjB2M3QycW85bGd0NXRmZ3YifQ.9LfHPsNoEXQH-xzz-81Ffw", diff --git a/preview/package.json b/preview/package.json index cb5209557..55e7c0703 100644 --- a/preview/package.json +++ b/preview/package.json @@ -5,7 +5,7 @@ "build": "pnpm run clean && pnpm run css && pnpm run js && pnpm run html", "dev": "pnpm run clean && pnpm run watch", "watch": "pnpm run watch-html & pnpm run watch-css & pnpm run watch-js", - "watch-html": "eleventy --serve --port=3000", + "watch-html": "eleventy --serve --port=3000 --incremental", "watch-js": "nodemon --watch js/ --ext js --exec 'pnpm run js'", "watch-css": "nodemon --watch scss/ --ext scss --exec 'pnpm run css'", "css": "pnpm run css-compile && pnpm run css-prefix && pnpm run css-minify", diff --git a/preview/pages/_data/pages.json b/preview/pages/_data/urls.json similarity index 100% rename from preview/pages/_data/pages.json rename to preview/pages/_data/urls.json diff --git a/preview/pages/_includes/cards/most-visited-pages.html b/preview/pages/_includes/cards/most-visited-pages.html index 8e4893af8..e39510fc6 100644 --- a/preview/pages/_includes/cards/most-visited-pages.html +++ b/preview/pages/_includes/cards/most-visited-pages.html @@ -14,17 +14,17 @@ - {% for page in pages limit: 6 %} - {% assign data = page.data | join: ',' %} + {% for url in urls limit: 6 %} + {% assign data = url.data | join: ',' %} {% assign id = "bounce-rate-" | append: forloop.index %} - {{ page.uri }} + {{ url.uri }} {% include "ui/icon.html" icon="link" %} - {{ page.visitors | format_number }} - {{ page.unique | format_number }} - {{ page.bounce }} + {{ url.visitors | format_number }} + {{ url.unique | format_number }} + {{ url.bounce }} {% include "ui/chart-sparkline.html" type="line" data=data id=id small=true color="primary" %} diff --git a/preview/pages/_includes/layout/header-actions/breadcrumb.html b/preview/pages/_includes/layout/header-actions/breadcrumb.html index 7d3925c41..ecad433a2 100644 --- a/preview/pages/_includes/layout/header-actions/breadcrumb.html +++ b/preview/pages/_includes/layout/header-actions/breadcrumb.html @@ -1,6 +1,6 @@ -{% assign pages = "Tabler,Pages," %} -{% assign pages = pages | append: page.page-header %} +{% assign breadcrumb-pages = "Tabler,Pages," %} +{% assign breadcrumb-pages = breadcrumb-pages | append: page.page-header %}
- {% include "ui/breadcrumb.html" pages=pages class="breadcrumb-arrows" %} + {% include "ui/breadcrumb.html" pages=breadcrumb-pages class="breadcrumb-arrows" %}
\ No newline at end of file diff --git a/preview/pages/_includes/ui/breadcrumb.html b/preview/pages/_includes/ui/breadcrumb.html index e3b970c35..9b7241b1d 100644 --- a/preview/pages/_includes/ui/breadcrumb.html +++ b/preview/pages/_includes/ui/breadcrumb.html @@ -1,6 +1,6 @@ -{% assign pages = include.pages | default: "Home,Library,Data" | split: "," %} +{% assign breadcrumb-pages = include.pages | default: "Home,Library,Data" | split: "," %}
    - {% for page in pages %} + {% for page in breadcrumb-pages %} {% if forloop.last %}
  1. {{ page }}
    2. {% else %} diff --git a/preview/pages/robots.liquid b/preview/pages/robots.liquid new file mode 100644 index 000000000..c652cc481 --- /dev/null +++ b/preview/pages/robots.liquid @@ -0,0 +1,13 @@ +--- +layout: null +permalink: robots.txt +--- +Sitemap: {% if environment != 'development' %}{{ site.previewUrl }}{% endif %}/sitemap.xml + +{% if environment == 'preview' %} +User-agent: * +Disallow: +{% else %} +User-agent: * +Disallow: / +{% endif %} \ No newline at end of file diff --git a/preview/pages/robots.txt b/preview/pages/robots.txt deleted file mode 100644 index 9a3d52369..000000000 --- a/preview/pages/robots.txt +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: none ---- -Sitemap: {% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ "sitemap.xml" | absolute_url }} diff --git a/preview/pages/sitemap.liquid b/preview/pages/sitemap.liquid new file mode 100644 index 000000000..5cccb6c2e --- /dev/null +++ b/preview/pages/sitemap.liquid @@ -0,0 +1,14 @@ +--- +layout: null +permalink: sitemap.xml +--- + + + +{% for page in pages %} + + {% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ page.url | replace: 'index.html', '' | xml_escape }} + {{ page.last_modified_at | default: 'now' | date_to_xmlschema }} + +{% endfor %} + \ No newline at end of file diff --git a/preview/pages/sitemap.xml b/preview/pages/sitemap.xml deleted file mode 100644 index d839c6651..000000000 --- a/preview/pages/sitemap.xml +++ /dev/null @@ -1,34 +0,0 @@ ---- -layout: none ---- - - -{% assign collections = site.collections | where_exp:'collection','collection.output != false' %} -{% for collection in collections %} -{% assign docs = collection.docs | where_exp:'doc','doc.sitemap != false' %} -{% for doc in docs %} - - {% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ doc.url | replace: '/index.html', '/' | absolute_url | xml_escape }} - {% if doc.last_modified_at or doc.date %} - {{ doc.last_modified_at | default: doc.date | date_to_xmlschema }} - {% endif %} - -{% endfor %} -{% endfor %} - -{% assign pages = site.html_pages | where_exp:'doc','doc.sitemap != false' | where_exp:'doc','doc.url != "/404.html"' %} -{% for page in pages %} - - {% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ page.url | replace: '/index.html', '/' | absolute_url | xml_escape }} - {{ page.last_modified_at | default: 'now' | date_to_xmlschema }} - -{% endfor %} - -{% assign static_files = page.static_files | where_exp:'page','page.sitemap != false' | where_exp:'page','page.name != "404.html"' %} -{% for file in static_files %} - - {% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ file.path | replace:'/index.html','/' | absolute_url | xml_escape }} - {{ file.modified_time | date_to_xmlschema }} - -{% endfor %} - \ No newline at end of file