Update robots.txt handling and improve sitemap URL generation (#2130)

* Update robots.txt handling and improve sitemap URL generation

* Add dynamic page data handling and update sitemap generation

* Remove debug output from sitemap generation

* Create thin-clouds-glow.md

.changeset/thin-clouds-glow.md

@@ -0,0 +1,5 @@
+---
+"preview": patch
+---
+
+Update `robots.txt` handling and improve sitemap URL generation

preview/eleventy.config.mjs

@@ -1,6 +1,7 @@
 import { readFileSync } from 'node:fs';
 import { EleventyRenderPlugin } from "@11ty/eleventy";
 import { join, dirname } from 'node:path';
+import { sync } from 'glob';
 
 /*
  * Copy list
@@ -75,6 +76,16 @@ export default function (eleventyConfig) {
 	eleventyConfig.addGlobalData("license", readFileSync(join("..", "LICENSE"), "utf-8"));
 	eleventyConfig.addGlobalData("changelog", readFileSync(join("..", "CHANGELOG.md"), "utf-8"));
 
+	eleventyConfig.addGlobalData("pages", () => {
+		return sync('pages/**/*.html').filter((file) => {
+			return !file.includes('pages/_') && !file.includes('pages/docs/index.html');
+		}).map((file) => {
+			return {
+				url: file.replace(/^pages\//, '/')
+			}
+});
+	});
+
 	eleventyConfig.addGlobalData("site", {
 		title: "Tabler",
 		description: "Premium and Open Source dashboard template with responsive and high quality UI.",
@@ -86,7 +97,7 @@ export default function (eleventyConfig) {
 		githubSponsorsUrl: "https://github.com/sponsors/codecalm",
 		changelogUrl: "https://github.com/tabler/tabler/releases",
 		sponsorUrl: "https://github.com/sponsors/codecalm",
-		previewUrl: "https://tabler.io/demo",
+		previewUrl: "https://preview.tabler.io",
 		docsUrl: "https://tabler.io/docs",
 
 		mapboxKey: "pk.eyJ1IjoidGFibGVyIiwiYSI6ImNscHh3dnhndjB2M3QycW85bGd0NXRmZ3YifQ.9LfHPsNoEXQH-xzz-81Ffw",

preview/package.json

@@ -5,7 +5,7 @@
     "build": "pnpm run clean && pnpm run css && pnpm run js && pnpm run html",
     "dev": "pnpm run clean && pnpm run watch",
     "watch": "pnpm run watch-html & pnpm run watch-css & pnpm run watch-js",
-    "watch-html": "eleventy --serve --port=3000",
+    "watch-html": "eleventy --serve --port=3000 --incremental",
     "watch-js": "nodemon --watch js/ --ext js --exec 'pnpm run js'",
     "watch-css": "nodemon --watch scss/ --ext scss --exec 'pnpm run css'",
     "css": "pnpm run css-compile && pnpm run css-prefix && pnpm run css-minify",

preview/pages/_includes/cards/most-visited-pages.html

@@ -14,17 +14,17 @@
 			</tr>
 			</thead>
 
-			{% for page in pages limit: 6 %}
-			{% assign data = page.data | join: ',' %}
+			{% for url in urls limit: 6 %}
+			{% assign data = url.data | join: ',' %}
 			{% assign id = "bounce-rate-" | append: forloop.index %}
 			<tr>
 				<td>
-					{{ page.uri }}
+					{{ url.uri }}
 					<a href="#" class="ms-1" aria-label="Open website">{% include "ui/icon.html" icon="link" %}</a>
 				</td>
-				<td class="text-secondary">{{ page.visitors | format_number }}</td>
-				<td class="text-secondary">{{ page.unique | format_number }}</td>
-				<td class="text-secondary">{{ page.bounce }}</td>
+				<td class="text-secondary">{{ url.visitors | format_number }}</td>
+				<td class="text-secondary">{{ url.unique | format_number }}</td>
+				<td class="text-secondary">{{ url.bounce }}</td>
 				<td class="text-end w-1">
 					{% include "ui/chart-sparkline.html" type="line" data=data id=id small=true color="primary" %}
 				</td>

preview/pages/_includes/layout/header-actions/breadcrumb.html

@@ -1,6 +1,6 @@
-{% assign pages = "Tabler,Pages," %}
-{% assign pages = pages | append: page.page-header %}
+{% assign breadcrumb-pages = "Tabler,Pages," %}
+{% assign breadcrumb-pages = breadcrumb-pages | append: page.page-header %}
 
 <div class="d-flex">
-	{% include "ui/breadcrumb.html" pages=pages class="breadcrumb-arrows" %}
+	{% include "ui/breadcrumb.html" pages=breadcrumb-pages class="breadcrumb-arrows" %}
 </div>

preview/pages/_includes/ui/breadcrumb.html

@@ -1,6 +1,6 @@
-{% assign pages = include.pages | default: "Home,Library,Data" | split: "," %}
+{% assign breadcrumb-pages = include.pages | default: "Home,Library,Data" | split: "," %}
 <ol class="breadcrumb{% if include.class %} {{ include.class }}{% endif %}" aria-label="breadcrumbs">
-   {% for page in pages %}
+   {% for page in breadcrumb-pages %}
    {% if forloop.last %}
    <li class="breadcrumb-item active" aria-current="page"><a href="#">{{ page }}</a></li>
    {% else %}

preview/pages/robots.liquid

@@ -0,0 +1,13 @@
+---
+layout: null
+permalink: robots.txt
+---
+Sitemap: {% if environment != 'development' %}{{ site.previewUrl }}{% endif %}/sitemap.xml
+
+{% if environment == 'preview' %}
+User-agent: *
+Disallow:
+{% else %}
+User-agent: *
+Disallow: /
+{% endif %}

preview/pages/robots.txt

@@ -1,4 +0,0 @@
----
-layout: none
----
-Sitemap: {% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ "sitemap.xml" | absolute_url }}

preview/pages/sitemap.liquid

@@ -0,0 +1,14 @@
+---
+layout: null
+permalink: sitemap.xml
+---
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+
+{% for page in pages %}
+<url>
+	<loc>{% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ page.url | replace: 'index.html', '' | xml_escape }}</loc>
+	<lastmod>{{ page.last_modified_at | default: 'now' | date_to_xmlschema }}</lastmod>
+</url>
+{% endfor %}
+</urlset>

preview/pages/sitemap.xml

@@ -1,34 +0,0 @@
----
-layout: none
----
-<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-{% assign collections = site.collections | where_exp:'collection','collection.output != false' %}
-{% for collection in collections %}
-{% assign docs = collection.docs | where_exp:'doc','doc.sitemap != false' %}
-{% for doc in docs %}
-<url>
-	<loc>{% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ doc.url | replace: '/index.html', '/' | absolute_url | xml_escape }}</loc>
-	{% if doc.last_modified_at or doc.date %}
-	<lastmod>{{ doc.last_modified_at | default: doc.date | date_to_xmlschema }}</lastmod>
-	{% endif %}
-</url>
-{% endfor %}
-{% endfor %}
-
-{% assign pages = site.html_pages | where_exp:'doc','doc.sitemap != false' | where_exp:'doc','doc.url != "/404.html"' %}
-{% for page in pages %}
-<url>
-	<loc>{% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ page.url | replace: '/index.html', '/' | absolute_url | xml_escape }}</loc>
-	<lastmod>{{ page.last_modified_at | default: 'now' | date_to_xmlschema }}</lastmod>
-</url>
-{% endfor %}
-
-{% assign static_files = page.static_files | where_exp:'page','page.sitemap != false' | where_exp:'page','page.name != "404.html"' %}
-{% for file in static_files %}
-<url>
-	<loc>{% if environment != 'development' %}{{ site.previewUrl }}{% endif %}{{ file.path | replace:'/index.html','/' | absolute_url | xml_escape }}</loc>
-	<lastmod>{{ file.modified_time | date_to_xmlschema }}</lastmod>
-</url>
-{% endfor %}
-</urlset>