From 359f85f5dacb78125bb66a710aa2f183a060b614 Mon Sep 17 00:00:00 2001 From: Bryan Frimin Date: Tue, 24 Mar 2026 15:55:31 +0100 Subject: [PATCH] Add SEO controls and sitemap for compliance pages Add search engine indexing toggle, robots.txt, and sitemap.xml generation for compliance pages. Replace checkboxes with toggle components in the compliance page UI and add an "Open" button in the page header to quickly access the live compliance page. The search engine indexing toggle is disabled when the compliance page is inactive. Signed-off-by: Bryan Frimin --- .../src/hooks/graph/TrustCenterGraph.ts | 1 + .../compliance-page/CompliancePageLayout.tsx | 25 +++- .../CompliancePageStatusSection.tsx | 101 ++++++++-------- packages/ui/src/Atoms/Toggle/Toggle.tsx | 47 ++++++++ packages/ui/src/index.ts | 1 + pkg/coredata/migrations/20260324T120000Z.sql | 5 + pkg/coredata/search_engine_indexing.go | 77 ++++++++++++ pkg/coredata/trust_center.go | 32 +++-- pkg/probo/trust_center_service.go | 4 + .../api/compliancepage/markdown_handler.go | 40 +++++++ pkg/server/api/console/v1/schema.graphql | 16 +++ .../api/console/v1/types/trust_center.go | 10 +- pkg/server/api/console/v1/v1_resolver.go | 5 +- pkg/server/server.go | 2 + pkg/trust/compliance_page_service.go | 113 ++++++++++++++++++ pkg/trust/robots.txt.tmpl | 9 ++ pkg/trust/sitemap.xml.tmpl | 18 +++ 17 files changed, 438 insertions(+), 68 deletions(-) create mode 100644 packages/ui/src/Atoms/Toggle/Toggle.tsx create mode 100644 pkg/coredata/migrations/20260324T120000Z.sql create mode 100644 pkg/coredata/search_engine_indexing.go create mode 100644 pkg/trust/robots.txt.tmpl create mode 100644 pkg/trust/sitemap.xml.tmpl diff --git a/apps/console/src/hooks/graph/TrustCenterGraph.ts b/apps/console/src/hooks/graph/TrustCenterGraph.ts index 70c0031d6..d7a65ae91 100644 --- a/apps/console/src/hooks/graph/TrustCenterGraph.ts +++ b/apps/console/src/hooks/graph/TrustCenterGraph.ts @@ -9,6 +9,7 @@ const updateTrustCenterMutation = graphql` trustCenter { id active + searchEngineIndexing updatedAt } } diff --git a/apps/console/src/pages/organizations/compliance-page/CompliancePageLayout.tsx b/apps/console/src/pages/organizations/compliance-page/CompliancePageLayout.tsx index beab92c07..80bbd30e9 100644 --- a/apps/console/src/pages/organizations/compliance-page/CompliancePageLayout.tsx +++ b/apps/console/src/pages/organizations/compliance-page/CompliancePageLayout.tsx @@ -1,6 +1,6 @@ import { usePageTitle } from "@probo/hooks"; import { useTranslate } from "@probo/i18n"; -import { Badge, IconBell2, IconCheckmark1, IconFolder2, IconMedal, IconPageTextLine, IconPencil, IconPeopleAdd, IconSettingsGear2, IconStore, PageHeader, TabLink, Tabs } from "@probo/ui"; +import { Badge, Button, IconBell2, IconCheckmark1, IconFolder2, IconMedal, IconPageTextLine, IconPencil, IconPeopleAdd, IconSettingsGear2, IconStore, PageHeader, TabLink, Tabs } from "@probo/ui"; import { type PreloadedQuery, usePreloadedQuery } from "react-relay"; import { Outlet } from "react-router"; import { graphql } from "relay-runtime"; @@ -13,7 +13,11 @@ export const compliancePageLayoutQuery = graphql` organization: node(id: $organizationId) { __typename ... on Organization { + customDomain { + domain + } compliancePage: trustCenter { + id active } } @@ -34,6 +38,12 @@ export function CompliancePageLayout(props: { queryRef: PreloadedQuery {organization.compliancePage?.active ? __("Active") : __("Inactive")} + {organization.compliancePage?.active && compliancePageUrl && ( + + )} diff --git a/apps/console/src/pages/organizations/compliance-page/overview/_components/CompliancePageStatusSection.tsx b/apps/console/src/pages/organizations/compliance-page/overview/_components/CompliancePageStatusSection.tsx index 911b5d4b9..64e04b977 100644 --- a/apps/console/src/pages/organizations/compliance-page/overview/_components/CompliancePageStatusSection.tsx +++ b/apps/console/src/pages/organizations/compliance-page/overview/_components/CompliancePageStatusSection.tsx @@ -1,5 +1,5 @@ import { useTranslate } from "@probo/i18n"; -import { Button, Card, Checkbox, Spinner, useToast } from "@probo/ui"; +import { Card, Spinner, Toggle, useToast } from "@probo/ui"; import { useFragment } from "react-relay"; import { graphql } from "relay-runtime"; @@ -8,12 +8,10 @@ import { useUpdateTrustCenterMutation } from "#/hooks/graph/TrustCenterGraph"; const fragment = graphql` fragment CompliancePageStatusSectionFragment on Organization { - customDomain { - domain - } compliancePage: trustCenter { id active + searchEngineIndexing canUpdate: permission(action: "core:trust-center:update") } } @@ -32,12 +30,6 @@ export function CompliancePageStatusSection(props: { fragmentRef, ); - const compliancePageUrl = organization.compliancePage?.id - ? organization.customDomain?.domain - ? `https://${organization.customDomain.domain}` - : `${window.location.origin}/trust/${organization.compliancePage.id}` - : null; - const [updateCompliancePage, isUpdating] = useUpdateTrustCenterMutation(); const handleToggleActive = async (active: boolean) => { @@ -60,10 +52,32 @@ export function CompliancePageStatusSection(props: { }); }; + const handleToggleSearchEngineIndexing = async (indexable: boolean) => { + if (!organization.compliancePage?.id) { + toast({ + title: __("Error"), + description: __("Compliance page not found"), + variant: "error", + }); + return; + } + + await updateCompliancePage({ + variables: { + input: { + trustCenterId: organization.compliancePage.id, + searchEngineIndexing: indexable ? "INDEXABLE" : "NOT_INDEXABLE", + }, + }, + }); + }; + return (
-

{__("Compliance Page Status")}

+

+ {__("Compliance Page Status")} +

{isUpdating && }
@@ -76,55 +90,42 @@ export function CompliancePageStatusSection(props: { )}

- void handleToggleActive(checked)} disabled={!organization.compliancePage?.canUpdate} /> - {organization.compliancePage?.active && compliancePageUrl && ( -
-
-
-

- {__("Your compliance page is live!")} -

-

- {__("Your customers can now access your compliance page at:")} -

- - {compliancePageUrl} - -
- -
-
- )} - - {!organization.compliancePage?.active && ( -
-

- {__("Compliance page is inactive")} -

-

+

+
+

{__("Search Engine Indexing")}

+

{__( - "Your compliance page is currently not accessible to the public. Enable it to start sharing your compliance status.", + "Allow search engines to index your compliance page and make it discoverable", )}

- )} + + + void handleToggleSearchEngineIndexing(checked)} + disabled={ + !organization.compliancePage?.canUpdate + || !organization.compliancePage?.active + } + /> + +
); diff --git a/packages/ui/src/Atoms/Toggle/Toggle.tsx b/packages/ui/src/Atoms/Toggle/Toggle.tsx new file mode 100644 index 000000000..53240c6c4 --- /dev/null +++ b/packages/ui/src/Atoms/Toggle/Toggle.tsx @@ -0,0 +1,47 @@ +type Props = { + checked: boolean; + onChange: (checked: boolean) => void; + disabled?: boolean; +}; + +export function Toggle({ checked, onChange, disabled = false }: Props) { + return ( + + ); +} diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index 8d5eceba1..686bb77d0 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -38,6 +38,7 @@ export { InfiniteScrollTrigger } from "./Atoms/InfiniteScrollTrigger/InfiniteScr export { PriorityLevel } from "./Atoms/PriorityLevel/PriorityLevel"; export { TaskStateIcon } from "./Atoms/Icons/TaskStateIcon"; export { Checkbox } from "./Atoms/Checkbox/Checkbox"; +export { Toggle } from "./Atoms/Toggle/Toggle"; export { Cell, CellHead, diff --git a/pkg/coredata/migrations/20260324T120000Z.sql b/pkg/coredata/migrations/20260324T120000Z.sql new file mode 100644 index 000000000..b9a2a28ee --- /dev/null +++ b/pkg/coredata/migrations/20260324T120000Z.sql @@ -0,0 +1,5 @@ +ALTER TABLE trust_centers +ADD COLUMN search_engine_indexing TEXT NOT NULL DEFAULT 'NOT_INDEXABLE'; + +ALTER TABLE trust_centers +ALTER COLUMN search_engine_indexing DROP DEFAULT; diff --git a/pkg/coredata/search_engine_indexing.go b/pkg/coredata/search_engine_indexing.go new file mode 100644 index 000000000..2380a05c8 --- /dev/null +++ b/pkg/coredata/search_engine_indexing.go @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package coredata + +import ( + "database/sql/driver" + "fmt" +) + +type SearchEngineIndexing string + +const ( + SearchEngineIndexingIndexable SearchEngineIndexing = "INDEXABLE" + SearchEngineIndexingNotIndexable SearchEngineIndexing = "NOT_INDEXABLE" +) + +func (s SearchEngineIndexing) String() string { + return string(s) +} + +func (s SearchEngineIndexing) IsValid() bool { + switch s { + case SearchEngineIndexingIndexable, SearchEngineIndexingNotIndexable: + return true + } + return false +} + +func (s *SearchEngineIndexing) UnmarshalText(text []byte) error { + *s = SearchEngineIndexing(text) + if !s.IsValid() { + return fmt.Errorf("%s is not a valid SearchEngineIndexing", string(text)) + } + return nil +} + +func (s SearchEngineIndexing) MarshalText() ([]byte, error) { + return []byte(s.String()), nil +} + +func (s *SearchEngineIndexing) Scan(value any) error { + var str string + switch v := value.(type) { + case string: + str = v + case []byte: + str = string(v) + default: + return fmt.Errorf("unsupported type for SearchEngineIndexing: %T", value) + } + + switch str { + case "INDEXABLE": + *s = SearchEngineIndexingIndexable + case "NOT_INDEXABLE": + *s = SearchEngineIndexingNotIndexable + default: + return fmt.Errorf("invalid SearchEngineIndexing value: %q", str) + } + return nil +} + +func (s SearchEngineIndexing) Value() (driver.Value, error) { + return s.String(), nil +} diff --git a/pkg/coredata/trust_center.go b/pkg/coredata/trust_center.go index 9a7388645..e9be8c863 100644 --- a/pkg/coredata/trust_center.go +++ b/pkg/coredata/trust_center.go @@ -30,17 +30,18 @@ import ( type ( TrustCenter struct { - ID gid.GID `db:"id"` - OrganizationID gid.GID `db:"organization_id"` - TenantID gid.TenantID `db:"tenant_id"` - Active bool `db:"active"` - Slug string `db:"slug"` - MailingListID *gid.GID `db:"mailing_list_id"` - LogoFileID *gid.GID `db:"logo_file_id"` - DarkLogoFileID *gid.GID `db:"dark_logo_file_id"` - NonDisclosureAgreementFileID *gid.GID `db:"non_disclosure_agreement_file_id"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID gid.GID `db:"id"` + OrganizationID gid.GID `db:"organization_id"` + TenantID gid.TenantID `db:"tenant_id"` + Active bool `db:"active"` + Slug string `db:"slug"` + SearchEngineIndexing SearchEngineIndexing `db:"search_engine_indexing"` + MailingListID *gid.GID `db:"mailing_list_id"` + LogoFileID *gid.GID `db:"logo_file_id"` + DarkLogoFileID *gid.GID `db:"dark_logo_file_id"` + NonDisclosureAgreementFileID *gid.GID `db:"non_disclosure_agreement_file_id"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } TrustCenters []*TrustCenter @@ -85,6 +86,7 @@ SELECT dark_logo_file_id, active, slug, + search_engine_indexing, non_disclosure_agreement_file_id, created_at, updated_at @@ -136,6 +138,7 @@ SELECT dark_logo_file_id, active, slug, + search_engine_indexing, non_disclosure_agreement_file_id, created_at, updated_at @@ -187,6 +190,7 @@ SELECT dark_logo_file_id, active, slug, + search_engine_indexing, non_disclosure_agreement_file_id, created_at, updated_at @@ -238,6 +242,7 @@ SELECT dark_logo_file_id, active, slug, + search_engine_indexing, non_disclosure_agreement_file_id, created_at, updated_at @@ -284,6 +289,7 @@ INSERT INTO trust_centers ( dark_logo_file_id, active, slug, + search_engine_indexing, non_disclosure_agreement_file_id, created_at, updated_at @@ -296,6 +302,7 @@ INSERT INTO trust_centers ( @dark_logo_file_id, @active, @slug, + @search_engine_indexing, @non_disclosure_agreement_file_id, @created_at, @updated_at @@ -311,6 +318,7 @@ INSERT INTO trust_centers ( "dark_logo_file_id": tc.DarkLogoFileID, "active": tc.Active, "slug": tc.Slug, + "search_engine_indexing": tc.SearchEngineIndexing, "non_disclosure_agreement_file_id": tc.NonDisclosureAgreementFileID, "created_at": tc.CreatedAt, "updated_at": tc.UpdatedAt, @@ -340,6 +348,7 @@ UPDATE trust_centers SET active = @active, slug = @slug, + search_engine_indexing = @search_engine_indexing, logo_file_id = @logo_file_id, dark_logo_file_id = @dark_logo_file_id, non_disclosure_agreement_file_id = @non_disclosure_agreement_file_id, @@ -357,6 +366,7 @@ WHERE "dark_logo_file_id": tc.DarkLogoFileID, "active": tc.Active, "slug": tc.Slug, + "search_engine_indexing": tc.SearchEngineIndexing, "non_disclosure_agreement_file_id": tc.NonDisclosureAgreementFileID, "updated_at": tc.UpdatedAt, } diff --git a/pkg/probo/trust_center_service.go b/pkg/probo/trust_center_service.go index a8bd3c85f..82fe4299b 100644 --- a/pkg/probo/trust_center_service.go +++ b/pkg/probo/trust_center_service.go @@ -43,6 +43,7 @@ type ( ID gid.GID Active *bool Slug *string + SearchEngineIndexing *coredata.SearchEngineIndexing NonDisclosureAgreementFileID *gid.GID } @@ -178,6 +179,9 @@ func (s TrustCenterService) Update( if req.Slug != nil { trustCenter.Slug = *req.Slug } + if req.SearchEngineIndexing != nil { + trustCenter.SearchEngineIndexing = *req.SearchEngineIndexing + } trustCenter.UpdatedAt = time.Now() if err := trustCenter.Update(ctx, conn, s.svc.scope); err != nil { diff --git a/pkg/server/api/compliancepage/markdown_handler.go b/pkg/server/api/compliancepage/markdown_handler.go index d0ad1e845..f61408cd3 100644 --- a/pkg/server/api/compliancepage/markdown_handler.go +++ b/pkg/server/api/compliancepage/markdown_handler.go @@ -41,3 +41,43 @@ func (h *Handler) HandleLLMsTxt(w http.ResponseWriter, r *http.Request) { http.Error(w, "internal server error", http.StatusInternalServerError) } } + +func (h *Handler) HandleRobotsTxt(w http.ResponseWriter, r *http.Request) { + tc := CompliancePageFromContext(r.Context()) + if tc == nil { + http.Error(w, "not found", http.StatusNotFound) + return + } + + baseURL := CompliancePageBaseURLFromContext(r.Context()) + if baseURL == nil { + http.Error(w, "not found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + if err := h.trustService.RenderRobotsTxt(r.Context(), w, tc.SearchEngineIndexing, *baseURL); err != nil { + http.Error(w, "internal server error", http.StatusInternalServerError) + } +} + +func (h *Handler) HandleSitemap(w http.ResponseWriter, r *http.Request) { + tc := CompliancePageFromContext(r.Context()) + if tc == nil { + http.Error(w, "not found", http.StatusNotFound) + return + } + + baseURL := CompliancePageBaseURLFromContext(r.Context()) + if baseURL == nil { + http.Error(w, "not found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + + if err := h.trustService.RenderSitemap(r.Context(), w, tc.ID, tc.TenantID, *baseURL); err != nil { + http.Error(w, "internal server error", http.StatusInternalServerError) + } +} diff --git a/pkg/server/api/console/v1/schema.graphql b/pkg/server/api/console/v1/schema.graphql index 46ed4a174..355b129aa 100644 --- a/pkg/server/api/console/v1/schema.graphql +++ b/pkg/server/api/console/v1/schema.graphql @@ -138,6 +138,20 @@ enum TrustCenterVisibility ) } +enum SearchEngineIndexing + @goModel( + model: "go.probo.inc/probo/pkg/coredata.SearchEngineIndexing" + ) { + INDEXABLE + @goEnum( + value: "go.probo.inc/probo/pkg/coredata.SearchEngineIndexingIndexable" + ) + NOT_INDEXABLE + @goEnum( + value: "go.probo.inc/probo/pkg/coredata.SearchEngineIndexingNotIndexable" + ) +} + enum ControlImplementationState @goModel( model: "go.probo.inc/probo/pkg/coredata.ControlImplementationState" @@ -1685,6 +1699,7 @@ type TrustCenter implements Node ) { id: ID! active: Boolean! + searchEngineIndexing: SearchEngineIndexing! logoFileUrl: String @goField(forceResolver: true) darkLogoFileUrl: String @goField(forceResolver: true) ndaFileName: String @@ -3893,6 +3908,7 @@ input UpdateOrganizationContextInput { input UpdateTrustCenterInput { trustCenterId: ID! active: Boolean + searchEngineIndexing: SearchEngineIndexing } input UploadTrustCenterNDAInput { diff --git a/pkg/server/api/console/v1/types/trust_center.go b/pkg/server/api/console/v1/types/trust_center.go index 04b463a71..da1695b1a 100644 --- a/pkg/server/api/console/v1/types/trust_center.go +++ b/pkg/server/api/console/v1/types/trust_center.go @@ -24,6 +24,7 @@ import ( type TrustCenter struct { ID gid.GID `json:"id"` Active bool `json:"active"` + SearchEngineIndexing coredata.SearchEngineIndexing `json:"searchEngineIndexing"` LogoFileURL *string `json:"logoFileUrl,omitempty"` DarkLogoFileURL *string `json:"darkLogoFileUrl,omitempty"` NdaFileName *string `json:"ndaFileName,omitempty"` @@ -53,9 +54,10 @@ func NewTrustCenter(tc *coredata.TrustCenter, file *coredata.File) *TrustCenter Organization: &Organization{ ID: tc.OrganizationID, }, - Active: tc.Active, - NdaFileName: ndaFileName, - CreatedAt: tc.CreatedAt, - UpdatedAt: tc.UpdatedAt, + Active: tc.Active, + SearchEngineIndexing: tc.SearchEngineIndexing, + NdaFileName: ndaFileName, + CreatedAt: tc.CreatedAt, + UpdatedAt: tc.UpdatedAt, } } diff --git a/pkg/server/api/console/v1/v1_resolver.go b/pkg/server/api/console/v1/v1_resolver.go index 0eeeadb3f..ae9c47735 100644 --- a/pkg/server/api/console/v1/v1_resolver.go +++ b/pkg/server/api/console/v1/v1_resolver.go @@ -2106,8 +2106,9 @@ func (r *mutationResolver) UpdateTrustCenter(ctx context.Context, input types.Up trustCenter, file, err := prb.TrustCenters.Update( ctx, &probo.UpdateTrustCenterRequest{ - ID: input.TrustCenterID, - Active: input.Active, + ID: input.TrustCenterID, + Active: input.Active, + SearchEngineIndexing: input.SearchEngineIndexing, }, ) if err != nil { diff --git a/pkg/server/server.go b/pkg/server/server.go index 3f02a6097..4d72255bc 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -178,6 +178,8 @@ func (s *Server) trustCenterRouter() chi.Router { r.Mount("/api/trust/v1", s.apiServer.CompliancePageHandler()) r.Get("/llms.txt", h.HandleLLMsTxt) + r.Get("/robots.txt", h.HandleRobotsTxt) + r.Get("/sitemap.xml", h.HandleSitemap) r.Handle("/*", s.trustWebServer) return r diff --git a/pkg/trust/compliance_page_service.go b/pkg/trust/compliance_page_service.go index d0d2b90a5..27b4b65de 100644 --- a/pkg/trust/compliance_page_service.go +++ b/pkg/trust/compliance_page_service.go @@ -31,6 +31,12 @@ import ( //go:embed compliance.md.tmpl var complianceTmplContent string +//go:embed sitemap.xml.tmpl +var sitemapTmplContent string + +//go:embed robots.txt.tmpl +var robotsTmplContent string + var complianceTmpl = template.Must( template.New("compliance"). Funcs(template.FuncMap{ @@ -44,6 +50,14 @@ var complianceTmpl = template.Must( Parse(complianceTmplContent), ) +var sitemapTmpl = template.Must( + template.New("sitemap").Parse(sitemapTmplContent), +) + +var robotsTmpl = template.Must( + template.New("robots").Parse(robotsTmplContent), +) + type ( compliancePageData struct { OrgName string @@ -166,6 +180,105 @@ func (s *Service) RenderCompliancePageMarkdown( return nil } +type ( + sitemapData struct { + BaseURL string + Documents []string + } + + robotsData struct { + Indexable bool + BaseURL string + } +) + +func (s *Service) RenderSitemap( + ctx context.Context, + w io.Writer, + trustCenterID gid.GID, + tenantID gid.TenantID, + baseURL string, +) error { + org, err := s.GetOrganizationByTrustCenterID(ctx, trustCenterID) + if err != nil { + return fmt.Errorf("cannot load organization for sitemap: %w", err) + } + + tenantSvc := s.WithTenant(tenantID) + + data := &sitemapData{ + BaseURL: baseURL, + } + + data.Documents, err = s.fetchDocumentIDs(ctx, tenantSvc, org.ID) + if err != nil { + return fmt.Errorf("cannot fetch document IDs for sitemap: %w", err) + } + + if err := sitemapTmpl.Execute(w, data); err != nil { + return fmt.Errorf("cannot render sitemap: %w", err) + } + + return nil +} + +func (s *Service) RenderRobotsTxt( + ctx context.Context, + w io.Writer, + searchEngineIndexing coredata.SearchEngineIndexing, + baseURL string, +) error { + data := &robotsData{ + Indexable: searchEngineIndexing == coredata.SearchEngineIndexingIndexable, + BaseURL: baseURL, + } + + if err := robotsTmpl.Execute(w, data); err != nil { + return fmt.Errorf("cannot render robots.txt: %w", err) + } + + return nil +} + +func (s *Service) fetchDocumentIDs(ctx context.Context, tenantSvc *TenantService, orgID gid.GID) ([]string, error) { + var ids []string + + var cursorKey *page.CursorKey + for { + cursor := page.NewCursor( + page.MaxCursorSize, + cursorKey, + page.Head, + page.OrderBy[coredata.DocumentOrderField]{ + Field: coredata.DocumentOrderFieldTitle, + Direction: page.OrderDirectionAsc, + }, + ) + + result, err := tenantSvc.Documents.ListForOrganizationId(ctx, orgID, cursor) + if err != nil { + return nil, fmt.Errorf("cannot list documents: %w", err) + } + + for _, doc := range result.Data { + if doc.TrustCenterVisibility == coredata.TrustCenterVisibilityNone { + continue + } + ids = append(ids, doc.ID.String()) + } + + if !result.Info.HasNext { + break + } + + last := result.Data[len(result.Data)-1] + ck := last.CursorKey(coredata.DocumentOrderFieldTitle) + cursorKey = &ck + } + + return ids, nil +} + func (s *Service) fetchComplianceFrameworks(ctx context.Context, tenantSvc *TenantService, trustCenterID gid.GID) ([]compliancePageFramework, error) { var frameworks []compliancePageFramework diff --git a/pkg/trust/robots.txt.tmpl b/pkg/trust/robots.txt.tmpl new file mode 100644 index 000000000..47167a95c --- /dev/null +++ b/pkg/trust/robots.txt.tmpl @@ -0,0 +1,9 @@ +{{- if .Indexable }} +User-Agent: * +Allow: / + +Sitemap: {{ .BaseURL }}/sitemap.xml +{{- else }} +User-Agent: * +Disallow: / +{{- end }} diff --git a/pkg/trust/sitemap.xml.tmpl b/pkg/trust/sitemap.xml.tmpl new file mode 100644 index 000000000..44af78c5b --- /dev/null +++ b/pkg/trust/sitemap.xml.tmpl @@ -0,0 +1,18 @@ + + + + {{ .BaseURL }}/overview + + + {{ .BaseURL }}/documents + + + {{ .BaseURL }}/subprocessors + + + {{ .BaseURL }}/updates + +{{ range .Documents }} + {{ $.BaseURL }}/documents/{{ . }} + +{{ end }}