Data as document: replace snapshot with publish workflow

Mirror the SOA-to-document migration for the data list. Remove data
from the snapshot system and add a publish workflow that generates a
ProseMirror document for the full organization data inventory.

Signed-off-by: Sacha Al Himdani <sacha@getprobo.com>

pkg/server/api/console/v1/asset_resolvers.go

@@ -220,12 +220,7 @@ func (r *datumConnectionResolver) TotalCount(ctx context.Context, obj *types.Dat
 
 	switch obj.Resolver.(type) {
 	case *organizationResolver:
-		datumFilter := coredata.NewDatumFilter(nil)
-		if obj.Filter != nil {
-			datumFilter = coredata.NewDatumFilter(&obj.Filter.SnapshotID)
-		}
-
-		count, err := prb.Data.CountForOrganizationID(ctx, obj.ParentID, datumFilter)
+		count, err := prb.Data.CountForOrganizationID(ctx, obj.ParentID)
 		if err != nil {
 			r.logger.ErrorCtx(ctx, "cannot count data", log.Error(err))
 			return 0, gqlutils.Internal(ctx)
@@ -405,6 +400,29 @@ func (r *mutationResolver) DeleteDatum(ctx context.Context, input types.DeleteDa
 	}, nil
 }
 
+// PublishDataList is the resolver for the publishDataList field.
+func (r *mutationResolver) PublishDataList(ctx context.Context, input types.PublishDataListInput) (*types.PublishDataListPayload, error) {
+	if err := r.authorize(ctx, input.OrganizationID, probo.ActionDatumPublish); err != nil {
+		return nil, err
+	}
+
+	prb := r.ProboService(ctx, input.OrganizationID.TenantID())
+
+	document, documentVersion, err := prb.GeneratedDocuments.PublishDataList(ctx, input.OrganizationID, input.ApproverIds)
+	if err != nil {
+		if errors.Is(err, coredata.ErrResourceAlreadyExists) {
+			return nil, gqlutils.Conflict(ctx, err)
+		}
+		r.logger.ErrorCtx(ctx, "cannot publish data list", log.Error(err))
+		return nil, gqlutils.Internal(ctx)
+	}
+
+	return &types.PublishDataListPayload{
+		DocumentEdge:        types.NewDocumentEdge(document, coredata.DocumentOrderFieldCreatedAt),
+		DocumentVersionEdge: types.NewDocumentVersionEdge(documentVersion, coredata.DocumentVersionOrderFieldCreatedAt),
+	}, nil
+}
+
 // Asset returns schema.AssetResolver implementation.
 func (r *Resolver) Asset() schema.AssetResolver { return &assetResolver{r} }
 

pkg/server/api/console/v1/graphql/asset.graphql

@@ -66,10 +66,6 @@ input AssetFilter {
     snapshotId: ID
 }
 
-input DatumFilter {
-    snapshotId: ID
-}
-
 type Asset implements Node {
     id: ID!
     snapshotId: ID
@@ -97,7 +93,6 @@ type Datum implements Node
         model: "go.probo.inc/probo/pkg/server/api/console/v1/types.Datum"
     ) {
     id: ID!
-    snapshotId: ID
     name: String!
     dataClassification: DataClassification!
     owner: Profile! @goField(forceResolver: true)
@@ -150,6 +145,9 @@ extend type Mutation {
     createDatum(input: CreateDatumInput!): CreateDatumPayload!
     updateDatum(input: UpdateDatumInput!): UpdateDatumPayload!
     deleteDatum(input: DeleteDatumInput!): DeleteDatumPayload!
+    publishDataList(
+        input: PublishDataListInput!
+    ): PublishDataListPayload!
 }
 
 input CreateAssetInput {
@@ -219,3 +217,13 @@ type UpdateDatumPayload {
 type DeleteDatumPayload {
     deletedDatumId: ID!
 }
+
+input PublishDataListInput {
+    organizationId: ID!
+    approverIds: [ID!]
+}
+
+type PublishDataListPayload {
+    documentEdge: DocumentEdge!
+    documentVersionEdge: DocumentVersionEdge!
+}

pkg/server/api/console/v1/graphql/organization.graphql

@@ -130,13 +130,14 @@ type Organization implements Node {
         filter: AssetFilter = { snapshotId: null }
     ): AssetConnection! @goField(forceResolver: true)
 
+    dataListDocument: Document @goField(forceResolver: true)
+
     data(
         first: Int
         after: CursorKey
         last: Int
         before: CursorKey
         orderBy: DatumOrder
-        filter: DatumFilter = { snapshotId: null }
     ): DatumConnection! @goField(forceResolver: true)
 
     audits(

pkg/server/api/console/v1/graphql/snapshot.graphql

@@ -4,7 +4,6 @@ enum SnapshotsType
     VENDORS
         @goEnum(value: "go.probo.inc/probo/pkg/coredata.SnapshotsTypeVendors")
     ASSETS @goEnum(value: "go.probo.inc/probo/pkg/coredata.SnapshotsTypeAssets")
-    DATA @goEnum(value: "go.probo.inc/probo/pkg/coredata.SnapshotsTypeData")
     FINDINGS
         @goEnum(
             value: "go.probo.inc/probo/pkg/coredata.SnapshotsTypeFindings"

pkg/server/api/console/v1/organization_resolvers.go

@@ -256,8 +256,32 @@ func (r *organizationResolver) Assets(ctx context.Context, obj *types.Organizati
 	return types.NewAssetConnection(page, r, obj.ID, filter), nil
 }
 
-// Assets is the resolver for the assets field.
-func (r *organizationResolver) Data(ctx context.Context, obj *types.Organization, first *int, after *page.CursorKey, last *int, before *page.CursorKey, orderBy *types.DatumOrderBy, filter *types.DatumFilter) (*types.DatumConnection, error) {
+// DataListDocument is the resolver for the dataListDocument field.
+func (r *organizationResolver) DataListDocument(ctx context.Context, obj *types.Organization) (*types.Document, error) {
+	if err := r.authorize(ctx, obj.ID, probo.ActionDocumentGet); err != nil {
+		return nil, err
+	}
+
+	prb := r.ProboService(ctx, obj.ID.TenantID())
+
+	dataDocumentID, err := prb.GeneratedDocuments.GetDataListDocumentID(ctx, obj.ID)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get data export document ID: %w", err)
+	}
+	if dataDocumentID == nil {
+		return nil, nil
+	}
+
+	doc, err := prb.Documents.Get(ctx, *dataDocumentID)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get data export document: %w", err)
+	}
+
+	return types.NewDocument(doc), nil
+}
+
+// Data is the resolver for the data field.
+func (r *organizationResolver) Data(ctx context.Context, obj *types.Organization, first *int, after *page.CursorKey, last *int, before *page.CursorKey, orderBy *types.DatumOrderBy) (*types.DatumConnection, error) {
 	if err := r.authorize(ctx, obj.ID, probo.ActionDatumList); err != nil {
 		return nil, err
 	}
@@ -277,18 +301,13 @@ func (r *organizationResolver) Data(ctx context.Context, obj *types.Organization
 
 	cursor := types.NewCursor(first, after, last, before, pageOrderBy)
 
-	datumFilter := coredata.NewDatumFilter(nil)
-	if filter != nil {
-		datumFilter = coredata.NewDatumFilter(&filter.SnapshotID)
-	}
-
-	page, err := prb.Data.ListForOrganizationID(ctx, obj.ID, cursor, datumFilter)
+	page, err := prb.Data.ListForOrganizationID(ctx, obj.ID, cursor)
 	if err != nil {
 		r.logger.ErrorCtx(ctx, "cannot list organization data", log.Error(err))
 		return nil, gqlutils.Internal(ctx)
 	}
 
-	return types.NewDataConnection(page, r, obj.ID, filter), nil
+	return types.NewDataConnection(page, r, obj.ID), nil
 }
 
 // Audits is the resolver for the audits field.

pkg/server/api/console/v1/types/datum.go

@@ -25,7 +25,6 @@ import (
 type Datum struct {
 	ID                 gid.GID                     `json:"id"`
 	OrganizationID     gid.GID                     `json:"-"`
-	SnapshotID         *gid.GID                    `json:"snapshotId,omitempty"`
 	Name               string                      `json:"name"`
 	DataClassification coredata.DataClassification `json:"dataClassification"`
 	Owner              *Profile                    `json:"owner"`
@@ -48,7 +47,6 @@ type (
 
 		Resolver any
 		ParentID gid.GID
-		Filter   *DatumFilter
 	}
 )
 
@@ -56,7 +54,6 @@ func NewDataConnection(
 	p *page.Page[*coredata.Datum, coredata.DatumOrderField],
 	parentType any,
 	parentID gid.GID,
-	filter *DatumFilter,
 ) *DatumConnection {
 	edges := make([]*DatumEdge, len(p.Data))
 	for i, datum := range p.Data {
@@ -69,7 +66,6 @@ func NewDataConnection(
 
 		Resolver: parentType,
 		ParentID: parentID,
-		Filter:   filter,
 	}
 }
 
@@ -84,7 +80,6 @@ func NewDatum(d *coredata.Datum) *Datum {
 		},
 		OrganizationID:     d.OrganizationID,
 		Name:               d.Name,
-		SnapshotID:         d.SnapshotID,
 		DataClassification: d.DataClassification,
 		CreatedAt:          d.CreatedAt,
 		UpdatedAt:          d.UpdatedAt,