Harden common third party enricher edge cases
Reject oversized logo responses instead of silently truncating them, which could persist corrupt image bytes as a valid logo. Tighten ownership substring matching with a length-ratio guard so a short label root no longer attributes unrelated domains to a vendor. Render the worker confidence threshold when set to zero by testing against nil, so an explicit "accept all" value is not dropped by Helm's falsy-numeric truthiness. Sanitize and bound per-agent error text before persisting it to the enrichment metadata column to avoid leaking unbounded internal detail. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -416,7 +416,7 @@ spec:
|
|||||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS
|
||||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns | quote }}
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns | quote }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold }}
|
{{- if ne .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold nil }}
|
||||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD
|
||||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold | quote }}
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold | quote }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
|||||||
@@ -73,8 +73,27 @@ const (
|
|||||||
defaultEnrichmentMaxAttempts = 3
|
defaultEnrichmentMaxAttempts = 3
|
||||||
|
|
||||||
enrichmentLogoUserAgent = "Probo-Enricher/1.0"
|
enrichmentLogoUserAgent = "Probo-Enricher/1.0"
|
||||||
|
|
||||||
|
// maxEnrichmentErrorLen bounds the per-agent error text persisted in
|
||||||
|
// enrichment metadata so a verbose or hostile agent/tool error cannot
|
||||||
|
// leak unbounded internal detail into the column.
|
||||||
|
maxEnrichmentErrorLen = 500
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// sanitizeAgentError reduces a raw agent or tool error to a single bounded
|
||||||
|
// line safe to persist in enrichment metadata: whitespace runs (including
|
||||||
|
// embedded newlines) collapse to single spaces and the result is truncated
|
||||||
|
// to maxEnrichmentErrorLen runes.
|
||||||
|
func sanitizeAgentError(err error) string {
|
||||||
|
msg := strings.Join(strings.Fields(err.Error()), " ")
|
||||||
|
|
||||||
|
if runes := []rune(msg); len(runes) > maxEnrichmentErrorLen {
|
||||||
|
msg = string(runes[:maxEnrichmentErrorLen]) + "…"
|
||||||
|
}
|
||||||
|
|
||||||
|
return msg
|
||||||
|
}
|
||||||
|
|
||||||
// EnrichmentConfig configures the common-third-party enrichment worker
|
// EnrichmentConfig configures the common-third-party enrichment worker
|
||||||
// and the two agents it runs. The worker no-ops when LLMClient is nil;
|
// and the two agents it runs. The worker no-ops when LLMClient is nil;
|
||||||
// callers gate registration on config presence. Browser tools for Agent
|
// callers gate registration on config presence. Browser tools for Agent
|
||||||
@@ -219,7 +238,7 @@ func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonTh
|
|||||||
company, err := h.runCompanyProfile(ctx, party)
|
company, err := h.runCompanyProfile(ctx, party)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.WarnCtx(ctx, "company profile agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
h.logger.WarnCtx(ctx, "company profile agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
||||||
runErrors = append(runErrors, "company_profile: "+err.Error())
|
runErrors = append(runErrors, "company_profile: "+sanitizeAgentError(err))
|
||||||
} else {
|
} else {
|
||||||
anySuccess = true
|
anySuccess = true
|
||||||
}
|
}
|
||||||
@@ -260,7 +279,7 @@ func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonTh
|
|||||||
compliance, err := h.runComplianceDocs(ctx, party.Name, website, legalName)
|
compliance, err := h.runComplianceDocs(ctx, party.Name, website, legalName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.WarnCtx(ctx, "compliance docs agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
h.logger.WarnCtx(ctx, "compliance docs agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
||||||
runErrors = append(runErrors, "compliance_docs: "+err.Error())
|
runErrors = append(runErrors, "compliance_docs: "+sanitizeAgentError(err))
|
||||||
} else {
|
} else {
|
||||||
anySuccess = true
|
anySuccess = true
|
||||||
}
|
}
|
||||||
@@ -272,7 +291,7 @@ func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonTh
|
|||||||
domainsResult, err := h.runDomains(ctx, party.Name, website)
|
domainsResult, err := h.runDomains(ctx, party.Name, website)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.WarnCtx(ctx, "domains agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
h.logger.WarnCtx(ctx, "domains agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
||||||
runErrors = append(runErrors, "domains: "+err.Error())
|
runErrors = append(runErrors, "domains: "+sanitizeAgentError(err))
|
||||||
} else {
|
} else {
|
||||||
anySuccess = true
|
anySuccess = true
|
||||||
owned = resolveOwnedDomains(party.Name, website, domainsResult, defaultEnrichmentDomainConfidenceThreshold)
|
owned = resolveOwnedDomains(party.Name, website, domainsResult, defaultEnrichmentDomainConfidenceThreshold)
|
||||||
|
|||||||
6
pkg/thirdparty/common_third_party_logo.go
vendored
6
pkg/thirdparty/common_third_party_logo.go
vendored
@@ -196,11 +196,15 @@ func downloadImage(
|
|||||||
return nil, "", fmt.Errorf("logo response is not an image: %q", contentType)
|
return nil, "", fmt.Errorf("logo response is not an image: %q", contentType)
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxLogoSize))
|
body, err := io.ReadAll(io.LimitReader(resp.Body, maxLogoSize+1))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("cannot read logo body: %w", err)
|
return nil, "", fmt.Errorf("cannot read logo body: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(body) > maxLogoSize {
|
||||||
|
return nil, "", fmt.Errorf("logo response exceeds max size %d bytes", maxLogoSize)
|
||||||
|
}
|
||||||
|
|
||||||
if len(body) == 0 {
|
if len(body) == 0 {
|
||||||
return nil, "", fmt.Errorf("logo response is empty")
|
return nil, "", fmt.Errorf("logo response is empty")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -167,10 +167,11 @@ func exactLabelMatch(label string, vendorLabels []string) bool {
|
|||||||
return slices.Contains(vendorLabels, label)
|
return slices.Contains(vendorLabels, label)
|
||||||
}
|
}
|
||||||
|
|
||||||
// relatedLabelMatch reports whether label is the same as, contains, or is
|
// relatedLabelMatch reports whether label is the same as, or shares a
|
||||||
// contained by any vendor label. Substring matches require the shorter
|
// dominant root with, any vendor label. A substring match requires the
|
||||||
// string to be at least minLabelOverlap characters to avoid spurious hits
|
// shorter label to be at least minLabelOverlap characters AND to cover at
|
||||||
// on very short labels.
|
// least half of the longer label, so a short root (e.g. "meta") no longer
|
||||||
|
// attributes an unrelated domain (e.g. "metallica") to the vendor.
|
||||||
func relatedLabelMatch(label string, vendorLabels []string) bool {
|
func relatedLabelMatch(label string, vendorLabels []string) bool {
|
||||||
for _, vl := range vendorLabels {
|
for _, vl := range vendorLabels {
|
||||||
if label == vl {
|
if label == vl {
|
||||||
@@ -182,7 +183,15 @@ func relatedLabelMatch(label string, vendorLabels []string) bool {
|
|||||||
shorter, longer = longer, shorter
|
shorter, longer = longer, shorter
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(shorter) >= minLabelOverlap && strings.Contains(longer, shorter) {
|
if len(shorter) < minLabelOverlap {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(shorter)*2 < len(longer) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(longer, shorter) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user