@@ -248,6 +248,7 @@ func normalizeCertifications(values []string) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
seen[key] = struct{}{}
|
seen[key] = struct{}{}
|
||||||
|
|
||||||
out = append(out, v)
|
out = append(out, v)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -250,6 +250,7 @@ func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonTh
|
|||||||
applyCertifications(&party, meta, prior, compliance.Certifications, h.cfg.ConfidenceThreshold, now)
|
applyCertifications(&party, meta, prior, compliance.Certifications, h.cfg.ConfidenceThreshold, now)
|
||||||
|
|
||||||
status := enrichmentStatusDone
|
status := enrichmentStatusDone
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case !anySuccess:
|
case !anySuccess:
|
||||||
status = enrichmentStatusFailed
|
status = enrichmentStatusFailed
|
||||||
|
|||||||
2
pkg/thirdparty/common_third_party_logo.go
vendored
2
pkg/thirdparty/common_third_party_logo.go
vendored
@@ -178,6 +178,7 @@ func downloadImage(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("cannot fetch logo: %w", err)
|
return nil, "", fmt.Errorf("cannot fetch logo: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
@@ -188,6 +189,7 @@ func downloadImage(
|
|||||||
if idx := strings.Index(contentType, ";"); idx != -1 {
|
if idx := strings.Index(contentType, ";"); idx != -1 {
|
||||||
contentType = contentType[:idx]
|
contentType = contentType[:idx]
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType = strings.TrimSpace(contentType)
|
contentType = strings.TrimSpace(contentType)
|
||||||
|
|
||||||
if !strings.HasPrefix(contentType, "image/") {
|
if !strings.HasPrefix(contentType, "image/") {
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ func FindLogoURL(info *PageInfo) (string, error) {
|
|||||||
|
|
||||||
for _, n := range findAllIn(head, "link") {
|
for _, n := range findAllIn(head, "link") {
|
||||||
rel := strings.ToLower(attrVal(n, "rel"))
|
rel := strings.ToLower(attrVal(n, "rel"))
|
||||||
|
|
||||||
href := attrVal(n, "href")
|
href := attrVal(n, "href")
|
||||||
if href == "" {
|
if href == "" {
|
||||||
continue
|
continue
|
||||||
@@ -64,6 +65,7 @@ func FindLogoURL(info *PageInfo) (string, error) {
|
|||||||
|
|
||||||
for _, n := range findAllIn(head, "meta") {
|
for _, n := range findAllIn(head, "meta") {
|
||||||
name := strings.ToLower(attrVal(n, "name"))
|
name := strings.ToLower(attrVal(n, "name"))
|
||||||
|
|
||||||
content := attrVal(n, "content")
|
content := attrVal(n, "content")
|
||||||
if name == "msapplication-tileimage" && content != "" {
|
if name == "msapplication-tileimage" && content != "" {
|
||||||
msTileImage = content
|
msTileImage = content
|
||||||
@@ -92,8 +94,10 @@ func parseSizeAttr(sizes string) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
best := 0
|
best := 0
|
||||||
|
|
||||||
for token := range strings.FieldsSeq(sizes) {
|
for token := range strings.FieldsSeq(sizes) {
|
||||||
token = strings.ToLower(token)
|
token = strings.ToLower(token)
|
||||||
|
|
||||||
parts := strings.SplitN(token, "x", 2)
|
parts := strings.SplitN(token, "x", 2)
|
||||||
if len(parts) != 2 {
|
if len(parts) != 2 {
|
||||||
continue
|
continue
|
||||||
@@ -117,6 +121,7 @@ func ExtensionForMIME(contentType string) string {
|
|||||||
if idx := strings.Index(ct, ";"); idx != -1 {
|
if idx := strings.Index(ct, ";"); idx != -1 {
|
||||||
ct = ct[:idx]
|
ct = ct[:idx]
|
||||||
}
|
}
|
||||||
|
|
||||||
ct = strings.TrimSpace(ct)
|
ct = strings.TrimSpace(ct)
|
||||||
|
|
||||||
switch ct {
|
switch ct {
|
||||||
@@ -146,6 +151,7 @@ func (p *PageInfo) HeadLinks(rel string) []*html.Node {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rel = strings.ToLower(rel)
|
rel = strings.ToLower(rel)
|
||||||
|
|
||||||
var matches []*html.Node
|
var matches []*html.Node
|
||||||
|
|
||||||
for _, n := range findAllIn(head, "link") {
|
for _, n := range findAllIn(head, "link") {
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageIn
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("cannot fetch page: %w", err)
|
return nil, fmt.Errorf("cannot fetch page: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
@@ -57,6 +58,7 @@ func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageIn
|
|||||||
}
|
}
|
||||||
|
|
||||||
const maxHTMLSize = 10 << 20 // 10 MiB
|
const maxHTMLSize = 10 << 20 // 10 MiB
|
||||||
|
|
||||||
return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize))
|
return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user