101 lines
2.9 KiB
Go
101 lines
2.9 KiB
Go
package scan
|
|
|
|
import (
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
// roleStopwords are seniority/modality words that add no signal when comparing
|
|
// whether two job titles describe the same underlying role. Mirrors the list
|
|
// in career-ops dedup-tracker.mjs.
|
|
var roleStopwords = map[string]struct{}{
|
|
"senior": {}, "junior": {}, "lead": {}, "staff": {}, "principal": {},
|
|
"head": {}, "chief": {}, "manager": {}, "director": {}, "associate": {},
|
|
"intern": {}, "contractor": {}, "remote": {}, "hybrid": {}, "onsite": {},
|
|
"engineer": {}, "engineering": {},
|
|
}
|
|
|
|
// locationStopwords catch city/region tokens that sometimes appear inside a
|
|
// title (e.g. "Security Engineer — Tokyo"). They should not count toward
|
|
// role-similarity overlap.
|
|
var locationStopwords = map[string]struct{}{
|
|
"tokyo": {}, "japan": {}, "london": {}, "berlin": {}, "paris": {},
|
|
"singapore": {}, "york": {}, "francisco": {}, "angeles": {}, "seattle": {},
|
|
"austin": {}, "boston": {}, "chicago": {}, "denver": {}, "toronto": {},
|
|
"amsterdam": {}, "dublin": {}, "sydney": {}, "global": {}, "emea": {},
|
|
"apac": {}, "latam": {},
|
|
}
|
|
|
|
var tokenSplitRE = regexp.MustCompile(`[^a-z0-9]+`)
|
|
|
|
// tokenizeRole lowercases, splits on non-alphanumeric, drops short words
|
|
// (≤ 2 chars) and stopwords. The return is a dedup-preserving slice.
|
|
func tokenizeRole(s string) []string {
|
|
s = strings.ToLower(s)
|
|
raw := tokenSplitRE.Split(s, -1)
|
|
seen := make(map[string]struct{}, len(raw))
|
|
out := make([]string, 0, len(raw))
|
|
for _, w := range raw {
|
|
if len(w) <= 2 {
|
|
continue
|
|
}
|
|
if _, stop := roleStopwords[w]; stop {
|
|
continue
|
|
}
|
|
if _, stop := locationStopwords[w]; stop {
|
|
continue
|
|
}
|
|
if _, dup := seen[w]; dup {
|
|
continue
|
|
}
|
|
seen[w] = struct{}{}
|
|
out = append(out, w)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// RoleSimilar reports whether two job titles describe what is plausibly the
|
|
// same underlying role. Uses token-overlap with stopwords stripped and a
|
|
// prefix-both-ways comparison so that abbreviations (pen → penetration)
|
|
// don't break the match.
|
|
//
|
|
// A match requires:
|
|
// - both titles yield at least one content word (after stopword strip);
|
|
// - at least 2 content words overlap OR every content word in the shorter
|
|
// side matches (handles single-word patterns like "Security");
|
|
// - overlap ratio ≥ 0.5 against the smaller tokenized side.
|
|
//
|
|
// Intended for dedup UX ("we've seen a similar role"), not for hard filtering.
|
|
func RoleSimilar(a, b string) bool {
|
|
wa := tokenizeRole(a)
|
|
wb := tokenizeRole(b)
|
|
if len(wa) == 0 || len(wb) == 0 {
|
|
return false
|
|
}
|
|
overlap := 0
|
|
for _, x := range wa {
|
|
for _, y := range wb {
|
|
if x == y || strings.HasPrefix(y, x) || strings.HasPrefix(x, y) {
|
|
overlap++
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if overlap == 0 {
|
|
return false
|
|
}
|
|
smaller := len(wa)
|
|
if len(wb) < smaller {
|
|
smaller = len(wb)
|
|
}
|
|
required := 2
|
|
if smaller < required {
|
|
required = smaller
|
|
}
|
|
if overlap < required {
|
|
return false
|
|
}
|
|
ratio := float64(overlap) / float64(smaller)
|
|
return ratio >= 0.5
|
|
}
|