initial public release

internal/scan/similarity.go

@@ -0,0 +1,100 @@
+package scan
+
+import (
+	"regexp"
+	"strings"
+)
+
+// roleStopwords are seniority/modality words that add no signal when comparing
+// whether two job titles describe the same underlying role. Mirrors the list
+// in career-ops dedup-tracker.mjs.
+var roleStopwords = map[string]struct{}{
+	"senior": {}, "junior": {}, "lead": {}, "staff": {}, "principal": {},
+	"head": {}, "chief": {}, "manager": {}, "director": {}, "associate": {},
+	"intern": {}, "contractor": {}, "remote": {}, "hybrid": {}, "onsite": {},
+	"engineer": {}, "engineering": {},
+}
+
+// locationStopwords catch city/region tokens that sometimes appear inside a
+// title (e.g. "Security Engineer — Tokyo"). They should not count toward
+// role-similarity overlap.
+var locationStopwords = map[string]struct{}{
+	"tokyo": {}, "japan": {}, "london": {}, "berlin": {}, "paris": {},
+	"singapore": {}, "york": {}, "francisco": {}, "angeles": {}, "seattle": {},
+	"austin": {}, "boston": {}, "chicago": {}, "denver": {}, "toronto": {},
+	"amsterdam": {}, "dublin": {}, "sydney": {}, "global": {}, "emea": {},
+	"apac": {}, "latam": {},
+}
+
+var tokenSplitRE = regexp.MustCompile(`[^a-z0-9]+`)
+
+// tokenizeRole lowercases, splits on non-alphanumeric, drops short words
+// (≤ 2 chars) and stopwords. The return is a dedup-preserving slice.
+func tokenizeRole(s string) []string {
+	s = strings.ToLower(s)
+	raw := tokenSplitRE.Split(s, -1)
+	seen := make(map[string]struct{}, len(raw))
+	out := make([]string, 0, len(raw))
+	for _, w := range raw {
+		if len(w) <= 2 {
+			continue
+		}
+		if _, stop := roleStopwords[w]; stop {
+			continue
+		}
+		if _, stop := locationStopwords[w]; stop {
+			continue
+		}
+		if _, dup := seen[w]; dup {
+			continue
+		}
+		seen[w] = struct{}{}
+		out = append(out, w)
+	}
+	return out
+}
+
+// RoleSimilar reports whether two job titles describe what is plausibly the
+// same underlying role. Uses token-overlap with stopwords stripped and a
+// prefix-both-ways comparison so that abbreviations (pen → penetration)
+// don't break the match.
+//
+// A match requires:
+//   - both titles yield at least one content word (after stopword strip);
+//   - at least 2 content words overlap OR every content word in the shorter
+//     side matches (handles single-word patterns like "Security");
+//   - overlap ratio ≥ 0.5 against the smaller tokenized side.
+//
+// Intended for dedup UX ("we've seen a similar role"), not for hard filtering.
+func RoleSimilar(a, b string) bool {
+	wa := tokenizeRole(a)
+	wb := tokenizeRole(b)
+	if len(wa) == 0 || len(wb) == 0 {
+		return false
+	}
+	overlap := 0
+	for _, x := range wa {
+		for _, y := range wb {
+			if x == y || strings.HasPrefix(y, x) || strings.HasPrefix(x, y) {
+				overlap++
+				break
+			}
+		}
+	}
+	if overlap == 0 {
+		return false
+	}
+	smaller := len(wa)
+	if len(wb) < smaller {
+		smaller = len(wb)
+	}
+	required := 2
+	if smaller < required {
+		required = smaller
+	}
+	if overlap < required {
+		return false
+	}
+	ratio := float64(overlap) / float64(smaller)
+	return ratio >= 0.5
+}