initial public release
This commit is contained in:
@@ -0,0 +1,668 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// maxResponseBytes caps any single upstream response we'll parse.
|
||||
// Guards against OOM from maliciously-large or buggy feeds.
|
||||
const maxResponseBytes = 25 * 1024 * 1024 // 25 MB. Bumped from 5 MB: OpenAI's Ashby board is ~12 MB.
|
||||
|
||||
// DetectAdapter resolves a Company into an API endpoint, or returns ok=false.
|
||||
func DetectAdapter(c Company) (AdapterInfo, bool) {
|
||||
// Explicit Greenhouse API override.
|
||||
if c.API != "" && strings.Contains(c.API, "greenhouse") {
|
||||
return AdapterInfo{Type: AdapterGreenhouse, URL: c.API}, true
|
||||
}
|
||||
// Explicit Workday config block.
|
||||
if c.Workday != nil && c.Workday.Host != "" && c.Workday.Tenant != "" && c.Workday.Site != "" {
|
||||
host := strings.TrimRight(c.Workday.Host, "/")
|
||||
return AdapterInfo{
|
||||
Type: AdapterWorkday,
|
||||
URL: fmt.Sprintf("%s/wday/cxs/%s/%s/jobs", host, c.Workday.Tenant, c.Workday.Site),
|
||||
Host: host,
|
||||
Site: c.Workday.Site,
|
||||
}, true
|
||||
}
|
||||
|
||||
// Any explicit API URL that isn't greenhouse — infer from domain.
|
||||
if c.API != "" {
|
||||
u := c.API
|
||||
switch {
|
||||
case strings.Contains(u, "boards-api.greenhouse.io"):
|
||||
return AdapterInfo{Type: AdapterGreenhouse, URL: u}, true
|
||||
case strings.Contains(u, "api.lever.co"):
|
||||
return AdapterInfo{Type: AdapterLever, URL: u}, true
|
||||
case strings.Contains(u, "api.ashbyhq.com"):
|
||||
return AdapterInfo{Type: AdapterAshby, URL: u}, true
|
||||
case strings.Contains(u, ".bamboohr.com"):
|
||||
return AdapterInfo{Type: AdapterBambooHR, URL: u}, true
|
||||
case strings.Contains(u, ".teamtailor.com"):
|
||||
return AdapterInfo{Type: AdapterTeamtailor, URL: u}, true
|
||||
}
|
||||
}
|
||||
|
||||
url := c.CareersURL
|
||||
if m := reAshby.FindStringSubmatch(url); len(m) == 2 {
|
||||
return AdapterInfo{
|
||||
Type: AdapterAshby,
|
||||
URL: fmt.Sprintf("https://api.ashbyhq.com/posting-api/job-board/%s?includeCompensation=true", m[1]),
|
||||
}, true
|
||||
}
|
||||
if m := reLever.FindStringSubmatch(url); len(m) == 2 {
|
||||
return AdapterInfo{
|
||||
Type: AdapterLever,
|
||||
URL: fmt.Sprintf("https://api.lever.co/v0/postings/%s", m[1]),
|
||||
}, true
|
||||
}
|
||||
if m := reGreenhouse.FindStringSubmatch(url); len(m) == 2 {
|
||||
return AdapterInfo{
|
||||
Type: AdapterGreenhouse,
|
||||
URL: fmt.Sprintf("https://boards-api.greenhouse.io/v1/boards/%s/jobs", m[1]),
|
||||
}, true
|
||||
}
|
||||
if m := reBambooHR.FindStringSubmatch(url); len(m) == 2 {
|
||||
return AdapterInfo{
|
||||
Type: AdapterBambooHR,
|
||||
URL: fmt.Sprintf("https://%s.bamboohr.com/careers/list", m[1]),
|
||||
}, true
|
||||
}
|
||||
if m := reTeamtailor.FindStringSubmatch(url); len(m) == 2 {
|
||||
return AdapterInfo{
|
||||
Type: AdapterTeamtailor,
|
||||
URL: fmt.Sprintf("https://%s.teamtailor.com/jobs.rss", m[1]),
|
||||
}, true
|
||||
}
|
||||
|
||||
// scan_method: linkedin — browser-based search; ScanQuery holds keywords.
|
||||
if c.ScanMethod == "linkedin" {
|
||||
return AdapterInfo{Type: AdapterLinkedIn}, true
|
||||
}
|
||||
|
||||
return AdapterInfo{Type: AdapterUnknown}, false
|
||||
}
|
||||
|
||||
var (
|
||||
reAshby = regexp.MustCompile(`jobs\.ashbyhq\.com/([^/?#]+)`)
|
||||
reLever = regexp.MustCompile(`jobs\.lever\.co/([^/?#]+)`)
|
||||
reGreenhouse = regexp.MustCompile(`job-boards(?:\.eu)?\.greenhouse\.io/([^/?#]+)`)
|
||||
reBambooHR = regexp.MustCompile(`https?://([^./]+)\.bamboohr\.com`)
|
||||
reTeamtailor = regexp.MustCompile(`https?://([^./]+)\.teamtailor\.com`)
|
||||
)
|
||||
|
||||
// Fetch runs the right HTTP dance per adapter and returns normalized jobs.
|
||||
func Fetch(ctx context.Context, client *http.Client, c Company, info AdapterInfo) ([]Job, error) {
|
||||
switch info.Type {
|
||||
case AdapterGreenhouse:
|
||||
return fetchGreenhouse(ctx, client, info.URL, c.Name)
|
||||
case AdapterAshby:
|
||||
return fetchAshby(ctx, client, info.URL, c.Name)
|
||||
case AdapterLever:
|
||||
return fetchLever(ctx, client, info.URL, c.Name)
|
||||
case AdapterWorkday:
|
||||
return fetchWorkday(ctx, client, info, c.Name)
|
||||
case AdapterBambooHR:
|
||||
return fetchBambooHR(ctx, client, info.URL, c.Name)
|
||||
case AdapterTeamtailor:
|
||||
return fetchTeamtailor(ctx, client, info.URL, c.Name)
|
||||
case AdapterRemotive:
|
||||
return fetchRemotive(ctx, client, info.URL)
|
||||
case AdapterRemoteOK:
|
||||
return fetchRemoteOK(ctx, client, info.URL, info.UserAgent)
|
||||
case AdapterUSAJobs:
|
||||
return fetchUSAJobs(ctx, client, info)
|
||||
case AdapterRSS:
|
||||
return fetchRSS(ctx, client, info.URL, info.DefaultCompany)
|
||||
case AdapterLinkedIn:
|
||||
return fetchLinkedIn(ctx, c.ScanQuery, "", c.Name)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported adapter type: %s", info.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// DetectAggregator resolves an Aggregator into a scan-ready AdapterInfo.
|
||||
// Returns ok=false for unknown or misconfigured aggregator types.
|
||||
func DetectAggregator(a Aggregator) (AdapterInfo, bool) {
|
||||
limit := a.Limit
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
switch strings.ToLower(a.Type) {
|
||||
case "remotive":
|
||||
q := url.QueryEscape(a.Query)
|
||||
return AdapterInfo{
|
||||
Type: AdapterRemotive,
|
||||
URL: fmt.Sprintf("https://remotive.com/api/remote-jobs?search=%s&limit=%d", q, limit),
|
||||
}, true
|
||||
case "remoteok":
|
||||
tag := a.Tag
|
||||
if tag == "" {
|
||||
tag = "security"
|
||||
}
|
||||
return AdapterInfo{
|
||||
Type: AdapterRemoteOK,
|
||||
URL: fmt.Sprintf("https://remoteok.io/api?tags=%s", url.QueryEscape(tag)),
|
||||
UserAgent: a.UserAgent,
|
||||
}, true
|
||||
case "usajobs":
|
||||
apiKey := a.APIKey
|
||||
if apiKey == "" {
|
||||
apiKey = os.Getenv("USAJOBS_API_KEY")
|
||||
}
|
||||
ua := a.UserAgent
|
||||
if ua == "" {
|
||||
ua = os.Getenv("USAJOBS_USER_AGENT")
|
||||
}
|
||||
return AdapterInfo{
|
||||
Type: AdapterUSAJobs,
|
||||
URL: fmt.Sprintf("https://data.usajobs.gov/api/search?Keyword=%s&ResultsPerPage=%d", url.QueryEscape(a.Query), limit),
|
||||
APIKey: apiKey,
|
||||
UserAgent: ua,
|
||||
}, true
|
||||
case "rss":
|
||||
if a.URL == "" {
|
||||
return AdapterInfo{Type: AdapterUnknown}, false
|
||||
}
|
||||
return AdapterInfo{
|
||||
Type: AdapterRSS,
|
||||
URL: a.URL,
|
||||
DefaultCompany: a.Company,
|
||||
}, true
|
||||
}
|
||||
return AdapterInfo{Type: AdapterUnknown}, false
|
||||
}
|
||||
|
||||
func getJSON(ctx context.Context, client *http.Client, url string, headers map[string]string, out interface{}) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
return json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(out)
|
||||
}
|
||||
|
||||
// Greenhouse: { jobs: [ { title, absolute_url, location:{name} } ] }
|
||||
func fetchGreenhouse(ctx context.Context, client *http.Client, url, company string) ([]Job, error) {
|
||||
var payload struct {
|
||||
Jobs []struct {
|
||||
Title string `json:"title"`
|
||||
AbsoluteURL string `json:"absolute_url"`
|
||||
Location struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"location"`
|
||||
} `json:"jobs"`
|
||||
}
|
||||
if err := getJSON(ctx, client, url, nil, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Job, 0, len(payload.Jobs))
|
||||
for _, j := range payload.Jobs {
|
||||
out = append(out, Job{
|
||||
Title: j.Title,
|
||||
URL: j.AbsoluteURL,
|
||||
Company: company,
|
||||
Location: j.Location.Name,
|
||||
Source: "greenhouse",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Ashby (posting-api): { jobs: [ { title, jobUrl, location } ] }
|
||||
func fetchAshby(ctx context.Context, client *http.Client, url, company string) ([]Job, error) {
|
||||
var payload struct {
|
||||
Jobs []struct {
|
||||
Title string `json:"title"`
|
||||
JobURL string `json:"jobUrl"`
|
||||
Location string `json:"location"`
|
||||
} `json:"jobs"`
|
||||
}
|
||||
if err := getJSON(ctx, client, url, nil, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Job, 0, len(payload.Jobs))
|
||||
for _, j := range payload.Jobs {
|
||||
out = append(out, Job{
|
||||
Title: j.Title,
|
||||
URL: j.JobURL,
|
||||
Company: company,
|
||||
Location: j.Location,
|
||||
Source: "ashby",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Lever: [ { text, hostedUrl, categories:{location} } ]
|
||||
func fetchLever(ctx context.Context, client *http.Client, url, company string) ([]Job, error) {
|
||||
var payload []struct {
|
||||
Text string `json:"text"`
|
||||
HostedURL string `json:"hostedUrl"`
|
||||
Categories struct {
|
||||
Location string `json:"location"`
|
||||
} `json:"categories"`
|
||||
}
|
||||
if err := getJSON(ctx, client, url, nil, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Job, 0, len(payload))
|
||||
for _, j := range payload {
|
||||
out = append(out, Job{
|
||||
Title: j.Text,
|
||||
URL: j.HostedURL,
|
||||
Company: company,
|
||||
Location: j.Categories.Location,
|
||||
Source: "lever",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Workday: POST with pagination.
|
||||
func fetchWorkday(ctx context.Context, client *http.Client, info AdapterInfo, company string) ([]Job, error) {
|
||||
const limit = 20
|
||||
const safetyCap = 1000
|
||||
var all []Job
|
||||
offset := 0
|
||||
for offset < safetyCap {
|
||||
body := fmt.Sprintf(`{"appliedFacets":{},"limit":%d,"offset":%d,"searchText":""}`, limit, offset)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, info.URL, bytes.NewBufferString(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
var payload struct {
|
||||
JobPostings []struct {
|
||||
Title string `json:"title"`
|
||||
ExternalPath string `json:"externalPath"`
|
||||
LocationsText string `json:"locationsText"`
|
||||
} `json:"jobPostings"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&payload); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, err
|
||||
}
|
||||
resp.Body.Close()
|
||||
for _, j := range payload.JobPostings {
|
||||
// Workday sometimes returns odd externalPath values. Reject any
|
||||
// that isn't a clean absolute segment (no traversal, no scheme).
|
||||
if !strings.HasPrefix(j.ExternalPath, "/") || strings.Contains(j.ExternalPath, "..") {
|
||||
continue
|
||||
}
|
||||
url := fmt.Sprintf("%s/en-US/%s%s", info.Host, info.Site, j.ExternalPath)
|
||||
all = append(all, Job{
|
||||
Title: j.Title,
|
||||
URL: url,
|
||||
Company: company,
|
||||
Location: j.LocationsText,
|
||||
Source: "workday",
|
||||
})
|
||||
}
|
||||
if len(payload.JobPostings) < limit {
|
||||
break
|
||||
}
|
||||
offset += limit
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// BambooHR list: { result: [ { id, jobOpeningName, jobOpeningShareUrl|null, locationCity } ] }
|
||||
func fetchBambooHR(ctx context.Context, client *http.Client, url, company string) ([]Job, error) {
|
||||
var payload struct {
|
||||
Result []struct {
|
||||
ID int `json:"id"`
|
||||
JobOpeningName string `json:"jobOpeningName"`
|
||||
JobOpeningShareURL string `json:"jobOpeningShareUrl"`
|
||||
LocationCity string `json:"locationCity"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := getJSON(ctx, client, url, map[string]string{"Accept": "application/json"}, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
host := url // list endpoint lives on the same subdomain
|
||||
if i := strings.Index(host, "/careers"); i > 0 {
|
||||
host = host[:i]
|
||||
}
|
||||
out := make([]Job, 0, len(payload.Result))
|
||||
for _, j := range payload.Result {
|
||||
link := j.JobOpeningShareURL
|
||||
if link == "" {
|
||||
link = fmt.Sprintf("%s/careers/%d/detail", host, j.ID)
|
||||
}
|
||||
out = append(out, Job{
|
||||
Title: j.JobOpeningName,
|
||||
URL: link,
|
||||
Company: company,
|
||||
Location: j.LocationCity,
|
||||
Source: "bamboohr",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Teamtailor RSS: simple XML feed.
|
||||
func fetchTeamtailor(ctx context.Context, client *http.Client, url, company string) ([]Job, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/rss+xml, application/xml")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Minimal RSS 2.0 parse. Disable entity expansion to avoid XXE.
|
||||
return parseRSSItems(body, company)
|
||||
}
|
||||
|
||||
// Minimal RSS <item><title>…</title><link>…</link></item> extractor with XXE off.
|
||||
func parseRSSItems(body []byte, company string) ([]Job, error) {
|
||||
type item struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
}
|
||||
type rss struct {
|
||||
Channel struct {
|
||||
Items []item `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
dec := newSafeXMLDecoder(bytes.NewReader(body))
|
||||
var feed rss
|
||||
if err := dec.Decode(&feed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Job, 0, len(feed.Channel.Items))
|
||||
for _, it := range feed.Channel.Items {
|
||||
out = append(out, Job{
|
||||
Title: strings.TrimSpace(it.Title),
|
||||
URL: strings.TrimSpace(it.Link),
|
||||
Company: company,
|
||||
Source: "teamtailor",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Remotive: { jobs: [ { title, url, company_name, candidate_required_location } ] }
|
||||
func fetchRemotive(ctx context.Context, client *http.Client, url string) ([]Job, error) {
|
||||
var payload struct {
|
||||
Jobs []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
CompanyName string `json:"company_name"`
|
||||
Location string `json:"candidate_required_location"`
|
||||
JobType string `json:"job_type"`
|
||||
} `json:"jobs"`
|
||||
}
|
||||
if err := getJSON(ctx, client, url, nil, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Job, 0, len(payload.Jobs))
|
||||
for _, j := range payload.Jobs {
|
||||
loc := j.Location
|
||||
if loc == "" {
|
||||
loc = j.JobType
|
||||
}
|
||||
out = append(out, Job{
|
||||
Title: j.Title,
|
||||
URL: j.URL,
|
||||
Company: j.CompanyName,
|
||||
Location: loc,
|
||||
Source: "remotive",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// safeHeaderValue rejects values containing CRLF or NUL that could split HTTP
|
||||
// headers. Go's net/http panics on these, but we prefer a clear error.
|
||||
func safeHeaderValue(v string) error {
|
||||
if strings.ContainsAny(v, "\r\n\x00") {
|
||||
return fmt.Errorf("header value contains invalid control characters")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoteOK: array where first element is legal header — skip entries without
|
||||
// (id AND position|title). Bare fetch is rejected with 403 so we send a UA.
|
||||
func fetchRemoteOK(ctx context.Context, client *http.Client, url, userAgent string) ([]Job, error) {
|
||||
ua := userAgent
|
||||
if ua == "" {
|
||||
ua = "apex-scanner/1.0 (+https://github.com/cobr-ai/apex)"
|
||||
}
|
||||
if err := safeHeaderValue(ua); err != nil {
|
||||
return nil, fmt.Errorf("remoteok: user-agent: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", ua)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
var entries []struct {
|
||||
ID interface{} `json:"id"` // string or int depending on the posting
|
||||
Position string `json:"position"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Slug string `json:"slug"`
|
||||
Company string `json:"company"`
|
||||
Location string `json:"location"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&entries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Job, 0, len(entries))
|
||||
for _, j := range entries {
|
||||
if j.ID == nil {
|
||||
continue
|
||||
}
|
||||
title := j.Position
|
||||
if title == "" {
|
||||
title = j.Title
|
||||
}
|
||||
if title == "" {
|
||||
continue
|
||||
}
|
||||
jobURL := j.URL
|
||||
if jobURL == "" && j.Slug != "" {
|
||||
jobURL = "https://remoteok.io/remote-jobs/" + j.Slug
|
||||
}
|
||||
loc := j.Location
|
||||
if loc == "" {
|
||||
loc = "Remote"
|
||||
}
|
||||
out = append(out, Job{
|
||||
Title: title,
|
||||
URL: jobURL,
|
||||
Company: j.Company,
|
||||
Location: loc,
|
||||
Source: "remoteok",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// USAJobs: paginated SearchResult.SearchResultItems[].MatchedObjectDescriptor.
|
||||
// Requires both User-Agent (registered email) and Authorization-Key headers.
|
||||
func fetchUSAJobs(ctx context.Context, client *http.Client, info AdapterInfo) ([]Job, error) {
|
||||
if info.APIKey == "" {
|
||||
return nil, fmt.Errorf("usajobs: api key missing (set aggregator.api_key or USAJOBS_API_KEY env)")
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, info.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ua := info.UserAgent
|
||||
if ua == "" {
|
||||
ua = "apex-scanner/1.0"
|
||||
}
|
||||
if err := safeHeaderValue(ua); err != nil {
|
||||
return nil, fmt.Errorf("usajobs: user-agent: %w", err)
|
||||
}
|
||||
if err := safeHeaderValue(info.APIKey); err != nil {
|
||||
return nil, fmt.Errorf("usajobs: api-key: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", ua)
|
||||
req.Header.Set("Authorization-Key", info.APIKey)
|
||||
req.Header.Set("Host", "data.usajobs.gov")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
var payload struct {
|
||||
SearchResult struct {
|
||||
SearchResultItems []struct {
|
||||
MatchedObjectDescriptor struct {
|
||||
PositionTitle string `json:"PositionTitle"`
|
||||
PositionURI string `json:"PositionURI"`
|
||||
ApplyURI []string `json:"ApplyURI"`
|
||||
OrganizationName string `json:"OrganizationName"`
|
||||
PositionLocationDisplay string `json:"PositionLocationDisplay"`
|
||||
} `json:"MatchedObjectDescriptor"`
|
||||
} `json:"SearchResultItems"`
|
||||
} `json:"SearchResult"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBytes)).Decode(&payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := payload.SearchResult.SearchResultItems
|
||||
out := make([]Job, 0, len(items))
|
||||
for _, it := range items {
|
||||
md := it.MatchedObjectDescriptor
|
||||
jobURL := ""
|
||||
if len(md.ApplyURI) > 0 {
|
||||
jobURL = md.ApplyURI[0]
|
||||
}
|
||||
if jobURL == "" {
|
||||
jobURL = md.PositionURI
|
||||
}
|
||||
company := md.OrganizationName
|
||||
if company == "" {
|
||||
company = "US Federal"
|
||||
}
|
||||
out = append(out, Job{
|
||||
Title: md.PositionTitle,
|
||||
URL: jobURL,
|
||||
Company: company,
|
||||
Location: md.PositionLocationDisplay,
|
||||
Source: "usajobs",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Generic RSS aggregator: mirrors parseRSSItems but tags Source=rss and
|
||||
// splices defaultCompany for feeds that don't embed a company per item.
|
||||
func fetchRSS(ctx context.Context, client *http.Client, feedURL, defaultCompany string) ([]Job, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/rss+xml, application/xml, text/xml")
|
||||
req.Header.Set("User-Agent", "apex-scanner/1.0")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type item struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
GUID string `xml:"guid"`
|
||||
Description string `xml:"description"`
|
||||
}
|
||||
type rss struct {
|
||||
Channel struct {
|
||||
Items []item `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
dec := newSafeXMLDecoder(bytes.NewReader(body))
|
||||
var feed rss
|
||||
if err := dec.Decode(&feed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Job, 0, len(feed.Channel.Items))
|
||||
for _, it := range feed.Channel.Items {
|
||||
link := strings.TrimSpace(it.Link)
|
||||
if link == "" {
|
||||
link = strings.TrimSpace(it.GUID)
|
||||
}
|
||||
title := strings.TrimSpace(it.Title)
|
||||
if title == "" || link == "" {
|
||||
continue
|
||||
}
|
||||
desc := strings.TrimSpace(it.Description)
|
||||
if len(desc) > 80 {
|
||||
desc = desc[:80]
|
||||
}
|
||||
out = append(out, Job{
|
||||
Title: title,
|
||||
URL: link,
|
||||
Company: defaultCompany,
|
||||
Location: desc,
|
||||
Source: "rss",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HTTPClient returns a timeout-limited client with safe defaults.
|
||||
func HTTPClient(timeout time.Duration) *http.Client {
|
||||
return &http.Client{Timeout: timeout}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
# Apex Tier-1 API Scan Adapters
|
||||
# Targets: Red Team, Penetration Testing, OT/ICS Pentest, AI Security, Adversary Emulation
|
||||
# Strategy: zero-token direct JSON API hits
|
||||
|
||||
title_filter:
|
||||
positive:
|
||||
- "Red Team"
|
||||
- "Penetration Test"
|
||||
- "Pen Test"
|
||||
- "Pentest"
|
||||
- "Offensive Security"
|
||||
- "Adversary Emulation"
|
||||
- "Adversary Simulation"
|
||||
- "Purple Team"
|
||||
- "AI Security"
|
||||
- "AI Red Team"
|
||||
- "LLM Security"
|
||||
- "AI/ML Security"
|
||||
- "ML Security"
|
||||
- "GenAI Security"
|
||||
- "AI Adversarial"
|
||||
- "Prompt Security"
|
||||
- "OT Security"
|
||||
- "OT Penetration"
|
||||
- "ICS Security"
|
||||
- "ICS Penetration"
|
||||
- "SCADA"
|
||||
- "Industrial Control"
|
||||
- "Critical Infrastructure"
|
||||
- "OT/ICS"
|
||||
- "OT Pentest"
|
||||
- "Cloud Security"
|
||||
- "Cloud Penetration"
|
||||
- "AWS Security"
|
||||
- "Security Engineer"
|
||||
- "Security Consultant"
|
||||
- "Security Researcher"
|
||||
- "Vulnerability Research"
|
||||
- "Exploit Development"
|
||||
- "Threat Emulation"
|
||||
- "Threat Hunter"
|
||||
- "Application Security"
|
||||
- "Product Security"
|
||||
- "AppSec"
|
||||
- "Principal Security"
|
||||
- "Staff Security"
|
||||
- "Ethical Hacker"
|
||||
|
||||
negative:
|
||||
- "SOC Analyst I"
|
||||
- "SOC Analyst Tier 1"
|
||||
- "Tier 1"
|
||||
- "T1 Analyst"
|
||||
- "GRC"
|
||||
- "Compliance Analyst"
|
||||
- "Auditor"
|
||||
- "Privacy Analyst"
|
||||
- "Risk Analyst"
|
||||
- "Governance"
|
||||
- "Sales"
|
||||
- "Account Executive"
|
||||
- "BDR"
|
||||
- "SDR"
|
||||
- "Marketing"
|
||||
- "Recruiter"
|
||||
- "Customer Success"
|
||||
- "Renewals"
|
||||
- "Intern"
|
||||
- "Internship"
|
||||
- "Co-op"
|
||||
- "Apprentice"
|
||||
- "Junior"
|
||||
- "Entry Level"
|
||||
- "Entry-Level"
|
||||
- "Graduate Program"
|
||||
- ".NET Developer"
|
||||
- "Java Developer"
|
||||
- "iOS Developer"
|
||||
- "Android Developer"
|
||||
- "PHP Developer"
|
||||
- "Ruby Developer"
|
||||
- "COBOL"
|
||||
- "Mainframe"
|
||||
- "SAP "
|
||||
- "Salesforce Admin"
|
||||
- "Web3"
|
||||
- "Blockchain"
|
||||
- "Crypto"
|
||||
|
||||
seniority_boost:
|
||||
- "Senior"
|
||||
- "Staff"
|
||||
- "Principal"
|
||||
- "Lead"
|
||||
- "Associate Principal"
|
||||
- "Head"
|
||||
- "Director"
|
||||
|
||||
tracked_companies:
|
||||
# === OT / ICS Security ===
|
||||
- name: Dragos
|
||||
careers_url: https://job-boards.greenhouse.io/dragos
|
||||
api: https://boards-api.greenhouse.io/v1/boards/dragos/jobs
|
||||
enabled: true
|
||||
notes: "OT/ICS security market leader — dream company"
|
||||
|
||||
- name: Nozomi Networks
|
||||
careers_url: https://job-boards.greenhouse.io/nozominetworks
|
||||
api: https://boards-api.greenhouse.io/v1/boards/nozominetworks/jobs
|
||||
enabled: true
|
||||
notes: "OT/IoT visibility & security"
|
||||
|
||||
- name: Shift5
|
||||
careers_url: https://job-boards.greenhouse.io/shift5
|
||||
api: https://boards-api.greenhouse.io/v1/boards/shift5/jobs
|
||||
enabled: true
|
||||
notes: "OT security for transportation/defense"
|
||||
|
||||
# === Pure-Play Red Team / Offensive ===
|
||||
- name: SpecterOps
|
||||
careers_url: https://job-boards.greenhouse.io/specterops
|
||||
api: https://boards-api.greenhouse.io/v1/boards/specterops/jobs
|
||||
enabled: true
|
||||
notes: "BloodHound creators, elite adversary emulation"
|
||||
|
||||
- name: Bishop Fox
|
||||
careers_url: https://job-boards.greenhouse.io/bishopfox
|
||||
api: https://boards-api.greenhouse.io/v1/boards/bishopfox/jobs
|
||||
enabled: true
|
||||
|
||||
- name: Praetorian
|
||||
careers_url: https://job-boards.greenhouse.io/praetorian
|
||||
api: https://boards-api.greenhouse.io/v1/boards/praetorian/jobs
|
||||
enabled: true
|
||||
notes: "Offensive security + managed services"
|
||||
|
||||
# === Major Cyber Vendors ===
|
||||
- name: Rapid7
|
||||
careers_url: https://careers.rapid7.com/
|
||||
enabled: false
|
||||
notes: "Metasploit stewards, consulting services. Uses Clinch (PageUp) ATS — no public API adapter yet."
|
||||
|
||||
- name: Arctic Wolf
|
||||
careers_url: https://arcticwolf.wd1.myworkdayjobs.com/External
|
||||
enabled: true
|
||||
workday:
|
||||
host: https://arcticwolf.wd1.myworkdayjobs.com
|
||||
tenant: arcticwolf
|
||||
site: External
|
||||
|
||||
- name: Recorded Future
|
||||
careers_url: https://job-boards.greenhouse.io/recordedfuture
|
||||
api: https://boards-api.greenhouse.io/v1/boards/recordedfuture/jobs
|
||||
enabled: true
|
||||
|
||||
- name: CrowdStrike
|
||||
careers_url: https://crowdstrike.wd5.myworkdayjobs.com/crowdstrikecareers
|
||||
enabled: true
|
||||
workday:
|
||||
host: https://crowdstrike.wd5.myworkdayjobs.com
|
||||
tenant: crowdstrike
|
||||
site: crowdstrikecareers
|
||||
notes: "Dream company — Falcon, Services red team (Workday)"
|
||||
|
||||
- name: Sophos
|
||||
careers_url: https://jobs.lever.co/sophos
|
||||
enabled: true
|
||||
notes: "Lever-hosted"
|
||||
|
||||
- name: Tenable
|
||||
careers_url: https://www.tenable.com/careers
|
||||
enabled: true
|
||||
|
||||
# === Crowdsourced / Bug Bounty ===
|
||||
- name: HackerOne
|
||||
careers_url: https://jobs.ashbyhq.com/hackerone
|
||||
api: https://api.ashbyhq.com/posting-api/job-board/hackerone?includeCompensation=true
|
||||
enabled: true
|
||||
|
||||
- name: Bugcrowd
|
||||
careers_url: https://job-boards.greenhouse.io/bugcrowd
|
||||
api: https://boards-api.greenhouse.io/v1/boards/bugcrowd/jobs
|
||||
enabled: true
|
||||
|
||||
- name: Synack
|
||||
careers_url: https://job-boards.greenhouse.io/synack
|
||||
api: https://boards-api.greenhouse.io/v1/boards/synack/jobs
|
||||
enabled: true
|
||||
|
||||
- name: Horizon3.ai
|
||||
careers_url: https://jobs.ashbyhq.com/horizon3ai
|
||||
enabled: true
|
||||
notes: "Ashby-hosted"
|
||||
|
||||
# === Federal / Cleared Contractors ===
|
||||
- name: Booz Allen Hamilton
|
||||
careers_url: https://bah.wd1.myworkdayjobs.com/BAH_Jobs
|
||||
enabled: true
|
||||
workday:
|
||||
host: https://bah.wd1.myworkdayjobs.com
|
||||
tenant: bah
|
||||
site: BAH_Jobs
|
||||
notes: "SECRET clearance asset (Workday)"
|
||||
|
||||
- name: Leidos
|
||||
careers_url: https://leidos.wd5.myworkdayjobs.com/External
|
||||
enabled: true
|
||||
workday:
|
||||
host: https://leidos.wd5.myworkdayjobs.com
|
||||
tenant: leidos
|
||||
site: External
|
||||
notes: "SECRET clearance asset (Workday)"
|
||||
|
||||
- name: Two Six Technologies
|
||||
careers_url: https://job-boards.greenhouse.io/twosixtechnologies
|
||||
api: https://boards-api.greenhouse.io/v1/boards/twosixtechnologies/jobs
|
||||
enabled: true
|
||||
|
||||
# === AI Labs ===
|
||||
- name: Anthropic
|
||||
careers_url: https://job-boards.greenhouse.io/anthropic
|
||||
api: https://boards-api.greenhouse.io/v1/boards/anthropic/jobs
|
||||
enabled: true
|
||||
|
||||
- name: OpenAI
|
||||
careers_url: https://jobs.ashbyhq.com/openai
|
||||
enabled: true
|
||||
notes: "Ashby-hosted job board (653 live postings)"
|
||||
|
||||
- name: HiddenLayer
|
||||
careers_url: https://job-boards.greenhouse.io/hiddenlayer
|
||||
api: https://boards-api.greenhouse.io/v1/boards/hiddenlayer/jobs
|
||||
enabled: true
|
||||
|
||||
aggregators:
|
||||
# Remotive — remote-first jobs, free public API, search-by-keyword
|
||||
- name: Remotive — Red Team
|
||||
type: remotive
|
||||
query: "red team"
|
||||
limit: 50
|
||||
enabled: true
|
||||
|
||||
- name: Remotive — Penetration Tester
|
||||
type: remotive
|
||||
query: "penetration tester"
|
||||
limit: 50
|
||||
enabled: true
|
||||
|
||||
- name: Remotive — AI Security
|
||||
type: remotive
|
||||
query: "AI security"
|
||||
limit: 50
|
||||
enabled: true
|
||||
|
||||
- name: Remotive — Offensive Security
|
||||
type: remotive
|
||||
query: "offensive security"
|
||||
limit: 50
|
||||
enabled: true
|
||||
|
||||
# RemoteOK — remote tech, free JSON API, tag-based
|
||||
- name: RemoteOK — Security
|
||||
type: remoteok
|
||||
tag: security
|
||||
enabled: true
|
||||
|
||||
# USAJobs — federal roles, free JSON API, requires registered API key.
|
||||
# Creds live in Infisical (apex folder): USAJOBS_API_KEY, USAJOBS_USER_AGENT.
|
||||
# Pull with: eval $(~/.local/bin/creds env apex) before running scan.
|
||||
# NOTE: USAJobs keyword search uses substring AND matching. Keep queries
|
||||
# single-keyword and let the global title_filter downselect.
|
||||
- name: USAJobs — penetration
|
||||
type: usajobs
|
||||
query: "penetration"
|
||||
limit: 100
|
||||
enabled: true
|
||||
|
||||
- name: USAJobs — red team
|
||||
type: usajobs
|
||||
query: "red team"
|
||||
limit: 100
|
||||
enabled: true
|
||||
|
||||
- name: USAJobs — offensive security
|
||||
type: usajobs
|
||||
query: "offensive security"
|
||||
limit: 100
|
||||
enabled: true
|
||||
|
||||
- name: USAJobs — vulnerability
|
||||
type: usajobs
|
||||
query: "vulnerability"
|
||||
limit: 100
|
||||
enabled: true
|
||||
|
||||
- name: USAJobs — adversary
|
||||
type: usajobs
|
||||
query: "adversary"
|
||||
limit: 100
|
||||
enabled: true
|
||||
|
||||
- name: USAJobs — SCADA / ICS
|
||||
type: usajobs
|
||||
query: "SCADA"
|
||||
limit: 100
|
||||
enabled: true
|
||||
|
||||
- name: USAJobs — industrial control
|
||||
type: usajobs
|
||||
query: "industrial control"
|
||||
limit: 100
|
||||
enabled: true
|
||||
|
||||
- name: USAJobs — exploit research
|
||||
type: usajobs
|
||||
query: "exploit"
|
||||
limit: 100
|
||||
enabled: true
|
||||
|
||||
# Generic RSS — use this for company career pages that publish an RSS feed
|
||||
# but no JSON API. Only 20 most recent items per poll is common.
|
||||
- name: Deloitte — RSS (US External Careers)
|
||||
type: rss
|
||||
url: https://apply.deloitte.com/en_US/careers/Home/feed/
|
||||
company: Deloitte
|
||||
enabled: true
|
||||
notes: "Dream company. Custom Taleo-like ATS, generic RSS feed (20 most recent jobs)."
|
||||
@@ -0,0 +1,235 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchGreenhouse_ParsesFixture(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
payload := map[string]any{
|
||||
"jobs": []any{
|
||||
map[string]any{
|
||||
"id": 1,
|
||||
"title": "Security Engineer",
|
||||
"absolute_url": "https://dragos.greenhouse.io/jobs/1",
|
||||
"location": map[string]any{
|
||||
"name": "San Francisco, CA",
|
||||
},
|
||||
},
|
||||
map[string]any{
|
||||
"id": 2,
|
||||
"title": "Red Team Operator",
|
||||
"absolute_url": "https://dragos.greenhouse.io/jobs/2",
|
||||
"location": map[string]any{
|
||||
"name": "Remote",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
jobs, err := fetchGreenhouse(context.Background(), srv.Client(), srv.URL, "Dragos")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 2 {
|
||||
t.Fatalf("jobs = %d, want 2", len(jobs))
|
||||
}
|
||||
if jobs[0].Title != "Security Engineer" {
|
||||
t.Errorf("job[0].Title = %q, want Security Engineer", jobs[0].Title)
|
||||
}
|
||||
if jobs[0].URL != "https://dragos.greenhouse.io/jobs/1" {
|
||||
t.Errorf("job[0].URL = %q", jobs[0].URL)
|
||||
}
|
||||
if jobs[0].Company != "Dragos" {
|
||||
t.Errorf("job[0].Company = %q, want Dragos", jobs[0].Company)
|
||||
}
|
||||
if jobs[0].Location != "San Francisco, CA" {
|
||||
t.Errorf("job[0].Location = %q", jobs[0].Location)
|
||||
}
|
||||
if jobs[0].Source != "greenhouse" {
|
||||
t.Errorf("job[0].Source = %q, want greenhouse", jobs[0].Source)
|
||||
}
|
||||
if jobs[1].Title != "Red Team Operator" {
|
||||
t.Errorf("job[1].Title = %q", jobs[1].Title)
|
||||
}
|
||||
if jobs[1].Location != "Remote" {
|
||||
t.Errorf("job[1].Location = %q", jobs[1].Location)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchGreenhouse_EmptyJobs(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
payload := map[string]any{"jobs": []any{}}
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
jobs, err := fetchGreenhouse(context.Background(), srv.Client(), srv.URL, "Acme")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 0 {
|
||||
t.Fatalf("jobs = %d, want 0", len(jobs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAshby_ParsesFixture(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
payload := map[string]any{
|
||||
"jobs": []any{
|
||||
map[string]any{
|
||||
"id": "job1",
|
||||
"title": "Pentester",
|
||||
"jobUrl": "https://ashby.example.com/jobs/job1",
|
||||
"location": "New York, NY",
|
||||
},
|
||||
map[string]any{
|
||||
"id": "job2",
|
||||
"title": "AI Security Researcher",
|
||||
"jobUrl": "https://ashby.example.com/jobs/job2",
|
||||
"location": "Remote",
|
||||
},
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
jobs, err := fetchAshby(context.Background(), srv.Client(), srv.URL, "Acme")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 2 {
|
||||
t.Fatalf("jobs = %d, want 2", len(jobs))
|
||||
}
|
||||
if jobs[0].Title != "Pentester" {
|
||||
t.Errorf("job[0].Title = %q, want Pentester", jobs[0].Title)
|
||||
}
|
||||
if jobs[0].URL != "https://ashby.example.com/jobs/job1" {
|
||||
t.Errorf("job[0].URL = %q", jobs[0].URL)
|
||||
}
|
||||
if jobs[0].Company != "Acme" {
|
||||
t.Errorf("job[0].Company = %q, want Acme", jobs[0].Company)
|
||||
}
|
||||
if jobs[0].Location != "New York, NY" {
|
||||
t.Errorf("job[0].Location = %q", jobs[0].Location)
|
||||
}
|
||||
if jobs[0].Source != "ashby" {
|
||||
t.Errorf("job[0].Source = %q, want ashby", jobs[0].Source)
|
||||
}
|
||||
if jobs[1].Title != "AI Security Researcher" {
|
||||
t.Errorf("job[1].Title = %q", jobs[1].Title)
|
||||
}
|
||||
if jobs[1].Location != "Remote" {
|
||||
t.Errorf("job[1].Location = %q", jobs[1].Location)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAshby_EmptyJobs(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
payload := map[string]any{"jobs": []any{}}
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
jobs, err := fetchAshby(context.Background(), srv.Client(), srv.URL, "Acme")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 0 {
|
||||
t.Fatalf("jobs = %d, want 0", len(jobs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchLever_ParsesFixture(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
payload := []any{
|
||||
map[string]any{
|
||||
"id": "posting1",
|
||||
"text": "Red Team Lead",
|
||||
"hostedUrl": "https://lever.example.com/postings/posting1",
|
||||
"categories": map[string]any{
|
||||
"location": "San Francisco, CA",
|
||||
},
|
||||
},
|
||||
map[string]any{
|
||||
"id": "posting2",
|
||||
"text": "Offensive Security Engineer",
|
||||
"hostedUrl": "https://lever.example.com/postings/posting2",
|
||||
"categories": map[string]any{
|
||||
"location": "Los Angeles, CA",
|
||||
},
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
jobs, err := fetchLever(context.Background(), srv.Client(), srv.URL, "Bishop Fox")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 2 {
|
||||
t.Fatalf("jobs = %d, want 2", len(jobs))
|
||||
}
|
||||
if jobs[0].Title != "Red Team Lead" {
|
||||
t.Errorf("job[0].Title = %q, want Red Team Lead", jobs[0].Title)
|
||||
}
|
||||
if jobs[0].URL != "https://lever.example.com/postings/posting1" {
|
||||
t.Errorf("job[0].URL = %q", jobs[0].URL)
|
||||
}
|
||||
if jobs[0].Company != "Bishop Fox" {
|
||||
t.Errorf("job[0].Company = %q, want Bishop Fox", jobs[0].Company)
|
||||
}
|
||||
if jobs[0].Location != "San Francisco, CA" {
|
||||
t.Errorf("job[0].Location = %q", jobs[0].Location)
|
||||
}
|
||||
if jobs[0].Source != "lever" {
|
||||
t.Errorf("job[0].Source = %q, want lever", jobs[0].Source)
|
||||
}
|
||||
if jobs[1].Title != "Offensive Security Engineer" {
|
||||
t.Errorf("job[1].Title = %q", jobs[1].Title)
|
||||
}
|
||||
if jobs[1].Location != "Los Angeles, CA" {
|
||||
t.Errorf("job[1].Location = %q", jobs[1].Location)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchLever_EmptyPostings(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
payload := []any{}
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
jobs, err := fetchLever(context.Background(), srv.Client(), srv.URL, "Beta Corp")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 0 {
|
||||
t.Fatalf("jobs = %d, want 0", len(jobs))
|
||||
}
|
||||
}
|
||||
|
||||
// Sanity check: company API adapters dispatch correctly through the Fetch function
|
||||
func TestFetch_DispatachesCompanyAdapters(t *testing.T) {
|
||||
cases := []AdapterType{AdapterGreenhouse, AdapterAshby, AdapterLever}
|
||||
for _, typ := range cases {
|
||||
_, err := Fetch(context.Background(), &http.Client{},
|
||||
Company{Name: "test"}, AdapterInfo{Type: typ, URL: "http://127.0.0.1:1"})
|
||||
// Expect a real network error (connection refused), NOT "unsupported adapter type".
|
||||
if err == nil {
|
||||
t.Errorf("%s: expected connection error", typ)
|
||||
}
|
||||
if err.Error() == "unsupported adapter type: "+string(typ) {
|
||||
t.Errorf("%s: dispatch not wired", typ)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Round-trip fixture test: spin up a local server that returns each provider's
|
||||
// canned payload, then verify the adapter fetches + normalizes correctly.
|
||||
|
||||
func TestDetectAggregator_EachType(t *testing.T) {
|
||||
cases := []struct {
|
||||
agg Aggregator
|
||||
want AdapterType
|
||||
}{
|
||||
{Aggregator{Type: "remotive", Query: "security"}, AdapterRemotive},
|
||||
{Aggregator{Type: "remoteok", Tag: "security"}, AdapterRemoteOK},
|
||||
{Aggregator{Type: "usajobs", Query: "cyber", APIKey: "k"}, AdapterUSAJobs},
|
||||
{Aggregator{Type: "rss", URL: "https://example.com/feed.xml"}, AdapterRSS},
|
||||
}
|
||||
for _, c := range cases {
|
||||
info, ok := DetectAggregator(c.agg)
|
||||
if !ok {
|
||||
t.Errorf("%s: expected ok=true", c.agg.Type)
|
||||
continue
|
||||
}
|
||||
if info.Type != c.want {
|
||||
t.Errorf("%s: type = %q, want %q", c.agg.Type, info.Type, c.want)
|
||||
}
|
||||
if info.URL == "" {
|
||||
t.Errorf("%s: URL empty", c.agg.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectAggregator_RSSRequiresURL(t *testing.T) {
|
||||
if _, ok := DetectAggregator(Aggregator{Type: "rss"}); ok {
|
||||
t.Fatal("rss without URL must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectAggregator_USAJobsEnvKey(t *testing.T) {
|
||||
t.Setenv("USAJOBS_API_KEY", "env-key")
|
||||
info, ok := DetectAggregator(Aggregator{Type: "usajobs", Query: "x"})
|
||||
if !ok {
|
||||
t.Fatal("expected ok")
|
||||
}
|
||||
if info.APIKey != "env-key" {
|
||||
t.Errorf("expected env-key, got %q", info.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemotive_ParsesFixture(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintln(w, `{
|
||||
"jobs": [
|
||||
{"title":"Security Engineer","url":"https://remotive.com/r/1","company_name":"Acme","candidate_required_location":"Worldwide"},
|
||||
{"title":"DevOps","url":"https://remotive.com/r/2","company_name":"Beta","candidate_required_location":"","job_type":"full_time"}
|
||||
]
|
||||
}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
jobs, err := fetchRemotive(context.Background(), srv.Client(), srv.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 2 {
|
||||
t.Fatalf("jobs = %d, want 2", len(jobs))
|
||||
}
|
||||
if jobs[0].Title != "Security Engineer" || jobs[0].Source != "remotive" {
|
||||
t.Errorf("job[0] malformed: %+v", jobs[0])
|
||||
}
|
||||
if jobs[1].Location != "full_time" {
|
||||
t.Errorf("job[1] should fall back to job_type: %q", jobs[1].Location)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRemoteOK_SkipsHeaderRow(t *testing.T) {
|
||||
// RemoteOK's API: first element is a legal/license header with no id.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.Header.Get("User-Agent"), "apex-scanner") {
|
||||
t.Errorf("missing UA header: %q", r.Header.Get("User-Agent"))
|
||||
}
|
||||
_, _ = fmt.Fprintln(w, `[
|
||||
{"legal":"For personal use only"},
|
||||
{"id":"abc","position":"Red Team Operator","url":"https://remoteok.io/r/abc","company":"Acme","location":"Remote"},
|
||||
{"id":"def","slug":"xyz","title":"SOC","company":"Beta"}
|
||||
]`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
jobs, err := fetchRemoteOK(context.Background(), srv.Client(), srv.URL, "")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 2 {
|
||||
t.Fatalf("jobs = %d, want 2 (header row skipped)", len(jobs))
|
||||
}
|
||||
if jobs[0].Title != "Red Team Operator" {
|
||||
t.Errorf("job[0].Title = %q", jobs[0].Title)
|
||||
}
|
||||
if !strings.HasSuffix(jobs[1].URL, "/xyz") {
|
||||
t.Errorf("slug-derived URL expected, got %q", jobs[1].URL)
|
||||
}
|
||||
if jobs[1].Location != "Remote" {
|
||||
t.Errorf("default Remote location expected, got %q", jobs[1].Location)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchUSAJobs_RequiresKey(t *testing.T) {
|
||||
_, err := fetchUSAJobs(context.Background(), &http.Client{Timeout: time.Second},
|
||||
AdapterInfo{Type: AdapterUSAJobs, URL: "https://data.usajobs.gov/api/search"})
|
||||
if err == nil || !strings.Contains(err.Error(), "api key") {
|
||||
t.Fatalf("expected missing key error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchUSAJobs_ParsesSearchResult(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization-Key") != "test-key" {
|
||||
t.Errorf("missing auth header: %q", r.Header.Get("Authorization-Key"))
|
||||
}
|
||||
payload := map[string]any{
|
||||
"SearchResult": map[string]any{
|
||||
"SearchResultItems": []any{
|
||||
map[string]any{
|
||||
"MatchedObjectDescriptor": map[string]any{
|
||||
"PositionTitle": "Cyber Analyst",
|
||||
"PositionURI": "https://usajobs.gov/job/1",
|
||||
"ApplyURI": []any{"https://usajobs.gov/apply/1"},
|
||||
"OrganizationName": "DHS",
|
||||
"PositionLocationDisplay": "Arlington, VA",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}))
|
||||
defer srv.Close()
|
||||
jobs, err := fetchUSAJobs(context.Background(), srv.Client(), AdapterInfo{
|
||||
Type: AdapterUSAJobs,
|
||||
URL: srv.URL,
|
||||
APIKey: "test-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 1 {
|
||||
t.Fatalf("jobs = %d, want 1", len(jobs))
|
||||
}
|
||||
if jobs[0].URL != "https://usajobs.gov/apply/1" {
|
||||
t.Errorf("ApplyURI[0] should win, got %q", jobs[0].URL)
|
||||
}
|
||||
if jobs[0].Company != "DHS" {
|
||||
t.Errorf("company = %q, want DHS", jobs[0].Company)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRSS_GenericFeed(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/rss+xml")
|
||||
_, _ = fmt.Fprintln(w, `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0"><channel>
|
||||
<item>
|
||||
<title>Pentester</title>
|
||||
<link>https://example.com/pentester</link>
|
||||
<description>Remote, EU timezone preferred.</description>
|
||||
</item>
|
||||
<item>
|
||||
<title></title>
|
||||
<link>https://example.com/no-title</link>
|
||||
</item>
|
||||
<item>
|
||||
<title>Red Team Lead</title>
|
||||
<link></link>
|
||||
<guid>https://example.com/red-team-lead</guid>
|
||||
<description>Long description that is definitely going to be truncated at eighty characters for the location preview.</description>
|
||||
</item>
|
||||
</channel></rss>`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
jobs, err := fetchRSS(context.Background(), srv.Client(), srv.URL, "Acme Co")
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
if len(jobs) != 2 {
|
||||
t.Fatalf("jobs = %d, want 2 (empty-title dropped)", len(jobs))
|
||||
}
|
||||
if jobs[0].Company != "Acme Co" {
|
||||
t.Errorf("default company not applied: %+v", jobs[0])
|
||||
}
|
||||
if jobs[1].URL != "https://example.com/red-team-lead" {
|
||||
t.Errorf("guid fallback failed: %q", jobs[1].URL)
|
||||
}
|
||||
if len(jobs[1].Location) > 80 {
|
||||
t.Errorf("location not truncated: len=%d", len(jobs[1].Location))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRSS_XXEDefused(t *testing.T) {
|
||||
// newSafeXMLDecoder disables entity expansion. Feed a payload that would
|
||||
// trigger an external entity fetch under a naive decoder and confirm we
|
||||
// do NOT fetch anything from the "evil" server.
|
||||
evilHit := make(chan struct{}, 1)
|
||||
evil := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case evilHit <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
_, _ = fmt.Fprint(w, "pwn")
|
||||
}))
|
||||
defer evil.Close()
|
||||
feed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = fmt.Fprintf(w, `<?xml version="1.0"?>
|
||||
<!DOCTYPE foo [ <!ENTITY xxe SYSTEM "%s"> ]>
|
||||
<rss version="2.0"><channel>
|
||||
<item><title>&xxe;</title><link>https://ok.example/ok</link></item>
|
||||
</channel></rss>`, evil.URL)
|
||||
}))
|
||||
defer feed.Close()
|
||||
_, _ = fetchRSS(context.Background(), feed.Client(), feed.URL, "Test")
|
||||
select {
|
||||
case <-evilHit:
|
||||
t.Fatal("XXE: evil server was contacted — entity expansion is NOT defused")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
// good: evil host was not reached
|
||||
}
|
||||
}
|
||||
|
||||
// sanity check so the aggregator path wires into the runner dispatch table.
|
||||
func TestFetch_DispatchesAggregators(t *testing.T) {
|
||||
os.Setenv("USAJOBS_API_KEY", "")
|
||||
cases := []AdapterType{AdapterRemotive, AdapterRemoteOK, AdapterRSS}
|
||||
for _, typ := range cases {
|
||||
_, err := Fetch(context.Background(), &http.Client{Timeout: time.Second},
|
||||
Company{Name: "x"}, AdapterInfo{Type: typ, URL: "http://127.0.0.1:1"})
|
||||
// Expect a real network error, NOT "unsupported adapter type".
|
||||
if err == nil || strings.Contains(err.Error(), "unsupported adapter type") {
|
||||
t.Errorf("%s: dispatch failed, err=%v", typ, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// LoadConfig reads and validates an adapters.yaml file.
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
var cfg Config
|
||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("parse %s: %w", path, err)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WorkdayDiscovery is the result of resolving a company careers URL into
|
||||
// the Workday tenant/site pair that can be plugged into adapters.yaml.
|
||||
type WorkdayDiscovery struct {
|
||||
Input string `json:"input"`
|
||||
FinalURL string `json:"finalUrl"`
|
||||
Host string `json:"host"` // e.g. https://acme.wd3.myworkdayjobs.com
|
||||
Tenant string `json:"tenant"` // e.g. acme
|
||||
Site string `json:"site"` // e.g. Careers
|
||||
Source string `json:"source"` // "url" (direct parse) or "playwright" (shell-out)
|
||||
}
|
||||
|
||||
// YAMLSnippet renders a ready-to-paste adapters.yaml block for the result.
|
||||
func (d WorkdayDiscovery) YAMLSnippet(name string) string {
|
||||
if name == "" {
|
||||
name = d.Tenant
|
||||
}
|
||||
return fmt.Sprintf(`- name: %s
|
||||
careers_url: %s/%s
|
||||
workday:
|
||||
host: %s
|
||||
tenant: %s
|
||||
site: %s
|
||||
enabled: true`, name, d.Host, d.Site, d.Host, d.Tenant, d.Site)
|
||||
}
|
||||
|
||||
// workdayURLRE matches the canonical tenant-hosted Workday URL so we can
|
||||
// short-circuit the Playwright shell-out when the input already IS the
|
||||
// direct Workday URL.
|
||||
var workdayURLRE = regexp.MustCompile(`^(https?://([a-zA-Z0-9_-]+)\.wd[0-9]+\.myworkdayjobs\.com)/(?:en-US/)?([a-zA-Z0-9_-]+)`)
|
||||
|
||||
// DiscoverWorkday resolves a careers URL into Workday host/tenant/site.
|
||||
// Order of attempts:
|
||||
// 1. Parse the URL itself — handles the common case where the user already
|
||||
// pasted the direct *.myworkdayjobs.com/site URL.
|
||||
// 2. If DiscoverScript is non-empty and node is on PATH, invoke the
|
||||
// career-ops discover-workday.mjs helper which launches headless
|
||||
// Playwright and inspects XHR calls.
|
||||
//
|
||||
// Returns ErrNoDiscovery if neither path resolves.
|
||||
type Discoverer struct {
|
||||
// DiscoverScript is the absolute path to career-ops/discover-workday.mjs
|
||||
// (shell-out mode). Empty disables Playwright fallback.
|
||||
DiscoverScript string
|
||||
// Timeout caps the Playwright shell-out. Default 60s.
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func (d *Discoverer) Discover(ctx context.Context, input string) (WorkdayDiscovery, error) {
|
||||
if strings.TrimSpace(input) == "" {
|
||||
return WorkdayDiscovery{}, errors.New("discover: empty input")
|
||||
}
|
||||
if _, err := url.ParseRequestURI(input); err != nil {
|
||||
return WorkdayDiscovery{}, fmt.Errorf("discover: invalid url: %w", err)
|
||||
}
|
||||
// Fast path: direct Workday URL.
|
||||
if m := workdayURLRE.FindStringSubmatch(input); len(m) == 4 {
|
||||
return WorkdayDiscovery{
|
||||
Input: input,
|
||||
FinalURL: input,
|
||||
Host: m[1],
|
||||
Tenant: m[2],
|
||||
Site: m[3],
|
||||
Source: "url",
|
||||
}, nil
|
||||
}
|
||||
// Fallback: Playwright via career-ops shell-out.
|
||||
if d.DiscoverScript == "" {
|
||||
return WorkdayDiscovery{}, ErrNoDiscovery
|
||||
}
|
||||
if _, err := os.Stat(d.DiscoverScript); err != nil {
|
||||
return WorkdayDiscovery{}, fmt.Errorf("discover: script not found at %s: %w", d.DiscoverScript, err)
|
||||
}
|
||||
return d.runPlaywright(ctx, input)
|
||||
}
|
||||
|
||||
// ErrNoDiscovery is returned when the careers URL isn't a direct Workday URL
|
||||
// and no fallback script is configured. Callers can decide to prompt the user.
|
||||
var ErrNoDiscovery = errors.New("discover: unable to resolve (no direct Workday URL, no script configured)")
|
||||
|
||||
func (d *Discoverer) runPlaywright(ctx context.Context, input string) (WorkdayDiscovery, error) {
|
||||
timeout := d.Timeout
|
||||
if timeout == 0 {
|
||||
timeout = 60 * time.Second
|
||||
}
|
||||
runCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
// --json is a flag we rely on the script to implement or ignore; either
|
||||
// way we parse stdout defensively. For now we scan the text output for
|
||||
// the final YAML snippet and regex the Workday fields back out of it.
|
||||
// Absolute path so CWD mismatches don't matter — fall back to the original
|
||||
// path if resolution fails (e.g. broken CWD) instead of silently exec'ing
|
||||
// with an empty argv[1].
|
||||
abs, err := filepath.Abs(d.DiscoverScript)
|
||||
if err != nil {
|
||||
abs = d.DiscoverScript
|
||||
}
|
||||
cmd := exec.CommandContext(runCtx, "node", abs, input)
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &out
|
||||
if err := cmd.Run(); err != nil {
|
||||
return WorkdayDiscovery{}, fmt.Errorf("discover: node run: %w (output: %s)", err, tailString(out.String(), 400))
|
||||
}
|
||||
|
||||
result := parseDiscoverOutput(input, out.String())
|
||||
if result.Host == "" {
|
||||
// Fallback: attempt to parse JSON blob if the script printed one.
|
||||
var j WorkdayDiscovery
|
||||
if err := json.Unmarshal(out.Bytes(), &j); err == nil && j.Host != "" {
|
||||
j.Source = "playwright"
|
||||
return j, nil
|
||||
}
|
||||
return WorkdayDiscovery{}, fmt.Errorf("discover: no workday fields in output (tail: %s)", tailString(out.String(), 200))
|
||||
}
|
||||
result.Source = "playwright"
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Package-level regex patterns for parseDiscoverOutput — compiling once at
|
||||
// init time avoids reallocating on every Playwright shell-out.
|
||||
var (
|
||||
reDiscHost = regexp.MustCompile(`(?m)^\s*host:\s*(\S+)`)
|
||||
reDiscTenant = regexp.MustCompile(`(?m)^\s*tenant:\s*(\S+)`)
|
||||
reDiscSite = regexp.MustCompile(`(?m)^\s*site:\s*(\S+)`)
|
||||
)
|
||||
|
||||
// parseDiscoverOutput walks the discover-workday.mjs stdout looking for the
|
||||
// YAML snippet it prints and extracts host/tenant/site back out.
|
||||
func parseDiscoverOutput(input, output string) WorkdayDiscovery {
|
||||
d := WorkdayDiscovery{Input: input}
|
||||
if m := reDiscHost.FindStringSubmatch(output); len(m) > 1 {
|
||||
d.Host = m[1]
|
||||
}
|
||||
if m := reDiscTenant.FindStringSubmatch(output); len(m) > 1 {
|
||||
d.Tenant = m[1]
|
||||
}
|
||||
if m := reDiscSite.FindStringSubmatch(output); len(m) > 1 {
|
||||
d.Site = m[1]
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// tailString caps log snippets used in error messages so we don't dump
|
||||
// hundreds of KB of Playwright chatter into the user's terminal.
|
||||
func tailString(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return "…" + string(r[len(r)-n:])
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDiscoverer_DirectWorkdayURL(t *testing.T) {
|
||||
d := &Discoverer{}
|
||||
out, err := d.Discover(context.Background(),
|
||||
"https://accenture.wd3.myworkdayjobs.com/en-US/AccentureCareers")
|
||||
if err != nil {
|
||||
t.Fatalf("Discover: %v", err)
|
||||
}
|
||||
if out.Host != "https://accenture.wd3.myworkdayjobs.com" {
|
||||
t.Errorf("host = %q", out.Host)
|
||||
}
|
||||
if out.Tenant != "accenture" {
|
||||
t.Errorf("tenant = %q", out.Tenant)
|
||||
}
|
||||
if out.Site != "AccentureCareers" {
|
||||
t.Errorf("site = %q", out.Site)
|
||||
}
|
||||
if out.Source != "url" {
|
||||
t.Errorf("source = %q, want url", out.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverer_InvalidInput(t *testing.T) {
|
||||
d := &Discoverer{}
|
||||
if _, err := d.Discover(context.Background(), ""); err == nil {
|
||||
t.Fatal("empty input must error")
|
||||
}
|
||||
if _, err := d.Discover(context.Background(), "not a url"); err == nil {
|
||||
t.Fatal("bad url must error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverer_NoScript_NoMatch(t *testing.T) {
|
||||
d := &Discoverer{}
|
||||
_, err := d.Discover(context.Background(), "https://careers.example.com/jobs")
|
||||
if !errors.Is(err, ErrNoDiscovery) {
|
||||
t.Fatalf("expected ErrNoDiscovery, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDiscoverOutput_PullsWorkdayFromYAML(t *testing.T) {
|
||||
raw := `...some preamble...
|
||||
- name: KPMG
|
||||
careers_url: https://kpmg.wd5.myworkdayjobs.com/Global
|
||||
workday:
|
||||
host: https://kpmg.wd5.myworkdayjobs.com
|
||||
tenant: kpmg
|
||||
site: Global
|
||||
enabled: true`
|
||||
d := parseDiscoverOutput("https://kpmg.com/careers", raw)
|
||||
if d.Host != "https://kpmg.wd5.myworkdayjobs.com" {
|
||||
t.Errorf("host = %q", d.Host)
|
||||
}
|
||||
if d.Tenant != "kpmg" {
|
||||
t.Errorf("tenant = %q", d.Tenant)
|
||||
}
|
||||
if d.Site != "Global" {
|
||||
t.Errorf("site = %q", d.Site)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYAMLSnippet_IncludesAllFields(t *testing.T) {
|
||||
d := WorkdayDiscovery{
|
||||
Host: "https://acme.wd3.myworkdayjobs.com",
|
||||
Tenant: "acme",
|
||||
Site: "Careers",
|
||||
}
|
||||
snip := d.YAMLSnippet("Acme Corp")
|
||||
for _, want := range []string{
|
||||
"name: Acme Corp",
|
||||
"tenant: acme",
|
||||
"site: Careers",
|
||||
"host: https://acme.wd3.myworkdayjobs.com",
|
||||
"enabled: true",
|
||||
} {
|
||||
if !contains(snip, want) {
|
||||
t.Errorf("snippet missing %q: %s", want, snip)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// contains is a thin wrapper so I don't drag strings.Contains into every test.
|
||||
func contains(haystack, needle string) bool {
|
||||
return len(haystack) >= len(needle) && indexOf(haystack, needle) >= 0
|
||||
}
|
||||
|
||||
func indexOf(haystack, needle string) int {
|
||||
for i := 0; i+len(needle) <= len(haystack); i++ {
|
||||
if haystack[i:i+len(needle)] == needle {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package scan
|
||||
|
||||
import "strings"
|
||||
|
||||
// TitleMatcher returns true if a title should be kept.
|
||||
type TitleMatcher func(title string) bool
|
||||
|
||||
// NewTitleMatcher builds a case-insensitive positive/negative matcher.
|
||||
// At least one positive must match (or positives empty) AND zero negatives.
|
||||
func NewTitleMatcher(f TitleFilter) TitleMatcher {
|
||||
pos := lower(f.Positive)
|
||||
neg := lower(f.Negative)
|
||||
return func(title string) bool {
|
||||
lt := strings.ToLower(title)
|
||||
hasPos := len(pos) == 0
|
||||
for _, p := range pos {
|
||||
if strings.Contains(lt, p) {
|
||||
hasPos = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasPos {
|
||||
return false
|
||||
}
|
||||
for _, n := range neg {
|
||||
if strings.Contains(lt, n) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func lower(ss []string) []string {
|
||||
out := make([]string, 0, len(ss))
|
||||
for _, s := range ss {
|
||||
out = append(out, strings.ToLower(s))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/cobr-ai/apex/internal/linkedin"
|
||||
)
|
||||
|
||||
// fetchLinkedIn opens a browser session and runs a LinkedIn job search.
|
||||
// query comes from Company.ScanQuery; location defaults to empty (worldwide).
|
||||
func fetchLinkedIn(ctx context.Context, query, location, company string) ([]Job, error) {
|
||||
cli, err := linkedin.NewClient(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cli.Close()
|
||||
results, err := cli.SearchJobs(ctx, query, location, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Job, 0, len(results))
|
||||
for _, r := range results {
|
||||
c := r.Company
|
||||
if c == "" {
|
||||
c = company
|
||||
}
|
||||
out = append(out, Job{
|
||||
Title: r.Title,
|
||||
URL: r.URL,
|
||||
Company: c,
|
||||
Location: r.Location,
|
||||
Source: "linkedin",
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Runner scans all enabled companies with an adapter, filtered by title.
|
||||
type Runner struct {
|
||||
DB *sql.DB
|
||||
Client *http.Client
|
||||
Matcher TitleMatcher
|
||||
Workers int
|
||||
}
|
||||
|
||||
// NewRunner returns a Runner with sensible defaults.
|
||||
func NewRunner(db *sql.DB, cfg *Config) *Runner {
|
||||
return &Runner{
|
||||
DB: db,
|
||||
Client: HTTPClient(15 * time.Second),
|
||||
Matcher: NewTitleMatcher(cfg.TitleFilter),
|
||||
Workers: 6,
|
||||
}
|
||||
}
|
||||
|
||||
// Run scans every enabled, adapter-supported company. Results stream on the
|
||||
// returned channel; the channel closes when all companies are done.
|
||||
func (r *Runner) Run(ctx context.Context, companies []Company) <-chan ScanResult {
|
||||
return r.RunAll(ctx, companies, nil)
|
||||
}
|
||||
|
||||
// RunAll scans companies AND aggregators concurrently. Aggregators are
|
||||
// keyword/tag/feed sources (Remotive, RemoteOK, USAJobs, generic RSS) that
|
||||
// aren't tied to one company; each aggregator appears as its own ScanResult
|
||||
// row named after the aggregator label.
|
||||
func (r *Runner) RunAll(ctx context.Context, companies []Company, aggregators []Aggregator) <-chan ScanResult {
|
||||
out := make(chan ScanResult, len(companies)+len(aggregators))
|
||||
workers := r.Workers
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
sem := make(chan struct{}, workers)
|
||||
var wg sync.WaitGroup
|
||||
for _, c := range companies {
|
||||
if !c.Enabled {
|
||||
continue
|
||||
}
|
||||
info, ok := DetectAdapter(c)
|
||||
if !ok {
|
||||
out <- ScanResult{Company: c.Name, Err: errUnsupported}
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(c Company, info AdapterInfo) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
start := time.Now()
|
||||
res := r.scanOne(ctx, c, info)
|
||||
res.DurationMs = time.Since(start).Milliseconds()
|
||||
out <- res
|
||||
}(c, info)
|
||||
}
|
||||
for _, a := range aggregators {
|
||||
if !a.Enabled {
|
||||
continue
|
||||
}
|
||||
info, ok := DetectAggregator(a)
|
||||
if !ok {
|
||||
out <- ScanResult{Company: a.Name, Err: errUnsupported}
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(a Aggregator, info AdapterInfo) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
start := time.Now()
|
||||
// Aggregators reuse scanOne via a synthetic Company so the
|
||||
// title-filter and record path stay shared.
|
||||
res := r.scanOne(ctx, Company{Name: a.Name}, info)
|
||||
res.DurationMs = time.Since(start).Milliseconds()
|
||||
out <- res
|
||||
}(a, info)
|
||||
}
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(out)
|
||||
}()
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *Runner) scanOne(ctx context.Context, c Company, info AdapterInfo) ScanResult {
|
||||
res := ScanResult{Company: c.Name}
|
||||
jobs, err := Fetch(ctx, r.Client, c, info)
|
||||
if err != nil {
|
||||
res.Err = err
|
||||
return res
|
||||
}
|
||||
res.Found = len(jobs)
|
||||
|
||||
// Title filter.
|
||||
kept := make([]Job, 0, len(jobs))
|
||||
for _, j := range jobs {
|
||||
if r.Matcher(j.Title) {
|
||||
kept = append(kept, j)
|
||||
}
|
||||
}
|
||||
res.Filtered = res.Found - len(kept)
|
||||
|
||||
// Single transaction per company — batches all kept jobs in one commit
|
||||
// to eliminate the N+1 tx pattern under parallel workers.
|
||||
newJobs, err := r.recordJobs(ctx, kept)
|
||||
if err != nil {
|
||||
res.Err = err
|
||||
return res
|
||||
}
|
||||
res.New = len(newJobs)
|
||||
res.Duplicate = len(kept) - res.New
|
||||
res.NewJobs = newJobs
|
||||
return res
|
||||
}
|
||||
|
||||
// recordJobs inserts a batch of jobs under a single transaction and returns
|
||||
// the subset that were newly inserted (not duplicates).
|
||||
func (r *Runner) recordJobs(ctx context.Context, jobs []Job) ([]Job, error) {
|
||||
if len(jobs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tx, err := r.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback() // safe: no-op after explicit commit
|
||||
|
||||
insertJob, err := tx.PrepareContext(ctx,
|
||||
`INSERT OR IGNORE INTO jobs (url, company, title, source) VALUES (?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer insertJob.Close()
|
||||
|
||||
upsertScan, err := tx.PrepareContext(ctx,
|
||||
`INSERT INTO scanned_urls (url, company, title, source, status)
|
||||
VALUES (?, ?, ?, ?, 'added')
|
||||
ON CONFLICT(url) DO UPDATE SET last_seen = CURRENT_TIMESTAMP`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer upsertScan.Close()
|
||||
|
||||
var newJobs []Job
|
||||
for _, j := range jobs {
|
||||
result, err := insertJob.ExecContext(ctx, j.URL, j.Company, j.Title, j.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if rows > 0 {
|
||||
newJobs = append(newJobs, j)
|
||||
}
|
||||
if _, err := upsertScan.ExecContext(ctx, j.URL, j.Company, j.Title, j.Source); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newJobs, nil
|
||||
}
|
||||
|
||||
// errUnsupported is a sentinel error surfaced to the user when a company lacks
|
||||
// an adapter-compatible API. It carries through to Scan UI for visibility.
|
||||
var errUnsupported = &unsupportedErr{}
|
||||
|
||||
type unsupportedErr struct{}
|
||||
|
||||
func (u *unsupportedErr) Error() string { return "no tier-1 adapter available" }
|
||||
|
||||
// IsUnsupported lets UI code show a different row style for unsupported companies.
|
||||
func IsUnsupported(err error) bool {
|
||||
_, ok := err.(*unsupportedErr)
|
||||
return ok
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func TestDetectAdapter_Greenhouse(t *testing.T) {
|
||||
c := Company{
|
||||
Name: "Dragos",
|
||||
CareersURL: "https://job-boards.greenhouse.io/dragos",
|
||||
API: "https://boards-api.greenhouse.io/v1/boards/dragos/jobs",
|
||||
}
|
||||
info, ok := DetectAdapter(c)
|
||||
if !ok || info.Type != AdapterGreenhouse {
|
||||
t.Fatalf("expected greenhouse, got %v ok=%v", info.Type, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectAdapter_Ashby(t *testing.T) {
|
||||
c := Company{Name: "Acme", CareersURL: "https://jobs.ashbyhq.com/acme"}
|
||||
info, ok := DetectAdapter(c)
|
||||
if !ok || info.Type != AdapterAshby {
|
||||
t.Fatalf("expected ashby, got %v ok=%v", info.Type, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectAdapter_Lever(t *testing.T) {
|
||||
c := Company{Name: "Acme", CareersURL: "https://jobs.lever.co/acme"}
|
||||
info, ok := DetectAdapter(c)
|
||||
if !ok || info.Type != AdapterLever {
|
||||
t.Fatalf("expected lever, got %v ok=%v", info.Type, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectAdapter_Unknown(t *testing.T) {
|
||||
c := Company{Name: "Acme", CareersURL: "https://acme.com/careers"}
|
||||
_, ok := DetectAdapter(c)
|
||||
if ok {
|
||||
t.Fatalf("expected unsupported for arbitrary careers page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTitleMatcher_PositiveOnly(t *testing.T) {
|
||||
m := NewTitleMatcher(TitleFilter{Positive: []string{"Red Team", "Penetration Test"}})
|
||||
if !m("Senior Red Team Operator") {
|
||||
t.Fatal("expected match")
|
||||
}
|
||||
if m("Senior Software Engineer") {
|
||||
t.Fatal("expected no match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTitleMatcher_NegativeBlocks(t *testing.T) {
|
||||
m := NewTitleMatcher(TitleFilter{
|
||||
Positive: []string{"Security"},
|
||||
Negative: []string{"Intern", "Junior"},
|
||||
})
|
||||
if m("Security Intern") {
|
||||
t.Fatal("negative should block")
|
||||
}
|
||||
if !m("Security Engineer") {
|
||||
t.Fatal("positive should match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordJobs_BatchDedup(t *testing.T) {
|
||||
db, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(`
|
||||
CREATE TABLE jobs (url TEXT UNIQUE NOT NULL, company TEXT, title TEXT, source TEXT);
|
||||
CREATE TABLE scanned_urls (url TEXT UNIQUE NOT NULL, company TEXT, title TEXT, source TEXT, status TEXT, first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP, last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
|
||||
`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := &Runner{DB: db}
|
||||
ctx := context.Background()
|
||||
batch := []Job{
|
||||
{Title: "Senior Red Team", URL: "https://example.com/jobs/1", Company: "Acme", Source: "test"},
|
||||
{Title: "Staff Pentester", URL: "https://example.com/jobs/2", Company: "Acme", Source: "test"},
|
||||
}
|
||||
|
||||
newJobs, err := r.recordJobs(ctx, batch)
|
||||
if err != nil {
|
||||
t.Fatalf("first batch: %v", err)
|
||||
}
|
||||
if len(newJobs) != 2 {
|
||||
t.Fatalf("expected 2 new, got %d", len(newJobs))
|
||||
}
|
||||
// Re-run the same batch plus one fresh URL — only the new one should land.
|
||||
batch = append(batch, Job{Title: "Principal OT", URL: "https://example.com/jobs/3", Company: "Acme", Source: "test"})
|
||||
newJobs, err = r.recordJobs(ctx, batch)
|
||||
if err != nil {
|
||||
t.Fatalf("second batch: %v", err)
|
||||
}
|
||||
if len(newJobs) != 1 || newJobs[0].URL != "https://example.com/jobs/3" {
|
||||
t.Fatalf("expected only jobs/3 new, got %+v", newJobs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig(t *testing.T) {
|
||||
cfg, err := LoadConfig("adapters.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig: %v", err)
|
||||
}
|
||||
if len(cfg.TrackedCompanies) == 0 {
|
||||
t.Fatal("expected seeded companies")
|
||||
}
|
||||
if len(cfg.TitleFilter.Positive) == 0 {
|
||||
t.Fatal("expected positive keywords")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// roleStopwords are seniority/modality words that add no signal when comparing
|
||||
// whether two job titles describe the same underlying role. Mirrors the list
|
||||
// in career-ops dedup-tracker.mjs.
|
||||
var roleStopwords = map[string]struct{}{
|
||||
"senior": {}, "junior": {}, "lead": {}, "staff": {}, "principal": {},
|
||||
"head": {}, "chief": {}, "manager": {}, "director": {}, "associate": {},
|
||||
"intern": {}, "contractor": {}, "remote": {}, "hybrid": {}, "onsite": {},
|
||||
"engineer": {}, "engineering": {},
|
||||
}
|
||||
|
||||
// locationStopwords catch city/region tokens that sometimes appear inside a
|
||||
// title (e.g. "Security Engineer — Tokyo"). They should not count toward
|
||||
// role-similarity overlap.
|
||||
var locationStopwords = map[string]struct{}{
|
||||
"tokyo": {}, "japan": {}, "london": {}, "berlin": {}, "paris": {},
|
||||
"singapore": {}, "york": {}, "francisco": {}, "angeles": {}, "seattle": {},
|
||||
"austin": {}, "boston": {}, "chicago": {}, "denver": {}, "toronto": {},
|
||||
"amsterdam": {}, "dublin": {}, "sydney": {}, "global": {}, "emea": {},
|
||||
"apac": {}, "latam": {},
|
||||
}
|
||||
|
||||
var tokenSplitRE = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
// tokenizeRole lowercases, splits on non-alphanumeric, drops short words
|
||||
// (≤ 2 chars) and stopwords. The return is a dedup-preserving slice.
|
||||
func tokenizeRole(s string) []string {
|
||||
s = strings.ToLower(s)
|
||||
raw := tokenSplitRE.Split(s, -1)
|
||||
seen := make(map[string]struct{}, len(raw))
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, w := range raw {
|
||||
if len(w) <= 2 {
|
||||
continue
|
||||
}
|
||||
if _, stop := roleStopwords[w]; stop {
|
||||
continue
|
||||
}
|
||||
if _, stop := locationStopwords[w]; stop {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[w]; dup {
|
||||
continue
|
||||
}
|
||||
seen[w] = struct{}{}
|
||||
out = append(out, w)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RoleSimilar reports whether two job titles describe what is plausibly the
|
||||
// same underlying role. Uses token-overlap with stopwords stripped and a
|
||||
// prefix-both-ways comparison so that abbreviations (pen → penetration)
|
||||
// don't break the match.
|
||||
//
|
||||
// A match requires:
|
||||
// - both titles yield at least one content word (after stopword strip);
|
||||
// - at least 2 content words overlap OR every content word in the shorter
|
||||
// side matches (handles single-word patterns like "Security");
|
||||
// - overlap ratio ≥ 0.5 against the smaller tokenized side.
|
||||
//
|
||||
// Intended for dedup UX ("we've seen a similar role"), not for hard filtering.
|
||||
func RoleSimilar(a, b string) bool {
|
||||
wa := tokenizeRole(a)
|
||||
wb := tokenizeRole(b)
|
||||
if len(wa) == 0 || len(wb) == 0 {
|
||||
return false
|
||||
}
|
||||
overlap := 0
|
||||
for _, x := range wa {
|
||||
for _, y := range wb {
|
||||
if x == y || strings.HasPrefix(y, x) || strings.HasPrefix(x, y) {
|
||||
overlap++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if overlap == 0 {
|
||||
return false
|
||||
}
|
||||
smaller := len(wa)
|
||||
if len(wb) < smaller {
|
||||
smaller = len(wb)
|
||||
}
|
||||
required := 2
|
||||
if smaller < required {
|
||||
required = smaller
|
||||
}
|
||||
if overlap < required {
|
||||
return false
|
||||
}
|
||||
ratio := float64(overlap) / float64(smaller)
|
||||
return ratio >= 0.5
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package scan
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRoleSimilar_TruePositives(t *testing.T) {
|
||||
cases := [][2]string{
|
||||
{"Senior Red Team Operator", "Red Team Operator"},
|
||||
{"Staff Penetration Tester", "Senior Pen Tester"}, // prefix match pen→penetration
|
||||
{"Security Engineer, Tokyo", "Senior Security Engineer"}, // location + seniority stripped
|
||||
{"Principal ICS Security Consultant", "ICS Security Engineer"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if !RoleSimilar(c[0], c[1]) {
|
||||
t.Errorf("expected similar: %q vs %q", c[0], c[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleSimilar_TrueNegatives(t *testing.T) {
|
||||
cases := [][2]string{
|
||||
{"Red Team Operator", "Data Engineer"},
|
||||
{"Security Engineer", "Accounting Analyst"},
|
||||
{"Head of Engineering", "Head of Marketing"}, // both reduce to [marketing] vs [] after stopwords
|
||||
{"Senior Engineer", "Principal Engineer"}, // both reduce to empty after stopwords
|
||||
}
|
||||
for _, c := range cases {
|
||||
if RoleSimilar(c[0], c[1]) {
|
||||
t.Errorf("expected not similar: %q vs %q", c[0], c[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleSimilar_EmptyInputs(t *testing.T) {
|
||||
if RoleSimilar("", "") {
|
||||
t.Fatal("empty inputs must not match")
|
||||
}
|
||||
if RoleSimilar("Senior", "Engineer") {
|
||||
t.Fatal("all-stopword inputs must not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenizeRole_StripsCorrectly(t *testing.T) {
|
||||
got := tokenizeRole("Senior Red Team Operator, Tokyo (Remote)")
|
||||
// senior, tokyo, remote, engineer stopwords stripped; too-short (≤2) stripped.
|
||||
want := map[string]bool{"red": true, "team": true, "operator": true}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %v want keys %v", got, want)
|
||||
}
|
||||
for _, w := range got {
|
||||
if !want[w] {
|
||||
t.Errorf("unexpected token %q", w)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package scan
|
||||
|
||||
// Job is a single posting discovered from a portal.
|
||||
type Job struct {
|
||||
Title string
|
||||
URL string
|
||||
Company string
|
||||
Location string
|
||||
Source string
|
||||
}
|
||||
|
||||
// Company is a target scanned from adapters.yaml.
|
||||
type Company struct {
|
||||
Name string `yaml:"name"`
|
||||
CareersURL string `yaml:"careers_url"`
|
||||
API string `yaml:"api,omitempty"`
|
||||
ScanMethod string `yaml:"scan_method,omitempty"`
|
||||
ScanQuery string `yaml:"scan_query,omitempty"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Notes string `yaml:"notes,omitempty"`
|
||||
Workday *WorkdayConfig `yaml:"workday,omitempty"`
|
||||
}
|
||||
|
||||
// WorkdayConfig holds tenant/site for Workday POST API.
|
||||
type WorkdayConfig struct {
|
||||
Host string `yaml:"host"`
|
||||
Tenant string `yaml:"tenant"`
|
||||
Site string `yaml:"site"`
|
||||
}
|
||||
|
||||
// TitleFilter controls which job titles are accepted.
|
||||
type TitleFilter struct {
|
||||
Positive []string `yaml:"positive"`
|
||||
Negative []string `yaml:"negative"`
|
||||
SeniorityBoost []string `yaml:"seniority_boost,omitempty"`
|
||||
}
|
||||
|
||||
// Aggregator is a broad-search job source (not tied to one company).
|
||||
// Examples: Remotive keyword search, RemoteOK by tag, USAJobs keyword search,
|
||||
// a generic RSS feed. The runner scans aggregators alongside TrackedCompanies.
|
||||
type Aggregator struct {
|
||||
Name string `yaml:"name"`
|
||||
Type string `yaml:"type"` // remotive | remoteok | usajobs | rss
|
||||
Query string `yaml:"query,omitempty"` // keyword search (remotive, usajobs)
|
||||
Tag string `yaml:"tag,omitempty"` // remoteok tag (e.g. "security")
|
||||
Limit int `yaml:"limit,omitempty"` // results per call; default 50
|
||||
URL string `yaml:"url,omitempty"` // rss feed URL
|
||||
Company string `yaml:"company,omitempty"` // default company label for rss
|
||||
APIKey string `yaml:"api_key,omitempty"` // USAJobs API key; env USAJOBS_API_KEY also honored
|
||||
UserAgent string `yaml:"user_agent,omitempty"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// Config is the scan configuration loaded from adapters.yaml.
|
||||
type Config struct {
|
||||
TitleFilter TitleFilter `yaml:"title_filter"`
|
||||
TrackedCompanies []Company `yaml:"tracked_companies"`
|
||||
Aggregators []Aggregator `yaml:"aggregators,omitempty"`
|
||||
}
|
||||
|
||||
// AdapterType identifies the detected API kind.
|
||||
type AdapterType string
|
||||
|
||||
const (
|
||||
AdapterGreenhouse AdapterType = "greenhouse"
|
||||
AdapterAshby AdapterType = "ashby"
|
||||
AdapterLever AdapterType = "lever"
|
||||
AdapterWorkday AdapterType = "workday"
|
||||
AdapterBambooHR AdapterType = "bamboohr"
|
||||
AdapterTeamtailor AdapterType = "teamtailor"
|
||||
AdapterRemotive AdapterType = "remotive"
|
||||
AdapterRemoteOK AdapterType = "remoteok"
|
||||
AdapterUSAJobs AdapterType = "usajobs"
|
||||
AdapterRSS AdapterType = "rss"
|
||||
AdapterLinkedIn AdapterType = "linkedin"
|
||||
AdapterUnknown AdapterType = "unknown"
|
||||
)
|
||||
|
||||
// AdapterInfo is the resolved API endpoint for a company or aggregator.
|
||||
// RSS and USAJobs populate the extra fields used during fetch (e.g. default
|
||||
// company label, API key header). Zero values are fine for other adapters.
|
||||
type AdapterInfo struct {
|
||||
Type AdapterType
|
||||
URL string
|
||||
Host string
|
||||
Site string
|
||||
DefaultCompany string // rss: spliced into Job.Company when feed is generic
|
||||
APIKey string // usajobs: Authorization-Key header
|
||||
UserAgent string // usajobs + rss: optional UA override
|
||||
}
|
||||
|
||||
// ScanResult is reported per-company as the runner progresses.
|
||||
type ScanResult struct {
|
||||
Company string
|
||||
Found int
|
||||
Filtered int
|
||||
Duplicate int
|
||||
New int
|
||||
NewJobs []Job
|
||||
Err error
|
||||
DurationMs int64
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
)
|
||||
|
||||
// newSafeXMLDecoder disables external entity expansion so parseRSSItems is
|
||||
// XXE-safe even when a malicious feed is served.
|
||||
func newSafeXMLDecoder(r io.Reader) *xml.Decoder {
|
||||
d := xml.NewDecoder(r)
|
||||
d.Strict = false
|
||||
d.Entity = map[string]string{} // no external/named entity resolution
|
||||
return d
|
||||
}
|
||||
Reference in New Issue
Block a user