package scan import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "strings" "testing" "time" ) // Round-trip fixture test: spin up a local server that returns each provider's // canned payload, then verify the adapter fetches + normalizes correctly. func TestDetectAggregator_EachType(t *testing.T) { cases := []struct { agg Aggregator want AdapterType }{ {Aggregator{Type: "remotive", Query: "security"}, AdapterRemotive}, {Aggregator{Type: "remoteok", Tag: "security"}, AdapterRemoteOK}, {Aggregator{Type: "usajobs", Query: "cyber", APIKey: "k"}, AdapterUSAJobs}, {Aggregator{Type: "rss", URL: "https://example.com/feed.xml"}, AdapterRSS}, } for _, c := range cases { info, ok := DetectAggregator(c.agg) if !ok { t.Errorf("%s: expected ok=true", c.agg.Type) continue } if info.Type != c.want { t.Errorf("%s: type = %q, want %q", c.agg.Type, info.Type, c.want) } if info.URL == "" { t.Errorf("%s: URL empty", c.agg.Type) } } } func TestDetectAggregator_RSSRequiresURL(t *testing.T) { if _, ok := DetectAggregator(Aggregator{Type: "rss"}); ok { t.Fatal("rss without URL must be rejected") } } func TestDetectAggregator_USAJobsEnvKey(t *testing.T) { t.Setenv("USAJOBS_API_KEY", "env-key") info, ok := DetectAggregator(Aggregator{Type: "usajobs", Query: "x"}) if !ok { t.Fatal("expected ok") } if info.APIKey != "env-key" { t.Errorf("expected env-key, got %q", info.APIKey) } } func TestFetchRemotive_ParsesFixture(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprintln(w, `{ "jobs": [ {"title":"Security Engineer","url":"https://remotive.com/r/1","company_name":"Acme","candidate_required_location":"Worldwide"}, {"title":"DevOps","url":"https://remotive.com/r/2","company_name":"Beta","candidate_required_location":"","job_type":"full_time"} ] }`) })) defer srv.Close() jobs, err := fetchRemotive(context.Background(), srv.Client(), srv.URL) if err != nil { t.Fatalf("fetch: %v", err) } if len(jobs) != 2 { t.Fatalf("jobs = %d, want 2", len(jobs)) } if jobs[0].Title != "Security Engineer" || jobs[0].Source != "remotive" { t.Errorf("job[0] malformed: %+v", jobs[0]) } if jobs[1].Location != "full_time" { t.Errorf("job[1] should fall back to job_type: %q", jobs[1].Location) } } func TestFetchRemoteOK_SkipsHeaderRow(t *testing.T) { // RemoteOK's API: first element is a legal/license header with no id. srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.HasPrefix(r.Header.Get("User-Agent"), "apex-scanner") { t.Errorf("missing UA header: %q", r.Header.Get("User-Agent")) } _, _ = fmt.Fprintln(w, `[ {"legal":"For personal use only"}, {"id":"abc","position":"Red Team Operator","url":"https://remoteok.io/r/abc","company":"Acme","location":"Remote"}, {"id":"def","slug":"xyz","title":"SOC","company":"Beta"} ]`) })) defer srv.Close() jobs, err := fetchRemoteOK(context.Background(), srv.Client(), srv.URL, "") if err != nil { t.Fatalf("fetch: %v", err) } if len(jobs) != 2 { t.Fatalf("jobs = %d, want 2 (header row skipped)", len(jobs)) } if jobs[0].Title != "Red Team Operator" { t.Errorf("job[0].Title = %q", jobs[0].Title) } if !strings.HasSuffix(jobs[1].URL, "/xyz") { t.Errorf("slug-derived URL expected, got %q", jobs[1].URL) } if jobs[1].Location != "Remote" { t.Errorf("default Remote location expected, got %q", jobs[1].Location) } } func TestFetchUSAJobs_RequiresKey(t *testing.T) { _, err := fetchUSAJobs(context.Background(), &http.Client{Timeout: time.Second}, AdapterInfo{Type: AdapterUSAJobs, URL: "https://data.usajobs.gov/api/search"}) if err == nil || !strings.Contains(err.Error(), "api key") { t.Fatalf("expected missing key error, got %v", err) } } func TestFetchUSAJobs_ParsesSearchResult(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization-Key") != "test-key" { t.Errorf("missing auth header: %q", r.Header.Get("Authorization-Key")) } payload := map[string]any{ "SearchResult": map[string]any{ "SearchResultItems": []any{ map[string]any{ "MatchedObjectDescriptor": map[string]any{ "PositionTitle": "Cyber Analyst", "PositionURI": "https://usajobs.gov/job/1", "ApplyURI": []any{"https://usajobs.gov/apply/1"}, "OrganizationName": "DHS", "PositionLocationDisplay": "Arlington, VA", }, }, }, }, } _ = json.NewEncoder(w).Encode(payload) })) defer srv.Close() jobs, err := fetchUSAJobs(context.Background(), srv.Client(), AdapterInfo{ Type: AdapterUSAJobs, URL: srv.URL, APIKey: "test-key", }) if err != nil { t.Fatalf("fetch: %v", err) } if len(jobs) != 1 { t.Fatalf("jobs = %d, want 1", len(jobs)) } if jobs[0].URL != "https://usajobs.gov/apply/1" { t.Errorf("ApplyURI[0] should win, got %q", jobs[0].URL) } if jobs[0].Company != "DHS" { t.Errorf("company = %q, want DHS", jobs[0].Company) } } func TestFetchRSS_GenericFeed(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/rss+xml") _, _ = fmt.Fprintln(w, ` Pentester https://example.com/pentester Remote, EU timezone preferred. https://example.com/no-title Red Team Lead https://example.com/red-team-lead Long description that is definitely going to be truncated at eighty characters for the location preview. `) })) defer srv.Close() jobs, err := fetchRSS(context.Background(), srv.Client(), srv.URL, "Acme Co") if err != nil { t.Fatalf("fetch: %v", err) } if len(jobs) != 2 { t.Fatalf("jobs = %d, want 2 (empty-title dropped)", len(jobs)) } if jobs[0].Company != "Acme Co" { t.Errorf("default company not applied: %+v", jobs[0]) } if jobs[1].URL != "https://example.com/red-team-lead" { t.Errorf("guid fallback failed: %q", jobs[1].URL) } if len(jobs[1].Location) > 80 { t.Errorf("location not truncated: len=%d", len(jobs[1].Location)) } } func TestFetchRSS_XXEDefused(t *testing.T) { // newSafeXMLDecoder disables entity expansion. Feed a payload that would // trigger an external entity fetch under a naive decoder and confirm we // do NOT fetch anything from the "evil" server. evilHit := make(chan struct{}, 1) evil := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case evilHit <- struct{}{}: default: } _, _ = fmt.Fprint(w, "pwn") })) defer evil.Close() feed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/xml") _, _ = fmt.Fprintf(w, ` ]> &xxe;https://ok.example/ok `, evil.URL) })) defer feed.Close() _, _ = fetchRSS(context.Background(), feed.Client(), feed.URL, "Test") select { case <-evilHit: t.Fatal("XXE: evil server was contacted — entity expansion is NOT defused") case <-time.After(200 * time.Millisecond): // good: evil host was not reached } } // sanity check so the aggregator path wires into the runner dispatch table. func TestFetch_DispatchesAggregators(t *testing.T) { os.Setenv("USAJOBS_API_KEY", "") cases := []AdapterType{AdapterRemotive, AdapterRemoteOK, AdapterRSS} for _, typ := range cases { _, err := Fetch(context.Background(), &http.Client{Timeout: time.Second}, Company{Name: "x"}, AdapterInfo{Type: typ, URL: "http://127.0.0.1:1"}) // Expect a real network error, NOT "unsupported adapter type". if err == nil || strings.Contains(err.Error(), "unsupported adapter type") { t.Errorf("%s: dispatch failed, err=%v", typ, err) } } }