apex-public/internal/scan/aggregator_test.go

package scan

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"
	"time"
)

// Round-trip fixture test: spin up a local server that returns each provider's
// canned payload, then verify the adapter fetches + normalizes correctly.

func TestDetectAggregator_EachType(t *testing.T) {
	cases := []struct {
		agg  Aggregator
		want AdapterType
	}{
		{Aggregator{Type: "remotive", Query: "security"}, AdapterRemotive},
		{Aggregator{Type: "remoteok", Tag: "security"}, AdapterRemoteOK},
		{Aggregator{Type: "usajobs", Query: "cyber", APIKey: "k"}, AdapterUSAJobs},
		{Aggregator{Type: "rss", URL: "https://example.com/feed.xml"}, AdapterRSS},
	}
	for _, c := range cases {
		info, ok := DetectAggregator(c.agg)
		if !ok {
			t.Errorf("%s: expected ok=true", c.agg.Type)
			continue
		}
		if info.Type != c.want {
			t.Errorf("%s: type = %q, want %q", c.agg.Type, info.Type, c.want)
		}
		if info.URL == "" {
			t.Errorf("%s: URL empty", c.agg.Type)
		}
	}
}

func TestDetectAggregator_RSSRequiresURL(t *testing.T) {
	if _, ok := DetectAggregator(Aggregator{Type: "rss"}); ok {
		t.Fatal("rss without URL must be rejected")
	}
}

func TestDetectAggregator_USAJobsEnvKey(t *testing.T) {
	t.Setenv("USAJOBS_API_KEY", "env-key")
	info, ok := DetectAggregator(Aggregator{Type: "usajobs", Query: "x"})
	if !ok {
		t.Fatal("expected ok")
	}
	if info.APIKey != "env-key" {
		t.Errorf("expected env-key, got %q", info.APIKey)
	}
}

func TestFetchRemotive_ParsesFixture(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_, _ = fmt.Fprintln(w, `{
			"jobs": [
				{"title":"Security Engineer","url":"https://remotive.com/r/1","company_name":"Acme","candidate_required_location":"Worldwide"},
				{"title":"DevOps","url":"https://remotive.com/r/2","company_name":"Beta","candidate_required_location":"","job_type":"full_time"}
			]
		}`)
	}))
	defer srv.Close()
	jobs, err := fetchRemotive(context.Background(), srv.Client(), srv.URL)
	if err != nil {
		t.Fatalf("fetch: %v", err)
	}
	if len(jobs) != 2 {
		t.Fatalf("jobs = %d, want 2", len(jobs))
	}
	if jobs[0].Title != "Security Engineer" || jobs[0].Source != "remotive" {
		t.Errorf("job[0] malformed: %+v", jobs[0])
	}
	if jobs[1].Location != "full_time" {
		t.Errorf("job[1] should fall back to job_type: %q", jobs[1].Location)
	}
}

func TestFetchRemoteOK_SkipsHeaderRow(t *testing.T) {
	// RemoteOK's API: first element is a legal/license header with no id.
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if !strings.HasPrefix(r.Header.Get("User-Agent"), "apex-scanner") {
			t.Errorf("missing UA header: %q", r.Header.Get("User-Agent"))
		}
		_, _ = fmt.Fprintln(w, `[
			{"legal":"For personal use only"},
			{"id":"abc","position":"Red Team Operator","url":"https://remoteok.io/r/abc","company":"Acme","location":"Remote"},
			{"id":"def","slug":"xyz","title":"SOC","company":"Beta"}
		]`)
	}))
	defer srv.Close()
	jobs, err := fetchRemoteOK(context.Background(), srv.Client(), srv.URL, "")
	if err != nil {
		t.Fatalf("fetch: %v", err)
	}
	if len(jobs) != 2 {
		t.Fatalf("jobs = %d, want 2 (header row skipped)", len(jobs))
	}
	if jobs[0].Title != "Red Team Operator" {
		t.Errorf("job[0].Title = %q", jobs[0].Title)
	}
	if !strings.HasSuffix(jobs[1].URL, "/xyz") {
		t.Errorf("slug-derived URL expected, got %q", jobs[1].URL)
	}
	if jobs[1].Location != "Remote" {
		t.Errorf("default Remote location expected, got %q", jobs[1].Location)
	}
}

func TestFetchUSAJobs_RequiresKey(t *testing.T) {
	_, err := fetchUSAJobs(context.Background(), &http.Client{Timeout: time.Second},
		AdapterInfo{Type: AdapterUSAJobs, URL: "https://data.usajobs.gov/api/search"})
	if err == nil || !strings.Contains(err.Error(), "api key") {
		t.Fatalf("expected missing key error, got %v", err)
	}
}

func TestFetchUSAJobs_ParsesSearchResult(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization-Key") != "test-key" {
			t.Errorf("missing auth header: %q", r.Header.Get("Authorization-Key"))
		}
		payload := map[string]any{
			"SearchResult": map[string]any{
				"SearchResultItems": []any{
					map[string]any{
						"MatchedObjectDescriptor": map[string]any{
							"PositionTitle":           "Cyber Analyst",
							"PositionURI":             "https://usajobs.gov/job/1",
							"ApplyURI":                []any{"https://usajobs.gov/apply/1"},
							"OrganizationName":        "DHS",
							"PositionLocationDisplay": "Arlington, VA",
						},
					},
				},
			},
		}
		_ = json.NewEncoder(w).Encode(payload)
	}))
	defer srv.Close()
	jobs, err := fetchUSAJobs(context.Background(), srv.Client(), AdapterInfo{
		Type:   AdapterUSAJobs,
		URL:    srv.URL,
		APIKey: "test-key",
	})
	if err != nil {
		t.Fatalf("fetch: %v", err)
	}
	if len(jobs) != 1 {
		t.Fatalf("jobs = %d, want 1", len(jobs))
	}
	if jobs[0].URL != "https://usajobs.gov/apply/1" {
		t.Errorf("ApplyURI[0] should win, got %q", jobs[0].URL)
	}
	if jobs[0].Company != "DHS" {
		t.Errorf("company = %q, want DHS", jobs[0].Company)
	}
}

func TestFetchRSS_GenericFeed(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/rss+xml")
		_, _ = fmt.Fprintln(w, `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"><channel>
  <item>
    <title>Pentester</title>
    <link>https://example.com/pentester</link>
    <description>Remote, EU timezone preferred.</description>
  </item>
  <item>
    <title></title>
    <link>https://example.com/no-title</link>
  </item>
  <item>
    <title>Red Team Lead</title>
    <link></link>
    <guid>https://example.com/red-team-lead</guid>
    <description>Long description that is definitely going to be truncated at eighty characters for the location preview.</description>
  </item>
</channel></rss>`)
	}))
	defer srv.Close()
	jobs, err := fetchRSS(context.Background(), srv.Client(), srv.URL, "Acme Co")
	if err != nil {
		t.Fatalf("fetch: %v", err)
	}
	if len(jobs) != 2 {
		t.Fatalf("jobs = %d, want 2 (empty-title dropped)", len(jobs))
	}
	if jobs[0].Company != "Acme Co" {
		t.Errorf("default company not applied: %+v", jobs[0])
	}
	if jobs[1].URL != "https://example.com/red-team-lead" {
		t.Errorf("guid fallback failed: %q", jobs[1].URL)
	}
	if len(jobs[1].Location) > 80 {
		t.Errorf("location not truncated: len=%d", len(jobs[1].Location))
	}
}

func TestFetchRSS_XXEDefused(t *testing.T) {
	// newSafeXMLDecoder disables entity expansion. Feed a payload that would
	// trigger an external entity fetch under a naive decoder and confirm we
	// do NOT fetch anything from the "evil" server.
	evilHit := make(chan struct{}, 1)
	evil := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		select {
		case evilHit <- struct{}{}:
		default:
		}
		_, _ = fmt.Fprint(w, "pwn")
	}))
	defer evil.Close()
	feed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/xml")
		_, _ = fmt.Fprintf(w, `<?xml version="1.0"?>
<!DOCTYPE foo [ <!ENTITY xxe SYSTEM "%s"> ]>
<rss version="2.0"><channel>
  <item><title>&xxe;</title><link>https://ok.example/ok</link></item>
</channel></rss>`, evil.URL)
	}))
	defer feed.Close()
	_, _ = fetchRSS(context.Background(), feed.Client(), feed.URL, "Test")
	select {
	case <-evilHit:
		t.Fatal("XXE: evil server was contacted — entity expansion is NOT defused")
	case <-time.After(200 * time.Millisecond):
		// good: evil host was not reached
	}
}

// sanity check so the aggregator path wires into the runner dispatch table.
func TestFetch_DispatchesAggregators(t *testing.T) {
	os.Setenv("USAJOBS_API_KEY", "")
	cases := []AdapterType{AdapterRemotive, AdapterRemoteOK, AdapterRSS}
	for _, typ := range cases {
		_, err := Fetch(context.Background(), &http.Client{Timeout: time.Second},
			Company{Name: "x"}, AdapterInfo{Type: typ, URL: "http://127.0.0.1:1"})
		// Expect a real network error, NOT "unsupported adapter type".
		if err == nil || strings.Contains(err.Error(), "unsupported adapter type") {
			t.Errorf("%s: dispatch failed, err=%v", typ, err)
		}
	}
}