1239 lines
50 KiB
JavaScript
1239 lines
50 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* scan.mjs — Zero-token portal scanner with a plugin-based provider layer.
|
|
*
|
|
* Providers live in providers/*.mjs and are loaded at startup. Each provider
|
|
* exports a default object with:
|
|
* - id: string — matched against `provider:` in portals.yml
|
|
* - detect(entry): {url}|null — optional auto-detection from careers_url
|
|
* - fetch(entry, ctx): [{title,url,company,location}] — required
|
|
*
|
|
* Files prefixed with _ are shared helpers (e.g. _http.mjs) and are never
|
|
* loaded as providers. Adding a new HTTP/API source = drop a *.mjs into
|
|
* providers/. Local executable parsers use `providers/local-parser.mjs` when
|
|
* `parser.command` + `parser.script` are set in portals.yml.
|
|
*
|
|
* A tracked_companies entry can set `provider:` explicitly to bypass
|
|
* URL-based auto-detection. The `transport:` field is reserved for future
|
|
* transports — Phase A only ships the http transport.
|
|
*
|
|
* Zero Claude API tokens — pure HTTP + JSON.
|
|
*
|
|
* Usage:
|
|
* node scan.mjs # scan all enabled companies
|
|
* node scan.mjs --dry-run # preview without writing files
|
|
* node scan.mjs --company Cohere # scan a single company
|
|
* node scan.mjs --verify # Playwright-check each new URL; drop expired postings
|
|
* node scan.mjs --verify --headed-fallback # retry anti-bot-blocked URLs in a headed browser (needs a display)
|
|
* node scan.mjs --verify --throttle # jittered ~5-10s gap between checks (stay under rate limits)
|
|
* node scan.mjs --verify --throttle=8000 # custom base gap in ms (waits base..2*base)
|
|
*/
|
|
|
|
import { readFileSync, writeFileSync, appendFileSync, existsSync, mkdirSync } from 'fs';
|
|
import { pathToFileURL, fileURLToPath } from 'url';
|
|
import path from 'path';
|
|
import yaml from 'js-yaml';
|
|
|
|
import { makeHttpCtx } from './providers/_http.mjs';
|
|
import { buildTrustValidator } from './providers/_trust-validator.mjs';
|
|
import { loadProviders, resolveProvider } from './providers/_registry.mjs';
|
|
import { mergeProviderPlugins } from './plugins/_engine.mjs';
|
|
import { classifyFetchError } from './verify-portals.mjs';
|
|
|
|
try {
|
|
const { config } = await import('dotenv');
|
|
config();
|
|
} catch {
|
|
// dotenv is optional — fall back to process.env if not installed
|
|
}
|
|
|
|
const parseYaml = yaml.load;
|
|
|
|
// ── Config ──────────────────────────────────────────────────────────
|
|
|
|
const PORTALS_PATH = process.env.CAREER_OPS_PORTALS || 'portals.yml';
|
|
const PROFILE_PATH = process.env.CAREER_OPS_PROFILE || 'config/profile.yml';
|
|
const SCAN_HISTORY_PATH = 'data/scan-history.tsv';
|
|
const PIPELINE_PATH = 'data/pipeline.md';
|
|
const APPLICATIONS_PATH = 'data/applications.md';
|
|
const PROVIDERS_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), 'providers');
|
|
|
|
// Ensure required directories exist (fresh setup)
|
|
mkdirSync('data', { recursive: true });
|
|
|
|
const CONCURRENCY = 10;
|
|
|
|
// Provider loading + routing live in providers/_registry.mjs so the portal
|
|
// health check (verify-portals.mjs) can reuse the exact same layer without
|
|
// importing this module.
|
|
|
|
// ── Title filter ────────────────────────────────────────────────────
|
|
|
|
// Compile a lowercased keyword into a matcher. Short all-letter acronyms
|
|
// (2-3 chars: cfo, coo, sdr, bdr, gsi…) match on WORD BOUNDARIES so "COO" no
|
|
// longer matches "Coordinator", "SDR" no longer matches anything mid-word, etc.
|
|
// Multi-word phrases and keywords containing non-letters (".NET", "SAP ",
|
|
// "L&D") keep fast, permissive substring matching.
|
|
export function compileKeyword(kw) {
|
|
if (/^[a-z]{2,3}$/.test(kw)) {
|
|
const re = new RegExp(`\\b${kw}\\b`);
|
|
return (lower) => re.test(lower);
|
|
}
|
|
return (lower) => lower.includes(kw);
|
|
}
|
|
|
|
export function buildTitleFilter(titleFilter) {
|
|
// Normalize defensively: a malformed title_filter (a null, numeric, or otherwise
|
|
// non-string entry in the YAML) must not crash the scan via k.toLowerCase().
|
|
const normalize = (arr) => (Array.isArray(arr) ? arr : [])
|
|
.filter(k => typeof k === 'string')
|
|
.map(k => k.trim().toLowerCase())
|
|
.filter(k => k.length > 0)
|
|
.map(compileKeyword);
|
|
const positive = normalize(titleFilter?.positive);
|
|
const negative = normalize(titleFilter?.negative);
|
|
|
|
return (title) => {
|
|
const lower = (title || '').toLowerCase();
|
|
const hasPositive = positive.length === 0 || positive.some(m => m(lower));
|
|
const hasNegative = negative.some(m => m(lower));
|
|
return hasPositive && !hasNegative;
|
|
};
|
|
}
|
|
|
|
// ── Location filter ─────────────────────────────────────────────────
|
|
// Optional. If `location_filter` is absent from portals.yml, all locations pass.
|
|
// Semantics (case-insensitive substring, in this order):
|
|
// - Empty / whitespace-only / non-string location → pass (don't penalize
|
|
// missing or malformed provider data)
|
|
// - `always_allow` matches → pass (takes precedence over `block` — lets a
|
|
// multi-location string like "Remote, Belgium or France" through because
|
|
// the home region is an option, even though "france" is blocked)
|
|
// - `block` matches → reject
|
|
// - `allow` empty → pass (already cleared block)
|
|
// - `allow` non-empty → must match at least one keyword
|
|
|
|
// Normalize a keyword list from portals.yml: tolerates a bare string
|
|
// (wrapped to a 1-item array), null/undefined (→ []), and non-string
|
|
// entries (filtered out). Survivors are lowercased, trimmed, and any
|
|
// resulting empty strings are dropped — an empty keyword would otherwise
|
|
// match every location via String.includes(''), silently bypassing the
|
|
// other tiers.
|
|
function normalizeKeywordList(value) {
|
|
if (value == null) return [];
|
|
const arr = Array.isArray(value) ? value : [value];
|
|
return arr
|
|
.filter(k => typeof k === 'string')
|
|
.map(k => k.toLowerCase().trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
export function buildLocationFilter(locationFilter) {
|
|
if (!locationFilter) return () => true;
|
|
const alwaysAllow = normalizeKeywordList(locationFilter.always_allow);
|
|
const allow = normalizeKeywordList(locationFilter.allow);
|
|
const block = normalizeKeywordList(locationFilter.block);
|
|
|
|
return (location) => {
|
|
if (typeof location !== 'string' || location.trim() === '') return true;
|
|
const lower = location.toLowerCase();
|
|
if (alwaysAllow.length > 0 && alwaysAllow.some(k => lower.includes(k))) return true;
|
|
if (block.length > 0 && block.some(k => lower.includes(k))) return false;
|
|
if (allow.length === 0) return true;
|
|
return allow.some(k => lower.includes(k));
|
|
};
|
|
}
|
|
|
|
// ── Content filter ──────────────────────────────────────────────────
|
|
// Optional. If `content_filter` is absent from portals.yml, all jobs pass.
|
|
// Filters on the job DESCRIPTION text to separate same-titled roles with
|
|
// different stacks (a "Software Engineer" listing that mentions "PHP" vs one
|
|
// that mentions "Rust"). Semantics (case-insensitive substring, in order):
|
|
// - Empty / whitespace-only / non-string description → PASS. The scanner is
|
|
// zero-token and only sees descriptions a provider already returns in its
|
|
// list payload; providers without one must never be silently dropped.
|
|
// - any `negative` keyword present → reject
|
|
// - `positive` empty → pass (already cleared negatives)
|
|
// - `positive` non-empty → at least one keyword must be present
|
|
//
|
|
// Provider support: only providers whose list API ships the description for
|
|
// free (no extra per-job request, which would break the zero-token design)
|
|
// populate `job.description`. Lever (`descriptionPlain`) does today; others
|
|
// leave it empty and therefore always pass this filter.
|
|
|
|
export function buildContentFilter(contentFilter) {
|
|
if (!contentFilter) return () => true;
|
|
const positive = normalizeKeywordList(contentFilter.positive);
|
|
const negative = normalizeKeywordList(contentFilter.negative);
|
|
|
|
return (description) => {
|
|
if (typeof description !== 'string' || description.trim() === '') return true;
|
|
const lower = description.toLowerCase();
|
|
if (negative.length > 0 && negative.some(k => lower.includes(k))) return false;
|
|
if (positive.length === 0) return true;
|
|
return positive.some(k => lower.includes(k));
|
|
};
|
|
}
|
|
|
|
// ── Salary filter ───────────────────────────────────────────────────
|
|
// Optional. If `salary_filter` is absent from portals.yml, all salaries pass.
|
|
// Semantics:
|
|
// - min/max are annual compensation filters (use annualized values)
|
|
// - max: 0 means "no upper limit"
|
|
// - If no salary data exists on a job, it passes (conservative behavior)
|
|
// - If both currencies are known and mismatch (e.g., USD filter, EUR job), it fails
|
|
// - Partial ranges (min only or max only) work correctly via overlap logic
|
|
// Uses null-safe checks (!= null, ??) to preserve 0 values correctly.
|
|
|
|
export function buildSalaryFilter(salaryFilter) {
|
|
if (!salaryFilter) return () => true;
|
|
|
|
// Coerce and validate bounds — malformed YAML must not silently mis-filter
|
|
const min = Number(salaryFilter.min ?? 0);
|
|
const max = Number(salaryFilter.max ?? 0);
|
|
const filterCurrency = (salaryFilter.currency || '').trim().toUpperCase();
|
|
|
|
if (!Number.isFinite(min) || !Number.isFinite(max) || min < 0 || max < 0) {
|
|
console.error('Warning: salary_filter.min/max must be non-negative numbers — salary filter disabled');
|
|
return () => true;
|
|
}
|
|
if (max > 0 && min > max) {
|
|
console.error('Warning: salary_filter.min cannot exceed salary_filter.max — salary filter disabled');
|
|
return () => true;
|
|
}
|
|
|
|
// If both min and max are 0, no filtering applied
|
|
if (min === 0 && max === 0) return () => true;
|
|
|
|
return (salary) => {
|
|
// If no salary data exists, pass (conservative - many providers don't expose salary)
|
|
if (!salary) return true;
|
|
|
|
const jobMin = salary.min ?? salary.max ?? null;
|
|
const jobMax = salary.max ?? salary.min ?? null;
|
|
|
|
// If we have no usable salary values, pass conservatively
|
|
if (jobMin == null && jobMax == null) return true;
|
|
|
|
// Currency handling - reject only if BOTH currencies exist and mismatch
|
|
const jobCurrency = (salary.currency || '').trim().toUpperCase();
|
|
if (filterCurrency && jobCurrency && filterCurrency !== jobCurrency) {
|
|
return false;
|
|
}
|
|
|
|
// Range overlap logic - reject ONLY if job is completely outside filter range
|
|
// Job entirely below user minimum
|
|
if (min > 0 && jobMax != null && jobMax < min) {
|
|
return false;
|
|
}
|
|
// Job entirely above user maximum
|
|
if (max > 0 && jobMin != null && jobMin > max) {
|
|
return false;
|
|
}
|
|
|
|
// Otherwise pass (overlap exists or no valid range to compare)
|
|
return true;
|
|
};
|
|
}
|
|
|
|
export function companyMatch(jobCompany, windowCompany) {
|
|
const cleanNoSpaces = (str) => String(str || '').toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
const c1NoSpaces = cleanNoSpaces(jobCompany);
|
|
const c2NoSpaces = cleanNoSpaces(windowCompany);
|
|
if (c1NoSpaces === c2NoSpaces) return true;
|
|
|
|
const cleanWithSpaces = (str) => String(str || '').toLowerCase().replace(/[^a-z0-9\s]/g, ' ').replace(/\s+/g, ' ').trim();
|
|
const c1WithSpaces = cleanWithSpaces(jobCompany);
|
|
const c2WithSpaces = cleanWithSpaces(windowCompany);
|
|
if (!c1WithSpaces || !c2WithSpaces) return false;
|
|
|
|
const regex1 = new RegExp('\\b' + c2WithSpaces.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&') + '\\b');
|
|
const regex2 = new RegExp('\\b' + c1WithSpaces.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&') + '\\b');
|
|
return regex1.test(c1WithSpaces) || regex2.test(c2WithSpaces);
|
|
}
|
|
|
|
export function addDays(dateStr, days) {
|
|
const date = new Date(`${dateStr}T00:00:00Z`);
|
|
date.setUTCDate(date.getUTCDate() + days);
|
|
return date.toISOString().slice(0, 10);
|
|
}
|
|
|
|
export function loadReApplyWindows(profilePath = PROFILE_PATH) {
|
|
if (!existsSync(profilePath)) return {};
|
|
try {
|
|
const raw = yaml.load(readFileSync(profilePath, 'utf-8')) || {};
|
|
const windows = raw.re_apply_windows || {};
|
|
const validWindows = {};
|
|
for (const [company, win] of Object.entries(windows)) {
|
|
if (!win || typeof win !== 'object') continue;
|
|
const lastApplyDate = win.last_apply_date;
|
|
if (typeof lastApplyDate !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(lastApplyDate)) continue;
|
|
if (isNaN(Date.parse(lastApplyDate))) continue;
|
|
|
|
const sameRoleDays = win.same_role_days;
|
|
if (sameRoleDays !== undefined && (!Number.isInteger(sameRoleDays) || sameRoleDays < 0)) continue;
|
|
|
|
if (win.applied_to !== undefined && !Array.isArray(win.applied_to)) continue;
|
|
if (win.applied_to !== undefined && win.applied_to.some(x => typeof x !== 'string')) continue;
|
|
|
|
if (win.cross_role_bucket !== undefined && typeof win.cross_role_bucket !== 'string') continue;
|
|
|
|
validWindows[company] = win;
|
|
}
|
|
return validWindows;
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
export function buildCooldownFilter(windows, today) {
|
|
if (!windows || Object.keys(windows).length === 0) {
|
|
return () => ({ skip: false });
|
|
}
|
|
|
|
const genericKeywords = new Set(['all', 'roles', 'role', 'family', 'bucket', 'group', 'team']);
|
|
|
|
return (job) => {
|
|
const jobCompany = job.company || '';
|
|
const jobTitleLower = (job.title || '').toLowerCase();
|
|
|
|
for (const [windowCompany, window] of Object.entries(windows)) {
|
|
if (companyMatch(jobCompany, windowCompany)) {
|
|
const lastApplyDate = window.last_apply_date;
|
|
const sameRoleDays = Number(window.same_role_days || 0);
|
|
if (!lastApplyDate) continue;
|
|
|
|
const cooldownUntil = addDays(lastApplyDate, sameRoleDays);
|
|
if (today >= cooldownUntil) {
|
|
continue;
|
|
}
|
|
|
|
if (Array.isArray(window.applied_to)) {
|
|
const matchesApplied = window.applied_to.some(role => {
|
|
const roleLower = role.toLowerCase();
|
|
return jobTitleLower.includes(roleLower);
|
|
});
|
|
if (matchesApplied) {
|
|
return { skip: true, reason: `cooldown:${windowCompany}:${cooldownUntil}`, cooldownUntil };
|
|
}
|
|
}
|
|
|
|
if (window.cross_role_bucket) {
|
|
const bucketKeywords = window.cross_role_bucket
|
|
.toLowerCase()
|
|
.split('_')
|
|
.filter(kw => kw && !genericKeywords.has(kw));
|
|
|
|
const matchesBucket = bucketKeywords.some(kw => {
|
|
if (kw === 'em') {
|
|
return /\bem\b/i.test(jobTitleLower) || jobTitleLower.includes('engineering manager');
|
|
}
|
|
return jobTitleLower.includes(kw);
|
|
});
|
|
|
|
if (matchesBucket) {
|
|
return { skip: true, reason: `cooldown:${windowCompany}:${cooldownUntil}`, cooldownUntil };
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return { skip: false };
|
|
};
|
|
}
|
|
|
|
|
|
// ── URL rediscovery (--rediscover-404) ──────────────────────────────
|
|
// When a tracked company's job URL returns 404/410, the role may have just
|
|
// moved to a new URL (Workday/Greenhouse rotate URLs without closing roles).
|
|
// These helpers back an opt-in search-and-reverify fallback before giving up.
|
|
|
|
// extractCareersUrlDomain returns the hostname of a company's careers_url, or
|
|
// null when it's missing/unparseable. The presence of a domain is what gates
|
|
// the fallback — broad-discovery offers without a careers_url stay ineligible.
|
|
export function extractCareersUrlDomain(careersUrl) {
|
|
if (!careersUrl) return null;
|
|
try {
|
|
return new URL(careersUrl).hostname;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// resolveSearchHref unwraps a DuckDuckGo HTML redirect (`/l/?uddg=<encoded>`)
|
|
// to its real destination, so domain matching sees the actual host instead of
|
|
// duckduckgo.com. Non-redirect hrefs pass through unchanged.
|
|
function resolveSearchHref(href) {
|
|
try {
|
|
const u = new URL(href, 'https://duckduckgo.com');
|
|
const isDdgHost = u.hostname === 'duckduckgo.com' || u.hostname.endsWith('.duckduckgo.com');
|
|
if (isDdgHost && u.pathname === '/l/') {
|
|
const target = u.searchParams.get('uddg');
|
|
if (target) return target;
|
|
}
|
|
} catch {
|
|
/* fall through to the raw href */
|
|
}
|
|
return href;
|
|
}
|
|
|
|
// pickRediscoveredUrl chooses the first result whose hostname *exactly* equals
|
|
// the careers domain (no substring/look-alike matches), unwrapping search-engine
|
|
// redirects first. Pure + exported so result-matching is unit-testable without
|
|
// driving a real browser. Returns null when nothing matches.
|
|
export function pickRediscoveredUrl(hrefs, domain) {
|
|
if (!domain || !Array.isArray(hrefs)) return null;
|
|
for (const raw of hrefs) {
|
|
const href = resolveSearchHref(raw);
|
|
let host;
|
|
try {
|
|
host = new URL(href).hostname;
|
|
} catch {
|
|
continue;
|
|
}
|
|
if (host === domain) return href;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// REDISCOVER_TIMEOUT_MS bounds the single fallback search so a slow or blocked
|
|
// search engine can't stall the sequential verify loop.
|
|
const REDISCOVER_TIMEOUT_MS = 10_000;
|
|
|
|
// searchForNewUrl runs one site-scoped search for a moved tracked role and
|
|
// returns a same-domain URL if found, else null. Every failure path returns
|
|
// null — the fallback must never throw into the verify loop. Leaves the page on
|
|
// a blank document so the next checkUrlLiveness call starts clean.
|
|
async function searchForNewUrl(page, offer) {
|
|
const domain = offer.careersUrlDomain;
|
|
if (!domain) return null;
|
|
const query = `"${offer.title}" "${offer.company}" site:${domain}`;
|
|
try {
|
|
await page.goto(
|
|
`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
|
|
{ waitUntil: 'domcontentloaded', timeout: REDISCOVER_TIMEOUT_MS },
|
|
);
|
|
const hrefs = await page.evaluate(() =>
|
|
Array.from(document.querySelectorAll('a.result__a'))
|
|
.map((a) => a.getAttribute('href'))
|
|
.filter(Boolean),
|
|
);
|
|
return pickRediscoveredUrl(hrefs, domain);
|
|
} catch {
|
|
return null;
|
|
} finally {
|
|
try {
|
|
await page.goto('about:blank');
|
|
} catch {
|
|
/* ignore — best-effort cleanup */
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Dedup ───────────────────────────────────────────────────────────
|
|
|
|
const PERMANENT_SCAN_HISTORY_STATUSES = new Set([
|
|
'skipped_invalid_url',
|
|
'skipped_blocked_host',
|
|
]);
|
|
|
|
function daysBetweenIsoDates(start, end) {
|
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(start) || !/^\d{4}-\d{2}-\d{2}$/.test(end)) return null;
|
|
const startDate = new Date(`${start}T00:00:00Z`);
|
|
const endDate = new Date(`${end}T00:00:00Z`);
|
|
if (startDate.toISOString().slice(0, 10) !== start || endDate.toISOString().slice(0, 10) !== end) return null;
|
|
return Math.floor((endDate - startDate) / (1000 * 60 * 60 * 24));
|
|
}
|
|
|
|
export function shouldDedupScanHistoryRow({ firstSeen, status = 'added' }, { recheckAfterDays = null, today = new Date().toISOString().slice(0, 10) } = {}) {
|
|
if (PERMANENT_SCAN_HISTORY_STATUSES.has(status)) return true;
|
|
if (status.startsWith('cooldown:')) {
|
|
const parts = status.split(':');
|
|
const cooldownUntil = parts[parts.length - 1];
|
|
return today < cooldownUntil;
|
|
}
|
|
if (status !== 'added') return true;
|
|
if (recheckAfterDays == null) return true;
|
|
const ageDays = daysBetweenIsoDates(firstSeen, today);
|
|
if (ageDays == null) return true;
|
|
return ageDays < recheckAfterDays;
|
|
}
|
|
|
|
function scanHistoryPolicy(config = {}) {
|
|
const raw = config.scan_history?.recheck_after_days;
|
|
const parsed = Number.parseInt(raw, 10);
|
|
return {
|
|
recheckAfterDays: Number.isFinite(parsed) && parsed >= 0 ? parsed : null,
|
|
};
|
|
}
|
|
|
|
export function loadSeenUrls(policy = {}) {
|
|
const seen = new Set();
|
|
let recheckEligible = 0;
|
|
|
|
// scan-history.tsv
|
|
if (existsSync(SCAN_HISTORY_PATH)) {
|
|
const lines = readFileSync(SCAN_HISTORY_PATH, 'utf-8').split('\n');
|
|
for (const line of lines.slice(1)) { // skip header
|
|
const [url, firstSeen, , , , status = 'added'] = line.split('\t');
|
|
if (!url) continue;
|
|
if (shouldDedupScanHistoryRow({ firstSeen, status }, policy)) seen.add(url);
|
|
else recheckEligible++;
|
|
}
|
|
}
|
|
|
|
// pipeline.md — extract URLs from checkbox lines
|
|
if (existsSync(PIPELINE_PATH)) {
|
|
const text = readFileSync(PIPELINE_PATH, 'utf-8');
|
|
for (const match of text.matchAll(/- \[[ x]\] (https?:\/\/\S+)/g)) {
|
|
seen.add(match[1]);
|
|
}
|
|
}
|
|
|
|
// applications.md — extract URLs from report links and any inline URLs
|
|
if (existsSync(APPLICATIONS_PATH)) {
|
|
const text = readFileSync(APPLICATIONS_PATH, 'utf-8');
|
|
for (const match of text.matchAll(/https?:\/\/[^\s|)]+/g)) {
|
|
seen.add(match[0]);
|
|
}
|
|
}
|
|
|
|
return { seen, recheckEligible };
|
|
}
|
|
|
|
function loadSeenCompanyRoles() {
|
|
const seen = new Set();
|
|
if (existsSync(APPLICATIONS_PATH)) {
|
|
const text = readFileSync(APPLICATIONS_PATH, 'utf-8');
|
|
// Parse markdown table rows: | # | Date | Company | Role | ...
|
|
for (const match of text.matchAll(/\|[^|]+\|[^|]+\|\s*([^|]+)\s*\|\s*([^|]+)\s*\|/g)) {
|
|
const company = match[1].trim().toLowerCase();
|
|
const role = match[2].trim().toLowerCase();
|
|
if (company && role && company !== 'company') {
|
|
seen.add(`${company}::${role}`);
|
|
}
|
|
}
|
|
}
|
|
return seen;
|
|
}
|
|
|
|
// ── Pipeline writer ─────────────────────────────────────────────────
|
|
|
|
function normalizeScanScalar(value) {
|
|
return String(value ?? '')
|
|
.replace(/[\r\n\t]+/g, ' ')
|
|
.replace(/ {2,}/g, ' ')
|
|
.trim();
|
|
}
|
|
|
|
function normalizeScanUrl(value) {
|
|
return String(value ?? '').trim().split(/\s+/)[0] || '';
|
|
}
|
|
|
|
const MARKDOWN_ESCAPE_CHARS = {
|
|
'\\': '\\\\',
|
|
'[': '\\[',
|
|
']': '\\]',
|
|
};
|
|
|
|
export function sanitizeMarkdownField(value) {
|
|
return normalizeScanScalar(value)
|
|
.replace(/[\\[\]]/g, char => MARKDOWN_ESCAPE_CHARS[char])
|
|
.replace(/\|/g, '/');
|
|
}
|
|
|
|
function sanitizePipelineUrl(value) {
|
|
return normalizeScanUrl(value)
|
|
.replace(/[\\[\]]/g, char => MARKDOWN_ESCAPE_CHARS[char])
|
|
.replace(/\|/g, '%7C');
|
|
}
|
|
|
|
export function sanitizeTsvField(value) {
|
|
const normalized = normalizeScanScalar(value);
|
|
return /^[=+\-@]/.test(normalized) ? `'${normalized}` : normalized;
|
|
}
|
|
|
|
// Format an offer's parsed compensation (the annualized {min,max,currency} that
|
|
// providers like Ashby attach as `offer.salary`) into a compact, sanitized cell
|
|
// such as `120000-160000 USD`. Returns '' when there is no usable salary data.
|
|
// Non-positive bounds are dropped (a 0 min/max is meaningless comp data, not "$0").
|
|
export function formatCompensation(salary) {
|
|
if (!salary || typeof salary !== 'object') return '';
|
|
const num = (n) => (Number.isFinite(n) && n > 0 ? String(Math.round(n)) : null);
|
|
const lo = num(salary.min);
|
|
const hi = num(salary.max);
|
|
const range = lo && hi && lo !== hi ? `${lo}-${hi}` : (lo || hi || '');
|
|
if (!range) return '';
|
|
const currency = typeof salary.currency === 'string' ? salary.currency.trim() : '';
|
|
return sanitizeMarkdownField(currency ? `${range} ${currency}` : range);
|
|
}
|
|
|
|
export function formatPipelineOffer(offer) {
|
|
const url = sanitizePipelineUrl(offer.url);
|
|
const company = sanitizeMarkdownField(offer.company);
|
|
const title = sanitizeMarkdownField(offer.title);
|
|
// Optional trailing columns, each sanitized like every other field:
|
|
// 4th = location, 5th = compensation.
|
|
// Gate location on an actual string so malformed provider data (a number or
|
|
// object) degrades to the 3-column form instead of stringifying into a
|
|
// spurious column. The columns are positional, so a present compensation
|
|
// forces the (possibly empty) location cell to keep comp in column 5.
|
|
// loadSeenUrls dedups on the URL and ignores trailing columns (backward-compatible).
|
|
const location = typeof offer.location === 'string' ? sanitizeMarkdownField(offer.location) : '';
|
|
const compensation = formatCompensation(offer.salary);
|
|
const base = `- [ ] ${url} | ${company} | ${title}`;
|
|
let line = base;
|
|
if (compensation) line = `${base} | ${location} | ${compensation}`;
|
|
else if (location) line = `${base} | ${location}`;
|
|
// Optional free-text ranking signal (e.g. a curated-list flag an importer
|
|
// attaches). Labeled — not positional like location/compensation — so it can
|
|
// ride on any row shape (bare URL, 3-, 4-, or 5-column) without a reader
|
|
// confusing it for a positional cell, and it stays generic: nothing here is
|
|
// source-specific, and an offer without `note` produces byte-identical output.
|
|
const note = typeof offer.note === 'string' ? sanitizeMarkdownField(offer.note) : '';
|
|
return note ? `${line} | note: ${note}` : line;
|
|
}
|
|
|
|
export function formatScanHistoryRow(offer, date, status = 'added') {
|
|
return [
|
|
normalizeScanUrl(offer.url),
|
|
date,
|
|
offer.source,
|
|
offer.title,
|
|
offer.company,
|
|
status,
|
|
offer.location || '',
|
|
].map(sanitizeTsvField).join('\t');
|
|
}
|
|
|
|
// Standard skeleton created on fresh install — matches the format documented
|
|
// in modes/pipeline.md and expected by /career-ops pipeline.
|
|
const PIPELINE_SKELETON = `# Pipeline — Pending URLs
|
|
|
|
Paste job URLs below as \`- [ ] {url}\` then run \`/career-ops pipeline\`.
|
|
|
|
## Pending
|
|
|
|
## Processed
|
|
`;
|
|
|
|
// Current section names (English). Legacy Spanish names are checked as fallback
|
|
// so existing pipeline.md files created before this change keep working.
|
|
const PENDING_MARKERS = ['## Pending', '## Pendientes'];
|
|
const PROCESSED_MARKERS = ['## Processed', '## Procesadas'];
|
|
|
|
export function appendToPipeline(offers) {
|
|
if (offers.length === 0) return;
|
|
|
|
// Auto-create with standard skeleton if missing (fresh-install guard).
|
|
if (!existsSync(PIPELINE_PATH)) {
|
|
writeFileSync(PIPELINE_PATH, PIPELINE_SKELETON, 'utf-8');
|
|
}
|
|
|
|
let text = readFileSync(PIPELINE_PATH, 'utf-8');
|
|
|
|
const marker = PENDING_MARKERS.find(m => text.includes(m)) ?? null;
|
|
const idx = marker !== null ? text.indexOf(marker) : -1;
|
|
|
|
if (idx === -1) {
|
|
// No Pending section found — insert one before Processed (or at end)
|
|
const procIdx = PROCESSED_MARKERS.reduce((found, m) => {
|
|
const i = text.indexOf(m);
|
|
return (found === -1 || (i !== -1 && i < found)) ? i : found;
|
|
}, -1);
|
|
const insertAt = procIdx === -1 ? text.length : procIdx;
|
|
const block = `\n## Pending\n\n` + offers.map(formatPipelineOffer).join('\n') + '\n\n';
|
|
text = text.slice(0, insertAt) + block + text.slice(insertAt);
|
|
} else {
|
|
// Find the end of existing Pending content (next ## or end)
|
|
const afterMarker = idx + marker.length;
|
|
const nextSection = text.indexOf('\n## ', afterMarker);
|
|
const insertAt = nextSection === -1 ? text.length : nextSection;
|
|
|
|
const block = '\n' + offers.map(formatPipelineOffer).join('\n') + '\n';
|
|
text = text.slice(0, insertAt) + block + text.slice(insertAt);
|
|
}
|
|
|
|
writeFileSync(PIPELINE_PATH, text, 'utf-8');
|
|
}
|
|
|
|
export function appendToScanHistory(offers, date, status = 'added') {
|
|
// Ensure file + header exist. Location appended as 7th column for non-breaking
|
|
// backward compat — older scan-history.tsv files with 6 columns still parse fine
|
|
// since loadSeenUrls only reads column 0. `status` is parameterized so callers
|
|
// can record verify outcomes (`skipped_expired`, etc.) without the legacy
|
|
// `(expired)` suffix in `source`.
|
|
if (!existsSync(SCAN_HISTORY_PATH)) {
|
|
writeFileSync(SCAN_HISTORY_PATH, 'url\tfirst_seen\tportal\ttitle\tcompany\tstatus\tlocation\n', 'utf-8');
|
|
}
|
|
|
|
const lines = offers.map(o => formatScanHistoryRow(o, date, status)).join('\n') + '\n';
|
|
|
|
appendFileSync(SCAN_HISTORY_PATH, lines, 'utf-8');
|
|
}
|
|
|
|
// ── Parallel fetch with concurrency limit ───────────────────────────
|
|
|
|
async function parallelFetch(tasks, limit) {
|
|
const results = [];
|
|
let i = 0;
|
|
|
|
async function next() {
|
|
while (i < tasks.length) {
|
|
const task = tasks[i++];
|
|
results.push(await task());
|
|
}
|
|
}
|
|
|
|
const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => next());
|
|
await Promise.all(workers);
|
|
return results;
|
|
}
|
|
|
|
// ── Main ────────────────────────────────────────────────────────────
|
|
|
|
async function verifyOffers(offers, { headedFallback = false, throttleBaseMs = 0, rediscover = false } = {}) {
|
|
// Dynamic imports keep the default zero-token path free of Playwright startup
|
|
let chromium;
|
|
let checkUrlLiveness;
|
|
let checkUrlLivenessWithFallback;
|
|
let createHeadedPageProvider;
|
|
let newLivenessPage;
|
|
let jitteredDelayMs;
|
|
let sleep;
|
|
try {
|
|
({ chromium } = await import('playwright'));
|
|
({ checkUrlLiveness, checkUrlLivenessWithFallback, createHeadedPageProvider, newLivenessPage, jitteredDelayMs, sleep } = await import('./liveness-browser.mjs'));
|
|
} catch (err) {
|
|
throw new Error(
|
|
`--verify requires Playwright with Chromium (run "npx playwright install chromium"): ${err.message}`,
|
|
{ cause: err },
|
|
);
|
|
}
|
|
|
|
let browser;
|
|
try {
|
|
browser = await chromium.launch({ headless: true });
|
|
} catch (err) {
|
|
throw new Error(
|
|
`--verify could not launch Chromium (run "npx playwright install chromium" or re-run without --verify): ${err.message}`,
|
|
{ cause: err },
|
|
);
|
|
}
|
|
|
|
// Three permanent buckets + one transient passthrough:
|
|
// verified → active pages and transient nav errors (retry next scan)
|
|
// expired → classifier-confirmed dead postings (HTTP 4xx, redirect markers,
|
|
// body patterns, listing pages, insufficient content)
|
|
// dropped → page loaded but classifier saw no Apply control. --verify is an
|
|
// opt-in stricter filter; keeping these defeats the purpose.
|
|
// invalid → up-front URL guard rejections (malformed / non-http / private)
|
|
const verified = [];
|
|
const expired = [];
|
|
const dropped = [];
|
|
const invalid = [];
|
|
const migrated = [];
|
|
|
|
const headed = headedFallback ? createHeadedPageProvider(chromium) : null;
|
|
const getHeadedPage = headed ? () => headed.get() : undefined;
|
|
|
|
try {
|
|
const page = await newLivenessPage(browser);
|
|
// Sequential — project rule: never Playwright in parallel
|
|
for (let i = 0; i < offers.length; i++) {
|
|
const offer = offers[i];
|
|
const { result, code, reason } = headed
|
|
? await checkUrlLivenessWithFallback(page, offer.url, { getHeadedPage })
|
|
: await checkUrlLiveness(page, offer.url);
|
|
if (result === 'expired') {
|
|
// 404/410 on a tracked company may just be a moved role — run one
|
|
// search + re-verify before giving up (opt-in via --rediscover-404).
|
|
// Only http_gone (HTTP 404/410) qualifies; soft-expiry signals
|
|
// (redirect/body/listing) are real closures, not URL moves.
|
|
if (rediscover && code === 'http_gone' && offer.tracked && offer.careersUrlDomain) {
|
|
const newUrl = await searchForNewUrl(page, offer);
|
|
if (newUrl) {
|
|
// Mirror the primary check: without the headed fallback, a
|
|
// challenge-prone domain would flag the rediscovered URL as
|
|
// expired just because the recheck hit the same anti-bot wall.
|
|
const recheck = headed
|
|
? await checkUrlLivenessWithFallback(page, newUrl, { getHeadedPage })
|
|
: await checkUrlLiveness(page, newUrl);
|
|
// Require a *confirmed* live page before migrating. A transient
|
|
// 'uncertain' (timeout/DNS/5xx) must not commit an unverified URL —
|
|
// fall through to expired (the original 404/410 is a real closure).
|
|
if (recheck.result === 'active') {
|
|
migrated.push({ ...offer, url: newUrl, previousUrl: offer.url });
|
|
console.log(` 🔄 migrated ${offer.company} | ${offer.title} → ${newUrl}`);
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
expired.push({ ...offer, reason });
|
|
console.log(` ❌ expired ${offer.company} | ${offer.title} (${reason})`);
|
|
} else if (result === 'uncertain' && GUARD_CODES.has(code)) {
|
|
// Guard failures are permanent (not transient like a timeout) — record them
|
|
// separately so they don't end up in pipeline.md but DO appear in scan-history
|
|
// with a precise status, dedup-blocking them on subsequent scans.
|
|
invalid.push({ ...offer, code, reason });
|
|
console.log(` ⛔ invalid ${offer.company} | ${offer.title} (${reason})`);
|
|
} else if (result === 'uncertain' && code === 'no_apply_control') {
|
|
// Page loaded but classifier could not find an Apply control. Treat like
|
|
// expired for routing — drop from pipeline AND record in scan-history so
|
|
// we don't burn a verify cycle on the same URL next scan.
|
|
dropped.push({ ...offer, reason });
|
|
console.log(` ⚠️ no-apply ${offer.company} | ${offer.title} (${reason})`);
|
|
} else {
|
|
// 'active' or 'uncertain' due to navigation_error (transient — retry next scan)
|
|
verified.push(offer);
|
|
const icon = result === 'active' ? '✅' : '⚠️';
|
|
console.log(` ${icon} ${result.padEnd(9)} ${offer.company} | ${offer.title}`);
|
|
}
|
|
|
|
const wait = i < offers.length - 1 ? jitteredDelayMs(throttleBaseMs) : 0;
|
|
if (wait) await sleep(wait);
|
|
}
|
|
} finally {
|
|
if (headed) await headed.close();
|
|
await browser.close();
|
|
}
|
|
|
|
return { verified, expired, dropped, invalid, migrated };
|
|
}
|
|
|
|
// Stable codes from liveness-browser's up-front URL guard. Routing dispatches
|
|
// on these codes (not on regex over reason strings) so wording can change
|
|
// without breaking the pipeline.
|
|
const GUARD_CODES = new Set(['invalid_url', 'unsupported_protocol', 'blocked_host']);
|
|
|
|
// guardStatusFor maps a guard code to the canonical scan-history status string.
|
|
function guardStatusFor(code) {
|
|
if (code === 'blocked_host') return 'skipped_blocked_host';
|
|
// invalid_url and unsupported_protocol both surface as malformed input
|
|
return 'skipped_invalid_url';
|
|
}
|
|
|
|
async function main() {
|
|
const args = process.argv.slice(2);
|
|
const dryRun = args.includes('--dry-run');
|
|
const verify = args.includes('--verify');
|
|
// Opt-in: on an anti-bot challenge (e.g. pracuj.pl Cloudflare wall), retry the
|
|
// URL in a headed browser. Off by default — headed Chromium needs a display, so
|
|
// scheduled/unattended scans should not rely on it.
|
|
const headedFallback = args.includes('--headed-fallback');
|
|
// --throttle or --throttle=<ms>: jittered gap between --verify checks to stay
|
|
// under rate-based WAF limits (pracuj.pl flags the session after a few rapid
|
|
// hits). Default base 5000ms. Off by default — most ATS feeds don't need it.
|
|
const throttleArg = args.find((a) => a === '--throttle' || a.startsWith('--throttle='));
|
|
const throttleBaseMs = throttleArg ? (Number(throttleArg.split('=')[1]) || 5000) : 0;
|
|
// --rediscover-404: when a tracked company's URL 404/410s, search for the
|
|
// moved role and re-verify before marking it expired. Opt-in; rides on --verify.
|
|
const rediscover = args.includes('--rediscover-404');
|
|
const companyFlag = args.indexOf('--company');
|
|
const filterCompany = companyFlag !== -1 ? args[companyFlag + 1]?.toLowerCase() : null;
|
|
|
|
// 1. Load providers
|
|
const providers = await loadProviders(PROVIDERS_DIR);
|
|
// Opt-in: merge enabled keyed/auth-gated provider plugins. Returns immediately
|
|
// (no discovery, no dotenv, no process.env mutation) when config/plugins.yml is
|
|
// absent — so a plain scan with no plugins configured stays byte-identical.
|
|
await mergeProviderPlugins(providers, { root: path.dirname(PROVIDERS_DIR) });
|
|
if (providers.size === 0) {
|
|
console.error('Error: no providers loaded from providers/');
|
|
process.exit(1);
|
|
}
|
|
|
|
// 2. Read portals.yml
|
|
if (!existsSync(PORTALS_PATH)) {
|
|
console.error('Error: portals.yml not found. Run onboarding first.');
|
|
process.exit(1);
|
|
}
|
|
|
|
let rawConfig;
|
|
try {
|
|
rawConfig = parseYaml(readFileSync(PORTALS_PATH, 'utf-8'));
|
|
} catch (err) {
|
|
console.error(`Error: failed to parse ${PORTALS_PATH}: ${err.message}`);
|
|
process.exit(1);
|
|
}
|
|
const config = rawConfig && typeof rawConfig === 'object' ? rawConfig : {};
|
|
const companies = Array.isArray(config.tracked_companies) ? config.tracked_companies : [];
|
|
const boards = Array.isArray(config.job_boards) ? config.job_boards : [];
|
|
const titleFilter = buildTitleFilter(config.title_filter);
|
|
|
|
// Seniority tier classifier integration
|
|
let classifyTier = null;
|
|
const skipTiers = Array.isArray(config.skip_tiers)
|
|
? config.skip_tiers.filter(t => typeof t === 'string').map(t => t.toLowerCase())
|
|
: [];
|
|
if (skipTiers.length > 0) {
|
|
const mod = await import('./classify-tier.mjs');
|
|
classifyTier = mod.classifyTier || mod.default;
|
|
}
|
|
|
|
const locationFilter = buildLocationFilter(config.location_filter);
|
|
const salaryFilter = buildSalaryFilter(config.salary_filter);
|
|
const trustValidator = buildTrustValidator(config.trust_filter);
|
|
const contentFilter = buildContentFilter(config.content_filter);
|
|
|
|
// 3. Resolve a provider for each enabled company / board
|
|
const targets = [];
|
|
let skippedCount = 0;
|
|
let boardCount = 0;
|
|
const resolveErrors = [];
|
|
const agentHandoff = [];
|
|
|
|
/**
|
|
* Processes a list of configuration entries, resolves their appropriate data providers,
|
|
* and appends valid entries to the global scanning targets list.
|
|
* @param {Array<{ name?: string, enabled?: boolean, [key: string]: unknown }>} entries - List of entries.
|
|
* @param {{ isBoard?: boolean }} [options={}] - Configuration options.
|
|
*/
|
|
function resolveEntries(entries, { isBoard = false } = {}) {
|
|
for (const entry of entries) {
|
|
if (!entry || typeof entry !== 'object') continue;
|
|
if (entry.enabled === false) continue;
|
|
if (typeof entry.name !== 'string' || !entry.name.trim()) {
|
|
console.error(`⚠️ Skipping entry — missing or non-string 'name' field: ${JSON.stringify(entry)}`);
|
|
continue;
|
|
}
|
|
if (filterCompany && !entry.name.toLowerCase().includes(filterCompany)) continue;
|
|
|
|
const resolved = resolveProvider(entry, providers);
|
|
if (!resolved) {
|
|
skippedCount++;
|
|
if (entry.scan_method === 'websearch') {
|
|
agentHandoff.push({
|
|
company: entry.name,
|
|
method: 'websearch',
|
|
query: entry.scan_query || entry.search_query || entry.careers_url || '',
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (resolved.error) {
|
|
resolveErrors.push({ company: entry.name, error: resolved.error });
|
|
continue;
|
|
}
|
|
|
|
targets.push({ ...entry, _provider: resolved.provider, _isBoard: isBoard });
|
|
if (isBoard) boardCount++;
|
|
}
|
|
}
|
|
|
|
resolveEntries(companies);
|
|
resolveEntries(boards, { isBoard: true });
|
|
|
|
const localParserCount = targets.filter(t => t._provider.id === 'local-parser').length;
|
|
const companyCount = targets.length - boardCount;
|
|
const parts = [`${companyCount} companies`];
|
|
if (boardCount > 0) parts.push(`${boardCount} job boards`);
|
|
parts.push(`${localParserCount} local parser`);
|
|
parts.push(`${skippedCount} skipped — no provider matched`);
|
|
console.log(`Scanning ${parts.join('; ')} via providers`);
|
|
if (dryRun) console.log('(dry run — no files will be written)\n');
|
|
|
|
// 4. Load dedup sets
|
|
const historyPolicy = scanHistoryPolicy(config);
|
|
const seenUrlState = loadSeenUrls(historyPolicy);
|
|
const seenUrls = seenUrlState.seen;
|
|
const seenCompanyRoles = loadSeenCompanyRoles();
|
|
|
|
// 5. Fetch from each target
|
|
const date = new Date().toISOString().slice(0, 10);
|
|
const windows = loadReApplyWindows();
|
|
const cooldownFilter = buildCooldownFilter(windows, date);
|
|
let totalFilteredCooldown = 0;
|
|
const cooldownOffers = [];
|
|
let totalFound = 0;
|
|
let totalFilteredTitle = 0;
|
|
let totalFilteredTier = 0;
|
|
let totalFilteredLocation = 0;
|
|
let totalFilteredSalary = 0;
|
|
let totalFilteredContent = 0;
|
|
let totalDupes = 0;
|
|
const newOffers = [];
|
|
const errors = [...resolveErrors];
|
|
const emptyTargets = [];
|
|
|
|
const tasks = targets.map(company => async () => {
|
|
let provider = company._provider;
|
|
const ctx = makeHttpCtx();
|
|
let sourceName = provider.id === 'local-parser' ? 'local-parser' : `${provider.id}-api`;
|
|
try {
|
|
let jobs;
|
|
try {
|
|
jobs = await provider.fetch(company, ctx);
|
|
} catch (parserErr) {
|
|
if (provider.id !== 'local-parser') throw parserErr;
|
|
const fallback = resolveProvider(company, providers, { skipIds: ['local-parser'] });
|
|
if (!fallback || fallback.error) throw parserErr;
|
|
provider = fallback.provider;
|
|
sourceName = `${provider.id}-api`;
|
|
jobs = await provider.fetch(company, ctx);
|
|
errors.push({
|
|
company: company.name,
|
|
error: `local parser failed, used API fallback: ${parserErr.message}`,
|
|
});
|
|
}
|
|
if (!Array.isArray(jobs)) {
|
|
throw new Error(`${provider.id}: fetch() did not return an array`);
|
|
}
|
|
totalFound += jobs.length;
|
|
if (!company._isBoard && jobs.length === 0) {
|
|
emptyTargets.push(company.name);
|
|
}
|
|
|
|
for (const job of jobs) {
|
|
// Trust enrichment — runs before filters, never drops
|
|
const trustResult = trustValidator(job);
|
|
job.trustScore = trustResult.score;
|
|
job.trustFlags = trustResult.flags;
|
|
job.trustLevel = trustResult.level;
|
|
|
|
if (!titleFilter(job.title)) {
|
|
totalFilteredTitle++;
|
|
continue;
|
|
}
|
|
if (classifyTier && skipTiers.includes(classifyTier(job.title))) {
|
|
totalFilteredTier++;
|
|
continue;
|
|
}
|
|
if (!locationFilter(job.location)) {
|
|
totalFilteredLocation++;
|
|
continue;
|
|
}
|
|
if (!salaryFilter(job.salary)) {
|
|
totalFilteredSalary++;
|
|
continue;
|
|
}
|
|
if (!contentFilter(job.description)) {
|
|
totalFilteredContent++;
|
|
continue;
|
|
}
|
|
if (seenUrls.has(job.url)) {
|
|
totalDupes++;
|
|
continue;
|
|
}
|
|
const key = `${job.company.toLowerCase()}::${job.title.toLowerCase()}`;
|
|
if (seenCompanyRoles.has(key)) {
|
|
totalDupes++;
|
|
continue;
|
|
}
|
|
const cooldownResult = cooldownFilter(job);
|
|
if (cooldownResult.skip) {
|
|
totalFilteredCooldown++;
|
|
cooldownOffers.push({
|
|
job: { ...job, source: sourceName },
|
|
status: cooldownResult.reason,
|
|
});
|
|
continue;
|
|
}
|
|
// Mark as seen to avoid intra-scan dupes
|
|
seenUrls.add(job.url);
|
|
seenCompanyRoles.add(key);
|
|
// Tag with the company's careers domain so verify can offer a 404/410
|
|
// rediscovery fallback. A null domain (no careers_url) marks the offer
|
|
// as broad-discovery — ineligible for the fallback, per the issue scope.
|
|
const careersUrlDomain = extractCareersUrlDomain(company.careers_url);
|
|
newOffers.push({
|
|
...job,
|
|
source: sourceName,
|
|
tracked: Boolean(careersUrlDomain),
|
|
careersUrlDomain,
|
|
});
|
|
}
|
|
} catch (err) {
|
|
errors.push({
|
|
company: company.name,
|
|
error: err.message,
|
|
kind: classifyFetchError(err),
|
|
});
|
|
}
|
|
});
|
|
|
|
await parallelFetch(tasks, CONCURRENCY);
|
|
|
|
// 5.5. Optional liveness verification — drop expired and guard-rejected postings
|
|
let verifiedOffers = newOffers;
|
|
let expiredOffers = [];
|
|
let droppedOffers = [];
|
|
let invalidOffers = [];
|
|
let migratedOffers = [];
|
|
if (verify && newOffers.length > 0) {
|
|
console.log(`\nVerifying liveness of ${newOffers.length} new offer(s) with Playwright (sequential)...`);
|
|
const result = await verifyOffers(newOffers, { headedFallback, throttleBaseMs, rediscover });
|
|
verifiedOffers = result.verified;
|
|
expiredOffers = result.expired;
|
|
droppedOffers = result.dropped;
|
|
invalidOffers = result.invalid;
|
|
migratedOffers = result.migrated;
|
|
// Migrated offers re-enter the pipeline at their newly discovered URL.
|
|
if (migratedOffers.length > 0) {
|
|
verifiedOffers = [...verifiedOffers, ...migratedOffers];
|
|
}
|
|
}
|
|
|
|
// 6. Write results
|
|
if (!dryRun && verifiedOffers.length > 0) {
|
|
appendToPipeline(verifiedOffers);
|
|
appendToScanHistory(verifiedOffers, date);
|
|
}
|
|
if (!dryRun && cooldownOffers.length > 0) {
|
|
const cooldownGroups = {};
|
|
for (const item of cooldownOffers) {
|
|
if (!cooldownGroups[item.status]) {
|
|
cooldownGroups[item.status] = [];
|
|
}
|
|
cooldownGroups[item.status].push(item.job);
|
|
}
|
|
for (const [status, group] of Object.entries(cooldownGroups)) {
|
|
appendToScanHistory(group, date, status);
|
|
}
|
|
}
|
|
// Expired postings — plus the old URLs of migrated offers — are recorded as
|
|
// skipped_expired so subsequent scans dedup-skip the dead URLs.
|
|
const expiredForHistory = [
|
|
...expiredOffers,
|
|
...migratedOffers.map(o => ({ ...o, url: o.previousUrl })),
|
|
];
|
|
if (!dryRun && expiredForHistory.length > 0) {
|
|
appendToScanHistory(expiredForHistory, date, 'skipped_expired');
|
|
}
|
|
// Pages that loaded but had no Apply control: record so we don't re-verify
|
|
// them next scan, but never let them reach pipeline.md.
|
|
if (!dryRun && droppedOffers.length > 0) {
|
|
appendToScanHistory(droppedOffers, date, 'skipped_no_apply_control');
|
|
}
|
|
// Guard-rejected URLs (invalid / unsupported protocol / blocked host) are
|
|
// recorded with a precise status so subsequent scans dedup-skip them via
|
|
// loadSeenUrls, but they never reach pipeline.md.
|
|
if (!dryRun && invalidOffers.length > 0) {
|
|
// Group by code so the TSV reflects the actual reason category.
|
|
const byStatus = new Map();
|
|
for (const o of invalidOffers) {
|
|
const status = guardStatusFor(o.code);
|
|
if (!byStatus.has(status)) byStatus.set(status, []);
|
|
byStatus.get(status).push(o);
|
|
}
|
|
for (const [status, group] of byStatus) {
|
|
appendToScanHistory(group, date, status);
|
|
}
|
|
}
|
|
|
|
// 7. Print summary
|
|
console.log(`\n${'━'.repeat(45)}`);
|
|
console.log(`Portal Scan — ${date}`);
|
|
console.log(`${'━'.repeat(45)}`);
|
|
const summaryCompanies = targets.filter(t => !t._isBoard).length;
|
|
const summaryBoards = targets.filter(t => t._isBoard).length;
|
|
console.log(`Companies scanned: ${summaryCompanies}`);
|
|
if (summaryBoards > 0) console.log(`Job boards scanned: ${summaryBoards}`);
|
|
console.log(`Total jobs found: ${totalFound}`);
|
|
console.log(`Filtered by title: ${totalFilteredTitle} removed`);
|
|
if (skipTiers.length > 0) {
|
|
console.log(`Filtered by tier: ${totalFilteredTier} removed`);
|
|
}
|
|
console.log(`Filtered by location: ${totalFilteredLocation} removed`);
|
|
console.log(`Filtered by salary: ${totalFilteredSalary} removed`);
|
|
console.log(`Filtered by content: ${totalFilteredContent} removed`);
|
|
if (Object.keys(windows).length > 0 || totalFilteredCooldown > 0) {
|
|
console.log(`Filtered by cooldown: ${totalFilteredCooldown} removed`);
|
|
}
|
|
console.log(`Duplicates: ${totalDupes} skipped`);
|
|
if (historyPolicy.recheckAfterDays != null) {
|
|
console.log(`Recheck eligible: ${seenUrlState.recheckEligible} old scan-history URL(s)`);
|
|
}
|
|
if (verify) {
|
|
console.log(`Expired (verified): ${expiredOffers.length} dropped`);
|
|
console.log(`Rediscovered (moved): ${migratedOffers.length} migrated`);
|
|
console.log(`No apply control: ${droppedOffers.length} dropped`);
|
|
console.log(`Invalid (guarded): ${invalidOffers.length} dropped`);
|
|
}
|
|
console.log(`New offers added: ${verifiedOffers.length}`);
|
|
|
|
// Trust validation summary (only when trust_filter is configured)
|
|
if (config.trust_filter && config.trust_filter.enabled !== false && verifiedOffers.length > 0) {
|
|
const trustHigh = verifiedOffers.filter(o => o.trustLevel === 'high').length;
|
|
const trustMedium = verifiedOffers.filter(o => o.trustLevel === 'medium').length;
|
|
const trustLow = verifiedOffers.filter(o => o.trustLevel === 'low').length;
|
|
console.log(`Trust validation: ${trustHigh} high, ${trustMedium} medium, ${trustLow} low`);
|
|
// Flag breakdown
|
|
/** @type {Record<string, number>} */
|
|
const flagCounts = {};
|
|
for (const o of verifiedOffers) {
|
|
for (const f of (o.trustFlags || [])) {
|
|
flagCounts[f] = (flagCounts[f] || 0) + 1;
|
|
}
|
|
}
|
|
if (Object.keys(flagCounts).length > 0) {
|
|
const parts = Object.entries(flagCounts).map(([k, v]) => `${k}: ${v}`);
|
|
console.log(`Trust flags: ${parts.join(', ')}`);
|
|
}
|
|
}
|
|
|
|
if (agentHandoff.length > 0) {
|
|
console.log(`Agent/WebSearch handoff: ${agentHandoff.length} compan${agentHandoff.length === 1 ? 'y' : 'ies'} not handled by zero-token providers`);
|
|
for (const item of agentHandoff.slice(0, 25)) {
|
|
const hint = item.query ? ` — ${item.query}` : '';
|
|
console.log(` • ${item.company} (${item.method})${hint}`);
|
|
}
|
|
if (agentHandoff.length > 25) {
|
|
console.log(` … ${agentHandoff.length - 25} more omitted; narrow with --company or inspect portals.yml`);
|
|
}
|
|
}
|
|
|
|
const unreachableTargets = errors.filter((e) => e.kind === 'slug_gone');
|
|
const networkTargets = errors.filter((e) => e.kind === 'network');
|
|
const otherErrors = errors.filter((e) => e.kind !== 'slug_gone' && e.kind !== 'network');
|
|
|
|
if (unreachableTargets.length > 0) {
|
|
const names = unreachableTargets.map((e) => e.company).join(', ');
|
|
console.log(`\n⚠️ ${unreachableTargets.length} target(s) unreachable (slug?): ${names} — run: node verify-portals.mjs`);
|
|
}
|
|
if (emptyTargets.length > 0) {
|
|
console.log(`🟡 ${emptyTargets.length} target(s) live but empty: ${emptyTargets.join(', ')}`);
|
|
}
|
|
if (networkTargets.length > 0) {
|
|
console.log(`\nNetwork errors (${networkTargets.length}):`);
|
|
for (const e of networkTargets) {
|
|
console.log(` ✗ ${e.company}: ${e.error}`);
|
|
}
|
|
}
|
|
if (otherErrors.length > 0) {
|
|
console.log(`\nErrors (${otherErrors.length}):`);
|
|
for (const e of otherErrors) {
|
|
console.log(` ✗ ${e.company}: ${e.error}`);
|
|
}
|
|
}
|
|
|
|
if (verifiedOffers.length > 0) {
|
|
console.log('\nNew offers:');
|
|
for (const o of verifiedOffers) {
|
|
const trustSuffix = o.trustScore != null && o.trustScore < 100
|
|
? ` [Trust: ${o.trustScore}/100${o.trustFlags?.length ? ' — ' + o.trustFlags.join(', ') : ''}]`
|
|
: '';
|
|
console.log(` + ${o.company} | ${o.title} | ${o.location || 'N/A'}${trustSuffix}`);
|
|
}
|
|
if (dryRun) {
|
|
console.log('\n(dry run — run without --dry-run to save results)');
|
|
} else {
|
|
console.log(`\nResults saved to ${PIPELINE_PATH} and ${SCAN_HISTORY_PATH}`);
|
|
}
|
|
}
|
|
|
|
console.log(`\n→ Run /career-ops pipeline to evaluate new offers.`);
|
|
console.log('→ Share results and get help: https://discord.gg/8pRpHETxa4');
|
|
}
|
|
|
|
// Only run main() when invoked directly (`node scan.mjs`), not when imported by tests.
|
|
// `|| ''` guards the case where Node is invoked without a script arg (e.g. `node -e`).
|
|
if (import.meta.url === pathToFileURL(process.argv[1] || '').href) {
|
|
main().catch(err => {
|
|
console.error('Fatal:', err.message);
|
|
process.exit(1);
|
|
});
|
|
}
|