115 lines
4.3 KiB
JavaScript
115 lines
4.3 KiB
JavaScript
const HARD_EXPIRED_PATTERNS = [
|
|
/job (is )?no longer available/i,
|
|
/job.*no longer open/i,
|
|
/position has been filled/i,
|
|
/this job has expired/i,
|
|
/job posting has expired/i,
|
|
/no longer accepting applications/i,
|
|
/this (position|role|job) (is )?no longer/i,
|
|
/this job (listing )?is closed/i,
|
|
/job (listing )?not found/i,
|
|
/the page you are looking for doesn.t exist/i,
|
|
/applications?\s+(?:(?:have|are|is)\s+)?closed/i,
|
|
/closed on \d{1,2}\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i,
|
|
/closed on (?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+\d{1,2}/i,
|
|
/diese stelle (ist )?(nicht mehr|bereits) besetzt/i,
|
|
/offre (expirée|n'est plus disponible)/i,
|
|
];
|
|
|
|
const LISTING_PAGE_PATTERNS = [
|
|
/\d+\s+jobs?\s+found/i,
|
|
/search for jobs page is loaded/i,
|
|
];
|
|
|
|
// Anti-bot interstitials (Cloudflare "Just a moment...", hCaptcha walls, etc.)
|
|
// render a tiny challenge page instead of the posting. Headless Playwright trips
|
|
// these on portals like pracuj.pl. They must NOT be read as expired: the body is
|
|
// short and lacks an apply control, so without this guard they fall through to
|
|
// `insufficient_content` → expired, and scan --verify would write live jobs to
|
|
// scan-history and permanently filter them out. Treat as uncertain instead.
|
|
const BOT_CHALLENGE_PATTERNS = [
|
|
/just a moment/i,
|
|
/performing security verification/i,
|
|
/checking your browser before/i,
|
|
/verify you are (a |not a )?human/i,
|
|
/enable javascript and cookies to continue/i,
|
|
/attention required.*cloudflare/i,
|
|
/\bray id\b/i,
|
|
/\bcf-ray\b/i,
|
|
/please complete the security check/i,
|
|
];
|
|
|
|
const EXPIRED_URL_PATTERNS = [
|
|
/[?&]error=true/i,
|
|
];
|
|
|
|
const APPLY_PATTERNS = [
|
|
/\bapply\b/i,
|
|
/\bsolicitar\b/i,
|
|
/\bbewerben\b/i,
|
|
/\bpostuler\b/i,
|
|
/submit application/i,
|
|
/easy apply/i,
|
|
/start application/i,
|
|
/ich bewerbe mich/i,
|
|
// Polish (pracuj.pl, justjoin.it, bulldogjob.pl): "Aplikuj" / "Aplikuj teraz" /
|
|
// "Wyślij CV" / "Przejdź do panelu aplikowania". Without these, a fully-loaded
|
|
// Polish posting has no recognized apply control and falls to no_apply_control.
|
|
/\baplikuj\b/i,
|
|
/panelu aplikowania/i,
|
|
/wyślij (cv|aplikacj)/i,
|
|
];
|
|
|
|
const MIN_CONTENT_CHARS = 300;
|
|
|
|
function firstMatch(patterns, text = '') {
|
|
return patterns.find((pattern) => pattern.test(text));
|
|
}
|
|
|
|
function hasApplyControl(controls = []) {
|
|
return controls.some((control) => APPLY_PATTERNS.some((pattern) => pattern.test(control)));
|
|
}
|
|
|
|
export function classifyLiveness({ status = 0, finalUrl = '', bodyText = '', applyControls = [] } = {}) {
|
|
if (status === 404 || status === 410) {
|
|
return { result: 'expired', code: 'http_gone', reason: `HTTP ${status}` };
|
|
}
|
|
|
|
// Bot/anti-scraping walls — never expired. Check before the content-length and
|
|
// listing-page heuristics, which would otherwise misread the short challenge
|
|
// body as a dead posting. 403/503 are access-blocked signals, not "gone"
|
|
// (a genuinely removed posting returns 404/410 or a hard-expired banner).
|
|
const botChallenge = firstMatch(BOT_CHALLENGE_PATTERNS, bodyText);
|
|
if (botChallenge) {
|
|
return { result: 'uncertain', code: 'bot_challenge', reason: `anti-bot challenge: ${botChallenge.source}` };
|
|
}
|
|
if (status === 403 || status === 503) {
|
|
return { result: 'uncertain', code: 'access_blocked', reason: `HTTP ${status} (access blocked, likely anti-bot)` };
|
|
}
|
|
|
|
const expiredUrl = firstMatch(EXPIRED_URL_PATTERNS, finalUrl);
|
|
if (expiredUrl) {
|
|
return { result: 'expired', code: 'expired_url', reason: `redirect to ${finalUrl}` };
|
|
}
|
|
|
|
const expiredBody = firstMatch(HARD_EXPIRED_PATTERNS, bodyText);
|
|
if (expiredBody) {
|
|
return { result: 'expired', code: 'expired_body', reason: `pattern matched: ${expiredBody.source}` };
|
|
}
|
|
|
|
if (hasApplyControl(applyControls)) {
|
|
return { result: 'active', code: 'apply_control_visible', reason: 'visible apply control detected' };
|
|
}
|
|
|
|
const listingPage = firstMatch(LISTING_PAGE_PATTERNS, bodyText);
|
|
if (listingPage) {
|
|
return { result: 'expired', code: 'listing_page', reason: `pattern matched: ${listingPage.source}` };
|
|
}
|
|
|
|
if (bodyText.trim().length < MIN_CONTENT_CHARS) {
|
|
return { result: 'expired', code: 'insufficient_content', reason: 'insufficient content — likely nav/footer only' };
|
|
}
|
|
|
|
return { result: 'uncertain', code: 'no_apply_control', reason: 'content present but no visible apply control found' };
|
|
}
|